NGINX metrics

nixos/roles/monitoring-server/prometheus.nix

@@ -11,6 +11,10 @@
       domain = {
         enable = true;
       };
+      nginx = {
+        enable = true;
+        sslVerify = false;
+      };
     };
     ruleFiles = [
       ./synapse-v2.rules
@@ -24,6 +28,22 @@
           }
         ];
       }
+      {
+        job_name = "${config.networking.hostName}-telegraf";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:9125"];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}-nginx";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}"];
+          }
+        ];
+      }
       {
         job_name = "domains";
         metrics_path = "/probe";

nixos/roles/monitoring-server/scalpel.nix

@@ -0,0 +1,22 @@
+{
+  lib,
+  config,
+  prev,
+  ...
+}: let
+  inherit (lib.strings) addContextFrom;
+  start = prev.config.systemd.services.telegraf.serviceConfig.ExecStart;
+  telegraf_cfgfile = builtins.head (builtins.match "^.*-config ([^\ ]*).*$" "${start}");
+in {
+  systemd.services.telegraf.serviceConfig.ExecStart = lib.mkForce (
+    builtins.replaceStrings ["${telegraf_cfgfile}"] ["${config.scalpel.trafos."config.toml".destination} "] "${start}"
+  );
+  scalpel.trafos."config.toml" = {
+    source = addContextFrom start telegraf_cfgfile;
+    matchers."TELEGRAF_API_KEY".secret = config.sops.secrets.telegraf_api_key.path;
+    owner = "telegraf";
+    group = "telegraf";
+    mode = "0440";
+  };
+  #environment.etc."ensure_telegraf_trafos".source = telegraf_cfgfile;
+}

nixos/roles/monitoring-server/secrets.nix

@@ -0,0 +1,10 @@
+_: {
+  sops.secrets.telegraf_api_key = {
+    format = "yaml";
+    sopsFile = ./secrets.yaml;
+  };
+
+  scalpels = [
+    ./scalpel.nix
+  ];
+}

nixos/roles/monitoring-server/secrets.yaml

@@ -0,0 +1,42 @@
+telegraf_api_key: ENC[AES256_GCM,data:XXMLlIxtFYmURr6QuRdZFL+Z3OIm1nm8ReZq/sAML1DzFKO8U2sbdyHjXnqUWw==,iv:mMpzUrZozfcxUSpxXki64loHWtt7VwdilWTLpie01NI=,tag:a0iRgCemgDCUxKV0gMoKow==,type:str]
+sops:
+    shamir_threshold: 1
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1n4kdchmkk3rfkaknxhveqr2ftprdpgwckutt23y6u8639lazzuks77tgav
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnOTB2LzByVHU1T1pxWFZQ
+            b2JZMXEzWEY5ZjRNNnlqMW5UUmVWTk9kM2lRCjlpemIzb1FhWEE1WFNGNXZMK1Vz
+            YmRrYW91bno1alh3M0dZN3dyYUk5dWMKLS0tIDdWbFk2a2hiU0pLMitYeWZPYkkw
+            T0NKQzIzY2g3TnBoT00xa0xBUW1BNDgK/Uj+ldtdx1E+hQlKBUWo9TEPa8vmk3dZ
+            QWE6YSlY9kYjGNs+WHjnUXoO3VMmyzxNFFkrnOHLcfKQbi9p5Qrp0w==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-07-29T22:20:58Z"
+    mac: ENC[AES256_GCM,data:wRPzcBx4PJqK8ziR1oiVT8RrCwzlz9IugY0VMC6q7fuSBDEPrZjJ3wqpP0crNzQuZD2otEiB8ooYlL3j/lLT+vMPuUzitM5J8V3uyLwGV5FLfqC3AgbDAwb7r/x2okpSWEffhwuTMUVZ6jJo0+/XoAWS+D4IULfa77nHg6YBuu0=,iv:vft9e7pz1v5Jkxx2HnKg4+HAFZ9uRBe8OhT5DB7Yx10=,tag:nYkl5vRBG0BI/z+IERambg==,type:str]
+    pgp:
+        - created_at: "2023-07-29T22:10:05Z"
+          enc: |
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA82M54yws73UARAApdK00AgsnRCH34W9dESFQm4ji7jjP+E4b0UDP6bEdPmX
+            KtFGSp4jZoTJYBpN2HJzeuVGPFfHUVMc8iZz/bkO120n41si0mwUQA+eNt7350sj
+            qhzjsjgYRG+iogaDI/VwEkcEtuONa3GtBjXQnXXtcI2F0e+40imXhYqezmtvjH02
+            BNkY+rTvmg6LLIVrMhJXQmT+qXg+4iP/gIbCjezjO1ah16JY18dK45dqDJd+uWSN
+            WmqHFjqEXUJ6dzXPPkOpbGUeVkAs1OCqnNB7Hl5A5r3v8d47KPhYA9Sqkocag/NQ
+            Y/LMaLS6SJrugmtbNtC7FhmPHfgOnDG+8gz3m1XgP4QWKXkuOdbqoeSlXWCFlIPi
+            px3hXdeqaHYQvDYaUJpJqnwPbpgHIb29mTaPtP4RWbvXJzoBEshS4ONcGPMmemcg
+            qi+F24h6UdIDpFCguqLdf0SY10InmGB/5XCaN6Bd7zuLAq3iel5zvAW/u8Irt37J
+            QoUlB5OwgJds2MpBwd9RJOczlO63VJzVrGDNAVD0D6KBZHRkdEWOgpv3w8DxhIpF
+            lNLz78/XYvCsjgQCV+SjeJjxtQea0JOk2Xtt7nQVCrwDKh7TIIOdT8jI2EbKDAbi
+            bJgI1NGDxfyrk79ga7qyjLN9jhCubdKRibPPzKXqNdCahN5ldFlMvL8rZeJNYtjS
+            XgGaiB/wBjAmn863D4brJOH7KqALxP/tEKc4FM4uH8fcDOpsbPcgZ6Q4nQbIVHBa
+            9bt8heM8006oeLPQM2raWM0/ETf+4rQzEwIO+Av4q2Rypnv47q1Qxbmag6Sh5Yw=
+            =wOQn
+            -----END PGP MESSAGE-----
+          fp: CD8CE78CB0B3BDD4
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3

nixos/roles/monitoring-server/telegraf.nix

@@ -0,0 +1,58 @@
+{config, ...}: {
+  users.users.telegraf = {
+    extraGroups = [
+      "nginx"
+    ];
+  };
+
+  services.telegraf = {
+    enable = true;
+    extraConfig = {
+      inputs = {
+        nginx = {
+          urls = [
+            "http://localhost/nginx_status"
+          ];
+          response_timeout = "5s";
+        };
+        tail = {
+          name_override = "nginxlog";
+          files = [
+            "/var/log/nginx/access.log"
+          ];
+          from_beginning = true;
+          pipe = false;
+          data_format = "grok";
+          grok_patterns = ["%{COMBINED_LOG_FORMAT}"];
+        };
+        cpu = {
+          percpu = true;
+        };
+        disk = {
+        };
+        diskio = {
+        };
+        io = {
+        };
+        net = {
+        };
+        mem = {
+        };
+        system = {
+        };
+      };
+      outputs = {
+        prometheus_client = {
+          listen = "127.0.0.1:9125";
+        };
+        http = {
+          url = "http://localhost:${toString config.services.grafana.settings.server.http_port}/api/live/push/custom_stream_id";
+          data_format = "influx";
+          headers = {
+            Authorization = "Bearer !!TELEGRAF_API_KEY!!";
+          };
+        };
+      };
+    };
+  };
+}

nixos/roles/web-server/nginx.nix

@@ -5,5 +5,6 @@ _: {
     recommendedOptimisation = true;
     recommendedGzipSettings = true;
     recommendedProxySettings = true;
+    statusPage = true;
   };
 }