further refactor work

nixos/servers/grafana-stack/grafana.nix

@@ -0,0 +1,11 @@
+_: {
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      domain = "mon.kittywit.ch";
+      http_port = 2342;
+      http_addr = "127.0.0.1";
+      root_url = "https://mon.kittywit.ch/";
+    };
+  };
+}

nixos/servers/grafana-stack/nginx.nix

@@ -0,0 +1,10 @@
+{config, ...}: {
+  services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
+    enableACME = true;
+    forceSSL = true;
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+      proxyWebsockets = true;
+    };
+  };
+}

nixos/servers/grafana-stack/prometheus.nix

@@ -0,0 +1,115 @@
+{config, ...}: {
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = ["systemd"];
+        port = 9002;
+      };
+      postgres = {
+        enable = true;
+        port = 9187;
+        runAsLocalSuperUser = true;
+        extraFlags = ["--auto-discover-databases"];
+      };
+      domain = {
+        enable = true;
+      };
+      nginx = {
+        enable = true;
+        sslVerify = false;
+      };
+    };
+    ruleFiles = [
+      ./synapse-v2.rules
+    ];
+    scrapeConfigs = [
+      {
+        job_name = "tewi-hass";
+        scrape_interval = "60s";
+        metrics_path = "/api/prometheus";
+        scheme = "https";
+        bearer_token = "!!HOME_ASSISTANT_API_TOKEN!!";
+        static_configs = [
+          {
+            targets = ["home.gensokyo.zone:443"];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}-minecraft";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:25585"];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node.port}"];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}-telegraf";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:9125"];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}-postgres";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}"];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}-nginx";
+        static_configs = [
+          {
+            targets = ["127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}"];
+          }
+        ];
+      }
+      {
+        job_name = "domains";
+        metrics_path = "/probe";
+        relabel_configs = [
+          {
+            source_labels = ["__address__"];
+            target_label = "__param_target";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.domain.port}";
+          }
+        ];
+        static_configs = [
+          {
+            targets = [
+              "dork.dev"
+              "inskip.me"
+              "gensokyo.zone"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "${config.networking.hostName}-synapse";
+        metrics_path = "/_synapse/metrics";
+        static_configs = [
+          {
+            targets = ["[::1]:8009"];
+          }
+        ];
+      }
+    ];
+  };
+}

nixos/servers/grafana-stack/scalpel.nix

@@ -0,0 +1,34 @@
+{
+  lib,
+  config,
+  prev,
+  ...
+}: let
+  inherit (lib.strings) addContextFrom;
+  inherit (lib.modules) mkForce;
+  telegraf_start = prev.config.systemd.services.telegraf.serviceConfig.ExecStart;
+  telegraf_cfgfile = builtins.head (builtins.match "^.*-config ([^\ ]*).*$" "${telegraf_start}");
+  prometheus_start = prev.config.systemd.services.prometheus.serviceConfig.ExecStart;
+  prometheus_cfgfile = builtins.head (builtins.match "^.*-config\.file=([^\ ]*).*$" "${prometheus_start}");
+in {
+  systemd.services.telegraf.serviceConfig.ExecStart = mkForce (
+    builtins.replaceStrings ["${telegraf_cfgfile}"] ["${config.scalpel.trafos."config.toml".destination} "] "${telegraf_start}"
+  );
+  scalpel.trafos."config.toml" = {
+    source = addContextFrom telegraf_start telegraf_cfgfile;
+    matchers."TELEGRAF_API_KEY".secret = config.sops.secrets.telegraf_api_key.path;
+    owner = "telegraf";
+    group = "telegraf";
+    mode = "0440";
+  };
+  systemd.services.prometheus.serviceConfig.ExecStart = mkForce (
+    builtins.replaceStrings ["${prometheus_cfgfile}"] ["${config.scalpel.trafos."prometheus.yml".destination} "] "${prometheus_start}"
+  );
+  scalpel.trafos."prometheus.yml" = {
+    source = addContextFrom prometheus_start prometheus_cfgfile;
+    matchers."HOME_ASSISTANT_API_TOKEN".secret = config.sops.secrets.home_assistant_api_key.path;
+    owner = "prometheus";
+    group = "prometheus";
+    mode = "0440";
+  };
+}

nixos/servers/grafana-stack/secrets.nix

@@ -0,0 +1,13 @@
+_: {
+  sops.secrets.telegraf_api_key = {
+    format = "yaml";
+    sopsFile = ./secrets.yaml;
+  };
+  sops.secrets.home_assistant_api_key = {
+    format = "yaml";
+    sopsFile = ./secrets.yaml;
+  };
+  scalpels = [
+    ./scalpel.nix
+  ];
+}

nixos/servers/grafana-stack/secrets.yaml

@@ -0,0 +1,43 @@
+telegraf_api_key: ENC[AES256_GCM,data:XXMLlIxtFYmURr6QuRdZFL+Z3OIm1nm8ReZq/sAML1DzFKO8U2sbdyHjXnqUWw==,iv:mMpzUrZozfcxUSpxXki64loHWtt7VwdilWTLpie01NI=,tag:a0iRgCemgDCUxKV0gMoKow==,type:str]
+home_assistant_api_key: ENC[AES256_GCM,data:+RSRYTXro9vZChEwTZNcyqFnwwDfdOEcXMbp7AAH6wo+R3+bVhTHNvUJU3q78CZkIzXquDDczvySHho28EUaKyUNXLWtmlHG99SL6qXPwZLbTpcMX+5pA//qcRHFss17LrxTXXkAuepQqWrSq8rxXEYshMsbLVo/L38jrW5y13YknLovXzUItRf4lDoX/fDhq6OD2EH1G6GoCjOUtIurzeaHMRo4nc4aO1/k1s9rAAVRBoFnFknM,iv:k2UuicBJ4UFKO6QV15ZUQ3Asur00MLT/DJzgKeZ+I8U=,tag:Wz6FPlu5CFQ3anljvGxenw==,type:str]
+sops:
+    shamir_threshold: 1
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1n4kdchmkk3rfkaknxhveqr2ftprdpgwckutt23y6u8639lazzuks77tgav
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnOTB2LzByVHU1T1pxWFZQ
+            b2JZMXEzWEY5ZjRNNnlqMW5UUmVWTk9kM2lRCjlpemIzb1FhWEE1WFNGNXZMK1Vz
+            YmRrYW91bno1alh3M0dZN3dyYUk5dWMKLS0tIDdWbFk2a2hiU0pLMitYeWZPYkkw
+            T0NKQzIzY2g3TnBoT00xa0xBUW1BNDgK/Uj+ldtdx1E+hQlKBUWo9TEPa8vmk3dZ
+            QWE6YSlY9kYjGNs+WHjnUXoO3VMmyzxNFFkrnOHLcfKQbi9p5Qrp0w==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-08-12T16:28:10Z"
+    mac: ENC[AES256_GCM,data:7t9dAJPUiOD93Hyt+YLVjR/SdqIcuLi8TFP2/8gzem8Hrn97Yqx5Iow57alFcOWcb1ymhSQLIWjh5RydhlnoeLj/HbacSKxxFirFFv842mBVKqbVyfQcNozGl5D0oo0yd8gKzXQ6BaKqel7ZeOeIeY6XKAzH2RH2r8Gj1kPhkHY=,iv:bZuu+kFJcc8SDA1uShXroQcLMjUj+DTSvsbIABLddFs=,tag:bNLzxOll5UL5uFJeoq5XzA==,type:str]
+    pgp:
+        - created_at: "2023-07-29T22:10:05Z"
+          enc: |
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA82M54yws73UARAApdK00AgsnRCH34W9dESFQm4ji7jjP+E4b0UDP6bEdPmX
+            KtFGSp4jZoTJYBpN2HJzeuVGPFfHUVMc8iZz/bkO120n41si0mwUQA+eNt7350sj
+            qhzjsjgYRG+iogaDI/VwEkcEtuONa3GtBjXQnXXtcI2F0e+40imXhYqezmtvjH02
+            BNkY+rTvmg6LLIVrMhJXQmT+qXg+4iP/gIbCjezjO1ah16JY18dK45dqDJd+uWSN
+            WmqHFjqEXUJ6dzXPPkOpbGUeVkAs1OCqnNB7Hl5A5r3v8d47KPhYA9Sqkocag/NQ
+            Y/LMaLS6SJrugmtbNtC7FhmPHfgOnDG+8gz3m1XgP4QWKXkuOdbqoeSlXWCFlIPi
+            px3hXdeqaHYQvDYaUJpJqnwPbpgHIb29mTaPtP4RWbvXJzoBEshS4ONcGPMmemcg
+            qi+F24h6UdIDpFCguqLdf0SY10InmGB/5XCaN6Bd7zuLAq3iel5zvAW/u8Irt37J
+            QoUlB5OwgJds2MpBwd9RJOczlO63VJzVrGDNAVD0D6KBZHRkdEWOgpv3w8DxhIpF
+            lNLz78/XYvCsjgQCV+SjeJjxtQea0JOk2Xtt7nQVCrwDKh7TIIOdT8jI2EbKDAbi
+            bJgI1NGDxfyrk79ga7qyjLN9jhCubdKRibPPzKXqNdCahN5ldFlMvL8rZeJNYtjS
+            XgGaiB/wBjAmn863D4brJOH7KqALxP/tEKc4FM4uH8fcDOpsbPcgZ6Q4nQbIVHBa
+            9bt8heM8006oeLPQM2raWM0/ETf+4rQzEwIO+Av4q2Rypnv47q1Qxbmag6Sh5Yw=
+            =wOQn
+            -----END PGP MESSAGE-----
+          fp: CD8CE78CB0B3BDD4
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3

nixos/servers/grafana-stack/synapse-v2.rules

@@ -0,0 +1,74 @@
+groups:
+- name: synapse
+  rules:
+
+  ###
+  ### Prometheus Console Only
+  ### The following rules are only needed if you use the Prometheus Console
+  ### in contrib/prometheus/consoles/synapse.html
+  ###
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_client_sent_edus_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_server_received_edus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_server_received_pdus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_server_received_queries) by (job)'
+
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+  ###
+  ### End of 'Prometheus Console Only' rules block
+  ###
+
+
+  ###
+  ### Grafana Only
+  ### The following rules are only needed if you use the Grafana dashboard
+  ### in contrib/grafana/synapse.json
+  ###
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
+    labels:
+      type: remote
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
+    labels:
+      type: local
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
+    labels:
+      type: bridges
+
+  - record: synapse_storage_events_persisted_by_event_type
+    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
+
+  - record: synapse_storage_events_persisted_by_origin
+    expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
+  ###
+  ### End of 'Grafana Only' rules block
+  ###

nixos/servers/grafana-stack/telegraf.nix

@@ -0,0 +1,77 @@
+{config, ...}: {
+  users.users.telegraf = {
+    extraGroups = [
+      "nginx"
+    ];
+  };
+  systemd.services.telegraf = {
+    serviceConfig = {
+      AmbientCapabilities = [
+        "CAP_NET_RAW"
+      ];
+      CapabilityBoundingSet = [
+        "CAP_NET_RAW"
+      ];
+    };
+  };
+  services.telegraf = {
+    enable = true;
+    extraConfig = {
+      inputs = {
+        nginx = {
+          urls = [
+            "http://localhost/nginx_status"
+          ];
+          response_timeout = "5s";
+        };
+        tail = {
+          name_override = "nginxlog";
+          files = [
+            "/var/log/nginx/access.log"
+          ];
+          from_beginning = true;
+          pipe = false;
+          data_format = "grok";
+          grok_patterns = ["%{COMBINED_LOG_FORMAT}"];
+        };
+        cpu = {
+          percpu = true;
+        };
+        disk = {
+        };
+        diskio = {
+        };
+        io = {
+        };
+        net = {
+        };
+        mem = {
+        };
+        ping = {
+          interval = "60s";
+          method = "native";
+          urls = [
+            "8.8.8.8"
+            "2001:4860:4860:0:0:0:0:8888"
+          ];
+          count = 3;
+          timeout = 2.0;
+        };
+        system = {
+        };
+      };
+      outputs = {
+        prometheus_client = {
+          listen = "127.0.0.1:9125";
+        };
+        http = {
+          url = "http://localhost:${toString config.services.grafana.settings.server.http_port}/api/live/push/custom_stream_id";
+          data_format = "influx";
+          headers = {
+            Authorization = "Bearer !!TELEGRAF_API_KEY!!";
+          };
+        };
+      };
+    };
+  };
+}