services/{grafana,loki,node-exporter,promtail}.nix: init

Added monitoring stack and applied monitoring stack to each node worth
monitoring.

hosts/athame/nixos/default.nix

@@ -24,6 +24,11 @@ with lib;
     ../../../services/murmur.nix
     ../../../services/matrix.nix
     ../../../services/restic.nix
+    ../../../services/grafana.nix
+    ../../../services/prometheus.nix
+    ../../../services/loki.nix
+    ../../../services/node-exporter.nix
+    ../../../services/promtail.nix
   ];
 
   boot.loader.grub.enable = true;

hosts/samhain/nixos/default.nix

@@ -15,6 +15,8 @@ in {
     ../../../services/zfs.nix
     ../../../services/restic.nix
     ../../../services/nginx.nix
+    ../../../services/node-exporter.nix
+    ../../../services/promtail.nix
     ./thermal
     ./transmission.nix
     ./jellyfin.nix

hosts/yule/nixos/default.nix

@@ -1,10 +1,14 @@
-{ config, pkgs, profiles, ... }:
+{ config, pkgs, lib, profiles, ... }:
+
+with lib;
 
 {
   imports = [
     ./hw.nix
     ../../../services/zfs.nix
     ../../../services/restic.nix
+    ../../../services/node-exporter.nix
+    ../../../services/promtail.nix
     profiles.gui
     profiles.sway
     profiles.kat

services/grafana.nix

@@ -0,0 +1,36 @@
+{ config, ... }:
+
+{
+  services.postgresql = {
+    ensureDatabases = [ "grafana" ];
+    ensureUsers = [{
+      name = "grafana";
+      ensurePermissions."DATABASE grafana" = "ALL PRIVILEGES";
+    }];
+  };
+
+  services.grafana = {
+    enable = true;
+    port = 3001;
+    domain = "graph.kittywit.ch";
+    rootUrl = "https://graph.kittywit.ch/";
+    database = {
+      type = "postgres";
+      host = "/run/postgresql/";
+      user = "grafana";
+      name = "grafana";
+    };
+  };
+
+  services.nginx.virtualHosts."graph.kittywit.ch" = {
+    enableACME = true;
+    forceSSL = true;
+    locations = { "/".proxyPass = "http://127.0.0.1:3001"; };
+  };
+
+  deploy.tf.dns.records.kittywitch_graph = {
+    tld = "kittywit.ch.";
+    domain = "graph";
+    cname.target = "athame.kittywit.ch.";
+  };
+}

services/loki.nix

@@ -0,0 +1,62 @@
+{ config, pkgs, ... }:
+
+{
+  katnet.private.tcp.ports = [ 3100 ];
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+      chunk_store_config = { max_look_back_period = "0s"; };
+      ingester = {
+        chunk_idle_period = "1h";
+        chunk_retain_period = "30s";
+        chunk_target_size = 1048576;
+        lifecycler = {
+          address = "0.0.0.0";
+          final_sleep = "0s";
+          ring = {
+            kvstore = { store = "inmemory"; };
+            replication_factor = 1;
+          };
+        };
+        max_chunk_age = "1h";
+        max_transfer_retries = 0;
+      };
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+      schema_config = {
+        configs = [{
+          from = "2020-10-24";
+          index = {
+            period = "24h";
+            prefix = "index_";
+          };
+          object_store = "filesystem";
+          schema = "v11";
+          store = "boltdb-shipper";
+        }];
+      };
+      compactor = {
+        working_directory = "/tmp/loki-compactor-boltdb";
+        shared_store = "filesystem";
+      };
+      server = { http_listen_port = 3100; };
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+        filesystem = { directory = "/var/lib/loki/chunks"; };
+      };
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+    };
+  };
+
+}

services/node-exporter.nix

@@ -0,0 +1,15 @@
+{ config, ... }:
+
+{
+  katnet.private.tcp.ports = [ 9002 ];
+
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9002;
+      };
+    };
+  };
+}

services/prometheus.nix

@@ -0,0 +1,22 @@
+{ config, hosts, lib, ... }:
+
+with lib;
+
+let
+  prom_configs =
+    (mapAttrs (hostName: host: host.config.services.prometheus.exporters.node)
+      (filterAttrs
+        (_: host: host.config.services.prometheus.exporters.node.enable)
+        hosts));
+in {
+  services.prometheus = {
+    enable = true;
+    scrapeConfigs = mapAttrsToList (hostName: prom: {
+      job_name = hostName;
+      static_configs = [{
+        targets = [ "${hostName}.net.kittywit.ch:${toString prom.port}" ];
+      }];
+    }) prom_configs;
+  };
+}
+

services/promtail.nix

@@ -0,0 +1,40 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  promtail_config = pkgs.writeText "prom-config.json" (builtins.toJSON {
+    clients =
+      [{ url = "http://athame.net.kittywit.ch:3100/loki/api/v1/push"; }];
+    positions = { filename = "/tmp/positions.yaml"; };
+    scrape_configs = [{
+      job_name = "journal";
+      journal = {
+        labels = {
+          host = config.networking.hostName;
+          job = "systemd-journal";
+        };
+        max_age = "12h";
+      };
+      relabel_configs = [{
+        source_labels = [ "__journal__systemd_unit" ];
+        target_label = "unit";
+      }];
+    }];
+    server = {
+      grpc_listen_port = 0;
+      http_listen_port = 28183;
+    };
+  });
+in {
+  systemd.services.promtail = {
+    description = "Promtail service for Loki";
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.grafana-loki}/bin/promtail --config.file ${promtail_config}
+      '';
+    };
+  };
+}