feat(access): monitoring

modules/nixos/monitoring/ingest/grafana.nix

@@ -1,11 +1,19 @@
-_: {
+{
+  config,
+  system,
+  lib,
+  ...
+}: let
+  inherit (lib.modules) mkDefault;
+  cfg = config.services.grafana;
+  service = system.exports.services.grafana;
+in {
   services.grafana = {
-    #enable = true;
     settings.server = {
-      domain = "gensokyo.zone";
+      domain = mkDefault config.networking.domain;
-      http_port = 9092;
+      http_port = mkDefault 9092;
-      http_addr = "0.0.0.0";
+      http_addr = mkDefault "::";
-      root_url = "https://mon.gensokyo.zone";
+      root_url = mkDefault "https://${service.id}.${cfg.settings.server.domain}";
     };
   };
 }

modules/nixos/monitoring/ingest/loki.nix

@@ -1,5 +1,4 @@
 {
-  pkgs,
   config,
   lib,
   access,
@@ -7,6 +6,7 @@
   ...
 }: let
   inherit (gensokyo-zone) systems;
+  inherit (lib.modules) mkIf mkOptionDefault;
   inherit (lib.attrsets) filterAttrs mapAttrsToList;
   promtailSystems =
     filterAttrs (
@@ -15,61 +15,38 @@
         && system.config.exports.services.promtail.enable
     )
     systems;
-  inherit (builtins) toJSON;
-  inherit (lib.options) mkOption;
-  inherit (lib.types) port;
   cfg = config.services.loki;
 in {
-  options.services.loki.settings = {
-    httpListenPort = mkOption {
-      type = port;
-      description = "Port to listen on over HTTP";
-      default = 9093;
-    };
-    grpcListenPort = mkOption {
-      type = port;
-      description = "Port to listen on over gRPC";
-      default = 0;
-    };
-  };
   config = {
     services.loki = {
-      #enable = true;
+      configuration = {
-      configFile = pkgs.writeTextFile {
+        server = {
-        name = "config.yaml";
+          http_listen_port = mkOptionDefault 9093;
-        executable = false;
+          grpc_listen_port = mkOptionDefault 0;
-        text = toJSON {
-          server = {
-            http_listen_port = cfg.settings.httpListenPort;
-            grpc_listen_port = cfg.settings.grpcListenPort;
-          };
-          positions = {
-            filename = "/tmp/positions.yaml";
-          };
-          clients =
-            mapAttrsToList (_: system: {
-              url = "${access.getAddressFor system.config.name "lan"}:${toString system.config.exports.services.promtail.ports.default.port}";
-            })
-            promtailSystems;
-          scrape_configs =
-            mapAttrsToList (_: system: {
-              job_name = "${system.config.name}-journal";
-              journal = {
-                max_age = "${toString (24 * 7)}h";
-                labels = {
-                  job = "systemd-journal";
-                  host = system.config.name;
-                };
-              };
-              relabel_configs = [
-                {
-                  source_labels = ["__journal__systemd_unit"];
-                  target_label = "unit";
-                }
-              ];
-            })
-            promtailSystems;
         };
+        # https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#1-local-configuration-exampleyaml
+        auth_enabled = mkOptionDefault false;
+        common = {
+          ring = {
+            instance_addr = mkOptionDefault "127.0.0.1";
+            kvstore.store = mkOptionDefault "inmemory";
+          };
+          replication_factor = 1;
+          path_prefix = mkOptionDefault cfg.dataDir;
+        };
+        schema_config.configs = [
+          {
+            from = "2020-05-15";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+        storage_config.filesystem.directory = mkOptionDefault "${cfg.dataDir}/chunks";
       };
     };
   };

modules/nixos/monitoring/source/promtail.nix

@@ -1,27 +1,54 @@
 {
   config,
+  system,
+  access,
   lib,
   ...
 }: let
-  inherit (builtins) toString;
+  inherit (lib.modules) mkIf mkOptionDefault;
-  inherit (lib.options) mkOption;
-  inherit (lib.modules) mkIf;
-  inherit (lib.types) port;
   cfg = config.services.promtail;
 in {
-  options.services.promtail.settings = {
+  config.services.promtail = {
-    httpListenPort = mkOption {
+    configuration = {
-      type = port;
+      server = {
-      description = "Port to listen on over HTTP";
+        http_listen_port = mkOptionDefault 9094;
-      default = 9094;
+        grpc_listen_port = mkOptionDefault 0;
+      };
+      clients = let
+        baseUrl = access.proxyUrlFor { serviceName = "loki"; };
+      in [
+        {
+          url = "${baseUrl}/loki/api/v1/push";
+        }
+      ];
+      scrape_configs = [
+        {
+          job_name = "${system.name}-journald";
+          journal = {
+            max_age = "${toString (24 * 7)}h";
+            labels = {
+              job = "systemd-journald";
+              system = system.name;
+              host = config.networking.fqdn;
+            };
+          };
+          relabel_configs = [
+            {
+              source_labels = ["__journal__systemd_unit"];
+              target_label = "unit";
+            }
+          ];
+        }
+      ];
     };
   };
-  config.services.promtail = {
+  config.networking.firewall.interfaces.lan = let
-    extraFlags = [
+    inherit (cfg.configuration) server;
-      "--server.http-listen-port=${toString cfg.settings.httpListenPort}"
+  in
-    ];
+    mkIf cfg.enable {
-  };
+      allowedTCPPorts = [
-  config.networking.firewall.interfaces.lan = mkIf cfg.enable {
+        server.http_listen_port
-    allowedTCPPorts = [ cfg.settings.httpListenPort ];
+        (mkIf (server.grpc_listen_port != 0) server.grpc_listen_port)
-  };
+      ];
+    };
 }

modules/system/exports/monitoring.nix

@@ -99,7 +99,6 @@ in
     config.exports.services =
       {
         prometheus = {config, ...}: {
-          id = mkAlmostOptionDefault "prometheus";
           nixos = {
             serviceAttr = "prometheus";
             assertions = mkIf config.enable [
@@ -115,7 +114,7 @@ in
           };
         };
         grafana = {config, ...}: {
-          id = mkAlmostOptionDefault "grafana";
+          id = mkAlmostOptionDefault "mon";
           nixos = {
             serviceAttr = "grafana";
             assertions = mkIf config.enable [
@@ -131,26 +130,29 @@ in
           };
         };
         loki = {config, ...}: {
-          id = mkAlmostOptionDefault "loki";
+          id = mkAlmostOptionDefault "logs";
           nixos = {
             serviceAttr = "loki";
             assertions = mkIf config.enable [
-              (nixosConfig: {
+              (nixosConfig: let
-                assertion = config.ports.default.port == nixosConfig.services.loki.settings.httpListenPort;
+                inherit (nixosConfig.services.loki.configuration.server) http_listen_port;
+              in {
+                assertion = config.ports.default.port == http_listen_port;
                 message = "port mismatch";
               })
               (nixosConfig: let
-                inherit (nixosConfig.services.loki.settings) grpcListenPort;
+                inherit (nixosConfig.services.loki.configuration.server) grpc_listen_port;
               in {
-                assertion = !config.ports.grpc.enable || config.ports.grpc.port == grpcListenPort;
+                assertion = !config.ports.grpc.enable || config.ports.grpc.port == grpc_listen_port;
                 message = "gRPC port mismatch";
               })
               (nixosConfig: let
-                inherit (nixosConfig.services.loki.settings) grpcListenPort;
+                inherit (nixosConfig.services.loki.configuration.server) grpc_listen_port;
               in {
-                assertion = if config.ports.grpc.enable
+                assertion =
-                  then grpcListenPort != 0
+                  if config.ports.grpc.enable
-                  else grpcListenPort == 0;
+                  then grpc_listen_port != 0
+                  else grpc_listen_port == 0;
                 message = "gRPC enable mismatch";
               })
             ];
@@ -174,7 +176,7 @@ in
             serviceAttr = "promtail";
             assertions = mkIf config.enable [
               (nixosConfig: {
-                assertion = config.ports.default.port == nixosConfig.services.promtail.settings.httpListenPort;
+                assertion = config.ports.default.port == nixosConfig.services.promtail.configuration.server.http_listen_port;
                 message = "port mismatch";
               })
             ];
@@ -187,6 +189,7 @@ in
             // {
               prometheus.exporter.enable = true;
             };
+          #ports.grpc = ...
         };
       }
       // exporters;

nixos/access/grafana.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit (lib.modules) mkIf mkDefault;
+  inherit (config.services) grafana;
+  name.shortServer = mkDefault "mon";
+  upstreamName = "grafana'access";
+in {
+  config.services.nginx = {
+    upstreams'.${upstreamName}.servers = {
+      local = {
+        enable = mkDefault grafana.enable;
+        addr = mkDefault "localhost";
+        port = mkIf grafana.enable (mkDefault grafana.settings.server.http_port);
+      };
+      service = {upstream, ...}: {
+        enable = mkIf upstream.servers.local.enable (mkDefault false);
+        accessService = {
+          name = "grafana";
+        };
+      };
+    };
+    virtualHosts = let
+      copyFromVhost = mkDefault "grafana";
+      vouch.enable = mkDefault true;
+      locations = {
+        "/" = {
+          proxy.enable = true;
+        };
+      };
+    in {
+      grafana = {
+        inherit name locations vouch;
+        proxy.upstream = mkDefault upstreamName;
+      };
+      grafana'local = {
+        inherit name locations;
+        ssl.cert = {
+          inherit copyFromVhost;
+        };
+        proxy = {
+          inherit copyFromVhost;
+        };
+        local.enable = mkDefault true;
+      };
+    };
+  };
+}

nixos/access/loki.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit (lib.modules) mkIf mkDefault;
+  inherit (config.services) loki;
+  name.shortServer = mkDefault "logs";
+  upstreamName = "loki'access";
+in {
+  config.services.nginx = {
+    # TODO: gRPC port?
+    upstreams'.${upstreamName}.servers = {
+      local = {
+        enable = mkDefault loki.enable;
+        addr = mkDefault "localhost";
+        port = mkIf loki.enable (mkDefault loki.configuration.server.http_listen_port);
+      };
+      service = {upstream, ...}: {
+        enable = mkIf upstream.servers.local.enable (mkDefault false);
+        accessService = {
+          name = "loki";
+        };
+      };
+    };
+    virtualHosts = let
+      copyFromVhost = mkDefault "loki";
+      vouch.enable = mkDefault true;
+      locations = {
+        "/" = {
+          proxy.enable = true;
+        };
+      };
+    in {
+      loki = {
+        inherit name locations vouch;
+        proxy.upstream = mkDefault upstreamName;
+      };
+      loki'local = {
+        inherit name locations vouch;
+        ssl.cert = {
+          inherit copyFromVhost;
+        };
+        proxy = {
+          inherit copyFromVhost;
+        };
+        local.enable = mkDefault true;
+      };
+    };
+  };
+}

nixos/access/prometheus.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit (lib.modules) mkIf mkDefault;
+  inherit (config.services) prometheus;
+  name.shortServer = mkDefault "prometheus";
+  upstreamName = "prometheus'access";
+in {
+  config.services.nginx = {
+    upstreams'.${upstreamName}.servers = {
+      local = {
+        enable = mkDefault prometheus.enable;
+        addr = mkDefault "localhost";
+        port = mkIf prometheus.enable (mkDefault prometheus.port);
+      };
+      service = {upstream, ...}: {
+        enable = mkIf upstream.servers.local.enable (mkDefault false);
+        accessService = {
+          name = "prometheus";
+        };
+      };
+    };
+    virtualHosts = let
+      copyFromVhost = mkDefault "prometheus";
+      vouch.enable = mkDefault true;
+      locations = {
+        "/" = {
+          proxy.enable = true;
+        };
+      };
+    in {
+      prometheus = {
+        inherit name locations vouch;
+        proxy.upstream = mkDefault upstreamName;
+      };
+      prometheus'local = {
+        inherit name locations;
+        ssl.cert = {
+          inherit copyFromVhost;
+        };
+        proxy = {
+          inherit copyFromVhost;
+        };
+        local.enable = mkDefault true;
+      };
+    };
+  };
+}

nixos/base/monitoring.nix

@@ -1,6 +1,8 @@
-_: {
+{lib, ...}: let
+  inherit (lib.modules) mkDefault;
+in {
   services = {
-    prometheus.exporters.node.enable = true;
+    prometheus.exporters.node.enable = mkDefault true;
-    promtail.enable = true;
+    promtail.enable = mkDefault true;
   };
 }

nixos/monitoring.nix

@@ -12,11 +12,11 @@ in {
     prometheus.enable = true;
   };
   networking.firewall.interfaces.lan.allowedTCPPorts = mkMerge [
-    (mkIf grafana.enable [ grafana.port ])
+    (mkIf grafana.enable [grafana.settings.server.http_port])
     (mkIf loki.enable [
-      loki.settings.httpListenPort
+      loki.configuration.server.http_listen_port
-      (mkIf (loki.settings.grpcListenPort != 0) loki.settings.grpcListenPort)
+      (mkIf (loki.configuration.server.grpc_listen_port != 0) loki.configuration.server.grpc_listen_port)
     ])
-    (mkIf prometheus.enable [ prometheus.port ])
+    (mkIf prometheus.enable [prometheus.port])
   ];
 }

systems/hakurei/nixos.nix

@@ -37,6 +37,9 @@ in {
     nixos.access.freeipa
     nixos.access.freepbx
     nixos.access.unifi
+    nixos.access.prometheus
+    nixos.access.grafana
+    nixos.access.loki
     nixos.access.kitchencam
     nixos.access.openwebrx
     nixos.access.deluge
@@ -173,6 +176,30 @@ in {
         virtualHosts.unifi'local.allServerNames
       ];
     };
+    prometheus = {
+      inherit (nginx) group;
+      domain = virtualHosts.prometheus.serverName;
+      extraDomainNames = mkMerge [
+        virtualHosts.prometheus.otherServerNames
+        virtualHosts.prometheus'local.allServerNames
+      ];
+    };
+    mon = {
+      inherit (nginx) group;
+      domain = virtualHosts.grafana.serverName;
+      extraDomainNames = mkMerge [
+        virtualHosts.grafana.otherServerNames
+        virtualHosts.grafana'local.allServerNames
+      ];
+    };
+    logs = {
+      inherit (nginx) group;
+      domain = virtualHosts.loki.serverName;
+      extraDomainNames = mkMerge [
+        virtualHosts.loki.otherServerNames
+        virtualHosts.loki'local.allServerNames
+      ];
+    };
     idp = {
       inherit (nginx) group;
       domain = virtualHosts.freeipa.serverName;
@@ -292,6 +319,21 @@ in {
         local.denyGlobal = true;
         ssl.cert.enable = true;
       };
+      prometheus = {
+        # we're not the real prometheus record-holder, so don't respond globally..
+        local.denyGlobal = true;
+        ssl.cert.enable = true;
+      };
+      grafana = {
+        # we're not the real mon record-holder, so don't respond globally..
+        local.denyGlobal = true;
+        ssl.cert.enable = true;
+      };
+      loki = {
+        # we're not the real logs record-holder, so don't respond globally..
+        local.denyGlobal = true;
+        ssl.cert.enable = true;
+      };
       home-assistant = {
         # not the real hass record-holder, so don't respond globally..
         local.denyGlobal = true;

systems/utsuho/nixos.nix

@@ -1,8 +1,10 @@
 {
   meta,
   config,
+  lib,
   ...
 }: let
+  inherit (lib.modules) mkMerge;
   inherit (config.services) nginx;
 in {
   imports = let
@@ -16,6 +18,9 @@ in {
     nixos.cloudflared
     nixos.nginx
     nixos.access.unifi
+    nixos.access.prometheus
+    nixos.access.grafana
+    nixos.access.loki
     nixos.unifi
     nixos.dnsmasq
     nixos.mosquitto
@@ -29,7 +34,12 @@ in {
     tunnels.${tunnelId} = {
       default = "http_status:404";
       credentialsFile = config.sops.secrets.cloudflared-tunnel-utsuho.path;
-      ingress = virtualHosts.unifi.proxied.cloudflared.getIngress {};
+      ingress = mkMerge [
+        (virtualHosts.unifi.proxied.cloudflared.getIngress {})
+        (virtualHosts.prometheus.proxied.cloudflared.getIngress {})
+        (virtualHosts.grafana.proxied.cloudflared.getIngress {})
+        (virtualHosts.loki.proxied.cloudflared.getIngress {})
+      ];
     };
   };
 
@@ -37,6 +47,9 @@ in {
     proxied.enable = true;
     virtualHosts = {
       unifi.proxied.enable = "cloudflared";
+      prometheus.proxied.enable = "cloudflared";
+      grafana.proxied.enable = "cloudflared";
+      loki.proxied.enable = "cloudflared";
     };
   };
 

tf/cloudflare_records.tf

@@ -20,6 +20,9 @@ module "hakurei_system_records" {
     "ipa-cock",
     "bw",
     "unifi",
+    "prometheus",
+    "mon",
+    "logs",
     "pbx",
     "smb",
     "mqtt",

tf/cloudflare_tunnels.tf

@@ -74,6 +74,9 @@ module "utsuho" {
   zone_id    = cloudflare_zone.gensokyo-zone_zone.id
   subdomains = [
     "unifi",
+    "prometheus",
+    "mon",
+    "logs",
   ]
 }