feat(monitoring): service status lookup

modules/system/access.nix

@@ -161,11 +161,11 @@
             serviceId ? null,
             service ? system.exports.services.${serviceName},
             portName ? "default",
+            port ? service.ports.${portName},
             network ? "lan",
             scheme ? null,
             getAddressFor ? "getAddressFor",
           }: let
-            port = service.ports.${portName};
             scheme' =
               if scheme == null
               then port.protocol

modules/system/exports/home-assistant.nix

@@ -28,6 +28,7 @@ in {
       };
   in {
     id = mkAlmostOptionDefault "home";
+    displayName = mkAlmostOptionDefault "Home Assistant";
     nixos = {
       serviceAttr = "home-assistant";
       assertions = mkIf config.enable [
@@ -36,13 +37,14 @@ in {
       ];
     };
     defaults.port.listen = mkAlmostOptionDefault "lan";
-    ports = mapAttrs (_: mapAlmostOptionDefaults) {
+    ports = {
       default = {
-        port = 8123;
+        port = mkAlmostOptionDefault 8123;
         protocol = "http";
+        status.enable = true;
       };
       homekit0 = {
-        port = 21063;
+        port = mkAlmostOptionDefault 21063;
         transport = "tcp";
       };
       # TODO: cast udp port range 32768 to 60999

modules/system/exports/ldap.nix

@@ -12,6 +12,7 @@ in {
       default = {
         port = 389;
         transport = "tcp";
+        starttls = true;
       };
       ssl = {
         port = 636;

modules/system/exports/monitoring.nix

@@ -1,9 +1,61 @@
 let
-  portModule = {lib, ...}: let
+  portModule = {config, gensokyo-zone, lib, ...}: let
-    inherit (lib.options) mkEnableOption;
+    inherit (gensokyo-zone.lib) unmerged;
+    inherit (lib.options) mkOption mkEnableOption;
+    inherit (lib.modules) mkIf mkMerge mkOptionDefault;
   in {
-    options.prometheus = with lib.types; {
+    options = with lib.types; {
-      exporter.enable = mkEnableOption "prometheus metrics endpoint";
+      prometheus = {
+        exporter.enable = mkEnableOption "prometheus metrics endpoint";
+      };
+      status = {
+        enable = mkEnableOption "status checks";
+        alert = {
+          enable = mkEnableOption "health check alerts" // {
+            default = true;
+          };
+        };
+        gatus = {
+          enable = mkEnableOption "gatus" // {
+            default = true;
+          };
+          settings = mkOption {
+            type = unmerged.types.attrs;
+          };
+          protocol = mkOption {
+            type = str;
+          };
+        };
+      };
+    };
+    config = {
+      status.gatus = let
+        defaultProtocol =
+          if config.protocol != null then mkOptionDefault config.protocol
+          else if config.starttls then mkOptionDefault "starttls"
+          else if config.ssl then mkOptionDefault "tls"
+          else if config.transport != "unix" then mkOptionDefault config.transport
+          else mkIf false (throw "unreachable");
+      in {
+        protocol = defaultProtocol;
+        settings = mkMerge [
+          {
+            conditions = mkMerge [
+              (mkIf (config.ssl || config.starttls) (mkOptionDefault [
+                "[CERTIFICATE_EXPIRATION] > 72h"
+              ]))
+              (mkOptionDefault [
+                "[CONNECTED] == true"
+              ])
+            ];
+          }
+          (mkIf (config.protocol == "http" || config.protocol == "https") {
+            conditions = mkOptionDefault [
+              "[STATUS] == 200"
+            ];
+          })
+        ];
+      };
     };
   };
   serviceModule = {
@@ -18,6 +70,7 @@ let
     inherit (lib.modules) mkOptionDefault;
     inherit (lib.attrsets) attrNames filterAttrs;
     exporterPorts = filterAttrs (_: port: port.enable && port.prometheus.exporter.enable) config.ports;
+    statusPorts = filterAttrs (_: port: port.enable && port.status.enable) config.ports;
   in {
     options = with lib.types; {
       prometheus = {
@@ -34,14 +87,19 @@ let
           };
         };
       };
+      status = {
+        ports = mkOption {
+          type = listOf str;
+        };
+      };
       ports = mkOption {
         type = attrsOf (submoduleWith {
           modules = [portModule];
         });
       };
     };
-    config.prometheus = {
+    config = {
-      exporter = {
+      prometheus.exporter = {
         ports = mkOptionDefault (attrNames exporterPorts);
         labels = mapOptionDefaults {
           gensokyo_exports_service = config.name;
@@ -50,6 +108,9 @@ let
           gensokyo_host = system.access.fqdn;
         };
       };
+      status = {
+        ports = mkOptionDefault (attrNames statusPorts);
+      };
     };
   };
 in
@@ -83,7 +144,7 @@ in
             protocol = "http";
           }
           // {
-            prometheus.exporter.enable = true;
+            prometheus.exporter.enable = mkAlmostOptionDefault true;
           };
       });
     exporters = mapListToAttrs mkExporter [
@@ -99,6 +160,11 @@ in
           type = listOf str;
         };
       };
+      status = {
+        services = mkOption {
+          type = listOf str;
+        };
+      };
       services = mkOption {
         type = attrsOf (submoduleWith {
           modules = [serviceModule];
@@ -110,6 +176,11 @@ in
     in {
       exporter.services = mkOptionDefault (attrNames exporterServices);
     };
+    config.exports.status = let
+      statusServices = filterAttrs (_: service: service.enable && service.status.ports != []) config.exports.services;
+    in {
+      services = mkOptionDefault (attrNames statusServices);
+    };
     config.exports.services =
       {
         prometheus = {config, ...}: {
@@ -122,9 +193,10 @@ in
               })
             ];
           };
-          ports.default = mapAlmostOptionDefaults {
+          ports.default = {
-            port = 9090;
+            port = mkAlmostOptionDefault 9090;
             protocol = "http";
+            status.enable = mkAlmostOptionDefault true;
           };
         };
         grafana = {config, ...}: {
@@ -138,11 +210,10 @@ in
               })
             ];
           };
-          ports.default = mapAlmostOptionDefaults {
+          ports.default = {
-            port = 9092;
+            port = mkAlmostOptionDefault 9092;
             protocol = "http";
-          } // {
+            prometheus.exporter.enable = mkAlmostOptionDefault true;
-            prometheus.exporter.enable = true;
           };
         };
         loki = {config, ...}: {
@@ -196,18 +267,15 @@ in
               })
             ];
           };
-          ports.default =
+          ports.default = {
-            mapAlmostOptionDefaults {
+            port = mkAlmostOptionDefault 9094;
-              port = 9094;
+            protocol = "http";
-              protocol = "http";
+            prometheus.exporter.enable = mkAlmostOptionDefault true;
-            }
+          };
-            // {
-              prometheus.exporter.enable = true;
-            };
           #ports.grpc = ...
         };
         gatus = {config, ...}: {
-          id = mkAlmostOptionDefault "gatus";
+          id = mkAlmostOptionDefault "status";
           nixos = {
             serviceAttr = "gatus";
             assertions = mkIf config.enable [
@@ -217,11 +285,11 @@ in
               })
             ];
           };
-          ports.default =
+          ports.default = {
-            mapAlmostOptionDefaults {
+            port = mkAlmostOptionDefault 9095;
-              port = 9095;
+            protocol = "http";
-              protocol = "http";
+            prometheus.exporter.enable = mkAlmostOptionDefault true;
-            };
+          };
           #ports.grpc = ...
         };
       }

modules/system/exports/services.nix

@@ -42,6 +42,10 @@
         type = bool;
         default = false;
       };
+      starttls = mkOption {
+        type = bool;
+        default = false;
+      };
       port = mkOption {
         type = nullOr int;
       };
@@ -71,6 +75,10 @@
         type = str;
         default = name;
       };
+      displayName = mkOption {
+        type = str;
+        default = name;
+      };
       id = mkOption {
         type = str;
         default = config.name;

modules/system/exports/zigbee2mqtt.nix

@@ -8,6 +8,7 @@
 in {
   config.exports.services.zigbee2mqtt = {config, ...}: {
     id = mkAlmostOptionDefault "z2m";
+    displayName = mkAlmostOptionDefault "Zigbee2MQTT";
     nixos = {
       serviceAttr = "zigbee2mqtt";
       assertions = mkIf config.enable [
@@ -17,9 +18,10 @@ in {
         })
       ];
     };
-    ports.default = mapAlmostOptionDefaults {
+    ports.default = {
-      port = 8072;
+      port = mkAlmostOptionDefault 8072;
       protocol = "http";
+      status.enable = true;
     };
   };
 }

nixos/access/gatus.nix

@@ -0,0 +1,49 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  inherit (lib.modules) mkIf mkDefault;
+  inherit (config.services) gatus;
+  name.shortServer = mkDefault "status";
+  upstreamName = "gatus'access";
+in {
+  config.services.nginx = {
+    upstreams'.${upstreamName}.servers = {
+      local = {
+        enable = mkDefault gatus.enable;
+        addr = mkDefault "localhost";
+        port = mkIf gatus.enable (mkDefault gatus.settings.web.port);
+      };
+      service = {upstream, ...}: {
+        enable = mkIf upstream.servers.local.enable (mkDefault false);
+        accessService = {
+          name = "gatus";
+        };
+      };
+    };
+    virtualHosts = let
+      copyFromVhost = mkDefault "gatus";
+      locations = {
+        "/" = {
+          proxy.enable = true;
+        };
+      };
+    in {
+      gatus = {
+        inherit name locations;
+        proxy.upstream = mkDefault upstreamName;
+      };
+      gatus'local = {
+        inherit name locations;
+        ssl.cert = {
+          inherit copyFromVhost;
+        };
+        proxy = {
+          inherit copyFromVhost;
+        };
+        local.enable = mkDefault true;
+      };
+    };
+  };
+}

nixos/monitoring/gatus.nix

@@ -1,86 +1,107 @@
-{ config, ... }: {
+{
+  config,
+  access,
+  gensokyo-zone,
+  lib,
+  ...
+}: let
+  inherit (gensokyo-zone) systems;
+  inherit (gensokyo-zone.lib) mkAlmostOptionDefault unmerged;
+  inherit (lib.modules) mkMerge mkOptionDefault;
+  inherit (lib.attrsets) attrValues nameValuePair listToAttrs;
+  inherit (lib.lists) filter length optional concatMap;
+  cfg = config.services.gatus;
+  statusSystems = filter (system: system.config.access.online.enable) (attrValues systems);
+  mapSystem = system: let
+    statusServices = map (serviceName: system.config.exports.services.${serviceName}) system.config.exports.status.services;
+  in concatMap (mkServiceEndpoint system) statusServices;
+  mkPortEndpoint = { system, service, port, unique }: let
+    inherit (port.status) gatus;
+    name = if unique
+      then service.displayName
+      else "${service.displayName}: ${port.name}";
+    conf = {
+      url = mkOptionDefault (access.proxyUrlFor {
+        inherit service port;
+        system = system.config;
+        scheme = gatus.protocol;
+        #network = port.listen;
+      });
+    };
+  in nameValuePair name ({ ... }: {
+    imports =
+      optional port.status.alert.enable alertingConfigAlerts
+      ++ optional (gatus.protocol == "http" || gatus.protocol == "https") alertingConfigHttp;
+
+    config = mkMerge [
+      (unmerged.mergeAttrs gatus.settings)
+      conf
+    ];
+  });
+  mkServiceEndpoint = system: service: let
+    statusPorts = map /*lib.attrsets.getAttr*/(portName: service.ports.${portName}) service.status.ports;
+    gatusPorts = filter (port: port.status.gatus.enable) statusPorts;
+    unique = length gatusPorts == 1;
+  in map (port: mkPortEndpoint {
+    inherit system service port unique;
+  }) gatusPorts;
+  alertingConfigAlerts = {
+    alerts = [
+      {
+        type = "discord";
+        send-on-resolved = true;
+        description = "Healthcheck failed.";
+        failure-threshold = 1;
+        success-threshold = 3;
+      }
+    ];
+  };
+  alertingConfigHttp = {
+    # Common interval for refreshing all basic HTTP endpoints
+    interval = mkAlmostOptionDefault "30s";
+  };
+in {
   sops.secrets.gatus_environment_file = {
     sopsFile = ../secrets/gatus.yaml;
   };
   services.gatus = {
     enable = true;
     environmentFile = config.sops.secrets.gatus_environment_file.path;
-    settings = let
+    settings = {
-        # Common interval for refreshing all basic HTTP endpoints
+      # Environment variables are pulled in to be usable within the config.
-        gatusCommonHTTPInterval = "30s";
+      alerting.discord = {
+        webhook-url = "\${DISCORD_WEBHOOK_URL}";
+      };
 
-        # Shared between all endpoints
+      # Endpoint configuration
-        commonAlertingConfig = {
+      endpoints = listToAttrs (concatMap mapSystem statusSystems);
-            alerts = [
-                {
-                    type = "discord";
-                    send-on-resolved = true;
-                    description = "Healthcheck failed.";
-                    failure-threshold = 1;
-                    success-threshold = 3;
-                }
-            ];
-        };
-        # Used wherever a basic HTTP 200 up-check is required.
-        basicHTTPCheck = url: {
-            inherit url;
-            interval = gatusCommonHTTPInterval;
-            conditions = [
-                "[STATUS] == 200"
-            ];
-         };
-     in {
-        # Environment variables are pulled in to be usable within the config.
-        alerting.discord = {
-            webhook-url = "\${DISCORD_WEBHOOK_URL}";
-        };
 
-        # Endpoint configuration
+      # The actual status page configuration
-        endpoints = {
+      ui = {
-            # Home Assistant uses the common alerting config, combined with a basic HTTP check for its domain.
+        title = "Gensokyo Zone Status";
-            "Home Assistant" = commonAlertingConfig // (basicHTTPCheck "https://home.local.gensokyo.zone");
+        description = "The status of the various girls in Gensokyo!";
-        };
+        header = "Gensokyo Zone Status";
+      };
 
-        # The actual status page configuration
+      # Prometheus metrics...!
-        ui = {
+      metrics = true;
-            title = "Gensokyo Zone Status";
-            description = "The status of the various girls in Gensokyo!";
-            header = "Gensokyo Zone Status";
-        };
 
-        # Prometheus metrics...!
+      # We could've used Postgres, but it seems like less moving parts if our status page
-        metrics = true;
+      # doesn't depend upon another service, internal or external, other than what gets it to the internet.
-
+      storage = {
-        # We could've used Postgres, but it seems like less moving parts if our status page
+        type = "sqlite";
-        # doesn't depend upon another service, internal or external, other than what gets it to the internet.
+        path = "/var/lib/gatus/data.db";
-        storage = {
+      };
-          type = "sqlite";
-          path = "/var/lib/gatus/data.db";
-        };
-
-        # Bind on the local address for now, on the port after the last one allocated for the monitoring project.
-        web = {
-          address = "10.1.1.38";
-          port = 9095;
-        };
 
+      # Bind on the local address for now, on the port after the last one allocated for the monitoring project.
+      web = {
+        address = "[::]";
+        port = 9095;
+      };
     };
   };
 
-/*  services.nginx.virtualHosts."status.gensokyo.zone" = let
+  networking.firewall.interfaces.lan.allowedTCPPorts = [
-    gatusWebCfg = config.services.gatus.settings.web;
+    cfg.settings.web.port
-    upstream = "${gatusWebCfg.address}:${toString gatusWebCfg.port}";
-  in {
-    forceSSL = true;
-    useACMEHost = serverName;
-    kTLS = true;
-    locations."/" = {
-      proxyPass = "http://${upstream}";
-      proxyWebsockets = true;
-    };
-  }; */
-
-  networking.firewall.interfaces.local.allowedTCPPorts = [
-    config.services.gatus.settings.web.port
   ];
 }

systems/hakurei/nixos.nix

@@ -37,6 +37,7 @@ in {
     nixos.access.freeipa
     nixos.access.freepbx
     nixos.access.unifi
+    nixos.access.gatus
     nixos.access.prometheus
     nixos.access.grafana
     nixos.access.loki
@@ -176,6 +177,14 @@ in {
         virtualHosts.unifi'local.allServerNames
       ];
     };
+    status = {
+      inherit (nginx) group;
+      domain = virtualHosts.gatus.serverName;
+      extraDomainNames = mkMerge [
+        virtualHosts.gatus.otherServerNames
+        virtualHosts.gatus'local.allServerNames
+      ];
+    };
     prometheus = {
       inherit (nginx) group;
       domain = virtualHosts.prometheus.serverName;
@@ -319,6 +328,11 @@ in {
         local.denyGlobal = true;
         ssl.cert.enable = true;
       };
+      gatus = {
+        # we're not the real gatus record-holder, so don't respond globally..
+        local.denyGlobal = true;
+        ssl.cert.enable = true;
+      };
       prometheus = {
         # we're not the real prometheus record-holder, so don't respond globally..
         local.denyGlobal = true;

systems/utsuho/nixos.nix

@@ -18,6 +18,7 @@ in {
     nixos.cloudflared
     nixos.nginx
     nixos.access.unifi
+    nixos.access.gatus
     nixos.access.prometheus
     nixos.access.grafana
     nixos.access.loki
@@ -36,6 +37,7 @@ in {
       credentialsFile = config.sops.secrets.cloudflared-tunnel-utsuho.path;
       ingress = mkMerge [
         (virtualHosts.unifi.proxied.cloudflared.getIngress {})
+        (virtualHosts.gatus.proxied.cloudflared.getIngress {})
         (virtualHosts.prometheus.proxied.cloudflared.getIngress {})
         (virtualHosts.grafana.proxied.cloudflared.getIngress {})
         (virtualHosts.loki.proxied.cloudflared.getIngress {})
@@ -47,6 +49,7 @@ in {
     proxied.enable = true;
     virtualHosts = {
       unifi.proxied.enable = "cloudflared";
+      gatus.proxied.enable = "cloudflared";
       prometheus.proxied.enable = "cloudflared";
       grafana.proxied.enable = "cloudflared";
       loki.proxied.enable = "cloudflared";

tf/cloudflare_records.tf

@@ -20,6 +20,7 @@ module "hakurei_system_records" {
     "ipa-cock",
     "bw",
     "unifi",
+    "status",
     "prometheus",
     "mon",
     "logs",

tf/cloudflare_tunnels.tf

@@ -74,6 +74,7 @@ module "utsuho" {
   zone_id    = cloudflare_zone.gensokyo-zone_zone.id
   subdomains = [
     "unifi",
+    "status",
     "prometheus",
     "mon",
     "logs",