diff --git a/modules/system/access.nix b/modules/system/access.nix index 073d91b0..d3ac580e 100644 --- a/modules/system/access.nix +++ b/modules/system/access.nix @@ -161,11 +161,11 @@ serviceId ? null, service ? system.exports.services.${serviceName}, portName ? "default", + port ? service.ports.${portName}, network ? "lan", scheme ? null, getAddressFor ? "getAddressFor", }: let - port = service.ports.${portName}; scheme' = if scheme == null then port.protocol diff --git a/modules/system/exports/home-assistant.nix b/modules/system/exports/home-assistant.nix index fd46454f..d56b311e 100644 --- a/modules/system/exports/home-assistant.nix +++ b/modules/system/exports/home-assistant.nix @@ -28,6 +28,7 @@ in { }; in { id = mkAlmostOptionDefault "home"; + displayName = mkAlmostOptionDefault "Home Assistant"; nixos = { serviceAttr = "home-assistant"; assertions = mkIf config.enable [ @@ -36,13 +37,14 @@ in { ]; }; defaults.port.listen = mkAlmostOptionDefault "lan"; - ports = mapAttrs (_: mapAlmostOptionDefaults) { + ports = { default = { - port = 8123; + port = mkAlmostOptionDefault 8123; protocol = "http"; + status.enable = true; }; homekit0 = { - port = 21063; + port = mkAlmostOptionDefault 21063; transport = "tcp"; }; # TODO: cast udp port range 32768 to 60999 diff --git a/modules/system/exports/ldap.nix b/modules/system/exports/ldap.nix index de1a760a..dc2d0c43 100644 --- a/modules/system/exports/ldap.nix +++ b/modules/system/exports/ldap.nix @@ -12,6 +12,7 @@ in { default = { port = 389; transport = "tcp"; + starttls = true; }; ssl = { port = 636; diff --git a/modules/system/exports/monitoring.nix b/modules/system/exports/monitoring.nix index c03d9716..303b819d 100644 --- a/modules/system/exports/monitoring.nix +++ b/modules/system/exports/monitoring.nix @@ -1,9 +1,61 @@ let - portModule = {lib, ...}: let - inherit (lib.options) mkEnableOption; + portModule = {config, gensokyo-zone, lib, ...}: let + inherit (gensokyo-zone.lib) unmerged; + inherit (lib.options) mkOption mkEnableOption; + inherit (lib.modules) mkIf mkMerge mkOptionDefault; in { - options.prometheus = with lib.types; { - exporter.enable = mkEnableOption "prometheus metrics endpoint"; + options = with lib.types; { + prometheus = { + exporter.enable = mkEnableOption "prometheus metrics endpoint"; + }; + status = { + enable = mkEnableOption "status checks"; + alert = { + enable = mkEnableOption "health check alerts" // { + default = true; + }; + }; + gatus = { + enable = mkEnableOption "gatus" // { + default = true; + }; + settings = mkOption { + type = unmerged.types.attrs; + }; + protocol = mkOption { + type = str; + }; + }; + }; + }; + config = { + status.gatus = let + defaultProtocol = + if config.protocol != null then mkOptionDefault config.protocol + else if config.starttls then mkOptionDefault "starttls" + else if config.ssl then mkOptionDefault "tls" + else if config.transport != "unix" then mkOptionDefault config.transport + else mkIf false (throw "unreachable"); + in { + protocol = defaultProtocol; + settings = mkMerge [ + { + conditions = mkMerge [ + (mkIf (config.ssl || config.starttls) (mkOptionDefault [ + "[CERTIFICATE_EXPIRATION] > 72h" + ])) + (mkOptionDefault [ + "[CONNECTED] == true" + ]) + ]; + } + (mkIf (config.protocol == "http" || config.protocol == "https") { + conditions = mkOptionDefault [ + "[STATUS] == 200" + ]; + }) + ]; + }; }; }; serviceModule = { @@ -18,6 +70,7 @@ let inherit (lib.modules) mkOptionDefault; inherit (lib.attrsets) attrNames filterAttrs; exporterPorts = filterAttrs (_: port: port.enable && port.prometheus.exporter.enable) config.ports; + statusPorts = filterAttrs (_: port: port.enable && port.status.enable) config.ports; in { options = with lib.types; { prometheus = { @@ -34,14 +87,19 @@ let }; }; }; + status = { + ports = mkOption { + type = listOf str; + }; + }; ports = mkOption { type = attrsOf (submoduleWith { modules = [portModule]; }); }; }; - config.prometheus = { - exporter = { + config = { + prometheus.exporter = { ports = mkOptionDefault (attrNames exporterPorts); labels = mapOptionDefaults { gensokyo_exports_service = config.name; @@ -50,6 +108,9 @@ let gensokyo_host = system.access.fqdn; }; }; + status = { + ports = mkOptionDefault (attrNames statusPorts); + }; }; }; in @@ -83,7 +144,7 @@ in protocol = "http"; } // { - prometheus.exporter.enable = true; + prometheus.exporter.enable = mkAlmostOptionDefault true; }; }); exporters = mapListToAttrs mkExporter [ @@ -99,6 +160,11 @@ in type = listOf str; }; }; + status = { + services = mkOption { + type = listOf str; + }; + }; services = mkOption { type = attrsOf (submoduleWith { modules = [serviceModule]; @@ -110,6 +176,11 @@ in in { exporter.services = mkOptionDefault (attrNames exporterServices); }; + config.exports.status = let + statusServices = filterAttrs (_: service: service.enable && service.status.ports != []) config.exports.services; + in { + services = mkOptionDefault (attrNames statusServices); + }; config.exports.services = { prometheus = {config, ...}: { @@ -122,9 +193,10 @@ in }) ]; }; - ports.default = mapAlmostOptionDefaults { - port = 9090; + ports.default = { + port = mkAlmostOptionDefault 9090; protocol = "http"; + status.enable = mkAlmostOptionDefault true; }; }; grafana = {config, ...}: { @@ -138,11 +210,10 @@ in }) ]; }; - ports.default = mapAlmostOptionDefaults { - port = 9092; + ports.default = { + port = mkAlmostOptionDefault 9092; protocol = "http"; - } // { - prometheus.exporter.enable = true; + prometheus.exporter.enable = mkAlmostOptionDefault true; }; }; loki = {config, ...}: { @@ -196,18 +267,15 @@ in }) ]; }; - ports.default = - mapAlmostOptionDefaults { - port = 9094; - protocol = "http"; - } - // { - prometheus.exporter.enable = true; - }; + ports.default = { + port = mkAlmostOptionDefault 9094; + protocol = "http"; + prometheus.exporter.enable = mkAlmostOptionDefault true; + }; #ports.grpc = ... }; gatus = {config, ...}: { - id = mkAlmostOptionDefault "gatus"; + id = mkAlmostOptionDefault "status"; nixos = { serviceAttr = "gatus"; assertions = mkIf config.enable [ @@ -217,11 +285,11 @@ in }) ]; }; - ports.default = - mapAlmostOptionDefaults { - port = 9095; - protocol = "http"; - }; + ports.default = { + port = mkAlmostOptionDefault 9095; + protocol = "http"; + prometheus.exporter.enable = mkAlmostOptionDefault true; + }; #ports.grpc = ... }; } diff --git a/modules/system/exports/services.nix b/modules/system/exports/services.nix index 02530dd5..d3892d78 100644 --- a/modules/system/exports/services.nix +++ b/modules/system/exports/services.nix @@ -42,6 +42,10 @@ type = bool; default = false; }; + starttls = mkOption { + type = bool; + default = false; + }; port = mkOption { type = nullOr int; }; @@ -71,6 +75,10 @@ type = str; default = name; }; + displayName = mkOption { + type = str; + default = name; + }; id = mkOption { type = str; default = config.name; diff --git a/modules/system/exports/zigbee2mqtt.nix b/modules/system/exports/zigbee2mqtt.nix index 2e3f181f..7013de2d 100644 --- a/modules/system/exports/zigbee2mqtt.nix +++ b/modules/system/exports/zigbee2mqtt.nix @@ -8,6 +8,7 @@ in { config.exports.services.zigbee2mqtt = {config, ...}: { id = mkAlmostOptionDefault "z2m"; + displayName = mkAlmostOptionDefault "Zigbee2MQTT"; nixos = { serviceAttr = "zigbee2mqtt"; assertions = mkIf config.enable [ @@ -17,9 +18,10 @@ in { }) ]; }; - ports.default = mapAlmostOptionDefaults { - port = 8072; + ports.default = { + port = mkAlmostOptionDefault 8072; protocol = "http"; + status.enable = true; }; }; } diff --git a/nixos/access/gatus.nix b/nixos/access/gatus.nix new file mode 100644 index 00000000..0c9a2035 --- /dev/null +++ b/nixos/access/gatus.nix @@ -0,0 +1,49 @@ +{ + config, + lib, + ... +}: let + inherit (lib.modules) mkIf mkDefault; + inherit (config.services) gatus; + name.shortServer = mkDefault "status"; + upstreamName = "gatus'access"; +in { + config.services.nginx = { + upstreams'.${upstreamName}.servers = { + local = { + enable = mkDefault gatus.enable; + addr = mkDefault "localhost"; + port = mkIf gatus.enable (mkDefault gatus.settings.web.port); + }; + service = {upstream, ...}: { + enable = mkIf upstream.servers.local.enable (mkDefault false); + accessService = { + name = "gatus"; + }; + }; + }; + virtualHosts = let + copyFromVhost = mkDefault "gatus"; + locations = { + "/" = { + proxy.enable = true; + }; + }; + in { + gatus = { + inherit name locations; + proxy.upstream = mkDefault upstreamName; + }; + gatus'local = { + inherit name locations; + ssl.cert = { + inherit copyFromVhost; + }; + proxy = { + inherit copyFromVhost; + }; + local.enable = mkDefault true; + }; + }; + }; +} diff --git a/nixos/monitoring/gatus.nix b/nixos/monitoring/gatus.nix index 9c93dcfa..01990a7c 100644 --- a/nixos/monitoring/gatus.nix +++ b/nixos/monitoring/gatus.nix @@ -1,86 +1,107 @@ -{ config, ... }: { +{ + config, + access, + gensokyo-zone, + lib, + ... +}: let + inherit (gensokyo-zone) systems; + inherit (gensokyo-zone.lib) mkAlmostOptionDefault unmerged; + inherit (lib.modules) mkMerge mkOptionDefault; + inherit (lib.attrsets) attrValues nameValuePair listToAttrs; + inherit (lib.lists) filter length optional concatMap; + cfg = config.services.gatus; + statusSystems = filter (system: system.config.access.online.enable) (attrValues systems); + mapSystem = system: let + statusServices = map (serviceName: system.config.exports.services.${serviceName}) system.config.exports.status.services; + in concatMap (mkServiceEndpoint system) statusServices; + mkPortEndpoint = { system, service, port, unique }: let + inherit (port.status) gatus; + name = if unique + then service.displayName + else "${service.displayName}: ${port.name}"; + conf = { + url = mkOptionDefault (access.proxyUrlFor { + inherit service port; + system = system.config; + scheme = gatus.protocol; + #network = port.listen; + }); + }; + in nameValuePair name ({ ... }: { + imports = + optional port.status.alert.enable alertingConfigAlerts + ++ optional (gatus.protocol == "http" || gatus.protocol == "https") alertingConfigHttp; + + config = mkMerge [ + (unmerged.mergeAttrs gatus.settings) + conf + ]; + }); + mkServiceEndpoint = system: service: let + statusPorts = map /*lib.attrsets.getAttr*/(portName: service.ports.${portName}) service.status.ports; + gatusPorts = filter (port: port.status.gatus.enable) statusPorts; + unique = length gatusPorts == 1; + in map (port: mkPortEndpoint { + inherit system service port unique; + }) gatusPorts; + alertingConfigAlerts = { + alerts = [ + { + type = "discord"; + send-on-resolved = true; + description = "Healthcheck failed."; + failure-threshold = 1; + success-threshold = 3; + } + ]; + }; + alertingConfigHttp = { + # Common interval for refreshing all basic HTTP endpoints + interval = mkAlmostOptionDefault "30s"; + }; +in { sops.secrets.gatus_environment_file = { sopsFile = ../secrets/gatus.yaml; }; services.gatus = { enable = true; environmentFile = config.sops.secrets.gatus_environment_file.path; - settings = let - # Common interval for refreshing all basic HTTP endpoints - gatusCommonHTTPInterval = "30s"; + settings = { + # Environment variables are pulled in to be usable within the config. + alerting.discord = { + webhook-url = "\${DISCORD_WEBHOOK_URL}"; + }; - # Shared between all endpoints - commonAlertingConfig = { - alerts = [ - { - type = "discord"; - send-on-resolved = true; - description = "Healthcheck failed."; - failure-threshold = 1; - success-threshold = 3; - } - ]; - }; - # Used wherever a basic HTTP 200 up-check is required. - basicHTTPCheck = url: { - inherit url; - interval = gatusCommonHTTPInterval; - conditions = [ - "[STATUS] == 200" - ]; - }; - in { - # Environment variables are pulled in to be usable within the config. - alerting.discord = { - webhook-url = "\${DISCORD_WEBHOOK_URL}"; - }; + # Endpoint configuration + endpoints = listToAttrs (concatMap mapSystem statusSystems); - # Endpoint configuration - endpoints = { - # Home Assistant uses the common alerting config, combined with a basic HTTP check for its domain. - "Home Assistant" = commonAlertingConfig // (basicHTTPCheck "https://home.local.gensokyo.zone"); - }; + # The actual status page configuration + ui = { + title = "Gensokyo Zone Status"; + description = "The status of the various girls in Gensokyo!"; + header = "Gensokyo Zone Status"; + }; - # The actual status page configuration - ui = { - title = "Gensokyo Zone Status"; - description = "The status of the various girls in Gensokyo!"; - header = "Gensokyo Zone Status"; - }; + # Prometheus metrics...! + metrics = true; - # Prometheus metrics...! - metrics = true; - - # We could've used Postgres, but it seems like less moving parts if our status page - # doesn't depend upon another service, internal or external, other than what gets it to the internet. - storage = { - type = "sqlite"; - path = "/var/lib/gatus/data.db"; - }; - - # Bind on the local address for now, on the port after the last one allocated for the monitoring project. - web = { - address = "10.1.1.38"; - port = 9095; - }; + # We could've used Postgres, but it seems like less moving parts if our status page + # doesn't depend upon another service, internal or external, other than what gets it to the internet. + storage = { + type = "sqlite"; + path = "/var/lib/gatus/data.db"; + }; + # Bind on the local address for now, on the port after the last one allocated for the monitoring project. + web = { + address = "[::]"; + port = 9095; + }; }; }; -/* services.nginx.virtualHosts."status.gensokyo.zone" = let - gatusWebCfg = config.services.gatus.settings.web; - upstream = "${gatusWebCfg.address}:${toString gatusWebCfg.port}"; - in { - forceSSL = true; - useACMEHost = serverName; - kTLS = true; - locations."/" = { - proxyPass = "http://${upstream}"; - proxyWebsockets = true; - }; - }; */ - - networking.firewall.interfaces.local.allowedTCPPorts = [ - config.services.gatus.settings.web.port + networking.firewall.interfaces.lan.allowedTCPPorts = [ + cfg.settings.web.port ]; } diff --git a/systems/hakurei/nixos.nix b/systems/hakurei/nixos.nix index 05f0d0ec..a3c0a660 100644 --- a/systems/hakurei/nixos.nix +++ b/systems/hakurei/nixos.nix @@ -37,6 +37,7 @@ in { nixos.access.freeipa nixos.access.freepbx nixos.access.unifi + nixos.access.gatus nixos.access.prometheus nixos.access.grafana nixos.access.loki @@ -176,6 +177,14 @@ in { virtualHosts.unifi'local.allServerNames ]; }; + status = { + inherit (nginx) group; + domain = virtualHosts.gatus.serverName; + extraDomainNames = mkMerge [ + virtualHosts.gatus.otherServerNames + virtualHosts.gatus'local.allServerNames + ]; + }; prometheus = { inherit (nginx) group; domain = virtualHosts.prometheus.serverName; @@ -319,6 +328,11 @@ in { local.denyGlobal = true; ssl.cert.enable = true; }; + gatus = { + # we're not the real gatus record-holder, so don't respond globally.. + local.denyGlobal = true; + ssl.cert.enable = true; + }; prometheus = { # we're not the real prometheus record-holder, so don't respond globally.. local.denyGlobal = true; diff --git a/systems/utsuho/nixos.nix b/systems/utsuho/nixos.nix index a1489638..8c26dbf4 100644 --- a/systems/utsuho/nixos.nix +++ b/systems/utsuho/nixos.nix @@ -18,6 +18,7 @@ in { nixos.cloudflared nixos.nginx nixos.access.unifi + nixos.access.gatus nixos.access.prometheus nixos.access.grafana nixos.access.loki @@ -36,6 +37,7 @@ in { credentialsFile = config.sops.secrets.cloudflared-tunnel-utsuho.path; ingress = mkMerge [ (virtualHosts.unifi.proxied.cloudflared.getIngress {}) + (virtualHosts.gatus.proxied.cloudflared.getIngress {}) (virtualHosts.prometheus.proxied.cloudflared.getIngress {}) (virtualHosts.grafana.proxied.cloudflared.getIngress {}) (virtualHosts.loki.proxied.cloudflared.getIngress {}) @@ -47,6 +49,7 @@ in { proxied.enable = true; virtualHosts = { unifi.proxied.enable = "cloudflared"; + gatus.proxied.enable = "cloudflared"; prometheus.proxied.enable = "cloudflared"; grafana.proxied.enable = "cloudflared"; loki.proxied.enable = "cloudflared"; diff --git a/tf/cloudflare_records.tf b/tf/cloudflare_records.tf index 64116809..96530f9e 100644 --- a/tf/cloudflare_records.tf +++ b/tf/cloudflare_records.tf @@ -20,6 +20,7 @@ module "hakurei_system_records" { "ipa-cock", "bw", "unifi", + "status", "prometheus", "mon", "logs", diff --git a/tf/cloudflare_tunnels.tf b/tf/cloudflare_tunnels.tf index 9b3ce3c4..6a77766b 100644 --- a/tf/cloudflare_tunnels.tf +++ b/tf/cloudflare_tunnels.tf @@ -74,6 +74,7 @@ module "utsuho" { zone_id = cloudflare_zone.gensokyo-zone_zone.id subdomains = [ "unifi", + "status", "prometheus", "mon", "logs",