gensokyo-zone/infrastructure
1
0
Fork 0
mirror of https://github.com/gensokyo-zone/infrastructure.git synced 2026-02-09 04:19:19 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(monitoring): service status lookup

Browse source
This commit is contained in:
arcnmx 2024-05-31 18:14:37 -07:00
parent f97ab24f47
commit ebfd1f5a9a
12 changed files with 273 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/system/access.nix
View file

@ -161,11 +161,11 @@
serviceId ? null,
service ? system.exports.services.${serviceName},
portName ? "default",
port ? service.ports.${portName},
network ? "lan",
scheme ? null,
getAddressFor ? "getAddressFor",
}: let
port = service.ports.${portName};
scheme' =
if scheme == null
then port.protocol

8
modules/system/exports/home-assistant.nix
View file

@ -28,6 +28,7 @@ in {
};
in {
id = mkAlmostOptionDefault "home";
displayName = mkAlmostOptionDefault "Home Assistant";
nixos = {
serviceAttr = "home-assistant";
assertions = mkIf config.enable [
@ -36,13 +37,14 @@ in {
];
};
defaults.port.listen = mkAlmostOptionDefault "lan";
ports = mapAttrs (_: mapAlmostOptionDefaults) {
ports = {
default = {
port = 8123;
port = mkAlmostOptionDefault 8123;
protocol = "http";
status.enable = true;
};
homekit0 = {
port = 21063;
port = mkAlmostOptionDefault 21063;
transport = "tcp";
};
# TODO: cast udp port range 32768 to 60999

1
modules/system/exports/ldap.nix
View file

@ -12,6 +12,7 @@ in {
default = {
port = 389;
transport = "tcp";
starttls = true;
};
ssl = {
port = 636;

112
modules/system/exports/monitoring.nix
View file

@ -1,10 +1,62 @@
let
portModule = {lib, ...}: let
inherit (lib.options) mkEnableOption;
portModule = {config, gensokyo-zone, lib, ...}: let
inherit (gensokyo-zone.lib) unmerged;
inherit (lib.options) mkOption mkEnableOption;
inherit (lib.modules) mkIf mkMerge mkOptionDefault;
in {
options.prometheus = with lib.types; {
options = with lib.types; {
prometheus = {
exporter.enable = mkEnableOption "prometheus metrics endpoint";
};
status = {
enable = mkEnableOption "status checks";
alert = {
enable = mkEnableOption "health check alerts" // {
default = true;
};
};
gatus = {
enable = mkEnableOption "gatus" // {
default = true;
};
settings = mkOption {
type = unmerged.types.attrs;
};
protocol = mkOption {
type = str;
};
};
};
};
config = {
status.gatus = let
defaultProtocol =
if config.protocol != null then mkOptionDefault config.protocol
else if config.starttls then mkOptionDefault "starttls"
else if config.ssl then mkOptionDefault "tls"
else if config.transport != "unix" then mkOptionDefault config.transport
else mkIf false (throw "unreachable");
in {
protocol = defaultProtocol;
settings = mkMerge [
{
conditions = mkMerge [
(mkIf (config.ssl || config.starttls) (mkOptionDefault [
"[CERTIFICATE_EXPIRATION] > 72h"
]))
(mkOptionDefault [
"[CONNECTED] == true"
])
];
}
(mkIf (config.protocol == "http" || config.protocol == "https") {
conditions = mkOptionDefault [
"[STATUS] == 200"
];
})
];
};
};
};
serviceModule = {
system,
@ -18,6 +70,7 @@ let
inherit (lib.modules) mkOptionDefault;
inherit (lib.attrsets) attrNames filterAttrs;
exporterPorts = filterAttrs (_: port: port.enable && port.prometheus.exporter.enable) config.ports;
statusPorts = filterAttrs (_: port: port.enable && port.status.enable) config.ports;
in {
options = with lib.types; {
prometheus = {
@ -34,14 +87,19 @@ let
};
};
};
status = {
ports = mkOption {
type = listOf str;
};
};
ports = mkOption {
type = attrsOf (submoduleWith {
modules = [portModule];
});
};
};
config.prometheus = {
exporter = {
config = {
prometheus.exporter = {
ports = mkOptionDefault (attrNames exporterPorts);
labels = mapOptionDefaults {
gensokyo_exports_service = config.name;
@ -50,6 +108,9 @@ let
gensokyo_host = system.access.fqdn;
};
};
status = {
ports = mkOptionDefault (attrNames statusPorts);
};
};
};
in
@ -83,7 +144,7 @@ in
protocol = "http";
}
// {
prometheus.exporter.enable = true;
prometheus.exporter.enable = mkAlmostOptionDefault true;
};
});
exporters = mapListToAttrs mkExporter [
@ -99,6 +160,11 @@ in
type = listOf str;
};
};
status = {
services = mkOption {
type = listOf str;
};
};
services = mkOption {
type = attrsOf (submoduleWith {
modules = [serviceModule];
@ -110,6 +176,11 @@ in
in {
exporter.services = mkOptionDefault (attrNames exporterServices);
};
config.exports.status = let
statusServices = filterAttrs (_: service: service.enable && service.status.ports != []) config.exports.services;
in {
services = mkOptionDefault (attrNames statusServices);
};
config.exports.services =
{
prometheus = {config, ...}: {
@ -122,9 +193,10 @@ in
})
];
};
ports.default = mapAlmostOptionDefaults {
port = 9090;
ports.default = {
port = mkAlmostOptionDefault 9090;
protocol = "http";
status.enable = mkAlmostOptionDefault true;
};
};
grafana = {config, ...}: {
@ -138,11 +210,10 @@ in
})
];
};
ports.default = mapAlmostOptionDefaults {
port = 9092;
ports.default = {
port = mkAlmostOptionDefault 9092;
protocol = "http";
} // {
prometheus.exporter.enable = true;
prometheus.exporter.enable = mkAlmostOptionDefault true;
};
};
loki = {config, ...}: {
@ -196,18 +267,15 @@ in
})
];
};
ports.default =
mapAlmostOptionDefaults {
port = 9094;
ports.default = {
port = mkAlmostOptionDefault 9094;
protocol = "http";
}
// {
prometheus.exporter.enable = true;
prometheus.exporter.enable = mkAlmostOptionDefault true;
};
#ports.grpc = ...
};
gatus = {config, ...}: {
id = mkAlmostOptionDefault "gatus";
id = mkAlmostOptionDefault "status";
nixos = {
serviceAttr = "gatus";
assertions = mkIf config.enable [
@ -217,10 +285,10 @@ in
})
];
};
ports.default =
mapAlmostOptionDefaults {
port = 9095;
ports.default = {
port = mkAlmostOptionDefault 9095;
protocol = "http";
prometheus.exporter.enable = mkAlmostOptionDefault true;
};
#ports.grpc = ...
};

8
modules/system/exports/services.nix
View file

@ -42,6 +42,10 @@
type = bool;
default = false;
};
starttls = mkOption {
type = bool;
default = false;
};
port = mkOption {
type = nullOr int;
};
@ -71,6 +75,10 @@
type = str;
default = name;
};
displayName = mkOption {
type = str;
default = name;
};
id = mkOption {
type = str;
default = config.name;

6
modules/system/exports/zigbee2mqtt.nix
View file

@ -8,6 +8,7 @@
in {
config.exports.services.zigbee2mqtt = {config, ...}: {
id = mkAlmostOptionDefault "z2m";
displayName = mkAlmostOptionDefault "Zigbee2MQTT";
nixos = {
serviceAttr = "zigbee2mqtt";
assertions = mkIf config.enable [
@ -17,9 +18,10 @@ in {
})
];
};
ports.default = mapAlmostOptionDefaults {
port = 8072;
ports.default = {
port = mkAlmostOptionDefault 8072;
protocol = "http";
status.enable = true;
};
};
}

49
nixos/access/gatus.nix Normal file
View file

@ -0,0 +1,49 @@
{
config,
lib,
...
}: let
inherit (lib.modules) mkIf mkDefault;
inherit (config.services) gatus;
name.shortServer = mkDefault "status";
upstreamName = "gatus'access";
in {
config.services.nginx = {
upstreams'.${upstreamName}.servers = {
local = {
enable = mkDefault gatus.enable;
addr = mkDefault "localhost";
port = mkIf gatus.enable (mkDefault gatus.settings.web.port);
};
service = {upstream, ...}: {
enable = mkIf upstream.servers.local.enable (mkDefault false);
accessService = {
name = "gatus";
};
};
};
virtualHosts = let
copyFromVhost = mkDefault "gatus";
locations = {
"/" = {
proxy.enable = true;
};
};
in {
gatus = {
inherit name locations;
proxy.upstream = mkDefault upstreamName;
};
gatus'local = {
inherit name locations;
ssl.cert = {
inherit copyFromVhost;
};
proxy = {
inherit copyFromVhost;
};
local.enable = mkDefault true;
};
};
};
}

99
nixos/monitoring/gatus.nix
View file

@ -1,16 +1,51 @@
{ config, ... }: {
sops.secrets.gatus_environment_file = {
sopsFile = ../secrets/gatus.yaml;
{
config,
access,
gensokyo-zone,
lib,
...
}: let
inherit (gensokyo-zone) systems;
inherit (gensokyo-zone.lib) mkAlmostOptionDefault unmerged;
inherit (lib.modules) mkMerge mkOptionDefault;
inherit (lib.attrsets) attrValues nameValuePair listToAttrs;
inherit (lib.lists) filter length optional concatMap;
cfg = config.services.gatus;
statusSystems = filter (system: system.config.access.online.enable) (attrValues systems);
mapSystem = system: let
statusServices = map (serviceName: system.config.exports.services.${serviceName}) system.config.exports.status.services;
in concatMap (mkServiceEndpoint system) statusServices;
mkPortEndpoint = { system, service, port, unique }: let
inherit (port.status) gatus;
name = if unique
then service.displayName
else "${service.displayName}: ${port.name}";
conf = {
url = mkOptionDefault (access.proxyUrlFor {
inherit service port;
system = system.config;
scheme = gatus.protocol;
#network = port.listen;
});
};
services.gatus = {
enable = true;
environmentFile = config.sops.secrets.gatus_environment_file.path;
settings = let
# Common interval for refreshing all basic HTTP endpoints
gatusCommonHTTPInterval = "30s";
in nameValuePair name ({ ... }: {
imports =
optional port.status.alert.enable alertingConfigAlerts
++ optional (gatus.protocol == "http" || gatus.protocol == "https") alertingConfigHttp;
# Shared between all endpoints
commonAlertingConfig = {
config = mkMerge [
(unmerged.mergeAttrs gatus.settings)
conf
];
});
mkServiceEndpoint = system: service: let
statusPorts = map /*lib.attrsets.getAttr*/(portName: service.ports.${portName}) service.status.ports;
gatusPorts = filter (port: port.status.gatus.enable) statusPorts;
unique = length gatusPorts == 1;
in map (port: mkPortEndpoint {
inherit system service port unique;
}) gatusPorts;
alertingConfigAlerts = {
alerts = [
{
type = "discord";
@ -21,25 +56,25 @@
}
];
};
# Used wherever a basic HTTP 200 up-check is required.
basicHTTPCheck = url: {
inherit url;
interval = gatusCommonHTTPInterval;
conditions = [
"[STATUS] == 200"
];
alertingConfigHttp = {
# Common interval for refreshing all basic HTTP endpoints
interval = mkAlmostOptionDefault "30s";
};
in {
sops.secrets.gatus_environment_file = {
sopsFile = ../secrets/gatus.yaml;
};
services.gatus = {
enable = true;
environmentFile = config.sops.secrets.gatus_environment_file.path;
settings = {
# Environment variables are pulled in to be usable within the config.
alerting.discord = {
webhook-url = "\${DISCORD_WEBHOOK_URL}";
};
# Endpoint configuration
endpoints = {
# Home Assistant uses the common alerting config, combined with a basic HTTP check for its domain.
"Home Assistant" = commonAlertingConfig // (basicHTTPCheck "https://home.local.gensokyo.zone");
};
endpoints = listToAttrs (concatMap mapSystem statusSystems);
# The actual status page configuration
ui = {
@ -60,27 +95,13 @@
# Bind on the local address for now, on the port after the last one allocated for the monitoring project.
web = {
address = "10.1.1.38";
address = "[::]";
port = 9095;
};
};
};
/* services.nginx.virtualHosts."status.gensokyo.zone" = let
gatusWebCfg = config.services.gatus.settings.web;
upstream = "${gatusWebCfg.address}:${toString gatusWebCfg.port}";
in {
forceSSL = true;
useACMEHost = serverName;
kTLS = true;
locations."/" = {
proxyPass = "http://${upstream}";
proxyWebsockets = true;
};
}; */
networking.firewall.interfaces.local.allowedTCPPorts = [
config.services.gatus.settings.web.port
networking.firewall.interfaces.lan.allowedTCPPorts = [
cfg.settings.web.port
];
}

14
systems/hakurei/nixos.nix
View file

@ -37,6 +37,7 @@ in {
nixos.access.freeipa
nixos.access.freepbx
nixos.access.unifi
nixos.access.gatus
nixos.access.prometheus
nixos.access.grafana
nixos.access.loki
@ -176,6 +177,14 @@ in {
virtualHosts.unifi'local.allServerNames
];
};
status = {
inherit (nginx) group;
domain = virtualHosts.gatus.serverName;
extraDomainNames = mkMerge [
virtualHosts.gatus.otherServerNames
virtualHosts.gatus'local.allServerNames
];
};
prometheus = {
inherit (nginx) group;
domain = virtualHosts.prometheus.serverName;
@ -319,6 +328,11 @@ in {
local.denyGlobal = true;
ssl.cert.enable = true;
};
gatus = {
# we're not the real gatus record-holder, so don't respond globally..
local.denyGlobal = true;
ssl.cert.enable = true;
};
prometheus = {
# we're not the real prometheus record-holder, so don't respond globally..
local.denyGlobal = true;

3
systems/utsuho/nixos.nix
View file

@ -18,6 +18,7 @@ in {
nixos.cloudflared
nixos.nginx
nixos.access.unifi
nixos.access.gatus
nixos.access.prometheus
nixos.access.grafana
nixos.access.loki
@ -36,6 +37,7 @@ in {
credentialsFile = config.sops.secrets.cloudflared-tunnel-utsuho.path;
ingress = mkMerge [
(virtualHosts.unifi.proxied.cloudflared.getIngress {})
(virtualHosts.gatus.proxied.cloudflared.getIngress {})
(virtualHosts.prometheus.proxied.cloudflared.getIngress {})
(virtualHosts.grafana.proxied.cloudflared.getIngress {})
(virtualHosts.loki.proxied.cloudflared.getIngress {})
@ -47,6 +49,7 @@ in {
proxied.enable = true;
virtualHosts = {
unifi.proxied.enable = "cloudflared";
gatus.proxied.enable = "cloudflared";
prometheus.proxied.enable = "cloudflared";
grafana.proxied.enable = "cloudflared";
loki.proxied.enable = "cloudflared";

1
tf/cloudflare_records.tf
View file

@ -20,6 +20,7 @@ module "hakurei_system_records" {
"ipa-cock",
"bw",
"unifi",
"status",
"prometheus",
"mon",
"logs",

1
tf/cloudflare_tunnels.tf
View file

@ -74,6 +74,7 @@ module "utsuho" {
zone_id = cloudflare_zone.gensokyo-zone_zone.id
subdomains = [
"unifi",
"status",
"prometheus",
"mon",
"logs",