From 79ba879e6ddc9764d954823b6662cbde7a68806f Mon Sep 17 00:00:00 2001 From: Kat Inskip Date: Fri, 31 May 2024 14:16:21 -0700 Subject: [PATCH] feat(monitoring): gatus, grafana alerting to discord --- modules/nixos/gatus.nix | 362 ++++++++++++++++++++++++++ modules/system/exports/monitoring.nix | 18 ++ nixos/monitoring/gatus.nix | 86 ++++++ nixos/monitoring/grafana-alerting.nix | 26 ++ nixos/{ => monitoring}/monitoring.nix | 0 nixos/secrets/gatus.yaml | 138 ++++++++++ nixos/secrets/grafana.yaml | 138 ++++++++++ systems/utsuho/default.nix | 1 + 8 files changed, 769 insertions(+) create mode 100644 modules/nixos/gatus.nix create mode 100644 nixos/monitoring/gatus.nix create mode 100644 nixos/monitoring/grafana-alerting.nix rename nixos/{ => monitoring}/monitoring.nix (100%) create mode 100644 nixos/secrets/gatus.yaml create mode 100644 nixos/secrets/grafana.yaml diff --git a/modules/nixos/gatus.nix b/modules/nixos/gatus.nix new file mode 100644 index 00000000..eff7145d --- /dev/null +++ b/modules/nixos/gatus.nix @@ -0,0 +1,362 @@ +{ config, lib, pkgs, ... }: + +let + inherit (lib) types mkIf mkOption mkEnableOption mkPackageOption mkDefault; + + cfg = config.services.gatus; + + configFile = pkgs.writeText "gatus-config.yml" (builtins.toJSON (cfg.settings + // { + endpoints = builtins.attrValues cfg.settings.endpoints; + })); +in { + options.services.gatus = { + enable = mkEnableOption "a developer-oriented service status page"; + + package = mkPackageOption pkgs "gatus" { }; + + user = mkOption { + type = types.str; + default = "gatus"; + }; + + group = mkOption { + type = types.str; + default = "gatus"; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + }; + + # https://github.com/TwiN/gatus#configuration + + settings = { + debug = mkEnableOption "debug logs"; + + metrics = mkEnableOption "expose metrics at /metrics"; + + storage = { + path = mkOption { type = types.path; }; + type = mkOption { type = types.enum [ "memory" "sqlite" "postgres" ]; }; + caching = mkEnableOption "write-through caching"; + }; + + endpoints = mkOption { + type = types.attrsOf (types.submodule ({ name, ... }: { + options = { + enabled = mkOption { + type = types.bool; + default = true; + description = '' + Whether to monitor the endpoint. + ''; + }; + name = mkOption { + type = types.str; + description = '' + Name of the endpoint. Can be anything. + Defaults to attribute name in `endpoints`. + ''; + }; + group = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Group name. Used to group multiple endpoints together on the dashboard. + See [https://github.com/TwiN/gatus#endpoint-groups](Endpoint groups). + ''; + }; + url = mkOption { type = types.str; }; + method = mkOption { + type = types.enum [ + "GET" + "HEAD" + "POST" + "PUT" + "DELETE" + "CONNECT" + "OPTIONS" + "TRACE" + "PATCH" + ]; + default = "GET"; + description = '' + Request method. + ''; + }; + conditions = mkOption { + type = types.listOf types.str; + description = '' + Conditions used to determine the health of the endpoint. + See [https://github.com/TwiN/gatus#conditions](Conditions). + ''; + }; + interval = mkOption { + type = types.str; + default = "60s"; + description = '' + Duration to wait between every status check. + ''; + }; + graphql = + mkEnableOption "wrapping the body in a query param for GraphQL"; + body = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Request body. + ''; + }; + headers = mkOption { + type = types.submodule { + freeformType = (pkgs.formats.yaml { }).type; + }; + default = { }; + description = '' + Request headers. + ''; + }; + dns = mkOption { + type = types.nullOr (types.submodule { + options = { + query-type = mkOption { + type = types.enum [ "A" "AAAA" "CNAME" "MX" "NS" ]; + description = '' + Query type (e.g. MX) + ''; + }; + query-name = mkOption { + type = types.str; + description = '' + Query name (e.g. example.com) + ''; + }; + }; + }); + default = null; + }; + ssh = mkOption { + type = types.nullOr (types.submodule { + options = { + username = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + SSH username + ''; + }; + password = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + SSH password + ''; + }; + }; + }); + default = null; + }; + alerts = mkOption { + type = types.listOf (types.submodule { + options = { + type = mkOption { + type = types.enum [ + "custom" + "discord" + "email" + "github" + "gitlab" + "googlechat" + "gotify" + "matrix" + "mattermost" + "messagebird" + "ntfy" + "opsgenie" + "pagerduty" + "pushover" + "slack" + "teams" + "telegram" + "twilio" + ]; + }; + enabled = mkOption { + type = types.bool; + default = true; + }; + failure-threshold = mkOption { type = types.ints.positive; }; + success-threshold = mkOption { type = types.ints.positive; }; + send-on-resolved = mkEnableOption + "sending a notification once a triggered alert is marked as solved"; + description = mkOption { type = types.str; }; + }; + }); + default = [ ]; + }; + client = mkOption { + type = types.submodule { + freeformType = (pkgs.formats.yaml { }).type; + }; + default = { }; + description = '' + [https://github.com/TwiN/gatus#client-configuration](Client configuration). + ''; + }; + ui = { + hide-hostname = + mkEnableOption "hiding the hostname in the result"; + hide-url = mkEnableOption "hiding the URL in the results"; + dont-resolve-failed-conditions = + mkEnableOption "resolving failed conditions for the UI"; + badge.response-time.thresholds = mkOption { + type = types.listOf types.ints.positive; + default = [ 50 200 300 500 750 ]; + description = '' + List of response time thresholds. Each time a threshold is reached, + the badge has a different color. + ''; + }; + }; + }; + config = { name = mkDefault name; }; + })); + default = { }; + }; + alerting = mkOption { + type = types.submodule { freeformType = (pkgs.formats.yaml { }).type; }; + default = { }; + description = '' + [https://github.com/TwiN/gatus#alerting](Alerting configuration). + ''; + }; + security = mkOption { + type = types.nullOr + (types.submodule { freeformType = (pkgs.formats.yaml { }).type; }); + default = null; + description = '' + [https://github.com/TwiN/gatus#security](Security configuration). + ''; + }; + disable-monitoring-lock = mkOption { + type = types.bool; + default = false; + description = "Whether to disable the monitoring lock"; + }; + skip-invalid-config-update = mkOption { + type = types.bool; + default = false; + description = "Whether to ignore invalid configuration update"; + }; + web = { + address = mkOption { + type = types.str; + default = "0.0.0.0"; + description = "Address to listen on"; + }; + port = mkOption { + type = types.port; + default = 8080; + description = "Port to listen on"; + }; + tls = mkOption { + type = types.nullOr (types.submodule { + options = { + certificate-file = mkOption { + type = types.nullOr types.path; + default = null; + description = + "Optional public certificate file for TLS in PEM format"; + }; + private-key-file = mkOption { + type = types.nullOr types.path; + default = null; + description = "Optional private key file for TLS in PEM format"; + }; + }; + }); + default = null; + }; + }; + ui = { + title = mkOption { + type = types.nullOr types.str; + default = null; + description = "Title of the document"; + }; + description = mkOption { + type = types.nullOr types.str; + default = null; + description = "Meta description for the page"; + }; + header = mkOption { + type = types.nullOr types.str; + default = null; + description = "Header at the top of the dashboard"; + }; + }; + }; + }; + + config = mkIf cfg.enable { + systemd.services.gatus = { + description = "Automated developer-oriented status page"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment.GATUS_CONFIG_PATH = "${configFile}"; + + serviceConfig = { + Type = "simple"; + Restart = "on-failure"; + User = cfg.user; + Group = cfg.group; + StateDirectory = "gatus"; + LogsDirectory = "gatus"; + EnvironmentFile = + mkIf (cfg.environmentFile != null) cfg.environmentFile; + + AmbientCapabilities = "CAP_NET_RAW"; # needed for ICMP probes + DevicePolicy = "closed"; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + UMask = "0077"; + + ExecStart = "${cfg.package}/bin/gatus"; + }; + }; + + users.groups = mkIf (cfg.group == "gatus") { ${cfg.group} = { }; }; + + users.users = mkIf (cfg.user == "gatus") { + ${cfg.user} = { + inherit (cfg) group; + description = "gatus service user"; + isSystemUser = true; + }; + }; + }; + + meta.maintainers = with lib.maintainers; [ christoph-heiss ]; +} \ No newline at end of file diff --git a/modules/system/exports/monitoring.nix b/modules/system/exports/monitoring.nix index f0ae36ba..f7d23598 100644 --- a/modules/system/exports/monitoring.nix +++ b/modules/system/exports/monitoring.nix @@ -203,6 +203,24 @@ in }; #ports.grpc = ... }; + gatus = {config, ...}: { + id = mkAlmostOptionDefault "gatus"; + nixos = { + serviceAttr = "gatus"; + assertions = mkIf config.enable [ + (nixosConfig: { + assertion = config.ports.default.port == nixosConfig.services.gatus.settings.web.port; + message = "port mismatch"; + }) + ]; + }; + ports.default = + mapAlmostOptionDefaults { + port = 9095; + protocol = "http"; + }; + #ports.grpc = ... + }; } // exporters; } diff --git a/nixos/monitoring/gatus.nix b/nixos/monitoring/gatus.nix new file mode 100644 index 00000000..9c93dcfa --- /dev/null +++ b/nixos/monitoring/gatus.nix @@ -0,0 +1,86 @@ +{ config, ... }: { + sops.secrets.gatus_environment_file = { + sopsFile = ../secrets/gatus.yaml; + }; + services.gatus = { + enable = true; + environmentFile = config.sops.secrets.gatus_environment_file.path; + settings = let + # Common interval for refreshing all basic HTTP endpoints + gatusCommonHTTPInterval = "30s"; + + # Shared between all endpoints + commonAlertingConfig = { + alerts = [ + { + type = "discord"; + send-on-resolved = true; + description = "Healthcheck failed."; + failure-threshold = 1; + success-threshold = 3; + } + ]; + }; + # Used wherever a basic HTTP 200 up-check is required. + basicHTTPCheck = url: { + inherit url; + interval = gatusCommonHTTPInterval; + conditions = [ + "[STATUS] == 200" + ]; + }; + in { + # Environment variables are pulled in to be usable within the config. + alerting.discord = { + webhook-url = "\${DISCORD_WEBHOOK_URL}"; + }; + + # Endpoint configuration + endpoints = { + # Home Assistant uses the common alerting config, combined with a basic HTTP check for its domain. + "Home Assistant" = commonAlertingConfig // (basicHTTPCheck "https://home.local.gensokyo.zone"); + }; + + # The actual status page configuration + ui = { + title = "Gensokyo Zone Status"; + description = "The status of the various girls in Gensokyo!"; + header = "Gensokyo Zone Status"; + }; + + # Prometheus metrics...! + metrics = true; + + # We could've used Postgres, but it seems like less moving parts if our status page + # doesn't depend upon another service, internal or external, other than what gets it to the internet. + storage = { + type = "sqlite"; + path = "/var/lib/gatus/data.db"; + }; + + # Bind on the local address for now, on the port after the last one allocated for the monitoring project. + web = { + address = "10.1.1.38"; + port = 9095; + }; + + }; + }; + +/* services.nginx.virtualHosts."status.gensokyo.zone" = let + gatusWebCfg = config.services.gatus.settings.web; + upstream = "${gatusWebCfg.address}:${toString gatusWebCfg.port}"; + in { + forceSSL = true; + useACMEHost = serverName; + kTLS = true; + locations."/" = { + proxyPass = "http://${upstream}"; + proxyWebsockets = true; + }; + }; */ + + networking.firewall.interfaces.local.allowedTCPPorts = [ + config.services.gatus.settings.web.port + ]; +} diff --git a/nixos/monitoring/grafana-alerting.nix b/nixos/monitoring/grafana-alerting.nix new file mode 100644 index 00000000..04117bf3 --- /dev/null +++ b/nixos/monitoring/grafana-alerting.nix @@ -0,0 +1,26 @@ +{ config, ... }: { + sops.secrets.grafana_discord_webhook_url = { + sopsFile = ../secrets/grafana.yaml; + owner = "grafana"; + }; + services.grafana.provision.alerting.contactPoints.settings = { + apiVersion = 1; + contactPoints = [ + { + orgId = 1; + name = "Discord"; + receivers = [ + { + uid = "discord_alerting"; + type = "discord"; + disableResolveMessage = false; + settings = { + url = "$__file{${config.sops.secrets.grafana_discord_webhook_url.path}}"; + #avatar_url = ""; + }; + } + ]; + } + ]; + }; +} \ No newline at end of file diff --git a/nixos/monitoring.nix b/nixos/monitoring/monitoring.nix similarity index 100% rename from nixos/monitoring.nix rename to nixos/monitoring/monitoring.nix diff --git a/nixos/secrets/gatus.yaml b/nixos/secrets/gatus.yaml new file mode 100644 index 00000000..97b65982 --- /dev/null +++ b/nixos/secrets/gatus.yaml @@ -0,0 +1,138 @@ +gatus_environment_file: ENC[AES256_GCM,data:BqzEORFnatmNswKHT31xjPBoS8YUEhtoSyoZaxLeF0Jut8S6c3+fjVXN+GjJj1OzVnz0JnZyTorzYux1HZ6ZJf79JgBJPzAjCBCKkEfsNzAH1NNF5h11pXlyvCccbd4oYvsBaQryUyY2wnYw3ResRKHC8qwj52hfsntZJM6Zexj59+jEhmcqt/9H9TBhK1vs,iv:AIaA57L63iUZQd27kbxFXD+CJL0zP3DRRBAgcITYTJ0=,tag:Da4s7uqs8ConxpAXSRshJw==,type:str] +sops: + shamir_threshold: 1 + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age12ze362pu5mza6ef9akrptr7hfe4auaqul4rkta7kyy2tnrstqensgmujeq + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0TlE5dFc3UVBhZXR6UzZB + WGRzSjl4NFlpUnoyVXFLT2pqR05YL3c5NkdJCjkvR2dnT21zZytjeVEwQ2twZWtE + TDUxVHRVTTZqRHF4TmlIelNncERkcUkKLS0tIElObWVuaDdRZFV6aDlrZUJ1Q0lT + c0ZjSHFjY2YrZ0NISkhLRFVPWFNkQXcKhjkYcS3P1mKl92p5s6Im3Jp3xfSnn+FD + +tEUe3kcNeucUe/U84XNkAT9igWlllg3a+i+OMPUc3g3kkx9Mn0ziA== + -----END AGE ENCRYPTED FILE----- + - recipient: age176uyyyk7veqnzmm8xzwfhf0u23m6hm02cldlfkldunqe6std0gcq6lg057 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1OHdpdGlJK28zN0dGMHNo + cnRlamNRTUx6NHNXUHZpbTloSHAzcmxMbVNzCkxVbkNudnRVTmdwWWszNnFvMzJo + L0Y2cUpIQ1dNRmdnS2dMTjdKQXBnUkkKLS0tIENuRUNyYkx0VmNISG9HcVpEckdL + dkhHeVlHakxhTlV1U1NUMi9ONzF5ZEUKUhYzD3iPNjvS9VbpN6POOC0XlVIV+GG9 + Vyv3L7o9Uce74HorzayU/5jv1ZCYEgJbDe5SoW+Zl67YZ4f+oz5ixw== + -----END AGE ENCRYPTED FILE----- + - recipient: age15hmlkd9p5rladsjzpmvrh6u34xvggu9mzdsdxdj3ms43tltxeuhq4g7g9k + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhdzV5Y2hKTlFaQThTKzZ5 + WFd5T29lVzNGSitWSkpvUkIwTDBQRXhtbG4wCm53NGF2VE1zaXRyOU5ZVHJ5Q3Bm + VWcxYXFTVWcrT2ovYitJVEFvS2YyU00KLS0tIFdMeU4yYU54ZFlsbmpsV0RJdDRZ + Y2JSTmlRcVhYRmRmMzR6Ukp1T3VHWEUKruGRHNofwHlG2p1WqS8oc97Aofxu0uIf + yRCXYnai1k2OQiN6Lv/yXtanqlLh6DeYAqqfZOcmIQKtYgyV6z943w== + -----END AGE ENCRYPTED FILE----- + - recipient: age10t6kc5069cyky929vvxk8aznqyxpkx3k5h5rmlyz83xtjmr22ahqe8mzes + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0VDJKTVM1Tm5TOFR3eWta + dXhpbTA1ODJSMmlVU2ZrdDdJdVVycy9aWVhjCjRJNHlJbW1zNEsyMUh6bWtiaTk3 + WFlscEpyUHczaVlLalJJOWtmdTdCT0UKLS0tIC9kTkpucS9zWEV3dFErbVVsQ0dB + MzBPc2gvb2M3bUpHK1hVNXNlZkYxN2sKbQO9/wdb4KOT2xJP0rRHYh4HbtY7xxtd + RFyjZUYrKiby34C5Fs3CeBMh4QnlRvgpLUwW0mgZil0BQznEIgbcUQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1a2quf2ekkj94ygu7wgvhrvh44fwn32c0l2cwvgvjh23wst90s54szdsvgr + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxOWZuNDU0aytMRFNUellT + RWpxdEVWUFhobWtDY0FET2NEckZLTGVHSUJBCnRuUDFtekJwZ3hVTnVWYmR0cXJG + TlYvVk9TU3lDUmhmNndmSU44ZEJpY2MKLS0tIDh2TU5tOGFrOWlmditLd2lHa2Nn + Z0k3Qys1SmVSYm1kTzJPNWxOKzBuaTQKs7J/pVNIHghC0VTAysAZYq9IsO2B2PcU + ocKVjcEmW4347spxUsuifLIo5+XXwuGCIc3GAK9UxJcxAqopl/1Hiw== + -----END AGE ENCRYPTED FILE----- + - recipient: age16klpkaut5759dut8mdm3jn0rnp8w6kxyvs9n6ntqrdsayjtd7upqlvw489 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaeHlhZkhNRit4STdBbitW + RjRLRWFXQVZGb0d1QWNsMm83b3g3TXZkYlVNCjczaWdSVFNWdkNabEpTYkJQYTM1 + aittOHRDY0RwWlZPQk1jcVVqU2ZEc3MKLS0tIENJbFZpSGMrVzhrWTFxbDBKUkZh + K2tYQWVhOWphRndjUDBKaEVyem0yVTgKPM0G1JmcLUPrPyhkY0WdTDMZfcDulfLL + mZnVqTVeFd0BT9zkl+DaqxaoTH4stnJ71Kcg1mJ/qjxVpHjfMWOd2g== + -----END AGE ENCRYPTED FILE----- + - recipient: age13qgddr326g5je0fpq2r3k940vsr3fh9nlvl9xtcxk3xg2x0k3vsq7pvzaj + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEc1VYMklMRUJMSTltWTVB + dzkzbnNFcmQvQ1BnSWNLQ1MvaENkWUFOOWpRCmRPN0N5a0JjdUtIWHVFZEF3Mmov + Slp4ckw1UTdIbWtIMEdqbmdoZDBsZjQKLS0tIGE1emRHMk1ITUl6THMwbTNTWUR1 + Zk9zSlNtblYzQjVHdVpOQldIaUo3NjAKiRfJpIumq9gFeGNicriseTSRI6+Ffgjc + +JKyaMg5e96W+CHgj2sxrltCy0hYkqBGrIs4xlq5k5qEgrOPhYCjFA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1ktmx2szedfnpe5xumnzs8vkk0ffqgga6ved3drtksg9pye6ndsnsnqq488 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKL252MXhWTk5uQ1NBNk5P + TG1Jc1VVUkFkeVFqYzgvVVBXYmIrNEhYUVI4CjJNcFZDZ1FxSzg5RW5qeHE0ZHJr + NTRCNUdscXFDcXdZR3RqWjVIcWEzYzQKLS0tIGpwMVIzWTRTVGNJNnB3cm80czZq + QlNPVlBYczFKTnUvVFlTRnQzK2RqSHcK9vHoiAAwjdPTKAUd5NixEalNFq7feWPm + lLn5ZsLrf8OYNnnoI90RWrxFIAl/8pYgw6IBICLGY4ATldDbiLVcsg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1fjcafp0j45sz03zq5srnxyq2mujndmn25vceg3wj2cgzymqm73ssmhdgku + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBadlBzTkk2blhmNXBxMXFo + UE92U0dlYWl2ZUtjZUV2WFh2OGFOY3JsQVE4CjZUb1A0aWxON0NYekFjRTNrZ3E5 + U1VyYnB5RlB0MGEyU1A4ZkFiTTc2ajgKLS0tIFhlaHUxM0VNL3pRbVhlREtVZ015 + MGxYUXdxSGtrbExYM056VWdPekxxeDAKagLH2DHE7Ot8uJoEBObkCY954Pw250n/ + yBYjX/AhdpIIYAZmrioi262SHDEVaa+2sPmWHQpN11ir4YjRza0PwQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1tkkau8vk5h9dh3kemash4eghn7lk84j0hhpmvvf7j6phgcsm9vmsphv0py + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmTHJQQUxtejZBYThSMXZT + bnRJc21mY3pOWHdhR1gxY0trQmUvU2c3aGtZCmZ0enBMQlo5TTRoNDRMSHpWSjcv + TmhzVStOWjdkbkRZVk1Iem5YOFZUK2MKLS0tIGwxWlZUSjJQTTlvdEpEbUNjczVV + VjZ3cDZRTHA4ZHpQdzB3UWEyL3VRRTAK6cY44Bpv4KrNkTMZyfMHDMA1uFjN4nti + cmv50HcidAEC6/LsVweEL0/u3xjaiPBbfJl4QUohCieksGF/pHyATA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-05-31T19:01:16Z" + mac: ENC[AES256_GCM,data:P8ZFOtm3Avn0xKI928vbmq8ACJirxs/zsz8BB8ONDtz+1lNS8kHVc1Hn1D0kNUKh9JSedA0PoGR7ALofVq28ifiu/2LOa/S15EDGjBItPyTq0miWFe7W71igw5DVLIb86HrkFwDl60mGqrbEg+5ADfKA4q30pKVBI2kqw6bGV/8=,iv:OpmXsqUwZW6bmsqKxFuJZo+aW3ycNnCEld7NKZ4Vjtw=,tag:Rk5GtXwI6hRNqxbx77jHTA==,type:str] + pgp: + - created_at: "2024-05-31T18:58:53Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA82M54yws73UARAAlLrWySL1Jupc7imMJ9msPDyOW3mv5N8a+Xe9kRdBsl99 + CBJ7hD1TEaolOdjmPXLJI+Di90FgujGJL/yN4U79I31f8pqUiLfdYu9wJBPBktYc + ad7ybjsDzCUcr+TMmh9lbqLyns1NMB3H567NuepIkr4Sf0uesxqJDk5woCno+1Ot + KDQrSPSN+RtWB/0KiFqnE4JH/DFVPx6hpPm5LxqKmfAr7Q0cmh2rZ1YuuH/yub5u + Q+7O4kM1F0CLi9IAWn0SfgS43/h0Z3NW+HnX00yt5mLUsMMa4uVGLuiIUtHfmyS4 + 1TmJrpNAC2KbVTALeySPk7qXW99oNGvVaS8So/13kfuE1S/oPuGg+Q7B6ApONy7S + 7KTyzzqYYdsz0ND4FaBZmEjEwd18Dj7NkquQ8adZxlEFNGU6IUao/n2160E5QJiJ + ReFAKE431sDCG+BH0JPOen1+dBtilFhRvX4Ymdc1JSPIeG9DRxV0nTI/jKzDptlY + GqlBBx0mpCBIhoyTpzSeGCe0tqpT5uewmq8QgtXtq3s2UCYT4MfsEB97EGt0R5aa + MbT/0YrNSsqgk25xfZMe8ZygIdEffqTIJJycX6229ydO6ee2uonfM7axjxjgY5Ac + /XzOtocPAsDbVCz5AQBiL6pqms2UxfjBYQaiXGrHvuq28KToiVpDkqqKF5Gc02PS + XgG4JhUtujm7YwvuudC77XP55Lw5Ereg57A/k4zFBXFe2VFM+tAuZ8T2BwSQMsHI + 2EeWZhZfAbK4pENA4nrFgAgR06pL4k/vapTF/aQcipBTAIjYK1q/WhOFTaDazcE= + =kym6 + -----END PGP MESSAGE----- + fp: CD8CE78CB0B3BDD4 + - created_at: "2024-05-31T18:58:53Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMA2W9MER3HLb7AQgAgZgeADPOCAibss3J83Yj6xMKVXbVOjz9BCvfChyc4ar5 + +avl71T3V6kQD9qF0hu9TdPKpmXCTh6KR0rMxSk0ygE0sB3ZVJQqyTOxcbBmOwNo + GiciWOXAw3ON/lRWGxGKMidPSjTXX2dLGmK7Nqjzome1HrmdGh/wMSK1rDiNfLdw + axrQa7DVaYEZ9guFcFHa15TuN8ht+zZCaWINUfuahCDmoqVufGrEn6MbhB49BZh6 + NJjOW1wmFQQqA+2NmBVMDroMXxeXh6MyMFMoeGY8J/rcLkHAaQK2X5SxfgGlw4D7 + xlEnvTj9ApAMU1/une6jnCKpGuDcxIKvhXavSI+cdNJeAc5mjeK6ejbDB+/p3AQT + u3e6yPTbY/ta0FlW2KJLoAPY34CkrRS1XD/vAisMlr3+c4dUokwqNFzdo35Q3nBl + FW9aJFjxfrwsNqh+gFavlifYXHIr62YrxgxSPmbehw== + =aTJC + -----END PGP MESSAGE----- + fp: 65BD3044771CB6FB + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/nixos/secrets/grafana.yaml b/nixos/secrets/grafana.yaml new file mode 100644 index 00000000..472fedd1 --- /dev/null +++ b/nixos/secrets/grafana.yaml @@ -0,0 +1,138 @@ +grafana_discord_webhook_url: ENC[AES256_GCM,data:aeEsHCURrnToCsJbE/N6gsofvL0SLBj4ez3WW0Pzsez/n2UZMdDRf46li9AnGGvJCYv9otHwFy+mKh/tj04mO+QqgLSrypZVFsAysTAMGf23Wd9EVGGvSnbddnDqEGHna6C92sRH0Yapgy7CNoEoCt5tfxLuhBGaMA==,iv:havyeDW6RLHqNUd/bgxUqQAAsSsErGUbtnbYJvMkSj0=,tag:vAF4rDbn/i+Y4BjndB9ELg==,type:str] +sops: + shamir_threshold: 1 + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age12ze362pu5mza6ef9akrptr7hfe4auaqul4rkta7kyy2tnrstqensgmujeq + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvT2o5MjFFeDdJbEt2MEcz + QWJ2N0xadUVNUU5VU1BOZGNuWlNsek5VVmpVClVYZzdZUnJPTm1ZVEw4N2laZ0tq + UW1aNlZWK1I1bWFIVitpNk9DZFZtczgKLS0tIGk5cC9vVDNQc2tvd1ZVbjErdW0x + cVVieUJrS0huV2pBd2Vlei9XdjlOOWMKN+T5+h+NSz5paNj3AX8acEA9x4igJh9N + 7noaJAar+/5W86fuSaDRf5DKkJF+u5SbRZoVu2t++iBpJQmWjsCwJA== + -----END AGE ENCRYPTED FILE----- + - recipient: age176uyyyk7veqnzmm8xzwfhf0u23m6hm02cldlfkldunqe6std0gcq6lg057 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCL09uUWtWbUVIS0w2TjJr + bmRpeCtqQVBOSWNiREFjTThJNUFXSG9ZeVM0CllRbWF3NUpCbERzMzFjdzgydm01 + REdVY0c0ZGk2VEdvYVlpa0tlNklkcHMKLS0tIE1rZ3pmMnNld01MWjdmMFBoVlVK + WG9LRk1pbU8wK0tzRGh0SVVPa0sxSXMK/wx0Cd4mQZERZ6Jzm4T2H7lCib8Hbc7G + 15NoWjAnxYjp5HQuif6cxREL8c2gxCS5DotCr67USocpw8C5e2c3BA== + -----END AGE ENCRYPTED FILE----- + - recipient: age15hmlkd9p5rladsjzpmvrh6u34xvggu9mzdsdxdj3ms43tltxeuhq4g7g9k + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDTjJONE13M0dqeGF3RmMx + eGlJWEpidzEyWnMvZ2FtcEJMTjA0bE1pQWx3Ck1xbTZtODhRRi9PUkJQY3N3QUEw + QzhyQ1Y3eW1RRE1kbnZGaDY5UEF0NVEKLS0tIGZOTjdKQ2l5M0hQR3UrdlIxOXAw + OUsxc3ZOMThPaFpKcVJ0Q3h3U0VBQ2sKh5CDAjBT6mb7m5QC9kT6mHplABT2EwZ9 + c7jpEsoxQw0grwmHEOguOE6T/ZRjbkwihTUgY5WDZqppeI506EBkhQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age10t6kc5069cyky929vvxk8aznqyxpkx3k5h5rmlyz83xtjmr22ahqe8mzes + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxSzFUOXdNQk54QUJHTTNF + VlRsNzRtZ2ZoaWJIclN0TmJjNk1jWFFRMXcwCjlEMkkrOWs3eVNGcU9NSkdoVml2 + NFJlY3YvWk00SUJpT2UzTlZFVlMxdVUKLS0tIDhhZFN4UldQUkhaZnBhZXNYM0No + V0dGRDdsSE9PVGU2TjJJRCszcUcwK1EKC+hzR7K/9pwSUhNpGUULmk/z/5vTY7Wl + oHYt6beQvG01jYZZihdvKyR8UWYNTWb2Skj3eNFt38QuqExeixFVmQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1a2quf2ekkj94ygu7wgvhrvh44fwn32c0l2cwvgvjh23wst90s54szdsvgr + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2d3kvOWxPaDIrS211aEdn + TktMaEszUGZKcG5YUkNWcnU1LzJ6WnJMMWowCmphdVFaYVY5Ynl2MHhsUTVLeHpt + Q0IxVWQyRG94OENnVEV3dVE1M1Z3VUkKLS0tIEFWZWVoYjRKY0RyNURsZVJJcGlU + N2FubVBsYldpakpzZHJqUWdkdGE3akUKcWqrtQ5ucXjsA6mCqKT6jnvXBHEPRLue + lW7LxrqtYquCnPU/qWTmSkfAVMe/+BapKtFBwEz7xR7Kr7qsAIByYg== + -----END AGE ENCRYPTED FILE----- + - recipient: age16klpkaut5759dut8mdm3jn0rnp8w6kxyvs9n6ntqrdsayjtd7upqlvw489 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTZ3VuUGo5b3Q0NlRyVEdz + MXdiY3dZQjF6bmtkSUdkcUJHcmVsc3pQN2pVCnBycytuY1hMSm5ZNllpMlhWZ2Rj + UlhIaURyTTczN3N3b1NIN296ZGpBK1UKLS0tIGNYeXE5bWpDSDVvSkpUTjhnQjFS + Uzdtak8vd0Z1cVYzOHJObUpuSEhFOUkKObRIQ1UaRm8p4IUHgE2nHcXpI9aT3+87 + +Th+7pjfa6XQyf9FoSBHFsiGgksUoIAVRKVFXpHADC2j8qEy2xrEhQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age13qgddr326g5je0fpq2r3k940vsr3fh9nlvl9xtcxk3xg2x0k3vsq7pvzaj + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2TXc4ZEJ0RFh0L0JnQ0Zz + eXVzSzlDREovNW91cTdiZXZYVXdkMEl6UlZjCm4yUG85eXRhdURaTG5KQ0QvaGRl + cDVCL3AzbWlWVTZLdmlnK2ZkckEwSDAKLS0tIHUrT1h4SHhjUDhQZGxpK3k4U0Ni + S2NLbFNFMU1ubnBQUm12VHkvNHc3RE0Klhgn2ox5fiT7baLXOsdWdehAZqWob8ph + z1MnkROoZ8pfpM//Wp/CgaiAV+6euacPjgmNnQXdRjgxBFdJSSI0qA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1ktmx2szedfnpe5xumnzs8vkk0ffqgga6ved3drtksg9pye6ndsnsnqq488 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvNXVBUGd2WUpZSCtSdmVv + WVZpWU9heDlJV1laSVlBcUdtL0JpMG1XbkFnCi80R2F2ZXBUWVRkbVduSEQ1VWtW + Sy9iUjd2N2RlaWNDVm5sK1Fla0psVjQKLS0tIGVEWjZMV2J2cFRBNHpDSDVPdmZu + OVhWNjNONGh3UlNYbEYyeG5uS0d2OWcKFOHrlzLX9upreL2bu8bOgzeIc1Sde5If + /JkhIGlQB1FeRWixSNW9me25J4hd44BVjrhDWrQa7pnJdPbrAVQWHQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1fjcafp0j45sz03zq5srnxyq2mujndmn25vceg3wj2cgzymqm73ssmhdgku + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLZnhObWcrQlo2R3BieVpF + WVJUa3QyU2JXZlJJVGw3UVFUMmp5SlhKdlFFCktCbUk2aCswRG5laTdlOVVBd3c4 + bENpQ3Rqc3loM2k0NGJSRGgzNFNBMGMKLS0tIGFpaVdkTis3eVI0LzB3TXFmcDdV + UzJTMjZtQkdiU09nMlRmczJ4TjJUQTgK86ASzPvoQm4gncxUhsa+2ckoIlN/6tkK + KdBG+LQU5obfyde0mAQow08h1q0fbcfJmKnqECke/rUB8oyYrkeeNg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1tkkau8vk5h9dh3kemash4eghn7lk84j0hhpmvvf7j6phgcsm9vmsphv0py + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHZFE1d0FycytWcHlpdkVw + N2N3MWdaY2hMcG12ZkhoMDFmRG84d0YySWpnCmlGWktsSDBoV0Z6TDNYWUF3eEQ2 + NnJMVUNIWEl0MVlIWkVrUldkZHkwTzQKLS0tIHRMcXNwYTUwNDBxWWdmUytOZGo1 + ZjYrNzFUVk9kZU9tTUVGOGNldExXK0UKM+iKF5/oO5DWgQemcHPSgtwBAuCnBaCf + FfogeD9e+LnuENB9BdYRoYtG4YgtE3txK9gc2LrgVhEebmDYqWkNFA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-05-31T17:00:09Z" + mac: ENC[AES256_GCM,data:M7GlEHCdxnXmK0z+L7OQTy5jz/I4nSfXwOcWtsvS87B4uGxrYCrepvnifV6rdOYy3FXfo9So101RtdtPfiw4492tJE/IhHsR685jXD6tPwXzQddbjXgs3+3GgaCb5zYM6cNqHudgyc83l9YMS+O5Ex8A+wrBD/Cq9uaLk3JvBfA=,iv:vWmBplT+XNXQz+KMRTfkU2eMqEaAMuFUW/ovkEyTlak=,tag:7AMX3R99vnAhD1tW/xvpJA==,type:str] + pgp: + - created_at: "2024-05-31T16:59:50Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA82M54yws73UAQ/8DzIv5mr95kJjEyuH2K83J6u0V5GB1/6froD7YtP1K8Go + EQHFl6Z4tsjraLYeszRymYMV5d24paPZpbO767kIkOajKPPrIz9ZF2+2L3cC2i6R + R3XLQqYg5RO2stgWqp9eVPxZbDOpmcyP3vM3sxLfOCLLDU50ghOaag0Hly5A85nf + y23bP1/0JcLPKIrhEv2Maw6123jYJr784DeuyWMIFUXskj9nGwsk1a/x6rI0pYb7 + kHYDU24eVjZQ6sRz1zQJs7QjVHkwbABO4ijuuYpq6cFxGFvSFRis+8j+d4T/hS18 + U7AStwJd4N5zyjtCP77xB3UPB73kwALX2RCtyCctfxVfg8MYtZzD2SLG0/lXBYoH + HoHnM+lxl9UY7XHaILVMX7XZ+b91J+Fzy8o1YQnPqaQpYZnx33btpRqt2gi+2EIQ + CVu4AH41ZjQTZWL5/FSHjWf1OAZ6Io7XJSgbhoXrGLstQOtc77ugAjej7dnhU3lL + QUYfsXS+ksr+/Ila4j071qxrlqeIUV3L9ddPU73voCdaqFzftU+FzqNLZroCgNg3 + Jn8e8rdXq6R/x9xjSud+vbV3gero/bxsE8RJRGLBNPrkpePjEwYE337S5oZRpQvC + s0tJQ21Bfgu88N4GLVBHQ0IIxB2aMmz1L4q4kBF1nenx6tuHCTJ79aoh0+eiyHHS + XgFWZeIGoIotH5K+RIkI9BY+Wp/xsyGG79jsV0B1BRn7yo4UzHeXKkjAitFxgWKq + YCd/omwqaosAF8xc4t3iwqNFWWam1MDDvBQ9ao04Mw7GdDsrROuGYOzLH0hBQd8= + =673o + -----END PGP MESSAGE----- + fp: CD8CE78CB0B3BDD4 + - created_at: "2024-05-31T16:59:50Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMA2W9MER3HLb7AQf/Xai4Fg4iw4yQXpCGnfA/pldBMJow4RB6BNS0ei+IMT9a + aXdHbPA8BsfKEOnckUE1v4GPUDTl9CmDRK2sG+7Flk8wJgUg01wOy00J0knu4Mva + jkam0qD9ta31JeEQhMacGve32czgN4gxAGJxEAZQBU2mnIFrkGWTQxw1H2G8Be+l + I+SiITeVjFaI7+nCwXalD50b01nlD8jgfyh5rB6zxmUanoSMDdO3aPhMQ9oeqEOg + t2OAmOINWlhBf7qp4tgB5ZkGVibIPNjv47UJlUa4godVHhdlXwa58/qg0sK+ZLNG + mNrR4e+VPA/Iptd+JSG81lxBac2+TQ1J09GpbChi3NJeAU9VWV7WOKvQGFwgPyJK + B5lfwb7TAx/O1rKKOimPVO8QZxf78QZS428OMff6/xvPVhTm6nFe09m5DMhl39Xl + 8wJMwOSRTPBwoOG1X7U51jjoh5SXWjxvopyisLef6A== + =H/Ol + -----END PGP MESSAGE----- + fp: 65BD3044771CB6FB + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/systems/utsuho/default.nix b/systems/utsuho/default.nix index 3c371dc5..a56f8af6 100644 --- a/systems/utsuho/default.nix +++ b/systems/utsuho/default.nix @@ -19,6 +19,7 @@ _: { grafana.enable = true; loki.enable = true; prometheus.enable = true; + gatus.enable = true; }; }; }