feat(monitoring): cloudflared metrics

modules/nixos/cloudflared.nix

@@ -1,38 +1,67 @@
-{
+let
+  tunnelModule = {pkgs, config, lib, ...}: let
+    inherit (lib.options) mkOption mkEnableOption;
+    settingsFormat = pkgs.formats.json {};
+  in {
+    options = with lib.types; {
+      extraArgs = mkOption {
+        type = listOf str;
+        default = [];
+      };
+      extraTunnel = {
+        enable =
+          mkEnableOption "extra tunnels"
+          // {
+            default = config.extraTunnel.ingress != {};
+          };
+        ingress = mkOption {
+          inherit (settingsFormat) type;
+          default = {};
+        };
+      };
+    };
+  };
+in {
   pkgs,
   config,
   utils,
+  gensokyo-zone,
   lib,
   ...
 }: let
   inherit (lib.attrsets) mapAttrsToList mapAttrs' nameValuePair filterAttrsRecursive;
   inherit (lib.lists) singleton;
   inherit (lib.modules) mkIf mkMerge mkForce;
-  inherit (lib.options) mkOption mkEnableOption;
+  inherit (lib.options) mkOption;
   cfg = config.services.cloudflared;
-  settingsFormat = pkgs.formats.json {};
 in {
   options.services.cloudflared = with lib.types; {
-    tunnels = let
+    metricsPort = mkOption {
-      tunnelModule = {config, ...}: {
+      type = nullOr port;
-        options = {
+      default = null;
-          extraTunnel = {
+    };
-            enable =
+    metricsBind = mkOption {
-              mkEnableOption "extra tunnels"
+      type = str;
-              // {
+      default = "127.0.0.1";
-                default = config.extraTunnel.ingress != {};
+    };
-              };
+    extraArgs = mkOption {
-            ingress = mkOption {
+      type = listOf str;
-              inherit (settingsFormat) type;
+      default = [];
-              default = {};
+    };
-            };
+    tunnels = mkOption {
-          };
+      type = attrsOf (submoduleWith {
+        modules = [tunnelModule];
+        shorthandOnlyDefinesConfig = true;
+        specialArgs = {
+          inherit pkgs utils gensokyo-zone;
         };
-      };
+      });
-    in
+    };
-      mkOption {
+  };
-        type = attrsOf (submodule tunnelModule);
+  config.services.cloudflared = {
-      };
+    extraArgs = mkIf (cfg.metricsPort != null) [
+      "--metrics" "${cfg.metricsBind}:${toString cfg.metricsPort}"
+    ];
   };
   config.systemd.services = let
     filterConfig = filterAttrsRecursive (_: v: ! builtins.elem v [null [] {}]);
@@ -44,29 +73,38 @@ in {
   in
     mkIf cfg.enable (mapAttrs' (uuid: tunnel: let
       RuntimeDirectory = "cloudflared-tunnel-${uuid}";
-      configPath = "/run/${RuntimeDirectory}/config.yml";
       settings = {
         tunnel = uuid;
         credentials-file = tunnel.credentialsFile;
+        warp-routing = filterConfig tunnel.warp-routing;
+        originRequest = filterConfig tunnel.originRequest;
         ingress =
           mapAttrsToList mapIngress tunnel.ingress
           ++ mapAttrsToList mapIngress tunnel.extraTunnel.ingress
           ++ singleton {service = tunnel.default;};
       };
+      configPath =
+        if tunnel.extraTunnel.enable
+        then "/run/${RuntimeDirectory}/config.yml"
+        else pkgs.writeText "cloudflared.yml" (builtins.toJSON settings);
+      args = [
+        "--config=${configPath}"
+        "--no-autoupdate"
+      ] ++ cfg.extraArgs ++ tunnel.extraArgs;
     in
       nameValuePair "cloudflared-tunnel-${uuid}" (mkMerge [
         {
           after = mkIf config.services.tailscale.enable ["tailscale-autoconnect.service"];
           serviceConfig = {
             RestartSec = 10;
+            ExecStart = mkForce [
+              "${cfg.package}/bin/cloudflared tunnel ${utils.escapeSystemdExecArgs args} run"
+            ];
           };
         }
         (mkIf tunnel.extraTunnel.enable {
           serviceConfig = {
             inherit RuntimeDirectory;
-            ExecStart = mkForce [
-              "${cfg.package}/bin/cloudflared tunnel --config=${configPath} --no-autoupdate run"
-            ];
             ExecStartPre = [
               (pkgs.writeShellScript "cloudflared-tunnel-${uuid}-prepare" ''
                 ${utils.genJqSecretsReplacementSnippet settings configPath}

modules/system/exports/cloudflared.nix

@@ -0,0 +1,43 @@
+{
+  lib,
+  gensokyo-zone,
+  ...
+}: let
+  inherit (gensokyo-zone.lib) mkAlmostOptionDefault;
+  inherit (lib.modules) mkIf;
+in {
+  config.exports.services.cloudflared = {config, systemConfig, ...}: let
+    assertMetrics = nixosConfig: let
+      cfg = nixosConfig.services.cloudflared;
+      metricsPort =
+        if config.ports.metrics.enable
+        then config.ports.metrics.port
+        else null;
+    in {
+      assertion = metricsPort == cfg.metricsPort;
+      message = "metricsPort mismatch";
+    };
+  in {
+    displayName = mkAlmostOptionDefault "Cloudflare Tunnel/${systemConfig.name}";
+    nixos = {
+      serviceAttr = "cloudflared";
+      assertions = mkIf config.enable [
+        assertMetrics
+      ];
+    };
+    defaults.port.listen = mkAlmostOptionDefault "lan";
+    ports = {
+      metrics = {
+        port = mkAlmostOptionDefault 3011;
+        protocol = "http";
+        status = {
+          enable = true;
+          gatus.http = {
+            statusCondition = mkAlmostOptionDefault "[STATUS] == 404";
+          };
+        };
+        prometheus.exporter.enable = mkAlmostOptionDefault true;
+      };
+    };
+  };
+}

modules/system/exports/home-assistant.nix

@@ -3,9 +3,8 @@
   gensokyo-zone,
   ...
 }: let
-  inherit (gensokyo-zone.lib) mapAlmostOptionDefaults mkAlmostOptionDefault;
+  inherit (gensokyo-zone.lib) mkAlmostOptionDefault;
   inherit (lib.modules) mkIf;
-  inherit (lib.attrsets) mapAttrs;
   inherit (lib.lists) all imap0;
   inherit (lib.trivial) id;
 in {

nixos/cloudflared.nix

@@ -7,7 +7,16 @@
   cfg = config.services.cloudflared;
 in {
   config = {
-    services.cloudflared.enable = mkDefault true;
+    services.cloudflared = {
+      enable = mkDefault true;
+      metricsPort = mkDefault 3011;
+      metricsBind = "[::]";
+    };
+    networking.firewall = mkIf cfg.enable {
+      interfaces.lan.allowedTCPPorts = mkIf (cfg.metricsPort != null) [
+        cfg.metricsPort
+      ];
+    };
     boot.kernel.sysctl = mkIf (!config.boot.isContainer && cfg.enable) {
       # https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes
       "net.core.rmem_max" = mkDefault 2500000;

systems/hakurei/default.nix

@@ -19,6 +19,7 @@
   exports = {
     services = {
       tailscale.enable = true;
+      cloudflared.enable = true;
       samba.enable = true;
       syncplay.enable = true;
       vouch-proxy = {

systems/keycloak/default.nix

@@ -12,6 +12,7 @@ _: {
       keycloak.enable = true;
       vouch-proxy.enable = true;
       vaultwarden.enable = true;
+      cloudflared.enable = true;
       nginx = {
         enable = true;
         ports.proxied.enable = true;

systems/kuwubernetes/default.nix

@@ -23,6 +23,7 @@ _: {
   };
   exports = {
     services = {
+      cloudflared.enable = true;
     };
   };
 }

systems/mediabox/default.nix

@@ -13,6 +13,7 @@ _: {
         enable = true;
         ports.proxied.enable = true;
       };
+      cloudflared.enable = true;
       plex.enable = true;
       invidious.enable = true;
       deluge.enable = true;

systems/tei/default.nix

@@ -13,6 +13,7 @@ _: {
         enable = true;
         ports.proxied.enable = true;
       };
+      cloudflared.enable = true;
       tailscale.enable = true;
       home-assistant.enable = true;
       zigbee2mqtt.enable = true;

systems/utsuho/default.nix

@@ -13,6 +13,7 @@ _: {
         enable = true;
         ports.proxied.enable = true;
       };
+      cloudflared.enable = true;
       unifi.enable = true;
       mosquitto.enable = true;
       dnsmasq.enable = true;