fix(aya): runner

ci/flake-cron.nix

@@ -67,8 +67,11 @@ in {
           filteredHosts = [ "hakurei" "reimu" "aya" "tei" "mediabox" ];
           gcBetweenHosts = false;
           nodeBuildString = concatMapStringsSep " && " (node: "nix build -Lf . nixosConfigurations.${node}.config.system.build.toplevel -o result-${node}" + optionalString gcBetweenHosts " && nix-collect-garbage -d") filteredHosts;
+          hostPath = builtins.getEnv "PATH";
         in ''
           # ${toString builtins.currentTime}
+          export PATH="${hostPath}:$PATH"
+          export NIX_CONFIG="$(printf '%s\naccept-flake-config = true\n' "''${NIX_CONFIG-}")"
           nix flake update
 
           if git status --porcelain | grep -qF flake.lock; then

modules/nixos/github-runner.nix

@@ -5,8 +5,8 @@
   ...
 }: let
   inherit (lib.options) mkOption;
-  inherit (lib.modules) mkIf mkDefault;
+  inherit (lib.modules) mkIf mkDefault mkForce;
-  inherit (lib.attrsets) filterAttrs mapAttrs' nameValuePair;
+  inherit (lib.attrsets) attrNames attrValues filterAttrs mapAttrs' nameValuePair;
   inherit (inputs.self.lib.lib) unmerged;
   cfg = config.services.github-runners;
   nixosConfig = config;
@@ -23,14 +23,31 @@
       };
     };
     config = {
+      replace = mkIf config.ephemeral (mkDefault true);
       serviceSettings = mkIf (config.networkNamespace.name != null) {
         networkNamespace = {
           name = mkDefault config.networkNamespace.name;
           afterOnline = mkDefault true;
         };
+        restartTriggers = [
+          config.ephemeral
+          config.url
+          config.name
+          config.runnerGroup
+          config.extraLabels
+          config.noDefaultLabels
+          config.user
+          config.group
+          config.workDir
+          "${config.package}"
+          config.extraPackages
+          config.nodeRuntimes
+          (attrNames config.extraEnvironment)
+          (attrValues config.extraEnvironment)
+        ];
       };
-      serviceOverrides = mkIf (config.user != null && nixosConfig.users.users ? ${config.user}) {
+      serviceOverrides = mkIf (config.user != null || config.group != null) {
-        DynamicUser = false;
+        DynamicUser = mkForce true;
       };
     };
   };

modules/nixos/network/namespace.nix

@@ -236,6 +236,9 @@
           ExecStart = [
             ''${ip} netns add ${escapeSystemdExecArg config.name}''
           ];
+          ExecStartPost = [
+            ''-${ip-n config} link set dev lo up''
+          ];
           ExecStop = [
             ''${ip} netns delete ${escapeSystemdExecArg config.name}''
           ];
@@ -420,6 +423,10 @@
         type = bool;
         default = false;
       };
+      privateMounts = mkOption {
+        type = bool;
+        default = true;
+      };
       name = mkOption {
         type = nullOr str;
         default = null;
@@ -456,6 +463,7 @@
         ];
         serviceConfig = {
           NetworkNamespacePath = mkOptionDefault cfg.path;
+          PrivateMounts = mkIf (!cfg.privateMounts) (mkDefault false);
           BindReadOnlyPaths = mkIf (cfg.bindResolvConf != null) [
             "${cfg.bindResolvConf}:/etc/resolv.conf"
           ];

nixos/github-runner/zone.nix

@@ -10,16 +10,46 @@
   inherit (lib.lists) genList;
   inherit (inputs.self.lib.lib) unmerged;
   cfg = config.services.github-runner-zone;
+  genZone = f: genList f cfg.count;
+  genZoneAttrs = prefix: f: listToAttrs (genZone (i: nameValuePair "${prefix}${toString i}" (f i)));
 in {
   options.services.github-runner-zone = with lib.types; {
     enable = mkEnableOption "github-runners.zone" // {
       default = true;
     };
+    targetName = mkOption {
+      type = str;
+      default = "github-runner-zone";
+    };
+    networkNamespace.name = mkOption {
+      type = nullOr str;
+      default = null;
+    };
     count = mkOption {
       type = int;
       default = 4;
     };
-    user = mkOption {
+    ephemeral = mkOption {
+      type = bool;
+      default = true;
+    };
+    keyPrefix = mkOption {
+      type = str;
+      default = "zone-";
+    };
+    namePrefix = mkOption {
+      type = str;
+      default = "${config.networking.hostName}-";
+    };
+    userPrefix = mkOption {
+      type = nullOr str;
+      default = "github-runner-zone-";
+    };
+    dynamicUser = mkOption {
+      type = bool;
+      default = false;
+    };
+    group = mkOption {
       type = nullOr str;
       default = "github-runner-zone";
     };
@@ -32,35 +62,72 @@ in {
     services.github-runner-zone = {
       runnerSettings = {
         enable = mkDefault true;
+        ephemeral = mkDefault cfg.ephemeral;
+        replace = mkDefault true;
         extraLabels = [ "ubuntu-latest" ];
         tokenFile = mkDefault config.sops.secrets.github-runner-gensokyo-zone-token.path;
         url = mkDefault "https://github.com/gensokyo-zone";
-        user = mkDefault cfg.user;
+        group = mkDefault cfg.group;
         extraEnvironment = {
           GIT_TEXTDOMAINDIR = "${config.programs.git.package}/share/locale";
         };
+        networkNamespace.name = mkIf (cfg.networkNamespace.name != null) (mkDefault cfg.networkNamespace.name);
+        serviceSettings = {
+          wantedBy = [ "${cfg.targetName}.target" ];
+          unitConfig = {
+            StopPropagatedFrom = [ "${cfg.targetName}.target" ];
+          };
+        };
+        serviceOverrides = mkIf (!cfg.dynamicUser) {
+          # XXX: the ci sshd hack requires this for now :<
+          PrivateUsers = false;
+          InaccessiblePaths = [
+            "/run/wrappers"
+          ];
+        };
       };
     };
 
-    services.github-runners = listToAttrs (genList (i: nameValuePair "zone-${toString i}" (mkMerge [
+    services.github-runners = genZoneAttrs cfg.keyPrefix (i: mkMerge [
       (unmerged.merge cfg.runnerSettings)
       {
-        name = mkDefault "${config.networking.hostName}-${toString i}";
+        name = mkDefault "${cfg.namePrefix}${toString i}";
+        user = mkIf (cfg.userPrefix != null) (
+          mkDefault "${cfg.userPrefix}${toString i}"
+        );
       }
-    ])) cfg.count);
+    ]);
 
-    users = mkIf (cfg.enable && cfg.user != null) {
+    systemd = mkIf cfg.enable {
-      users.${cfg.user} = {
+      services.nix-daemon = mkIf cfg.enable {
-        group = cfg.user;
+        networkNamespace = mkIf (cfg.networkNamespace.name != null) {
-        isSystemUser = true;
+          name = mkDefault cfg.networkNamespace.name;
+          privateMounts = mkDefault false;
+        };
       };
-      groups.${cfg.user} = { };
+      targets.${cfg.targetName} = {
+        wantedBy = [ "multi-user.target" ];
+      };
+    };
+
+    users = mkIf cfg.enable {
+      groups = mkIf (cfg.group != null) {
+        ${toString cfg.group} = { };
+      };
+      users = mkMerge [
+        (mkIf (!cfg.dynamicUser) (genZoneAttrs cfg.userPrefix (i: {
+          isSystemUser = true;
+          useDefaultShell = mkDefault true;
+          group = mkIf (cfg.group != null) (mkDefault cfg.group);
+          createHome = false;
+          home = "/var/lib/github-runner/${cfg.keyPrefix}${toString i}";
+        })))
+      ];
     };
 
     sops.secrets = {
       github-runner-gensokyo-zone-token = mkIf cfg.enable {
         sopsFile = mkDefault ../secrets/github-runner.yaml;
-        owner = mkIf (cfg.user != null) cfg.user;
       };
     };
   };

nixos/secrets/github-runner.yaml

@@ -1,4 +1,4 @@
-github-runner-gensokyo-zone-token: ENC[AES256_GCM,data:FbOUFltX1sfyYrP1KQfLr4zt/pMmdfWa4Kb9yRM=,iv:63Wr1pVE1hwlxKQibkH/mmPKWQTT7bkQID2B0C+InZw=,tag:ldSrm8+UAQVjFliDktmTqA==,type:str]
+github-runner-gensokyo-zone-token: ENC[AES256_GCM,data:GQX/1IGCCFBUuFmup9MRph/KNVbQd17LNjCk8fb3I5YhYQ1lLIHSlc67Qrw8vxT0uGCvC1gWnTNW21IxJ02y1iqokKZda1l0ggrBssVmE+R1D9sGzSnCmDdkKcBP,iv:SXNlgDgKegDV+8sftl5nl7tzP/B4liM1I0gbU+aMtM4=,tag:UiiOhkDnUk94kfoLu6UwSA==,type:str]
 sops:
     shamir_threshold: 1
     kms: []
@@ -51,8 +51,8 @@ sops:
             OFFicTNyNi9NYWNrejNQMzJDdXI3amcKwDnLGpKuq+dVRxTy8YRuqOCDu0RyTjHF
             6vp6MRH+7W7wL+1bsgvcmAx64gFBoiRVkg4rlVq1jHGT3Pv524FRIA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-02-18T19:35:22Z"
+    lastmodified: "2024-02-20T23:11:36Z"
-    mac: ENC[AES256_GCM,data:Ow58/0zQTj4OP+rnRUupXsUEqD1YiT8iVcoGVbKYYTg8UJGiGaS5c5XjlwAKJsZZZw+g0V5hbnlLpz1oWPdqBdE+AShED9BuGsJkzULzx5wBdwlOgZ7iLDy4xWcrVHZBNL+k+TINLgoHwjxEAc6Z8QkPYqIK6MSAvblcNOViqIc=,iv:2822rfae+KVpfwOL7wck1GFIQCi7ZLNzaSyCT9kH5Uc=,tag:l29EJPG3ENBRPY/9odZQDQ==,type:str]
+    mac: ENC[AES256_GCM,data:PoTQOl3/V+vJXF++fObXxJJVrPqYwle6puuD73eliIDAtvhk/x24MhhrjA82uzsqhhMj0IyRdkD7WhG8V81gu2PAWV/Q8EMRk6g9lsqr65K08VB7dbHn7teKXR8+Eqsw7AHV2KcnqIsjMdKdh3rxeAO1jdATH9odTb7i6O7RNnY=,iv:Qz/ahGO+toY0Ibf0O7PRp+MKY/1M9yrjZ9+Vez2VdgI=,tag:zM2dWKotMOW3Rce5jMOeZQ==,type:str]
     pgp:
         - created_at: "2024-02-12T21:16:54Z"
           enc: |-

systems/aya/nixos.nix

@@ -19,7 +19,7 @@
 
   services.github-runner-zone = {
     count = 16;
-    runnerSettings.networkNamespace.name = "ns1";
+    networkNamespace.name = "ns1";
   };
 
   networking.namespaces.ns1 = {

tf/proxmox_vms.tf

@@ -119,7 +119,7 @@ resource "proxmox_virtual_environment_container" "aya" {
 
   memory {
     dedicated = 16384
-    swap      = 8192
+    swap      = 12288
   }
 
   cpu {