diff --git a/ci/flake-cron.nix b/ci/flake-cron.nix index f856ba8e..14918cd6 100644 --- a/ci/flake-cron.nix +++ b/ci/flake-cron.nix @@ -67,8 +67,11 @@ in { filteredHosts = [ "hakurei" "reimu" "aya" "tei" "mediabox" ]; gcBetweenHosts = false; nodeBuildString = concatMapStringsSep " && " (node: "nix build -Lf . nixosConfigurations.${node}.config.system.build.toplevel -o result-${node}" + optionalString gcBetweenHosts " && nix-collect-garbage -d") filteredHosts; + hostPath = builtins.getEnv "PATH"; in '' # ${toString builtins.currentTime} + export PATH="${hostPath}:$PATH" + export NIX_CONFIG="$(printf '%s\naccept-flake-config = true\n' "''${NIX_CONFIG-}")" nix flake update if git status --porcelain | grep -qF flake.lock; then diff --git a/modules/nixos/github-runner.nix b/modules/nixos/github-runner.nix index aafd4cbf..7f466026 100644 --- a/modules/nixos/github-runner.nix +++ b/modules/nixos/github-runner.nix @@ -5,8 +5,8 @@ ... }: let inherit (lib.options) mkOption; - inherit (lib.modules) mkIf mkDefault; - inherit (lib.attrsets) filterAttrs mapAttrs' nameValuePair; + inherit (lib.modules) mkIf mkDefault mkForce; + inherit (lib.attrsets) attrNames attrValues filterAttrs mapAttrs' nameValuePair; inherit (inputs.self.lib.lib) unmerged; cfg = config.services.github-runners; nixosConfig = config; @@ -23,14 +23,31 @@ }; }; config = { + replace = mkIf config.ephemeral (mkDefault true); serviceSettings = mkIf (config.networkNamespace.name != null) { networkNamespace = { name = mkDefault config.networkNamespace.name; afterOnline = mkDefault true; }; + restartTriggers = [ + config.ephemeral + config.url + config.name + config.runnerGroup + config.extraLabels + config.noDefaultLabels + config.user + config.group + config.workDir + "${config.package}" + config.extraPackages + config.nodeRuntimes + (attrNames config.extraEnvironment) + (attrValues config.extraEnvironment) + ]; }; - serviceOverrides = mkIf (config.user != null && nixosConfig.users.users ? ${config.user}) { - DynamicUser = false; + serviceOverrides = mkIf (config.user != null || config.group != null) { + DynamicUser = mkForce true; }; }; }; diff --git a/modules/nixos/network/namespace.nix b/modules/nixos/network/namespace.nix index 7e0e3169..35d245d8 100644 --- a/modules/nixos/network/namespace.nix +++ b/modules/nixos/network/namespace.nix @@ -236,6 +236,9 @@ ExecStart = [ ''${ip} netns add ${escapeSystemdExecArg config.name}'' ]; + ExecStartPost = [ + ''-${ip-n config} link set dev lo up'' + ]; ExecStop = [ ''${ip} netns delete ${escapeSystemdExecArg config.name}'' ]; @@ -420,6 +423,10 @@ type = bool; default = false; }; + privateMounts = mkOption { + type = bool; + default = true; + }; name = mkOption { type = nullOr str; default = null; @@ -456,6 +463,7 @@ ]; serviceConfig = { NetworkNamespacePath = mkOptionDefault cfg.path; + PrivateMounts = mkIf (!cfg.privateMounts) (mkDefault false); BindReadOnlyPaths = mkIf (cfg.bindResolvConf != null) [ "${cfg.bindResolvConf}:/etc/resolv.conf" ]; diff --git a/nixos/github-runner/zone.nix b/nixos/github-runner/zone.nix index 4fa8bf0c..3db4bb89 100644 --- a/nixos/github-runner/zone.nix +++ b/nixos/github-runner/zone.nix @@ -10,16 +10,46 @@ inherit (lib.lists) genList; inherit (inputs.self.lib.lib) unmerged; cfg = config.services.github-runner-zone; + genZone = f: genList f cfg.count; + genZoneAttrs = prefix: f: listToAttrs (genZone (i: nameValuePair "${prefix}${toString i}" (f i))); in { options.services.github-runner-zone = with lib.types; { enable = mkEnableOption "github-runners.zone" // { default = true; }; + targetName = mkOption { + type = str; + default = "github-runner-zone"; + }; + networkNamespace.name = mkOption { + type = nullOr str; + default = null; + }; count = mkOption { type = int; default = 4; }; - user = mkOption { + ephemeral = mkOption { + type = bool; + default = true; + }; + keyPrefix = mkOption { + type = str; + default = "zone-"; + }; + namePrefix = mkOption { + type = str; + default = "${config.networking.hostName}-"; + }; + userPrefix = mkOption { + type = nullOr str; + default = "github-runner-zone-"; + }; + dynamicUser = mkOption { + type = bool; + default = false; + }; + group = mkOption { type = nullOr str; default = "github-runner-zone"; }; @@ -32,35 +62,72 @@ in { services.github-runner-zone = { runnerSettings = { enable = mkDefault true; + ephemeral = mkDefault cfg.ephemeral; + replace = mkDefault true; extraLabels = [ "ubuntu-latest" ]; tokenFile = mkDefault config.sops.secrets.github-runner-gensokyo-zone-token.path; url = mkDefault "https://github.com/gensokyo-zone"; - user = mkDefault cfg.user; + group = mkDefault cfg.group; extraEnvironment = { GIT_TEXTDOMAINDIR = "${config.programs.git.package}/share/locale"; }; + networkNamespace.name = mkIf (cfg.networkNamespace.name != null) (mkDefault cfg.networkNamespace.name); + serviceSettings = { + wantedBy = [ "${cfg.targetName}.target" ]; + unitConfig = { + StopPropagatedFrom = [ "${cfg.targetName}.target" ]; + }; + }; + serviceOverrides = mkIf (!cfg.dynamicUser) { + # XXX: the ci sshd hack requires this for now :< + PrivateUsers = false; + InaccessiblePaths = [ + "/run/wrappers" + ]; + }; }; }; - services.github-runners = listToAttrs (genList (i: nameValuePair "zone-${toString i}" (mkMerge [ + services.github-runners = genZoneAttrs cfg.keyPrefix (i: mkMerge [ (unmerged.merge cfg.runnerSettings) { - name = mkDefault "${config.networking.hostName}-${toString i}"; + name = mkDefault "${cfg.namePrefix}${toString i}"; + user = mkIf (cfg.userPrefix != null) ( + mkDefault "${cfg.userPrefix}${toString i}" + ); } - ])) cfg.count); + ]); - users = mkIf (cfg.enable && cfg.user != null) { - users.${cfg.user} = { - group = cfg.user; - isSystemUser = true; + systemd = mkIf cfg.enable { + services.nix-daemon = mkIf cfg.enable { + networkNamespace = mkIf (cfg.networkNamespace.name != null) { + name = mkDefault cfg.networkNamespace.name; + privateMounts = mkDefault false; + }; }; - groups.${cfg.user} = { }; + targets.${cfg.targetName} = { + wantedBy = [ "multi-user.target" ]; + }; + }; + + users = mkIf cfg.enable { + groups = mkIf (cfg.group != null) { + ${toString cfg.group} = { }; + }; + users = mkMerge [ + (mkIf (!cfg.dynamicUser) (genZoneAttrs cfg.userPrefix (i: { + isSystemUser = true; + useDefaultShell = mkDefault true; + group = mkIf (cfg.group != null) (mkDefault cfg.group); + createHome = false; + home = "/var/lib/github-runner/${cfg.keyPrefix}${toString i}"; + }))) + ]; }; sops.secrets = { github-runner-gensokyo-zone-token = mkIf cfg.enable { sopsFile = mkDefault ../secrets/github-runner.yaml; - owner = mkIf (cfg.user != null) cfg.user; }; }; }; diff --git a/nixos/secrets/github-runner.yaml b/nixos/secrets/github-runner.yaml index fded6c35..2894306b 100644 --- a/nixos/secrets/github-runner.yaml +++ b/nixos/secrets/github-runner.yaml @@ -1,4 +1,4 @@ -github-runner-gensokyo-zone-token: ENC[AES256_GCM,data:FbOUFltX1sfyYrP1KQfLr4zt/pMmdfWa4Kb9yRM=,iv:63Wr1pVE1hwlxKQibkH/mmPKWQTT7bkQID2B0C+InZw=,tag:ldSrm8+UAQVjFliDktmTqA==,type:str] +github-runner-gensokyo-zone-token: ENC[AES256_GCM,data:GQX/1IGCCFBUuFmup9MRph/KNVbQd17LNjCk8fb3I5YhYQ1lLIHSlc67Qrw8vxT0uGCvC1gWnTNW21IxJ02y1iqokKZda1l0ggrBssVmE+R1D9sGzSnCmDdkKcBP,iv:SXNlgDgKegDV+8sftl5nl7tzP/B4liM1I0gbU+aMtM4=,tag:UiiOhkDnUk94kfoLu6UwSA==,type:str] sops: shamir_threshold: 1 kms: [] @@ -51,8 +51,8 @@ sops: OFFicTNyNi9NYWNrejNQMzJDdXI3amcKwDnLGpKuq+dVRxTy8YRuqOCDu0RyTjHF 6vp6MRH+7W7wL+1bsgvcmAx64gFBoiRVkg4rlVq1jHGT3Pv524FRIA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-02-18T19:35:22Z" - mac: ENC[AES256_GCM,data:Ow58/0zQTj4OP+rnRUupXsUEqD1YiT8iVcoGVbKYYTg8UJGiGaS5c5XjlwAKJsZZZw+g0V5hbnlLpz1oWPdqBdE+AShED9BuGsJkzULzx5wBdwlOgZ7iLDy4xWcrVHZBNL+k+TINLgoHwjxEAc6Z8QkPYqIK6MSAvblcNOViqIc=,iv:2822rfae+KVpfwOL7wck1GFIQCi7ZLNzaSyCT9kH5Uc=,tag:l29EJPG3ENBRPY/9odZQDQ==,type:str] + lastmodified: "2024-02-20T23:11:36Z" + mac: ENC[AES256_GCM,data:PoTQOl3/V+vJXF++fObXxJJVrPqYwle6puuD73eliIDAtvhk/x24MhhrjA82uzsqhhMj0IyRdkD7WhG8V81gu2PAWV/Q8EMRk6g9lsqr65K08VB7dbHn7teKXR8+Eqsw7AHV2KcnqIsjMdKdh3rxeAO1jdATH9odTb7i6O7RNnY=,iv:Qz/ahGO+toY0Ibf0O7PRp+MKY/1M9yrjZ9+Vez2VdgI=,tag:zM2dWKotMOW3Rce5jMOeZQ==,type:str] pgp: - created_at: "2024-02-12T21:16:54Z" enc: |- diff --git a/systems/aya/nixos.nix b/systems/aya/nixos.nix index 6be17472..7e8c365c 100644 --- a/systems/aya/nixos.nix +++ b/systems/aya/nixos.nix @@ -19,7 +19,7 @@ services.github-runner-zone = { count = 16; - runnerSettings.networkNamespace.name = "ns1"; + networkNamespace.name = "ns1"; }; networking.namespaces.ns1 = { diff --git a/tf/proxmox_vms.tf b/tf/proxmox_vms.tf index 0b5574b0..a67ae2de 100644 --- a/tf/proxmox_vms.tf +++ b/tf/proxmox_vms.tf @@ -119,7 +119,7 @@ resource "proxmox_virtual_environment_container" "aya" { memory { dedicated = 16384 - swap = 8192 + swap = 12288 } cpu {