feat(ci): unify nf-update command for actions use

.github/workflows/flake-update.yml

@@ -43,6 +43,14 @@ jobs:
     - id: nix-install
       name: nix install
       uses: arcnmx/ci/actions/nix/install@v0.7
+    - env:
+        CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
+        NF_CONFIG_ROOT: ${{ github.workspace }}
+        NF_UPDATE_CACHIX_PUSH: '1'
+        NF_UPDATE_GIT_COMMIT: '1'
+      id: flake-update
+      name: flake update build
+      run: nix run .#nf-update
     - id: ci-dirty
       name: nix test dirty
       uses: arcnmx/ci/actions/nix/run@v0.7

.github/workflows/nodes.yml

@@ -181,6 +181,57 @@ jobs:
         command: ci-build-cache
         quiet: false
         stdin: ${{ runner.temp }}/ci.build.cache
+  keycloak:
+    name: nodes-keycloak
+    runs-on: ubuntu-latest
+    steps:
+    - id: checkout
+      name: git clone
+      uses: actions/checkout@v4
+      with:
+        submodules: false
+    - id: nix-install
+      name: nix install
+      uses: arcnmx/ci/actions/nix/install@v0.7
+    - id: ci-dirty
+      name: nix test dirty
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.keycloak.run.test
+        command: ci-build-dirty
+        quiet: false
+        stdout: ${{ runner.temp }}/ci.build.dirty
+    - id: ci-test
+      name: nix test build
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.keycloak.run.test
+        command: ci-build-realise
+        ignore-exit-code: true
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.dirty
+    - env:
+        CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }}
+      id: ci-summary
+      name: nix test results
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.keycloak.run.test
+        command: ci-build-summarise
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.dirty
+        stdout: ${{ runner.temp }}/ci.build.cache
+    - env:
+        CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
+      id: ci-cache
+      if: always()
+      name: nix test cache
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.keycloak.run.test
+        command: ci-build-cache
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.cache
   litterbox:
     name: nodes-litterbox
     runs-on: ubuntu-latest

ci/actions-test.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -eu
+
+if [[ ${GITHUB_ACTIONS-} = true && ${RUNNER_NAME-} = "Github Actions"* ]]; then
+	# low disk space available on public runners...
+	echo "enabled GC between builds due to restricted disk space..." >&2
+	export NF_ACTIONS_TEST_GC=1
+fi
+
+NIX_BUILD_ARGS=(
+	--show-trace
+)
+
+for nfsystem in "${NF_NIX_SYSTEMS[@]}"; do
+	nfargs=(
+		"${NIX_BUILD_ARGS[@]}"
+	)
+	if [[ -n "${NF_ACTIONS_TEST_OUTLINK-}" || -n "${NF_UPDATE_CACHIX_PUSH-}" ]]; then
+		nfargs+=(
+			-o "${NF_ACTIONS_TEST_OUTLINK-result}-$nfsystem"
+		)
+	else
+		nfargs+=(
+			--no-link
+		)
+	fi
+
+	echo "building ${nfsystem}..." >&2
+
+	nix build \
+		"${NF_CONFIG_ROOT}#nixosConfigurations.${nfsystem}.config.system.build.toplevel" \
+		"${nfargs[@]}" \
+		"$@"
+
+	if [[ -n "${NF_ACTIONS_TEST_GC-}" ]]; then
+		if [[ -n "${NF_UPDATE_CACHIX_PUSH-}" ]]; then
+			cachix push gensokyo-infrastructure "./${NF_ACTIONS_TEST_OUTLINK-result}-$nfsystem"*/
+			rm -f "./${NF_ACTIONS_TEST_OUTLINK-result}-$nfsystem"*
+		fi
+		nix-collect-garbage -d
+	fi
+done

ci/flake-cron.nix

@@ -10,6 +10,7 @@ in {
   name = "flake-update";
 
   nixpkgs.args.localSystem = "x86_64-linux";
+  nixpkgs.args.config.checkMetaRecursively = false;
 
   ci = {
     version = "v0.7";
@@ -21,7 +22,6 @@ in {
   gh-actions.env.CACHIX_SIGNING_KEY = "\${{ secrets.CACHIX_SIGNING_KEY }}";
 
   nix.config = {
-    accept-flake-config = true;
     extra-platforms = ["aarch64-linux" "armv6l-linux" "armv7l-linux"];
     #extra-sandbox-paths = with channels.cipkgs; map (package: builtins.unsafeDiscardStringContext "${package}?") [bash qemu "/run/binfmt"];
   };
@@ -47,6 +47,20 @@ in {
       ];
       workflow_dispatch = {};
     };
+    jobs.flake-update = {
+      # TODO: split this up into two phases, then push at the end so other CI tests can run first
+      step.flake-update = {
+        name = "flake update build";
+        order = 500;
+        run = "nix run .#nf-update";
+        env = {
+          CACHIX_SIGNING_KEY = "\${{ secrets.CACHIX_SIGNING_KEY }}";
+          NF_UPDATE_GIT_COMMIT = "1";
+          NF_UPDATE_CACHIX_PUSH = "1";
+          NF_CONFIG_ROOT = "\${{ github.workspace }}";
+        };
+      };
+    };
   };
 
   channels = {
@@ -55,50 +69,6 @@ in {
   };
 
   jobs.flake-update = {
-    tasks.flake-build.inputs = with channels.cipkgs;
-      ci.command {
-        name = "flake-update-build";
-        allowSubstitutes = false;
-        cache = {
-          enable = false;
-        };
-        displayName = "flake update build";
-        environment = ["CACHIX_SIGNING_KEY" "GITHUB_REF"];
-        command = let
-          filteredHosts = ["hakurei" "reimu" "aya" "tei" "litterbox" "mediabox"];
-          gcBetweenHosts = false;
-          nodeBuildString = concatMapStringsSep " && " (node: "nix build --show-trace -Lf . nixosConfigurations.${node}.config.system.build.toplevel -o result-${node}" + optionalString gcBetweenHosts " && nix-collect-garbage -d") filteredHosts;
-          hostPath = builtins.getEnv "PATH";
-        in ''
-          # ${toString builtins.currentTime}
-          export PATH="${hostPath}:$PATH"
-          export NIX_CONFIG="$(printf '%s\naccept-flake-config = true\n' "''${NIX_CONFIG-}")"
-          nix flake update
-
-          if git status --porcelain | grep -qF flake.lock; then
-            git -P diff flake.lock
-            echo "checking that nodes still build..." >&2
-            if ${nodeBuildString}; then
-              if [[ -n $CACHIX_SIGNING_KEY ]]; then
-                cachix push gensokyo-infrastructure result*/ &
-                CACHIX_PUSH=$!
-              fi
-              git add flake.lock
-              export GIT_{COMMITTER,AUTHOR}_EMAIL=github@kittywit.ch
-              export GIT_{COMMITTER,AUTHOR}_NAME="flake cron job"
-              git commit --message="ci: flake update"
-              if [[ $GITHUB_REF = refs/heads/${gitBranch} ]]; then
-                git push origin HEAD:${gitBranch}
-              fi
-
-              wait ''${CACHIX_PUSH-}
-            fi
-          else
-            echo "no source changes" >&2
-          fi
-        '';
-        impure = true;
-      };
   };
 
   ci.gh-actions.checkoutOptions = {

ci/generate.sh

@@ -4,3 +4,8 @@ set -eu
 for node in reisen; do
 	nix eval --json "${NF_CONFIG_ROOT}#lib.generate.$node.users" | jq -M . > "$NF_CONFIG_ROOT/systems/$node/users.json"
 done
+
+for ciconfig in "${NF_CONFIG_FILES[@]}"; do
+	echo "processing ${ciconfig}..." >&2
+	nix run --argstr config "$NF_CONFIG_ROOT/ci/$ciconfig" -f "$NF_INPUT_CI" run.gh-actions-generate
+done

ci/nix.nix

@@ -0,0 +1,17 @@
+{
+  ci = {
+    workflowConfigs = [
+      "nodes.nix"
+      "flake-cron.nix"
+    ];
+    nixosSystems = [
+      "hakurei"
+      "reimu"
+      "aya"
+      "tei"
+      "litterbox"
+      "keycloak"
+      "mediabox"
+    ];
+  };
+}

ci/nodes.nix

@@ -9,6 +9,7 @@ with lib; {
   name = "nodes";
 
   nixpkgs.args.localSystem = "x86_64-linux";
+  nixpkgs.args.config.checkMetaRecursively = false;
 
   ci = {
     version = "v0.7";
@@ -19,13 +20,12 @@ with lib; {
   channels.nixfiles.path = ../.;
 
   nix.config = {
-    accept-flake-config = true;
     extra-platforms = ["aarch64-linux" "armv6l-linux" "armv7l-linux"];
     #extra-sandbox-paths = with channels.cipkgs; map (package: builtins.unsafeDiscardStringContext "${package}?") [bash qemu "/run/binfmt"];
   };
 
   jobs = let
-    enabledHosts = ["hakurei" "reimu" "aya" "tei" "litterbox" "mediabox" "ct"];
+    enabledHosts = ["hakurei" "reimu" "aya" "tei" "litterbox" "keycloak" "mediabox" "ct"];
   in
     mapAttrs' (k: nameValuePair "${k}") (genAttrs enabledHosts (host: {
       tasks.${host}.inputs = channels.nixfiles.nixosConfigurations.${host}.config.system.build.toplevel;

ci/update.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -eu
+
+if [[ -n ${CACHIX_SIGNING_KEY-} ]]; then
+	export NF_UPDATE_CACHIX_PUSH=1
+fi
+
+cd "$NF_CONFIG_ROOT"
+
+nix flake update "$@"
+
+if [[ -n $(git status --porcelain ./flake.lock) ]]; then
+	git -P diff ./flake.lock
+else
+	echo "no source changes" >&2
+	exit
+fi
+
+echo "checking that nodes still build..." >&2
+if [[ -n ${NF_UPDATE_CACHIX_PUSH-} ]]; then
+	export NF_ACTIONS_TEST_OUTLINK=${NF_ACTIONS_TEST_OUTLINK-result}
+fi
+nf-actions-test
+
+if [[ -n ${NF_UPDATE_CACHIX_PUSH-} ]]; then
+	cachix push gensokyo-infrastructure "./${NF_ACTIONS_TEST_OUTLINK}"*/ &
+	CACHIX_PUSH=$!
+fi
+
+if [[ -z ${NF_UPDATE_GIT_COMMIT-} ]]; then
+	exit
+fi
+
+if [[ -n $(git diff --staged) ]]; then
+	echo "git working tree dirty, refusing to commit..." >&2
+	exit 1
+fi
+
+git add flake.lock
+env \
+	GIT_{COMMITTER,AUTHOR}_EMAIL=github@kittywit.ch \
+	GIT_{COMMITTER,AUTHOR}_NAME="flake cron job" \
+git commit --message="ci: flake update"
+
+if [[ ${GITHUB_REF-} = refs/heads/${NF_UPDATE_BRANCH-main} ]]; then
+	git push origin HEAD:${NF_UPDATE_BRANCH-main}
+fi
+
+wait ${CACHIX_PUSH-}

devShells.nix

@@ -23,23 +23,6 @@
       ${optionalString (subdir != null) ''cd "$NF_CONFIG_ROOT${subdir}"''}
       exec nix ${subcommand} ''${FLAKE_OPTS-} "$NF_CONFIG_ROOT#${attr}" ${exeArg} "$@"
     '';
-  nf-actions = pkgs.writeShellScriptBin "nf-actions" ''
-    NF_CONFIG_FILES=($NF_CONFIG_ROOT/ci/{nodes,flake-cron}.nix)
-    for f in "''${NF_CONFIG_FILES[@]}"; do
-      echo $f
-      nix run --argstr config "$f" -f '${inputs.ci}' run.gh-actions-generate
-    done
-  '';
-  nf-actions-test = pkgs.writeShellScriptBin "nf-actions-test" ''
-    set -eu
-    for host in hakurei reimu aya tei litterbox mediabox ct; do
-      echo testing $host...
-      nix run --argstr config "$NF_CONFIG_ROOT/ci/nodes.nix" -f '${inputs.ci}' job.$host.test
-    done
-  '';
-  nf-update = pkgs.writeShellScriptBin "nf-update" ''
-    exec nix flake update "$@"
-  '';
   nf-tf = pkgs.writeShellScriptBin "nf-tf" ''
     cd "$NF_CONFIG_ROOT/tf"
     if [[ $# -eq 0 ]]; then
@@ -52,10 +35,9 @@
     nativeBuildInputs = with pkgs; [
       inetutils
       sops
-      nf-actions
-      nf-actions-test
-      nf-update
       nf-tf
+      (mkWrapper {name = "nf-update";})
+      (mkWrapper {name = "nf-actions-test";})
       (mkWrapper {name = "nf-docs";})
       (mkWrapper {name = "nf-generate";})
       (mkWrapper {name = "nf-setup-node";})

packages/default.nix

@@ -8,6 +8,7 @@
   packages = inputs.self.packages.${system};
   inherit (inputs.self.legacyPackages.${system}) pkgs;
   fmt = import ../ci/fmt.nix;
+  inherit (import ../ci/nix.nix) ci;
   exports = ''
     export NF_CONFIG_ROOT=''${NF_CONFIG_ROOT-${toString ../.}}
   '';
@@ -65,6 +66,16 @@
         )
         source ${../ci/setup.sh}
       '';
+    nf-actions-test = pkgs.writeShellScriptBin "nf-actions-test" ''
+      ${exports}
+      NF_NIX_SYSTEMS=(${string.concatMapSep " " string.escapeShellArg ci.nixosSystems})
+      source ${../ci/actions-test.sh}
+    '';
+    nf-update = pkgs.writeShellScriptBin "nf-update" ''
+      ${exports}
+      export PATH="${makeBinPath [packages.nf-actions-test pkgs.cachix]}:$PATH"
+      source ${../ci/update.sh}
+    '';
     nf-hostname = pkgs.writeShellScriptBin "nf-hostname" ''
       ${exports}
       source ${../ci/hostname.sh}
@@ -101,6 +112,8 @@
     nf-generate = pkgs.writeShellScriptBin "nf-generate" ''
       ${exports}
       export PATH="$PATH:${makeBinPath [pkgs.jq]}"
+      NF_INPUT_CI=${string.escapeShellArg inputs.ci}
+      NF_CONFIG_FILES=(${string.concatMapSep " " string.escapeShellArg ci.workflowConfigs})
       source ${../ci/generate.sh}
     '';
     nf-statix = pkgs.writeShellScriptBin "nf-statix" ''