merge #2

.github/workflows/niv-update.yml

@@ -92,10 +92,14 @@ name: niv-update
   pull_request:
     paths:
     - nix/*
-    - ci/*
+    - default.nix
+    - ./ci/niv-cron.nix
+    - .github/workflows/niv-update.yml
   push:
     paths:
     - nix/*
-    - ci/*
+    - default.nix
+    - ./ci/niv-cron.nix
+    - .github/workflows/niv-update.yml
   schedule:
   - cron: 0 */6 * * *

ci/hosts.nix

@@ -1,13 +1,22 @@
-{ lib, channels, env, ... }: with lib; let
-  nixfiles = import ../.; 
-in {
+{ lib, channels, env, ... }: with lib; {
   name = "hosts";
   ci.gh-actions.enable = true;
   ci.gh-actions.export = true;
+  channels.nixfiles.path = ../.;
+
+  # ensure sources are fetched and available in the local store before evaluating host configs
+  environment.bootstrap = {
+    sourceCache = channels.cipkgs.runCommand "sources" {
+      srcs = attrNames channels.nixfiles.sourceCache.local;
+    } ''
+      mkdir -p $out/share/sources
+      ln -s $srcs $out/share/sources/
+    '';
+  };
 
   jobs = let hostnames = [ "samhain" "yule" "athame" ];
   in mapAttrs' (k: nameValuePair "host-${k}") (genAttrs hostnames (host: {
-      tasks.${host}.inputs = nixfiles.hosts.${host}.config.system.build.toplevel;
+      tasks.${host}.inputs = channels.nixfiles.hosts.${host}.config.system.build.toplevel;
   }));
 
   ci.gh-actions.checkoutOptions.submodules = false;

ci/niv-cron.nix

@@ -1,4 +1,4 @@
-{ lib, channels, ... }:
+{ lib, channels, config, ... }:
 with lib; {
   name = "niv-update";
   ci.gh-actions.enable = true;
@@ -9,7 +9,11 @@ with lib; {
 
   gh-actions = {
     on = let
-      paths = [ "nix/*" "ci/*" ];
+      paths = [
+        "nix/*" # niv and sources.json
+        "default.nix" # sourceCache
+        config.ci.configPath config.ci.gh-actions.path
+      ];
     in {
       push = {
         inherit paths;
@@ -23,34 +27,71 @@ with lib; {
     };
   };
 
+  channels = {
+    nixfiles.path = ../.;
+    nixpkgs.path = "${channels.nixfiles.sources.nixpkgs}";
+  };
+
+  environment.test = {
+    inherit (channels.cipkgs) cachix;
+    inherit (channels.nixpkgs) niv;
+  };
+
   jobs.niv-update = {
       tasks.niv-build.inputs = with channels.cipkgs;
         ci.command {
           name = "niv-update-build";
           displayName = "niv update build";
-          nativeBuildInputs = [ nix cachix ];
           environment = [ "OPENSSH_PRIVATE_KEY" "CACHIX_SIGNING_KEY" ];
-          command = let sources = (import ../.).sources; in
-          ''
-            mkdir ~/.ssh
-            echo "$OPENSSH_PRIVATE_KEY" > ~/.ssh/id_rsa
-            chmod 0600 ~/.ssh/id_rsa
-            for source in ${toString (attrNames sources)}; do
-              nix run -f . pkgs.niv  -c niv update $source || true
-            done
+          command = ''
+            if [[ -n $OPENSSH_PRIVATE_KEY ]]; then
+              mkdir ~/.ssh
+              echo "$OPENSSH_PRIVATE_KEY" > ~/.ssh/id_rsa
+              chmod 0600 ~/.ssh/id_rsa
+            fi
+
+            ${concatStringsSep "\n" (mapAttrsToList (source: spec: let
+              update = "niv update ${source}";
+              fetch = "timeout 30 git fetch -q --depth 1 ${spec.repo} ${spec.branch}:source-${source}";
+              revision = "$(git show-ref -s source-${source})";
+              isGit = hasPrefix "https://" spec.repo or "";
+              git = ''
+                if ${fetch}; then
+                  echo "${source}:${spec.branch} HEAD at ${revision}" >&2
+                  ${update} -r ${revision} || true
+                else
+                  echo "failed to fetch latest revision from ${spec.repo}" >&2
+                fi
+              '';
+              auto = "${update} || true";
+            in if isGit then git else auto) channels.nixfiles.sources)}
+
             if git status --porcelain | grep -qF nix/sources.json; then
+              git -P diff nix/sources.json
+              nix build --no-link -Lf . sourceCache.local
+              echo "checking that hosts still build..." >&2
               if nix build -Lf . hosts.{athame,yule,samhain}.config.system.build.toplevel; then
-                nix build -f ../. sourceCache
-                ${cachix}/bin/cachix push kittywitch $(nix eval '(toString (import ../.).sourceCache)')
-                nix-build $(echo "-A hosts."{athame,yule,samhain}.config.system.build.toplevel) | ${cachix}/bin/cachix push kittywitch
-                git add nix/sources.json
-                export GIT_{COMMITTER,AUTHOR}_EMAIL=kat@kittywit.ch
-                export GIT_{COMMITTER,AUTHOR}_NAME=kat witch
-                git commit --message="ci-trusted: niv update"
-                git remote add gitea ssh://gitea@git.kittywit.ch:62954/kat/nixfiles.git
-                GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
-                  git push gitea master
+                if [[ -n $CACHIX_SIGNING_KEY ]]; then
+                  nix build --no-link -Lf . sourceCache.all
+                  cachix push kittywitch $(nix eval -f . sourceCache.allStr)
+
+                  cachix push kittywitch result*/ &
+                  CACHIX_PUSH=$!
+                fi
+                if [[ -n $OPENSSH_PRIVATE_KEY ]]; then
+                  git add nix/sources.json
+                  export GIT_{COMMITTER,AUTHOR}_EMAIL=kat@kittywit.ch
+                  export GIT_{COMMITTER,AUTHOR}_NAME=kat witch
+                  git commit --message="ci-trusted: niv update"
+                  git remote add gitea ssh://gitea@git.kittywit.ch:62954/kat/nixfiles.git
+                  GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
+                    git push gitea master
+                fi
+
+                wait ''${CACHIX_PUSH-}
               fi
+            else
+              echo "no source changes" >&2
             fi
           '';
           impure = true;

default.nix

@@ -19,8 +19,16 @@ rec {
 
     runners = import ./runners.nix { inherit lib; inherit (deploy) target; };
 
-    getSources = sources: lib.attrValues (lib.removeAttrs sources [ "__functor" ]);
-    sourceCache = map(value: if lib.isDerivation value.outPath then value.outPath else value) (getSources sources ++ getSources (import sources.nix-hexchen {}).sources);
+    sourceCache = with lib; let
+      getSources = sources: removeAttrs sources [ "__functor" "dorkfiles" ];
+      source2drv = value: if isDerivation value.outPath then value.outPath else value;
+      sources2drvs = sources: mapAttrs (_: source2drv) (getSources sources);
+    in recurseIntoAttrs rec {
+      local = sources2drvs sources;
+      hexchen = sources2drvs (import sources.nix-hexchen {}).sources;
+      all = attrValues local ++ attrValues hexchen;
+      allStr = toString all;
+    };
 
   deploy = import ./lib/deploy.nix {
     inherit pkgs sources;

nix/sources.json

@@ -78,9 +78,9 @@
         "ref": "main",
         "repo": "https://gitlab.com/hexchen/nixfiles",
         "rev": "8a152b95d09c9729bfe4582c89a8900a0d32caf1",
-        "sha256": "0zz0fxdf3k4nmm5l2raw35wd1m27d5qp46r4aq4lc124bm30qqab",
+        "sha256": "0yqh49v0212v6h66xk9j5rbvm454pyfp10mbqcqs9sf67lndga4m",
         "type": "tarball",
-        "url": "https://gitlab.com/hexchen/nixfiles/-/archive/1dee1474f147ce8d04ec4b200df1c6255ae59bdc/nixfiles-1dee1474f147ce8d04ec4b200df1c6255ae59bdc.tar.gz",
+        "url": "https://gitlab.com/hexchen/nixfiles/-/archive/8a152b95d09c9729bfe4582c89a8900a0d32caf1/nixfiles-8a152b95d09c9729bfe4582c89a8900a0d32caf1.tar.gz",
         "url_template": "<repo>/-/archive/<rev>.tar.gz"
     },
     "nixos-mailserver": {
@@ -143,6 +143,7 @@
         "url_template": "<repo>/archive/<rev>.tar.gz"
     },
     "qyliss-nixlib": {
+        "branch": "master",
         "builtin": false,
         "repo": "https://git.qyliss.net/nixlib",
         "rev": "e14330c5be9b005d4310cd4dc0d384cff882aedc",