From 34a41a64ba7dc419af61a4daaec89cd8e70445c0 Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 08:13:45 -0700 Subject: [PATCH 1/8] use ci channels --- ci/hosts.nix | 7 +++---- ci/niv-cron.nix | 24 +++++++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/ci/hosts.nix b/ci/hosts.nix index 829d0ddf..e197458e 100644 --- a/ci/hosts.nix +++ b/ci/hosts.nix @@ -1,13 +1,12 @@ -{ lib, channels, env, ... }: with lib; let - nixfiles = import ../.; -in { +{ lib, channels, env, ... }: with lib; { name = "hosts"; ci.gh-actions.enable = true; ci.gh-actions.export = true; + channels.nixfiles.path = ../.; jobs = let hostnames = [ "samhain" "yule" "athame" ]; in mapAttrs' (k: nameValuePair "host-${k}") (genAttrs hostnames (host: { - tasks.${host}.inputs = nixfiles.hosts.${host}.config.system.build.toplevel; + tasks.${host}.inputs = channels.nixfiles.hosts.${host}.config.system.build.toplevel; })); ci.gh-actions.checkoutOptions.submodules = false; diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 888d2ba1..4ec832f4 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -23,26 +23,36 @@ with lib; { }; }; + channels = { + nixfiles.path = ../.; + nixpkgs.path = "${channels.nixfiles.sources.nixpkgs}"; + }; + + environment.test = { + inherit (channels.cipkgs) cachix; + inherit (channels.nixpkgs) niv; + }; + jobs.niv-update = { tasks.niv-build.inputs = with channels.cipkgs; ci.command { name = "niv-update-build"; displayName = "niv update build"; - nativeBuildInputs = [ nix cachix ]; environment = [ "OPENSSH_PRIVATE_KEY" "CACHIX_SIGNING_KEY" ]; - command = let sources = (import ../.).sources; in - '' + command = '' mkdir ~/.ssh echo "$OPENSSH_PRIVATE_KEY" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa - for source in ${toString (attrNames sources)}; do - nix run -f . pkgs.niv -c niv update $source || true + + for source in ${toString (attrNames channels.nixfiles.sources)}; do + niv update $source || true done + if git status --porcelain | grep -qF nix/sources.json; then if nix build -Lf . hosts.{athame,yule,samhain}.config.system.build.toplevel; then nix build -f ../. sourceCache - ${cachix}/bin/cachix push kittywitch $(nix eval '(toString (import ../.).sourceCache)') - nix-build $(echo "-A hosts."{athame,yule,samhain}.config.system.build.toplevel) | ${cachix}/bin/cachix push kittywitch + cachix push kittywitch $(nix eval '(toString (import ../.).sourceCache)') + nix-build $(echo "-A hosts."{athame,yule,samhain}.config.system.build.toplevel) | cachix push kittywitch git add nix/sources.json export GIT_{COMMITTER,AUTHOR}_EMAIL=kat@kittywit.ch export GIT_{COMMITTER,AUTHOR}_NAME=kat witch From b262605e5ab4b06a33e9838ffa7f32dd7bf52dee Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 08:20:43 -0700 Subject: [PATCH 2/8] log when there's nothing to do --- ci/niv-cron.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 4ec832f4..4407bf00 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -61,6 +61,8 @@ with lib; { GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ git push gitea master fi + else + echo "no source changes" >&2 fi ''; impure = true; From 6a3b8ef8ca75793bc8ee110d82861fc83d76007f Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 08:25:16 -0700 Subject: [PATCH 3/8] skip ci steps that require (missing) secrets --- ci/niv-cron.nix | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 4407bf00..611f0d0e 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -40,9 +40,11 @@ with lib; { displayName = "niv update build"; environment = [ "OPENSSH_PRIVATE_KEY" "CACHIX_SIGNING_KEY" ]; command = '' - mkdir ~/.ssh - echo "$OPENSSH_PRIVATE_KEY" > ~/.ssh/id_rsa - chmod 0600 ~/.ssh/id_rsa + if [[ -n $OPENSSH_PRIVATE_KEY ]]; then + mkdir ~/.ssh + echo "$OPENSSH_PRIVATE_KEY" > ~/.ssh/id_rsa + chmod 0600 ~/.ssh/id_rsa + fi for source in ${toString (attrNames channels.nixfiles.sources)}; do niv update $source || true @@ -51,15 +53,19 @@ with lib; { if git status --porcelain | grep -qF nix/sources.json; then if nix build -Lf . hosts.{athame,yule,samhain}.config.system.build.toplevel; then nix build -f ../. sourceCache - cachix push kittywitch $(nix eval '(toString (import ../.).sourceCache)') - nix-build $(echo "-A hosts."{athame,yule,samhain}.config.system.build.toplevel) | cachix push kittywitch - git add nix/sources.json - export GIT_{COMMITTER,AUTHOR}_EMAIL=kat@kittywit.ch - export GIT_{COMMITTER,AUTHOR}_NAME=kat witch - git commit --message="ci-trusted: niv update" - git remote add gitea ssh://gitea@git.kittywit.ch:62954/kat/nixfiles.git - GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ - git push gitea master + if [[ -n $CACHIX_SIGNING_KEY ]]; then + cachix push kittywitch $(nix eval '(toString (import ../.).sourceCache)') + nix-build $(echo "-A hosts."{athame,yule,samhain}.config.system.build.toplevel) | cachix push kittywitch + fi + if [[ -n $OPENSSH_PRIVATE_KEY ]]; then + git add nix/sources.json + export GIT_{COMMITTER,AUTHOR}_EMAIL=kat@kittywit.ch + export GIT_{COMMITTER,AUTHOR}_NAME=kat witch + git commit --message="ci-trusted: niv update" + git remote add gitea ssh://gitea@git.kittywit.ch:62954/kat/nixfiles.git + GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ + git push gitea master + fi fi else echo "no source changes" >&2 From a8bfcdf17f991f5550e664f947e89b6cd5ec423d Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 08:30:38 -0700 Subject: [PATCH 4/8] ensure sources are cached locally before building --- ci/hosts.nix | 10 ++++++++++ ci/niv-cron.nix | 11 ++++++++--- default.nix | 12 ++++++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ci/hosts.nix b/ci/hosts.nix index e197458e..83569450 100644 --- a/ci/hosts.nix +++ b/ci/hosts.nix @@ -4,6 +4,16 @@ ci.gh-actions.export = true; channels.nixfiles.path = ../.; + # ensure sources are fetched and available in the local store before evaluating host configs + environment.bootstrap = { + sourceCache = channels.cipkgs.runCommand "sources" { + srcs = attrNames channels.nixfiles.sourceCache.local; + } '' + mkdir -p $out/share/sources + ln -s $srcs $out/share/sources/ + ''; + }; + jobs = let hostnames = [ "samhain" "yule" "athame" ]; in mapAttrs' (k: nameValuePair "host-${k}") (genAttrs hostnames (host: { tasks.${host}.inputs = channels.nixfiles.hosts.${host}.config.system.build.toplevel; diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 611f0d0e..68991f0c 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -51,11 +51,14 @@ with lib; { done if git status --porcelain | grep -qF nix/sources.json; then + nix build --no-link -f . sourceCache.local if nix build -Lf . hosts.{athame,yule,samhain}.config.system.build.toplevel; then - nix build -f ../. sourceCache if [[ -n $CACHIX_SIGNING_KEY ]]; then - cachix push kittywitch $(nix eval '(toString (import ../.).sourceCache)') - nix-build $(echo "-A hosts."{athame,yule,samhain}.config.system.build.toplevel) | cachix push kittywitch + nix build --no-link -f . sourceCache.all + cachix push kittywitch $(nix eval -f . sourceCache.allStr) + + cachix push kittywitch result*/ & + CACHIX_PUSH=$! fi if [[ -n $OPENSSH_PRIVATE_KEY ]]; then git add nix/sources.json @@ -66,6 +69,8 @@ with lib; { GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ git push gitea master fi + + wait ''${CACHIX_PUSH-} fi else echo "no source changes" >&2 diff --git a/default.nix b/default.nix index 1b4ae31a..f5393170 100644 --- a/default.nix +++ b/default.nix @@ -19,8 +19,16 @@ rec { runners = import ./runners.nix { inherit lib; inherit (deploy) target; }; - getSources = sources: lib.attrValues (lib.removeAttrs sources [ "__functor" ]); - sourceCache = map(value: if lib.isDerivation value.outPath then value.outPath else value) (getSources sources ++ getSources (import sources.nix-hexchen {}).sources); + sourceCache = with lib; let + getSources = sources: removeAttrs sources [ "__functor" "dorkfiles" ]; + source2drv = value: if isDerivation value.outPath then value.outPath else value; + sources2drvs = sources: mapAttrs (_: source2drv) (getSources sources); + in recurseIntoAttrs rec { + local = sources2drvs sources; + hexchen = sources2drvs (import sources.nix-hexchen {}).sources; + all = attrValues local ++ attrValues hexchen; + allStr = toString all; + }; deploy = import ./lib/deploy.nix { inherit pkgs sources; From 8cd7a0170cf63e2f41e3ada106cbcdd48470ab3e Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 08:54:19 -0700 Subject: [PATCH 5/8] fix cron trigger paths --- .github/workflows/niv-update.yml | 8 ++++++-- ci/niv-cron.nix | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/niv-update.yml b/.github/workflows/niv-update.yml index 19122da1..8237b5a9 100644 --- a/.github/workflows/niv-update.yml +++ b/.github/workflows/niv-update.yml @@ -92,10 +92,14 @@ name: niv-update pull_request: paths: - nix/* - - ci/* + - default.nix + - ./ci/niv-cron.nix + - .github/workflows/niv-update.yml push: paths: - nix/* - - ci/* + - default.nix + - ./ci/niv-cron.nix + - .github/workflows/niv-update.yml schedule: - cron: 0 */6 * * * diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 68991f0c..4171d8d2 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -1,4 +1,4 @@ -{ lib, channels, ... }: +{ lib, channels, config, ... }: with lib; { name = "niv-update"; ci.gh-actions.enable = true; @@ -9,7 +9,11 @@ with lib; { gh-actions = { on = let - paths = [ "nix/*" "ci/*" ]; + paths = [ + "nix/*" # niv and sources.json + "default.nix" # sourceCache + config.ci.configPath config.ci.gh-actions.path + ]; in { push = { inherit paths; From 0d3b44c6a9b40edeb31912e69fc02a41d9ec16fd Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 09:13:26 -0700 Subject: [PATCH 6/8] update hexchen source --- nix/sources.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 1e74ebc8..da388678 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -78,9 +78,9 @@ "ref": "main", "repo": "https://gitlab.com/hexchen/nixfiles", "rev": "8a152b95d09c9729bfe4582c89a8900a0d32caf1", - "sha256": "0zz0fxdf3k4nmm5l2raw35wd1m27d5qp46r4aq4lc124bm30qqab", + "sha256": "0yqh49v0212v6h66xk9j5rbvm454pyfp10mbqcqs9sf67lndga4m", "type": "tarball", - "url": "https://gitlab.com/hexchen/nixfiles/-/archive/1dee1474f147ce8d04ec4b200df1c6255ae59bdc/nixfiles-1dee1474f147ce8d04ec4b200df1c6255ae59bdc.tar.gz", + "url": "https://gitlab.com/hexchen/nixfiles/-/archive/8a152b95d09c9729bfe4582c89a8900a0d32caf1/nixfiles-8a152b95d09c9729bfe4582c89a8900a0d32caf1.tar.gz", "url_template": "/-/archive/.tar.gz" }, "nixos-mailserver": { From d712176441f2f8259618dd6e5ad2ed750ff6870e Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 09:50:55 -0700 Subject: [PATCH 7/8] show niv update diff --- ci/niv-cron.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 4171d8d2..47c389aa 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -55,10 +55,12 @@ with lib; { done if git status --porcelain | grep -qF nix/sources.json; then - nix build --no-link -f . sourceCache.local + git -P diff nix/sources.json + nix build --no-link -Lf . sourceCache.local + echo "checking that hosts still build..." >&2 if nix build -Lf . hosts.{athame,yule,samhain}.config.system.build.toplevel; then if [[ -n $CACHIX_SIGNING_KEY ]]; then - nix build --no-link -f . sourceCache.all + nix build --no-link -Lf . sourceCache.all cachix push kittywitch $(nix eval -f . sourceCache.allStr) cachix push kittywitch result*/ & From 2a6edb865607958d21b46f26fe9f419b8b48165a Mon Sep 17 00:00:00 2001 From: arcnmx Date: Wed, 19 May 2021 09:48:24 -0700 Subject: [PATCH 8/8] support updating arbitrary git repos via niv --- ci/niv-cron.nix | 18 +++++++++++++++--- nix/sources.json | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ci/niv-cron.nix b/ci/niv-cron.nix index 47c389aa..4491fce3 100644 --- a/ci/niv-cron.nix +++ b/ci/niv-cron.nix @@ -50,9 +50,21 @@ with lib; { chmod 0600 ~/.ssh/id_rsa fi - for source in ${toString (attrNames channels.nixfiles.sources)}; do - niv update $source || true - done + ${concatStringsSep "\n" (mapAttrsToList (source: spec: let + update = "niv update ${source}"; + fetch = "timeout 30 git fetch -q --depth 1 ${spec.repo} ${spec.branch}:source-${source}"; + revision = "$(git show-ref -s source-${source})"; + isGit = hasPrefix "https://" spec.repo or ""; + git = '' + if ${fetch}; then + echo "${source}:${spec.branch} HEAD at ${revision}" >&2 + ${update} -r ${revision} || true + else + echo "failed to fetch latest revision from ${spec.repo}" >&2 + fi + ''; + auto = "${update} || true"; + in if isGit then git else auto) channels.nixfiles.sources)} if git status --porcelain | grep -qF nix/sources.json; then git -P diff nix/sources.json diff --git a/nix/sources.json b/nix/sources.json index da388678..78498004 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -143,6 +143,7 @@ "url_template": "/archive/.tar.gz" }, "qyliss-nixlib": { + "branch": "master", "builtin": false, "repo": "https://git.qyliss.net/nixlib", "rev": "e14330c5be9b005d4310cd4dc0d384cff882aedc",