CI: make work with aarch64 possibly. nixdirfmt

.github/workflows/niv-update.yml

@@ -5,6 +5,15 @@ env:
   CI_PLATFORM: gh-actions
   OPENSSH_PRIVATE_KEY: ${{ secrets.OPENSSH_PRIVATE_KEY }}
 jobs:
+  ci:
+    name: ci
+    runs-on: ubuntu-latest
+    steps:
+    - id: aarch64
+      name: prepare for aarch64 builds
+      run: 'sudo aarch64binfmt
+
+        '
   ci-check:
     name: niv-update check
     runs-on: ubuntu-latest

.github/workflows/nodes.yml

@@ -117,6 +117,15 @@ jobs:
         command: ci-build-cache
         quiet: false
         stdin: ${{ runner.temp }}/ci.build.cache
+  ci:
+    name: ci
+    runs-on: ubuntu-latest
+    steps:
+    - id: aarch64
+      name: prepare for aarch64 builds
+      run: 'sudo aarch64binfmt
+
+        '
   ci-check:
     name: nodes check
     runs-on: ubuntu-latest
@@ -142,6 +151,63 @@ jobs:
         args: -u .github/workflows/nodes.yml .ci/workflow.yml
         attrs: nixpkgs.diffutils
         command: diff
+  daiyousei:
+    name: nodes-daiyousei
+    runs-on: ubuntu-latest
+    steps:
+    - id: checkout
+      name: git clone
+      uses: actions/checkout@v1
+      with:
+        submodules: false
+    - id: nix-install
+      name: nix install
+      uses: arcnmx/ci/actions/nix/install@master
+    - id: ci-setup
+      name: nix setup
+      uses: arcnmx/ci/actions/nix/run@master
+      with:
+        attrs: ci.job.daiyousei.run.bootstrap
+        quiet: false
+    - id: ci-dirty
+      name: nix test dirty
+      uses: arcnmx/ci/actions/nix/run@master
+      with:
+        attrs: ci.job.daiyousei.run.test
+        command: ci-build-dirty
+        quiet: false
+        stdout: ${{ runner.temp }}/ci.build.dirty
+    - id: ci-test
+      name: nix test build
+      uses: arcnmx/ci/actions/nix/run@master
+      with:
+        attrs: ci.job.daiyousei.run.test
+        command: ci-build-realise
+        ignore-exit-code: true
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.dirty
+    - env:
+        CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }}
+      id: ci-summary
+      name: nix test results
+      uses: arcnmx/ci/actions/nix/run@master
+      with:
+        attrs: ci.job.daiyousei.run.test
+        command: ci-build-summarise
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.dirty
+        stdout: ${{ runner.temp }}/ci.build.cache
+    - env:
+        CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
+      id: ci-cache
+      if: always()
+      name: nix test cache
+      uses: arcnmx/ci/actions/nix/run@master
+      with:
+        attrs: ci.job.daiyousei.run.test
+        command: ci-build-cache
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.cache
   ostara:
     name: nodes-ostara
     runs-on: ubuntu-latest

ci/niv-cron.nix

@@ -4,9 +4,31 @@ with lib; {
   ci.gh-actions.enable = true;
   ci.gh-actions.export = true;
 
+  nix.config.extraPlatforms = "aarch64-linux";
+
   gh-actions.env.OPENSSH_PRIVATE_KEY = "\${{ secrets.OPENSSH_PRIVATE_KEY }}";
   gh-actions.env.CACHIX_SIGNING_KEY = "\${{ secrets.CACHIX_SIGNING_KEY }}";
 
+  # ensure sources are fetched and available in the local store before evaluating host configs
+  environment.bootstrap = {
+    aarch64binfmt =
+      let
+        makeQemuWrapper = name: ''
+          mkdir -f /run/binfmt
+          rm -f /run/binfmt/${name}
+          cat > /run/binfmt/${name} << 'EOF'
+          #!${channels.cipkgs.bash}/bin/sh
+          exec -- ${channels.cipkgs.qemu}/bin/qemu-${name} "$@"
+          EOF
+          chmod +x /run/binfmt/${name}
+        ''; in
+      channels.cipkgs.writeShellScriptBin "aarch64binfmt" ''
+        ${makeQemuWrapper "aarch64"}
+        mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
+        echo ':aarch64-linux:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\x00\xff\xfe\xff\xff\xff:/run/binfmt/aarch64:' > /proc/sys/fs/binfmt_misc/register
+      '';
+  };
+
   gh-actions = {
     on =
       let
@@ -27,6 +49,13 @@ with lib; {
           cron = "0 0 * * *";
         }];
       };
+    jobs.ci.step.aarch64 = {
+      order = 201;
+      name = "prepare for aarch64 builds";
+      run = ''
+        sudo aarch64binfmt
+      '';
+    };
   };
 
   channels = {

ci/nodes.nix

@@ -1,11 +1,39 @@
-{ lib, channels, env, ... }: with lib; {
+{ lib, config, channels, env, ... }: with lib; {
   name = "nodes";
   ci.gh-actions.enable = true;
   ci.gh-actions.export = true;
   channels.nixfiles.path = ../.;
 
+  nix.config.extraPlatforms = "aarch64-linux";
+
+  gh-actions = {
+    jobs.ci.step.aarch64 = {
+      order = 201;
+      name = "prepare for aarch64 builds";
+      run = ''
+        sudo aarch64binfmt
+      '';
+    };
+  };
+
   # ensure sources are fetched and available in the local store before evaluating host configs
   environment.bootstrap = {
+    aarch64binfmt =
+      let
+        makeQemuWrapper = name: ''
+          mkdir -f /run/binfmt
+          rm -f /run/binfmt/${name}
+          cat > /run/binfmt/${name} << 'EOF'
+          #!${channels.cipkgs.bash}/bin/sh
+          exec -- ${channels.cipkgs.qemu}/bin/qemu-${name} "$@"
+          EOF
+          chmod +x /run/binfmt/${name}
+        ''; in
+      channels.cipkgs.writeShellScriptBin "aarch64binfmt" ''
+        ${makeQemuWrapper "aarch64"}
+        mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
+        echo ':aarch64-linux:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\x00\xff\xfe\xff\xff\xff:/run/binfmt/aarch64:' > /proc/sys/fs/binfmt_misc/register
+      '';
     sourceCache = channels.cipkgs.runCommand "sources"
       {
         srcs = attrNames channels.nixfiles.sourceCache.local;

config/modules/meta/network.nix

@@ -31,12 +31,14 @@ with lib;
           config = {
             nixpkgs = {
               system = mkDefault pkgs.system;
-              pkgs = let
+              pkgs =
+                let
                   pkgsReval = import pkgs.path {
                     inherit (config.nixpkgs) localSystem crossSystem;
                     inherit (pkgs) overlays config;
                   };
-              in mkDefault (if config.nixpkgs.config == pkgs.config && config.nixpkgs.localSystem.system == pkgs.targetPlatform.system then pkgs else pkgsReval);
+                in
+                mkDefault (if config.nixpkgs.config == pkgs.config && config.nixpkgs.localSystem.system == pkgs.targetPlatform.system then pkgs else pkgsReval);
             };
           };
         };

config/modules/nixos/secrets.nix

@@ -45,13 +45,15 @@ in
       kw.secrets.variables = lib.mkMerge (mapAttrsToList (username: user: user.kw.secrets.variables) config.home-manager.users);
     }
     (mkIf (cfg.variables != { }) {
-      deploy.tf.variables = mapAttrs' (name: content:
+      deploy.tf.variables = mapAttrs'
+        (name: content:
           nameValuePair name ({
             value.shellCommand = "${cfg.command} ${content.path}" + optionalString (content.field != "") " -f ${content.field}";
             type = "string";
             sensitive = true;
           })
-      ) cfg.variables;
+        )
+        cfg.variables;
     })
   ];
 }

config/modules/nixos/yggdrasil.nix

@@ -7,7 +7,8 @@ let
   calcAddr = pubkey: lib.readFile (pkgs.runCommandNoCC "calcaddr-${pubkey}" { } ''
     echo '{ EncryptionPublicKey: "${pubkey}" }' | ${config.services.yggdrasil.package}/bin/yggdrasil -useconf -address | tr -d '\n' > $out
   '').outPath;
-in {
+in
+{
   options.network.yggdrasil = {
     enable = mkEnableOption "Enable the yggdrasil-based private hexnet";
     pubkey = mkOption {
@@ -92,14 +93,18 @@ in {
     };
   };
 
-  config = mkIf cfg.enable (let
-    yggConfigs = filter (
+  config = mkIf cfg.enable (
+    let
+      yggConfigs = filter
+        (
           c: c.enable && (cfg.pubkey != c.pubkey)
-      ) (
+        )
+        (
           mapAttrsToList (_: node: node.network.yggdrasil or { enable = false; pubkey = null; }) meta.network.nodes
         );
       pubkeys = flatten (map (c: [ c.pubkey ] ++ (attrValues c.extra.pubkeys)) yggConfigs);
-  in {
+    in
+    {
       assertions = [
         {
           assertion = !(cfg.listen.enable && (cfg.listen.endpoints == [ ]));
@@ -122,11 +127,13 @@ in {
             AllowFromRemote = false;
             WhitelistEncryptionPublicKeys = pubkeys;
           };
-        TunnelRouting = let
+          TunnelRouting =
+            let
               subnets = v: (
                 listToAttrs (flatten (map (c: map (net: nameValuePair net c.pubkey) c.tunnel."localV${toString v}") yggConfigs))
               ) // cfg.tunnel."remoteV${toString v}";
-        in {
+            in
+            {
               Enable = true;
               IPv4LocalSubnets = cfg.tunnel.localV4 ++ cfg.extra.localV4;
               IPv6LocalSubnets = cfg.tunnel.localV6 ++ cfg.extra.localV6;
@@ -136,12 +143,15 @@ in {
         };
       };
 
-    systemd.services.yggdrasil.postStart = let
+      systemd.services.yggdrasil.postStart =
+        let
           yggTun = config.services.yggdrasil.config.TunnelRouting;
           addNets = v: nets: concatMapStringsSep "\n" (net: "${pkgs.iproute}/bin/ip -${toString v} route add ${net} dev yggdrasil") (attrNames nets);
-    in "sleep 1\n" + (concatMapStringsSep "\n" (v: addNets v yggTun."IPv${toString v}RemoteSubnets") [ 4 6 ]);
+        in
+        "sleep 1\n" + (concatMapStringsSep "\n" (v: addNets v yggTun."IPv${toString v}RemoteSubnets") [ 4 6 ]);
 
-    system.build.yggdrasilTemplate = let
+      system.build.yggdrasilTemplate =
+        let
           json = builtins.toJSON {
             inherit (config.services.yggdrasil.config) Peers SessionFirewall TunnelRouting;
             EncryptionPublicKey = "";
@@ -149,7 +159,9 @@ in {
             SigningPublicKey = "";
             SigningPrivateKey = "";
           };
-    in pkgs.runCommandNoCC "yggdrasil-template.json" {}
+        in
+        pkgs.runCommandNoCC "yggdrasil-template.json" { }
           "echo '${json}' | ${config.services.yggdrasil.package}/bin/yggdrasil -useconf -normaliseconf > $out";
-  });
+    }
+  );
 }

config/profiles/base/secrets.nix

@@ -7,7 +7,9 @@
     external = true;
   };
 
-  kw.secrets.command = let
+  kw.secrets.command =
+    let
       bitw = pkgs.writeShellScriptBin "bitw" ''${pkgs.rbw-bitw}/bin/bitw -p gpg://${config.kw.secrets.repo.bitw.source} "$@"'';
-  in "${bitw}/bin/bitw get";
+    in
+    "${bitw}/bin/bitw get";
 }

config/profiles/cross/aarch64.nix

@@ -1,4 +1,3 @@
-
 { pkgs, config, lib, ... }: with lib; {
   boot.binfmt = {
     emulatedSystems = [ "aarch64-linux" ];

config/services/fusionpbx/default.nix

@@ -7,7 +7,8 @@
     cname = { inherit (config.network.addresses.private) target; };
   };
 
-  kw.secrets.variables = mapListToAttrs (field:
+  kw.secrets.variables = mapListToAttrs
+    (field:
       nameValuePair "fusionpbx-${field}" {
         path = "services/fusionpbx";
         inherit field;

config/services/mail/default.nix

@@ -4,23 +4,30 @@ with lib;
 
 let
   domains = [ "kittywitch" "dork" ];
-in {
+in
+{
   imports = [ sources.nixos-mailserver.outPath ];
 
-  kw.secrets.variables = listToAttrs (map (field:
+  kw.secrets.variables = listToAttrs (map
+    (field:
       nameValuePair "mail-${field}-hash" {
         path = "secrets/mail-kittywitch";
         field = "${field}-hash";
       }) [ "gitea" "kat" ]
-    ++ map (domain:
+  ++ map
+    (domain:
       nameValuePair "mail-domainkey-${domain}" {
         path = "secrets/mail-${domain}";
         field = "notes";
-      }) domains);
+      })
+    domains);
 
-  deploy.tf.dns.records = lib.mkMerge (map (domain: let
+  deploy.tf.dns.records = lib.mkMerge (map
+    (domain:
+      let
         zoneGet = domain: if domain == "dork" then "dork.dev." else config.network.dns.zone;
-    in {
+      in
+      {
         "services_mail_${domain}_mx" = {
           zone = zoneGet domain;
           mx = {
@@ -45,7 +52,8 @@ in {
           domain = "mail._domainkey";
           txt.value = tf.variables."mail-domainkey-${domain}".ref;
         };
-    }) domains);
+      })
+    domains);
 
   secrets.files = {
     mail-kat-hash = {

config/services/matrix/default.nix

@@ -13,14 +13,17 @@ with lib;
     LC_CTYPE = "C";
   '';
 
-  kw.secrets.variables = (mapListToAttrs (field:
+  kw.secrets.variables = (mapListToAttrs
+    (field:
       nameValuePair "mautrix-telegram-${field}" {
         path = "secrets/mautrix-telegram";
         inherit field;
       }) [ "api-hash" "api-id" "as-token" "hs-token" ]
-    // { matrix-registration = {
+  // {
+    matrix-registration = {
       path = "secrets/matrix-registration";
-    }; });
+    };
+  });
 
   secrets.files.mautrix-telegram-env = {
     text = ''

config/services/syncplay/default.nix

@@ -4,9 +4,12 @@ with lib;
 
 {
 
-  kw.secrets.variables = let
+  kw.secrets.variables =
+    let
       fieldAdapt = field: if field == "pass" then "password" else field;
-  in mapListToAttrs (field:
+    in
+    mapListToAttrs
+      (field:
         nameValuePair "syncplay-${field}" {
           path = "services/media/syncplay";
           field = fieldAdapt field;

config/services/znc/default.nix

@@ -50,16 +50,21 @@ in
 {
   network.firewall.public.tcp.ports = singleton 5001;
 
-  kw.secrets.variables = let
+  kw.secrets.variables =
+    let
       fieldAdapt = field: if field == "cert" then "notes" else if field == "pass" then "password" else field;
-  in listToAttrs (concatMap (network:
-    map (field:
+    in
+    listToAttrs (concatMap
+      (network:
+        map
+          (field:
             nameValuePair "znc-${network}-${field}" {
               path = "social/irc/${network}";
               field = fieldAdapt field;
             }) [ "cert" "pass" ]
       ) [ "liberachat" "espernet" ]
-  ++ map (field:
+    ++ map
+      (field:
         nameValuePair "znc-softnet-${field}" {
           path = "social/irc/softnet";
           field = fieldAdapt field;

config/targets/rinnosuke-domains/default.nix

@@ -1,6 +1,7 @@
 { config, ... }:
 
-let rinnosuke = config.network.nodes.rinnosuke; in {
+let rinnosuke = config.network.nodes.rinnosuke; in
+{
   deploy.targets.rinnosuke-domains.tf = {
     dns.records = {
       node_public_rinnosuke_v4 = {

config/users/kat/personal/taskwarrior.nix

@@ -3,9 +3,12 @@
 with lib;
 
 {
-  kw.secrets.variables = let
+  kw.secrets.variables =
+    let
       fieldAdapt = field: if field == "key" then "notes" else field;
-  in mapListToAttrs (field:
+    in
+    mapListToAttrs
+      (field:
         nameValuePair "taskwarrior-${field}" {
           path = "services/taskwarrior";
           field = fieldAdapt field;

nix/sources.json

@@ -17,10 +17,10 @@
         "homepage": null,
         "owner": "arcnmx",
         "repo": "nixexprs",
-        "rev": "c294222dc0cfeb77fc1ee281e21121581b831e08",
-        "sha256": "06z5ql2sfwh4526pgwnx93firyaaa4l2xsqxivkm2diyhmr1jm65",
+        "rev": "5bf9fb9e07b6dfefd0134c57db667c7c75ed0119",
+        "sha256": "0pvz1swflmyyyzx5yzfflr1llsfw38pax03vac83pc8y0r78f84h",
         "type": "tarball",
-        "url": "https://github.com/arcnmx/nixexprs/archive/c294222dc0cfeb77fc1ee281e21121581b831e08.tar.gz",
+        "url": "https://github.com/arcnmx/nixexprs/archive/5bf9fb9e07b6dfefd0134c57db667c7c75ed0119.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "ci": {
@@ -41,10 +41,10 @@
         "homepage": "",
         "owner": "nix-community",
         "repo": "emacs-overlay",
-        "rev": "9f17ba1ed9c8b2012c0a1a41edf3d02936aa4def",
-        "sha256": "1qsgzswm04c7wnkk7d0ik7dahgxyxm1c0kpv8yhjk9g1gikwnxbz",
+        "rev": "3475b81d1c14509c9ebfae1e53366b28acaa2b6b",
+        "sha256": "0amksnxwk86pbb8w2zab7sycijyn0d3636877dfhdbdqkzck233g",
         "type": "tarball",
-        "url": "https://github.com/nix-community/emacs-overlay/archive/9f17ba1ed9c8b2012c0a1a41edf3d02936aa4def.tar.gz",
+        "url": "https://github.com/nix-community/emacs-overlay/archive/3475b81d1c14509c9ebfae1e53366b28acaa2b6b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "hexchen": {
@@ -159,10 +159,10 @@
         "homepage": "https://nur.nix-community.org/",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "45f8470f72f1c8989ae5fe23c6cc343ae3463a97",
-        "sha256": "0py76bjfiqncvxvald35gjgvhlg304d1yhmk0qvc22ar8k3dqckm",
+        "rev": "d2cdb44f12b82d9d207b6e6596cc9d2a2e1361eb",
+        "sha256": "01gs4vvhq4i71l9q7xlb3py4kadq2s7acdndkvdk2lk3l8d13qm1",
         "type": "tarball",
-        "url": "https://github.com/nix-community/NUR/archive/45f8470f72f1c8989ae5fe23c6cc343ae3463a97.tar.gz",
+        "url": "https://github.com/nix-community/NUR/archive/d2cdb44f12b82d9d207b6e6596cc9d2a2e1361eb.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "tf-nix": {
@@ -171,10 +171,10 @@
         "homepage": null,
         "owner": "arcnmx",
         "repo": "tf-nix",
-        "rev": "509bdbb0768e033fc1d8b7a4bc7b2ec8e141806a",
-        "sha256": "1h48lh55j8jpknl4zica6b6q402r5nj816sz1k7dqy607y0pfcrb",
+        "rev": "d57b4335aa35781420b7d064d3d77141004c44e9",
+        "sha256": "0xfl305yaz6xlgaz2jxp4qpibqsyh2hjgibvyxbqqcx0frs9bvya",
         "type": "tarball",
-        "url": "https://github.com/arcnmx/tf-nix/archive/509bdbb0768e033fc1d8b7a4bc7b2ec8e141806a.tar.gz",
+        "url": "https://github.com/arcnmx/tf-nix/archive/d57b4335aa35781420b7d064d3d77141004c44e9.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

overlays/exprs

@@ -1 +1 @@
-Subproject commit 68a371ce344cd66e646d58dd3658fc7c0615f768
+Subproject commit a78026175487423769ac2667e618a896af63c5b0

shell.nix

@@ -70,6 +70,7 @@ pkgs.mkShell {
   shellHook = ''
     export HOME_HOSTNAME=$(hostname -s)
     export HOME_UID=$(id -u)
+    export CI_PLATFORM="impure"
     export NIX_PATH="$NIX_PATH:home=${toString ./.}"
   '';
 }