project-wide: A move to targets, getting closer to production

default.nix

@@ -7,12 +7,15 @@ rec {
     defaultFile = "nixos.nix";
   };
 
-  hosts = import ./lib/hosts.nix { inherit pkgs sources witch profiles; inherit (deploy) tf;  };
+  hosts = import ./lib/hosts.nix {
+    inherit pkgs sources witch profiles;
+    inherit (deploy) tf;
+  };
 
   inherit (pkgs) lib;
 
   deploy = import ./lib/deploy.nix {
     inherit pkgs sources;
-    inherit (hosts) hosts groups;
+    inherit (hosts) hosts targets;
   };
 }

hosts/athame/meta.nix

@@ -1,37 +1,85 @@
-{ config, hosts, lib, ... }: {
-  config = {
-    resources.athame = {
-      provider = "null";
-      type = "resource";
-      connection = {
-        port = 62954;
-        host = "athame.kittywit.ch";
-      };
+{ config, hosts, lib, ... }:
+with config.resources; {
+  resources.athame = {
+    provider = "hcloud";
+    type = "server";
+    inputs = {
+      name = "athame";
+      image = "ubuntu-20.04";
+      server_type = "cpx21";
+      location = "nbg1";
+      backups = false;
+      ssh_keys = [ (hcloud_ssh_key.refAttr "id") ];
     };
-
-    dns.records.kittywitch_athame = {
-      tld = "kittywit.ch.";
-      domain = "athame";
-      a.address = "168.119.126.111";
-    };
-
-    dns.records.kittywitch_root = {
-      tld = "kittywit.ch.";
-      domain = "@";
-      a.address = "168.119.126.111";
-    };
-
-    deploy.systems.athame = with config.resources; {
-      nixosConfig = hosts.athame.config;
-      connection = athame.connection.set;
-      triggers.copy.athame = athame.refAttr "id";
-      triggers.secrets.athame = athame.refAttr "id";
-      #triggers.switch = lib.mapAttrs (name: record: { 
-      #A = config.lib.tf.terraformExpr ''join(",", ${record.out.resource.namedRef}.addresses)'';
-      #AAAA = config.lib.tf.terraformExpr ''join(",", ${record.out.resource.namedRef}.addresses)'';
-      #CNAME = record.out.resource.refAttr "cname";
-      #SRV = record.out.resource.refAttr "id";
-      #}.${record.out.type}) config.dns.records;
+    connection = { host = config.lib.tf.terraformSelf "ipv4_address"; };
+    provisioners = [
+      {
+        file = {
+          destination = "/tmp/sshportfix.nix";
+          content = "{ config, ...}: { services.openssh.ports = [ 62954 ]; }";
+        };
+      }
+      {
+        remote-exec.command =
+          "curl https://raw.githubusercontent.com/elitak/nixos-infect/master/nixos-infect | NIXOS_IMPORT=/tmp/sshportfix.nix NO_REBOOT=true PROVIDER=hetznercloud NIX_CHANNEL=nixos-20.09 bash 2>&1 | tee /tmp/infect.log";
+      }
+      {
+        remote-exec.command = "reboot";
+        onFailure = "continue";
+      }
+    ];
   };
+
+  resources.athame_rdns = {
+    provider = "hcloud";
+    type = "rdns";
+    inputs = {
+      server_id = athame.refAttr "id";
+      dns_ptr = "athame.kittywit.ch";
+      ip_address = athame.refAttr "ipv4_address";
+    };
+  };
+
+  dns.records.kittywitch_athame = {
+    tld = "kittywit.ch.";
+    domain = "athame";
+    a.address = athame.refAttr "ipv4_address";
+  };
+
+  dns.records.kittywitch_root = {
+    tld = "kittywit.ch.";
+    domain = "@";
+    a.address = athame.refAttr "ipv4_address";
+  };
+
+  dns.records.kittywitch_athame_v6 = {
+    tld = "kittywit.ch.";
+    domain = "athame";
+    aaaa.address = athame.refAttr "ipv6_address";
+  };
+
+  dns.records.kittywitch_root_v6 = {
+    tld = "kittywit.ch.";
+    domain = "@";
+    aaaa.address = athame.refAttr "ipv6_address";
+  };
+
+  deploy.systems.athame = {
+    nixosConfig = hosts.athame.config;
+    connection = {
+      host = athame.refAttr "ipv4_address";
+      port = 62954;
+    };
+    triggers.copy.athame = athame.refAttr "id";
+    triggers.secrets.athame = athame.refAttr "id";
+    triggers.switch = lib.mapAttrs (name: record:
+      {
+        A = config.lib.tf.terraformExpr
+          ''join(",", ${record.out.resource.namedRef}.addresses)'';
+        AAAA = config.lib.tf.terraformExpr
+          ''join(",", ${record.out.resource.namedRef}.addresses)'';
+        CNAME = record.out.resource.refAttr "cname";
+        SRV = record.out.resource.refAttr "id";
+      }.${record.out.type}) config.dns.records;
   };
 }

hosts/athame/nixos/default.nix

@@ -6,25 +6,27 @@
     # profiles
     profiles.kat
     # host-specific services 
-    ./postgres.nix
-    ./virtualhosts.nix
-    ./fail2ban.nix
+    #./postgres.nix
+    #./virtualhosts.nix
+    #./fail2ban.nix
     # services 
-    ../../../services/nginx.nix
-    ../../../services/mail.nix
-    ../../../services/asterisk.nix
-    ../../../services/gitea
-    ../../../services/syncplay.nix
-    ../../../services/bitwarden.nix
-    ../../../services/taskserver.nix
-    ../../../services/murmur.nix
-    ../../../services/znc.nix
-    ../../../services/matrix.nix
+    #../../../services/nginx.nix
+    #../../../services/mail.nix
+    #../../../services/asterisk.nix
+    #../../../services/gitea
+    #../../../services/syncplay.nix
+    #../../../services/bitwarden.nix
+    #../../../services/taskserver.nix
+    #../../../services/murmur.nix
+    #../../../services/znc.nix
+    #../../../services/matrix.nix
   ];
 
   boot.loader.grub.enable = true;
   boot.loader.grub.version = 2;
 
+  deploy.target = "infra";
+
   networking = {
     hostName = "athame";
     domain = "kittywit.ch";

hosts/samhain/meta.nix

@@ -11,8 +11,8 @@
     deploy.systems.samhain = with config.resources; {
       nixosConfig = hosts.samhain.config;
       connection = samhain.connection.set;
-      triggers.copy.samhain = athame.refAttr "id";
-      triggers.secrets.samhain = athame.refAttr "id";
+      triggers.copy.samhain = samhain.refAttr "id";
+      triggers.secrets.samhain = samhain.refAttr "id";
     };
   };
 }

hosts/samhain/nixos/default.nix

@@ -13,7 +13,7 @@
     ./torrenting.nix
   ];
 
-  deploy.groups = [ "gui" ];
+  deploy.target = "personal";
 
   # graphics tablet
   services.xserver.wacom.enable = true;

hosts/yule/meta.nix

@@ -11,8 +11,8 @@
     deploy.systems.yule = with config.resources; {
       nixosConfig = hosts.yule.config;
       connection = yule.connection.set;
-      triggers.copy.yule = athame.refAttr "id";
-      triggers.secrets.yule = athame.refAttr "id";
+      triggers.copy.yule = yule.refAttr "id";
+      triggers.secrets.yule = yule.refAttr "id";
     };
   };
 }

hosts/yule/nixos/default.nix

@@ -10,7 +10,7 @@
     profiles.laptop
   ];
 
-  deploy.groups = [ "gui" ];
+  deploy.target = "personal";
 
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;

lib/deploy.nix

@@ -1,4 +1,4 @@
-{ pkgs, hosts, sources, groups }:
+{ pkgs, hosts, sources, targets }:
 
 with pkgs.lib;
 
@@ -11,41 +11,38 @@ let
       specialArgs = { inherit hosts; };
     }).config;
 
-  tf = makeOverridable ({ group ? null, host ? null }:
+  tf = { targetName, target }:
     tfEval ({ config, ... }: {
-      imports =
-        mapAttrsToList (name: host: import (../hosts + "/${name}/meta.nix"))
-        hosts ++ [{ config = mkMerge (mapAttrsToList (_: host: mapAttrs (_: mkMerge) host.config.deploy.tf.out.set) hosts); }];
+      imports = map (hostName: ../hosts + "/${hostName}/meta.nix") target ++ [{
+        config = mkMerge (map (hostName:
+          mapAttrs (_: mkMerge) hosts.${hostName}.config.deploy.tf.out.set)
+          target);
+      }] ++ concatMap (hostName:
+        filter builtins.pathExists
+        (map (profile: ../profiles + "/${profile}/meta.nix") (attrNames
+          (filterAttrs (_: id) hosts.${hostName}.config.deploy.profile))))
+        target;
 
-        deps = {
+      deps = {
         select.allProviders = true;
         enable = true;
-        select.hclPaths =
-          (map (name: config.resources."${name}_system_switch".out.hclPathStr)
-            (if host != null then
-              [ host ]
-            else
-              (if group != null then groups.${group} else [ ])));
       };
 
-      state = { file = toString ../private/files/tf/terraform.tfstate; };
+      state = {
+        file = ../private/files/tf + "/terraform-${targetName}.tfstate";
+      };
 
       runners.lazy = {
         file = ../.;
         args = [ "--show-trace" ];
-        attrPrefix = let
-          attr = if host != null then
-            "host.${host}"
-          else if group != null then
-            "group.${group}"
-          else
-            "tf";
-        in "deploy.${attr}.runners.run.";
+        attrPrefix =
+          let attr = if target != null then "target.${targetName}" else "tf";
+          in "deploy.${attr}.runners.run.";
       };
 
       terraform = {
-        dataDir = toString ../private/files/tf/tfdata;
-        logPath = toString ../private/files/tf/terraform.log;
+        dataDir = ../private/files/tf + "/tfdata/${targetName}";
+        logPath = ../private/files/tf + "/terraform-${targetName}.log";
       };
 
       variables.hcloud_token = {
@@ -87,9 +84,9 @@ let
           };
         };
       };
-    })) { };
+    });
 in {
   inherit tf;
-  group = genAttrs (attrNames groups) (group: (tf.override { inherit group; }));
-  host = genAttrs (attrNames hosts) (host: (tf.override { inherit host; }));
+  target =
+    mapAttrs (targetName: target: tf { inherit target targetName; }) targets;
 }

lib/hosts.nix

@@ -9,7 +9,7 @@ rec {
 
   hostConfig = hostName:
     { config, ... }: {
-      _module.args = { inherit hosts groups; };
+      _module.args = { inherit hosts targets; };
       imports = [ ../nixos.nix ../modules/nixos ];
       networking = { inherit hostName; };
       nixpkgs.pkgs = import pkgsPath {
@@ -30,11 +30,6 @@ rec {
       specialArgs = { inherit sources tf profiles witch hostName; };
     })) hostNames);
 
-  groupNames = unique (concatLists
-    (mapAttrsToList (name: host: host.config.deploy.groups) hosts));
-
-  groups = listToAttrs (map (groupName:
-    nameValuePair groupName (attrNames
-      (filterAttrs (name: host: elem groupName host.config.deploy.groups)
-        hosts))) groupNames);
+  targets = foldAttrs (host: hosts: [ host ] ++ hosts) [ ] (mapAttrsToList
+    (hostName: host: { ${host.config.deploy.target} = hostName; }) hosts);
 }

modules/nixos/deploy/default.nix

@@ -5,12 +5,10 @@ with lib;
 {
   options = {
     deploy = {
-      groups = mkOption {
-        type = with types; listOf str;
-        default = [ ];
+      target = mkOption {
+        type = with types; str;
+        default = "";
       };
     };
   };
-
-  config = { deploy.groups = [ "all" ]; };
 }

modules/nixos/tf-glue/default.nix

@@ -1,4 +1,6 @@
-{ config, lib, ... }: with lib; let
+{ config, lib, ... }:
+with lib;
+let
   cfg = config.deploy.tf;
   unmergedValues = types.mkOptionType {
     name = "unmergedValues";
@@ -14,9 +16,7 @@ in {
           type = types.listOf types.str;
           default = [ ];
         };
-        out.set = mkOption {
-          type = types.unspecified;
-        };
+        out.set = mkOption { type = types.unspecified; };
       };
     };
   };
@@ -25,6 +25,6 @@ in {
     deploy.tf = {
       attrs = [ "out" "attrs" ];
       out.set = removeAttrs cfg cfg.attrs;
-    }; 
+    };
   };
 }

services/bitwarden.nix

@@ -5,9 +5,7 @@
     ensureDatabases = [ "bitwarden_rs" ];
     ensureUsers = [{
       name = "bitwarden_rs";
-      ensurePermissions = {
-        "DATABASE bitwarden_rs" = "ALL PRIVILEGES";
-      };
+      ensurePermissions = { "DATABASE bitwarden_rs" = "ALL PRIVILEGES"; };
     }];
   };
 
@@ -32,7 +30,7 @@
       "/notifications/hub".proxyPass = "http://127.0.0.1:3012";
       "/notifications/hub/negotiate".proxyPass = "http://127.0.0.1:80";
     };
-  }; 
+  };
 
   deploy.tf.dns.records.kittywitch_vault = {
     tld = "kittywit.ch.";

services/gitea/default.nix

@@ -4,11 +4,10 @@
   services.postgresql = {
     enable = true;
     ensureDatabases = [ "gitea" ];
-    ensureUsers = [
-      { name = "gitea";
-        ensurePermissions."DATABASE gitea" = "ALL PRIVILEGES";
-      }
-    ];
+    ensureUsers = [{
+      name = "gitea";
+      ensurePermissions."DATABASE gitea" = "ALL PRIVILEGES";
+    }];
   };
 
   services.gitea = {

services/murmur.nix

@@ -14,7 +14,7 @@
       sslCert=/var/lib/acme/kittywit.ch/fullchain.pem
       sslKey=/var/lib/acme/kittywit.ch/key.pem
     '';
-  };  
+  };
 
   deploy.tf.dns.records.kittywitch_voice = {
     tld = "kittywit.ch.";

services/syncserver.nix

@@ -12,7 +12,7 @@
     enableACME = true;
     forceSSL = true;
     locations = { "/".proxyPass = "http://127.0.0.1:5001"; };
-  };  
+  };
 
   deploy.tf.dns.records.kittywitch_sync = {
     tld = "kittywit.ch.";

services/weechat.nix

@@ -35,7 +35,7 @@ in {
         proxyWebsockets = true;
       };
     };
-  };  
+  };
 
   deploy.tf.dns.records.kittywitch_irc = {
     tld = "kittywit.ch.";