feat: monitoring, add logistics node

.github/workflows/nodes.yml

@@ -385,6 +385,57 @@ jobs:
         command: ci-build-cache
         quiet: false
         stdin: ${{ runner.temp }}/ci.build.cache
+  logistics:
+    name: nodes-logistics
+    runs-on: ubuntu-latest
+    steps:
+    - id: checkout
+      name: git clone
+      uses: actions/checkout@v4
+      with:
+        submodules: false
+    - id: nix-install
+      name: nix install
+      uses: arcnmx/ci/actions/nix/install@v0.7
+    - id: ci-dirty
+      name: nix test dirty
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.logistics.run.test
+        command: ci-build-dirty
+        quiet: false
+        stdout: ${{ runner.temp }}/ci.build.dirty
+    - id: ci-test
+      name: nix test build
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.logistics.run.test
+        command: ci-build-realise
+        ignore-exit-code: true
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.dirty
+    - env:
+        CI_EXIT_CODE: ${{ steps.ci-test.outputs.exit-code }}
+      id: ci-summary
+      name: nix test results
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.logistics.run.test
+        command: ci-build-summarise
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.dirty
+        stdout: ${{ runner.temp }}/ci.build.cache
+    - env:
+        CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }}
+      id: ci-cache
+      if: always()
+      name: nix test cache
+      uses: arcnmx/ci/actions/nix/run@v0.7
+      with:
+        attrs: ci.job.logistics.run.test
+        command: ci-build-cache
+        quiet: false
+        stdin: ${{ runner.temp }}/ci.build.cache
   mediabox:
     name: nodes-mediabox
     runs-on: ubuntu-latest

modules/nixos/monitoring/ingest/grafana.nix

@@ -0,0 +1,11 @@
+_: {
+    services.grafana = {
+        #enable = true;
+        settings.server = {
+            domain = "gensokyo.zone";
+            http_port = 9092;
+            http_addr = "0.0.0.0";
+            root_url = "https://mon.gensokyo.zone";
+        };
+    };
+}

modules/nixos/monitoring/ingest/loki.nix

@@ -0,0 +1,61 @@
+{ pkgs, config, lib, access, gensokyo-zone, ... }: let
+  inherit (gensokyo-zone) systems;
+  inherit (lib.attrsets) filterAttrs mapAttrsToList attrNames;
+  promtailSystems = filterAttrs (_: system:
+    system.config.exporters.promtail.enable or false
+  ) systems;
+    inherit (builtins) toJSON;
+    inherit (lib.options) mkOption;
+    inherit (lib.types) port;
+    cfg = config.services.loki;
+in {
+    options.services.loki.settings = {
+        httpListenPort = mkOption {
+            type = port;
+            description = "Port to listen on over HTTP";
+            default = 9093;
+        };
+        grpcListenPort = mkOption {
+            type = port;
+            description = "Port to listen on over gRPC";
+            default = 0;
+        };
+   };
+    config = {
+        services.loki = {
+            #enable = true;
+            configFile = pkgs.writeTextFile {
+                name = "config.yaml";
+                executable = false;
+                text = (toJSON {
+                    server = {
+                        http_listen_port = cfg.settings.httpListenPort;
+                        grpc_listen_port = cfg.settings.grpcListenPort;
+                    };
+                    positions = {
+                        filename = "/tmp/positions.yaml";
+                    };
+                    clients = mapAttrsToList (system: systemConfig: {
+                            url = "${access.getAddressFor system.config.name "local"}:${system.config.exporters.promtail.port}";
+                    }) promtailSystems;
+                    scrape_configs = mapAttrsToList (system: systemConfig: {
+                        job_name = "${system.config.name}-journal";
+                        journal = {
+                            max_age = "${24*7}h";
+                            labels = {
+                                job = "systemd-journal";
+                                host = system.config.name;
+                            };
+                        };
+                        relabel_configs = [
+                            {
+                                source_labels = [ "__journal__systemd_unit" ];
+                                target_label = "unit";
+                            }
+                        ];
+                   }) promtailSystems;
+                });
+            };
+        };
+    };
+}

modules/nixos/monitoring/ingest/prometheus.nix

@@ -0,0 +1,20 @@
+{ access, lib, gensokyo-zone, ... }: let
+  inherit (gensokyo-zone) systems;
+  inherit (lib.attrsets) filterAttrs mapAttrsToList attrNames;
+  nodeExporterSystems = filterAttrs (_: system:
+    system.config.exporters.prometheus-exporters-node.enable or false
+  ) systems;
+ in {
+    services.prometheus = {
+        #enable = true;
+        port = 9090;
+        scrapeConfigs = mapAttrsToList (system: systemConfig: {
+                job_name = "${system.config.name}-node-exporter";
+                static_configs = {
+                     targets = [
+                        "${access.getAddressFor system.config.name "local"}:${system.config.exporters.prometheus-exporters-node.port}"
+                     ];
+                };
+           }) nodeExporterSystems;
+    };
+}

modules/nixos/monitoring/source/prometheus.nix

@@ -0,0 +1,53 @@
+{ config, lib, ... }: let
+    inherit (lib.modules) mkIf mkMerge;
+ in {
+    config = {
+        services.prometheus.exporters = {
+            node = mkMerge [
+                {
+                    #enable = true;
+                    port = 9091;
+                    enabledCollectors = [
+                        "nfs"
+                    ];
+                }
+               (mkIf config.services.nfs.server.enable {
+                    enabledCollectors = [
+                        "nfsd"
+                    ];
+               })
+               (mkIf (!config.boot.isContainer) {
+                    enabledCollectors = [
+                        "nvme"
+                        "hwmon"
+                    ];
+               })
+               {
+                    enabledCollectors = [
+                        "arp"
+                        "boottime"
+                        "cpu"
+                        "cpufreq"
+                        "diskstats"
+                        "dmi"
+                        "entropy"
+                        "filesystem"
+                        "netdev"
+                        "systemd"
+                        "sysctl"
+                        "systemd"
+                        "loadavg"
+                        "meminfo"
+                        "netstat"
+                        "os"
+                        "stat"
+                        "time"
+                        "uname"
+                        "vmstat"
+                        "zfs"
+                       ];
+                }
+            ];
+        };
+    };
+}

modules/nixos/monitoring/source/promtail.nix

@@ -0,0 +1,19 @@
+{ config, lib, ... }: let
+    inherit (builtins) toJSON;
+    inherit (lib.options) mkOption;
+    inherit (lib.types) port;
+    cfg = config.services.promtail;
+in {
+    options.services.promtail.settings = {
+        httpListenPort = mkOption {
+            type = port;
+            description = "Port to listen on over HTTP";
+            default = 9094;
+        };
+    };
+    config.services.promtail = {
+        extraFlags = [
+            "--server.http-listen-port=${cfg.settings.httpListenPort}"
+        ];
+    };
+}

modules/system/exports/monitoring.nix

@@ -0,0 +1,87 @@
+{lib, gensokyo-zone, ...}: let
+  inherit (gensokyo-zone.lib) mapAlmostOptionDefaults mkAlmostOptionDefault;
+  inherit (lib.modules) mkIf;
+in {
+   config.exports.services = {
+          prometheus = { config, ... }: {
+            id = mkAlmostOptionDefault "prometheus";
+            nixos = {
+              serviceAttr = "prometheus";
+              assertions = mkIf config.enable [
+                (nixosConfig: {
+                  assertion = config.ports.default.port == nixosConfig.services.prometheus.port;
+                  message = "port mismatch";
+                })
+              ];
+            };
+            ports.default = mapAlmostOptionDefaults {
+              port = 9090;
+              protocol = "http";
+            };
+          };
+          prometheus-exporters-node = { config, ... }: {
+            id = mkAlmostOptionDefault "prometheus-exporters-node";
+            nixos = {
+              serviceAttrPath = [ "services" "prometheus" "exporters" "node" ];
+              assertions = mkIf config.enable [
+                (nixosConfig: {
+                  assertion = config.ports.default.port == nixosConfig.services.prometheus.exporters.node.port;
+                  message = "port mismatch";
+                })
+              ];
+            };
+            ports.default = mapAlmostOptionDefaults {
+              port = 9091;
+              protocol = "http";
+            };
+          };
+          grafana = { config, ... }: {
+            id = mkAlmostOptionDefault "grafana";
+            nixos = {
+              serviceAttr = "grafana";
+              assertions = mkIf config.enable [
+                (nixosConfig: {
+                  assertion = config.ports.default.port == nixosConfig.services.grafana.settings.server.http_port;
+                  message = "port mismatch";
+                })
+              ];
+            };
+            ports.default = mapAlmostOptionDefaults {
+              port = 9092;
+              protocol = "http";
+            };
+          };
+          loki = { config, ... }: {
+            id = mkAlmostOptionDefault "loki";
+            nixos = {
+              serviceAttr = "loki";
+              assertions = mkIf config.enable [
+                (nixosConfig: {
+                  assertion = config.ports.default.port == nixosConfig.services.loki.settings.httpListenPort;
+                  message = "port mismatch";
+                })
+              ];
+            };
+            ports.default = mapAlmostOptionDefaults {
+              port = 9093;
+              protocol = "http";
+            };
+          };
+          promtail = { config, ... }: {
+            id = mkAlmostOptionDefault "promtail";
+            nixos = {
+              serviceAttr = "promtail";
+              assertions = mkIf config.enable [
+                (nixosConfig: {
+                  assertion = config.ports.default.port == nixosConfig.services.promtail.settings.httpListenPort;
+                  message = "port mismatch";
+                })
+              ];
+            };
+            ports.default = mapAlmostOptionDefaults {
+              port = 9094;
+              protocol = "http";
+            };
+          };
+      };
+     }

systems/logistics/default.nix

@@ -0,0 +1,13 @@
+_: {
+  arch = "x86_64";
+  type = "NixOS";
+  modules = [
+    ./nixos.nix
+  ];
+  deploy.hostname = "10.1.1.63";
+  exports = {
+    services = {
+      sshd.enable = true;
+    };
+  };
+}

systems/logistics/hardware-configuration.nix

@@ -0,0 +1,39 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/nixos.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "nvme" "usb_storage" "sd_mod" "rtsx_pci_sdmmc" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/3331f9a0-6b86-411c-8574-63de28046cf2";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/8DC2-0DAE";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp0s31f6.useDHCP = lib.mkDefault true;
+  # networking.interfaces.wlp3s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

systems/logistics/nixos.nix

@@ -0,0 +1,119 @@
+# Edit this configuration file to define what should be installed on
+# your system.  Help is available in the nixos.nix(5) man page
+# and in the NixOS manual (accessible by running ‘nixos-help’).
+
+{ config, pkgs, ... }:
+
+{
+  imports =
+    [ # Include the results of the hardware scan.
+      ./hardware-configuration.nix
+    ];
+
+  # Bootloader.
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+
+  networking.hostName = "nixos"; # Define your hostname.
+  # networking.wireless.enable = true;  # Enables wireless support via wpa_supplicant.
+
+  # Configure network proxy if necessary
+  # networking.proxy.default = "http://user:password@proxy:port/";
+  # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";
+
+  # Enable networking
+  networking.networkmanager.enable = true;
+
+  # Set your time zone.
+  time.timeZone = "America/Vancouver";
+
+  # Select internationalisation properties.
+  i18n.defaultLocale = "en_CA.UTF-8";
+
+  # Enable the X11 windowing system.
+  services.xserver.enable = true;
+
+  # Enable the XFCE Desktop Environment.
+  services.xserver.displayManager.lightdm.enable = true;
+  services.xserver.desktopManager.xfce.enable = true;
+
+  # Configure keymap in X11
+  services.xserver = {
+    layout = "us";
+    xkbVariant = "";
+  };
+
+  # Enable CUPS to print documents.
+  services.printing.enable = true;
+
+  # Enable sound with pipewire.
+  sound.enable = true;
+  hardware.pulseaudio.enable = false;
+  security.rtkit.enable = true;
+  services.pipewire = {
+    enable = true;
+    alsa.enable = true;
+    alsa.support32Bit = true;
+    pulse.enable = true;
+    # If you want to use JACK applications, uncomment this
+    #jack.enable = true;
+
+    # use the example session manager (no others are packaged yet so this is enabled by default,
+    # no need to redefine it in your config for now)
+    #media-session.enable = true;
+  };
+
+  # Enable touchpad support (enabled default in most desktopManager).
+  # services.xserver.libinput.enable = true;
+
+  # Define a user account. Don't forget to set a password with ‘passwd’.
+  users.users.logistics = {
+    isNormalUser = true;
+    description = "Logistics";
+    extraGroups = [ "networkmanager" "wheel" ];
+    packages = with pkgs; [
+    #  thunderbird
+    ];
+  };
+
+  # Install firefox.
+  programs.firefox.enable = true;
+
+  # Allow unfree packages
+  nixpkgs.config.allowUnfree = true;
+
+  # List packages installed in system profile. To search, run:
+  # $ nix search wget
+  environment.systemPackages = with pkgs; [
+  #  vim # Do not forget to add an editor to edit nixos.nix! The Nano editor is also installed by default.
+  #  wget
+  ];
+
+  # Some programs need SUID wrappers, can be configured further or are
+  # started in user sessions.
+  # programs.mtr.enable = true;
+  # programs.gnupg.agent = {
+  #   enable = true;
+  #   enableSSHSupport = true;
+  # };
+
+  # List services that you want to enable:
+
+  # Enable the OpenSSH daemon.
+  services.openssh.enable = true;
+
+  # Open ports in the firewall.
+  # networking.firewall.allowedTCPPorts = [ ... ];
+  # networking.firewall.allowedUDPPorts = [ ... ];
+  # Or disable the firewall altogether.
+  # networking.firewall.enable = false;
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man nixos.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "23.11"; # Did you read the comment?
+
+}

tree.nix

@@ -56,6 +56,9 @@
     };
     "modules/nixos/access".functor.enable = true;
     "modules/nixos/ldap".functor.enable = true;
+    "modules/nixos/monitoring".functor.enable = true;
+    "modules/nixos/monitoring/ingest".functor.enable = true;
+    "modules/nixos/monitoring/source".functor.enable = true;
     "modules/nixos/krb5".functor.enable = true;
     "modules/nixos/sssd".functor.enable = true;
     "modules/nixos/network".functor.enable = true;