diff --git a/hosts/athame/nixos/default.nix b/hosts/athame/nixos/default.nix index a4be8853..b69bd391 100644 --- a/hosts/athame/nixos/default.nix +++ b/hosts/athame/nixos/default.nix @@ -24,6 +24,11 @@ with lib; ../../../services/murmur.nix ../../../services/matrix.nix ../../../services/restic.nix + ../../../services/grafana.nix + ../../../services/prometheus.nix + ../../../services/loki.nix + ../../../services/node-exporter.nix + ../../../services/promtail.nix ]; boot.loader.grub.enable = true; diff --git a/hosts/samhain/nixos/default.nix b/hosts/samhain/nixos/default.nix index 9d661fd9..031532e8 100644 --- a/hosts/samhain/nixos/default.nix +++ b/hosts/samhain/nixos/default.nix @@ -15,6 +15,8 @@ in { ../../../services/zfs.nix ../../../services/restic.nix ../../../services/nginx.nix + ../../../services/node-exporter.nix + ../../../services/promtail.nix ./thermal ./transmission.nix ./jellyfin.nix diff --git a/hosts/yule/nixos/default.nix b/hosts/yule/nixos/default.nix index 4cd4de05..8f999265 100644 --- a/hosts/yule/nixos/default.nix +++ b/hosts/yule/nixos/default.nix @@ -1,10 +1,14 @@ -{ config, pkgs, profiles, ... }: +{ config, pkgs, lib, profiles, ... }: + +with lib; { imports = [ ./hw.nix ../../../services/zfs.nix ../../../services/restic.nix + ../../../services/node-exporter.nix + ../../../services/promtail.nix profiles.gui profiles.sway profiles.kat diff --git a/services/grafana.nix b/services/grafana.nix new file mode 100644 index 00000000..e5c7d367 --- /dev/null +++ b/services/grafana.nix @@ -0,0 +1,36 @@ +{ config, ... }: + +{ + services.postgresql = { + ensureDatabases = [ "grafana" ]; + ensureUsers = [{ + name = "grafana"; + ensurePermissions."DATABASE grafana" = "ALL PRIVILEGES"; + }]; + }; + + services.grafana = { + enable = true; + port = 3001; + domain = "graph.kittywit.ch"; + rootUrl = "https://graph.kittywit.ch/"; + database = { + type = "postgres"; + host = "/run/postgresql/"; + user = "grafana"; + name = "grafana"; + }; + }; + + services.nginx.virtualHosts."graph.kittywit.ch" = { + enableACME = true; + forceSSL = true; + locations = { "/".proxyPass = "http://127.0.0.1:3001"; }; + }; + + deploy.tf.dns.records.kittywitch_graph = { + tld = "kittywit.ch."; + domain = "graph"; + cname.target = "athame.kittywit.ch."; + }; +} diff --git a/services/loki.nix b/services/loki.nix new file mode 100644 index 00000000..8716814b --- /dev/null +++ b/services/loki.nix @@ -0,0 +1,62 @@ +{ config, pkgs, ... }: + +{ + katnet.private.tcp.ports = [ 3100 ]; + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + chunk_store_config = { max_look_back_period = "0s"; }; + ingester = { + chunk_idle_period = "1h"; + chunk_retain_period = "30s"; + chunk_target_size = 1048576; + lifecycler = { + address = "0.0.0.0"; + final_sleep = "0s"; + ring = { + kvstore = { store = "inmemory"; }; + replication_factor = 1; + }; + }; + max_chunk_age = "1h"; + max_transfer_retries = 0; + }; + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + }; + schema_config = { + configs = [{ + from = "2020-10-24"; + index = { + period = "24h"; + prefix = "index_"; + }; + object_store = "filesystem"; + schema = "v11"; + store = "boltdb-shipper"; + }]; + }; + compactor = { + working_directory = "/tmp/loki-compactor-boltdb"; + shared_store = "filesystem"; + }; + server = { http_listen_port = 3100; }; + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + shared_store = "filesystem"; + }; + filesystem = { directory = "/var/lib/loki/chunks"; }; + }; + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + }; + }; + +} diff --git a/services/node-exporter.nix b/services/node-exporter.nix new file mode 100644 index 00000000..701bbf48 --- /dev/null +++ b/services/node-exporter.nix @@ -0,0 +1,15 @@ +{ config, ... }: + +{ + katnet.private.tcp.ports = [ 9002 ]; + + services.prometheus = { + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9002; + }; + }; + }; +} diff --git a/services/prometheus.nix b/services/prometheus.nix new file mode 100644 index 00000000..e6414851 --- /dev/null +++ b/services/prometheus.nix @@ -0,0 +1,22 @@ +{ config, hosts, lib, ... }: + +with lib; + +let + prom_configs = + (mapAttrs (hostName: host: host.config.services.prometheus.exporters.node) + (filterAttrs + (_: host: host.config.services.prometheus.exporters.node.enable) + hosts)); +in { + services.prometheus = { + enable = true; + scrapeConfigs = mapAttrsToList (hostName: prom: { + job_name = hostName; + static_configs = [{ + targets = [ "${hostName}.net.kittywit.ch:${toString prom.port}" ]; + }]; + }) prom_configs; + }; +} + diff --git a/services/promtail.nix b/services/promtail.nix new file mode 100644 index 00000000..030a8c43 --- /dev/null +++ b/services/promtail.nix @@ -0,0 +1,40 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + promtail_config = pkgs.writeText "prom-config.json" (builtins.toJSON { + clients = + [{ url = "http://athame.net.kittywit.ch:3100/loki/api/v1/push"; }]; + positions = { filename = "/tmp/positions.yaml"; }; + scrape_configs = [{ + job_name = "journal"; + journal = { + labels = { + host = config.networking.hostName; + job = "systemd-journal"; + }; + max_age = "12h"; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + server = { + grpc_listen_port = 0; + http_listen_port = 28183; + }; + }); +in { + systemd.services.promtail = { + description = "Promtail service for Loki"; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + ExecStart = '' + ${pkgs.grafana-loki}/bin/promtail --config.file ${promtail_config} + ''; + }; + }; +}