From af760a543a564d7a1847d00e1be74227dc7100b3 Mon Sep 17 00:00:00 2001
From: arcnmx <git@git.arcn.mx>
Date: Sun, 23 Jun 2024 19:09:52 -0700
Subject: [PATCH] feat(monitoring): loki nginx access.log source

---
 modules/nixos/monitoring/ingest/loki.nix     | 10 +++
 modules/nixos/monitoring/source/promtail.nix | 91 ++++++++++++++++++--
 2 files changed, 94 insertions(+), 7 deletions(-)

diff --git a/modules/nixos/monitoring/ingest/loki.nix b/modules/nixos/monitoring/ingest/loki.nix
index 9112ece5..7f7c1dd2 100644
--- a/modules/nixos/monitoring/ingest/loki.nix
+++ b/modules/nixos/monitoring/ingest/loki.nix
@@ -6,6 +6,7 @@
   ...
 }: let
   inherit (gensokyo-zone) systems;
+  inherit (gensokyo-zone.lib) mapOptionDefaults;
   inherit (lib.modules) mkIf mkOptionDefault;
   inherit (lib.attrsets) filterAttrs mapAttrsToList;
   promtailSystems =
@@ -24,6 +25,15 @@ in {
           http_listen_port = mkOptionDefault 9093;
           grpc_listen_port = mkOptionDefault 0;
         };
+        limits_config = mapOptionDefaults {
+          ingestion_rate_mb = 32;
+          ingestion_burst_size_mb = 64;
+          max_label_value_length = 8192;
+          max_label_names_per_series = 32;
+          max_line_size = "512KB";
+          per_stream_rate_limit = "32MB";
+          per_stream_rate_limit_burst = "64MB";
+        };
         # https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#1-local-configuration-exampleyaml
         auth_enabled = mkOptionDefault false;
         common = {
diff --git a/modules/nixos/monitoring/source/promtail.nix b/modules/nixos/monitoring/source/promtail.nix
index 7c182683..fb7224f6 100644
--- a/modules/nixos/monitoring/source/promtail.nix
+++ b/modules/nixos/monitoring/source/promtail.nix
@@ -5,7 +5,9 @@
   lib,
   ...
 }: let
-  inherit (lib.modules) mkIf mkOptionDefault;
+  inherit (lib.modules) mkIf mkMerge mkOptionDefault;
+  inherit (lib.strings) concatStringsSep;
+  inherit (config.services) nginx;
   cfg = config.services.promtail;
 in {
   config.services.promtail = {
@@ -21,27 +23,102 @@ in {
           url = "${baseUrl}/loki/api/v1/push";
         }
       ];
-      scrape_configs = [
+      scrape_configs = let
+        labels = {
+          system = systemConfig.name;
+          host = config.networking.fqdn;
+        };
+      in [
         {
           job_name = "${systemConfig.name}-journald";
           journal = {
+            json = true;
             max_age = "${toString (24 * 7)}h";
-            labels = {
-              job = "systemd-journald";
-              system = systemConfig.name;
-              host = config.networking.fqdn;
-            };
+            labels = mkMerge [
+              {
+                job = "systemd-journald";
+              }
+              labels
+            ];
           };
           relabel_configs = [
             {
               source_labels = ["__journal__systemd_unit"];
               target_label = "unit";
             }
+            {
+              source_labels = ["__journal_syslog_identifier"];
+              target_label = "syslog_identifier";
+            }
           ];
         }
+        (mkIf nginx.enable {
+          job_name = "${systemConfig.name}-nginx-access";
+          static_configs = [
+            {
+              labels = mkMerge [
+                {
+                  job = "nginx-access";
+                  __path__ = "${nginx.accessLog.path}";
+                }
+                labels
+              ];
+            }
+          ];
+          # see https://grafana.com/docs/loki/latest/send-data/promtail/pipelines/
+          # and https://grafana.com/docs/loki/latest/send-data/promtail/stages/
+          pipeline_stages = [
+            {
+              match = {
+                selector = ''{job="nginx-access"}'';
+                pipeline_name = "access";
+                stages = [
+                  {
+                    regex.expression = concatStringsSep " " [
+                      ''(?P<remote_addr>.*?)''
+                      ''(?P<remote_log_name>.*?)''
+                      ''(?P<userid>.*?)(@(?P<virtual_host>.*?))?''
+                      ''\[(?P<timestamp>.*?)\]''
+                      ''\"(?P<request_method>.*?) (?P<path>.*?)( (?P<request_version>HTTP/.*))?\"''
+                      ''(?P<status>.*?)''
+                      ''(?P<length>.*?)''
+                      ''\"(?P<referrer>.*?)\"''
+                      ''\"(?P<user_agent>.*?)\"''
+                    ];
+                  }
+                  {
+                    labels = {
+                      remote_addr = null;
+                      remote_log_name = null;
+                      userid = null;
+                      virtual_host = null;
+                      request_method = null;
+                      path = null;
+                      request_version = null;
+                      status = null;
+                      length = null;
+                      referrer = null;
+                      user_agent = null;
+                    };
+                  }
+                  {
+                    timestamp = {
+                      source = "timestamp";
+                      format = "2/Jan/2006:15:04:05 -0700";
+                    };
+                  }
+                ];
+              };
+            }
+          ];
+        })
       ];
     };
   };
+  config.systemd.services.promtail = mkIf cfg.enable {
+    # TODO: there must be a better way to provide promtail access to these logs!
+    serviceConfig.Group = mkIf nginx.enable (lib.mkForce nginx.group);
+  };
   config.networking.firewall.interfaces.lan = let
     inherit (cfg.configuration) server;
   in