From e6a890915f1a4fcad7e8f62c2a43fd1b2a13d382 Mon Sep 17 00:00:00 2001
From: arcnmx <git@git.arcn.mx>
Date: Fri, 20 Sep 2024 16:26:29 -0700
Subject: [PATCH] fix(loki): ingest

---
 modules/nixos/monitoring/ingest/loki.nix     | 26 +++++++++++++++-----
 modules/nixos/monitoring/source/promtail.nix |  9 -------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/modules/nixos/monitoring/ingest/loki.nix b/modules/nixos/monitoring/ingest/loki.nix
index 7f7c1dd2..81d6fc7a 100644
--- a/modules/nixos/monitoring/ingest/loki.nix
+++ b/modules/nixos/monitoring/ingest/loki.nix
@@ -26,13 +26,17 @@ in {
           grpc_listen_port = mkOptionDefault 0;
         };
         limits_config = mapOptionDefaults {
-          ingestion_rate_mb = 32;
-          ingestion_burst_size_mb = 64;
-          max_label_value_length = 8192;
-          max_label_names_per_series = 32;
+          ingestion_rate_mb = 256;
+          ingestion_burst_size_mb = 512;
+          max_label_value_length = 8192 * 4;
+          max_label_names_per_series = 128;
+          max_entries_limit_per_query = 1000000;
+          #cardinality_limit: 200000
           max_line_size = "512KB";
-          per_stream_rate_limit = "32MB";
-          per_stream_rate_limit_burst = "64MB";
+          per_stream_rate_limit = "128MB";
+          per_stream_rate_limit_burst = "256MB";
+          reject_old_samples = true;
+          reject_old_samples_max_age = "${toString (24 * 9)}h";
         };
         # https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#1-local-configuration-exampleyaml
         auth_enabled = mkOptionDefault false;
@@ -59,5 +63,15 @@ in {
         storage_config.filesystem.directory = mkOptionDefault "${cfg.dataDir}/chunks";
       };
     };
+    networking.firewall.interfaces.local = let
+      inherit (cfg.configuration) server;
+    in
+      mkIf cfg.enable {
+        allowedTCPPorts = [
+          # for nodes on the lan outside of reisen...
+          server.http_listen_port
+          (mkIf (server.grpc_listen_port != 0) server.grpc_listen_port)
+        ];
+      };
   };
 }
diff --git a/modules/nixos/monitoring/source/promtail.nix b/modules/nixos/monitoring/source/promtail.nix
index cfcce811..eff542de 100644
--- a/modules/nixos/monitoring/source/promtail.nix
+++ b/modules/nixos/monitoring/source/promtail.nix
@@ -182,13 +182,4 @@ in {
     # TODO: there must be a better way to provide promtail access to these logs!
     serviceConfig.Group = mkIf nginx.enable (lib.mkForce nginx.group);
   };
-  config.networking.firewall.interfaces.lan = let
-    inherit (cfg.configuration) server;
-  in
-    mkIf cfg.enable {
-      allowedTCPPorts = [
-        server.http_listen_port
-        (mkIf (server.grpc_listen_port != 0) server.grpc_listen_port)
-      ];
-    };
 }