From aa5c67c13a5e5a85d8a0c9b4bec482f356ce83cf Mon Sep 17 00:00:00 2001 From: Kat Inskip Date: Sat, 29 Jul 2023 16:16:49 -0700 Subject: [PATCH] NGINX metrics --- nixos/roles/monitoring-server/prometheus.nix | 20 +++++++ nixos/roles/monitoring-server/scalpel.nix | 22 ++++++++ nixos/roles/monitoring-server/secrets.nix | 10 ++++ nixos/roles/monitoring-server/secrets.yaml | 42 ++++++++++++++ nixos/roles/monitoring-server/telegraf.nix | 58 ++++++++++++++++++++ nixos/roles/web-server/nginx.nix | 1 + 6 files changed, 153 insertions(+) create mode 100644 nixos/roles/monitoring-server/scalpel.nix create mode 100644 nixos/roles/monitoring-server/secrets.nix create mode 100644 nixos/roles/monitoring-server/secrets.yaml create mode 100644 nixos/roles/monitoring-server/telegraf.nix diff --git a/nixos/roles/monitoring-server/prometheus.nix b/nixos/roles/monitoring-server/prometheus.nix index c54ef9a5..d12ff3ba 100644 --- a/nixos/roles/monitoring-server/prometheus.nix +++ b/nixos/roles/monitoring-server/prometheus.nix @@ -11,6 +11,10 @@ domain = { enable = true; }; + nginx = { + enable = true; + sslVerify = false; + }; }; ruleFiles = [ ./synapse-v2.rules @@ -24,6 +28,22 @@ } ]; } + { + job_name = "${config.networking.hostName}-telegraf"; + static_configs = [ + { + targets = ["127.0.0.1:9125"]; + } + ]; + } + { + job_name = "${config.networking.hostName}-nginx"; + static_configs = [ + { + targets = ["127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}"]; + } + ]; + } { job_name = "domains"; metrics_path = "/probe"; diff --git a/nixos/roles/monitoring-server/scalpel.nix b/nixos/roles/monitoring-server/scalpel.nix new file mode 100644 index 00000000..54774c65 --- /dev/null +++ b/nixos/roles/monitoring-server/scalpel.nix @@ -0,0 +1,22 @@ +{ + lib, + config, + prev, + ... +}: let + inherit (lib.strings) addContextFrom; + start = prev.config.systemd.services.telegraf.serviceConfig.ExecStart; + telegraf_cfgfile = builtins.head (builtins.match "^.*-config ([^\ ]*).*$" "${start}"); +in { + systemd.services.telegraf.serviceConfig.ExecStart = lib.mkForce ( + builtins.replaceStrings ["${telegraf_cfgfile}"] ["${config.scalpel.trafos."config.toml".destination} "] "${start}" + ); + scalpel.trafos."config.toml" = { + source = addContextFrom start telegraf_cfgfile; + matchers."TELEGRAF_API_KEY".secret = config.sops.secrets.telegraf_api_key.path; + owner = "telegraf"; + group = "telegraf"; + mode = "0440"; + }; + #environment.etc."ensure_telegraf_trafos".source = telegraf_cfgfile; +} diff --git a/nixos/roles/monitoring-server/secrets.nix b/nixos/roles/monitoring-server/secrets.nix new file mode 100644 index 00000000..4f84c158 --- /dev/null +++ b/nixos/roles/monitoring-server/secrets.nix @@ -0,0 +1,10 @@ +_: { + sops.secrets.telegraf_api_key = { + format = "yaml"; + sopsFile = ./secrets.yaml; + }; + + scalpels = [ + ./scalpel.nix + ]; +} diff --git a/nixos/roles/monitoring-server/secrets.yaml b/nixos/roles/monitoring-server/secrets.yaml new file mode 100644 index 00000000..586cec03 --- /dev/null +++ b/nixos/roles/monitoring-server/secrets.yaml @@ -0,0 +1,42 @@ +telegraf_api_key: ENC[AES256_GCM,data:XXMLlIxtFYmURr6QuRdZFL+Z3OIm1nm8ReZq/sAML1DzFKO8U2sbdyHjXnqUWw==,iv:mMpzUrZozfcxUSpxXki64loHWtt7VwdilWTLpie01NI=,tag:a0iRgCemgDCUxKV0gMoKow==,type:str] +sops: + shamir_threshold: 1 + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1n4kdchmkk3rfkaknxhveqr2ftprdpgwckutt23y6u8639lazzuks77tgav + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnOTB2LzByVHU1T1pxWFZQ + b2JZMXEzWEY5ZjRNNnlqMW5UUmVWTk9kM2lRCjlpemIzb1FhWEE1WFNGNXZMK1Vz + YmRrYW91bno1alh3M0dZN3dyYUk5dWMKLS0tIDdWbFk2a2hiU0pLMitYeWZPYkkw + T0NKQzIzY2g3TnBoT00xa0xBUW1BNDgK/Uj+ldtdx1E+hQlKBUWo9TEPa8vmk3dZ + QWE6YSlY9kYjGNs+WHjnUXoO3VMmyzxNFFkrnOHLcfKQbi9p5Qrp0w== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2023-07-29T22:20:58Z" + mac: ENC[AES256_GCM,data:wRPzcBx4PJqK8ziR1oiVT8RrCwzlz9IugY0VMC6q7fuSBDEPrZjJ3wqpP0crNzQuZD2otEiB8ooYlL3j/lLT+vMPuUzitM5J8V3uyLwGV5FLfqC3AgbDAwb7r/x2okpSWEffhwuTMUVZ6jJo0+/XoAWS+D4IULfa77nHg6YBuu0=,iv:vft9e7pz1v5Jkxx2HnKg4+HAFZ9uRBe8OhT5DB7Yx10=,tag:nYkl5vRBG0BI/z+IERambg==,type:str] + pgp: + - created_at: "2023-07-29T22:10:05Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQIMA82M54yws73UARAApdK00AgsnRCH34W9dESFQm4ji7jjP+E4b0UDP6bEdPmX + KtFGSp4jZoTJYBpN2HJzeuVGPFfHUVMc8iZz/bkO120n41si0mwUQA+eNt7350sj + qhzjsjgYRG+iogaDI/VwEkcEtuONa3GtBjXQnXXtcI2F0e+40imXhYqezmtvjH02 + BNkY+rTvmg6LLIVrMhJXQmT+qXg+4iP/gIbCjezjO1ah16JY18dK45dqDJd+uWSN + WmqHFjqEXUJ6dzXPPkOpbGUeVkAs1OCqnNB7Hl5A5r3v8d47KPhYA9Sqkocag/NQ + Y/LMaLS6SJrugmtbNtC7FhmPHfgOnDG+8gz3m1XgP4QWKXkuOdbqoeSlXWCFlIPi + px3hXdeqaHYQvDYaUJpJqnwPbpgHIb29mTaPtP4RWbvXJzoBEshS4ONcGPMmemcg + qi+F24h6UdIDpFCguqLdf0SY10InmGB/5XCaN6Bd7zuLAq3iel5zvAW/u8Irt37J + QoUlB5OwgJds2MpBwd9RJOczlO63VJzVrGDNAVD0D6KBZHRkdEWOgpv3w8DxhIpF + lNLz78/XYvCsjgQCV+SjeJjxtQea0JOk2Xtt7nQVCrwDKh7TIIOdT8jI2EbKDAbi + bJgI1NGDxfyrk79ga7qyjLN9jhCubdKRibPPzKXqNdCahN5ldFlMvL8rZeJNYtjS + XgGaiB/wBjAmn863D4brJOH7KqALxP/tEKc4FM4uH8fcDOpsbPcgZ6Q4nQbIVHBa + 9bt8heM8006oeLPQM2raWM0/ETf+4rQzEwIO+Av4q2Rypnv47q1Qxbmag6Sh5Yw= + =wOQn + -----END PGP MESSAGE----- + fp: CD8CE78CB0B3BDD4 + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/nixos/roles/monitoring-server/telegraf.nix b/nixos/roles/monitoring-server/telegraf.nix new file mode 100644 index 00000000..ed02cdb7 --- /dev/null +++ b/nixos/roles/monitoring-server/telegraf.nix @@ -0,0 +1,58 @@ +{config, ...}: { + users.users.telegraf = { + extraGroups = [ + "nginx" + ]; + }; + + services.telegraf = { + enable = true; + extraConfig = { + inputs = { + nginx = { + urls = [ + "http://localhost/nginx_status" + ]; + response_timeout = "5s"; + }; + tail = { + name_override = "nginxlog"; + files = [ + "/var/log/nginx/access.log" + ]; + from_beginning = true; + pipe = false; + data_format = "grok"; + grok_patterns = ["%{COMBINED_LOG_FORMAT}"]; + }; + cpu = { + percpu = true; + }; + disk = { + }; + diskio = { + }; + io = { + }; + net = { + }; + mem = { + }; + system = { + }; + }; + outputs = { + prometheus_client = { + listen = "127.0.0.1:9125"; + }; + http = { + url = "http://localhost:${toString config.services.grafana.settings.server.http_port}/api/live/push/custom_stream_id"; + data_format = "influx"; + headers = { + Authorization = "Bearer !!TELEGRAF_API_KEY!!"; + }; + }; + }; + }; + }; +} diff --git a/nixos/roles/web-server/nginx.nix b/nixos/roles/web-server/nginx.nix index cf0fbe39..289f6be3 100644 --- a/nixos/roles/web-server/nginx.nix +++ b/nixos/roles/web-server/nginx.nix @@ -5,5 +5,6 @@ _: { recommendedOptimisation = true; recommendedGzipSettings = true; recommendedProxySettings = true; + statusPage = true; }; }