gensokyo-zone/infrastructure
1
0
Fork 0
mirror of https://github.com/gensokyo-zone/infrastructure.git synced 2026-02-09 04:19:19 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(access): monitoring

Browse source
This commit is contained in:
arcnmx 2024-05-30 15:09:27 -07:00
parent 511a02931a
commit 0397043f88
13 changed files with 323 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

20
modules/nixos/monitoring/ingest/grafana.nix
View file

@ -1,11 +1,19 @@
_: {
{
config,
system,
lib,
...
}: let
inherit (lib.modules) mkDefault;
cfg = config.services.grafana;
service = system.exports.services.grafana;
in {
services.grafana = {
#enable = true;
settings.server = {
domain = "gensokyo.zone";
http_port = 9092;
http_addr = "0.0.0.0";
root_url = "https://mon.gensokyo.zone";
domain = mkDefault config.networking.domain;
http_port = mkDefault 9092;
http_addr = mkDefault "::";
root_url = mkDefault "https://${service.id}.${cfg.settings.server.domain}";
};
};
}

79
modules/nixos/monitoring/ingest/loki.nix
View file

@ -1,5 +1,4 @@
{
pkgs,
config,
lib,
access,
@ -7,6 +6,7 @@
...
}: let
inherit (gensokyo-zone) systems;
inherit (lib.modules) mkIf mkOptionDefault;
inherit (lib.attrsets) filterAttrs mapAttrsToList;
promtailSystems =
filterAttrs (
@ -15,61 +15,38 @@
&& system.config.exports.services.promtail.enable
)
systems;
inherit (builtins) toJSON;
inherit (lib.options) mkOption;
inherit (lib.types) port;
cfg = config.services.loki;
in {
options.services.loki.settings = {
httpListenPort = mkOption {
type = port;
description = "Port to listen on over HTTP";
default = 9093;
};
grpcListenPort = mkOption {
type = port;
description = "Port to listen on over gRPC";
default = 0;
};
};
config = {
services.loki = {
#enable = true;
configFile = pkgs.writeTextFile {
name = "config.yaml";
executable = false;
text = toJSON {
server = {
http_listen_port = cfg.settings.httpListenPort;
grpc_listen_port = cfg.settings.grpcListenPort;
};
positions = {
filename = "/tmp/positions.yaml";
};
clients =
mapAttrsToList (_: system: {
url = "${access.getAddressFor system.config.name "lan"}:${toString system.config.exports.services.promtail.ports.default.port}";
})
promtailSystems;
scrape_configs =
mapAttrsToList (_: system: {
job_name = "${system.config.name}-journal";
journal = {
max_age = "${toString (24 * 7)}h";
labels = {
job = "systemd-journal";
host = system.config.name;
};
};
relabel_configs = [
{
source_labels = ["__journal__systemd_unit"];
target_label = "unit";
}
];
})
promtailSystems;
configuration = {
server = {
http_listen_port = mkOptionDefault 9093;
grpc_listen_port = mkOptionDefault 0;
};
# https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#1-local-configuration-exampleyaml
auth_enabled = mkOptionDefault false;
common = {
ring = {
instance_addr = mkOptionDefault "127.0.0.1";
kvstore.store = mkOptionDefault "inmemory";
};
replication_factor = 1;
path_prefix = mkOptionDefault cfg.dataDir;
};
schema_config.configs = [
{
from = "2020-05-15";
store = "tsdb";
object_store = "filesystem";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
storage_config.filesystem.directory = mkOptionDefault "${cfg.dataDir}/chunks";
};
};
};

61
modules/nixos/monitoring/source/promtail.nix
View file

@ -1,27 +1,54 @@
{
config,
system,
access,
lib,
...
}: let
inherit (builtins) toString;
inherit (lib.options) mkOption;
inherit (lib.modules) mkIf;
inherit (lib.types) port;
inherit (lib.modules) mkIf mkOptionDefault;
cfg = config.services.promtail;
in {
options.services.promtail.settings = {
httpListenPort = mkOption {
type = port;
description = "Port to listen on over HTTP";
default = 9094;
config.services.promtail = {
configuration = {
server = {
http_listen_port = mkOptionDefault 9094;
grpc_listen_port = mkOptionDefault 0;
};
clients = let
baseUrl = access.proxyUrlFor { serviceName = "loki"; };
in [
{
url = "${baseUrl}/loki/api/v1/push";
}
];
scrape_configs = [
{
job_name = "${system.name}-journald";
journal = {
max_age = "${toString (24 * 7)}h";
labels = {
job = "systemd-journald";
system = system.name;
host = config.networking.fqdn;
};
};
relabel_configs = [
{
source_labels = ["__journal__systemd_unit"];
target_label = "unit";
}
];
}
];
};
};
config.services.promtail = {
extraFlags = [
"--server.http-listen-port=${toString cfg.settings.httpListenPort}"
];
};
config.networking.firewall.interfaces.lan = mkIf cfg.enable {
allowedTCPPorts = [ cfg.settings.httpListenPort ];
};
config.networking.firewall.interfaces.lan = let
inherit (cfg.configuration) server;
in
mkIf cfg.enable {
allowedTCPPorts = [
server.http_listen_port
(mkIf (server.grpc_listen_port != 0) server.grpc_listen_port)
];
};
}

27
modules/system/exports/monitoring.nix
View file

@ -99,7 +99,6 @@ in
config.exports.services =
{
prometheus = {config, ...}: {
id = mkAlmostOptionDefault "prometheus";
nixos = {
serviceAttr = "prometheus";
assertions = mkIf config.enable [
@ -115,7 +114,7 @@ in
};
};
grafana = {config, ...}: {
id = mkAlmostOptionDefault "grafana";
id = mkAlmostOptionDefault "mon";
nixos = {
serviceAttr = "grafana";
assertions = mkIf config.enable [
@ -131,26 +130,29 @@ in
};
};
loki = {config, ...}: {
id = mkAlmostOptionDefault "loki";
id = mkAlmostOptionDefault "logs";
nixos = {
serviceAttr = "loki";
assertions = mkIf config.enable [
(nixosConfig: {
assertion = config.ports.default.port == nixosConfig.services.loki.settings.httpListenPort;
(nixosConfig: let
inherit (nixosConfig.services.loki.configuration.server) http_listen_port;
in {
assertion = config.ports.default.port == http_listen_port;
message = "port mismatch";
})
(nixosConfig: let
inherit (nixosConfig.services.loki.settings) grpcListenPort;
inherit (nixosConfig.services.loki.configuration.server) grpc_listen_port;
in {
assertion = !config.ports.grpc.enable || config.ports.grpc.port == grpcListenPort;
assertion = !config.ports.grpc.enable || config.ports.grpc.port == grpc_listen_port;
message = "gRPC port mismatch";
})
(nixosConfig: let
inherit (nixosConfig.services.loki.settings) grpcListenPort;
inherit (nixosConfig.services.loki.configuration.server) grpc_listen_port;
in {
assertion = if config.ports.grpc.enable
then grpcListenPort != 0
else grpcListenPort == 0;
assertion =
if config.ports.grpc.enable
then grpc_listen_port != 0
else grpc_listen_port == 0;
message = "gRPC enable mismatch";
})
];
@ -174,7 +176,7 @@ in
serviceAttr = "promtail";
assertions = mkIf config.enable [
(nixosConfig: {
assertion = config.ports.default.port == nixosConfig.services.promtail.settings.httpListenPort;
assertion = config.ports.default.port == nixosConfig.services.promtail.configuration.server.http_listen_port;
message = "port mismatch";
})
];
@ -187,6 +189,7 @@ in
// {
prometheus.exporter.enable = true;
};
#ports.grpc = ...
};
}
// exporters;

50
nixos/access/grafana.nix Normal file
View file

@ -0,0 +1,50 @@
{
config,
lib,
...
}: let
inherit (lib.modules) mkIf mkDefault;
inherit (config.services) grafana;
name.shortServer = mkDefault "mon";
upstreamName = "grafana'access";
in {
config.services.nginx = {
upstreams'.${upstreamName}.servers = {
local = {
enable = mkDefault grafana.enable;
addr = mkDefault "localhost";
port = mkIf grafana.enable (mkDefault grafana.settings.server.http_port);
};
service = {upstream, ...}: {
enable = mkIf upstream.servers.local.enable (mkDefault false);
accessService = {
name = "grafana";
};
};
};
virtualHosts = let
copyFromVhost = mkDefault "grafana";
vouch.enable = mkDefault true;
locations = {
"/" = {
proxy.enable = true;
};
};
in {
grafana = {
inherit name locations vouch;
proxy.upstream = mkDefault upstreamName;
};
grafana'local = {
inherit name locations;
ssl.cert = {
inherit copyFromVhost;
};
proxy = {
inherit copyFromVhost;
};
local.enable = mkDefault true;
};
};
};
}

51
nixos/access/loki.nix Normal file
View file

@ -0,0 +1,51 @@
{
config,
lib,
...
}: let
inherit (lib.modules) mkIf mkDefault;
inherit (config.services) loki;
name.shortServer = mkDefault "logs";
upstreamName = "loki'access";
in {
config.services.nginx = {
# TODO: gRPC port?
upstreams'.${upstreamName}.servers = {
local = {
enable = mkDefault loki.enable;
addr = mkDefault "localhost";
port = mkIf loki.enable (mkDefault loki.configuration.server.http_listen_port);
};
service = {upstream, ...}: {
enable = mkIf upstream.servers.local.enable (mkDefault false);
accessService = {
name = "loki";
};
};
};
virtualHosts = let
copyFromVhost = mkDefault "loki";
vouch.enable = mkDefault true;
locations = {
"/" = {
proxy.enable = true;
};
};
in {
loki = {
inherit name locations vouch;
proxy.upstream = mkDefault upstreamName;
};
loki'local = {
inherit name locations vouch;
ssl.cert = {
inherit copyFromVhost;
};
proxy = {
inherit copyFromVhost;
};
local.enable = mkDefault true;
};
};
};
}

50
nixos/access/prometheus.nix Normal file
View file

@ -0,0 +1,50 @@
{
config,
lib,
...
}: let
inherit (lib.modules) mkIf mkDefault;
inherit (config.services) prometheus;
name.shortServer = mkDefault "prometheus";
upstreamName = "prometheus'access";
in {
config.services.nginx = {
upstreams'.${upstreamName}.servers = {
local = {
enable = mkDefault prometheus.enable;
addr = mkDefault "localhost";
port = mkIf prometheus.enable (mkDefault prometheus.port);
};
service = {upstream, ...}: {
enable = mkIf upstream.servers.local.enable (mkDefault false);
accessService = {
name = "prometheus";
};
};
};
virtualHosts = let
copyFromVhost = mkDefault "prometheus";
vouch.enable = mkDefault true;
locations = {
"/" = {
proxy.enable = true;
};
};
in {
prometheus = {
inherit name locations vouch;
proxy.upstream = mkDefault upstreamName;
};
prometheus'local = {
inherit name locations;
ssl.cert = {
inherit copyFromVhost;
};
proxy = {
inherit copyFromVhost;
};
local.enable = mkDefault true;
};
};
};
}

8
nixos/base/monitoring.nix
View file

@ -1,6 +1,8 @@
_: {
{lib, ...}: let
inherit (lib.modules) mkDefault;
in {
services = {
prometheus.exporters.node.enable = true;
promtail.enable = true;
prometheus.exporters.node.enable = mkDefault true;
promtail.enable = mkDefault true;
};
}

8
nixos/monitoring.nix
View file

@ -12,11 +12,11 @@ in {
prometheus.enable = true;
};
networking.firewall.interfaces.lan.allowedTCPPorts = mkMerge [
(mkIf grafana.enable [ grafana.port ])
(mkIf grafana.enable [grafana.settings.server.http_port])
(mkIf loki.enable [
loki.settings.httpListenPort
(mkIf (loki.settings.grpcListenPort != 0) loki.settings.grpcListenPort)
loki.configuration.server.http_listen_port
(mkIf (loki.configuration.server.grpc_listen_port != 0) loki.configuration.server.grpc_listen_port)
])
(mkIf prometheus.enable [ prometheus.port ])
(mkIf prometheus.enable [prometheus.port])
];
}

42
systems/hakurei/nixos.nix
View file

@ -37,6 +37,9 @@ in {
nixos.access.freeipa
nixos.access.freepbx
nixos.access.unifi
nixos.access.prometheus
nixos.access.grafana
nixos.access.loki
nixos.access.kitchencam
nixos.access.openwebrx
nixos.access.deluge
@ -173,6 +176,30 @@ in {
virtualHosts.unifi'local.allServerNames
];
};
prometheus = {
inherit (nginx) group;
domain = virtualHosts.prometheus.serverName;
extraDomainNames = mkMerge [
virtualHosts.prometheus.otherServerNames
virtualHosts.prometheus'local.allServerNames
];
};
mon = {
inherit (nginx) group;
domain = virtualHosts.grafana.serverName;
extraDomainNames = mkMerge [
virtualHosts.grafana.otherServerNames
virtualHosts.grafana'local.allServerNames
];
};
logs = {
inherit (nginx) group;
domain = virtualHosts.loki.serverName;
extraDomainNames = mkMerge [
virtualHosts.loki.otherServerNames
virtualHosts.loki'local.allServerNames
];
};
idp = {
inherit (nginx) group;
domain = virtualHosts.freeipa.serverName;
@ -292,6 +319,21 @@ in {
local.denyGlobal = true;
ssl.cert.enable = true;
};
prometheus = {
# we're not the real prometheus record-holder, so don't respond globally..
local.denyGlobal = true;
ssl.cert.enable = true;
};
grafana = {
# we're not the real mon record-holder, so don't respond globally..
local.denyGlobal = true;
ssl.cert.enable = true;
};
loki = {
# we're not the real logs record-holder, so don't respond globally..
local.denyGlobal = true;
ssl.cert.enable = true;
};
home-assistant = {
# not the real hass record-holder, so don't respond globally..
local.denyGlobal = true;

15
systems/utsuho/nixos.nix
View file

@ -1,8 +1,10 @@
{
meta,
config,
lib,
...
}: let
inherit (lib.modules) mkMerge;
inherit (config.services) nginx;
in {
imports = let
@ -16,6 +18,9 @@ in {
nixos.cloudflared
nixos.nginx
nixos.access.unifi
nixos.access.prometheus
nixos.access.grafana
nixos.access.loki
nixos.unifi
nixos.dnsmasq
nixos.mosquitto
@ -29,7 +34,12 @@ in {
tunnels.${tunnelId} = {
default = "http_status:404";
credentialsFile = config.sops.secrets.cloudflared-tunnel-utsuho.path;
ingress = virtualHosts.unifi.proxied.cloudflared.getIngress {};
ingress = mkMerge [
(virtualHosts.unifi.proxied.cloudflared.getIngress {})
(virtualHosts.prometheus.proxied.cloudflared.getIngress {})
(virtualHosts.grafana.proxied.cloudflared.getIngress {})
(virtualHosts.loki.proxied.cloudflared.getIngress {})
];
};
};
@ -37,6 +47,9 @@ in {
proxied.enable = true;
virtualHosts = {
unifi.proxied.enable = "cloudflared";
prometheus.proxied.enable = "cloudflared";
grafana.proxied.enable = "cloudflared";
loki.proxied.enable = "cloudflared";
};
};

3
tf/cloudflare_records.tf
View file

@ -20,6 +20,9 @@ module "hakurei_system_records" {
"ipa-cock",
"bw",
"unifi",
"prometheus",
"mon",
"logs",
"pbx",
"smb",
"mqtt",

3
tf/cloudflare_tunnels.tf
View file

@ -74,6 +74,9 @@ module "utsuho" {
zone_id = cloudflare_zone.gensokyo-zone_zone.id
subdomains = [
"unifi",
"prometheus",
"mon",
"logs",
]
}