feat: add improved alerting for various things

nixos/profiles/server/nix.nix

@@ -5,20 +5,63 @@ in {
         automatic = true;
         dates = "weekly";
     };
+
     sops.secrets.nix-gc-environment = {
         sopsFile = ./secrets.yaml;
     };
+
     systemd.services.nix-gc = {
         script = let
             cfg = config.nix.gc;
         in mkForce ''
-            ${pkgs.curl}/bin/curl -vvvv -i -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "{\"content\": \"Beginning nix garbage collection on ${config.networking.hostName}.${config.networking.domain}\"}" $DISCORD_WEBHOOK_LINK
-            OUTPUT=$(${config.nix.package.out}/bin/nix-collect-garbage ${cfg.options});
-            ${pkgs.curl}/bin/curl -vvvv -i -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "{\"content\": \"Finished nix garbage collection on ${config.networking.hostName}.${config.networking.domain}\"}" $DISCORD_WEBHOOK_LINK
-            ${pkgs.curl}/bin/curl -vvvv -i -H "Accept: application/json" -H "Content-Type:application/json" -X POST --data "{\"content\": \''${OUTPUT}\"}" $DISCORD_WEBHOOK_LINK
+            #!/usr/bin/env bash
+            set -euo pipefail
+
+            # Helper functions
+            send_discord_message() {
+                local message="$1"
+                local escaped_message=$(printf '%s' "$message" | ${pkgs.jq}/bin/jq -R -s '.')
+                ${pkgs.curl}/bin/curl -s -H "Accept: application/json" -H "Content-Type: application/json" \
+                     -X POST --data "{\"content\": $escaped_message}" "$DISCORD_WEBHOOK_LINK"
+            }
+
+            get_filesystem_usage() {
+                ${pkgs.coreutils}/bin/df -h / | ${pkgs.gawk}/bin/awk 'NR==2 {print $5 " (" $3 ")"}' | tr -d '\n'
+            }
+
+            calculate_ratio() {
+                local before="$1"
+                local after="$2"
+                ${pkgs.gawk}/bin/awk "BEGIN {printf \"%.2f\", ($after / $before) * 100}"
+            }
+
+            # Initial filesystem usage
+            FS_BEFORE_USAGE=$(get_filesystem_usage)
+
+            send_discord_message "Beginning nix garbage collection on ${config.networking.hostName} - Filesystem usage before: $FS_BEFORE_USAGE"
+
+            # Perform garbage collection
+            OUTPUT=$(${config.nix.package.out}/bin/nix-collect-garbage ${cfg.options})
+
+            # Get filesystem usage after garbage collection
+            FS_AFTER_USAGE=$(get_filesystem_usage)
+
+            # Extract numeric values for calculation (assuming format like "75% (15G)")
+            BEFORE_PERCENT=$(echo $FS_BEFORE_USAGE | ${pkgs.coreutils}/bin/cut -d'%' -f1)
+            AFTER_PERCENT=$(echo $FS_AFTER_USAGE | ${pkgs.coreutils}/bin/cut -d'%' -f1)
+
+            # Calculate ratio
+            RATIO=$(calculate_ratio $BEFORE_PERCENT $AFTER_PERCENT)
+
+            send_discord_message "Finished nix garbage collection on ${config.networking.hostName} - Filesystem usage: $FS_BEFORE_USAGE -> $FS_AFTER_USAGE ($RATIO%)"
+
+            # Send the output of nix-collect-garbage
+            send_discord_message "$OUTPUT"
         '';
+
         serviceConfig = {
             EnvironmentFile = config.sops.secrets.nix-gc-environment.path;
+            Type = "oneshot";
         };
     };
-}
+}

nixos/profiles/server/secrets.yaml

@@ -9,60 +9,87 @@ sops:
         - recipient: age1n4kdchmkk3rfkaknxhveqr2ftprdpgwckutt23y6u8639lazzuks77tgav
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuQ3piaWxpa1N4ZFRTdkhw
-            d3BLOXdCd2NDeDJmcGl2UkxlV2RGMUdlYzFVCndmNk44aUVHRExJUmJXU2RpeHN2
-            c0Y5bnQyZ2IyaFVuTHBkdHR2cFlldEEKLS0tIGpjUkZpL01BemdQb3JFL3crQS8w
-            dlZmMjJtcHl2NUU3bzV1dzBQK0FmY1UKiKRO7lTSpF7DYhR6eO0AhW4jsWMC9Etm
-            Bcc6Zpec0QKgmoy63aDj6+Fx0V5fCVX1Lis0PADpeNIn9Dshv5ouGg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrM1ZZbHlVazdHUWVtK1NT
+            ZmYyN1JNa2E2S3NldWR6dEFiNElCcWUvMXg0Ck5TeUs3REtzMVkvR2V2QlhvUWdB
+            eHJ4Nkl2MitIeitSci9KS1RRalRoMWsKLS0tIGdwL3RnNno0d1hRNFhRSUthU2hH
+            YUhWVjZiaTVyYmhZUDQrcUJ1T0Q2aUEKtkAw4R9MFUviuJkdXxHJyUzA2Syf23d8
+            vPTA71uwvKYHu49/xmkV8Dw06V0nIC8DVkoiraeiElP0c49msbuqaw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1cnu37d5fqyahh9vvc4hj6z6k8ur9ksuefln7sr6g3emmn927eutqxdawuh
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKbFRvTXQrK1ZNWWxPblB5
-            MGVsaUx4MzRlcW4xVkZNczFRdzBlM3VQQVV3CjdXUk9IVC9NRDBNeUMrSUo2anFS
-            eUhNYWZvdEhJamVYcXJXUExwdFQwb1kKLS0tIHZqNlFhWXZHSDAvdkFtMVhSdnlI
-            amhncGFzbktNVThyTHl6NFdMc3N5SFkK9NDy5U7Bfl6t8sSZem+EbqD5yW3ZHiex
-            PUac2UJvy5Q8QA3knQUUtLuLAuE5WrpIOzV8w8YnMYpDBhZtwO9uDg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzeUdIOWx2R3V5QjArR0Y2
+            amJNUVg5d3NBcmNaNE9LWkFOeU9UcnV6WEJBClNFTzQ0TWxhNnBSOE55UjRuSXV4
+            dWJmdzZ5ZEVURmRNaVRqQSttZUwzaGMKLS0tIHorc3lBYkNLYWtWSzVJczlJS2VH
+            cDhxTmJzclM2c3Z4SEJGaDkrWUFJMzAKJpMErKgrSfibubv9FkPVGqM5+nyB8tqb
+            FKqpd/p/jDVYFTH2RmwmzSQZP0Pjjl0nKYfi0pC/K9716HE7uTy7Ww==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1nr0qds8w3gldmdvhwu0p6w2ys8f4sd0h3xy94h9dsafjzttaypxquzmswc
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIWFlRUFl2OHhuWnFWblBH
-            bWtRamd4ZDRURHRSYWRFc2tabWg1QTk1Vm1FCkV0akNpNTRxUURzQjQ3RHJMOFVI
-            T3lDZkFzdER3bmszcVVWZ1h1eWxwZXMKLS0tIFJianRjUm1tOWlxTGkxTkJ4a2hq
-            Z2lERWpVaXhqRDQ3YlpndTdKUklUcjgK5XCk4qbAerT2AfOlpjKK4sUTdAN3Edt0
-            XleLhGq+bPG3CHUEN7SIaoHh4fyCpwcNGJPAcmeGY1yJZh8y0UQvSw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHSTNSWnVBT2pyTTdCYVRV
+            TWNNek1TNzI2dkJzVGZDUjlFbkF2Um54QkJRClcyU096ZUFhRXMyd3o1RTFkK0xx
+            K00xcTBTakxaa3hJRWpPWTV2aXdpQ2MKLS0tIHFvOERJRThQVStCejN2VVh0eENT
+            WGRPV05WZHR0RzNWZ3NHV1RJMEZsRVUKbJR3qG7KTGgUsnsajndrUN+FNW+E3Rfu
+            4bHisR1/sFINs9P25E4F353Ld5fVyt9+zkUO+GuHd2WEc1Hgge8HvQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age18hpxz0ghvswv9k30cle73prvnzrsuczqh87jjdk9fl50j3ddndmq9xae0n
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2YlMzam1CMmFoSXVwWjdY
-            bC9hem5manl2RngvMm1FdDU0anZlL0pDdHpZClhOdlVrM05aek1sMUdQdHNvRTRp
-            UEZ4LzFXM3NtRzA0Nm80OFlGSWlnMW8KLS0tIGRPZWhRVStiUm9tYjErWmpZa3A4
-            aDJmdGUxZWdqbXFjeCt4dHlSVDE4TEkKz+z1s1MvGcyVIPLQEnFFm1YpDDUc2KBf
-            p92AFO+1CXZsQTKY6eRPIUxkXPKXsBYPosy7Z34mBKmjlrvxrM+2OA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzTG1UUG51RVFXQTdHOEVO
+            L3ptclplbVgvUktMTDgrSENVL2FVaEc2TEN3Ci9CL0JQNnFzdjR6aFJHc3g2VUN0
+            V1UxUVkzcjI0aXZYb0Y3RlZBK3lSM0UKLS0tIGZpbi9Ba0dXY1E4c0Q1ZkZOYjlG
+            Nk1mbWR2MGFWZEdWbThWc2lNNWpwU28KDvpGGsTyRjyHvOjVyMzvjZa36y0WXcej
+            FLjDVQt4MGQ6u/r91MMPk2rT5N1UPWDoraKKC6HZ+cw/UcgGgd4CNQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1xgy03g3vjydsxcl0qpdgm8rahjcjq95ucxfwlgr22zwjx3p7jf2s9jk6u5
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0dGdUeDZ3bzRDYVE0ZXIy
+            Ylc0Nk10UmpCV3JLWnY3SjdKKy9jTWpzSWxvClpyNTN3RnZiTmxGVjZIQzkyWXhq
+            NlgrV3RXMWZyMjRsUzltSTgxeThBbVEKLS0tIEtJZTJaekZkK1lpVThDTXZTUmRN
+            d1BES0pXSzZGV0xybjc4N2w5RTV5NU0K2EY6/uS0ZR9TxFywTXrbWwlQZ7M7NzxI
+            dDyeK+kMVhBXEyVO4j+uZPBAs8b2lih7AZPAioTiz/wh9PieaI2k5g==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1fv5dafs4n3r5n83qm2hfz7xmnflsz0xf9r3saralrptpgf8mvuxq4t8k3u
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SGpEZmFTOWRtT25XM0ll
+            SXVMVXBPTC9LQ3d6cG5NY2dqSmtldWJMdENFCjVyWm4vM2lHQW5nS0FkZFVjZGhV
+            eFk1NWRPZVJVaEN2ZXJXTUlEaHRnQTgKLS0tIGRQek41bTNXZXBJRDdtWklRM3pC
+            NzRyWXAyOFVlYXZOc0lxTGl5b3d0RWsKH0+TdY7D/mApS+110QGE09MdZh/RcSyh
+            9oNy3EDpB7GOy/UcMLz8Cl6rgMg8gsQwvDfRRig9HsCWY5lNXs/W1g==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age120530yclr75k6nrzp6k5jjftj8j4q9v3533guupzk4ct86mjxszqg9e5t5
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaUGphZU9XL2M1Zm8ydG52
+            b0dYeThiaitUMkZjSkdacG1tOXM4YlE3MzFFCllZOVBSaGx1NkxINGhqRkhtZjNC
+            OU8vSzFxdkpBV2pNNzBFN0t3S1hyc2sKLS0tIGRJMXhzVlBUd091THJLSXUxVXVn
+            OGVicjRPMG1IcFdMckw4QmVyd05Lb1UKjbtiQonzA3nKWxRCcseRQsNmG+qgN71j
+            YSsTOP1ClhKnbBdldiRjGDyyuZ0XQQ1nXjcEwntlQ7PP08O/zwSsOw==
             -----END AGE ENCRYPTED FILE-----
     lastmodified: "2024-06-25T03:22:44Z"
     mac: ENC[AES256_GCM,data:2uVqMaPYgG2hbkMZCd3xJjjoEJqsGhFEXAq4p+X7YWO4hwB+H/REJJkHCUBegggWJtKA1zDKDIVzvZv3BeRaIe63Kaj2A/7c3qwjCsBpzm5DdJ3WrlAIffFSgOs7jUyFwQtP0ZsbHigsr/rA5NqDeC+4hVHg9XKgLXKyPoVk+iM=,iv:rzf0xQGfGMirg1wwe3paq1+lNdISerFXRUsPLtZ09m0=,tag:6xkM9kvN/8NqzTYB5eHbVA==,type:str]
     pgp:
-        - created_at: "2024-06-25T03:21:52Z"
+        - created_at: "2024-07-03T16:38:04Z"
           enc: |-
             -----BEGIN PGP MESSAGE-----
 
-            hQIMA82M54yws73UAQ/8CQAUzNv2BxCf8d+XPW+NeV5XsTqk06/QdFmyhguS4fn7
-            eyclxiww6FBspxX8WxfLsE3qLjA1cGRv8W8kvZMzuIiJW7BECnzUvANNci3STl3w
-            Ei4zkWCuXYdgO0nbfzvv2MyXSdw5nnJIRpbh/QyR7UOJkHHkurtLXCupNImZUN0d
-            FKzM+Y0rM/rDQvNxk216T0eAE68su+wzNbPEgYzMSq/0N5kFl+31JU7hRdXf1+Kd
-            MFFwu8owk/G0pqkOx3jIV5sia97CZbG7pZLNwfXTngVum/neRGCwNf+Ub4S51K0s
-            pQZHDFgacRUCKkJs2XXZcYQHEn2NQ+z+6rvnmOEsMMRM2X+g1+6SocL2Rf6VZgDo
-            UNr6oUplzMdJFRM8ymqP6IsVK/L8NQF9sna2MevtDGxoFV6Dl2mOzyHUCCaHyp0O
-            sWiIsnkogFDGOH7OjUSvTjv/o5RbeHGyLzzAYg8ZKRyqhdhzF+QFToQ4mqzyjrAd
-            NEqDgAYolgOPg2NmDpuBBnHwJhNQDaWA3wDDSEtH++xrjgZy0vovM79HUwYOGyPK
-            mOjl2CM52QFaORmSj561TgfOAO2ulVPIjXa88w9mFyyNqsecqWevQFBYn9/V7Yz0
-            5SpnUpxhJ50ZeY/IZa5rz+JoZmX+Gg+dwqvG58o1Nh21tQzFemApi7FC1HqwukPS
-            XgEhEqzHm2ayA9wTLyFkaZeIMQyCm/bm3i0PN4N9yojq6/g3wXK2k/tld208ro5m
-            682qNj7bIeWqwdfZxdmdgzutqojV1zrfaC2iYLd58waxua6w9UbE9jvkg0cz6H8=
-            =ceQ3
+            hQIMA82M54yws73UARAArVDBsn02FJbl6OXIW1YrT4O96F7wMuJYF5w9Qw1sMudi
+            BKnZeiM9gJQPPgr9J/Pb5FGR6KQQzcz7ogYgZHGvxdDgIdqwSXWpP9Y3W0qCEZfc
+            y/BDfdGyOWa3cTMVQg7gO6nnhu/02hjUT/+dRe+kDwbm7Jn5o+SZBM+136YaQeiO
+            8Wqfa2pGPCMkh/CzrvywuD66Y8nYm75ViqzFsh0SzCw3huOQBn5tGYWbaoLz7IPk
+            +j9Yl7FAeWPVCV2mlQ+G0szZiZ7ouYv7e/xkDk7n+Z6hxzuqAg2LgCgVOhH2bDfa
+            LdWzdOD7wEdLwfT7hAf8EnaF5CcjShox0XqsUptXqBUu84A/8JH3vKFVN4JxO9YP
+            hviLuXk+VsIiFoIL6qvGbdtaqWPG8JN+OdtekLX3S0OHFltdPh1Jxyomh4+CCvLh
+            KlrkGMRC8xcff/p/mTSr+aiHbjqU3aEf+tNJyk/2ghHCCOcXTc3FuFhx87+NhVW5
+            u75Swb4u05cZSzDY3Ie3xmim5kvM7IyNwSJ4dyEHpGDmHUXQxGQPVsNGtImeyM7l
+            AmVkSi3LfyV/DGBDy3iQbqotREd7OQEHnPFH0YFlr1PsM17Y6JrXHlSxDT7FsIg0
+            6q593i+BV5tdfKc1UF77FOvxlr3wnxy9pXxKSNUoOTLzoeGadaJ3aV8ukVzNyXLS
+            XgELL1usQe/o03hxjEeQJuy1VEl0QRk8Y/6wtQDuJXG5Y/fwxl8XRn1ck12soU5P
+            3tV5aAiPjLrAFz1gopYHel+pSaKTUDavD5TBJ2jR+oswWRdFOlF5qYoEwlA7ADI=
+            =TE06
             -----END PGP MESSAGE-----
           fp: CD8CE78CB0B3BDD4
     unencrypted_suffix: _unencrypted