Add healthchecks configuration

2023-09-13 23:01:51 +01:00 · 2023-09-13 23:01:51 +01:00 · b9280f2e5c
parent 18941f17e0
commit b9280f2e5c
7 changed files with 130 additions and 22 deletions
--- a/flake.lock
+++ b/flake.lock
@ -117,11 +117,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1694134858,
-        "narHash": "sha256-fG/ESauOGmiojKlpJG8gB62dJa5Wd+ZIuiDMKK/HD3g=",
+        "lastModified": 1694375657,
+        "narHash": "sha256-32X8dcty4vPXx+D4yJPQZBo5hJ1NQikALhevGv6elO4=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "19c6a4081b14443420358262f8416149bd79561a",
+        "rev": "f7848d3e5f15ed02e3f286029697e41ee31662d7",
        "type": "github"
      },
      "original": {
@ -218,11 +218,11 @@
    },
    "nixpkgs_3": {
      "locked": {
-        "lastModified": 1694062546,
-        "narHash": "sha256-PiGI4f2BGnZcedP6slLjCLGLRLXPa9+ogGGgVPfGxys=",
+        "lastModified": 1694343207,
+        "narHash": "sha256-jWi7OwFxU5Owi4k2JmiL1sa/OuBCQtpaAesuj5LXC8w=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "b200e0df08f80c32974a6108ce431d8a8a5e6547",
+        "rev": "78058d810644f5ed276804ce7ea9e82d92bee293",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -106,6 +106,7 @@
          system = "aarch64-linux";
          modules = [
            (import ./nix/monitoring/configuration.nix)
+            agenix.nixosModules.default
            {
              _module.args.nixinate = {
                host = "monitoring";
--- a/nix/monitoring/configuration.nix
+++ b/nix/monitoring/configuration.nix
@ -4,6 +4,11 @@
    ./networking.nix # generated at runtime by nixos-infect
  ];

+  age.secrets.healthchecks-secret = {
+    file = ../../secrets/monitoring-healthchecks-secret.age;
+    owner = "healthchecks";
+  };
+
  boot.tmp.cleanOnBoot = true;
  zramSwap.enable = true;
  networking.hostName = "monitoring";
@ -31,16 +36,8 @@
    extraFlags = [ "--web.enable-remote-write-receiver" ];
    scrapeConfigs = [
      {
-        job_name = "desktop";
-        static_configs = [ { targets = [ "london:9100" ]; } ];
-      }
-      {
-	      job_name = "nas";
-        static_configs = [ { targets = [ "vancouver:9100" ]; } ];
-      }
-      {
-        job_name = "monitoring";
-        static_configs = [ { targets = [ "localhost:9100" ]; } ];
+        job_name = "personal_hardware";
+        static_configs = [ { targets = [ "london:9100" "vancouver:9100" "localhost:9100" ]; } ];
      }
      {
 	      job_name = "speedtest-exporter";
@ -48,10 +45,20 @@
 	      scrape_timeout = "1m";
        static_configs = [ { targets = [ "vancouver:9798" ]; } ];
      }
+      {
+	      job_name = "syncthing";
+        static_configs = [ { targets = [ "vancouver:8384" "london:8384" ]; } ];
+      }
      {
        job_name = "forgejo";
        static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
      }
+      {
+        job_name = "healthchecks";
+        scrape_interval = "60s";
+        metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/qyitrbccSwyuvZEISGdBHSBQaEwLcaFu";
+        static_configs = [ { targets = [ "localhost:8000" ]; } ];
+      }
      {
 	      job_name = "blackbox";
        metrics_path = "/probe";
@ -72,6 +79,14 @@
    };
  };
  services.tailscale.enable = true;
+
+  services.healthchecks = {
+    enable = true;
+    settings = {
+      SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
+      SITE_ROOT = "https://healthchecks.gmem.ca";
+    };
+  };
  
  # nginx reverse proxy
  services.nginx = {
@ -82,13 +97,20 @@
    recommendedOptimisation = true;
    recommendedTlsSettings = true;
    recommendedProxySettings = true;
-    virtualHosts.${config.services.grafana.domain} = {
+    virtualHosts.${config.services.grafana.settings.server.domain} = {
      default = true;
      enableACME = true;
      forceSSL = true;
      locations."/" = {
-	      
-        proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+        proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+        proxyWebsockets = true;
+      };
+    };
+    virtualHosts."healthchecks.gmem.ca" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:8000";
        proxyWebsockets = true;
      };
    };
--- a/nix/nas/configuration.nix
+++ b/nix/nas/configuration.nix
@ -34,6 +34,11 @@ in
    group = "users";
    mode = "770";
  };
+  age.secrets.healthcheck-ping = {
+    file = ../../secrets/healthchecks-ping.sh.age;
+    group = "users";
+    mode = "770";
+  };
  nix = {
    settings = {
      auto-optimise-store = true;
@ -62,13 +67,27 @@ in
            "/Primary/gabriel/projects"
            "/Primary/gabriel/org"
            "/Primary/gabriel/Backup/Pictures"
+            "/Primary/gabriel/Photos"
+            "/Primary/shared"
          ];
          timerConfig = {
            OnCalendar = "daily";
            Persistent = true;
-            RandomizedDelaySec = "5h";
+            RandomizedDelaySec = "6h";
          };
+          pruneOpts = [
+            "--keep-daily 7"
+            "--keep-weekly 5"
+            "--keep-monthly 12"
+            "--keep-yearly 75"
+          ];
          passwordFile = config.age.secrets.restic-password.path;
+          backupPrepareCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup/start'
+          '';
+          backupCleanupCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup'
+          '';
        };
        "becki" = {
          user = "becki";
@ -80,8 +99,48 @@ in
          timerConfig = {
            OnCalendar = "daily";
            Persistent = true;
-            RandomizedDelaySec = "5h";
+            RandomizedDelaySec = "6h";
          };
+          pruneOpts = [
+            "--keep-daily 7"
+            "--keep-weekly 5"
+            "--keep-monthly 12"
+            "--keep-yearly 75"
+          ];
+          passwordFile = config.age.secrets.restic-password.path;
+          initialize = true;
+          backupPrepareCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup/start'
+          '';
+          backupCleanupCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup'
+          '';
+        };
+        "apps" = {
+          user = "root";
+          environmentFile = config.age.secrets.restic-b2-credentials.path;
+          repository = "s3:s3.us-west-000.backblazeb2.com/gsimmer-app-backup";
+          paths = [
+            "/Primary/k3scluster"
+            "/Primary/gitea"
+          ];
+          timerConfig = {
+            OnCalendar = "daily";
+            Persistent = true;
+            RandomizedDelaySec = "12h";
+          };
+          pruneOpts = [
+            "--keep-daily 7"
+            "--keep-weekly 5"
+            "--keep-monthly 12"
+            "--keep-yearly 75"
+          ];
+          backupPrepareCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup/start'
+          '';
+          backupCleanupCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup'
+          '';
          passwordFile = config.age.secrets.restic-password.path;
          initialize = true;
        };
--- a/secrets.nix
+++ b/secrets.nix
@ -1,6 +1,7 @@
 let
  vancouver = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC30kDkktiWJGaJEIrqgneQx1SmIYUEVPm7w1F9p//CB";
-  machines = [ vancouver ];
+  monitoring = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEDtzsbxKgZ/NBYlYO2EJQZhBy3nVBVERWebbsP9mLcy";
+  machines = [ vancouver monitoring ];

  gsimmer = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIztwQxt+jqroFONSgq+xzPMuE2I5Dq/zWPQ8RcTYJr gabriel@gitgalaxy.com";
  users = [ gsimmer ];
@ -9,4 +10,7 @@ in
  "secrets/vancouver-action-runner.age".publicKeys = [ vancouver gsimmer ];
  "secrets/vancouver-restic-b2.age".publicKeys = [ vancouver gsimmer ];
  "secrets/vancouver-restic-password.age".publicKeys = [ vancouver gsimmer ];
+  "secrets/monitoring-healthchecks-secret.age".publicKeys = [ monitoring gsimmer ];
+
+  "secrets/healthchecks-ping.sh.age".publicKeys = machines ++ users;
 }
--- a/secrets/healthchecks-ping.sh.age
+++ b/secrets/healthchecks-ping.sh.age
@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 oN6OTQ vxYv5Y/7huGJzb4wW2I0BtHvRRxq5CyNfwDD++xIYn4
+LH2I/s1OvXgaQJy6+gu7qvB6gxRZb6sQmyS8D3HcuaA
+-> ssh-ed25519 J+a91w J0wCpPEStRdvvBFIlgabxjqUEuimFwclUXzPJEaX7UI
+KsrVMVyIEKbCKUKEglGZq5Xx0uPa1M8+wknZMlr3TBU
+-> ssh-ed25519 qbziOw uCnSeIqXoc7W9fqqkykWfgNVI8tC3JDP0a/CijnALw4
+R0qBsH6JDV9q6yIgBzeOOhaZOR4tficWmMm/hTclqxw
+-> d|h-grease !k-" ["gH/ 8o27gAs
+/o/XVGDK2ObI67QHgwKZywXVOblMAgDTnPtigKAW3qpi8xB6
+--- vbYaWEYM88qenBbbgeSoExaW/rYj0EqJgPHoUEAhuws
+òcÝÀ` @èÏb7Ô<
 ¹v5¨}ˆ`YùïÊ•o€#éíè¢•š‡¼òÑNøçìEIèècÓmÛ(*aÓ”ï“	B‰`•Â†<C382>q¤ô$$
--- a/secrets/monitoring-healthchecks-secret.age
+++ b/secrets/monitoring-healthchecks-secret.age
@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 J+a91w RgOoAtLvK4dWBSkVb/x5SsFly+iZ97spe5DsecEfPTY
+t6+YkVuRfsP+2gFRH8vdtm+i43B3ynb3Q5ZYhVEC8fE
+-> ssh-ed25519 qbziOw HcPjy1TabEYEV2mY9jywSIfZKYlGPYRY55t/4BVQQ1A
+TFHyFZ8iJXBo4/YqFq6FP7P7I0Q5vwcC30l3PCVx3Ik
+-> WwaHgi3'-grease Wnu
+HKx04wWgJ9ilCeFZ+4Ug1cf0b75WantwoQt4P2vPeF21w9Q3box+j71hfdjKyw5K
+pJbkinzkeZqnxi35bbNil+/AtSOdbbgYyRAOx3z8JOcOu5zHh724
+--- 1STXaZFxI1AGWzz/86QSos4U5WIG+nct8/BqSZq6bjs
+¯33(žëCbPŽ<50>9nü¦ƒ
+¬Þ`<60>Ü11ÌWŠØ™D‚2H\,Ÿy·ÆPy3hPÓ ~