Add healthchecks configuration

flake.lock

@@ -117,11 +117,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1694134858,
+        "lastModified": 1694375657,
-        "narHash": "sha256-fG/ESauOGmiojKlpJG8gB62dJa5Wd+ZIuiDMKK/HD3g=",
+        "narHash": "sha256-32X8dcty4vPXx+D4yJPQZBo5hJ1NQikALhevGv6elO4=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "19c6a4081b14443420358262f8416149bd79561a",
+        "rev": "f7848d3e5f15ed02e3f286029697e41ee31662d7",
         "type": "github"
       },
       "original": {
@@ -218,11 +218,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1694062546,
+        "lastModified": 1694343207,
-        "narHash": "sha256-PiGI4f2BGnZcedP6slLjCLGLRLXPa9+ogGGgVPfGxys=",
+        "narHash": "sha256-jWi7OwFxU5Owi4k2JmiL1sa/OuBCQtpaAesuj5LXC8w=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b200e0df08f80c32974a6108ce431d8a8a5e6547",
+        "rev": "78058d810644f5ed276804ce7ea9e82d92bee293",
         "type": "github"
       },
       "original": {

flake.nix

@@ -106,6 +106,7 @@
           system = "aarch64-linux";
           modules = [
             (import ./nix/monitoring/configuration.nix)
+            agenix.nixosModules.default
             {
               _module.args.nixinate = {
                 host = "monitoring";

nix/london/configuration.nix

@@ -1,4 +1,24 @@
 { config, pkgs, ... }:
+let
+
+  syncthingLatest =
+    let
+      version = "1.24.0";
+      src = pkgs.fetchFromGitHub {
+        owner = "syncthing";
+        repo = "syncthing";
+        rev = "v1.24.0";
+        hash = "sha256-5vr9qWMHBYpu8wHpV1JZcX1kEPi+mYeZ7ZQBqXASp9I=";
+      };
+    in
+      (pkgs.syncthing.override rec {
+        buildGoModule = args: pkgs.buildGoModule.override {} (args // {
+          inherit src version;
+          vendorHash = "sha256-BZwZ6npmWFU0lvynjRZOBOhtxqic0djoSUdCOLbUwjE=";
+        });
+      });
+
+in
 {
   imports =
     [
@@ -70,6 +90,16 @@
   i18n.defaultLocale = "en_GB.utf8";
 
   services = {
+    syncthing = {
+      enable = true;
+      overrideDevices = false;
+      overrideFolders = false;
+      user = "gsimmer";
+      dataDir = "/home/gsimmer";
+      guiAddress = "100.95.77.62:8384";
+      package = syncthingLatest;
+    };
+    usbmuxd.enable = true;
     prometheus.exporters.node = {
       enable = true;
       listenAddress = "100.95.77.62";

nix/monitoring/configuration.nix

@@ -4,6 +4,11 @@
     ./networking.nix # generated at runtime by nixos-infect
   ];
 
+  age.secrets.healthchecks-secret = {
+    file = ../../secrets/monitoring-healthchecks-secret.age;
+    owner = "healthchecks";
+  };
+
   boot.tmp.cleanOnBoot = true;
   zramSwap.enable = true;
   networking.hostName = "monitoring";
@@ -31,16 +36,8 @@
     extraFlags = [ "--web.enable-remote-write-receiver" ];
     scrapeConfigs = [
       {
-        job_name = "desktop";
+        job_name = "personal_hardware";
-        static_configs = [ { targets = [ "london:9100" ]; } ];
+        static_configs = [ { targets = [ "london:9100" "vancouver:9100" "localhost:9100" ]; } ];
-      }
-      {
-	      job_name = "nas";
-        static_configs = [ { targets = [ "vancouver:9100" ]; } ];
-      }
-      {
-        job_name = "monitoring";
-        static_configs = [ { targets = [ "localhost:9100" ]; } ];
       }
       {
 	      job_name = "speedtest-exporter";
@@ -48,10 +45,20 @@
 	      scrape_timeout = "1m";
         static_configs = [ { targets = [ "vancouver:9798" ]; } ];
       }
+      {
+	      job_name = "syncthing";
+        static_configs = [ { targets = [ "vancouver:8384" "london:8384" ]; } ];
+      }
       {
         job_name = "forgejo";
         static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
       }
+      {
+        job_name = "healthchecks";
+        scrape_interval = "60s";
+        metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/qyitrbccSwyuvZEISGdBHSBQaEwLcaFu";
+        static_configs = [ { targets = [ "localhost:8000" ]; } ];
+      }
       {
 	      job_name = "blackbox";
         metrics_path = "/probe";
@@ -73,6 +80,14 @@
   };
   services.tailscale.enable = true;
 
+  services.healthchecks = {
+    enable = true;
+    settings = {
+      SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
+      SITE_ROOT = "https://healthchecks.gmem.ca";
+    };
+  };
+  
   # nginx reverse proxy
   services.nginx = {
     enable = true;
@@ -82,13 +97,20 @@
     recommendedOptimisation = true;
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
-    virtualHosts.${config.services.grafana.domain} = {
+    virtualHosts.${config.services.grafana.settings.server.domain} = {
       default = true;
       enableACME = true;
       forceSSL = true;
       locations."/" = {
-	      
+        proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
-        proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+        proxyWebsockets = true;
+      };
+    };
+    virtualHosts."healthchecks.gmem.ca" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:8000";
         proxyWebsockets = true;
       };
     };

nix/nas/configuration.nix

@@ -1,5 +1,22 @@
 { config, pkgs, ... }:
-
+let
+  syncthingLatest =
+    let
+      version = "1.24.0";
+      src = pkgs.fetchFromGitHub {
+        owner = "syncthing";
+        repo = "syncthing";
+        rev = "v1.24.0";
+        hash = "sha256-5vr9qWMHBYpu8wHpV1JZcX1kEPi+mYeZ7ZQBqXASp9I=";
+      };
+    in
+      (pkgs.syncthing.override rec {
+        buildGoModule = args: pkgs.buildGoModule.override {} (args // {
+          inherit src version;
+          vendorHash = "sha256-BZwZ6npmWFU0lvynjRZOBOhtxqic0djoSUdCOLbUwjE=";
+        });
+      });
+in
 {
   imports =
     [ # Include the results of the hardware scan.
@@ -17,6 +34,11 @@
     group = "users";
     mode = "770";
   };
+  age.secrets.healthcheck-ping = {
+    file = ../../secrets/healthchecks-ping.sh.age;
+    group = "users";
+    mode = "770";
+  };
   nix = {
     settings = {
       auto-optimise-store = true;
@@ -45,13 +67,27 @@
             "/Primary/gabriel/projects"
             "/Primary/gabriel/org"
             "/Primary/gabriel/Backup/Pictures"
+            "/Primary/gabriel/Photos"
+            "/Primary/shared"
           ];
           timerConfig = {
             OnCalendar = "daily";
             Persistent = true;
-            RandomizedDelaySec = "5h";
+            RandomizedDelaySec = "6h";
           };
+          pruneOpts = [
+            "--keep-daily 7"
+            "--keep-weekly 5"
+            "--keep-monthly 12"
+            "--keep-yearly 75"
+          ];
           passwordFile = config.age.secrets.restic-password.path;
+          backupPrepareCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup/start'
+          '';
+          backupCleanupCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup'
+          '';
         };
         "becki" = {
           user = "becki";
@@ -63,8 +99,48 @@
           timerConfig = {
             OnCalendar = "daily";
             Persistent = true;
-            RandomizedDelaySec = "5h";
+            RandomizedDelaySec = "6h";
           };
+          pruneOpts = [
+            "--keep-daily 7"
+            "--keep-weekly 5"
+            "--keep-monthly 12"
+            "--keep-yearly 75"
+          ];
+          passwordFile = config.age.secrets.restic-password.path;
+          initialize = true;
+          backupPrepareCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup/start'
+          '';
+          backupCleanupCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup'
+          '';
+        };
+        "apps" = {
+          user = "root";
+          environmentFile = config.age.secrets.restic-b2-credentials.path;
+          repository = "s3:s3.us-west-000.backblazeb2.com/gsimmer-app-backup";
+          paths = [
+            "/Primary/k3scluster"
+            "/Primary/gitea"
+          ];
+          timerConfig = {
+            OnCalendar = "daily";
+            Persistent = true;
+            RandomizedDelaySec = "12h";
+          };
+          pruneOpts = [
+            "--keep-daily 7"
+            "--keep-weekly 5"
+            "--keep-monthly 12"
+            "--keep-yearly 75"
+          ];
+          backupPrepareCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup/start'
+          '';
+          backupCleanupCommand = ''
+            ${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup'
+          '';
           passwordFile = config.age.secrets.restic-password.path;
           initialize = true;
         };
@@ -77,6 +153,7 @@
       user = "gsimmer";
       dataDir = "/Primary/gabriel";
       guiAddress = "100.116.48.47:8384";
+      package = syncthingLatest;
     };
     prometheus.exporters = {
       blackbox = {

secrets.nix

@@ -1,6 +1,7 @@
 let
   vancouver = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC30kDkktiWJGaJEIrqgneQx1SmIYUEVPm7w1F9p//CB";
-  machines = [ vancouver ];
+  monitoring = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEDtzsbxKgZ/NBYlYO2EJQZhBy3nVBVERWebbsP9mLcy";
+  machines = [ vancouver monitoring ];
 
   gsimmer = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIztwQxt+jqroFONSgq+xzPMuE2I5Dq/zWPQ8RcTYJr gabriel@gitgalaxy.com";
   users = [ gsimmer ];
@@ -9,4 +10,7 @@ in
   "secrets/vancouver-action-runner.age".publicKeys = [ vancouver gsimmer ];
   "secrets/vancouver-restic-b2.age".publicKeys = [ vancouver gsimmer ];
   "secrets/vancouver-restic-password.age".publicKeys = [ vancouver gsimmer ];
+  "secrets/monitoring-healthchecks-secret.age".publicKeys = [ monitoring gsimmer ];
+
+  "secrets/healthchecks-ping.sh.age".publicKeys = machines ++ users;
 }

secrets/healthchecks-ping.sh.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 oN6OTQ vxYv5Y/7huGJzb4wW2I0BtHvRRxq5CyNfwDD++xIYn4
+LH2I/s1OvXgaQJy6+gu7qvB6gxRZb6sQmyS8D3HcuaA
+-> ssh-ed25519 J+a91w J0wCpPEStRdvvBFIlgabxjqUEuimFwclUXzPJEaX7UI
+KsrVMVyIEKbCKUKEglGZq5Xx0uPa1M8+wknZMlr3TBU
+-> ssh-ed25519 qbziOw uCnSeIqXoc7W9fqqkykWfgNVI8tC3JDP0a/CijnALw4
+R0qBsH6JDV9q6yIgBzeOOhaZOR4tficWmMm/hTclqxw
+-> d|h-grease !k-" ["gH/ 8o27gAs
+/o/XVGDK2ObI67QHgwKZywXVOblMAgDTnPtigKAW3qpi8xB6
+--- vbYaWEYM88qenBbbgeSoExaW/rYj0EqJgPHoUEAhuws
+òcÝÀ` @èÏb7Ô<
 ¹v5¨}ˆ`
YùïÊ•o€#éí袕š‡¼òÑNøçìEIèècÓmÛ(*aÓ”ï“	B‰`•Â†<C382>q¤ô$$

secrets/monitoring-healthchecks-secret.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 J+a91w RgOoAtLvK4dWBSkVb/x5SsFly+iZ97spe5DsecEfPTY
+t6+YkVuRfsP+2gFRH8vdtm+i43B3ynb3Q5ZYhVEC8fE
+-> ssh-ed25519 qbziOw HcPjy1TabEYEV2mY9jywSIfZKYlGPYRY55t/4BVQQ1A
+TFHyFZ8iJXBo4/YqFq6FP7P7I0Q5vwcC30l3PCVx3Ik
+-> WwaHgi3'-grease Wnu
+HKx04wWgJ9ilCeFZ+4Ug1cf0b75WantwoQt4P2vPeF21w9Q3box+j71hfdjKyw5K
+pJbkinzkeZqnxi35bbNil+/AtSOdbbgYyRAOx3z8JOcOu5zHh724
+--- 1STXaZFxI1AGWzz/86QSos4U5WIG+nct8/BqSZq6bjs
+¯33(žëCbPŽ<50>9nü¦ƒ
+¬Þ`<60>Ü11ÌWŠØ™D‚2H\,Ÿy·ÆPy3hPÓ ~