monitoring-hosted blackbox exporter, uptime kuma

nix/monitoring/configuration.nix

@@ -39,6 +39,8 @@
     owner = "grafana";
   };
 
+  nix.settings.auto-optimise-store = true;
+
   boot.tmp.cleanOnBoot = true;
   zramSwap.enable = true;
   networking.hostName = "monitoring";
@@ -244,7 +246,7 @@
             rules = [
               {
                 alert = "LowInternetDownload";
-                expr = ''speedtest_download_bits_per_second{} < 500000000'';
+                expr = "speedtest_download_bits_per_second{} < 500000000";
                 for = "2h";
                 labels.severity = "page";
                 annotations = {
@@ -343,10 +345,15 @@
         static_configs = [{targets = ["vancouver:6534"];}];
       }
       {
-        job_name = "blackbox";
+        job_name = "blackbox_home";
         metrics_path = "/probe";
         params = {"modules" = ["http_2xx"];};
-        static_configs = [{targets = ["google.com" "gabrielsimmer.com" "artbybecki.com"];}];
+        static_configs = [{
+          labels = {
+            location = "home";
+          };
+          targets = ["floofy.tech" "1.1.1.1" "waterwolf.club"];
+        }];
         relabel_configs = [
           {
             source_labels = ["__address__"];
@@ -363,6 +370,32 @@
           }
         ];
       }
+      {
+        job_name = "blackbox_hetzner";
+        metrics_path = "/probe";
+        params = {"modules" = ["http_2xx"];};
+        static_configs = [{
+          labels = {
+            location = "hetzner";
+          };
+          targets = ["floofy.tech" "1.1.1.1" "waterwolf.club"];
+        }];
+        relabel_configs = [
+          {
+            source_labels = ["__address__"];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = ["__param_target"];
+            target_label = "instance";
+          }
+          {
+            source_labels = [];
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
     ];
     exporters.node = {
       enable = true;
@@ -372,6 +405,10 @@
         "processes"
       ];
     };
+    exporters.blackbox = {
+      enable = true;
+      configFile = "/var/lib/blackbox/config.yml";
+    };
   };
   services.tailscale.enable = true;
 
@@ -391,6 +428,13 @@
     };
   };
 
+  services.uptime-kuma = {
+    enable = true;
+    settings = {
+      PORT = "4000";
+    };
+  };
+
   # nginx reverse proxy
   services.nginx = {
     enable = true;
@@ -418,6 +462,15 @@
         proxyWebsockets = true;
       };
     };
+
+    virtualHosts."uptime.gmem.ca" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString config.services.uptime-kuma.settings.PORT}";
+        proxyWebsockets = true;
+      };
+    };
   };
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "acme@gmem.ca";