Add monitoring server

flake.nix

@@ -65,8 +65,23 @@
             }
           ];
         };
-        nas = nixpkgs.lib.nixosSystem {
+        monitoring = nixpkgs.lib.nixosSystem {
           system = "aarch64-linux";
+          modules = [
+            (import ./krops/monitoring/configuration.nix)
+            {
+              _module.args.nixinate = {
+                host = "monitoring";
+                sshUser = "root";
+                buildOn = "remote";
+                substituteOnTarget = true;
+                hermetic = false;
+              };
+            }
+          ];
+        };
+        nas = nixpkgs.lib.nixosSystem {
+          system = "x86_64-linux";
           modules = [
             (import ./krops/nas/configuration.nix)
             home-manager.nixosModules.home-manager

krops/monitoring/configuration.nix

@@ -0,0 +1,100 @@
+{ config, pkgs, ... }: {
+  imports = [
+    ./hardware.nix
+    ./networking.nix # generated at runtime by nixos-infect
+  ];
+
+  boot.tmp.cleanOnBoot = true;
+  zramSwap.enable = true;
+  networking.hostName = "monitoring";
+  networking.domain = "";
+  services.openssh.enable = true;
+  users.users.root.openssh.authorizedKeys.keys = [
+    ''ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjEgtIWPA5Ncs/KOcMeT6Q/HACJJetDOLjMvXXwUE+08oTX1EpHrWPpy8J+UHKIyErCNPYq8dgtrbhnMRlxHqI='' 
+  ];
+  networking.firewall.enable = false;
+
+  services.grafana = {
+    enable = true;
+    settings.server = {
+      domain = "grafana.gmem.ca";
+      http_port = 2342;
+      http_addr = "127.0.0.1";
+    };
+  };
+  services.prometheus = {
+    enable = true;
+    globalConfig = {
+      scrape_interval = "15s";
+    };
+    port = 9001;
+    extraFlags = [ "--web.enable-remote-write-receiver" ];
+    scrapeConfigs = [
+      {
+        job_name = "desktop";
+        static_configs = [ { targets = [ "london:9100" ]; } ];
+      }
+      {
+	      job_name = "nas";
+        static_configs = [ { targets = [ "vancouver:9100" ]; } ];
+      }
+      {
+        job_name = "monitoring";
+        static_configs = [ { targets = [ "localhost:9100" ]; } ];
+      }
+      {
+	      job_name = "speedtest-exporter";
+	      scrape_interval = "1h";
+	      scrape_timeout = "1m";
+        static_configs = [ { targets = [ "vancouver:9798" ]; } ];
+      }
+      {
+        job_name = "forgejo";
+        static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
+      }
+      {
+	      job_name = "blackbox";
+        metrics_path = "/probe";
+        params = { "modules" = [ "http_2xx" ]; };
+        static_configs = [ { targets = [ "google.com" "gabrielsimmer.com" "artbybecki.com" ]; } ];
+  	    relabel_configs = [ 
+          { source_labels = ["__address__"];    target_label = "__param_target"; }
+          { source_labels = ["__param_target"]; target_label = "instance"; }
+          { source_labels = []; target_label = "__address__"; replacement = "vancouver:9115"; } ];
+      }
+    ];
+    exporters.node = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      enabledCollectors = [
+        "systemd" "processes"
+      ];
+    };
+  };
+  services.tailscale.enable = true;
+  
+  # nginx reverse proxy
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedBrotliSettings = true;
+    recommendedZstdSettings = true;
+    recommendedOptimisation = true;
+    recommendedTlsSettings = true;
+    recommendedProxySettings = true;
+    virtualHosts.${config.services.grafana.domain} = {
+      default = true;
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+	      
+        proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+        proxyWebsockets = true;
+      };
+    };
+  };
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "acme@gmem.ca";
+
+  system.stateVersion = "23.11";
+}

krops/monitoring/hardware.nix

@@ -0,0 +1,14 @@
+{ modulesPath, ... }:
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+  boot.loader.grub = {
+    efiSupport = true;
+    efiInstallAsRemovable = true;
+    device = "nodev";
+  };
+  fileSystems."/boot" = { device = "/dev/disk/by-uuid/77CF-345D"; fsType = "vfat"; };
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" ];
+  boot.initrd.kernelModules = [ "nvme" ];
+  fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
+  
+}

krops/monitoring/networking.nix

@@ -0,0 +1,33 @@
+{ lib, ... }: {
+  # This file was populated at runtime with the networking
+  # details gathered from the active system.
+  networking = {
+    nameservers = [ "8.8.8.8"
+ ];
+    defaultGateway = "172.31.1.1";
+    defaultGateway6 = {
+      address = "fe80::1";
+      interface = "eth0";
+    };
+    dhcpcd.enable = false;
+    usePredictableInterfaceNames = lib.mkForce false;
+    interfaces = {
+      eth0 = {
+        ipv4.addresses = [
+          { address="91.107.206.145"; prefixLength=32; }
+        ];
+        ipv6.addresses = [
+          { address="2a01:4f8:c012:5ec6::1"; prefixLength=64; }
+{ address="fe80::9400:2ff:fe83:3979"; prefixLength=64; }
+        ];
+        ipv4.routes = [ { address = "172.31.1.1"; prefixLength = 32; } ];
+        ipv6.routes = [ { address = "fe80::1"; prefixLength = 128; } ];
+      };
+      
+    };
+  };
+  services.udev.extraRules = ''
+    ATTR{address}=="96:00:02:83:39:79", NAME="eth0"
+    
+  '';
+}

krops/nas/configuration.nix

@@ -24,13 +24,26 @@
   };
 
   services = {
+    prometheus.exporters = {
+      blackbox = {
+        enable = true;
+        configFile = "/var/lib/blackbox/config.yml";
+      };
+      node = {
+        enable = true;
+        listenAddress = "100.116.48.47";
+        enabledCollectors = [
+          "systemd" "zfs" "processes"
+        ];
+      };
+    };
     coredns = {
       enable = true;
       config =
         ''
         .:53 {
           health
-          file /var/src/dns.db git.gmem.ca
+          file /var/src/dns.db git.gmem.ca food.gmem.ca
           forward . 45.90.28.116 45.90.30.116
           bind tailscale0
         }
@@ -201,6 +214,9 @@
         federation = {
           ENABLED = true;
         };
+        metrics = {
+          ENABLED = true;
+        };
       };
     };
     gitea-actions-runner = {
@@ -215,6 +231,7 @@
             "nix:docker://nixos/nix"
           ];
           url = "https://git.gmem.ca/";
+          token = "ilTf824bWmTbDtjr0oE2riSzaoUliS54RwjrVDdR";
           settings = {
             cache.port = 4328;
           };
@@ -268,7 +285,7 @@
       trustedInterfaces = ["tailscale0" "virbr0"];
       checkReversePath = "loose";
       enable = true;
-      allowedTCPPorts = [ 22 53 80 443 2049 4328 ];
+      allowedTCPPorts = [ 22 53 80 443 2049 4328 5432 9100 ];
       allowedUDPPorts = [ 53 41641 ];
     };
     useDHCP = false;
@@ -366,6 +383,13 @@
     libvirtd.enable = true;
   };
 
+  virtualisation.oci-containers.containers = {
+    speedtest = {
+      image = "ghcr.io/miguelndecarvalho/speedtest-exporter";
+      ports = [ "9798:9798" ];
+    };
+  };
+
   sound.enable = true;
   security.rtkit.enable = true;