From 1813986ad4a7b77f912e5ee46aff9ef899e7fe86 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Tue, 5 Sep 2023 17:08:43 +0100 Subject: [PATCH] Add monitoring server --- flake.nix | 17 ++++- krops/monitoring/configuration.nix | 100 +++++++++++++++++++++++++++++ krops/monitoring/hardware.nix | 14 ++++ krops/monitoring/networking.nix | 33 ++++++++++ krops/nas/configuration.nix | 28 +++++++- 5 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 krops/monitoring/configuration.nix create mode 100644 krops/monitoring/hardware.nix create mode 100644 krops/monitoring/networking.nix diff --git a/flake.nix b/flake.nix index b296d14..3c0fbad 100644 --- a/flake.nix +++ b/flake.nix @@ -65,8 +65,23 @@ } ]; }; - nas = nixpkgs.lib.nixosSystem { + monitoring = nixpkgs.lib.nixosSystem { system = "aarch64-linux"; + modules = [ + (import ./krops/monitoring/configuration.nix) + { + _module.args.nixinate = { + host = "monitoring"; + sshUser = "root"; + buildOn = "remote"; + substituteOnTarget = true; + hermetic = false; + }; + } + ]; + }; + nas = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; modules = [ (import ./krops/nas/configuration.nix) home-manager.nixosModules.home-manager diff --git a/krops/monitoring/configuration.nix b/krops/monitoring/configuration.nix new file mode 100644 index 0000000..1d0f953 --- /dev/null +++ b/krops/monitoring/configuration.nix @@ -0,0 +1,100 @@ +{ config, pkgs, ... }: { + imports = [ + ./hardware.nix + ./networking.nix # generated at runtime by nixos-infect + ]; + + boot.tmp.cleanOnBoot = true; + zramSwap.enable = true; + networking.hostName = "monitoring"; + networking.domain = ""; + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keys = [ + ''ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjEgtIWPA5Ncs/KOcMeT6Q/HACJJetDOLjMvXXwUE+08oTX1EpHrWPpy8J+UHKIyErCNPYq8dgtrbhnMRlxHqI='' + ]; + networking.firewall.enable = false; + + services.grafana = { + enable = true; + settings.server = { + domain = "grafana.gmem.ca"; + http_port = 2342; + http_addr = "127.0.0.1"; + }; + }; + services.prometheus = { + enable = true; + globalConfig = { + scrape_interval = "15s"; + }; + port = 9001; + extraFlags = [ "--web.enable-remote-write-receiver" ]; + scrapeConfigs = [ + { + job_name = "desktop"; + static_configs = [ { targets = [ "london:9100" ]; } ]; + } + { + job_name = "nas"; + static_configs = [ { targets = [ "vancouver:9100" ]; } ]; + } + { + job_name = "monitoring"; + static_configs = [ { targets = [ "localhost:9100" ]; } ]; + } + { + job_name = "speedtest-exporter"; + scrape_interval = "1h"; + scrape_timeout = "1m"; + static_configs = [ { targets = [ "vancouver:9798" ]; } ]; + } + { + job_name = "forgejo"; + static_configs = [ { targets = [ "git.gmem.ca" ]; } ]; + } + { + job_name = "blackbox"; + metrics_path = "/probe"; + params = { "modules" = [ "http_2xx" ]; }; + static_configs = [ { targets = [ "google.com" "gabrielsimmer.com" "artbybecki.com" ]; } ]; + relabel_configs = [ + { source_labels = ["__address__"]; target_label = "__param_target"; } + { source_labels = ["__param_target"]; target_label = "instance"; } + { source_labels = []; target_label = "__address__"; replacement = "vancouver:9115"; } ]; + } + ]; + exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + enabledCollectors = [ + "systemd" "processes" + ]; + }; + }; + services.tailscale.enable = true; + + # nginx reverse proxy + services.nginx = { + enable = true; + recommendedGzipSettings = true; + recommendedBrotliSettings = true; + recommendedZstdSettings = true; + recommendedOptimisation = true; + recommendedTlsSettings = true; + recommendedProxySettings = true; + virtualHosts.${config.services.grafana.domain} = { + default = true; + enableACME = true; + forceSSL = true; + locations."/" = { + + proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}"; + proxyWebsockets = true; + }; + }; + }; + security.acme.acceptTerms = true; + security.acme.defaults.email = "acme@gmem.ca"; + + system.stateVersion = "23.11"; +} diff --git a/krops/monitoring/hardware.nix b/krops/monitoring/hardware.nix new file mode 100644 index 0000000..8e067c1 --- /dev/null +++ b/krops/monitoring/hardware.nix @@ -0,0 +1,14 @@ +{ modulesPath, ... }: +{ + imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; + boot.loader.grub = { + efiSupport = true; + efiInstallAsRemovable = true; + device = "nodev"; + }; + fileSystems."/boot" = { device = "/dev/disk/by-uuid/77CF-345D"; fsType = "vfat"; }; + boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" ]; + boot.initrd.kernelModules = [ "nvme" ]; + fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; + +} diff --git a/krops/monitoring/networking.nix b/krops/monitoring/networking.nix new file mode 100644 index 0000000..61a8426 --- /dev/null +++ b/krops/monitoring/networking.nix @@ -0,0 +1,33 @@ +{ lib, ... }: { + # This file was populated at runtime with the networking + # details gathered from the active system. + networking = { + nameservers = [ "8.8.8.8" + ]; + defaultGateway = "172.31.1.1"; + defaultGateway6 = { + address = "fe80::1"; + interface = "eth0"; + }; + dhcpcd.enable = false; + usePredictableInterfaceNames = lib.mkForce false; + interfaces = { + eth0 = { + ipv4.addresses = [ + { address="91.107.206.145"; prefixLength=32; } + ]; + ipv6.addresses = [ + { address="2a01:4f8:c012:5ec6::1"; prefixLength=64; } +{ address="fe80::9400:2ff:fe83:3979"; prefixLength=64; } + ]; + ipv4.routes = [ { address = "172.31.1.1"; prefixLength = 32; } ]; + ipv6.routes = [ { address = "fe80::1"; prefixLength = 128; } ]; + }; + + }; + }; + services.udev.extraRules = '' + ATTR{address}=="96:00:02:83:39:79", NAME="eth0" + + ''; +} diff --git a/krops/nas/configuration.nix b/krops/nas/configuration.nix index d59e36e..c2d78cd 100644 --- a/krops/nas/configuration.nix +++ b/krops/nas/configuration.nix @@ -24,13 +24,26 @@ }; services = { + prometheus.exporters = { + blackbox = { + enable = true; + configFile = "/var/lib/blackbox/config.yml"; + }; + node = { + enable = true; + listenAddress = "100.116.48.47"; + enabledCollectors = [ + "systemd" "zfs" "processes" + ]; + }; + }; coredns = { enable = true; config = '' .:53 { health - file /var/src/dns.db git.gmem.ca + file /var/src/dns.db git.gmem.ca food.gmem.ca forward . 45.90.28.116 45.90.30.116 bind tailscale0 } @@ -201,6 +214,9 @@ federation = { ENABLED = true; }; + metrics = { + ENABLED = true; + }; }; }; gitea-actions-runner = { @@ -215,6 +231,7 @@ "nix:docker://nixos/nix" ]; url = "https://git.gmem.ca/"; + token = "ilTf824bWmTbDtjr0oE2riSzaoUliS54RwjrVDdR"; settings = { cache.port = 4328; }; @@ -268,7 +285,7 @@ trustedInterfaces = ["tailscale0" "virbr0"]; checkReversePath = "loose"; enable = true; - allowedTCPPorts = [ 22 53 80 443 2049 4328 ]; + allowedTCPPorts = [ 22 53 80 443 2049 4328 5432 9100 ]; allowedUDPPorts = [ 53 41641 ]; }; useDHCP = false; @@ -366,6 +383,13 @@ libvirtd.enable = true; }; + virtualisation.oci-containers.containers = { + speedtest = { + image = "ghcr.io/miguelndecarvalho/speedtest-exporter"; + ports = [ "9798:9798" ]; + }; + }; + sound.enable = true; security.rtkit.enable = true;