Compare commits
2 commits
9202371681
...
b9280f2e5c
Author | SHA1 | Date | |
---|---|---|---|
Gabriel Simmer | b9280f2e5c | ||
Gabriel Simmer | 18941f17e0 |
12
flake.lock
12
flake.lock
|
@ -117,11 +117,11 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1694134858,
|
"lastModified": 1694375657,
|
||||||
"narHash": "sha256-fG/ESauOGmiojKlpJG8gB62dJa5Wd+ZIuiDMKK/HD3g=",
|
"narHash": "sha256-32X8dcty4vPXx+D4yJPQZBo5hJ1NQikALhevGv6elO4=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "home-manager",
|
"repo": "home-manager",
|
||||||
"rev": "19c6a4081b14443420358262f8416149bd79561a",
|
"rev": "f7848d3e5f15ed02e3f286029697e41ee31662d7",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -218,11 +218,11 @@
|
||||||
},
|
},
|
||||||
"nixpkgs_3": {
|
"nixpkgs_3": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1694062546,
|
"lastModified": 1694343207,
|
||||||
"narHash": "sha256-PiGI4f2BGnZcedP6slLjCLGLRLXPa9+ogGGgVPfGxys=",
|
"narHash": "sha256-jWi7OwFxU5Owi4k2JmiL1sa/OuBCQtpaAesuj5LXC8w=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "b200e0df08f80c32974a6108ce431d8a8a5e6547",
|
"rev": "78058d810644f5ed276804ce7ea9e82d92bee293",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
|
|
@ -106,6 +106,7 @@
|
||||||
system = "aarch64-linux";
|
system = "aarch64-linux";
|
||||||
modules = [
|
modules = [
|
||||||
(import ./nix/monitoring/configuration.nix)
|
(import ./nix/monitoring/configuration.nix)
|
||||||
|
agenix.nixosModules.default
|
||||||
{
|
{
|
||||||
_module.args.nixinate = {
|
_module.args.nixinate = {
|
||||||
host = "monitoring";
|
host = "monitoring";
|
||||||
|
|
|
@ -1,4 +1,24 @@
|
||||||
{ config, pkgs, ... }:
|
{ config, pkgs, ... }:
|
||||||
|
let
|
||||||
|
|
||||||
|
syncthingLatest =
|
||||||
|
let
|
||||||
|
version = "1.24.0";
|
||||||
|
src = pkgs.fetchFromGitHub {
|
||||||
|
owner = "syncthing";
|
||||||
|
repo = "syncthing";
|
||||||
|
rev = "v1.24.0";
|
||||||
|
hash = "sha256-5vr9qWMHBYpu8wHpV1JZcX1kEPi+mYeZ7ZQBqXASp9I=";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
(pkgs.syncthing.override rec {
|
||||||
|
buildGoModule = args: pkgs.buildGoModule.override {} (args // {
|
||||||
|
inherit src version;
|
||||||
|
vendorHash = "sha256-BZwZ6npmWFU0lvynjRZOBOhtxqic0djoSUdCOLbUwjE=";
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
in
|
||||||
{
|
{
|
||||||
imports =
|
imports =
|
||||||
[
|
[
|
||||||
|
@ -70,6 +90,16 @@
|
||||||
i18n.defaultLocale = "en_GB.utf8";
|
i18n.defaultLocale = "en_GB.utf8";
|
||||||
|
|
||||||
services = {
|
services = {
|
||||||
|
syncthing = {
|
||||||
|
enable = true;
|
||||||
|
overrideDevices = false;
|
||||||
|
overrideFolders = false;
|
||||||
|
user = "gsimmer";
|
||||||
|
dataDir = "/home/gsimmer";
|
||||||
|
guiAddress = "100.95.77.62:8384";
|
||||||
|
package = syncthingLatest;
|
||||||
|
};
|
||||||
|
usbmuxd.enable = true;
|
||||||
prometheus.exporters.node = {
|
prometheus.exporters.node = {
|
||||||
enable = true;
|
enable = true;
|
||||||
listenAddress = "100.95.77.62";
|
listenAddress = "100.95.77.62";
|
||||||
|
|
|
@ -4,6 +4,11 @@
|
||||||
./networking.nix # generated at runtime by nixos-infect
|
./networking.nix # generated at runtime by nixos-infect
|
||||||
];
|
];
|
||||||
|
|
||||||
|
age.secrets.healthchecks-secret = {
|
||||||
|
file = ../../secrets/monitoring-healthchecks-secret.age;
|
||||||
|
owner = "healthchecks";
|
||||||
|
};
|
||||||
|
|
||||||
boot.tmp.cleanOnBoot = true;
|
boot.tmp.cleanOnBoot = true;
|
||||||
zramSwap.enable = true;
|
zramSwap.enable = true;
|
||||||
networking.hostName = "monitoring";
|
networking.hostName = "monitoring";
|
||||||
|
@ -31,16 +36,8 @@
|
||||||
extraFlags = [ "--web.enable-remote-write-receiver" ];
|
extraFlags = [ "--web.enable-remote-write-receiver" ];
|
||||||
scrapeConfigs = [
|
scrapeConfigs = [
|
||||||
{
|
{
|
||||||
job_name = "desktop";
|
job_name = "personal_hardware";
|
||||||
static_configs = [ { targets = [ "london:9100" ]; } ];
|
static_configs = [ { targets = [ "london:9100" "vancouver:9100" "localhost:9100" ]; } ];
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "nas";
|
|
||||||
static_configs = [ { targets = [ "vancouver:9100" ]; } ];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "monitoring";
|
|
||||||
static_configs = [ { targets = [ "localhost:9100" ]; } ];
|
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
job_name = "speedtest-exporter";
|
job_name = "speedtest-exporter";
|
||||||
|
@ -48,10 +45,20 @@
|
||||||
scrape_timeout = "1m";
|
scrape_timeout = "1m";
|
||||||
static_configs = [ { targets = [ "vancouver:9798" ]; } ];
|
static_configs = [ { targets = [ "vancouver:9798" ]; } ];
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
job_name = "syncthing";
|
||||||
|
static_configs = [ { targets = [ "vancouver:8384" "london:8384" ]; } ];
|
||||||
|
}
|
||||||
{
|
{
|
||||||
job_name = "forgejo";
|
job_name = "forgejo";
|
||||||
static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
|
static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
job_name = "healthchecks";
|
||||||
|
scrape_interval = "60s";
|
||||||
|
metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/qyitrbccSwyuvZEISGdBHSBQaEwLcaFu";
|
||||||
|
static_configs = [ { targets = [ "localhost:8000" ]; } ];
|
||||||
|
}
|
||||||
{
|
{
|
||||||
job_name = "blackbox";
|
job_name = "blackbox";
|
||||||
metrics_path = "/probe";
|
metrics_path = "/probe";
|
||||||
|
@ -72,6 +79,14 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
services.tailscale.enable = true;
|
services.tailscale.enable = true;
|
||||||
|
|
||||||
|
services.healthchecks = {
|
||||||
|
enable = true;
|
||||||
|
settings = {
|
||||||
|
SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
|
||||||
|
SITE_ROOT = "https://healthchecks.gmem.ca";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
# nginx reverse proxy
|
# nginx reverse proxy
|
||||||
services.nginx = {
|
services.nginx = {
|
||||||
|
@ -82,13 +97,20 @@
|
||||||
recommendedOptimisation = true;
|
recommendedOptimisation = true;
|
||||||
recommendedTlsSettings = true;
|
recommendedTlsSettings = true;
|
||||||
recommendedProxySettings = true;
|
recommendedProxySettings = true;
|
||||||
virtualHosts.${config.services.grafana.domain} = {
|
virtualHosts.${config.services.grafana.settings.server.domain} = {
|
||||||
default = true;
|
default = true;
|
||||||
enableACME = true;
|
enableACME = true;
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations."/" = {
|
locations."/" = {
|
||||||
|
proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
|
||||||
proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
|
proxyWebsockets = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
virtualHosts."healthchecks.gmem.ca" = {
|
||||||
|
enableACME = true;
|
||||||
|
forceSSL = true;
|
||||||
|
locations."/" = {
|
||||||
|
proxyPass = "http://127.0.0.1:8000";
|
||||||
proxyWebsockets = true;
|
proxyWebsockets = true;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,5 +1,22 @@
|
||||||
{ config, pkgs, ... }:
|
{ config, pkgs, ... }:
|
||||||
|
let
|
||||||
|
syncthingLatest =
|
||||||
|
let
|
||||||
|
version = "1.24.0";
|
||||||
|
src = pkgs.fetchFromGitHub {
|
||||||
|
owner = "syncthing";
|
||||||
|
repo = "syncthing";
|
||||||
|
rev = "v1.24.0";
|
||||||
|
hash = "sha256-5vr9qWMHBYpu8wHpV1JZcX1kEPi+mYeZ7ZQBqXASp9I=";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
(pkgs.syncthing.override rec {
|
||||||
|
buildGoModule = args: pkgs.buildGoModule.override {} (args // {
|
||||||
|
inherit src version;
|
||||||
|
vendorHash = "sha256-BZwZ6npmWFU0lvynjRZOBOhtxqic0djoSUdCOLbUwjE=";
|
||||||
|
});
|
||||||
|
});
|
||||||
|
in
|
||||||
{
|
{
|
||||||
imports =
|
imports =
|
||||||
[ # Include the results of the hardware scan.
|
[ # Include the results of the hardware scan.
|
||||||
|
@ -17,6 +34,11 @@
|
||||||
group = "users";
|
group = "users";
|
||||||
mode = "770";
|
mode = "770";
|
||||||
};
|
};
|
||||||
|
age.secrets.healthcheck-ping = {
|
||||||
|
file = ../../secrets/healthchecks-ping.sh.age;
|
||||||
|
group = "users";
|
||||||
|
mode = "770";
|
||||||
|
};
|
||||||
nix = {
|
nix = {
|
||||||
settings = {
|
settings = {
|
||||||
auto-optimise-store = true;
|
auto-optimise-store = true;
|
||||||
|
@ -45,13 +67,27 @@
|
||||||
"/Primary/gabriel/projects"
|
"/Primary/gabriel/projects"
|
||||||
"/Primary/gabriel/org"
|
"/Primary/gabriel/org"
|
||||||
"/Primary/gabriel/Backup/Pictures"
|
"/Primary/gabriel/Backup/Pictures"
|
||||||
|
"/Primary/gabriel/Photos"
|
||||||
|
"/Primary/shared"
|
||||||
];
|
];
|
||||||
timerConfig = {
|
timerConfig = {
|
||||||
OnCalendar = "daily";
|
OnCalendar = "daily";
|
||||||
Persistent = true;
|
Persistent = true;
|
||||||
RandomizedDelaySec = "5h";
|
RandomizedDelaySec = "6h";
|
||||||
};
|
};
|
||||||
|
pruneOpts = [
|
||||||
|
"--keep-daily 7"
|
||||||
|
"--keep-weekly 5"
|
||||||
|
"--keep-monthly 12"
|
||||||
|
"--keep-yearly 75"
|
||||||
|
];
|
||||||
passwordFile = config.age.secrets.restic-password.path;
|
passwordFile = config.age.secrets.restic-password.path;
|
||||||
|
backupPrepareCommand = ''
|
||||||
|
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup/start'
|
||||||
|
'';
|
||||||
|
backupCleanupCommand = ''
|
||||||
|
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup'
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
"becki" = {
|
"becki" = {
|
||||||
user = "becki";
|
user = "becki";
|
||||||
|
@ -63,8 +99,48 @@
|
||||||
timerConfig = {
|
timerConfig = {
|
||||||
OnCalendar = "daily";
|
OnCalendar = "daily";
|
||||||
Persistent = true;
|
Persistent = true;
|
||||||
RandomizedDelaySec = "5h";
|
RandomizedDelaySec = "6h";
|
||||||
};
|
};
|
||||||
|
pruneOpts = [
|
||||||
|
"--keep-daily 7"
|
||||||
|
"--keep-weekly 5"
|
||||||
|
"--keep-monthly 12"
|
||||||
|
"--keep-yearly 75"
|
||||||
|
];
|
||||||
|
passwordFile = config.age.secrets.restic-password.path;
|
||||||
|
initialize = true;
|
||||||
|
backupPrepareCommand = ''
|
||||||
|
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup/start'
|
||||||
|
'';
|
||||||
|
backupCleanupCommand = ''
|
||||||
|
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup'
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
"apps" = {
|
||||||
|
user = "root";
|
||||||
|
environmentFile = config.age.secrets.restic-b2-credentials.path;
|
||||||
|
repository = "s3:s3.us-west-000.backblazeb2.com/gsimmer-app-backup";
|
||||||
|
paths = [
|
||||||
|
"/Primary/k3scluster"
|
||||||
|
"/Primary/gitea"
|
||||||
|
];
|
||||||
|
timerConfig = {
|
||||||
|
OnCalendar = "daily";
|
||||||
|
Persistent = true;
|
||||||
|
RandomizedDelaySec = "12h";
|
||||||
|
};
|
||||||
|
pruneOpts = [
|
||||||
|
"--keep-daily 7"
|
||||||
|
"--keep-weekly 5"
|
||||||
|
"--keep-monthly 12"
|
||||||
|
"--keep-yearly 75"
|
||||||
|
];
|
||||||
|
backupPrepareCommand = ''
|
||||||
|
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup/start'
|
||||||
|
'';
|
||||||
|
backupCleanupCommand = ''
|
||||||
|
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup'
|
||||||
|
'';
|
||||||
passwordFile = config.age.secrets.restic-password.path;
|
passwordFile = config.age.secrets.restic-password.path;
|
||||||
initialize = true;
|
initialize = true;
|
||||||
};
|
};
|
||||||
|
@ -77,6 +153,7 @@
|
||||||
user = "gsimmer";
|
user = "gsimmer";
|
||||||
dataDir = "/Primary/gabriel";
|
dataDir = "/Primary/gabriel";
|
||||||
guiAddress = "100.116.48.47:8384";
|
guiAddress = "100.116.48.47:8384";
|
||||||
|
package = syncthingLatest;
|
||||||
};
|
};
|
||||||
prometheus.exporters = {
|
prometheus.exporters = {
|
||||||
blackbox = {
|
blackbox = {
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
let
|
let
|
||||||
vancouver = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC30kDkktiWJGaJEIrqgneQx1SmIYUEVPm7w1F9p//CB";
|
vancouver = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC30kDkktiWJGaJEIrqgneQx1SmIYUEVPm7w1F9p//CB";
|
||||||
machines = [ vancouver ];
|
monitoring = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEDtzsbxKgZ/NBYlYO2EJQZhBy3nVBVERWebbsP9mLcy";
|
||||||
|
machines = [ vancouver monitoring ];
|
||||||
|
|
||||||
gsimmer = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIztwQxt+jqroFONSgq+xzPMuE2I5Dq/zWPQ8RcTYJr gabriel@gitgalaxy.com";
|
gsimmer = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIztwQxt+jqroFONSgq+xzPMuE2I5Dq/zWPQ8RcTYJr gabriel@gitgalaxy.com";
|
||||||
users = [ gsimmer ];
|
users = [ gsimmer ];
|
||||||
|
@ -9,4 +10,7 @@ in
|
||||||
"secrets/vancouver-action-runner.age".publicKeys = [ vancouver gsimmer ];
|
"secrets/vancouver-action-runner.age".publicKeys = [ vancouver gsimmer ];
|
||||||
"secrets/vancouver-restic-b2.age".publicKeys = [ vancouver gsimmer ];
|
"secrets/vancouver-restic-b2.age".publicKeys = [ vancouver gsimmer ];
|
||||||
"secrets/vancouver-restic-password.age".publicKeys = [ vancouver gsimmer ];
|
"secrets/vancouver-restic-password.age".publicKeys = [ vancouver gsimmer ];
|
||||||
|
"secrets/monitoring-healthchecks-secret.age".publicKeys = [ monitoring gsimmer ];
|
||||||
|
|
||||||
|
"secrets/healthchecks-ping.sh.age".publicKeys = machines ++ users;
|
||||||
}
|
}
|
||||||
|
|
11
secrets/healthchecks-ping.sh.age
Normal file
11
secrets/healthchecks-ping.sh.age
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
age-encryption.org/v1
|
||||||
|
-> ssh-ed25519 oN6OTQ vxYv5Y/7huGJzb4wW2I0BtHvRRxq5CyNfwDD++xIYn4
|
||||||
|
LH2I/s1OvXgaQJy6+gu7qvB6gxRZb6sQmyS8D3HcuaA
|
||||||
|
-> ssh-ed25519 J+a91w J0wCpPEStRdvvBFIlgabxjqUEuimFwclUXzPJEaX7UI
|
||||||
|
KsrVMVyIEKbCKUKEglGZq5Xx0uPa1M8+wknZMlr3TBU
|
||||||
|
-> ssh-ed25519 qbziOw uCnSeIqXoc7W9fqqkykWfgNVI8tC3JDP0a/CijnALw4
|
||||||
|
R0qBsH6JDV9q6yIgBzeOOhaZOR4tficWmMm/hTclqxw
|
||||||
|
-> d|h-grease !k-" ["gH/ 8o27gAs
|
||||||
|
/o/XVGDK2ObI67QHgwKZywXVOblMAgDTnPtigKAW3qpi8xB6
|
||||||
|
--- vbYaWEYM88qenBbbgeSoExaW/rYj0EqJgPHoUEAhuws
|
||||||
|
òcÝÀ` @èÏb7Ô<
¹v5¨}ˆ`YùïÊ•o€#éí袕š‡¼òÑNøçìEIèècÓmÛ(*aÓ”ï“ B‰`•Â†<C382>q¤ô$$
|
11
secrets/monitoring-healthchecks-secret.age
Normal file
11
secrets/monitoring-healthchecks-secret.age
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
age-encryption.org/v1
|
||||||
|
-> ssh-ed25519 J+a91w RgOoAtLvK4dWBSkVb/x5SsFly+iZ97spe5DsecEfPTY
|
||||||
|
t6+YkVuRfsP+2gFRH8vdtm+i43B3ynb3Q5ZYhVEC8fE
|
||||||
|
-> ssh-ed25519 qbziOw HcPjy1TabEYEV2mY9jywSIfZKYlGPYRY55t/4BVQQ1A
|
||||||
|
TFHyFZ8iJXBo4/YqFq6FP7P7I0Q5vwcC30l3PCVx3Ik
|
||||||
|
-> WwaHgi3'-grease Wnu
|
||||||
|
HKx04wWgJ9ilCeFZ+4Ug1cf0b75WantwoQt4P2vPeF21w9Q3box+j71hfdjKyw5K
|
||||||
|
pJbkinzkeZqnxi35bbNil+/AtSOdbbgYyRAOx3z8JOcOu5zHh724
|
||||||
|
--- 1STXaZFxI1AGWzz/86QSos4U5WIG+nct8/BqSZq6bjs
|
||||||
|
¯33(žëCbPŽ<50>9nü¦ƒ
|
||||||
|
¬Þ`<60>Ü11ÌWŠØ™D‚2H\,Ÿy·ÆPy3hPÓ ~
|
Loading…
Reference in a new issue