1
0
Fork 0

Add healthchecks configuration
Lint / lint (push) Successful in 15s Details

trunk
Gabriel Simmer 2023-09-13 23:01:51 +01:00
parent 18941f17e0
commit b9280f2e5c
Signed by: arch
SSH Key Fingerprint: SHA256:m3OEcdtrnBpMX+2BDGh/byv3hrCekCLzDYMdvGEKPPQ
7 changed files with 130 additions and 22 deletions

View File

@ -117,11 +117,11 @@
]
},
"locked": {
"lastModified": 1694134858,
"narHash": "sha256-fG/ESauOGmiojKlpJG8gB62dJa5Wd+ZIuiDMKK/HD3g=",
"lastModified": 1694375657,
"narHash": "sha256-32X8dcty4vPXx+D4yJPQZBo5hJ1NQikALhevGv6elO4=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "19c6a4081b14443420358262f8416149bd79561a",
"rev": "f7848d3e5f15ed02e3f286029697e41ee31662d7",
"type": "github"
},
"original": {
@ -218,11 +218,11 @@
},
"nixpkgs_3": {
"locked": {
"lastModified": 1694062546,
"narHash": "sha256-PiGI4f2BGnZcedP6slLjCLGLRLXPa9+ogGGgVPfGxys=",
"lastModified": 1694343207,
"narHash": "sha256-jWi7OwFxU5Owi4k2JmiL1sa/OuBCQtpaAesuj5LXC8w=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "b200e0df08f80c32974a6108ce431d8a8a5e6547",
"rev": "78058d810644f5ed276804ce7ea9e82d92bee293",
"type": "github"
},
"original": {

View File

@ -106,6 +106,7 @@
system = "aarch64-linux";
modules = [
(import ./nix/monitoring/configuration.nix)
agenix.nixosModules.default
{
_module.args.nixinate = {
host = "monitoring";

View File

@ -4,6 +4,11 @@
./networking.nix # generated at runtime by nixos-infect
];
age.secrets.healthchecks-secret = {
file = ../../secrets/monitoring-healthchecks-secret.age;
owner = "healthchecks";
};
boot.tmp.cleanOnBoot = true;
zramSwap.enable = true;
networking.hostName = "monitoring";
@ -31,16 +36,8 @@
extraFlags = [ "--web.enable-remote-write-receiver" ];
scrapeConfigs = [
{
job_name = "desktop";
static_configs = [ { targets = [ "london:9100" ]; } ];
}
{
job_name = "nas";
static_configs = [ { targets = [ "vancouver:9100" ]; } ];
}
{
job_name = "monitoring";
static_configs = [ { targets = [ "localhost:9100" ]; } ];
job_name = "personal_hardware";
static_configs = [ { targets = [ "london:9100" "vancouver:9100" "localhost:9100" ]; } ];
}
{
job_name = "speedtest-exporter";
@ -48,10 +45,20 @@
scrape_timeout = "1m";
static_configs = [ { targets = [ "vancouver:9798" ]; } ];
}
{
job_name = "syncthing";
static_configs = [ { targets = [ "vancouver:8384" "london:8384" ]; } ];
}
{
job_name = "forgejo";
static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
}
{
job_name = "healthchecks";
scrape_interval = "60s";
metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/qyitrbccSwyuvZEISGdBHSBQaEwLcaFu";
static_configs = [ { targets = [ "localhost:8000" ]; } ];
}
{
job_name = "blackbox";
metrics_path = "/probe";
@ -72,6 +79,14 @@
};
};
services.tailscale.enable = true;
services.healthchecks = {
enable = true;
settings = {
SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
SITE_ROOT = "https://healthchecks.gmem.ca";
};
};
# nginx reverse proxy
services.nginx = {
@ -82,13 +97,20 @@
recommendedOptimisation = true;
recommendedTlsSettings = true;
recommendedProxySettings = true;
virtualHosts.${config.services.grafana.domain} = {
virtualHosts.${config.services.grafana.settings.server.domain} = {
default = true;
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
proxyWebsockets = true;
};
};
virtualHosts."healthchecks.gmem.ca" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://127.0.0.1:8000";
proxyWebsockets = true;
};
};

View File

@ -34,6 +34,11 @@ in
group = "users";
mode = "770";
};
age.secrets.healthcheck-ping = {
file = ../../secrets/healthchecks-ping.sh.age;
group = "users";
mode = "770";
};
nix = {
settings = {
auto-optimise-store = true;
@ -62,13 +67,27 @@ in
"/Primary/gabriel/projects"
"/Primary/gabriel/org"
"/Primary/gabriel/Backup/Pictures"
"/Primary/gabriel/Photos"
"/Primary/shared"
];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
RandomizedDelaySec = "5h";
RandomizedDelaySec = "6h";
};
pruneOpts = [
"--keep-daily 7"
"--keep-weekly 5"
"--keep-monthly 12"
"--keep-yearly 75"
];
passwordFile = config.age.secrets.restic-password.path;
backupPrepareCommand = ''
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup/start'
'';
backupCleanupCommand = ''
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/gsimmer-backup'
'';
};
"becki" = {
user = "becki";
@ -80,8 +99,48 @@ in
timerConfig = {
OnCalendar = "daily";
Persistent = true;
RandomizedDelaySec = "5h";
RandomizedDelaySec = "6h";
};
pruneOpts = [
"--keep-daily 7"
"--keep-weekly 5"
"--keep-monthly 12"
"--keep-yearly 75"
];
passwordFile = config.age.secrets.restic-password.path;
initialize = true;
backupPrepareCommand = ''
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup/start'
'';
backupCleanupCommand = ''
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/becki-backup'
'';
};
"apps" = {
user = "root";
environmentFile = config.age.secrets.restic-b2-credentials.path;
repository = "s3:s3.us-west-000.backblazeb2.com/gsimmer-app-backup";
paths = [
"/Primary/k3scluster"
"/Primary/gitea"
];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
RandomizedDelaySec = "12h";
};
pruneOpts = [
"--keep-daily 7"
"--keep-weekly 5"
"--keep-monthly 12"
"--keep-yearly 75"
];
backupPrepareCommand = ''
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup/start'
'';
backupCleanupCommand = ''
${pkgs.bash}/bin/bash -c '${pkgs.curl}/bin/curl -fsS -m 10 --retry 5 -o /dev/null $(${pkgs.coreutils}/bin/cat ${config.age.secrets.healthcheck-ping.path})/apps-backup'
'';
passwordFile = config.age.secrets.restic-password.path;
initialize = true;
};

View File

@ -1,6 +1,7 @@
let
vancouver = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC30kDkktiWJGaJEIrqgneQx1SmIYUEVPm7w1F9p//CB";
machines = [ vancouver ];
monitoring = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEDtzsbxKgZ/NBYlYO2EJQZhBy3nVBVERWebbsP9mLcy";
machines = [ vancouver monitoring ];
gsimmer = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIztwQxt+jqroFONSgq+xzPMuE2I5Dq/zWPQ8RcTYJr gabriel@gitgalaxy.com";
users = [ gsimmer ];
@ -9,4 +10,7 @@ in
"secrets/vancouver-action-runner.age".publicKeys = [ vancouver gsimmer ];
"secrets/vancouver-restic-b2.age".publicKeys = [ vancouver gsimmer ];
"secrets/vancouver-restic-password.age".publicKeys = [ vancouver gsimmer ];
"secrets/monitoring-healthchecks-secret.age".publicKeys = [ monitoring gsimmer ];
"secrets/healthchecks-ping.sh.age".publicKeys = machines ++ users;
}

View File

@ -0,0 +1,11 @@
age-encryption.org/v1
-> ssh-ed25519 oN6OTQ vxYv5Y/7huGJzb4wW2I0BtHvRRxq5CyNfwDD++xIYn4
LH2I/s1OvXgaQJy6+gu7qvB6gxRZb6sQmyS8D3HcuaA
-> ssh-ed25519 J+a91w J0wCpPEStRdvvBFIlgabxjqUEuimFwclUXzPJEaX7UI
KsrVMVyIEKbCKUKEglGZq5Xx0uPa1M8+wknZMlr3TBU
-> ssh-ed25519 qbziOw uCnSeIqXoc7W9fqqkykWfgNVI8tC3JDP0a/CijnALw4
R0qBsH6JDV9q6yIgBzeOOhaZOR4tficWmMm/hTclqxw
-> d|h-grease !k-" ["gH/ 8o27gAs
/o/XVGDK2ObI67QHgwKZywXVOblMAgDTnPtigKAW3qpi8xB6
--- vbYaWEYM88qenBbbgeSoExaW/rYj0EqJgPHoUEAhuws
òÀ` @ èÏb7Ô<  ¹v5¨}ˆ`YùïÊ•o#éí袕š‡¼òÑNøçìEIèècÓmÛ(*aÓ”ï“ B‰`•Â†<C382>q¤ô$$

View File

@ -0,0 +1,11 @@
age-encryption.org/v1
-> ssh-ed25519 J+a91w RgOoAtLvK4dWBSkVb/x5SsFly+iZ97spe5DsecEfPTY
t6+YkVuRfsP+2gFRH8vdtm+i43B3ynb3Q5ZYhVEC8fE
-> ssh-ed25519 qbziOw HcPjy1TabEYEV2mY9jywSIfZKYlGPYRY55t/4BVQQ1A
TFHyFZ8iJXBo4/YqFq6FP7P7I0Q5vwcC30l3PCVx3Ik
-> WwaHgi3'-grease Wnu
HKx04wWgJ9ilCeFZ+4Ug1cf0b75WantwoQt4P2vPeF21w9Q3box+j71hfdjKyw5K
pJbkinzkeZqnxi35bbNil+/AtSOdbbgYyRAOx3z8JOcOu5zHh724
--- 1STXaZFxI1AGWzz/86QSos4U5WIG+nct8/BqSZq6bjs
¯33(ž ëCbPŽ<50>9nü¦ ƒ
¬Þ`<60>Ü11ÌWŠØ™D2H\,Ÿy·ÆPy3hPÓ ~