2023-09-05 17:08:43 +01:00
|
|
|
{ config, pkgs, ... }: {
|
|
|
|
imports = [
|
|
|
|
./hardware.nix
|
|
|
|
./networking.nix # generated at runtime by nixos-infect
|
|
|
|
];
|
|
|
|
|
2023-09-13 23:01:51 +01:00
|
|
|
age.secrets.healthchecks-secret = {
|
|
|
|
file = ../../secrets/monitoring-healthchecks-secret.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
|
|
|
|
2023-09-20 00:25:38 +01:00
|
|
|
age.secrets.healthchecks-smtp = {
|
|
|
|
file = ../../secrets/fastmail-smtp.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
2023-09-15 09:21:50 +01:00
|
|
|
age.secrets.prometheus-webconfig-secret = {
|
|
|
|
file = ../../secrets/monitoring-prometheus-webconfig.age;
|
|
|
|
owner = "prometheus";
|
2023-09-19 23:59:44 +01:00
|
|
|
mode = "775";
|
|
|
|
};
|
|
|
|
|
|
|
|
age.secrets.prometheus-password-secret = {
|
|
|
|
file = ../../secrets/monitoring-prometheus-password.age;
|
|
|
|
owner = "prometheus";
|
2023-09-15 09:21:50 +01:00
|
|
|
};
|
|
|
|
|
2023-09-05 17:08:43 +01:00
|
|
|
boot.tmp.cleanOnBoot = true;
|
|
|
|
zramSwap.enable = true;
|
|
|
|
networking.hostName = "monitoring";
|
|
|
|
networking.domain = "";
|
|
|
|
services.openssh.enable = true;
|
|
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
|
|
|
''ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjEgtIWPA5Ncs/KOcMeT6Q/HACJJetDOLjMvXXwUE+08oTX1EpHrWPpy8J+UHKIyErCNPYq8dgtrbhnMRlxHqI=''
|
|
|
|
];
|
|
|
|
networking.firewall.enable = false;
|
|
|
|
|
|
|
|
services.grafana = {
|
|
|
|
enable = true;
|
|
|
|
settings.server = {
|
|
|
|
domain = "grafana.gmem.ca";
|
|
|
|
http_port = 2342;
|
|
|
|
http_addr = "127.0.0.1";
|
|
|
|
};
|
|
|
|
};
|
2023-09-19 23:59:44 +01:00
|
|
|
services.alertmanager-ntfy = {
|
|
|
|
enable = true;
|
|
|
|
settings = {
|
|
|
|
http = {
|
|
|
|
addr = "127.0.0.1:8111";
|
|
|
|
};
|
|
|
|
ntfy = {
|
|
|
|
baseurl = "https://ntfy.gmem.ca";
|
|
|
|
notification = {
|
|
|
|
topic = "alerts";
|
|
|
|
priority = ''
|
|
|
|
status == "firing" ? "high" : "default"
|
2023-09-20 00:25:38 +01:00
|
|
|
'';
|
2023-09-19 23:59:44 +01:00
|
|
|
templates = {
|
|
|
|
title = ''{{ if eq .Status "resolved" }}Resolved: {{ end }}{{ index .Annotations "summary" }}'';
|
|
|
|
description = ''{{ index .Annotations "description" }}'';
|
|
|
|
click = ''http://grafana.gmem.ca/d/{{ index .Annotations "dashboard" }}'';
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
services.prometheus = {
|
|
|
|
enable = true;
|
2023-09-15 09:21:50 +01:00
|
|
|
webConfigFile = config.age.secrets.prometheus-webconfig-secret.path;
|
2023-09-05 17:08:43 +01:00
|
|
|
globalConfig = {
|
|
|
|
scrape_interval = "15s";
|
|
|
|
};
|
2023-09-19 23:59:44 +01:00
|
|
|
alertmanagers = [ {
|
|
|
|
basic_auth = {
|
|
|
|
username = "homelab";
|
|
|
|
password_file = config.age.secrets.prometheus-password-secret.path;
|
|
|
|
};
|
|
|
|
static_configs = [ {
|
|
|
|
targets = [
|
|
|
|
"localhost:9093"
|
|
|
|
];
|
|
|
|
} ];
|
|
|
|
} ];
|
|
|
|
rules = [(builtins.toJSON {
|
|
|
|
groups = [{
|
|
|
|
name = "healthchecks";
|
|
|
|
rules = [
|
|
|
|
{
|
|
|
|
alert = "HealthcheckFailedCheckin";
|
|
|
|
expr = ''hc_check_up < 1'';
|
|
|
|
for = "5m";
|
|
|
|
labels.severity = "page";
|
|
|
|
annotations = {
|
|
|
|
summary = "{{ $labels.name }} healthcheck failed";
|
|
|
|
description = "The {{ $labels.name }} healthcheck failed to check in.";
|
|
|
|
dashboard = "f594ea85-45f2-4019-b988-2d17638b5cf3";
|
|
|
|
};
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}];
|
|
|
|
})];
|
|
|
|
alertmanager = {
|
|
|
|
enable = true;
|
|
|
|
extraFlags = [ "--web.config.file=${config.age.secrets.prometheus-webconfig-secret.path}" ];
|
|
|
|
webExternalUrl = "https://alerts.gmem.ca";
|
|
|
|
configText = ''
|
|
|
|
global: {}
|
|
|
|
|
|
|
|
# The directory from which notification templates are read.
|
|
|
|
templates:
|
|
|
|
- '/etc/alertmanager/template/*.tmpl'
|
|
|
|
|
|
|
|
# The root route on which each incoming alert enters.
|
|
|
|
route:
|
|
|
|
group_by: ['alertname', 'cluster', 'service']
|
|
|
|
group_wait: 0s
|
|
|
|
group_interval: 5m
|
|
|
|
repeat_interval: 3h
|
|
|
|
# A default receiver
|
|
|
|
receiver: ntfy
|
|
|
|
|
|
|
|
receivers:
|
|
|
|
- name: ntfy
|
|
|
|
webhook_configs:
|
|
|
|
- url: http://localhost:8111/hook
|
|
|
|
'';
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
port = 9001;
|
|
|
|
extraFlags = [ "--web.enable-remote-write-receiver" ];
|
|
|
|
scrapeConfigs = [
|
|
|
|
{
|
2023-09-13 23:01:51 +01:00
|
|
|
job_name = "personal_hardware";
|
|
|
|
static_configs = [ { targets = [ "london:9100" "vancouver:9100" "localhost:9100" ]; } ];
|
2023-09-05 17:08:43 +01:00
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "speedtest-exporter";
|
|
|
|
scrape_interval = "1h";
|
|
|
|
scrape_timeout = "1m";
|
|
|
|
static_configs = [ { targets = [ "vancouver:9798" ]; } ];
|
|
|
|
}
|
2023-09-13 23:01:51 +01:00
|
|
|
{
|
|
|
|
job_name = "syncthing";
|
|
|
|
static_configs = [ { targets = [ "vancouver:8384" "london:8384" ]; } ];
|
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
|
|
|
job_name = "forgejo";
|
|
|
|
static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
|
|
|
|
}
|
2023-09-15 09:22:02 +01:00
|
|
|
{
|
|
|
|
job_name = "coredns";
|
|
|
|
static_configs = [ { targets = [ "vancouver:9253" ]; } ];
|
|
|
|
}
|
2023-09-13 23:01:51 +01:00
|
|
|
{
|
|
|
|
job_name = "healthchecks";
|
|
|
|
scrape_interval = "60s";
|
2023-09-13 23:17:37 +01:00
|
|
|
metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/TbMoU7SUdknzMe-H5Q4HzmKl3itOIrJk";
|
2023-09-13 23:01:51 +01:00
|
|
|
static_configs = [ { targets = [ "localhost:8000" ]; } ];
|
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
|
|
|
job_name = "blackbox";
|
|
|
|
metrics_path = "/probe";
|
|
|
|
params = { "modules" = [ "http_2xx" ]; };
|
|
|
|
static_configs = [ { targets = [ "google.com" "gabrielsimmer.com" "artbybecki.com" ]; } ];
|
|
|
|
relabel_configs = [
|
|
|
|
{ source_labels = ["__address__"]; target_label = "__param_target"; }
|
|
|
|
{ source_labels = ["__param_target"]; target_label = "instance"; }
|
|
|
|
{ source_labels = []; target_label = "__address__"; replacement = "vancouver:9115"; } ];
|
|
|
|
}
|
|
|
|
];
|
|
|
|
exporters.node = {
|
|
|
|
enable = true;
|
|
|
|
listenAddress = "127.0.0.1";
|
|
|
|
enabledCollectors = [
|
|
|
|
"systemd" "processes"
|
|
|
|
];
|
|
|
|
};
|
|
|
|
};
|
|
|
|
services.tailscale.enable = true;
|
2023-09-13 23:01:51 +01:00
|
|
|
|
|
|
|
services.healthchecks = {
|
|
|
|
enable = true;
|
|
|
|
settings = {
|
|
|
|
SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
|
|
|
|
SITE_ROOT = "https://healthchecks.gmem.ca";
|
2023-09-13 23:17:37 +01:00
|
|
|
SITE_NAME = "Arch's Healthchecks";
|
2023-09-20 00:25:38 +01:00
|
|
|
EMAIL_HOST = "smtp.fastmail.com";
|
|
|
|
EMAIL_HOST_PASSWORD_FILE = config.age.secrets.healthchecks-smtp.path;
|
|
|
|
EMAIL_HOST_USER = "g@gmem.ca";
|
|
|
|
DEFAULT_FROM_EMAIL = "healthchecks@gmem.ca";
|
2023-09-13 23:01:51 +01:00
|
|
|
};
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
|
|
|
|
# nginx reverse proxy
|
|
|
|
services.nginx = {
|
|
|
|
enable = true;
|
|
|
|
recommendedGzipSettings = true;
|
|
|
|
recommendedBrotliSettings = true;
|
|
|
|
recommendedZstdSettings = true;
|
|
|
|
recommendedOptimisation = true;
|
|
|
|
recommendedTlsSettings = true;
|
|
|
|
recommendedProxySettings = true;
|
2023-09-13 23:01:51 +01:00
|
|
|
virtualHosts.${config.services.grafana.settings.server.domain} = {
|
2023-09-05 17:08:43 +01:00
|
|
|
default = true;
|
|
|
|
enableACME = true;
|
|
|
|
forceSSL = true;
|
|
|
|
locations."/" = {
|
2023-09-13 23:01:51 +01:00
|
|
|
proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
|
|
|
|
proxyWebsockets = true;
|
|
|
|
};
|
|
|
|
};
|
2023-09-15 09:22:02 +01:00
|
|
|
|
2023-09-13 23:01:51 +01:00
|
|
|
virtualHosts."healthchecks.gmem.ca" = {
|
|
|
|
enableACME = true;
|
|
|
|
forceSSL = true;
|
|
|
|
locations."/" = {
|
|
|
|
proxyPass = "http://127.0.0.1:8000";
|
2023-09-05 17:08:43 +01:00
|
|
|
proxyWebsockets = true;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
security.acme.acceptTerms = true;
|
|
|
|
security.acme.defaults.email = "acme@gmem.ca";
|
|
|
|
|
|
|
|
system.stateVersion = "23.11";
|
|
|
|
}
|