2023-09-25 10:48:52 +01:00
|
|
|
{ config, pkgs, ... }:
|
|
|
|
|
|
|
|
{
|
2023-09-05 17:08:43 +01:00
|
|
|
imports = [
|
|
|
|
./hardware.nix
|
|
|
|
./networking.nix # generated at runtime by nixos-infect
|
|
|
|
];
|
|
|
|
|
2023-09-13 23:01:51 +01:00
|
|
|
age.secrets.healthchecks-secret = {
|
|
|
|
file = ../../secrets/monitoring-healthchecks-secret.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
|
|
|
|
2023-09-20 00:25:38 +01:00
|
|
|
age.secrets.healthchecks-smtp = {
|
|
|
|
file = ../../secrets/fastmail-smtp.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
2023-09-25 10:48:52 +01:00
|
|
|
|
|
|
|
age.secrets.healthchecks-telegram = {
|
|
|
|
file = ../../secrets/healthchecks-telegram.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
|
|
|
|
2023-09-15 09:21:50 +01:00
|
|
|
age.secrets.prometheus-webconfig-secret = {
|
|
|
|
file = ../../secrets/monitoring-prometheus-webconfig.age;
|
|
|
|
owner = "prometheus";
|
2023-09-19 23:59:44 +01:00
|
|
|
mode = "775";
|
|
|
|
};
|
|
|
|
|
|
|
|
age.secrets.prometheus-password-secret = {
|
|
|
|
file = ../../secrets/monitoring-prometheus-password.age;
|
|
|
|
owner = "prometheus";
|
2023-09-15 09:21:50 +01:00
|
|
|
};
|
|
|
|
|
2023-09-05 17:08:43 +01:00
|
|
|
boot.tmp.cleanOnBoot = true;
|
|
|
|
zramSwap.enable = true;
|
|
|
|
networking.hostName = "monitoring";
|
|
|
|
networking.domain = "";
|
|
|
|
services.openssh.enable = true;
|
|
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
|
|
|
''ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjEgtIWPA5Ncs/KOcMeT6Q/HACJJetDOLjMvXXwUE+08oTX1EpHrWPpy8J+UHKIyErCNPYq8dgtrbhnMRlxHqI=''
|
|
|
|
];
|
|
|
|
networking.firewall.enable = false;
|
|
|
|
|
|
|
|
services.grafana = {
|
|
|
|
enable = true;
|
2023-10-18 15:48:42 +01:00
|
|
|
settings = {
|
|
|
|
feature_toggles = {
|
|
|
|
publicDashboards = true;
|
|
|
|
};
|
|
|
|
server = {
|
|
|
|
domain = "grafana.gmem.ca";
|
|
|
|
http_port = 2342;
|
|
|
|
http_addr = "127.0.0.1";
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
};
|
|
|
|
};
|
2023-10-08 22:55:20 +01:00
|
|
|
services.loki = {
|
|
|
|
enable = true;
|
|
|
|
configuration = {
|
|
|
|
server.http_listen_port = 3030;
|
|
|
|
auth_enabled = false;
|
|
|
|
|
|
|
|
ingester = {
|
|
|
|
lifecycler = {
|
|
|
|
address = "127.0.0.1";
|
|
|
|
ring = {
|
|
|
|
kvstore = {
|
|
|
|
store = "inmemory";
|
|
|
|
};
|
|
|
|
replication_factor = 1;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
chunk_idle_period = "1h";
|
|
|
|
max_chunk_age = "1h";
|
|
|
|
chunk_target_size = 999999;
|
|
|
|
chunk_retain_period = "30s";
|
|
|
|
max_transfer_retries = 0;
|
|
|
|
};
|
|
|
|
|
|
|
|
schema_config = {
|
|
|
|
configs = [{
|
|
|
|
from = "2022-06-06";
|
|
|
|
store = "boltdb-shipper";
|
|
|
|
object_store = "filesystem";
|
|
|
|
schema = "v11";
|
|
|
|
index = {
|
|
|
|
prefix = "index_";
|
|
|
|
period = "24h";
|
|
|
|
};
|
|
|
|
}];
|
|
|
|
};
|
|
|
|
|
|
|
|
storage_config = {
|
|
|
|
boltdb_shipper = {
|
|
|
|
active_index_directory = "/var/lib/loki/boltdb-shipper-active";
|
|
|
|
cache_location = "/var/lib/loki/boltdb-shipper-cache";
|
|
|
|
cache_ttl = "24h";
|
|
|
|
shared_store = "filesystem";
|
|
|
|
};
|
|
|
|
|
|
|
|
filesystem = {
|
|
|
|
directory = "/var/lib/loki/chunks";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
limits_config = {
|
|
|
|
reject_old_samples = true;
|
|
|
|
reject_old_samples_max_age = "168h";
|
|
|
|
};
|
|
|
|
|
|
|
|
chunk_store_config = {
|
|
|
|
max_look_back_period = "0s";
|
|
|
|
};
|
|
|
|
|
|
|
|
table_manager = {
|
|
|
|
retention_deletes_enabled = false;
|
|
|
|
retention_period = "0s";
|
|
|
|
};
|
|
|
|
|
|
|
|
compactor = {
|
|
|
|
working_directory = "/var/lib/loki";
|
|
|
|
shared_store = "filesystem";
|
|
|
|
compactor_ring = {
|
|
|
|
kvstore = {
|
|
|
|
store = "inmemory";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
services.promtail = {
|
|
|
|
enable = true;
|
|
|
|
configuration = {
|
|
|
|
server = {
|
|
|
|
http_listen_port = 3031;
|
|
|
|
grpc_listen_port = 0;
|
|
|
|
};
|
|
|
|
positions = {
|
|
|
|
filename = "/tmp/positions.yaml";
|
|
|
|
};
|
|
|
|
clients = [{
|
|
|
|
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
|
|
|
|
}];
|
|
|
|
scrape_configs = [{
|
|
|
|
job_name = "journal";
|
|
|
|
journal = {
|
|
|
|
max_age = "12h";
|
|
|
|
labels = {
|
|
|
|
job = "systemd-journal";
|
|
|
|
host = "monitoring";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
relabel_configs = [{
|
|
|
|
source_labels = [ "__journal__systemd_unit" ];
|
|
|
|
target_label = "unit";
|
|
|
|
}];
|
|
|
|
}];
|
|
|
|
};
|
|
|
|
};
|
2023-09-19 23:59:44 +01:00
|
|
|
services.alertmanager-ntfy = {
|
|
|
|
enable = true;
|
|
|
|
settings = {
|
|
|
|
http = {
|
|
|
|
addr = "127.0.0.1:8111";
|
|
|
|
};
|
|
|
|
ntfy = {
|
|
|
|
baseurl = "https://ntfy.gmem.ca";
|
|
|
|
notification = {
|
|
|
|
topic = "alerts";
|
|
|
|
priority = ''
|
|
|
|
status == "firing" ? "high" : "default"
|
2023-09-20 00:25:38 +01:00
|
|
|
'';
|
2023-09-19 23:59:44 +01:00
|
|
|
templates = {
|
|
|
|
title = ''{{ if eq .Status "resolved" }}Resolved: {{ end }}{{ index .Annotations "summary" }}'';
|
|
|
|
description = ''{{ index .Annotations "description" }}'';
|
|
|
|
click = ''http://grafana.gmem.ca/d/{{ index .Annotations "dashboard" }}'';
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
services.prometheus = {
|
|
|
|
enable = true;
|
2023-09-15 09:21:50 +01:00
|
|
|
webConfigFile = config.age.secrets.prometheus-webconfig-secret.path;
|
2023-09-05 17:08:43 +01:00
|
|
|
globalConfig = {
|
|
|
|
scrape_interval = "15s";
|
|
|
|
};
|
2023-09-19 23:59:44 +01:00
|
|
|
alertmanagers = [ {
|
|
|
|
basic_auth = {
|
|
|
|
username = "homelab";
|
|
|
|
password_file = config.age.secrets.prometheus-password-secret.path;
|
|
|
|
};
|
|
|
|
static_configs = [ {
|
|
|
|
targets = [
|
|
|
|
"localhost:9093"
|
|
|
|
];
|
|
|
|
} ];
|
|
|
|
} ];
|
|
|
|
rules = [(builtins.toJSON {
|
|
|
|
groups = [{
|
|
|
|
name = "healthchecks";
|
|
|
|
rules = [
|
|
|
|
{
|
|
|
|
alert = "HealthcheckFailedCheckin";
|
|
|
|
expr = ''hc_check_up < 1'';
|
|
|
|
for = "5m";
|
|
|
|
labels.severity = "page";
|
|
|
|
annotations = {
|
|
|
|
summary = "{{ $labels.name }} healthcheck failed";
|
|
|
|
description = "The {{ $labels.name }} healthcheck failed to check in.";
|
|
|
|
dashboard = "f594ea85-45f2-4019-b988-2d17638b5cf3";
|
|
|
|
};
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}];
|
|
|
|
})];
|
|
|
|
alertmanager = {
|
|
|
|
enable = true;
|
|
|
|
extraFlags = [ "--web.config.file=${config.age.secrets.prometheus-webconfig-secret.path}" ];
|
|
|
|
webExternalUrl = "https://alerts.gmem.ca";
|
|
|
|
configText = ''
|
|
|
|
global: {}
|
|
|
|
|
|
|
|
# The directory from which notification templates are read.
|
|
|
|
templates:
|
|
|
|
- '/etc/alertmanager/template/*.tmpl'
|
|
|
|
|
|
|
|
# The root route on which each incoming alert enters.
|
|
|
|
route:
|
|
|
|
group_by: ['alertname', 'cluster', 'service']
|
|
|
|
group_wait: 0s
|
|
|
|
group_interval: 5m
|
|
|
|
repeat_interval: 3h
|
|
|
|
# A default receiver
|
|
|
|
receiver: ntfy
|
|
|
|
|
|
|
|
receivers:
|
|
|
|
- name: ntfy
|
|
|
|
webhook_configs:
|
|
|
|
- url: http://localhost:8111/hook
|
|
|
|
'';
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
port = 9001;
|
|
|
|
extraFlags = [ "--web.enable-remote-write-receiver" ];
|
|
|
|
scrapeConfigs = [
|
|
|
|
{
|
2023-09-13 23:01:51 +01:00
|
|
|
job_name = "personal_hardware";
|
|
|
|
static_configs = [ { targets = [ "london:9100" "vancouver:9100" "localhost:9100" ]; } ];
|
2023-09-05 17:08:43 +01:00
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "speedtest-exporter";
|
|
|
|
scrape_interval = "1h";
|
|
|
|
scrape_timeout = "1m";
|
|
|
|
static_configs = [ { targets = [ "vancouver:9798" ]; } ];
|
|
|
|
}
|
2023-09-13 23:01:51 +01:00
|
|
|
{
|
|
|
|
job_name = "syncthing";
|
|
|
|
static_configs = [ { targets = [ "vancouver:8384" "london:8384" ]; } ];
|
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
|
|
|
job_name = "forgejo";
|
|
|
|
static_configs = [ { targets = [ "git.gmem.ca" ]; } ];
|
|
|
|
}
|
2023-09-15 09:22:02 +01:00
|
|
|
{
|
|
|
|
job_name = "coredns";
|
|
|
|
static_configs = [ { targets = [ "vancouver:9253" ]; } ];
|
|
|
|
}
|
2023-09-13 23:01:51 +01:00
|
|
|
{
|
|
|
|
job_name = "healthchecks";
|
|
|
|
scrape_interval = "60s";
|
2023-09-13 23:17:37 +01:00
|
|
|
metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/TbMoU7SUdknzMe-H5Q4HzmKl3itOIrJk";
|
2023-09-13 23:01:51 +01:00
|
|
|
static_configs = [ { targets = [ "localhost:8000" ]; } ];
|
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
|
|
|
job_name = "blackbox";
|
|
|
|
metrics_path = "/probe";
|
|
|
|
params = { "modules" = [ "http_2xx" ]; };
|
|
|
|
static_configs = [ { targets = [ "google.com" "gabrielsimmer.com" "artbybecki.com" ]; } ];
|
|
|
|
relabel_configs = [
|
|
|
|
{ source_labels = ["__address__"]; target_label = "__param_target"; }
|
|
|
|
{ source_labels = ["__param_target"]; target_label = "instance"; }
|
|
|
|
{ source_labels = []; target_label = "__address__"; replacement = "vancouver:9115"; } ];
|
|
|
|
}
|
|
|
|
];
|
|
|
|
exporters.node = {
|
|
|
|
enable = true;
|
|
|
|
listenAddress = "127.0.0.1";
|
|
|
|
enabledCollectors = [
|
|
|
|
"systemd" "processes"
|
|
|
|
];
|
|
|
|
};
|
|
|
|
};
|
|
|
|
services.tailscale.enable = true;
|
2023-09-13 23:01:51 +01:00
|
|
|
|
|
|
|
services.healthchecks = {
|
|
|
|
enable = true;
|
2023-09-25 10:48:52 +01:00
|
|
|
# package = healthchecks-edge;
|
2023-09-13 23:01:51 +01:00
|
|
|
settings = {
|
|
|
|
SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
|
|
|
|
SITE_ROOT = "https://healthchecks.gmem.ca";
|
2023-09-25 10:48:52 +01:00
|
|
|
SITE_NAME = "Archs Healthchecks";
|
2023-09-20 00:25:38 +01:00
|
|
|
EMAIL_HOST = "smtp.fastmail.com";
|
|
|
|
EMAIL_HOST_PASSWORD_FILE = config.age.secrets.healthchecks-smtp.path;
|
|
|
|
EMAIL_HOST_USER = "g@gmem.ca";
|
|
|
|
DEFAULT_FROM_EMAIL = "healthchecks@gmem.ca";
|
2023-09-25 10:48:52 +01:00
|
|
|
TELEGRAM_BOT_NAME = "arch_healthchecks_bot";
|
|
|
|
TELEGRAM_TOKEN_FILE = config.age.secrets.healthchecks-telegram.path;
|
2023-09-13 23:01:51 +01:00
|
|
|
};
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
|
|
|
|
# nginx reverse proxy
|
|
|
|
services.nginx = {
|
|
|
|
enable = true;
|
|
|
|
recommendedGzipSettings = true;
|
|
|
|
recommendedBrotliSettings = true;
|
|
|
|
recommendedZstdSettings = true;
|
|
|
|
recommendedOptimisation = true;
|
|
|
|
recommendedTlsSettings = true;
|
|
|
|
recommendedProxySettings = true;
|
2023-09-13 23:01:51 +01:00
|
|
|
virtualHosts.${config.services.grafana.settings.server.domain} = {
|
2023-09-05 17:08:43 +01:00
|
|
|
default = true;
|
|
|
|
enableACME = true;
|
|
|
|
forceSSL = true;
|
|
|
|
locations."/" = {
|
2023-09-13 23:01:51 +01:00
|
|
|
proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
|
|
|
|
proxyWebsockets = true;
|
|
|
|
};
|
|
|
|
};
|
2023-09-15 09:22:02 +01:00
|
|
|
|
2023-09-13 23:01:51 +01:00
|
|
|
virtualHosts."healthchecks.gmem.ca" = {
|
|
|
|
enableACME = true;
|
|
|
|
forceSSL = true;
|
|
|
|
locations."/" = {
|
|
|
|
proxyPass = "http://127.0.0.1:8000";
|
2023-09-05 17:08:43 +01:00
|
|
|
proxyWebsockets = true;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
security.acme.acceptTerms = true;
|
|
|
|
security.acme.defaults.email = "acme@gmem.ca";
|
|
|
|
|
|
|
|
system.stateVersion = "23.11";
|
|
|
|
}
|