2023-09-25 10:48:52 +01:00
|
|
|
{
|
2024-02-05 13:13:44 +00:00
|
|
|
config,
|
|
|
|
pkgs,
|
|
|
|
...
|
|
|
|
}: {
|
2023-09-05 17:08:43 +01:00
|
|
|
imports = [
|
|
|
|
./hardware.nix
|
|
|
|
./networking.nix # generated at runtime by nixos-infect
|
|
|
|
];
|
|
|
|
|
2023-09-13 23:01:51 +01:00
|
|
|
age.secrets.healthchecks-secret = {
|
|
|
|
file = ../../secrets/monitoring-healthchecks-secret.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
|
|
|
|
2023-09-20 00:25:38 +01:00
|
|
|
age.secrets.healthchecks-smtp = {
|
|
|
|
file = ../../secrets/fastmail-smtp.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
2023-09-25 10:48:52 +01:00
|
|
|
|
|
|
|
age.secrets.healthchecks-telegram = {
|
|
|
|
file = ../../secrets/healthchecks-telegram.age;
|
|
|
|
owner = "healthchecks";
|
|
|
|
};
|
2024-02-05 13:13:44 +00:00
|
|
|
|
2023-09-15 09:21:50 +01:00
|
|
|
age.secrets.prometheus-webconfig-secret = {
|
|
|
|
file = ../../secrets/monitoring-prometheus-webconfig.age;
|
|
|
|
owner = "prometheus";
|
2023-09-19 23:59:44 +01:00
|
|
|
mode = "775";
|
|
|
|
};
|
|
|
|
|
|
|
|
age.secrets.prometheus-password-secret = {
|
|
|
|
file = ../../secrets/monitoring-prometheus-password.age;
|
|
|
|
owner = "prometheus";
|
2023-09-15 09:21:50 +01:00
|
|
|
};
|
|
|
|
|
2023-10-30 12:27:14 +00:00
|
|
|
age.secrets.grafana-client-secret = {
|
|
|
|
file = ../../secrets/monitoring-grafana-client-secret.age;
|
|
|
|
owner = "grafana";
|
|
|
|
};
|
|
|
|
|
2023-09-05 17:08:43 +01:00
|
|
|
boot.tmp.cleanOnBoot = true;
|
|
|
|
zramSwap.enable = true;
|
|
|
|
networking.hostName = "monitoring";
|
|
|
|
networking.domain = "";
|
|
|
|
services.openssh.enable = true;
|
|
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
2024-02-05 13:13:44 +00:00
|
|
|
''ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjEgtIWPA5Ncs/KOcMeT6Q/HACJJetDOLjMvXXwUE+08oTX1EpHrWPpy8J+UHKIyErCNPYq8dgtrbhnMRlxHqI=''
|
2023-09-05 17:08:43 +01:00
|
|
|
];
|
|
|
|
networking.firewall.enable = false;
|
|
|
|
|
|
|
|
services.grafana = {
|
|
|
|
enable = true;
|
2023-10-18 15:48:42 +01:00
|
|
|
settings = {
|
|
|
|
feature_toggles = {
|
|
|
|
publicDashboards = true;
|
|
|
|
};
|
2023-10-30 12:27:14 +00:00
|
|
|
log = {
|
|
|
|
filters = "oauth.generic_oauth:debug";
|
|
|
|
};
|
2023-10-18 15:48:42 +01:00
|
|
|
server = {
|
|
|
|
domain = "grafana.gmem.ca";
|
|
|
|
http_port = 2342;
|
|
|
|
http_addr = "127.0.0.1";
|
2023-10-30 12:27:14 +00:00
|
|
|
root_url = "https://grafana.gmem.ca";
|
|
|
|
};
|
|
|
|
auth = {
|
|
|
|
signout_redirect_url = "https://authentik.gmem.ca/application/o/grafana/end-session/";
|
|
|
|
oauth_auto_login = true;
|
|
|
|
};
|
|
|
|
"auth.generic_oauth" = {
|
|
|
|
name = "authentik";
|
|
|
|
client_id = "VbOQzwuf0UK9AUGrWvaVaWWHvX2fJsZChxJNGt61";
|
|
|
|
client_secret = "$__file{${config.age.secrets.grafana-client-secret.path}}";
|
|
|
|
auth_url = "https://authentik.gmem.ca/application/o/authorize/";
|
|
|
|
api_url = "https://authentik.gmem.ca/application/o/userinfo/";
|
|
|
|
token_url = "https://authentik.gmem.ca/application/o/token/";
|
|
|
|
enabled = true;
|
|
|
|
scopes = "openid email grafana-user";
|
|
|
|
role_attribute_path = "contains(info.groups[*], 'Grafana Admins') && 'Admin' || contains(info.groups[*], 'Grafana Editors') && 'Editor' || 'Viewer'";
|
|
|
|
role_attribute_strict = true;
|
2023-10-18 15:48:42 +01:00
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
};
|
|
|
|
};
|
2023-10-08 22:55:20 +01:00
|
|
|
services.loki = {
|
|
|
|
enable = true;
|
|
|
|
configuration = {
|
|
|
|
server.http_listen_port = 3030;
|
|
|
|
auth_enabled = false;
|
|
|
|
|
|
|
|
ingester = {
|
|
|
|
lifecycler = {
|
|
|
|
address = "127.0.0.1";
|
|
|
|
ring = {
|
|
|
|
kvstore = {
|
|
|
|
store = "inmemory";
|
|
|
|
};
|
|
|
|
replication_factor = 1;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
chunk_idle_period = "1h";
|
|
|
|
max_chunk_age = "1h";
|
|
|
|
chunk_target_size = 999999;
|
|
|
|
chunk_retain_period = "30s";
|
|
|
|
max_transfer_retries = 0;
|
|
|
|
};
|
|
|
|
|
|
|
|
schema_config = {
|
2024-02-05 13:13:44 +00:00
|
|
|
configs = [
|
|
|
|
{
|
|
|
|
from = "2022-06-06";
|
|
|
|
store = "boltdb-shipper";
|
|
|
|
object_store = "filesystem";
|
|
|
|
schema = "v11";
|
|
|
|
index = {
|
|
|
|
prefix = "index_";
|
|
|
|
period = "24h";
|
|
|
|
};
|
|
|
|
}
|
|
|
|
];
|
2023-10-08 22:55:20 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
storage_config = {
|
|
|
|
boltdb_shipper = {
|
|
|
|
active_index_directory = "/var/lib/loki/boltdb-shipper-active";
|
|
|
|
cache_location = "/var/lib/loki/boltdb-shipper-cache";
|
|
|
|
cache_ttl = "24h";
|
|
|
|
shared_store = "filesystem";
|
|
|
|
};
|
|
|
|
|
|
|
|
filesystem = {
|
|
|
|
directory = "/var/lib/loki/chunks";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
limits_config = {
|
|
|
|
reject_old_samples = true;
|
|
|
|
reject_old_samples_max_age = "168h";
|
|
|
|
};
|
|
|
|
|
|
|
|
chunk_store_config = {
|
|
|
|
max_look_back_period = "0s";
|
|
|
|
};
|
|
|
|
|
|
|
|
table_manager = {
|
|
|
|
retention_deletes_enabled = false;
|
|
|
|
retention_period = "0s";
|
|
|
|
};
|
|
|
|
|
|
|
|
compactor = {
|
|
|
|
working_directory = "/var/lib/loki";
|
|
|
|
shared_store = "filesystem";
|
|
|
|
compactor_ring = {
|
|
|
|
kvstore = {
|
|
|
|
store = "inmemory";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
services.promtail = {
|
|
|
|
enable = true;
|
|
|
|
configuration = {
|
|
|
|
server = {
|
|
|
|
http_listen_port = 3031;
|
|
|
|
grpc_listen_port = 0;
|
|
|
|
};
|
|
|
|
positions = {
|
|
|
|
filename = "/tmp/positions.yaml";
|
|
|
|
};
|
2024-02-05 13:13:44 +00:00
|
|
|
clients = [
|
|
|
|
{
|
|
|
|
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
|
|
|
|
}
|
|
|
|
];
|
|
|
|
scrape_configs = [
|
|
|
|
{
|
|
|
|
job_name = "journal";
|
|
|
|
journal = {
|
|
|
|
max_age = "12h";
|
|
|
|
labels = {
|
|
|
|
job = "systemd-journal";
|
|
|
|
host = "monitoring";
|
|
|
|
};
|
2023-10-08 22:55:20 +01:00
|
|
|
};
|
2024-02-05 13:13:44 +00:00
|
|
|
relabel_configs = [
|
|
|
|
{
|
|
|
|
source_labels = ["__journal__systemd_unit"];
|
|
|
|
target_label = "unit";
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}
|
|
|
|
];
|
2023-10-08 22:55:20 +01:00
|
|
|
};
|
|
|
|
};
|
2023-09-19 23:59:44 +01:00
|
|
|
services.alertmanager-ntfy = {
|
2024-02-05 13:13:44 +00:00
|
|
|
enable = true;
|
2023-09-19 23:59:44 +01:00
|
|
|
settings = {
|
|
|
|
http = {
|
|
|
|
addr = "127.0.0.1:8111";
|
|
|
|
};
|
|
|
|
ntfy = {
|
|
|
|
baseurl = "https://ntfy.gmem.ca";
|
|
|
|
notification = {
|
|
|
|
topic = "alerts";
|
|
|
|
priority = ''
|
|
|
|
status == "firing" ? "high" : "default"
|
2024-02-05 13:13:44 +00:00
|
|
|
'';
|
2023-09-19 23:59:44 +01:00
|
|
|
templates = {
|
|
|
|
title = ''{{ if eq .Status "resolved" }}Resolved: {{ end }}{{ index .Annotations "summary" }}'';
|
|
|
|
description = ''{{ index .Annotations "description" }}'';
|
|
|
|
click = ''http://grafana.gmem.ca/d/{{ index .Annotations "dashboard" }}'';
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
services.prometheus = {
|
|
|
|
enable = true;
|
2023-09-15 09:21:50 +01:00
|
|
|
webConfigFile = config.age.secrets.prometheus-webconfig-secret.path;
|
2023-09-05 17:08:43 +01:00
|
|
|
globalConfig = {
|
|
|
|
scrape_interval = "15s";
|
|
|
|
};
|
2024-02-05 13:13:44 +00:00
|
|
|
alertmanagers = [
|
|
|
|
{
|
|
|
|
basic_auth = {
|
|
|
|
username = "homelab";
|
|
|
|
password_file = config.age.secrets.prometheus-password-secret.path;
|
|
|
|
};
|
|
|
|
static_configs = [
|
|
|
|
{
|
|
|
|
targets = [
|
|
|
|
"localhost:9093"
|
|
|
|
];
|
|
|
|
}
|
2023-09-19 23:59:44 +01:00
|
|
|
];
|
2024-02-05 13:13:44 +00:00
|
|
|
}
|
|
|
|
];
|
|
|
|
rules = [
|
|
|
|
(builtins.toJSON {
|
|
|
|
groups = [
|
|
|
|
{
|
|
|
|
name = "internet_connection";
|
|
|
|
rules = [
|
|
|
|
{
|
|
|
|
alert = "LowInternetDownload";
|
|
|
|
expr = ''speedtest_download_bits_per_second{} < 500000000'';
|
|
|
|
for = "2h";
|
|
|
|
labels.severity = "page";
|
|
|
|
annotations = {
|
|
|
|
summary = "Internet speed low";
|
|
|
|
description = "Home internet speed dropped below 500Mb/s.";
|
|
|
|
dashboard = "o9mIe_Aik";
|
|
|
|
};
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}
|
|
|
|
{
|
|
|
|
name = "healthchecks";
|
|
|
|
rules = [
|
|
|
|
{
|
|
|
|
alert = "HealthcheckFailedCheckin";
|
|
|
|
expr = ''hc_check_up < 1'';
|
|
|
|
for = "5m";
|
|
|
|
labels.severity = "page";
|
|
|
|
annotations = {
|
|
|
|
summary = "{{ $labels.name }} healthcheck failed";
|
|
|
|
description = "The {{ $labels.name }} healthcheck failed to check in.";
|
|
|
|
dashboard = "f594ea85-45f2-4019-b988-2d17638b5cf3";
|
|
|
|
};
|
|
|
|
}
|
|
|
|
];
|
|
|
|
}
|
|
|
|
];
|
|
|
|
})
|
|
|
|
];
|
2023-09-19 23:59:44 +01:00
|
|
|
alertmanager = {
|
|
|
|
enable = true;
|
2024-02-05 13:13:44 +00:00
|
|
|
extraFlags = ["--web.config.file=${config.age.secrets.prometheus-webconfig-secret.path}"];
|
2023-09-19 23:59:44 +01:00
|
|
|
webExternalUrl = "https://alerts.gmem.ca";
|
|
|
|
configText = ''
|
2024-02-05 13:13:44 +00:00
|
|
|
global: {}
|
2023-09-19 23:59:44 +01:00
|
|
|
|
2024-02-05 13:13:44 +00:00
|
|
|
# The directory from which notification templates are read.
|
|
|
|
templates:
|
|
|
|
- '/etc/alertmanager/template/*.tmpl'
|
2023-09-19 23:59:44 +01:00
|
|
|
|
2024-02-05 13:13:44 +00:00
|
|
|
# The root route on which each incoming alert enters.
|
|
|
|
route:
|
|
|
|
group_by: ['alertname', 'cluster', 'service']
|
|
|
|
group_wait: 0s
|
|
|
|
group_interval: 5m
|
|
|
|
repeat_interval: 3h
|
|
|
|
# A default receiver
|
|
|
|
receiver: ntfy
|
2023-09-19 23:59:44 +01:00
|
|
|
|
2024-02-05 13:13:44 +00:00
|
|
|
receivers:
|
|
|
|
- name: ntfy
|
|
|
|
webhook_configs:
|
|
|
|
- url: http://localhost:8111/hook
|
2023-09-19 23:59:44 +01:00
|
|
|
'';
|
|
|
|
};
|
2023-09-05 17:08:43 +01:00
|
|
|
port = 9001;
|
2024-02-05 13:13:44 +00:00
|
|
|
extraFlags = ["--web.enable-remote-write-receiver"];
|
2023-09-05 17:08:43 +01:00
|
|
|
scrapeConfigs = [
|
2023-12-08 23:42:59 +00:00
|
|
|
{
|
|
|
|
job_name = "proxmox";
|
|
|
|
metrics_path = "/pve";
|
2024-02-05 13:13:44 +00:00
|
|
|
params = {"target" = ["localhost"];};
|
|
|
|
static_configs = [{targets = ["proxmox:9221"];}];
|
2023-12-08 23:42:59 +00:00
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
2023-09-13 23:01:51 +01:00
|
|
|
job_name = "personal_hardware";
|
2024-02-05 13:13:44 +00:00
|
|
|
static_configs = [{targets = ["london:9100" "vancouver:9100" "localhost:9100"];}];
|
2023-09-05 17:08:43 +01:00
|
|
|
}
|
|
|
|
{
|
2024-02-05 13:13:44 +00:00
|
|
|
job_name = "speedtest-exporter";
|
|
|
|
scrape_interval = "1h";
|
|
|
|
scrape_timeout = "1m";
|
|
|
|
static_configs = [{targets = ["vancouver:9798"];}];
|
2023-09-05 17:08:43 +01:00
|
|
|
}
|
2023-09-13 23:01:51 +01:00
|
|
|
{
|
2024-02-05 13:13:44 +00:00
|
|
|
job_name = "syncthing";
|
|
|
|
static_configs = [{targets = ["vancouver:8384" "london:8384"];}];
|
2023-09-13 23:01:51 +01:00
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
|
|
|
job_name = "forgejo";
|
2024-02-05 13:13:44 +00:00
|
|
|
static_configs = [{targets = ["git.gmem.ca"];}];
|
2023-09-05 17:08:43 +01:00
|
|
|
}
|
2023-09-15 09:22:02 +01:00
|
|
|
{
|
|
|
|
job_name = "coredns";
|
2024-02-05 13:13:44 +00:00
|
|
|
static_configs = [{targets = ["vancouver:9253"];}];
|
2023-09-15 09:22:02 +01:00
|
|
|
}
|
2023-09-13 23:01:51 +01:00
|
|
|
{
|
|
|
|
job_name = "healthchecks";
|
|
|
|
scrape_interval = "60s";
|
2023-09-13 23:17:37 +01:00
|
|
|
metrics_path = "/projects/5f1de50f-a52d-4215-961f-aae7cc6cf6c9/metrics/TbMoU7SUdknzMe-H5Q4HzmKl3itOIrJk";
|
2024-02-05 13:13:44 +00:00
|
|
|
static_configs = [{targets = ["localhost:8000"];}];
|
2023-09-13 23:01:51 +01:00
|
|
|
}
|
2023-10-27 14:55:04 +01:00
|
|
|
{
|
|
|
|
job_name = "vrc_events";
|
|
|
|
scrape_interval = "60s";
|
2024-02-05 13:13:44 +00:00
|
|
|
static_configs = [{targets = ["vancouver:6534"];}];
|
2023-10-27 14:55:04 +01:00
|
|
|
}
|
2023-09-05 17:08:43 +01:00
|
|
|
{
|
2024-02-05 13:13:44 +00:00
|
|
|
job_name = "blackbox";
|
2023-09-05 17:08:43 +01:00
|
|
|
metrics_path = "/probe";
|
2024-02-05 13:13:44 +00:00
|
|
|
params = {"modules" = ["http_2xx"];};
|
|
|
|
static_configs = [{targets = ["google.com" "gabrielsimmer.com" "artbybecki.com"];}];
|
|
|
|
relabel_configs = [
|
|
|
|
{
|
|
|
|
source_labels = ["__address__"];
|
|
|
|
target_label = "__param_target";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
source_labels = ["__param_target"];
|
|
|
|
target_label = "instance";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
source_labels = [];
|
|
|
|
target_label = "__address__";
|
|
|
|
replacement = "vancouver:9115";
|
|
|
|
}
|
|
|
|
];
|
2023-09-05 17:08:43 +01:00
|
|
|
}
|
|
|
|
];
|
|
|
|
exporters.node = {
|
|
|
|
enable = true;
|
|
|
|
listenAddress = "127.0.0.1";
|
|
|
|
enabledCollectors = [
|
2024-02-05 13:13:44 +00:00
|
|
|
"systemd"
|
|
|
|
"processes"
|
2023-09-05 17:08:43 +01:00
|
|
|
];
|
|
|
|
};
|
|
|
|
};
|
|
|
|
services.tailscale.enable = true;
|
2023-09-13 23:01:51 +01:00
|
|
|
|
|
|
|
services.healthchecks = {
|
|
|
|
enable = true;
|
2023-09-25 10:48:52 +01:00
|
|
|
# package = healthchecks-edge;
|
2023-09-13 23:01:51 +01:00
|
|
|
settings = {
|
|
|
|
SECRET_KEY_FILE = config.age.secrets.healthchecks-secret.path;
|
|
|
|
SITE_ROOT = "https://healthchecks.gmem.ca";
|
2023-09-25 10:48:52 +01:00
|
|
|
SITE_NAME = "Archs Healthchecks";
|
2023-09-20 00:25:38 +01:00
|
|
|
EMAIL_HOST = "smtp.fastmail.com";
|
|
|
|
EMAIL_HOST_PASSWORD_FILE = config.age.secrets.healthchecks-smtp.path;
|
|
|
|
EMAIL_HOST_USER = "g@gmem.ca";
|
|
|
|
DEFAULT_FROM_EMAIL = "healthchecks@gmem.ca";
|
2023-09-25 10:48:52 +01:00
|
|
|
TELEGRAM_BOT_NAME = "arch_healthchecks_bot";
|
|
|
|
TELEGRAM_TOKEN_FILE = config.age.secrets.healthchecks-telegram.path;
|
2023-09-13 23:01:51 +01:00
|
|
|
};
|
|
|
|
};
|
2024-02-05 13:13:44 +00:00
|
|
|
|
2023-09-05 17:08:43 +01:00
|
|
|
# nginx reverse proxy
|
|
|
|
services.nginx = {
|
|
|
|
enable = true;
|
|
|
|
recommendedGzipSettings = true;
|
|
|
|
recommendedBrotliSettings = true;
|
|
|
|
recommendedZstdSettings = true;
|
|
|
|
recommendedOptimisation = true;
|
|
|
|
recommendedTlsSettings = true;
|
|
|
|
recommendedProxySettings = true;
|
2023-09-13 23:01:51 +01:00
|
|
|
virtualHosts.${config.services.grafana.settings.server.domain} = {
|
2023-09-05 17:08:43 +01:00
|
|
|
default = true;
|
|
|
|
enableACME = true;
|
|
|
|
forceSSL = true;
|
|
|
|
locations."/" = {
|
2023-09-13 23:01:51 +01:00
|
|
|
proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
|
|
|
|
proxyWebsockets = true;
|
|
|
|
};
|
|
|
|
};
|
2023-09-15 09:22:02 +01:00
|
|
|
|
2023-09-13 23:01:51 +01:00
|
|
|
virtualHosts."healthchecks.gmem.ca" = {
|
|
|
|
enableACME = true;
|
|
|
|
forceSSL = true;
|
|
|
|
locations."/" = {
|
|
|
|
proxyPass = "http://127.0.0.1:8000";
|
2023-09-05 17:08:43 +01:00
|
|
|
proxyWebsockets = true;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
|
|
|
security.acme.acceptTerms = true;
|
|
|
|
security.acme.defaults.email = "acme@gmem.ca";
|
|
|
|
|
|
|
|
system.stateVersion = "23.11";
|
|
|
|
}
|