EH22/nox
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

alertmanager pain

Browse source
This commit is contained in:
chris 2025-04-14 23:37:55 +02:00
parent 62ad1df154
commit c0241036c6
Signed by: c6ristian
SSH key fingerprint: SHA256:B3m+yzpaxGXSEcDBpPHfvza/DNC0wuX+CKMeGq8wgak
8 changed files with 413 additions and 94 deletions

2
modules/alloy.nix
View file

@ -15,7 +15,7 @@
sopsFile = ../secrets/passwords.yaml;
};
secrets."services/mimir/basic_auth" = {
mode = "0440";
mode = "0444";
owner = "alloy";
group = "alloy";
restartUnits = [ "alloy.service" ];

6
secrets/passwords.yaml
View file

@ -14,6 +14,8 @@ services:
nginx: ENC[AES256_GCM,data:1vQE9z+w6RGwFK8mOuhzW3NArwthEcY/c/yQzAA6IwXtsDuxu7AS11M=,iv:jMtkHRR/+DVX1FREGUVTSTVUUUzOpjrLbNabHnw8Dq0=,tag:HbtQyuLlW5AbYqYDrtrbJA==,type:str]
vyos:
vyos: ENC[AES256_GCM,data:IUeOs7fPj008Iz31QLC7iCxG5wQ=,iv:FlZcvasmJBR3l6Pk1DELgF0f/lBfrxxoS6jeRsGunmQ=,tag:CMZr2AvzxDO3Vo1FKzoQtw==,type:str]
telegrambot:
token: ENC[AES256_GCM,data:Q6POluc038dHP6EatuVzNx4kDSSCQfxY9A2W88S0ZOBkA8483f8xqvaT8MWQgw==,iv:n8EDaxO4DHKQwz3g3IbXUyQSd3xME1KpaOjdIFAnCW4=,tag:3qUQ1XdgJ8/xWtanJ4fTeQ==,type:str]
hardware:
proxmox_server:
ipmi:
@ -69,7 +71,7 @@ sops:
OGU5LzlYeFJLc3BBY3BScjdRcGU4MmcKW2ASw7TpDmlXymYIgSihGpF6rkrx08Aj
vpyqwi2Z6cUvdF6DoqfMU5NaLoLsGRTVYlalvPHZs3tfoY9/SVyoXA==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-04-07T04:26:22Z"
mac: ENC[AES256_GCM,data:YtmcWGAXnOXgzczhEj6qCYC+b+fuNQz5p9RKwolX2sEuC6lGqQBgNyYfLf05hIDE1QfhTZgjTGtdRfCo2fUzeouZ5ggLnAFgXAfqJQ1EMxN4gTEKEYWNEYc2b/EOQ+jb/gJdi/ih7tGY09OuHmyCorg/CTOY/M49I3G+79AVUP8=,iv:+3E/Ur/Dbdz359Q5DREuogEJnAGob8BpfH4D2YCUQy8=,tag:FbHH8/qN9VUjcvPQCiP8PA==,type:str]
lastmodified: "2025-04-14T21:15:35Z"
mac: ENC[AES256_GCM,data:rWOcAH6yKgvgusH+UkabBW1kcWIX6K0LmKkwGBIPnODtMkzeZT5E/6oLFlyp/vWI2FcWh1woo/Yf5sI7c74R5NJlJt5HtIkJbZhvxt9xOHEpi53aciEnL6sohNTohQ7n9WCamPKozf00P/VAVhWYWHfdibCJ4k7jD9eNZXPEc8s=,iv:asuUocdSJJJPfLBRgv+9KuXbpCpMewTRyRJ+zcTlWzE=,tag:zRV7pAI+TJBVmfWh9FXsvA==,type:str]
unencrypted_suffix: _unencrypted
version: 3.10.1

15
systems/monitoring.noc.eh22.intern/alerts/embedded-exporter.yml Normal file
View file

@ -0,0 +1,15 @@
# wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/master/dist/rules/grafana-alloy/embedded-exporter.yml
groups:
- name: EmbeddedExporter
rules:
- alert: GrafanaAlloyServiceDown
expr: 'count by (instance) (alloy_build_info) unless count by (instance) (alloy_build_info offset 2m) '
for: 0m
labels:
severity: critical
annotations:
summary: Grafana Alloy service down (instance {{ $labels.instance }})
description: "Alloy on (instance {{ $labels.instance }}) is not responding or has stopped running.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"

399
systems/monitoring.noc.eh22.intern/alerts/resources.yaml
View file

@ -1,89 +1,312 @@
#https://samber.github.io/awesome-prometheus-alerts/rules#host-and-hardware
groups:
- name: Host & hardware
rules:
- alert: HostOutOfMemory
expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Node memory is filling up (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of memory (instance {{ $labels.instance }})
- alert: HostMemoryUnderMemoryPressure
expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
The node is under heavy memory pressure. High rate of major page faults
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host memory under memory pressure (instance {{ $labels.instance }})
- alert: HostOutOfDiskSpace
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Disk is almost full (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of disk space (instance {{ $labels.instance }})
- alert: HostDiskWillFillIn24Hours
expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Filesystem is predicted to run out of space within the next 24 hours at current write rate
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
- alert: HostCpuIsUnderutilized
expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1w
labels:
severity: info
annotations:
description: |-
CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host CPU is underutilized (instance {{ $labels.instance }})
- alert: HostCpuStealNoisyNeighbor
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: warning
annotations:
description: |-
CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
- alert: HostOomKillDetected
expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: warning
annotations:
description: |-
OOM kill detected
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host OOM kill detected (instance {{ $labels.instance }})
- alert: HostNetworkInterfaceSaturated
expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1m
labels:
severity: warning
annotations:
description: |-
The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Network Interface Saturated (instance {{ $labels.instance }})
- name: NodeExporter
rules:
- alert: HostOutOfMemory
expr: '(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10)'
for: 2m
labels:
severity: warning
annotations:
summary: Host out of memory (instance {{ $labels.instance }})
description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostMemoryUnderMemoryPressure
expr: '(rate(node_vmstat_pgmajfault[5m]) > 1000)'
for: 0m
labels:
severity: warning
annotations:
summary: Host memory under memory pressure (instance {{ $labels.instance }})
description: "The node is under heavy memory pressure. High rate of loading memory pages from disk.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualNetworkThroughputIn
expr: '((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)'
for: 0m
labels:
severity: warning
annotations:
summary: Host unusual network throughput in (instance {{ $labels.instance }})
description: "Host receive bandwidth is high (>80%).\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualNetworkThroughputOut
expr: '((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)'
for: 0m
labels:
severity: warning
annotations:
summary: Host unusual network throughput out (instance {{ $labels.instance }})
description: "Host transmit bandwidth is high (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskReadRate
expr: '(rate(node_disk_io_time_seconds_total[5m]) > .80)'
for: 0m
labels:
severity: warning
annotations:
summary: Host unusual disk read rate (instance {{ $labels.instance }})
description: "Disk is too busy (IO wait > 80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOutOfDiskSpace
expr: '(node_filesystem_avail_bytes{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0)'
for: 2m
labels:
severity: critical
annotations:
summary: Host out of disk space (instance {{ $labels.instance }})
description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostDiskMayFillIn24Hours
expr: 'predict_linear(node_filesystem_avail_bytes{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"}[1h], 86400) <= 0 and node_filesystem_avail_bytes > 0'
for: 2m
labels:
severity: warning
annotations:
summary: Host disk may fill in 24 hours (instance {{ $labels.instance }})
description: "Filesystem will likely run out of space within the next 24 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOutOfInodes
expr: '(node_filesystem_files_free / node_filesystem_files < .10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0)'
for: 2m
labels:
severity: critical
annotations:
summary: Host out of inodes (instance {{ $labels.instance }})
description: "Disk is almost running out of available inodes (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostFilesystemDeviceError
expr: 'node_filesystem_device_error{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"} == 1'
for: 2m
labels:
severity: critical
annotations:
summary: Host filesystem device error (instance {{ $labels.instance }})
description: "Error stat-ing the {{ $labels.mountpoint }} filesystem\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostInodesMayFillIn24Hours
expr: 'predict_linear(node_filesystem_files_free{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"}[1h], 86400) <= 0 and node_filesystem_files_free > 0'
for: 2m
labels:
severity: warning
annotations:
summary: Host inodes may fill in 24 hours (instance {{ $labels.instance }})
description: "Filesystem will likely run out of inodes within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskReadLatency
expr: '(rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0)'
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk read latency (instance {{ $labels.instance }})
description: "Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskWriteLatency
expr: '(rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0)'
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk write latency (instance {{ $labels.instance }})
description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostHighCpuLoad
expr: '(avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > .80'
for: 10m
labels:
severity: warning
annotations:
summary: Host high CPU load (instance {{ $labels.instance }})
description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostCpuStealNoisyNeighbor
expr: 'avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10'
for: 0m
labels:
severity: warning
annotations:
summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostCpuHighIowait
expr: 'avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) > .10'
for: 0m
labels:
severity: warning
annotations:
summary: Host CPU high iowait (instance {{ $labels.instance }})
description: "CPU iowait > 10%. Your CPU is idling waiting for storage to respond.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskIo
expr: 'rate(node_disk_io_time_seconds_total[5m]) > 0.8'
for: 5m
labels:
severity: warning
annotations:
summary: Host unusual disk IO (instance {{ $labels.instance }})
description: "Disk usage >80%. Check storage for issues or increase IOPS capabilities. Check storage for issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostContextSwitchingHigh
expr: '(rate(node_context_switches_total[15m])/count without(mode,cpu) (node_cpu_seconds_total{mode="idle"})) / (rate(node_context_switches_total[1d])/count without(mode,cpu) (node_cpu_seconds_total{mode="idle"})) > 2'
for: 0m
labels:
severity: warning
annotations:
summary: Host context switching high (instance {{ $labels.instance }})
description: "Context switching is growing on the node (twice the daily average during the last 15m)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostSwapIsFillingUp
expr: '((1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80)'
for: 2m
labels:
severity: warning
annotations:
summary: Host swap is filling up (instance {{ $labels.instance }})
description: "Swap is filling up (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostSystemdServiceCrashed
expr: '(node_systemd_unit_state{state="failed"} == 1)'
for: 0m
labels:
severity: warning
annotations:
summary: Host systemd service crashed (instance {{ $labels.instance }})
description: "systemd service crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostPhysicalComponentTooHot
expr: 'node_hwmon_temp_celsius > node_hwmon_temp_max_celsius'
for: 5m
labels:
severity: warning
annotations:
summary: Host physical component too hot (instance {{ $labels.instance }})
description: "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNodeOvertemperatureAlarm
expr: '((node_hwmon_temp_crit_alarm_celsius == 1) or (node_hwmon_temp_alarm == 1))'
for: 0m
labels:
severity: critical
annotations:
summary: Host node overtemperature alarm (instance {{ $labels.instance }})
description: "Physical node temperature alarm triggered\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostSoftwareRaidInsufficientDrives
expr: '((node_md_disks_required - on(device, instance) node_md_disks{state="active"}) > 0)'
for: 0m
labels:
severity: critical
annotations:
summary: Host software RAID insufficient drives (instance {{ $labels.instance }})
description: "MD RAID array {{ $labels.device }} on {{ $labels.instance }} has insufficient drives remaining.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostSoftwareRaidDiskFailure
expr: '(node_md_disks{state="failed"} > 0)'
for: 2m
labels:
severity: warning
annotations:
summary: Host software RAID disk failure (instance {{ $labels.instance }})
description: "MD RAID array {{ $labels.device }} on {{ $labels.instance }} needs attention.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostKernelVersionDeviations
expr: 'changes(node_uname_info[1h]) > 0'
for: 0m
labels:
severity: info
annotations:
summary: Host kernel version deviations (instance {{ $labels.instance }})
description: "Kernel version for {{ $labels.instance }} has changed.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOomKillDetected
expr: '(increase(node_vmstat_oom_kill[1m]) > 0)'
for: 0m
labels:
severity: warning
annotations:
summary: Host OOM kill detected (instance {{ $labels.instance }})
description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostEdacCorrectableErrorsDetected
expr: '(increase(node_edac_correctable_errors_total[1m]) > 0)'
for: 0m
labels:
severity: info
annotations:
summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostEdacUncorrectableErrorsDetected
expr: '(node_edac_uncorrectable_errors_total > 0)'
for: 0m
labels:
severity: warning
annotations:
summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkReceiveErrors
expr: '(rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01)'
for: 2m
labels:
severity: warning
annotations:
summary: Host Network Receive Errors (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkTransmitErrors
expr: '(rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01)'
for: 2m
labels:
severity: warning
annotations:
summary: Host Network Transmit Errors (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkBondDegraded
expr: '((node_bonding_active - node_bonding_slaves) != 0)'
for: 2m
labels:
severity: warning
annotations:
summary: Host Network Bond Degraded (instance {{ $labels.instance }})
description: "Bond \"{{ $labels.device }}\" degraded on \"{{ $labels.instance }}\".\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostConntrackLimit
expr: '(node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8)'
for: 5m
labels:
severity: warning
annotations:
summary: Host conntrack limit (instance {{ $labels.instance }})
description: "The number of conntrack is approaching limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostClockSkew
expr: '((node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0))'
for: 10m
labels:
severity: warning
annotations:
summary: Host clock skew (instance {{ $labels.instance }})
description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostClockNotSynchronising
expr: '(min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16)'
for: 2m
labels:
severity: warning
annotations:
summary: Host clock not synchronising (instance {{ $labels.instance }})
description: "Clock not synchronising. Ensure NTP is configured on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostRequiresReboot
expr: '(node_reboot_required > 0)'
for: 4h
labels:
severity: info
annotations:
summary: Host requires reboot (instance {{ $labels.instance }})
description: "{{ $labels.instance }} requires a reboot.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"

7
systems/monitoring.noc.eh22.intern/grafana.nix
View file

@ -23,7 +23,7 @@
user = "grafana";
host = "/run/postgresql";
};
feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
feature_toggles.enable = "newVizTooltips";
security.angular_support_enabled = false;
};
provision = {
@ -48,7 +48,10 @@
type = "alertmanager";
uid = "mimir-alertmanager";
access = "proxy";
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/";
url = "http://mimir.noc.eh22.intern";
basicAuth = true;
basicAuthUser = "noc";
secureJsonData.basicAuthPassword = "$__file{${config.sops.secrets."services/mimir/basic_auth".path}}";
jsonData = {
handleGrafanaManagedAlerts = true;
implementation = "mimir";

37
systems/monitoring.noc.eh22.intern/mimir.nix
View file

@ -17,8 +17,11 @@ let
mkdir $out
cp -R $src $out/anonymous/
'';
template_alert = ./templates;
in
{
environment.etc."alertmanager/templates".source = template_alert;
services.mimir = {
enable = true;
configuration = {
@ -36,7 +39,6 @@ in
server = {
http_listen_port = 9009;
http_listen_address = "127.0.0.1";
log_level = "warn";
grpc_listen_port = 9096;
grpc_listen_address = "127.0.0.1";
};
@ -61,9 +63,32 @@ in
replication_factor = 1;
instance_addr = "127.0.0.1";
};
utf8_strict_mode = true;
fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
route = {
group_by = ["alertname"];
receiver = "telegram";
group_wait = "30s";
group_interval = "1m";
repeat_interval = "3m";
};
receivers = [
{
name = "telegram";
telegram_configs = [{
bot_token_file = config.sops.secrets."services/telegrambot/token".path;
chat_id = -1002579132187;
}];
}
];
templates = [
"/etc/alertmanager/templates/*.tmpl"
];
};
};
ruler = {
alertmanager_url = "http://localhost/alertmanager";
ring = {
instance_addr = "127.0.0.1";
};
@ -90,6 +115,16 @@ in
proxyPass = "http://mimir";
basicAuthFile = config.sops.secrets."services/mimir/nginx".path;
};
locations."/" = {
proxyPass = "http://mimir/";
basicAuthFile = config.sops.secrets."services/mimir/nginx".path;
};
};
virtualHosts."localhost" = {
locations."/" = {
proxyPass = "http://mimir/";
};
};
};
}

6
systems/monitoring.noc.eh22.intern/system.nix
View file

@ -31,6 +31,12 @@
group = "nginx";
restartUnits = [ "nginx.service" ];
};
secrets."services/telegrambot/token" = {
mode = "0444";
owner = "root";
group = "root";
restartUnits = [ "mimir.service" ];
};
};
networking.firewall.allowedTCPPorts = [ 80 ];

35
systems/monitoring.noc.eh22.intern/templates/alertmanager_alert_templates.tmpl Normal file
View file

@ -0,0 +1,35 @@
{{/*
Links & Resources
- https://prometheus.io/blog/2016/03/03/custom-alertmanager-templates/
- https://prometheus.io/docs/alerting/latest/notifications/
- https://gist.github.com/jidckii/5ac5f8f20368b56de72af70222509b7b
*/}}
{{ define "alert-item.telegram.eh22noc.internal" }}
<b>[{{ .Labels.alertname }}] {{ .Labels.nodename }}</b>
{{- if .Annotations.summary }}
<i>Summary</i>: {{ .Annotations.summary }}
{{- end }}
{{- if .Annotations.description }}
<i>Description</i>: {{ .Annotations.description }}
{{- end }}
<i>Labels</i>:
{{ range .Labels.SortedPairs -}}
• <i>{{ .Name }}</i>: <code>{{ .Value }}</code>
{{ end }}
{{- end }}
{{ define "alert-message.telegram.eh22noc" }}
{{- if .Alerts.Firing }}
<u>🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥</u>
{{ range .Alerts.Firing -}}
{{ template "alert-item.telegram.eh22noc.internal" . }}
{{- end }}
{{- end }}
{{- if .Alerts.Resolved }}
<u>✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅</u>
{{ range .Alerts.Resolved -}}
{{ template "alert-item.telegram.eh22noc.internal" . }}
{{- end }}
{{- end }}
{{- end }}