diff --git a/modules/alloy.nix b/modules/alloy.nix
index 3a06fba..32ddfdb 100644
--- a/modules/alloy.nix
+++ b/modules/alloy.nix
@@ -15,7 +15,7 @@
sopsFile = ../secrets/passwords.yaml;
};
secrets."services/mimir/basic_auth" = {
- mode = "0440";
+ mode = "0444";
owner = "alloy";
group = "alloy";
restartUnits = [ "alloy.service" ];
diff --git a/secrets/passwords.yaml b/secrets/passwords.yaml
index 2584efd..f6328e1 100644
--- a/secrets/passwords.yaml
+++ b/secrets/passwords.yaml
@@ -14,6 +14,8 @@ services:
nginx: ENC[AES256_GCM,data:1vQE9z+w6RGwFK8mOuhzW3NArwthEcY/c/yQzAA6IwXtsDuxu7AS11M=,iv:jMtkHRR/+DVX1FREGUVTSTVUUUzOpjrLbNabHnw8Dq0=,tag:HbtQyuLlW5AbYqYDrtrbJA==,type:str]
vyos:
vyos: ENC[AES256_GCM,data:IUeOs7fPj008Iz31QLC7iCxG5wQ=,iv:FlZcvasmJBR3l6Pk1DELgF0f/lBfrxxoS6jeRsGunmQ=,tag:CMZr2AvzxDO3Vo1FKzoQtw==,type:str]
+ telegrambot:
+ token: ENC[AES256_GCM,data:Q6POluc038dHP6EatuVzNx4kDSSCQfxY9A2W88S0ZOBkA8483f8xqvaT8MWQgw==,iv:n8EDaxO4DHKQwz3g3IbXUyQSd3xME1KpaOjdIFAnCW4=,tag:3qUQ1XdgJ8/xWtanJ4fTeQ==,type:str]
hardware:
proxmox_server:
ipmi:
@@ -69,7 +71,7 @@ sops:
OGU5LzlYeFJLc3BBY3BScjdRcGU4MmcKW2ASw7TpDmlXymYIgSihGpF6rkrx08Aj
vpyqwi2Z6cUvdF6DoqfMU5NaLoLsGRTVYlalvPHZs3tfoY9/SVyoXA==
-----END AGE ENCRYPTED FILE-----
- lastmodified: "2025-04-07T04:26:22Z"
- mac: ENC[AES256_GCM,data:YtmcWGAXnOXgzczhEj6qCYC+b+fuNQz5p9RKwolX2sEuC6lGqQBgNyYfLf05hIDE1QfhTZgjTGtdRfCo2fUzeouZ5ggLnAFgXAfqJQ1EMxN4gTEKEYWNEYc2b/EOQ+jb/gJdi/ih7tGY09OuHmyCorg/CTOY/M49I3G+79AVUP8=,iv:+3E/Ur/Dbdz359Q5DREuogEJnAGob8BpfH4D2YCUQy8=,tag:FbHH8/qN9VUjcvPQCiP8PA==,type:str]
+ lastmodified: "2025-04-14T21:15:35Z"
+ mac: ENC[AES256_GCM,data:rWOcAH6yKgvgusH+UkabBW1kcWIX6K0LmKkwGBIPnODtMkzeZT5E/6oLFlyp/vWI2FcWh1woo/Yf5sI7c74R5NJlJt5HtIkJbZhvxt9xOHEpi53aciEnL6sohNTohQ7n9WCamPKozf00P/VAVhWYWHfdibCJ4k7jD9eNZXPEc8s=,iv:asuUocdSJJJPfLBRgv+9KuXbpCpMewTRyRJ+zcTlWzE=,tag:zRV7pAI+TJBVmfWh9FXsvA==,type:str]
unencrypted_suffix: _unencrypted
version: 3.10.1
diff --git a/systems/monitoring.noc.eh22.intern/alerts/embedded-exporter.yml b/systems/monitoring.noc.eh22.intern/alerts/embedded-exporter.yml
new file mode 100644
index 0000000..83ec9df
--- /dev/null
+++ b/systems/monitoring.noc.eh22.intern/alerts/embedded-exporter.yml
@@ -0,0 +1,15 @@
+# wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/master/dist/rules/grafana-alloy/embedded-exporter.yml
+groups:
+
+- name: EmbeddedExporter
+
+ rules:
+
+ - alert: GrafanaAlloyServiceDown
+ expr: 'count by (instance) (alloy_build_info) unless count by (instance) (alloy_build_info offset 2m) '
+ for: 0m
+ labels:
+ severity: critical
+ annotations:
+ summary: Grafana Alloy service down (instance {{ $labels.instance }})
+ description: "Alloy on (instance {{ $labels.instance }}) is not responding or has stopped running.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
diff --git a/systems/monitoring.noc.eh22.intern/alerts/resources.yaml b/systems/monitoring.noc.eh22.intern/alerts/resources.yaml
index 5804cc3..b7459ac 100644
--- a/systems/monitoring.noc.eh22.intern/alerts/resources.yaml
+++ b/systems/monitoring.noc.eh22.intern/alerts/resources.yaml
@@ -1,89 +1,312 @@
+#https://samber.github.io/awesome-prometheus-alerts/rules#host-and-hardware
groups:
- - name: Host & hardware
- rules:
- - alert: HostOutOfMemory
- expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- for: 2m
- labels:
- severity: warning
- annotations:
- description: |-
- Node memory is filling up (< 10% left)
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host out of memory (instance {{ $labels.instance }})
- - alert: HostMemoryUnderMemoryPressure
- expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- for: 2m
- labels:
- severity: warning
- annotations:
- description: |-
- The node is under heavy memory pressure. High rate of major page faults
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host memory under memory pressure (instance {{ $labels.instance }})
- - alert: HostOutOfDiskSpace
- expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- for: 2m
- labels:
- severity: warning
- annotations:
- description: |-
- Disk is almost full (< 10% left)
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host out of disk space (instance {{ $labels.instance }})
- - alert: HostDiskWillFillIn24Hours
- expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- for: 2m
- labels:
- severity: warning
- annotations:
- description: |-
- Filesystem is predicted to run out of space within the next 24 hours at current write rate
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
- - alert: HostCpuIsUnderutilized
- expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- for: 1w
- labels:
- severity: info
- annotations:
- description: |-
- CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host CPU is underutilized (instance {{ $labels.instance }})
- - alert: HostCpuStealNoisyNeighbor
- expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- labels:
- severity: warning
- annotations:
- description: |-
- CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
- - alert: HostOomKillDetected
- expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- labels:
- severity: warning
- annotations:
- description: |-
- OOM kill detected
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host OOM kill detected (instance {{ $labels.instance }})
- - alert: HostNetworkInterfaceSaturated
- expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
- for: 1m
- labels:
- severity: warning
- annotations:
- description: |-
- The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
- VALUE = {{ $value }}
- LABELS = {{ $labels }}
- summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+
+- name: NodeExporter
+
+ rules:
+
+ - alert: HostOutOfMemory
+ expr: '(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host out of memory (instance {{ $labels.instance }})
+ description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostMemoryUnderMemoryPressure
+ expr: '(rate(node_vmstat_pgmajfault[5m]) > 1000)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host memory under memory pressure (instance {{ $labels.instance }})
+ description: "The node is under heavy memory pressure. High rate of loading memory pages from disk.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostUnusualNetworkThroughputIn
+ expr: '((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host unusual network throughput in (instance {{ $labels.instance }})
+ description: "Host receive bandwidth is high (>80%).\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostUnusualNetworkThroughputOut
+ expr: '((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host unusual network throughput out (instance {{ $labels.instance }})
+ description: "Host transmit bandwidth is high (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostUnusualDiskReadRate
+ expr: '(rate(node_disk_io_time_seconds_total[5m]) > .80)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host unusual disk read rate (instance {{ $labels.instance }})
+ description: "Disk is too busy (IO wait > 80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostOutOfDiskSpace
+ expr: '(node_filesystem_avail_bytes{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0)'
+ for: 2m
+ labels:
+ severity: critical
+ annotations:
+ summary: Host out of disk space (instance {{ $labels.instance }})
+ description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostDiskMayFillIn24Hours
+ expr: 'predict_linear(node_filesystem_avail_bytes{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"}[1h], 86400) <= 0 and node_filesystem_avail_bytes > 0'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host disk may fill in 24 hours (instance {{ $labels.instance }})
+ description: "Filesystem will likely run out of space within the next 24 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostOutOfInodes
+ expr: '(node_filesystem_files_free / node_filesystem_files < .10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0)'
+ for: 2m
+ labels:
+ severity: critical
+ annotations:
+ summary: Host out of inodes (instance {{ $labels.instance }})
+ description: "Disk is almost running out of available inodes (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostFilesystemDeviceError
+ expr: 'node_filesystem_device_error{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"} == 1'
+ for: 2m
+ labels:
+ severity: critical
+ annotations:
+ summary: Host filesystem device error (instance {{ $labels.instance }})
+ description: "Error stat-ing the {{ $labels.mountpoint }} filesystem\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostInodesMayFillIn24Hours
+ expr: 'predict_linear(node_filesystem_files_free{fstype!~"^(fuse.*|tmpfs|cifs|nfs)"}[1h], 86400) <= 0 and node_filesystem_files_free > 0'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host inodes may fill in 24 hours (instance {{ $labels.instance }})
+ description: "Filesystem will likely run out of inodes within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostUnusualDiskReadLatency
+ expr: '(rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host unusual disk read latency (instance {{ $labels.instance }})
+ description: "Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostUnusualDiskWriteLatency
+ expr: '(rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host unusual disk write latency (instance {{ $labels.instance }})
+ description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostHighCpuLoad
+ expr: '(avg by (instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > .80'
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host high CPU load (instance {{ $labels.instance }})
+ description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostCpuStealNoisyNeighbor
+ expr: 'avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+ description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostCpuHighIowait
+ expr: 'avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) > .10'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host CPU high iowait (instance {{ $labels.instance }})
+ description: "CPU iowait > 10%. Your CPU is idling waiting for storage to respond.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostUnusualDiskIo
+ expr: 'rate(node_disk_io_time_seconds_total[5m]) > 0.8'
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host unusual disk IO (instance {{ $labels.instance }})
+ description: "Disk usage >80%. Check storage for issues or increase IOPS capabilities. Check storage for issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostContextSwitchingHigh
+ expr: '(rate(node_context_switches_total[15m])/count without(mode,cpu) (node_cpu_seconds_total{mode="idle"})) / (rate(node_context_switches_total[1d])/count without(mode,cpu) (node_cpu_seconds_total{mode="idle"})) > 2'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host context switching high (instance {{ $labels.instance }})
+ description: "Context switching is growing on the node (twice the daily average during the last 15m)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostSwapIsFillingUp
+ expr: '((1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host swap is filling up (instance {{ $labels.instance }})
+ description: "Swap is filling up (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostSystemdServiceCrashed
+ expr: '(node_systemd_unit_state{state="failed"} == 1)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host systemd service crashed (instance {{ $labels.instance }})
+ description: "systemd service crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostPhysicalComponentTooHot
+ expr: 'node_hwmon_temp_celsius > node_hwmon_temp_max_celsius'
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host physical component too hot (instance {{ $labels.instance }})
+ description: "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostNodeOvertemperatureAlarm
+ expr: '((node_hwmon_temp_crit_alarm_celsius == 1) or (node_hwmon_temp_alarm == 1))'
+ for: 0m
+ labels:
+ severity: critical
+ annotations:
+ summary: Host node overtemperature alarm (instance {{ $labels.instance }})
+ description: "Physical node temperature alarm triggered\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostSoftwareRaidInsufficientDrives
+ expr: '((node_md_disks_required - on(device, instance) node_md_disks{state="active"}) > 0)'
+ for: 0m
+ labels:
+ severity: critical
+ annotations:
+ summary: Host software RAID insufficient drives (instance {{ $labels.instance }})
+ description: "MD RAID array {{ $labels.device }} on {{ $labels.instance }} has insufficient drives remaining.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostSoftwareRaidDiskFailure
+ expr: '(node_md_disks{state="failed"} > 0)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host software RAID disk failure (instance {{ $labels.instance }})
+ description: "MD RAID array {{ $labels.device }} on {{ $labels.instance }} needs attention.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostKernelVersionDeviations
+ expr: 'changes(node_uname_info[1h]) > 0'
+ for: 0m
+ labels:
+ severity: info
+ annotations:
+ summary: Host kernel version deviations (instance {{ $labels.instance }})
+ description: "Kernel version for {{ $labels.instance }} has changed.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostOomKillDetected
+ expr: '(increase(node_vmstat_oom_kill[1m]) > 0)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host OOM kill detected (instance {{ $labels.instance }})
+ description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostEdacCorrectableErrorsDetected
+ expr: '(increase(node_edac_correctable_errors_total[1m]) > 0)'
+ for: 0m
+ labels:
+ severity: info
+ annotations:
+ summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
+ description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostEdacUncorrectableErrorsDetected
+ expr: '(node_edac_uncorrectable_errors_total > 0)'
+ for: 0m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
+ description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostNetworkReceiveErrors
+ expr: '(rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host Network Receive Errors (instance {{ $labels.instance }})
+ description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostNetworkTransmitErrors
+ expr: '(rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host Network Transmit Errors (instance {{ $labels.instance }})
+ description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostNetworkBondDegraded
+ expr: '((node_bonding_active - node_bonding_slaves) != 0)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host Network Bond Degraded (instance {{ $labels.instance }})
+ description: "Bond \"{{ $labels.device }}\" degraded on \"{{ $labels.instance }}\".\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostConntrackLimit
+ expr: '(node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8)'
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host conntrack limit (instance {{ $labels.instance }})
+ description: "The number of conntrack is approaching limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostClockSkew
+ expr: '((node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0))'
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host clock skew (instance {{ $labels.instance }})
+ description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostClockNotSynchronising
+ expr: '(min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16)'
+ for: 2m
+ labels:
+ severity: warning
+ annotations:
+ summary: Host clock not synchronising (instance {{ $labels.instance }})
+ description: "Clock not synchronising. Ensure NTP is configured on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
+
+ - alert: HostRequiresReboot
+ expr: '(node_reboot_required > 0)'
+ for: 4h
+ labels:
+ severity: info
+ annotations:
+ summary: Host requires reboot (instance {{ $labels.instance }})
+ description: "{{ $labels.instance }} requires a reboot.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
diff --git a/systems/monitoring.noc.eh22.intern/grafana.nix b/systems/monitoring.noc.eh22.intern/grafana.nix
index 4128294..738023e 100644
--- a/systems/monitoring.noc.eh22.intern/grafana.nix
+++ b/systems/monitoring.noc.eh22.intern/grafana.nix
@@ -23,7 +23,7 @@
user = "grafana";
host = "/run/postgresql";
};
- feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
+ feature_toggles.enable = "newVizTooltips";
security.angular_support_enabled = false;
};
provision = {
@@ -48,7 +48,10 @@
type = "alertmanager";
uid = "mimir-alertmanager";
access = "proxy";
- url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/";
+ url = "http://mimir.noc.eh22.intern";
+ basicAuth = true;
+ basicAuthUser = "noc";
+ secureJsonData.basicAuthPassword = "$__file{${config.sops.secrets."services/mimir/basic_auth".path}}";
jsonData = {
handleGrafanaManagedAlerts = true;
implementation = "mimir";
diff --git a/systems/monitoring.noc.eh22.intern/mimir.nix b/systems/monitoring.noc.eh22.intern/mimir.nix
index 4c51b85..73508be 100644
--- a/systems/monitoring.noc.eh22.intern/mimir.nix
+++ b/systems/monitoring.noc.eh22.intern/mimir.nix
@@ -17,8 +17,11 @@ let
mkdir $out
cp -R $src $out/anonymous/
'';
+ template_alert = ./templates;
in
{
+ environment.etc."alertmanager/templates".source = template_alert;
+
services.mimir = {
enable = true;
configuration = {
@@ -36,7 +39,6 @@ in
server = {
http_listen_port = 9009;
http_listen_address = "127.0.0.1";
- log_level = "warn";
grpc_listen_port = 9096;
grpc_listen_address = "127.0.0.1";
};
@@ -61,9 +63,32 @@ in
replication_factor = 1;
instance_addr = "127.0.0.1";
};
+ utf8_strict_mode = true;
+ fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
+ route = {
+ group_by = ["alertname"];
+ receiver = "telegram";
+ group_wait = "30s";
+ group_interval = "1m";
+ repeat_interval = "3m";
+ };
+ receivers = [
+ {
+ name = "telegram";
+ telegram_configs = [{
+ bot_token_file = config.sops.secrets."services/telegrambot/token".path;
+ chat_id = -1002579132187;
+ }];
+ }
+ ];
+ templates = [
+ "/etc/alertmanager/templates/*.tmpl"
+ ];
+ };
};
ruler = {
+ alertmanager_url = "http://localhost/alertmanager";
ring = {
instance_addr = "127.0.0.1";
};
@@ -90,6 +115,16 @@ in
proxyPass = "http://mimir";
basicAuthFile = config.sops.secrets."services/mimir/nginx".path;
};
+ locations."/" = {
+ proxyPass = "http://mimir/";
+ basicAuthFile = config.sops.secrets."services/mimir/nginx".path;
+ };
+ };
+
+ virtualHosts."localhost" = {
+ locations."/" = {
+ proxyPass = "http://mimir/";
+ };
};
};
}
diff --git a/systems/monitoring.noc.eh22.intern/system.nix b/systems/monitoring.noc.eh22.intern/system.nix
index 09e4811..8cbc19b 100644
--- a/systems/monitoring.noc.eh22.intern/system.nix
+++ b/systems/monitoring.noc.eh22.intern/system.nix
@@ -31,6 +31,12 @@
group = "nginx";
restartUnits = [ "nginx.service" ];
};
+ secrets."services/telegrambot/token" = {
+ mode = "0444";
+ owner = "root";
+ group = "root";
+ restartUnits = [ "mimir.service" ];
+ };
};
networking.firewall.allowedTCPPorts = [ 80 ];
diff --git a/systems/monitoring.noc.eh22.intern/templates/alertmanager_alert_templates.tmpl b/systems/monitoring.noc.eh22.intern/templates/alertmanager_alert_templates.tmpl
new file mode 100644
index 0000000..c4e17f7
--- /dev/null
+++ b/systems/monitoring.noc.eh22.intern/templates/alertmanager_alert_templates.tmpl
@@ -0,0 +1,35 @@
+{{/*
+Links & Resources
+- https://prometheus.io/blog/2016/03/03/custom-alertmanager-templates/
+- https://prometheus.io/docs/alerting/latest/notifications/
+- https://gist.github.com/jidckii/5ac5f8f20368b56de72af70222509b7b
+*/}}
+{{ define "alert-item.telegram.eh22noc.internal" }}
+<b>[{{ .Labels.alertname }}] {{ .Labels.nodename }}</b>
+{{- if .Annotations.summary }}
+<i>Summary</i>: {{ .Annotations.summary }}
+{{- end }}
+{{- if .Annotations.description }}
+<i>Description</i>: {{ .Annotations.description }}
+{{- end }}
+<i>Labels</i>:
+{{ range .Labels.SortedPairs -}}
+• <i>{{ .Name }}</i>: <code>{{ .Value }}</code>
+{{ end }}
+{{- end }}
+
+
+{{ define "alert-message.telegram.eh22noc" }}
+{{- if .Alerts.Firing }}
+<u>🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥</u>
+{{ range .Alerts.Firing -}}
+{{ template "alert-item.telegram.eh22noc.internal" . }}
+{{- end }}
+{{- end }}
+{{- if .Alerts.Resolved }}
+<u>✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅</u>
+{{ range .Alerts.Resolved -}}
+{{ template "alert-item.telegram.eh22noc.internal" . }}
+{{- end }}
+{{- end }}
+{{- end }}