alertmanager

modules/alloy.nix

@@ -54,7 +54,19 @@
 
         rule {
           source_labels = ["__journal__systemd_unit"]
-          target_label  = "unit"
+          target_label  = "systemd_unit"
+        }
+        rule {
+          source_labels = ["__journal__hostname"]
+          target_label = "systemd_hostname"
+        }
+        rule {
+          source_labels = ["__journal__transport"]
+          target_label = "systemd_transport"
+        }
+        rule {
+          source_labels = ["__journal_syslog_identifier"]
+          target_label = "syslog_identifier"
         }
       }
 
@@ -65,7 +77,7 @@
       }
 
       logging {
-        level = "info"
+        level = "warn"
         format = "logfmt"
       }
 

systems/monitoring.noc.eh22.intern/alerts/resources.yaml

@@ -0,0 +1,89 @@
+groups:
+    - name: Host & hardware
+      rules:
+        - alert: HostOutOfMemory
+          expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Node memory is filling up (< 10% left)
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host out of memory (instance {{ $labels.instance }})
+        - alert: HostMemoryUnderMemoryPressure
+          expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The node is under heavy memory pressure. High rate of major page faults
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host memory under memory pressure (instance {{ $labels.instance }})
+        - alert: HostOutOfDiskSpace
+          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Disk is almost full (< 10% left)
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host out of disk space (instance {{ $labels.instance }})
+        - alert: HostDiskWillFillIn24Hours
+          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Filesystem is predicted to run out of space within the next 24 hours at current write rate
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+        - alert: HostCpuIsUnderutilized
+          expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1w
+          labels:
+            severity: info
+          annotations:
+            description: |-
+                CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host CPU is underutilized (instance {{ $labels.instance }})
+        - alert: HostCpuStealNoisyNeighbor
+          expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+        - alert: HostOomKillDetected
+          expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                OOM kill detected
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host OOM kill detected (instance {{ $labels.instance }})
+        - alert: HostNetworkInterfaceSaturated
+          expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host Network Interface Saturated (instance {{ $labels.instance }})

systems/monitoring.noc.eh22.intern/grafana.nix

@@ -41,7 +41,18 @@
             name = "Loki";
             type = "loki";
             uid = "loki";
-            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/";
+          }
+          {
+            name = "Mimir Alertmanager";
+            type = "alertmanager";
+            uid = "mimir-alertmanager";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/";
+            jsonData = {
+              handleGrafanaManagedAlerts = true;
+              implementation = "mimir";
+            };
           }
         ];
       };

systems/monitoring.noc.eh22.intern/mimir.nix

@@ -1,8 +1,23 @@
 {
   pkgs,
+  lib,
   config,
   ...
 }:
+let
+  inherit (lib) mkEnableOption mkIf;
+  alerts =
+    pkgs.runCommand "mimir-alerts-checked"
+      {
+        src = ./alerts;
+        nativeBuildInputs = with pkgs; [ prometheus.cli ];
+      }
+      ''
+        promtool check rules $src/*
+        mkdir $out
+        cp -R $src $out/anonymous/
+      '';
+in
 {
   services.mimir = {
     enable = true;
@@ -28,6 +43,16 @@
         max_global_series_per_user = 0; # unlimited
         max_label_value_length = 10000; # we have pgscv queries that are LONG
       };
+
+      alertmanager_storage.backend = "filesystem";
+      alertmanager = {
+        sharding_ring.replication_factor = 1;
+      };
+      ruler_storage = {
+        backend = "local";
+        local.directory = alerts;
+      };
+
     };
   };
 

systems/monitoring.noc.eh22.intern/system.nix

@@ -48,6 +48,12 @@
     ./loki.nix
   ];
 
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    logError = "syslog:server=unix:/dev/log,nohostname";
+  };
+
   # DO NOT CHANGE
   # this defines the first version of NixOS that was installed on the machine so that programs with non-migratable data files are kept compatible
   home-manager.users.noc.home.stateVersion = "24.11";