ntfy alerts

resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2

@@ -10,24 +10,21 @@ route:
   repeat_interval: 3h
   routes:
     - matchers:
-        - org="ccchh"
-      receiver: 'ccchh-infrastructure-alerts'
-
-
-{# Disable these for now, but might be interesting in the future.
-# Inhibition rules allow to mute a set of alerts given that another alert is
-# firing.
-# We use this to mute any warning-level notifications if the same alert is
-# already critical.
-inhibit_rules:
-  - source_matchers: [severity="critical"]
-    target_matchers: [severity="warning"]
-    # Apply inhibition if the alertname is the same.
-    # CAUTION:
-    #   If all label names listed in `equal` are missing
-    #   from both the source and target alerts,
-    #   the inhibition rule will apply!
-    equal: [alertname, cluster, service] #}
+        - org = "ccchh"
+        - severity = "critical",
+      receiver: ntfy-ccchh-critical
+    - matchers:
+        - org = "fux"
+        - severity = "critical",
+      receiver: ntfy-fux-critical
+    - matchers:
+      - org = "ccchh"
+      - severity =~ "info|warning",
+      receiver: ntfy-ccchh
+    - matchers:
+      - org = "fux"
+      - severity =~ "info|warning",
+      receiver: ntfy-fux
 
 templates:
   - "/etc/alertmanager/templates/*.tmpl"
@@ -40,3 +37,19 @@ receivers:
         chat_id: -1002434372415
         parse_mode: HTML
         message: {{ "'{{ template \"alert-message.telegram.ccchh\" . }}'" }}
+
+  - name: "ntfy-ccchh-critical"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-ccchh-critical:8000"
+
+  - name: "ntfy-fux-critical"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-fux-critical:8001"
+
+  - name: "ntfy-ccchh"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-ccchh:8010"
+
+  - name: "ntfy-fux"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-fux:8011"

resources/chaosknoten/grafana/docker_compose/compose.yaml.j2

@@ -14,7 +14,7 @@ services:
       - ./configs/prometheus.yml:/etc/prometheus/prometheus.yml
       - ./configs/prometheus_alerts.rules.yaml:/etc/prometheus/rules/alerts.rules.yaml
       - prom_data:/prometheus
-  
+
   alertmanager:
     image: prom/alertmanager
     container_name: alertmanager
@@ -66,6 +66,42 @@ services:
       - ./configs/loki.yaml:/etc/loki/local-config.yaml
       - loki_data:/var/loki
 
+  ntfy-alertmanager-ccchh-critical:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-ccchh-critical
+    volumes:
+      - ./configs/ntfy-alertmanager-ccchh-critical:/etc/ntfy-alertmanager/config
+    ports:
+      - 8000:8000
+    restart: unless-stopped
+
+  ntfy-alertmanager-fux-critical:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-fux-critical
+    volumes:
+      - ./configs/ntfy-alertmanager-fux-critical:/etc/ntfy-alertmanager/config
+    ports:
+      - 8001:8001
+    restart: unless-stopped
+
+  ntfy-alertmanager-ccchh:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-ccchh
+    volumes:
+      - ./configs/ntfy-alertmanager-ccchh:/etc/ntfy-alertmanager/config
+    ports:
+      - 8010:8010
+    restart: unless-stopped
+
+  ntfy-alertmanager-fux:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-fux
+    volumes:
+      - ./configs/ntfy-alertmanager-fux:/etc/ntfy-alertmanager/config
+    ports:
+      - 8011:8011
+    restart: unless-stopped
+
 volumes:
   graf_data: {}
   prom_data: {}

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2

@@ -0,0 +1,39 @@
+http-address :8000
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic ccchh-alertmanager-critical
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2

@@ -0,0 +1,39 @@
+http-address :8010
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic ccchh-alertmanager
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2

@@ -0,0 +1,39 @@
+http-address :8001
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic fux-alertmanager-critical
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2

@@ -0,0 +1,39 @@
+http-address :8011
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic fux-alertmanager
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+