ntfy alerts

inventories/chaosknoten/host_vars/grafana.sops.yaml

@@ -8,14 +8,10 @@ secret__metrics_chaos: ENC[AES256_GCM,data:GDLtKMuExpedDFWLew68JMbdaxy1aEep2j4/X
 secret__metrics_chaos_basic_auth: ENC[AES256_GCM,data:eT39ijCsheJZP3D335EIRdeVR4nSX7APw9e4iQ40NtXz8EEfGg==,iv:+OxDeTOF8PLxSFT5ZKkUwWYZfuBgv5YUJSGWsURL2kk=,tag:0nIroxvAjTG0vB/lwq09LA==,type:str]
 secret__metrics_fux: ENC[AES256_GCM,data:aV6zeZ/XsVlA3QepSfVd/cOr+tqFVhlAxRO9SHx7,iv:fxo0o9amrh5ivPTxRVkvymB3fr5dLFVE7EqIpBlNZBk=,tag:41dm29mrV/jmqj5IkuNAaw==,type:str]
 secret__metrics_fux_basic_auth: ENC[AES256_GCM,data:YL+QLzZyyObzDcz+FcefViMrvdkVSwRhDsBx/AwoDX3RLHCDjg==,iv:GADdMa7FHMM1FnyPp8DUHElpXsJeqD+gN5Slw0R9bgs=,tag:KGCoEud2JLU5s1gurrbywg==,type:str]
+secret__ntfy_token: ENC[AES256_GCM,data:0tuPJVmxHcdDWOMIo0QQXgIEkJo+p9A5emH+kc+U5tw=,iv:NZcfiz3UFw2fMcMf+q1GRp4Fsxpxbptsx9n8wPR54z0=,tag:SJYFtXccCbPrXjECiKUOUA==,type:str]
 sops:
-  kms: []
-  gcp_kms: []
-  azure_kv: []
-  hc_vault: []
-  age: []
-  lastmodified: "2025-05-04T14:18:24Z"
-  mac: ENC[AES256_GCM,data:z0Fy/06LsPPCCc9tcejcq1g/Ieq/PI5/JedJQ8rKrpeBjKJ7rKUMbZipj2CCTpID8fYf75M6ekceMS9sAjNnPIRU4oJGjwp/nem0U5sjAVqNwW66X0JFSjkM2RRyn8rWR8wKRxEnZp3o1zp6bs0wDDd2nNckB+DocrlBbiRwbgc=,iv:AlN8MPHWPc3Boz4PqQOHDG//Hvu5jAQSy37rbnCOr3M=,tag:KCqbUChASbnKPUg628si0A==,type:str]
+  lastmodified: "2025-06-02T20:28:07Z"
+  mac: ENC[AES256_GCM,data:mrA/ytnxpotGkGLCLRAGEEEiQmhcVtsCcSguZ1hnF9Qw+sIt/QULImP0yTVpQIfn3nVYBKn06+ZfRab7hTO48YuF+w1l/hkqYIcfoiikREtO9IO+Z4LBRoh59SpfQuAFAfmdegu5iTp6cXCWrEg5LElQQP3yg930kNN/HIEpZhM=,iv:3MdudOS5QaEaRQUyFANXBga8gyrTkD/CTM6qrcH8nL4=,tag:AvxWzNVLD8gOF93LXoSavA==,type:str]
   pgp:
     - created_at: "2025-05-04T13:15:49Z"
       enc: |-
@@ -242,4 +238,4 @@ sops:
         -----END PGP MESSAGE-----
       fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
   unencrypted_suffix: _unencrypted
-  version: 3.9.4
+  version: 3.10.2

inventories/chaosknoten/host_vars/grafana.yaml

@@ -14,6 +14,14 @@ docker_compose__configuration_files:
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl') }}"
   - name: loki.yaml
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/loki.yaml') }}"
+  - name: ntfy-alertmanager-ccchh-critical
+    content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2') }}"
+  - name: ntfy-alertmanager-ccchh
+    content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2') }}"
+  - name: ntfy-alertmanager-fux-critical
+    content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2') }}"
+  - name: ntfy-alertmanager-fux
+    content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2') }}"
 
 certbot__version_spec: ""
 certbot__acme_account_email_address: le-admin@hamburg.ccc.de

resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2

@@ -10,24 +10,21 @@ route:
   repeat_interval: 3h
   routes:
     - matchers:
-        - org="ccchh"
-      receiver: 'ccchh-infrastructure-alerts'
-
-
-{# Disable these for now, but might be interesting in the future.
-# Inhibition rules allow to mute a set of alerts given that another alert is
-# firing.
-# We use this to mute any warning-level notifications if the same alert is
-# already critical.
-inhibit_rules:
-  - source_matchers: [severity="critical"]
-    target_matchers: [severity="warning"]
-    # Apply inhibition if the alertname is the same.
-    # CAUTION:
-    #   If all label names listed in `equal` are missing
-    #   from both the source and target alerts,
-    #   the inhibition rule will apply!
-    equal: [alertname, cluster, service] #}
+        - org = "ccchh"
+        - severity = "critical",
+      receiver: ntfy-ccchh-critical
+    - matchers:
+        - org = "fux"
+        - severity = "critical",
+      receiver: ntfy-fux-critical
+    - matchers:
+      - org = "ccchh"
+      - severity =~ "info|warning",
+      receiver: ntfy-ccchh
+    - matchers:
+      - org = "fux"
+      - severity =~ "info|warning",
+      receiver: ntfy-fux
 
 templates:
   - "/etc/alertmanager/templates/*.tmpl"
@@ -40,3 +37,19 @@ receivers:
         chat_id: -1002434372415
         parse_mode: HTML
         message: {{ "'{{ template \"alert-message.telegram.ccchh\" . }}'" }}
+
+  - name: "ntfy-ccchh-critical"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-ccchh-critical:8000"
+
+  - name: "ntfy-fux-critical"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-fux-critical:8001"
+
+  - name: "ntfy-ccchh"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-ccchh:8010"
+
+  - name: "ntfy-fux"
+    webhook_configs:
+      - url: "http://ntfy-alertmanager-fux:8011"

resources/chaosknoten/grafana/docker_compose/compose.yaml.j2

@@ -14,7 +14,7 @@ services:
       - ./configs/prometheus.yml:/etc/prometheus/prometheus.yml
       - ./configs/prometheus_alerts.rules.yaml:/etc/prometheus/rules/alerts.rules.yaml
       - prom_data:/prometheus
-  
+
   alertmanager:
     image: prom/alertmanager
     container_name: alertmanager
@@ -66,6 +66,42 @@ services:
       - ./configs/loki.yaml:/etc/loki/local-config.yaml
       - loki_data:/var/loki
 
+  ntfy-alertmanager-ccchh-critical:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-ccchh-critical
+    volumes:
+      - ./configs/ntfy-alertmanager-ccchh-critical:/etc/ntfy-alertmanager/config
+    ports:
+      - 8000:8000
+    restart: unless-stopped
+
+  ntfy-alertmanager-fux-critical:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-fux-critical
+    volumes:
+      - ./configs/ntfy-alertmanager-fux-critical:/etc/ntfy-alertmanager/config
+    ports:
+      - 8001:8001
+    restart: unless-stopped
+
+  ntfy-alertmanager-ccchh:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-ccchh
+    volumes:
+      - ./configs/ntfy-alertmanager-ccchh:/etc/ntfy-alertmanager/config
+    ports:
+      - 8010:8010
+    restart: unless-stopped
+
+  ntfy-alertmanager-fux:
+    image: xenrox/ntfy-alertmanager:latest
+    container_name: ntfy-alertmanager-fux
+    volumes:
+      - ./configs/ntfy-alertmanager-fux:/etc/ntfy-alertmanager/config
+    ports:
+      - 8011:8011
+    restart: unless-stopped
+
 volumes:
   graf_data: {}
   prom_data: {}

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2

@@ -0,0 +1,39 @@
+http-address :8000
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic ccchh-alertmanager-critical
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2

@@ -0,0 +1,39 @@
+http-address :8010
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic ccchh-alertmanager
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2

@@ -0,0 +1,39 @@
+http-address :8001
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic fux-alertmanager-critical
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+

resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2

@@ -0,0 +1,39 @@
+http-address :8011
+log-level info
+log-format text
+# When multiple alerts are grouped together by Alertmanager, they can either be sent
+# each on their own (single mode) or be kept together (multi mode)
+# Options: single, multi
+# Default: multi
+alert-mode single
+
+labels {
+    order "severity"
+
+    severity "critical" {
+        priority 4
+    }
+
+    severity "warning" {
+        priority 3
+    }
+
+    severity "info" {
+        priority 1
+    }
+}
+
+resolved {
+    tags "resolved"
+}
+
+ntfy {
+    server https://ntfy.hamburg.ccc.de
+    topic fux-alertmanager
+    access-token {{ secret__ntfy_token }}
+}
+
+alertmanager {
+    silence-duration 1m
+}
+