diff --git a/inventories/chaosknoten/host_vars/grafana.sops.yaml b/inventories/chaosknoten/host_vars/grafana.sops.yaml index 2c24fa9..8d5e665 100644 --- a/inventories/chaosknoten/host_vars/grafana.sops.yaml +++ b/inventories/chaosknoten/host_vars/grafana.sops.yaml @@ -8,14 +8,10 @@ secret__metrics_chaos: ENC[AES256_GCM,data:GDLtKMuExpedDFWLew68JMbdaxy1aEep2j4/X secret__metrics_chaos_basic_auth: ENC[AES256_GCM,data:eT39ijCsheJZP3D335EIRdeVR4nSX7APw9e4iQ40NtXz8EEfGg==,iv:+OxDeTOF8PLxSFT5ZKkUwWYZfuBgv5YUJSGWsURL2kk=,tag:0nIroxvAjTG0vB/lwq09LA==,type:str] secret__metrics_fux: ENC[AES256_GCM,data:aV6zeZ/XsVlA3QepSfVd/cOr+tqFVhlAxRO9SHx7,iv:fxo0o9amrh5ivPTxRVkvymB3fr5dLFVE7EqIpBlNZBk=,tag:41dm29mrV/jmqj5IkuNAaw==,type:str] secret__metrics_fux_basic_auth: ENC[AES256_GCM,data:YL+QLzZyyObzDcz+FcefViMrvdkVSwRhDsBx/AwoDX3RLHCDjg==,iv:GADdMa7FHMM1FnyPp8DUHElpXsJeqD+gN5Slw0R9bgs=,tag:KGCoEud2JLU5s1gurrbywg==,type:str] +secret__ntfy_token: ENC[AES256_GCM,data:0tuPJVmxHcdDWOMIo0QQXgIEkJo+p9A5emH+kc+U5tw=,iv:NZcfiz3UFw2fMcMf+q1GRp4Fsxpxbptsx9n8wPR54z0=,tag:SJYFtXccCbPrXjECiKUOUA==,type:str] sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2025-05-04T14:18:24Z" - mac: ENC[AES256_GCM,data:z0Fy/06LsPPCCc9tcejcq1g/Ieq/PI5/JedJQ8rKrpeBjKJ7rKUMbZipj2CCTpID8fYf75M6ekceMS9sAjNnPIRU4oJGjwp/nem0U5sjAVqNwW66X0JFSjkM2RRyn8rWR8wKRxEnZp3o1zp6bs0wDDd2nNckB+DocrlBbiRwbgc=,iv:AlN8MPHWPc3Boz4PqQOHDG//Hvu5jAQSy37rbnCOr3M=,tag:KCqbUChASbnKPUg628si0A==,type:str] + lastmodified: "2025-06-02T20:28:07Z" + mac: ENC[AES256_GCM,data:mrA/ytnxpotGkGLCLRAGEEEiQmhcVtsCcSguZ1hnF9Qw+sIt/QULImP0yTVpQIfn3nVYBKn06+ZfRab7hTO48YuF+w1l/hkqYIcfoiikREtO9IO+Z4LBRoh59SpfQuAFAfmdegu5iTp6cXCWrEg5LElQQP3yg930kNN/HIEpZhM=,iv:3MdudOS5QaEaRQUyFANXBga8gyrTkD/CTM6qrcH8nL4=,tag:AvxWzNVLD8gOF93LXoSavA==,type:str] pgp: - created_at: "2025-05-04T13:15:49Z" enc: |- @@ -242,4 +238,4 @@ sops: -----END PGP MESSAGE----- fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 unencrypted_suffix: _unencrypted - version: 3.9.4 + version: 3.10.2 diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index b1b17fa..d2304f9 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -14,6 +14,14 @@ docker_compose__configuration_files: content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl') }}" - name: loki.yaml content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/loki.yaml') }}" + - name: ntfy-alertmanager-ccchh-critical + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2') }}" + - name: ntfy-alertmanager-ccchh + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2') }}" + - name: ntfy-alertmanager-fux-critical + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2') }}" + - name: ntfy-alertmanager-fux + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2') }}" certbot__version_spec: "" certbot__acme_account_email_address: le-admin@hamburg.ccc.de diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 index 2219d3b..93fb68b 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 @@ -10,24 +10,21 @@ route: repeat_interval: 3h routes: - matchers: - - org="ccchh" - receiver: 'ccchh-infrastructure-alerts' - - -{# Disable these for now, but might be interesting in the future. -# Inhibition rules allow to mute a set of alerts given that another alert is -# firing. -# We use this to mute any warning-level notifications if the same alert is -# already critical. -inhibit_rules: - - source_matchers: [severity="critical"] - target_matchers: [severity="warning"] - # Apply inhibition if the alertname is the same. - # CAUTION: - # If all label names listed in `equal` are missing - # from both the source and target alerts, - # the inhibition rule will apply! - equal: [alertname, cluster, service] #} + - org = "ccchh" + - severity = "critical", + receiver: ntfy-ccchh-critical + - matchers: + - org = "fux" + - severity = "critical", + receiver: ntfy-fux-critical + - matchers: + - org = "ccchh" + - severity =~ "info|warning", + receiver: ntfy-ccchh + - matchers: + - org = "fux" + - severity =~ "info|warning", + receiver: ntfy-fux templates: - "/etc/alertmanager/templates/*.tmpl" @@ -40,3 +37,19 @@ receivers: chat_id: -1002434372415 parse_mode: HTML message: {{ "'{{ template \"alert-message.telegram.ccchh\" . }}'" }} + + - name: "ntfy-ccchh-critical" + webhook_configs: + - url: "http://ntfy-alertmanager-ccchh-critical:8000" + + - name: "ntfy-fux-critical" + webhook_configs: + - url: "http://ntfy-alertmanager-fux-critical:8001" + + - name: "ntfy-ccchh" + webhook_configs: + - url: "http://ntfy-alertmanager-ccchh:8010" + + - name: "ntfy-fux" + webhook_configs: + - url: "http://ntfy-alertmanager-fux:8011" diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 index 825871e..8e22dc1 100644 --- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 @@ -14,7 +14,7 @@ services: - ./configs/prometheus.yml:/etc/prometheus/prometheus.yml - ./configs/prometheus_alerts.rules.yaml:/etc/prometheus/rules/alerts.rules.yaml - prom_data:/prometheus - + alertmanager: image: prom/alertmanager container_name: alertmanager @@ -66,6 +66,42 @@ services: - ./configs/loki.yaml:/etc/loki/local-config.yaml - loki_data:/var/loki + ntfy-alertmanager-ccchh-critical: + image: xenrox/ntfy-alertmanager:latest + container_name: ntfy-alertmanager-ccchh-critical + volumes: + - ./configs/ntfy-alertmanager-ccchh-critical:/etc/ntfy-alertmanager/config + ports: + - 8000:8000 + restart: unless-stopped + + ntfy-alertmanager-fux-critical: + image: xenrox/ntfy-alertmanager:latest + container_name: ntfy-alertmanager-fux-critical + volumes: + - ./configs/ntfy-alertmanager-fux-critical:/etc/ntfy-alertmanager/config + ports: + - 8001:8001 + restart: unless-stopped + + ntfy-alertmanager-ccchh: + image: xenrox/ntfy-alertmanager:latest + container_name: ntfy-alertmanager-ccchh + volumes: + - ./configs/ntfy-alertmanager-ccchh:/etc/ntfy-alertmanager/config + ports: + - 8010:8010 + restart: unless-stopped + + ntfy-alertmanager-fux: + image: xenrox/ntfy-alertmanager:latest + container_name: ntfy-alertmanager-fux + volumes: + - ./configs/ntfy-alertmanager-fux:/etc/ntfy-alertmanager/config + ports: + - 8011:8011 + restart: unless-stopped + volumes: graf_data: {} prom_data: {} diff --git a/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2 b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2 new file mode 100644 index 0000000..03cc955 --- /dev/null +++ b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2 @@ -0,0 +1,39 @@ +http-address :8000 +log-level info +log-format text +# When multiple alerts are grouped together by Alertmanager, they can either be sent +# each on their own (single mode) or be kept together (multi mode) +# Options: single, multi +# Default: multi +alert-mode single + +labels { + order "severity" + + severity "critical" { + priority 4 + } + + severity "warning" { + priority 3 + } + + severity "info" { + priority 1 + } +} + +resolved { + tags "resolved" +} + +ntfy { + server https://ntfy.hamburg.ccc.de + topic ccchh-alertmanager-critical + access-token {{ secret__ntfy_token }} +} + +alertmanager { + silence-duration 1m +} + diff --git a/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2 b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2 new file mode 100644 index 0000000..e65b20c --- /dev/null +++ b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2 @@ -0,0 +1,39 @@ +http-address :8010 +log-level info +log-format text +# When multiple alerts are grouped together by Alertmanager, they can either be sent +# each on their own (single mode) or be kept together (multi mode) +# Options: single, multi +# Default: multi +alert-mode single + +labels { + order "severity" + + severity "critical" { + priority 4 + } + + severity "warning" { + priority 3 + } + + severity "info" { + priority 1 + } +} + +resolved { + tags "resolved" +} + +ntfy { + server https://ntfy.hamburg.ccc.de + topic ccchh-alertmanager + access-token {{ secret__ntfy_token }} +} + +alertmanager { + silence-duration 1m +} + diff --git a/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2 b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2 new file mode 100644 index 0000000..bede36a --- /dev/null +++ b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2 @@ -0,0 +1,39 @@ +http-address :8001 +log-level info +log-format text +# When multiple alerts are grouped together by Alertmanager, they can either be sent +# each on their own (single mode) or be kept together (multi mode) +# Options: single, multi +# Default: multi +alert-mode single + +labels { + order "severity" + + severity "critical" { + priority 4 + } + + severity "warning" { + priority 3 + } + + severity "info" { + priority 1 + } +} + +resolved { + tags "resolved" +} + +ntfy { + server https://ntfy.hamburg.ccc.de + topic fux-alertmanager-critical + access-token {{ secret__ntfy_token }} +} + +alertmanager { + silence-duration 1m +} + diff --git a/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2 b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2 new file mode 100644 index 0000000..df41e90 --- /dev/null +++ b/resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2 @@ -0,0 +1,39 @@ +http-address :8011 +log-level info +log-format text +# When multiple alerts are grouped together by Alertmanager, they can either be sent +# each on their own (single mode) or be kept together (multi mode) +# Options: single, multi +# Default: multi +alert-mode single + +labels { + order "severity" + + severity "critical" { + priority 4 + } + + severity "warning" { + priority 3 + } + + severity "info" { + priority 1 + } +} + +resolved { + tags "resolved" +} + +ntfy { + server https://ntfy.hamburg.ccc.de + topic fux-alertmanager + access-token {{ secret__ntfy_token }} +} + +alertmanager { + silence-duration 1m +} +