This commit is contained in:
parent
15e200d96d
commit
3a9673b113
SSH key fingerprint:
SHA256:B3m+yzpaxGXSEcDBpPHfvza/DNC0wuX+CKMeGq8wgak
8 changed files with 236 additions and 27 deletions
|
|
@ -8,14 +8,10 @@ secret__metrics_chaos: ENC[AES256_GCM,data:GDLtKMuExpedDFWLew68JMbdaxy1aEep2j4/X
|
|||
secret__metrics_chaos_basic_auth: ENC[AES256_GCM,data:eT39ijCsheJZP3D335EIRdeVR4nSX7APw9e4iQ40NtXz8EEfGg==,iv:+OxDeTOF8PLxSFT5ZKkUwWYZfuBgv5YUJSGWsURL2kk=,tag:0nIroxvAjTG0vB/lwq09LA==,type:str]
|
||||
secret__metrics_fux: ENC[AES256_GCM,data:aV6zeZ/XsVlA3QepSfVd/cOr+tqFVhlAxRO9SHx7,iv:fxo0o9amrh5ivPTxRVkvymB3fr5dLFVE7EqIpBlNZBk=,tag:41dm29mrV/jmqj5IkuNAaw==,type:str]
|
||||
secret__metrics_fux_basic_auth: ENC[AES256_GCM,data:YL+QLzZyyObzDcz+FcefViMrvdkVSwRhDsBx/AwoDX3RLHCDjg==,iv:GADdMa7FHMM1FnyPp8DUHElpXsJeqD+gN5Slw0R9bgs=,tag:KGCoEud2JLU5s1gurrbywg==,type:str]
|
||||
secret__ntfy_token: ENC[AES256_GCM,data:0tuPJVmxHcdDWOMIo0QQXgIEkJo+p9A5emH+kc+U5tw=,iv:NZcfiz3UFw2fMcMf+q1GRp4Fsxpxbptsx9n8wPR54z0=,tag:SJYFtXccCbPrXjECiKUOUA==,type:str]
|
||||
sops:
|
||||
kms: []
|
||||
gcp_kms: []
|
||||
azure_kv: []
|
||||
hc_vault: []
|
||||
age: []
|
||||
lastmodified: "2025-05-04T14:18:24Z"
|
||||
mac: ENC[AES256_GCM,data:z0Fy/06LsPPCCc9tcejcq1g/Ieq/PI5/JedJQ8rKrpeBjKJ7rKUMbZipj2CCTpID8fYf75M6ekceMS9sAjNnPIRU4oJGjwp/nem0U5sjAVqNwW66X0JFSjkM2RRyn8rWR8wKRxEnZp3o1zp6bs0wDDd2nNckB+DocrlBbiRwbgc=,iv:AlN8MPHWPc3Boz4PqQOHDG//Hvu5jAQSy37rbnCOr3M=,tag:KCqbUChASbnKPUg628si0A==,type:str]
|
||||
lastmodified: "2025-06-02T20:28:07Z"
|
||||
mac: ENC[AES256_GCM,data:mrA/ytnxpotGkGLCLRAGEEEiQmhcVtsCcSguZ1hnF9Qw+sIt/QULImP0yTVpQIfn3nVYBKn06+ZfRab7hTO48YuF+w1l/hkqYIcfoiikREtO9IO+Z4LBRoh59SpfQuAFAfmdegu5iTp6cXCWrEg5LElQQP3yg930kNN/HIEpZhM=,iv:3MdudOS5QaEaRQUyFANXBga8gyrTkD/CTM6qrcH8nL4=,tag:AvxWzNVLD8gOF93LXoSavA==,type:str]
|
||||
pgp:
|
||||
- created_at: "2025-05-04T13:15:49Z"
|
||||
enc: |-
|
||||
|
|
@ -242,4 +238,4 @@ sops:
|
|||
-----END PGP MESSAGE-----
|
||||
fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.9.4
|
||||
version: 3.10.2
|
||||
|
|
|
|||
|
|
@ -14,6 +14,14 @@ docker_compose__configuration_files:
|
|||
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl') }}"
|
||||
- name: loki.yaml
|
||||
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/loki.yaml') }}"
|
||||
- name: ntfy-alertmanager-ccchh-critical
|
||||
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh-critical.j2') }}"
|
||||
- name: ntfy-alertmanager-ccchh
|
||||
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-ccchh.j2') }}"
|
||||
- name: ntfy-alertmanager-fux-critical
|
||||
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2') }}"
|
||||
- name: ntfy-alertmanager-fux
|
||||
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2') }}"
|
||||
|
||||
certbot__version_spec: ""
|
||||
certbot__acme_account_email_address: le-admin@hamburg.ccc.de
|
||||
|
|
|
|||
|
|
@ -10,24 +10,21 @@ route:
|
|||
repeat_interval: 3h
|
||||
routes:
|
||||
- matchers:
|
||||
- org="ccchh"
|
||||
receiver: 'ccchh-infrastructure-alerts'
|
||||
|
||||
|
||||
{# Disable these for now, but might be interesting in the future.
|
||||
# Inhibition rules allow to mute a set of alerts given that another alert is
|
||||
# firing.
|
||||
# We use this to mute any warning-level notifications if the same alert is
|
||||
# already critical.
|
||||
inhibit_rules:
|
||||
- source_matchers: [severity="critical"]
|
||||
target_matchers: [severity="warning"]
|
||||
# Apply inhibition if the alertname is the same.
|
||||
# CAUTION:
|
||||
# If all label names listed in `equal` are missing
|
||||
# from both the source and target alerts,
|
||||
# the inhibition rule will apply!
|
||||
equal: [alertname, cluster, service] #}
|
||||
- org = "ccchh"
|
||||
- severity = "critical",
|
||||
receiver: ntfy-ccchh-critical
|
||||
- matchers:
|
||||
- org = "fux"
|
||||
- severity = "critical",
|
||||
receiver: ntfy-fux-critical
|
||||
- matchers:
|
||||
- org = "ccchh"
|
||||
- severity =~ "info|warning",
|
||||
receiver: ntfy-ccchh
|
||||
- matchers:
|
||||
- org = "fux"
|
||||
- severity =~ "info|warning",
|
||||
receiver: ntfy-fux
|
||||
|
||||
templates:
|
||||
- "/etc/alertmanager/templates/*.tmpl"
|
||||
|
|
@ -40,3 +37,19 @@ receivers:
|
|||
chat_id: -1002434372415
|
||||
parse_mode: HTML
|
||||
message: {{ "'{{ template \"alert-message.telegram.ccchh\" . }}'" }}
|
||||
|
||||
- name: "ntfy-ccchh-critical"
|
||||
webhook_configs:
|
||||
- url: "http://ntfy-alertmanager-ccchh-critical:8000"
|
||||
|
||||
- name: "ntfy-fux-critical"
|
||||
webhook_configs:
|
||||
- url: "http://ntfy-alertmanager-fux-critical:8001"
|
||||
|
||||
- name: "ntfy-ccchh"
|
||||
webhook_configs:
|
||||
- url: "http://ntfy-alertmanager-ccchh:8010"
|
||||
|
||||
- name: "ntfy-fux"
|
||||
webhook_configs:
|
||||
- url: "http://ntfy-alertmanager-fux:8011"
|
||||
|
|
|
|||
|
|
@ -66,6 +66,42 @@ services:
|
|||
- ./configs/loki.yaml:/etc/loki/local-config.yaml
|
||||
- loki_data:/var/loki
|
||||
|
||||
ntfy-alertmanager-ccchh-critical:
|
||||
image: xenrox/ntfy-alertmanager:latest
|
||||
container_name: ntfy-alertmanager-ccchh-critical
|
||||
volumes:
|
||||
- ./configs/ntfy-alertmanager-ccchh-critical:/etc/ntfy-alertmanager/config
|
||||
ports:
|
||||
- 8000:8000
|
||||
restart: unless-stopped
|
||||
|
||||
ntfy-alertmanager-fux-critical:
|
||||
image: xenrox/ntfy-alertmanager:latest
|
||||
container_name: ntfy-alertmanager-fux-critical
|
||||
volumes:
|
||||
- ./configs/ntfy-alertmanager-fux-critical:/etc/ntfy-alertmanager/config
|
||||
ports:
|
||||
- 8001:8001
|
||||
restart: unless-stopped
|
||||
|
||||
ntfy-alertmanager-ccchh:
|
||||
image: xenrox/ntfy-alertmanager:latest
|
||||
container_name: ntfy-alertmanager-ccchh
|
||||
volumes:
|
||||
- ./configs/ntfy-alertmanager-ccchh:/etc/ntfy-alertmanager/config
|
||||
ports:
|
||||
- 8010:8010
|
||||
restart: unless-stopped
|
||||
|
||||
ntfy-alertmanager-fux:
|
||||
image: xenrox/ntfy-alertmanager:latest
|
||||
container_name: ntfy-alertmanager-fux
|
||||
volumes:
|
||||
- ./configs/ntfy-alertmanager-fux:/etc/ntfy-alertmanager/config
|
||||
ports:
|
||||
- 8011:8011
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
graf_data: {}
|
||||
prom_data: {}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
http-address :8000
|
||||
log-level info
|
||||
log-format text
|
||||
# When multiple alerts are grouped together by Alertmanager, they can either be sent
|
||||
# each on their own (single mode) or be kept together (multi mode)
|
||||
# Options: single, multi
|
||||
# Default: multi
|
||||
alert-mode single
|
||||
|
||||
labels {
|
||||
order "severity"
|
||||
|
||||
severity "critical" {
|
||||
priority 4
|
||||
}
|
||||
|
||||
severity "warning" {
|
||||
priority 3
|
||||
}
|
||||
|
||||
severity "info" {
|
||||
priority 1
|
||||
}
|
||||
}
|
||||
|
||||
resolved {
|
||||
tags "resolved"
|
||||
}
|
||||
|
||||
ntfy {
|
||||
server https://ntfy.hamburg.ccc.de
|
||||
topic ccchh-alertmanager-critical
|
||||
access-token {{ secret__ntfy_token }}
|
||||
}
|
||||
|
||||
alertmanager {
|
||||
silence-duration 1m
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
http-address :8010
|
||||
log-level info
|
||||
log-format text
|
||||
# When multiple alerts are grouped together by Alertmanager, they can either be sent
|
||||
# each on their own (single mode) or be kept together (multi mode)
|
||||
# Options: single, multi
|
||||
# Default: multi
|
||||
alert-mode single
|
||||
|
||||
labels {
|
||||
order "severity"
|
||||
|
||||
severity "critical" {
|
||||
priority 4
|
||||
}
|
||||
|
||||
severity "warning" {
|
||||
priority 3
|
||||
}
|
||||
|
||||
severity "info" {
|
||||
priority 1
|
||||
}
|
||||
}
|
||||
|
||||
resolved {
|
||||
tags "resolved"
|
||||
}
|
||||
|
||||
ntfy {
|
||||
server https://ntfy.hamburg.ccc.de
|
||||
topic ccchh-alertmanager
|
||||
access-token {{ secret__ntfy_token }}
|
||||
}
|
||||
|
||||
alertmanager {
|
||||
silence-duration 1m
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
http-address :8001
|
||||
log-level info
|
||||
log-format text
|
||||
# When multiple alerts are grouped together by Alertmanager, they can either be sent
|
||||
# each on their own (single mode) or be kept together (multi mode)
|
||||
# Options: single, multi
|
||||
# Default: multi
|
||||
alert-mode single
|
||||
|
||||
labels {
|
||||
order "severity"
|
||||
|
||||
severity "critical" {
|
||||
priority 4
|
||||
}
|
||||
|
||||
severity "warning" {
|
||||
priority 3
|
||||
}
|
||||
|
||||
severity "info" {
|
||||
priority 1
|
||||
}
|
||||
}
|
||||
|
||||
resolved {
|
||||
tags "resolved"
|
||||
}
|
||||
|
||||
ntfy {
|
||||
server https://ntfy.hamburg.ccc.de
|
||||
topic fux-alertmanager-critical
|
||||
access-token {{ secret__ntfy_token }}
|
||||
}
|
||||
|
||||
alertmanager {
|
||||
silence-duration 1m
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
http-address :8011
|
||||
log-level info
|
||||
log-format text
|
||||
# When multiple alerts are grouped together by Alertmanager, they can either be sent
|
||||
# each on their own (single mode) or be kept together (multi mode)
|
||||
# Options: single, multi
|
||||
# Default: multi
|
||||
alert-mode single
|
||||
|
||||
labels {
|
||||
order "severity"
|
||||
|
||||
severity "critical" {
|
||||
priority 4
|
||||
}
|
||||
|
||||
severity "warning" {
|
||||
priority 3
|
||||
}
|
||||
|
||||
severity "info" {
|
||||
priority 1
|
||||
}
|
||||
}
|
||||
|
||||
resolved {
|
||||
tags "resolved"
|
||||
}
|
||||
|
||||
ntfy {
|
||||
server https://ntfy.hamburg.ccc.de
|
||||
topic fux-alertmanager
|
||||
access-token {{ secret__ntfy_token }}
|
||||
}
|
||||
|
||||
alertmanager {
|
||||
silence-duration 1m
|
||||
}
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue