June
30876f821c
For now introduce node-exporter/hosts alert rules, which got taken from https://samber.github.io/awesome-prometheus-alerts/rules However with the labels removed from the description, since they don't render correctly (at least in Telegram) and don't seem to provide much value, as we render the labels in the notification anyway. Also only have Telegram as the notification channel for now, as it was the easiest to set up.
40 lines
1.4 KiB
Django/Jinja
40 lines
1.4 KiB
Django/Jinja
# Links & References:
|
|
# - https://prometheus.io/docs/alerting/latest/configuration/
|
|
# - https://github.com/prometheus/alertmanager/blob/48a99764a1fc9279fc828de83e7a03ae2219abc7/doc/examples/simple.yml
|
|
|
|
route:
|
|
group_by: ["alertname", "site", "type", "hypervisor"]
|
|
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 3h
|
|
|
|
receiver: ccchh-infrastructure-alerts
|
|
|
|
|
|
{# Disable these for now, but might be interesting in the future.
|
|
# Inhibition rules allow to mute a set of alerts given that another alert is
|
|
# firing.
|
|
# We use this to mute any warning-level notifications if the same alert is
|
|
# already critical.
|
|
inhibit_rules:
|
|
- source_matchers: [severity="critical"]
|
|
target_matchers: [severity="warning"]
|
|
# Apply inhibition if the alertname is the same.
|
|
# CAUTION:
|
|
# If all label names listed in `equal` are missing
|
|
# from both the source and target alerts,
|
|
# the inhibition rule will apply!
|
|
equal: [alertname, cluster, service] #}
|
|
|
|
templates:
|
|
- "/etc/alertmanager/templates/*.tmpl"
|
|
|
|
receivers:
|
|
- name: "ccchh-infrastructure-alerts"
|
|
telegram_configs:
|
|
- send_resolved: true
|
|
bot_token: {{ lookup("community.general.passwordstore", "noc/vm-secrets/chaosknoten/grafana/alertmanager_telegram_bot_token", create=false, missing="error") }}
|
|
chat_id: -1002434372415
|
|
parse_mode: HTML
|
|
message: {{ "'{{ template \"alert-message.telegram.ccchh\" . }}'" }}
|