From 0e61131c1bf4da86724f44ab3c3b9de054032461 Mon Sep 17 00:00:00 2001 From: c6ristian Date: Sun, 1 Jun 2025 01:33:10 +0200 Subject: [PATCH] prometheus: pre filtering setup --- .../grafana/docker_compose/alertmanager.yaml.j2 | 10 ++++++---- .../alertmanager_alert_templates.tmpl | 4 ++-- .../grafana/docker_compose/prometheus.yml | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 index 6d550ed..2219d3b 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 @@ -3,13 +3,15 @@ # - https://github.com/prometheus/alertmanager/blob/48a99764a1fc9279fc828de83e7a03ae2219abc7/doc/examples/simple.yml route: - group_by: ["alertname", "site", "type", "hypervisor"] - + receiver: 'ccchh-infrastructure-alerts' + group_by: [ "alertname", "site", "type", "hypervisor" ] group_wait: 30s group_interval: 5m repeat_interval: 3h - - receiver: ccchh-infrastructure-alerts + routes: + - matchers: + - org="ccchh" + receiver: 'ccchh-infrastructure-alerts' {# Disable these for now, but might be interesting in the future. diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl b/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl index 4651496..3e97e6e 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl @@ -22,7 +22,7 @@ Links & Resources {{ define "alert-message.telegram.ccchh" }} {{- if .Alerts.Firing }} 🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥 - {{- if le (len .Alerts.Firing) 6 }} + {{- if le (len .Alerts.Firing) 5 }} {{- range .Alerts.Firing }} {{ template "alert-item.telegram.ccchh.internal" . }} {{- end }} @@ -33,7 +33,7 @@ Links & Resources {{- if .Alerts.Resolved }} ✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅ - {{- if le (len .Alerts.Resolved) 6 }} + {{- if le (len .Alerts.Resolved) 5 }} {{- range .Alerts.Resolved }} {{ template "alert-item.telegram.ccchh.internal" . }} {{- end }} diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus.yml b/resources/chaosknoten/grafana/docker_compose/prometheus.yml index cf7f594..769cdc8 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus.yml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus.yml @@ -22,6 +22,8 @@ scrape_configs: static_configs: - targets: - localhost:9090 + labels: + org: ccchh - job_name: alertmanager honor_timestamps: true metrics_path: /metrics @@ -29,6 +31,8 @@ scrape_configs: static_configs: - targets: - alertmanager:9093 + labels: + org: ccchh - job_name: mumble honor_timestamps: true scrape_interval: 5s @@ -38,6 +42,8 @@ scrape_configs: static_configs: - targets: - mumble.hamburg.ccc.de:443 + labels: + org: ccchh - job_name: opnsense-ccchh honor_timestamps: true metrics_path: /metrics @@ -45,6 +51,8 @@ scrape_configs: static_configs: - targets: - 185.161.129.132:9100 + labels: + org: ccchh - job_name: jitsi honor_timestamps: true scrape_interval: 5s @@ -54,10 +62,14 @@ scrape_configs: static_configs: - targets: - jitsi.hamburg.ccc.de:9888 # Jitsi Video Bridge + labels: + org: ccchh - job_name: 'pve' static_configs: - targets: - 212.12.48.126 # chaosknoten + labels: + org: ccchh metrics_path: /pve params: module: [ default ] @@ -74,6 +86,7 @@ scrape_configs: static_configs: # Wieske Chaosknoten VMs - labels: + org: ccchh site: wieske type: virtual_machine hypervisor: chaosknoten @@ -98,6 +111,7 @@ scrape_configs: - zammad-intern.hamburg.ccc.de:9100 - pretalx-intern.hamburg.ccc.de:9100 - labels: + org: ccchh site: wieske type: physical_machine targets: