prometheus: pre filtering setup

resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2

@@ -3,13 +3,15 @@
 # - https://github.com/prometheus/alertmanager/blob/48a99764a1fc9279fc828de83e7a03ae2219abc7/doc/examples/simple.yml
 
 route:
-  group_by: ["alertname", "site", "type", "hypervisor"]
+  receiver: 'ccchh-infrastructure-alerts'
-
+  group_by: [ "alertname", "site", "type", "hypervisor" ]
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 3h
-
+  routes:
-  receiver: ccchh-infrastructure-alerts
+    - matchers:
+        - org="ccchh"
+      receiver: 'ccchh-infrastructure-alerts'
 
 
 {# Disable these for now, but might be interesting in the future.

resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl

@@ -22,7 +22,7 @@ Links & Resources
 {{ define "alert-message.telegram.ccchh" }}
   {{- if .Alerts.Firing }}
     <u>🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥</u>
-    {{- if le (len .Alerts.Firing) 6 }}
+    {{- if le (len .Alerts.Firing) 5 }}
       {{- range .Alerts.Firing }}
         {{ template "alert-item.telegram.ccchh.internal" . }}
       {{- end }}
@@ -33,7 +33,7 @@ Links & Resources
 
   {{- if .Alerts.Resolved }}
     <u>✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅</u>
-    {{- if le (len .Alerts.Resolved) 6 }}
+    {{- if le (len .Alerts.Resolved) 5 }}
       {{- range .Alerts.Resolved }}
         {{ template "alert-item.telegram.ccchh.internal" . }}
       {{- end }}

resources/chaosknoten/grafana/docker_compose/prometheus.yml

@@ -22,6 +22,8 @@ scrape_configs:
     static_configs:
       - targets:
           - localhost:9090
+        labels:
+          org: ccchh
   - job_name: alertmanager
     honor_timestamps: true
     metrics_path: /metrics
@@ -29,6 +31,8 @@ scrape_configs:
     static_configs:
       - targets:
           - alertmanager:9093
+        labels:
+          org: ccchh
   - job_name: mumble
     honor_timestamps: true
     scrape_interval: 5s
@@ -38,6 +42,8 @@ scrape_configs:
     static_configs:
       - targets:
           - mumble.hamburg.ccc.de:443
+        labels:
+          org: ccchh
   - job_name: opnsense-ccchh
     honor_timestamps: true
     metrics_path: /metrics
@@ -45,6 +51,8 @@ scrape_configs:
     static_configs:
       - targets:
           - 185.161.129.132:9100
+        labels:
+          org: ccchh
   - job_name: jitsi
     honor_timestamps: true
     scrape_interval: 5s
@@ -54,10 +62,14 @@ scrape_configs:
     static_configs:
       - targets:
           - jitsi.hamburg.ccc.de:9888 # Jitsi Video Bridge
+        labels:
+          org: ccchh
   - job_name: 'pve'
     static_configs:
       - targets:
           - 212.12.48.126  # chaosknoten
+        labels:
+          org: ccchh
     metrics_path: /pve
     params:
       module: [ default ]
@@ -74,6 +86,7 @@ scrape_configs:
     static_configs:
       # Wieske Chaosknoten VMs
       - labels:
+          org: ccchh
           site: wieske
           type: virtual_machine
           hypervisor: chaosknoten
@@ -98,6 +111,7 @@ scrape_configs:
           - zammad-intern.hamburg.ccc.de:9100
           - pretalx-intern.hamburg.ccc.de:9100
       - labels:
+          org: ccchh
           site: wieske
           type: physical_machine
         targets: