WIP

inventories/chaosknoten/host_vars/grafana.yaml

@@ -10,6 +10,8 @@ docker_compose__configuration_files:
     content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2') }}"
   - name: prometheus_alerts.rules.yaml
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml') }}"
+  - name: prometheus_alerts-fux.rules.yaml
+    content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/prometheus_alerts-fux.rules.yaml') }}"
   - name: alertmanager_alert_templates.tmpl
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl') }}"
   - name: loki.yaml

resources/chaosknoten/grafana/docker_compose/prometheus_alerts-fux.rules.yaml

@@ -0,0 +1,37 @@
+groups:
+  - name: SNMP
+    rules:
+      - alert: SnmpTargetMissing
+        expr: up{job=~".*snmp|SNMP.*"} == 0
+        for: 15m
+        labels:
+          severity: critical
+        annotations:
+          summary: SNMP target missing (instance {{ $labels.instance }})
+          description: "A SNMP target has disappeared for more the 30 min.\n  VALUE = {{ $value }}"
+      - alert: SnmpTargetFalky
+        expr: changes(up{job=~"snmp"}[24h]) > 5
+        for: 0m
+        labels:
+            severity: info
+        annotations:
+          summary: SNMP target flaky (instance {{ $labels.instance }})
+          description: "A SNMP target is has a flaky respons over the last 24 hours.\n VALUE = {{ $value }}"
+  - name: DHCP
+    rules:
+      - alert: DhcpFuxSharedFailed
+        expr: script_success{script="check_dhcp_fux_shared"} == 0
+        for: 0m
+        labels:
+          severity: critical
+        annotations:
+          summary: DHCP for Fux Shared stoped working
+          description: "No DHCP lease for the Fux Shared range was received"
+      - alert: DhcpFuxAdminFailed
+        expr: script_success{script_success="check_dhcp_fux_admin"} == 0
+        for: 0m
+        labels:
+          severity: critical
+        annotations:
+          summary: DHCP for Fux Admin stoped working
+          description: "No DHCP lease for the Fux Admin range was received"

resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml

@@ -410,7 +410,7 @@ groups:
           summary: Prometheus job missing (instance {{ $labels.instance }})
           description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}"
       - alert: PrometheusTargetMissing
-        expr: up == 0
+        expr: up{job!~"snmp|noc_room_temp"} == 0
         for: 0m
         labels:
           severity: critical
@@ -418,7 +418,7 @@ groups:
           summary: Prometheus target missing (instance {{ $labels.instance }})
           description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}"
       - alert: PrometheusAllTargetsMissing
-        expr: sum by (job) (up) == 0
+        expr: sum by (job) (up{job!~"snmp|noc_room_temp"}) == 0
         for: 0m
         labels:
           severity: critical