prometheus & alertmanager: add self-alerting

Add self-alerting for Prometheus and Alertmanager using rules from
https://samber.github.io/awesome-prometheus-alerts/rules

playbooks/files/chaosknoten/configs/grafana/docker_compose/prometheus.yml

@@ -22,6 +22,13 @@ scrape_configs:
   static_configs:
   - targets:
     - localhost:9090
+- job_name: alertmanager
+  honor_timestamps: true
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - alertmanager:9093
 - job_name: c3lingo
   honor_timestamps: true
   scrape_interval: 5s
@@ -58,15 +65,6 @@ scrape_configs:
   static_configs:
   - targets:
     - jitsi.hamburg.ccc.de:9888 # Jitsi Video Bridge
-- job_name: chaosknoten
-  honor_timestamps: true
-  scrape_interval: 5s
-  scrape_timeout: 1s
-  metrics_path: /metrics
-  scheme: http
-  static_configs:
-  - targets:
-    - chaosknoten.hamburg.ccc.de:9100 # Node Exporter
 - job_name: 'pve'
   static_configs:
     - targets:
@@ -85,6 +83,7 @@ scrape_configs:
       replacement: pve-exporter:9221
 - job_name: hosts
   static_configs:
+    # Wieske Chaosknoten VMs
     - labels:
         site: wieske
         type: virtual_machine
@@ -101,3 +100,9 @@ scrape_configs:
         - woodpecker-intern.hamburg.ccc.de:9100
         - penpot-intern.hamburg.ccc.de:9100
         - jitsi.hamburg.ccc.de:9100
+    # Wieske Physical Machines
+    - labels:
+        site: wieske
+        type: physical_machine
+      targets:
+        - chaosknoten.hamburg.ccc.de:9100