.sops.yaml

@@ -51,6 +51,10 @@ creation_rules:
           *admin_gpg_keys
         age:
           *host_external_age_keys
+  - path_regex: inventories/z9/group_vars/all.*
+    key_groups:
+      - pgp:
+          *admin_gpg_keys
   ## host vars
   # chaosknoten hosts
   - path_regex: inventories/chaosknoten/host_vars/cloud.*

inventories/chaosknoten/group_vars/all.sops.yaml

@@ -1,4 +1,5 @@
 msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str]
+metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str]
 sops:
   age:
     - recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3
@@ -163,8 +164,8 @@ sops:
         SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn
         mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2025-10-13T23:45:06Z"
+  lastmodified: "2026-01-25T18:06:26Z"
-  mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str]
+  mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str]
   pgp:
     - created_at: "2025-10-20T19:03:07Z"
       enc: |-
@@ -360,4 +361,4 @@ sops:
         -----END PGP MESSAGE-----
       fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
   unencrypted_suffix: _unencrypted
-  version: 3.10.2
+  version: 3.11.0

inventories/chaosknoten/group_vars/all.yaml

@@ -14,3 +14,46 @@ msmtp__smtp_port: 465
 msmtp__smtp_tls_method: smtps
 msmtp__smtp_user: any@hosts.hamburg.ccc.de
 msmtp__smtp_from: "{{ inventory_hostname }}@hosts.hamburg.ccc.de"
+
+alloy_config_default: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "{{ metrics__chaos_password }}"
+      }
+    }
+  }
+
+  prometheus.relabel "chaosknoten_common" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      target_label = "org"
+      replacement = "ccchh"
+    }
+    rule {
+      target_label = "site"
+      replacement = "wieske"
+    }
+    rule {
+      source_labels = ["instance"]
+      target_label = "instance"
+      regex  = "([^:]+)"
+      replacement = "${1}.hosts.hamburg.ccc.de"
+      action = "replace"
+    }
+  }
+
+  logging {
+    level = "info"
+  }
+
+  prometheus.exporter.unix "local_system" {
+    enable_collectors = ["systemd"]
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets           = prometheus.exporter.unix.local_system.targets
+    forward_to        = [prometheus.relabel.chaosknoten_common.receiver]
+  }

inventories/chaosknoten/host_vars/grafana.yaml

@@ -53,16 +53,7 @@ nginx__configurations:
   - name: metrics.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
 
-alloy_config: |
+alloy_config_additional: |
-  prometheus.remote_write "default" {
-    endpoint {
-      url = "https://metrics.hamburg.ccc.de/api/v1/write"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__metrics_chaos }}"
-      }
-    }
-  }
   loki.write "default" {
     endpoint {
       url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
@@ -98,9 +89,9 @@ alloy_config: |
     }
     rule {
       source_labels = ["__journal__hostname"]
-      target_label = "host"
+      target_label = "instance"
       regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
+      replacement = "${1}.hosts.hamburg.ccc.de"
       action = "replace"
     }
   }
@@ -111,30 +102,3 @@ alloy_config: |
     format_as_json = true
     labels        = {component = "loki.source.journal", org = "ccchh"}
   }
-
-  logging {
-    level = "info"
-  }
-  prometheus.exporter.unix "local_system" {
-    enable_collectors = ["systemd"]
-  }
-
-  prometheus.relabel "default" {
-    forward_to = [prometheus.remote_write.default.receiver]
-    rule {
-      target_label = "org"
-      replacement = "ccchh"
-    }
-    rule {
-      source_labels = ["instance"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  prometheus.scrape "scrape_metrics" {
-    targets           = prometheus.exporter.unix.local_system.targets
-    forward_to        = [prometheus.relabel.default.receiver]
-  }

inventories/chaosknoten/host_vars/ntfy.sops.yaml

@@ -1,5 +1,3 @@
-secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str]
-secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str]
 secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str]
 ntfy:
   user:
@@ -18,8 +16,8 @@ sops:
         bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM
         sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2025-10-20T19:01:39Z"
+  lastmodified: "2026-01-25T18:41:48Z"
-  mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str]
+  mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str]
   pgp:
     - created_at: "2025-10-20T19:03:04Z"
       enc: |-

inventories/chaosknoten/host_vars/ntfy.yaml

@@ -15,90 +15,8 @@ nginx__configurations:
   - name: ntfy.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}"
 
-alloy_config: |
+alloy_config_additional: |
-  prometheus.remote_write "default" {
-    endpoint {
-      url = "https://metrics.hamburg.ccc.de/api/v1/write"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__metrics_chaos }}"
-      }
-    }
-  }
-  loki.write "default" {
-    endpoint {
-      url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__loki_chaos }}"
-      }
-    }
-  }
-
-  loki.relabel "journal" {
-    forward_to = []
-
-    rule {
-      source_labels = ["__journal__systemd_unit"]
-      target_label  = "systemd_unit"
-    }
-    rule {
-      source_labels = ["__journal__hostname"]
-      target_label = "instance"
-    }
-    rule {
-      source_labels = ["__journal__transport"]
-      target_label = "systemd_transport"
-    }
-    rule {
-      source_labels = ["__journal_syslog_identifier"]
-      target_label = "syslog_identifier"
-    }
-    rule {
-      source_labels = ["__journal_priority_keyword"]
-      target_label  = "level"
-    }
-    rule {
-      source_labels = ["__journal__hostname"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  loki.source.journal "read_journal"  {
-    forward_to    = [loki.write.default.receiver]
-    relabel_rules = loki.relabel.journal.rules
-    format_as_json = true
-    labels        = {component = "loki.source.journal", org = "ccchh"}
-  }
-
-  prometheus.exporter.unix "local_system" {
-    enable_collectors = ["systemd"]
-  }
-
-  prometheus.relabel "default" {
-    forward_to = [prometheus.remote_write.default.receiver]
-    rule {
-      target_label = "org"
-      replacement = "ccchh"
-    }
-    rule {
-      source_labels = ["instance"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  prometheus.scrape "unix_metrics" {
-    targets           = prometheus.exporter.unix.local_system.targets
-    forward_to        = [prometheus.relabel.default.receiver]
-  }
-
   prometheus.scrape "ntfy_metrics" {
     targets     = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}]
-    forward_to  = [prometheus.relabel.default.receiver]
+    forward_to  = [prometheus.relabel.chaosknoten_common.receiver]
   }

inventories/chaosknoten/hosts.yaml

@@ -166,11 +166,10 @@ certbot_hosts:
     ntfy:
     sunders:
     spaceapiccc:
-prometheus_node_exporter_hosts:
+alloy_hosts:
   hosts:
     ccchoir:
     eh22-wiki:
-    tickets:
     keycloak:
     netbox:
     onlyoffice:
@@ -178,6 +177,15 @@ prometheus_node_exporter_hosts:
     pretalx:
     wiki:
     zammad:
+    grafana:
+    ntfy:
+    tickets:
+    renovate:
+    cloud:
+    public-reverse-proxy:
+    router:
+    sunders:
+    spaceapiccc:
 infrastructure_authorized_keys_hosts:
   hosts:
     ccchoir:
@@ -208,10 +216,6 @@ netbox_hosts:
 proxmox_vm_template_hosts:
   hosts:
     chaosknoten:
-alloy_hosts:
-  hosts:
-    grafana:
-    ntfy:
 ansible_pull_hosts:
   hosts:
     netbox:

inventories/z9/group_vars/all.sops.yaml

@@ -0,0 +1,200 @@
+metrics__chaos_password: ENC[AES256_GCM,data:seOU504dZ9K21+NK1MBf9isee2L2rueP6Bl0F66R,iv:ZtQ516gzJQSSgvOOAzPF9MuarXqHSLXy37/9z85KoQ8=,tag:dIal6OxPLli+7DbzhjNFsA==,type:str]
+sops:
+  lastmodified: "2026-01-25T19:52:03Z"
+  mac: ENC[AES256_GCM,data:6JXc+K8fmANf22puWyllV5wVSxZSVnN+U7GM9lNhkxbUBM4AaIedIHOXz9zDaZh/nT6onrW2nhKNC00kWziaddOnBxBUCWUk7bDGea6qJMIk4GfyU0f/xX7mHpgYorF/KmQP1uvNNAryn7zeSfS8Vk27GFDPbBO3GvYlOZFUJD8=,iv:6X6uf9obhNix/qLrpiP3bw1CWM7dY+XAEdfhuTTmuVc=,tag:KJHK1Hc/uV8DOw/7txHfEw==,type:str]
+  pgp:
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMAxK/JaB2/SdtAQ/9GKyJ+6SzK5xucxIxUKPRdsxirJwd6LHuIDkVTr7JTjfi
+        sXQZKVtQ7ZXbbVgZKURLtsdbhayZoU/8xYCQsX4vzDeAKc4bS6X25PLxs2oBKCYB
+        2oWl/jhKSAtVjtgnPnxljiEGxkDKW2sKlfjdjMj9yOYyif35AoQ8pIr2Tg4U8Z9C
+        ofaWBejvqxgaIShXe4jio3SIiOLYwTnaYmkoSY3QEA3RjckzNmqRE4aX+csB27cI
+        Vt8aGrcNzM8gCfi8IM1ypLHLw7Fg0OntF91RAUExG9OZJm2rGZabUixxhOCf/ttk
+        UOq7Eq29xFr9mTzyoZC2zmaOt7O+PIu8FDOvkvCgNv89ewn00DjT7DYSXB0AnPRD
+        VahAi4VAjKU2RXXbfZArdCXJpCTM2OPnXBh8Bfx/7eTnu2O8EK8OFbWuOWja8Ogr
+        7z9bgsoK4Uva6F3BQcLlZppKmkLk0P8detZihvwNbS55kkkdsA9LiyYEoHpasWpG
+        HW4dcQOqyuKwGjLE7FsqPtlxmWD6psCK3GdHzKGQR9fbXfUyD+c0DmPgPh6roFW8
+        XzvRGw6YUrP7/wtvUH4zSLQbB6kqz6nO88isPoLpClyQ/3Khj9QLljCDQB+yRBIu
+        p3a2HISwt4HQzuckk8W4yKIDdzf86dXVEMqUe4JTe+vW9PPobnUEXrPgRBNZYD7S
+        XgHOfGiWknFPa8s4KCHZK9sLB2joWAJTtQnk4cuaXoIgamiXB0qgiArc43PsjstE
+        N6kvVXrFVgQ9Xlrp8XDJHOsUYAy8admA8KNQF+XQ+KeHgQGKKX1RjbBFunIkaOc=
+        =1Rdp
+        -----END PGP MESSAGE-----
+      fp: EF643F59E008414882232C78FFA8331EEB7D6B70
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMA6EyPtWBEI+2AQ//QgGTp9DYSoNWI5n0+9gMUP7asTRdRl7T+xT88k7cCO9Z
+        lNi86qeqiGGkqTffARBJNaq+ut/D7EKc6ivp+ewfySimM5E3ape3C6rulHybE/j8
+        3EpP4VW+5yA4Nq+CbZvzQb60oXR8LnGzVX1gePWwyQozVzRS72/hnxyecQYsc1IJ
+        OTQPSfclZXJO3k6fK+BrfHsjJkOpHYoLAnI/9ty9JBSuwGfzI+ur2Jimn0Y783Ou
+        orZNzxrRKfkIPjdFnWGY3i31nW9tnkEXLdsdOHfOu8Ahtdi2NhwReSv5hMKPXbOw
+        lxhL/Y1bG4ChgAAFVG5QYZ69tuzSov8XP2Wv6auVA7HC3H+689fNeO0C6GhDcVgc
+        LBF1nK/zJq95uxlSNy5dpTSzKqwlwRzvLOCPByXc3pLcDDW9Zp194bS/iDGfnfqe
+        UUPK9e0gX8TYOeQhF3K+H6JMdFO/uYbiaeVZWmvOV6jSiii0CXoGe4oVZqcfcfA3
+        RScUjLx0f2w4xQwU60ZmHsmvs2PmdBsNDPeQXrqeyAfgFReDoI1RLo+k+3khoAJE
+        LzzNFg6bVBx3rRazWoASlHYK8i6dTHpMBompPC+kmjorZnoqnTRX4bix0atsFY7g
+        vt8CVfqy8YKrVIGPZnDAsrZ3ecShIQFB6OfxgSb6nqN1K3NwFcjXWH05SJOfFR3S
+        XgGrU49/hKqHTmAGHbWoe54qkPj+WvRkeGccEnvtum8yrPpDpmYg+wyEm3JeQf1S
+        gCHS/j0pJS/CnnfgoUgkYCMokGvtSoTJgIE/2XTA/BFNRg0vc1Dgk/WonG33PDU=
+        =ev2Q
+        -----END PGP MESSAGE-----
+      fp: F155144FC925A1BEA1F8A2C59A2A4CD59BFDC5EC
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMAz5uSgHG2iMJAQ//SxqxshraAR0pQf1lzhtZ5RHoNZJnaZKwic/pvJvIDUCA
+        6zotOpu478rK4w8zWdX1gfjve+iu4BkaB16lZqvsV6lLq80dT3yfeil9ETFElKuA
+        womIEdAafq8W71eTffUZ3Klrg/WjDVjeDRRKqz8vv9pd9MQbYmDhrjRG/ySP5qgZ
+        +/apRAOrYnbtzgjlPAfLIGD65jvS3JRE3gbZfpzzkLB8P5M1JVOUf15FxAZ2tyhO
+        PZ3FYC2JbCzftp0Iiksq8sl42Fl0FzOTLFQb0GhQ60tJatFVWhG31NeXdRRuLnQU
+        5bmanb2nJBroQJWM/8piG8npG8jhzRzeMTHboW5TezYAXBLxRQJct7pR9ZwDje2U
+        5j9VkNyKQ+wMJ2vMiyshserEe6gjc2/E+XYDheAPrFPqwGNklb6OSemm4vWwd6GK
+        HNqDxA/C0du1b1vm9CTLgk7utbEpspnNQnZh32iifSfiQ3Zl7FwTxnA/2Bj0csQ/
+        xrck7T2gzY39tOqXbq0QqIQA31BW4ukmxcAKn8pmJpguW0cBxDTaGNXQ4jo+8YtQ
+        MYYT4dR9S95MsOKWGREvMA0GMxzwbA2eMwZ7yUARCLVGD48MMiiDZcYqd03cnOO3
+        hGj+vy0FbsVdknztBDeGttUYHOtjb+XO4gF4sHdpaWxdF7kVVknNUtciWa+Kw4LS
+        XgGqWekdWhsKZ+bPboinUPY9e5vkgLueSWrQ0aqi5Pte9lQ3pYPqT2U51fJG5G9/
+        tYiofc0K7CB/qyxB7LpF5rtUla9oQQJd36xC0eO7laSapWiag2rzuIsMxR+4egA=
+        =a2qJ
+        -----END PGP MESSAGE-----
+      fp: 18DFCE01456DAB52EA38A6584EDC64F35FA1D6A5
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMAw5vwmoEJHQ1AQ//U6R7Cb1kfojFwzVy9ky+y7Puhk7Jyog+jLabWhurkV30
+        RO4EeTXMEJ1gVU9qJDeHn7Fr1HbYr9cdTf1yw5Y23p+pBZA1wRTkxHctEk1KNDRW
+        8QdCmQOu8jTDc5cq0F6d6lD6vJpfjaQez5cT7dN9Eqp0jUsmQCSLmqmXvbNEv3fW
+        n+o/VsvtaqMTjhlPUHvhe/d0/YWvkp4xPycDVW/oYh2KE3QUV5AKUJSOLuSIFENy
+        f0hjnoz8xmY7eA20uXvTWPgUU9J+KCSgmy87wM/WkM3kjKWOUkijDFCsCWSEIx4W
+        E2iY6N7yaYBkKfQ6s+f77xg+vc7g1plheK3pdyYkYvgfeqg27QFV/3m80f0gULS4
+        bNrZKNRrMD+grgjB75cj14PRHGQcaZEouE7l2uCUNbR/hFIF1M2F91HAW61mVLv6
+        ZNluofRYqHf+YWUO4KtJwpfgfh0gsCF3KaeHnAA/Xy9e+7KRgWbAbsDIQr40Nqm1
+        Cbv/HqjHCeS7ylw3TmYcwFoGO1XoL/toSQQ4/y0JPMCae+MGslDm2o/1X1VqAnIZ
+        sdhcTKY8HJWxn8uc5MFG4Mr0PhMIXirhBBQLYXdVJ/tOj9yVU+gJZe3lv64uQgDZ
+        Ey1KESfJu98uwrPS3Dzy2YPbT7Gh58sOfHeaDoeZAu+YzQMOQ7V260vu0XXMgSfS
+        XgE3PiMBjbW4eypClEK6H5iSL4SjEm+NweQNkwGaxqLSsb7LuOtSkiEmf4mdQEnI
+        SI14d0nNv7ki0T59Ssmi65A49SXjvLzsCBE1DgeqVD8IwKCewma0dgkNErdyG4s=
+        =rV/a
+        -----END PGP MESSAGE-----
+      fp: 87AB00D45D37C9E9167B5A5A333448678B60E505
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hF4DerEtaFuTeewSAQdAdDLPgKw0A+eoKiYGIKxOFZHYVg0V4UmuIti0XC5RJCIw
+        IPu2/Y45X9L40RRhH8N9lazjLeJv5Lbo08hMlo/CgshZ0BJVot8mBAiH/R2DsVRC
+        0l4BL6ctQ/xivjWQBBhy/DCYVtDRv8JXIEXNJgU/+UjkSE8Auh4NASANg9GTcBBn
+        lukzOBGYF9nH69fAkVtZbNL5+dFoPLDPUzZTU19D15J6IJkt+gKPSzjbtWaJqIsQ
+        =dGU4
+        -----END PGP MESSAGE-----
+      fp: 057870A2C72CD82566A3EC983695F4FCBCAE4912
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMAxjNhCKPP69fAQ/+LQ3yEzhjYXoDkE9olsl0lVxQ9FdCbqDHFJZAIReI0jqg
+        WF+0GmoMuG4kFZu+ju3vCWpT5kH84SUxOFXyaXp1TCfcJ1zCUno93fVssOL/9Sma
+        vVPVIXpTqJqFBOWJNOe7wNjvQiDE4TxjGC/FXr+hOoLZXrf7gdNaUXxZOb+vPZih
+        t79eZhxALGwmwsMdZxkA8ERCmNJet1/wn7s5vUjwrDYRZL2zGf4yocSCjwGYHOCs
+        j+DcrYG7Cd5J+CS8rKu2Yh5KEAfMhgMxGjK0HKUVPk1cQxOgronnM1vrij30S4+9
+        avNlOwAerg3RaFhXPj9UY7FGV+rZQY1CQKEWqr4AANkdDXb/LnLalwMBMcm+EDwT
+        zHxBhJ69QJmsZUP3Z5WQqxmyAux9+oodgehWKkY4sCR2huHuysbJNEStuI1HaTRj
+        ZJafiniHkFyQyTqc4wwJrRxkwJM6mVvcZdXuV7+QaEWr3FEF0t7tyEYUIRkUlJOQ
+        IUPDClDRLJnQGq11XT/QOlGfxET9fGoAkij1LrPqpvHxJ6IEGLMOPN4kw1yg02yO
+        u0HiB2wIUzKHJJ6vMR6zK3WY4MXCQISTZXpK7mILleRIIOWhw7C7gvlfuYkMT3fM
+        dXUQRhTblZXaeTxRuCUPqa0eGzac4UJBVoRAWXYiZWhdKxNJbyCMRQDcaOeho9/S
+        XgGENH9zFjI//pveCrlxx5BKDxTdqIn9R3iskbKbZRhVr+pU3IK4uCsUQlOBG3++
+        zxQinHgNbqA22clcuRMZ1NeDrzDfBLndsWuSeyWaAA9qEG9XjmjCRRzPGACoDLs=
+        =dywj
+        -----END PGP MESSAGE-----
+      fp: F38C9D4228FC6F674E322D9C3326D914EB9B8F55
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMA1Hthzn+T1OoAQ/+P4h9d4LPElJbVr4L3cE+6/9mhLF6n1hre14cCTSxTe20
+        HqpY7U9yE4G/1IfAE3pueOgc5FsLRPn8VnHAzy6ygt4/xXXzH6aACDiUtweqyExE
+        K39J9PLjczWb7XGbZc3YKoz5x0Q/93s+CdjK7piamb41bMRVbmffNtEceH8Gld9u
+        t8SI18F1yrRtXpK8FvzFLvA1jEt9sduPVq51bWG2LjMFaMrsm7hXt1ArUGsOoGN3
+        36E4VrVp6gp8BVa+apsUY3mHBi3hUE0h2tO4iEsi1qYr1OTn3v4Dn9oxKrYIIom2
+        hHszqVSq0fnIqoKOZbyUe2AdXtnTGpQQRxCBvtIEBNB1FS/CKCe7ceXVBZujU2Kd
+        JD3Lg5uXgkolfyjFCPzOp292xvJ83i7QMoTuVEw14PSjux3jAa4K3wpKUvF80ja2
+        ugGj3zTLAHdAV37lKO2WYZuMMJLKWKX1p9yKZqteJdiLQHH8f24dFZ2Vtoly/GKM
+        KzGJ1fimB6divQ8TOHVFAr1qDksk8zf1PBJ/IlWoBKv5IWwoikf42IOL/P2c/nk7
+        4pYwHrlk8y71Cgjw50K9/T/Ul0Ov6ay4FK+0vy+zbokSVczZKsrL4/Tc6s0S9ty6
+        SVKm7yL+BSGgZWmDNesYoRzboBT2mSb1N45ThUaeW7AwMo3hDJPjEkaFtZN3bqzS
+        XgEIFYvxWH6OEIl+VZ/J5qxxmi3Cz6XVzTliCnoTFUoVxHyOxN6HX0Jn/qRqmmlN
+        mJX4OT1FJ3WnqOHQ5Cm4403bm2H79mGCBKYiXPQeO/bVBh0mTbeYKRr8bjsm+rc=
+        =lIEt
+        -----END PGP MESSAGE-----
+      fp: 5DA93D5C9D7320E1BD3522C79C78172B3551C9FD
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMA46L6MuPqfJqAQ//Ue74vm+2K7chQcfdrY5GR0rPUKNiX9MMw9zgHpdyHlXa
+        FfL0NQbG8Rfc/6Yf7LH1sWMGED5Yvci3z+YkTWg8Fcv1pYQGj6Kul2L0aL4tvq/a
+        tdIJi+Ajp92jr+5jdae+GvJZedaHZYxykyeSe4/nk1j6k4u3TiVMBczk7z9701F+
+        ZFtG1SBRcqjZ/Vx36B6s+10f9ft7TWCIJfeTimBJV3fFt7r698vTQ+S1/uxo5Ik0
+        kQOtYDxsigBBeE0OX2ZBDyeGhfl/PZgN7GD7bgpjnfzDi+8kMXEMu4z88tOQulhk
+        qj+63irEydFCsMEC22XhLaqW8bjld0VAnkXv7DfoEbMt84XN6SejjDy6aXK9C0Qd
+        BIyQwTvsmgbInluw8Qu+GJPLLbY9qYjjuo5BbwUeBfiVxQaBYcm5lmPSKM0lq+Uq
+        fUYowpMS417L5kkp2yE/NmKOzi2ZuiFWMCpvPIvKea9zJxvEtIjohwtM86b4LH+j
+        7yie9gWu0bhBw9keKtIbRmgbsilp8E5OUHXgOT0sNWTLenQDsWQ9dmgvtpeEb9ax
+        6mw1QUpFz4CcHhuQixoI+q4y0SXcWxyN0U78U8igaELUtwpaRR7yf4VUJOEid+m1
+        Rzu5jLCuhlLmmW03W6Y/Vl+n0QOyEl0uPCRiRgYeUzKiYw6NRYHPezbJnmNAeKnS
+        XgFCdzLc5Jl6OqJfy4V0yJucGq72oKK1wdJi74PqTNs44CaeEW8tDhxVWm367e5b
+        Ve88LJyDhOrMi57aKcAJ12HoL8pI5hambJ0qSs9RKpnQIJH7U60MBBTCBHN3H4k=
+        =az/q
+        -----END PGP MESSAGE-----
+      fp: 8996B62CBD159DCADD3B6DC08BB33A8ABCF7BC4A
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hF4DQrf1tCqiJxoSAQdA04/sIHLMEFJO8wCB5+N5QWPzwyefW49JuNr/O2A+tTcw
+        Rm/CybmXPnSCx7p8QLruOG0tz8kM+YoSthSWlC9/B6TZgKLyrMOvx1U/fSNjKC4Q
+        0l4BDFhVCnXKTQmfZtj5Qpwuj3H5fZ7QzKUQz542pvqN/fJVnc0Q4rQapKcU4AOx
+        JTdXpu6gP3FRGviA342GHJU0gq220vSzPu889dsdmtgNfAEQWPLVKKwjigDQN+SV
+        =2Eki
+        -----END PGP MESSAGE-----
+      fp: B71138A6A8964A3C3B8899857B4F70C356765BAB
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hF4DzAGzViGx4qcSAQdAxLZsKVzF30df0Zk7Eg9u7fLJzApid00aEcZVxHQnZ0kw
+        5SDeSOzzTue71lKcCyunbO1/e20jMrNVvYKQp0kKkNHpTWgjN0hW3vZt6zeLcrSo
+        0lgBTOoJykoj24Y9WvIaQbae2K6M35drO2c7nhVmTzibUe7XEJ3C+vbUySdSTd+0
+        WL1IjqZUGSUL4SUIW6kW0WFdSJ01O6vbXhw1gw7KwKMfBHgIUAzpENTW
+        =S45t
+        -----END PGP MESSAGE-----
+      fp: D2E9C0807BF681F5E164DAFC5EE1B61CD90954CD
+    - created_at: "2026-01-25T19:51:13Z"
+      enc: |-
+        -----BEGIN PGP MESSAGE-----
+
+        hQIMA2pVdGTIrZI+AQ/8Dd/YnYUP9OA6qxJcerf9mCkQkC5PLehCPAOLOn4sNV5x
+        DnpdfAvgej1Vuy0CHK8//PAiEC7idLN+ictIdQgy0RaObp50tca44U2ssQOkmxcd
+        j5WpKKunHsKomksr7bRpwm/vtN4LoldQc7g1qaBlsaJE7iEOrB8I3n3fFhWD6xBG
+        TUzRe8r/M2/c25Agky9caILYvjm/etCsf/gZq3RwVvV48912JNqb+7o04vpj3MbO
+        AOsiEBCTNSZqN5XuRi/jCpQNe0p18M9irYkFWVe2be6Cb4wE2cdg904rC3K+v0QA
+        nwD6/bXWGI7WAF6nhvuiAS0vxmbvOePNI3KZ1CdEDeScqnAWUdkFuuAwmw0K7tHt
+        UJe/SlML6strnnjOGR6p3eeIjoDKtGBiqEjXYyEcXPVi8vFSd7muGcjLieyJUmfH
+        FVGA7bF+a6c4iTFSM2GNpANFV0qzZ/wa4aj9MqzOATTglQnr2LZJP7chnzoLyzx6
+        7cjTcWHsb3E+D7X37yF+mZAT6yvOoxaQNqTY6u1ZoY9NrGdJ1reudAlzg6k10cpf
+        O4Zww2Jjz5yEhvS9cTh8+bKOJYgKnbg/LLty/ade+rio4E0jn+a6VgRCqIMbGwgx
+        vf9ATU8S10/Es2cT6f99EpPgV0w9QCfhAGel/sjXk/zIT8rF2SbIlXf0/GK3yaXS
+        XgGrocZNe2RNZd3ZjsvtU6bBsPd9tekQLjC1vE6U/WXXPKapb6aOq2eL7Qb7QFu7
+        XSGN+YA/c9OwmtJLP3y5mGBowa6vWT1Uf6NweamPYJBpNG27Bt5yLlnEnaDZokw=
+        =9ri7
+        -----END PGP MESSAGE-----
+      fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
+  unencrypted_suffix: _unencrypted
+  version: 3.11.0

inventories/z9/group_vars/all.yaml

@@ -0,0 +1,43 @@
+alloy_config_default: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "{{ metrics__chaos_password }}"
+      }
+    }
+  }
+
+  prometheus.relabel "z9_common" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      target_label = "org"
+      replacement = "ccchh"
+    }
+    rule {
+      target_label = "site"
+      replacement = "z9"
+    }
+    rule {
+      source_labels = ["instance"]
+      target_label = "instance"
+      regex  = "([^:]+)"
+      replacement = "${1}.z9.ccchh.net"
+      action = "replace"
+    }
+  }
+
+  logging {
+    level = "info"
+  }
+
+  prometheus.exporter.unix "local_system" {
+    enable_collectors = ["systemd"]
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets           = prometheus.exporter.unix.local_system.targets
+    forward_to        = [prometheus.relabel.z9_common.receiver]
+  }
+

inventories/z9/hosts.yaml

@@ -49,5 +49,11 @@ ola_hosts:
 proxmox_vm_template_hosts:
   hosts:
     thinkcccore0:
+alloy_hosts:
+  hosts:
+    authoritative-dns:
+    light:
+    yate:
+    dooris:
 ansible_pull_hosts:
   hosts:

playbooks/deploy.yaml

@@ -64,11 +64,6 @@
   roles:
     - nginx
 
-- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts
-  hosts: prometheus_node_exporter_hosts
-  roles:
-    - prometheus_node_exporter
-
 - name: Configure unattended upgrades for all non-hypervisors
   hosts: all:!hypervisors
   become: true
@@ -83,10 +78,8 @@
 - name: Ensure Alloy is installed and Setup on alloy_hosts
   hosts: alloy_hosts
   become: true
-  tasks:
+  roles:
-    - name: Setup Alloy
+    - alloy
-      ansible.builtin.include_role:
-        name: grafana.grafana.alloy
 
 - name: Ensure ansible_pull deployment on ansible_pull_hosts
   hosts: ansible_pull_hosts

resources/chaosknoten/grafana/docker_compose/prometheus.yml

@@ -82,41 +82,6 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: pve-exporter:9221
-  - job_name: hosts
-    static_configs:
-      # Wieske Chaosknoten VMs
-      - labels:
-          org: ccchh
-          site: wieske
-          type: virtual_machine
-          hypervisor: chaosknoten
-        targets:
-          - netbox-intern.hamburg.ccc.de:9100
-          - matrix-intern.hamburg.ccc.de:9100
-          - public-web-static-intern.hamburg.ccc.de:9100
-          - git-intern.hamburg.ccc.de:9100
-          - forgejo-actions-runner-intern.hamburg.ccc.de:9100
-          - eh22-wiki-intern.hamburg.ccc.de:9100
-          - mjolnir-intern.hamburg.ccc.de:9100
-          - woodpecker-intern.hamburg.ccc.de:9100
-          - penpot-intern.hamburg.ccc.de:9100
-          - jitsi.hamburg.ccc.de:9100
-          - onlyoffice-intern.hamburg.ccc.de:9100
-          - ccchoir-intern.hamburg.ccc.de:9100
-          - tickets-intern.hamburg.ccc.de:9100
-          - keycloak-intern.hamburg.ccc.de:9100
-          - onlyoffice-intern.hamburg.ccc.de:9100
-          - pad-intern.hamburg.ccc.de:9100
-          - wiki-intern.hamburg.ccc.de:9100
-          - zammad-intern.hamburg.ccc.de:9100
-          - pretalx-intern.hamburg.ccc.de:9100
-      - labels:
-          org: ccchh
-          site: wieske
-          type: physical_machine
-        targets:
-          - chaosknoten.hamburg.ccc.de:9100
-
 
 storage:
   tsdb:

resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf

@@ -9,7 +9,6 @@ server {
     allow 2a00:14b0:4200:3380::/64;
     allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing
     # Z9
-    allow 2a07:c480:0:100::/56;
     allow 2a07:c481:1::/48;
     # fuxnoc
     allow 2a07:c481:0:1::/64;

roles/alloy/defaults/main.yaml

@@ -0,0 +1,44 @@
+alloy_config_default: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "chaos_password"
+      }
+    }
+  }
+
+  prometheus.relabel "common" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      target_label = "org"
+      replacement = "noorg"
+    }
+    rule {
+      target_label = "site"
+      replacement = "nosite"
+    }
+    rule {
+      source_labels = ["instance"]
+      target_label = "instance"
+      regex  = "([^:]+)"
+      replacement = "${1}.hosts.test"
+      action = "replace"
+    }
+  }
+
+  logging {
+    level = "info"
+  }
+
+  prometheus.exporter.unix "local_system" {
+    enable_collectors = ["systemd"]
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets           = prometheus.exporter.unix.local_system.targets
+    forward_to        = [prometheus.relabel.common.receiver]
+  }
+
+alloy_config_additional: ""

roles/alloy/tasks/main.yaml

@@ -0,0 +1,50 @@
+# https://github.com/grafana/grafana-ansible-collection/blob/main/roles/alloy/tasks/deploy.yml#L124
+- name: ensure alloy user exists
+  ansible.builtin.user:
+    name: alloy
+    system: true
+    append: true
+    create_home: false
+    state: present
+
+- name: ensure the `/etc/alloy/` config directory exists
+  ansible.builtin.file:
+    path: /etc/alloy
+    state: directory
+    mode: "0770"
+    owner: root
+    group: alloy
+  become: true
+
+- name: synchronize the additional configuration files directory, if present
+  when: alloy__additional_configs_dir is defined and alloy__additional_configs_dir != ""
+  block:
+    - name: ensure rsync is installed
+      ansible.builtin.apt:
+        name: rsync
+      become: true
+
+    - name: synchronize the additional configuration files directory, if present
+      ansible.posix.synchronize:
+        src: "{{ alloy__additional_configs_dir }}"
+        dest: /etc/alloy/additional
+        delete: true
+        recursive: true
+        use_ssh_args: true
+        rsync_opts:
+          - "--chown=root:alloy"
+      become: true
+
+- name: delete the additional configuration files directory, if not present
+  when: alloy__additional_configs_dir is not defined or alloy__additional_configs_dir == ""
+  ansible.builtin.file:
+    path: /etc/alloy/additional
+    state: absent
+  become: true
+
+- name: Setup Alloy
+  ansible.builtin.import_role:
+    name: grafana.grafana.alloy
+  vars:
+    alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}"
+  become: true

roles/prometheus_node_exporter/meta/main.yaml

@@ -1,10 +0,0 @@
----
-dependencies:
-  - role: distribution_check
-    vars:
-      distribution_check__distribution_support_spec:
-        - name: Debian
-          major_versions:
-            - "11"
-            - "12"
-            - "13"

roles/prometheus_node_exporter/tasks/main.yaml

@@ -1,14 +0,0 @@
-- name: make sure the `prometheus-node-exporter` package is installed
-  ansible.builtin.apt:
-    name: prometheus-node-exporter
-    state: present
-    allow_change_held_packages: true
-    update_cache: true
-  become: true
-
-- name: make sure `prometheus-node-exporter.service` is started and ansibled
-  ansible.builtin.systemd:
-    name: prometheus-node-exporter.service
-    state: started
-    enabled: true
-  become: true