rollout Alloy to replace prometheus_node_exporter

With the new network we need to deploy a push based solution in order to get metrics into prometheus

inventories/chaosknoten/group_vars/all.sops.yaml

@@ -1,4 +1,5 @@
 msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str]
+metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str]
 sops:
   age:
     - recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3
@@ -163,8 +164,8 @@ sops:
         SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn
         mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2025-10-13T23:45:06Z"
+  lastmodified: "2026-01-25T18:06:26Z"
-  mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str]
+  mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str]
   pgp:
     - created_at: "2025-10-20T19:03:07Z"
       enc: |-
@@ -360,4 +361,4 @@ sops:
         -----END PGP MESSAGE-----
       fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
   unencrypted_suffix: _unencrypted
-  version: 3.10.2
+  version: 3.11.0

inventories/chaosknoten/host_vars/grafana.yaml

@@ -53,16 +53,7 @@ nginx__configurations:
   - name: metrics.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
 
-alloy_config: |
+alloy_config_additional: |
-  prometheus.remote_write "default" {
-    endpoint {
-      url = "https://metrics.hamburg.ccc.de/api/v1/write"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__metrics_chaos }}"
-      }
-    }
-  }
   loki.write "default" {
     endpoint {
       url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
@@ -98,9 +89,9 @@ alloy_config: |
     }
     rule {
       source_labels = ["__journal__hostname"]
-      target_label = "host"
+      target_label = "instance"
       regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
+      replacement = "${1}.hosts.hamburg.ccc.de"
       action = "replace"
     }
   }
@@ -111,30 +102,3 @@ alloy_config: |
     format_as_json = true
     labels        = {component = "loki.source.journal", org = "ccchh"}
   }
-
-  logging {
-    level = "info"
-  }
-  prometheus.exporter.unix "local_system" {
-    enable_collectors = ["systemd"]
-  }
-
-  prometheus.relabel "default" {
-    forward_to = [prometheus.remote_write.default.receiver]
-    rule {
-      target_label = "org"
-      replacement = "ccchh"
-    }
-    rule {
-      source_labels = ["instance"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  prometheus.scrape "scrape_metrics" {
-    targets           = prometheus.exporter.unix.local_system.targets
-    forward_to        = [prometheus.relabel.default.receiver]
-  }

inventories/chaosknoten/host_vars/ntfy.sops.yaml

@@ -1,5 +1,3 @@
-secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str]
-secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str]
 secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str]
 ntfy:
   user:
@@ -18,8 +16,8 @@ sops:
         bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM
         sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2025-10-20T19:01:39Z"
+  lastmodified: "2026-01-25T18:41:48Z"
-  mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str]
+  mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str]
   pgp:
     - created_at: "2025-10-20T19:03:04Z"
       enc: |-

inventories/chaosknoten/host_vars/ntfy.yaml

@@ -15,90 +15,8 @@ nginx__configurations:
   - name: ntfy.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}"
 
-alloy_config: |
+alloy_config_additional: |
-  prometheus.remote_write "default" {
-    endpoint {
-      url = "https://metrics.hamburg.ccc.de/api/v1/write"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__metrics_chaos }}"
-      }
-    }
-  }
-  loki.write "default" {
-    endpoint {
-      url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__loki_chaos }}"
-      }
-    }
-  }
-
-  loki.relabel "journal" {
-    forward_to = []
-
-    rule {
-      source_labels = ["__journal__systemd_unit"]
-      target_label  = "systemd_unit"
-    }
-    rule {
-      source_labels = ["__journal__hostname"]
-      target_label = "instance"
-    }
-    rule {
-      source_labels = ["__journal__transport"]
-      target_label = "systemd_transport"
-    }
-    rule {
-      source_labels = ["__journal_syslog_identifier"]
-      target_label = "syslog_identifier"
-    }
-    rule {
-      source_labels = ["__journal_priority_keyword"]
-      target_label  = "level"
-    }
-    rule {
-      source_labels = ["__journal__hostname"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  loki.source.journal "read_journal"  {
-    forward_to    = [loki.write.default.receiver]
-    relabel_rules = loki.relabel.journal.rules
-    format_as_json = true
-    labels        = {component = "loki.source.journal", org = "ccchh"}
-  }
-
-  prometheus.exporter.unix "local_system" {
-    enable_collectors = ["systemd"]
-  }
-
-  prometheus.relabel "default" {
-    forward_to = [prometheus.remote_write.default.receiver]
-    rule {
-      target_label = "org"
-      replacement = "ccchh"
-    }
-    rule {
-      source_labels = ["instance"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  prometheus.scrape "unix_metrics" {
-    targets           = prometheus.exporter.unix.local_system.targets
-    forward_to        = [prometheus.relabel.default.receiver]
-  }
-
   prometheus.scrape "ntfy_metrics" {
     targets     = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}]
-    forward_to  = [prometheus.relabel.default.receiver]
+    forward_to  = [prometheus.relabel.chaosknoten_common.receiver]
   }

inventories/chaosknoten/hosts.yaml

@@ -166,11 +166,10 @@ certbot_hosts:
     ntfy:
     sunders:
     spaceapiccc:
-prometheus_node_exporter_hosts:
+alloy_hosts:
   hosts:
     ccchoir:
     eh22-wiki:
-    tickets:
     keycloak:
     netbox:
     onlyoffice:
@@ -178,6 +177,15 @@ prometheus_node_exporter_hosts:
     pretalx:
     wiki:
     zammad:
+    grafana:
+    ntfy:
+    tickets:
+    renovate:
+    cloud:
+    public-reverse-proxy:
+    router:
+    sunders:
+    spaceapiccc:
 infrastructure_authorized_keys_hosts:
   hosts:
     ccchoir:
@@ -208,10 +216,6 @@ netbox_hosts:
 proxmox_vm_template_hosts:
   hosts:
     chaosknoten:
-alloy_hosts:
-  hosts:
-    grafana:
-    ntfy:
 ansible_pull_hosts:
   hosts:
     netbox:

playbooks/deploy.yaml

@@ -64,11 +64,6 @@
   roles:
     - nginx
 
-- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts
-  hosts: prometheus_node_exporter_hosts
-  roles:
-    - prometheus_node_exporter
-
 - name: Configure unattended upgrades for all non-hypervisors
   hosts: all:!hypervisors
   become: true
@@ -83,10 +78,8 @@
 - name: Ensure Alloy is installed and Setup on alloy_hosts
   hosts: alloy_hosts
   become: true
-  tasks:
+  roles:
-    - name: Setup Alloy
+    - alloy
-      ansible.builtin.include_role:
-        name: grafana.grafana.alloy
 
 - name: Ensure ansible_pull deployment on ansible_pull_hosts
   hosts: ansible_pull_hosts

resources/chaosknoten/grafana/docker_compose/prometheus.yml

@@ -82,41 +82,6 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: pve-exporter:9221
-  - job_name: hosts
-    static_configs:
-      # Wieske Chaosknoten VMs
-      - labels:
-          org: ccchh
-          site: wieske
-          type: virtual_machine
-          hypervisor: chaosknoten
-        targets:
-          - netbox-intern.hamburg.ccc.de:9100
-          - matrix-intern.hamburg.ccc.de:9100
-          - public-web-static-intern.hamburg.ccc.de:9100
-          - git-intern.hamburg.ccc.de:9100
-          - forgejo-actions-runner-intern.hamburg.ccc.de:9100
-          - eh22-wiki-intern.hamburg.ccc.de:9100
-          - mjolnir-intern.hamburg.ccc.de:9100
-          - woodpecker-intern.hamburg.ccc.de:9100
-          - penpot-intern.hamburg.ccc.de:9100
-          - jitsi.hamburg.ccc.de:9100
-          - onlyoffice-intern.hamburg.ccc.de:9100
-          - ccchoir-intern.hamburg.ccc.de:9100
-          - tickets-intern.hamburg.ccc.de:9100
-          - keycloak-intern.hamburg.ccc.de:9100
-          - onlyoffice-intern.hamburg.ccc.de:9100
-          - pad-intern.hamburg.ccc.de:9100
-          - wiki-intern.hamburg.ccc.de:9100
-          - zammad-intern.hamburg.ccc.de:9100
-          - pretalx-intern.hamburg.ccc.de:9100
-      - labels:
-          org: ccchh
-          site: wieske
-          type: physical_machine
-        targets:
-          - chaosknoten.hamburg.ccc.de:9100
-
 
 storage:
   tsdb:

resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf

@@ -9,7 +9,6 @@ server {
     allow 2a00:14b0:4200:3380::/64;
     allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing
     # Z9
-    allow 2a07:c480:0:100::/56;
     allow 2a07:c481:1::/48;
     # fuxnoc
     allow 2a07:c481:0:1::/64;

roles/alloy/defaults/main.yaml

@@ -0,0 +1,44 @@
+alloy_config_default: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "{{ metrics__chaos_password }}"
+      }
+    }
+  }
+
+  prometheus.relabel "chaosknoten_common" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      target_label = "org"
+      replacement = "ccchh"
+    }
+    rule {
+      target_label = "site"
+      replacement = "wieske"
+    }
+    rule {
+      source_labels = ["instance"]
+      target_label = "instance"
+      regex  = "([^:]+)"
+      replacement = "${1}.hosts.hamburg.ccc.de"
+      action = "replace"
+    }
+  }
+
+  logging {
+    level = "info"
+  }
+
+  prometheus.exporter.unix "local_system" {
+    enable_collectors = ["systemd"]
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets           = prometheus.exporter.unix.local_system.targets
+    forward_to        = [prometheus.relabel.chaosknoten_common.receiver]
+  }
+
+alloy_config_additional: ""

roles/alloy/tasks/main.yaml

@@ -45,4 +45,6 @@
 - name: Setup Alloy
   ansible.builtin.import_role:
     name: grafana.grafana.alloy
+  vars:
+    alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}"
   become: true

roles/prometheus_node_exporter/meta/main.yaml

@@ -1,10 +0,0 @@
----
-dependencies:
-  - role: distribution_check
-    vars:
-      distribution_check__distribution_support_spec:
-        - name: Debian
-          major_versions:
-            - "11"
-            - "12"
-            - "13"

roles/prometheus_node_exporter/tasks/main.yaml

@@ -1,14 +0,0 @@
-- name: make sure the `prometheus-node-exporter` package is installed
-  ansible.builtin.apt:
-    name: prometheus-node-exporter
-    state: present
-    allow_change_held_packages: true
-    update_cache: true
-  become: true
-
-- name: make sure `prometheus-node-exporter.service` is started and ansibled
-  ansible.builtin.systemd:
-    name: prometheus-node-exporter.service
-    state: started
-    enabled: true
-  become: true