rollout Alloy to replace prometheus_node_exporter

With the new network we need to deploy a push based solution in order to get metrics into prometheus

inventories/chaosknoten/host_vars/grafana.yaml

@@ -53,16 +53,7 @@ nginx__configurations:
   - name: metrics.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
 
-alloy_config: |
-  prometheus.remote_write "default" {
-    endpoint {
-      url = "https://metrics.hamburg.ccc.de/api/v1/write"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__metrics_chaos }}"
-      }
-    }
-  }
+alloy_config_additional: |
   loki.write "default" {
     endpoint {
       url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
@@ -98,9 +89,9 @@ alloy_config: |
     }
     rule {
       source_labels = ["__journal__hostname"]
-      target_label = "host"
+      target_label = "instance"
       regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
+      replacement = "${1}.hosts.hamburg.ccc.de"
       action = "replace"
     }
   }
@@ -111,30 +102,3 @@ alloy_config: |
     format_as_json = true
     labels        = {component = "loki.source.journal", org = "ccchh"}
   }
-
-  logging {
-    level = "info"
-  }
-  prometheus.exporter.unix "local_system" {
-    enable_collectors = ["systemd"]
-  }
-
-  prometheus.relabel "default" {
-    forward_to = [prometheus.remote_write.default.receiver]
-    rule {
-      target_label = "org"
-      replacement = "ccchh"
-    }
-    rule {
-      source_labels = ["instance"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  prometheus.scrape "scrape_metrics" {
-    targets           = prometheus.exporter.unix.local_system.targets
-    forward_to        = [prometheus.relabel.default.receiver]
-  }

inventories/chaosknoten/host_vars/ntfy.sops.yaml

@@ -1,5 +1,3 @@
-secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str]
-secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str]
 secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str]
 ntfy:
   user:
@@ -18,8 +16,8 @@ sops:
         bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM
         sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2025-10-20T19:01:39Z"
-  mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str]
+  lastmodified: "2026-01-25T18:41:48Z"
+  mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str]
   pgp:
     - created_at: "2025-10-20T19:03:04Z"
       enc: |-

inventories/chaosknoten/host_vars/ntfy.yaml

@@ -15,90 +15,8 @@ nginx__configurations:
   - name: ntfy.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}"
 
-alloy_config: |
-  prometheus.remote_write "default" {
-    endpoint {
-      url = "https://metrics.hamburg.ccc.de/api/v1/write"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__metrics_chaos }}"
-      }
-    }
-  }
-  loki.write "default" {
-    endpoint {
-      url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
-      basic_auth {
-        username = "chaos"
-        password = "{{ secret__loki_chaos }}"
-      }
-    }
-  }
-
-  loki.relabel "journal" {
-    forward_to = []
-
-    rule {
-      source_labels = ["__journal__systemd_unit"]
-      target_label  = "systemd_unit"
-    }
-    rule {
-      source_labels = ["__journal__hostname"]
-      target_label = "instance"
-    }
-    rule {
-      source_labels = ["__journal__transport"]
-      target_label = "systemd_transport"
-    }
-    rule {
-      source_labels = ["__journal_syslog_identifier"]
-      target_label = "syslog_identifier"
-    }
-    rule {
-      source_labels = ["__journal_priority_keyword"]
-      target_label  = "level"
-    }
-    rule {
-      source_labels = ["__journal__hostname"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  loki.source.journal "read_journal"  {
-    forward_to    = [loki.write.default.receiver]
-    relabel_rules = loki.relabel.journal.rules
-    format_as_json = true
-    labels        = {component = "loki.source.journal", org = "ccchh"}
-  }
-
-  prometheus.exporter.unix "local_system" {
-    enable_collectors = ["systemd"]
-  }
-
-  prometheus.relabel "default" {
-    forward_to = [prometheus.remote_write.default.receiver]
-    rule {
-      target_label = "org"
-      replacement = "ccchh"
-    }
-    rule {
-      source_labels = ["instance"]
-      target_label = "host"
-      regex  = "([^:]+)"
-      replacement = "${1}.hamburg.ccc.de"
-      action = "replace"
-    }
-  }
-
-  prometheus.scrape "unix_metrics" {
-    targets           = prometheus.exporter.unix.local_system.targets
-    forward_to        = [prometheus.relabel.default.receiver]
-  }
-
+alloy_config_additional: |
   prometheus.scrape "ntfy_metrics" {
     targets     = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}]
-    forward_to  = [prometheus.relabel.default.receiver]
+    forward_to  = [prometheus.relabel.chaosknoten_common.receiver]
   }