From c620f0f86bb61b3a099e43c1955c56eb0ffa8053 Mon Sep 17 00:00:00 2001
From: c6ristian <c6ristian@christian.moe>
Date: Mon, 2 Jun 2025 00:47:55 +0200
Subject: [PATCH] ntfy: alloy

---
 .../chaosknoten/host_vars/ntfy.sops.yaml      |  6 +-
 inventories/chaosknoten/host_vars/ntfy.yaml   | 74 +++++++++++++++++++
 inventories/chaosknoten/hosts.yaml            |  3 +-
 3 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/inventories/chaosknoten/host_vars/ntfy.sops.yaml b/inventories/chaosknoten/host_vars/ntfy.sops.yaml
index 7c30930..e14402a 100644
--- a/inventories/chaosknoten/host_vars/ntfy.sops.yaml
+++ b/inventories/chaosknoten/host_vars/ntfy.sops.yaml
@@ -1,10 +1,12 @@
+secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str]
+secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str]
 ntfy:
   user:
     admin: ENC[AES256_GCM,data:kwGLrQXBiqKRoHkStGzYiC0fbcGgQHdZrrk9NyZtcZcI4nrKTGx1sxrHOMI=,iv:ACrBFMOP6rkfshOgB+a32TFWH1OKhQaoHcYgwHx+tao=,tag:2QTWmH/vAzIWAjaOHOkrXg==,type:str]
     fuxnoc: ENC[AES256_GCM,data:HVqo1GLaZfDi3ZfAxEJBudFZ+KooBaXk7fr6SsDBZr8=,iv:KziV5OXAtMABqWDPsTRdHM+Ibatp8p5UDoOBUdznx7Y=,tag:kmwSzjaJFBheQcs7181+Jw==,type:str]
 sops:
-  lastmodified: "2025-06-01T21:43:36Z"
-  mac: ENC[AES256_GCM,data:Ssv3QazPopQFN+6ZpoUuaDgVacFmv+VovkptUAybv3ia+03EQOTO5c6FtQf7o2n3M8J839LtOC6WDb34/0WK7aJZkrmnFAuqanJVjlQy5QUHvhSyhHO8/MQwPYnr2hVKHnVyHdKr9KJFilCCu2oP062a+U3eT8BVIeFGyVOqi9s=,iv:q4F5q5Q+6mtzzyYfqH1thNe2nV0eoS7fdoMUxKPNMz0=,tag:1cMSMILpcgFE84nOv+fSNQ==,type:str]
+  lastmodified: "2025-06-01T22:46:40Z"
+  mac: ENC[AES256_GCM,data:WEClJXDgBs6F0g9vbWDI0ytY7SV6GPKiaxO0CghcYVKaPMyLMuJBXWAMahfw/HVpljdiERQAf4J8Chq2r7sKgWwDsHmRldnsuFIb0jFDAefldVqG6MdlZQ75xpCpRBfvE4HTuT7M1PPU9syH/M5mYfVhXsRCwzMpIUIHrgigHhs=,iv:xtov4LekqYl7ofs5DMz193FW1C1vnDjcYXtLxVqRXWQ=,tag:QCV4czWcXeMv42eKEQ38Gw==,type:str]
   pgp:
     - created_at: "2025-06-01T21:41:02Z"
       enc: |-
diff --git a/inventories/chaosknoten/host_vars/ntfy.yaml b/inventories/chaosknoten/host_vars/ntfy.yaml
index 96cb530..6d0e0eb 100644
--- a/inventories/chaosknoten/host_vars/ntfy.yaml
+++ b/inventories/chaosknoten/host_vars/ntfy.yaml
@@ -14,3 +14,77 @@ nginx__version_spec: ""
 nginx__configurations:
   - name: ntfy.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}"
+
+alloy_config: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "{{ secret__metrics_chaos }}"
+      }
+    }
+  }
+  loki.write "default" {
+    endpoint {
+      url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
+      basic_auth {
+        username = "chaos"
+        password = "{{ secret__loki_chaos }}"
+      }
+    }
+  }
+
+  loki.relabel "journal" {
+    forward_to = []
+
+    rule {
+      source_labels = ["__journal__systemd_unit"]
+      target_label  = "systemd_unit"
+    }
+    rule {
+      source_labels = ["__journal__hostname"]
+      target_label = "instance"
+    }
+    rule {
+      source_labels = ["__journal__transport"]
+      target_label = "systemd_transport"
+    }
+    rule {
+      source_labels = ["__journal_syslog_identifier"]
+      target_label = "syslog_identifier"
+    }
+    rule {
+      source_labels = ["__journal_priority_keyword"]
+      target_label  = "level"
+    }
+  }
+
+  loki.source.journal "read_journal"  {
+    forward_to    = [loki.write.default.receiver]
+    relabel_rules = loki.relabel.journal.rules
+    format_as_json = true
+    labels        = {component = "loki.source.journal", org = "ccchh"}
+  }
+
+  prometheus.exporter.unix "local_system" { }
+
+  prometheus.relabel "default" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      target_label = "org"
+      replacement = "ccchh"
+    }
+    rule {
+      source_labels = ["instance"]
+      target_label = "host"
+      regex  = "([^:]+)"
+      replacement = "${1}.hamburg.ccc.det"
+      action = "replace"
+    }
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets         = prometheus.exporter.unix.local_system.targets
+    forward_to      = [prometheus.relabel.default.receiver]
+  }
diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml
index 74684ba..93ea984 100644
--- a/inventories/chaosknoten/hosts.yaml
+++ b/inventories/chaosknoten/hosts.yaml
@@ -60,7 +60,7 @@ all:
       ansible_user: chaos
       ansible_ssh_common_args: -J ssh://chaos@public-reverse-proxy.hamburg.ccc.de
     ntfy:
-      ansible_host: 172.31.17.149
+      ansible_host: ntfy-intern.hamburg.ccc.de
       ansible_user: chaos
       ansible_ssh_common_args: -J ssh://chaos@public-reverse-proxy.hamburg.ccc.de
 hypervisors:
@@ -179,3 +179,4 @@ ansible_pull_hosts:
 alloy_hosts:
   hosts:
     grafana:
+    ntfy: