prometheus remote write with alloy using it

inventories/chaosknoten/host_vars/grafana.yaml

@@ -20,6 +20,7 @@ certbot__acme_account_email_address: le-admin@hamburg.ccc.de
 certbot__certificate_domains:
   - "grafana.hamburg.ccc.de"
   - "loki.hamburg.ccc.de"
+  - "metrics.hamburg.ccc.de"
 
 certbot__new_cert_commands:
   - "systemctl reload nginx.service"
@@ -30,6 +31,8 @@ nginx__deploy_htpasswds: true
 nginx__htpasswds:
   - name: loki
     content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/loki.htpasswd.j2') }}"
+  - name: metrics
+    content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2') }}"
 nginx__configurations:
   - name: redirectv6
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/redirect.conf') }}"
@@ -37,3 +40,81 @@ nginx__configurations:
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/grafana.hamburg.ccc.de.conf') }}"
   - name: loki.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}"
+  - name: metrics.hamburg.ccc.de
+    content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
+
+
+alloy_config: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/metrics_chaos', create=false, missing='error') }}"
+      }
+    }
+  }
+  loki.write "default" {
+    endpoint {
+      url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
+      basic_auth {
+        username = "chaos"
+        password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/loki_chaos', create=false, missing='error') }}"
+      }
+    }
+  }
+
+  loki.relabel "journal" {
+    forward_to = []
+
+    rule {
+      source_labels = ["__journal__systemd_unit"]
+      target_label  = "systemd_unit"
+    }
+    rule {
+      source_labels = ["__journal__hostname"]
+      target_label = "instance"
+    }
+    rule {
+      source_labels = ["__journal__transport"]
+      target_label = "systemd_transport"
+    }
+    rule {
+      source_labels = ["__journal_syslog_identifier"]
+      target_label = "syslog_identifier"
+    }
+    rule {
+      source_labels = ["__journal_priority_keyword"]
+      target_label  = "level"
+    }
+  }
+
+  loki.source.journal "read_journal"  {
+    forward_to    = [loki.write.default.receiver]
+    relabel_rules = loki.relabel.journal.rules
+    format_as_json = true
+    labels        = {component = "loki.source.journal", host = "grafana", org = "ccchh"}
+  }
+
+  logging {
+    level = "info"
+  }
+  prometheus.exporter.unix "local_system" { }
+
+  prometheus.relabel "default" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      replacement = "org"
+      target_label = "ccchh"
+    }
+    rule {
+      target_label = "host"
+      replacement = "grafana"
+    }
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets         = prometheus.exporter.unix.local_system.targets
+    forward_to      = [prometheus.relabel.default.receiver]
+    scrape_interval = "15s"
+  }

inventories/chaosknoten/hosts.yaml

@@ -177,3 +177,6 @@ proxmox_vm_template_hosts:
 ansible_pull_hosts:
   hosts:
     netbox:
+alloy_hosts:
+  hosts:
+    grafana: