prometheus remote write with alloy using it

collections/requirements.yaml

@@ -1,3 +1,4 @@
 ---
 collections:
  - community.general
+ - grafana.grafana.alloy

inventories/chaosknoten/host_vars/grafana.yaml

@@ -20,6 +20,7 @@ certbot__acme_account_email_address: le-admin@hamburg.ccc.de
 certbot__certificate_domains:
   - "grafana.hamburg.ccc.de"
   - "loki.hamburg.ccc.de"
+  - "metrics.hamburg.ccc.de"
 
 certbot__new_cert_commands:
   - "systemctl reload nginx.service"
@@ -30,6 +31,8 @@ nginx__deploy_htpasswds: true
 nginx__htpasswds:
   - name: loki
     content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/loki.htpasswd.j2') }}"
+  - name: metrics
+    content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2') }}"
 nginx__configurations:
   - name: redirectv6
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/redirect.conf') }}"
@@ -37,3 +40,81 @@ nginx__configurations:
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/grafana.hamburg.ccc.de.conf') }}"
   - name: loki.hamburg.ccc.de
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}"
+  - name: metrics.hamburg.ccc.de
+    content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
+
+
+alloy_config: |
+  prometheus.remote_write "default" {
+    endpoint {
+      url = "https://metrics.hamburg.ccc.de/api/v1/write"
+      basic_auth {
+        username = "chaos"
+        password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/metrics_chaos', create=false, missing='error') }}"
+      }
+    }
+  }
+  loki.write "default" {
+    endpoint {
+      url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
+      basic_auth {
+        username = "chaos"
+        password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/loki_chaos', create=false, missing='error') }}"
+      }
+    }
+  }
+
+  loki.relabel "journal" {
+    forward_to = []
+
+    rule {
+      source_labels = ["__journal__systemd_unit"]
+      target_label  = "systemd_unit"
+    }
+    rule {
+      source_labels = ["__journal__hostname"]
+      target_label = "instance"
+    }
+    rule {
+      source_labels = ["__journal__transport"]
+      target_label = "systemd_transport"
+    }
+    rule {
+      source_labels = ["__journal_syslog_identifier"]
+      target_label = "syslog_identifier"
+    }
+    rule {
+      source_labels = ["__journal_priority_keyword"]
+      target_label  = "level"
+    }
+  }
+
+  loki.source.journal "read_journal"  {
+    forward_to    = [loki.write.default.receiver]
+    relabel_rules = loki.relabel.journal.rules
+    format_as_json = true
+    labels        = {component = "loki.source.journal", host = "grafana", org = "ccchh"}
+  }
+
+  logging {
+    level = "info"
+  }
+  prometheus.exporter.unix "local_system" { }
+
+  prometheus.relabel "default" {
+    forward_to = [prometheus.remote_write.default.receiver]
+    rule {
+      replacement = "org"
+      target_label = "ccchh"
+    }
+    rule {
+      target_label = "host"
+      replacement = "grafana"
+    }
+  }
+
+  prometheus.scrape "scrape_metrics" {
+    targets         = prometheus.exporter.unix.local_system.targets
+    forward_to      = [prometheus.relabel.default.receiver]
+    scrape_interval = "15s"
+  }

inventories/chaosknoten/hosts.yaml

@@ -177,3 +177,6 @@ proxmox_vm_template_hosts:
 ansible_pull_hosts:
   hosts:
     netbox:
+alloy_hosts:
+  hosts:
+    grafana:

playbooks/deploy.yaml

@@ -70,5 +70,13 @@
           - "o=Docker,n=${distro_codename}"
           - "o=nginx,n=${distro_codename}"
 
+- name: Ensure Alloy is installed and Setup on alloy_hosts
+  hosts: alloy_hosts
+  become: true
+  tasks:
+    - name: Setup Alloy
+      ansible.builtin.include_role:
+        name: grafana.grafana.alloy
+
 - name: Run ensure_eh22_styleguide_dir Playbook
   ansible.builtin.import_playbook: ensure_eh22_styleguide_dir.yaml

resources/chaosknoten/grafana/docker_compose/compose.yaml.j2

@@ -6,6 +6,7 @@ services:
     container_name: prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-remote-write-receiver'
     ports:
       - 9090:9090
     restart: unless-stopped
@@ -54,7 +55,6 @@ services:
     volumes:
       - /dev/null:/etc/prometheus/pve.yml
 
-
   loki:
     image: grafana/loki:3
     container_name: loki
@@ -71,3 +71,4 @@ volumes:
   prom_data: {}
   alertmanager_data: {}
   loki_data: {}
+  mimir_data: {}

resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf

@@ -3,13 +3,16 @@ server {
     allow 172.31.17.128/25;
     allow 212.12.51.128/28;
     allow 2a00:14b0:42:100::/56;
+    allow 2a00:14b0:4200:3380::/64;
     # Z9
     allow 2a07:c480:0:100::/56;
     allow 2a07:c481:1::/48;
+    
     deny all;
 
-    listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:9099 ssl http2;
+    listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:50051 ssl;
-    listen 172.31.17.145:9099 ssl http2;
+    listen 172.31.17.145:50051 ssl;
+    http2 on;
 
     server_name loki.hamburg.ccc.de;
 
@@ -27,7 +30,7 @@ server {
         # This is https in any case.
         proxy_set_header X-Forwarded-Proto https;
         proxy_set_header X-Scope-OrgID $remote_user;
-        grpc_pass grpc://localhost:19009;
+        grpc_pass grpc://localhost:19099;
     }
 }
 
@@ -36,13 +39,15 @@ server {
     allow 172.31.17.128/25;
     allow 212.12.51.128/28;
     allow 2a00:14b0:42:100::/56;
+    allow 2a00:14b0:4200:3380::/64;
     # Z9
     allow 2a07:c480:0:100::/56;
     allow 2a07:c481:1::/48;
     deny all;
 
-    listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:3100 ssl http2;
+    listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl;
-    listen 172.31.17.145:3100 ssl http2;
+    listen 172.31.17.145:443 ssl;
+    http2 on;
 
     server_name loki.hamburg.ccc.de;
 
@@ -62,7 +67,6 @@ server {
         proxy_set_header X-Forwarded-Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Port 3100;
         # This is https in any case.
         proxy_set_header X-Forwarded-Proto https;
         proxy_set_header X-Scope-OrgID $remote_user;

resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf

@@ -0,0 +1,55 @@
+server {
+    # Wieske
+    allow 172.31.17.128/25;
+    allow 212.12.51.128/28;
+    allow 2a00:14b0:42:100::/56;
+    allow 2a00:14b0:4200:3380::/64;
+    # Z9
+    allow 2a07:c480:0:100::/56;
+    allow 2a07:c481:1::/48;
+    deny all;
+
+    listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl;
+    listen 172.31.17.145:443 ssl;
+    http2 on;
+
+    server_name metrics.hamburg.ccc.de;
+
+    client_body_buffer_size 32k;
+
+    ssl_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/metrics.hamburg.ccc.de/privkey.pem;
+    # verify chain of trust of OCSP response using Root CA and Intermediate certs
+    ssl_trusted_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/chain.pem;
+
+    # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+    add_header Strict-Transport-Security "max-age=63072000" always;
+
+    auth_basic  "metrics";
+    auth_basic_user_file metrics.htpasswd;
+
+    location /api/v1/write {
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Port 3100;
+        # This is https in any case.
+        proxy_set_header X-Forwarded-Proto https;
+
+        proxy_pass http://127.0.0.1:9090;
+    }
+
+    location /ready {
+        rewrite ^ /-/ready break;
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        # This is https in any case.
+        proxy_set_header X-Forwarded-Proto https;
+
+        proxy_pass http://127.0.0.1:9090;
+    }
+}

resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2

@@ -0,0 +1 @@
+chaos:{{ lookup("community.general.passwordstore", "noc/vm-secrets/chaosknoten/grafana/metrics_chaos_basic_auth", create=false, missing="error") }}