diff --git a/collections/requirements.yaml b/collections/requirements.yaml
index a24c121..cec061f 100644
--- a/collections/requirements.yaml
+++ b/collections/requirements.yaml
@@ -1,3 +1,4 @@
---
collections:
- community.general
+ - grafana.grafana.alloy
diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml
index 2e8cb71..c0f2ad9 100644
--- a/inventories/chaosknoten/host_vars/grafana.yaml
+++ b/inventories/chaosknoten/host_vars/grafana.yaml
@@ -20,6 +20,7 @@ certbot__acme_account_email_address: le-admin@hamburg.ccc.de
certbot__certificate_domains:
- "grafana.hamburg.ccc.de"
- "loki.hamburg.ccc.de"
+ - "metrics.hamburg.ccc.de"
certbot__new_cert_commands:
- "systemctl reload nginx.service"
@@ -30,6 +31,8 @@ nginx__deploy_htpasswds: true
nginx__htpasswds:
- name: loki
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/loki.htpasswd.j2') }}"
+ - name: metrics
+ content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2') }}"
nginx__configurations:
- name: redirectv6
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/redirect.conf') }}"
@@ -37,3 +40,81 @@ nginx__configurations:
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/grafana.hamburg.ccc.de.conf') }}"
- name: loki.hamburg.ccc.de
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}"
+ - name: metrics.hamburg.ccc.de
+ content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
+
+
+alloy_config: |
+ prometheus.remote_write "default" {
+ endpoint {
+ url = "https://metrics.hamburg.ccc.de/api/v1/write"
+ basic_auth {
+ username = "chaos"
+ password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/metrics_chaos', create=false, missing='error') }}"
+ }
+ }
+ }
+ loki.write "default" {
+ endpoint {
+ url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
+ basic_auth {
+ username = "chaos"
+ password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/loki_chaos', create=false, missing='error') }}"
+ }
+ }
+ }
+
+ loki.relabel "journal" {
+ forward_to = []
+
+ rule {
+ source_labels = ["__journal__systemd_unit"]
+ target_label = "systemd_unit"
+ }
+ rule {
+ source_labels = ["__journal__hostname"]
+ target_label = "instance"
+ }
+ rule {
+ source_labels = ["__journal__transport"]
+ target_label = "systemd_transport"
+ }
+ rule {
+ source_labels = ["__journal_syslog_identifier"]
+ target_label = "syslog_identifier"
+ }
+ rule {
+ source_labels = ["__journal_priority_keyword"]
+ target_label = "level"
+ }
+ }
+
+ loki.source.journal "read_journal" {
+ forward_to = [loki.write.default.receiver]
+ relabel_rules = loki.relabel.journal.rules
+ format_as_json = true
+ labels = {component = "loki.source.journal", host = "grafana", org = "ccchh"}
+ }
+
+ logging {
+ level = "info"
+ }
+ prometheus.exporter.unix "local_system" { }
+
+ prometheus.relabel "default" {
+ forward_to = [prometheus.remote_write.default.receiver]
+ rule {
+ replacement = "org"
+ target_label = "ccchh"
+ }
+ rule {
+ target_label = "host"
+ replacement = "grafana"
+ }
+ }
+
+ prometheus.scrape "scrape_metrics" {
+ targets = prometheus.exporter.unix.local_system.targets
+ forward_to = [prometheus.relabel.default.receiver]
+ scrape_interval = "15s"
+ }
diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml
index 92185ed..ed0d042 100644
--- a/inventories/chaosknoten/hosts.yaml
+++ b/inventories/chaosknoten/hosts.yaml
@@ -177,3 +177,6 @@ proxmox_vm_template_hosts:
ansible_pull_hosts:
hosts:
netbox:
+alloy_hosts:
+ hosts:
+ grafana:
diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml
index d7dcdac..952aeec 100644
--- a/playbooks/deploy.yaml
+++ b/playbooks/deploy.yaml
@@ -70,5 +70,13 @@
- "o=Docker,n=${distro_codename}"
- "o=nginx,n=${distro_codename}"
+- name: Ensure Alloy is installed and Setup on alloy_hosts
+ hosts: alloy_hosts
+ become: true
+ tasks:
+ - name: Setup Alloy
+ ansible.builtin.include_role:
+ name: grafana.grafana.alloy
+
- name: Run ensure_eh22_styleguide_dir Playbook
ansible.builtin.import_playbook: ensure_eh22_styleguide_dir.yaml
diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2
index cbe6e81..3ef0a0c 100644
--- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2
+++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2
@@ -6,6 +6,7 @@ services:
container_name: prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
+ - '--web.enable-remote-write-receiver'
ports:
- 9090:9090
restart: unless-stopped
@@ -54,7 +55,6 @@ services:
volumes:
- /dev/null:/etc/prometheus/pve.yml
-
loki:
image: grafana/loki:3
container_name: loki
@@ -71,3 +71,4 @@ volumes:
prom_data: {}
alertmanager_data: {}
loki_data: {}
+ mimir_data: {}
diff --git a/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf
index f62dc4e..cb51782 100644
--- a/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf
+++ b/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf
@@ -3,13 +3,16 @@ server {
allow 172.31.17.128/25;
allow 212.12.51.128/28;
allow 2a00:14b0:42:100::/56;
+ allow 2a00:14b0:4200:3380::/64;
# Z9
allow 2a07:c480:0:100::/56;
allow 2a07:c481:1::/48;
+
deny all;
- listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:9099 ssl http2;
- listen 172.31.17.145:9099 ssl http2;
+ listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:50051 ssl;
+ listen 172.31.17.145:50051 ssl;
+ http2 on;
server_name loki.hamburg.ccc.de;
@@ -27,7 +30,7 @@ server {
# This is https in any case.
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Scope-OrgID $remote_user;
- grpc_pass grpc://localhost:19009;
+ grpc_pass grpc://localhost:19099;
}
}
@@ -36,13 +39,15 @@ server {
allow 172.31.17.128/25;
allow 212.12.51.128/28;
allow 2a00:14b0:42:100::/56;
+ allow 2a00:14b0:4200:3380::/64;
# Z9
allow 2a07:c480:0:100::/56;
allow 2a07:c481:1::/48;
deny all;
- listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:3100 ssl http2;
- listen 172.31.17.145:3100 ssl http2;
+ listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl;
+ listen 172.31.17.145:443 ssl;
+ http2 on;
server_name loki.hamburg.ccc.de;
@@ -62,7 +67,6 @@ server {
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Port 3100;
# This is https in any case.
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Scope-OrgID $remote_user;
diff --git a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf
new file mode 100644
index 0000000..a431821
--- /dev/null
+++ b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf
@@ -0,0 +1,55 @@
+server {
+ # Wieske
+ allow 172.31.17.128/25;
+ allow 212.12.51.128/28;
+ allow 2a00:14b0:42:100::/56;
+ allow 2a00:14b0:4200:3380::/64;
+ # Z9
+ allow 2a07:c480:0:100::/56;
+ allow 2a07:c481:1::/48;
+ deny all;
+
+ listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl;
+ listen 172.31.17.145:443 ssl;
+ http2 on;
+
+ server_name metrics.hamburg.ccc.de;
+
+ client_body_buffer_size 32k;
+
+ ssl_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/metrics.hamburg.ccc.de/privkey.pem;
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ ssl_trusted_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/chain.pem;
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ add_header Strict-Transport-Security "max-age=63072000" always;
+
+ auth_basic "metrics";
+ auth_basic_user_file metrics.htpasswd;
+
+ location /api/v1/write {
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Port 3100;
+ # This is https in any case.
+ proxy_set_header X-Forwarded-Proto https;
+
+ proxy_pass http://127.0.0.1:9090;
+ }
+
+ location /ready {
+ rewrite ^ /-/ready break;
+
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ # This is https in any case.
+ proxy_set_header X-Forwarded-Proto https;
+
+ proxy_pass http://127.0.0.1:9090;
+ }
+}
diff --git a/resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2 b/resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2
new file mode 100644
index 0000000..1dfcd6c
--- /dev/null
+++ b/resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2
@@ -0,0 +1 @@
+chaos:{{ lookup("community.general.passwordstore", "noc/vm-secrets/chaosknoten/grafana/metrics_chaos_basic_auth", create=false, missing="error") }}