CCCHH/ansible-infra
CCCHH/ansible-infra
4
2
Fork 0
Code Issues3 Pull requests Wiki Activity Actions

prometheus remote write with alloy using it

Browse source
This commit is contained in:
chris 2025-04-30 01:08:43 +02:00
parent e21ff26f36
commit e183f1a2c3
Signed by: c6ristian
SSH key fingerprint: SHA256:B3m+yzpaxGXSEcDBpPHfvza/DNC0wuX+CKMeGq8wgak
8 changed files with 161 additions and 7 deletions
collections
requirements.yaml
inventories/chaosknoten
host_vars
grafana.yaml
hosts.yaml
playbooks
deploy.yaml
resources/chaosknoten/grafana
docker_compose
compose.yaml.j2
nginx
loki.hamburg.ccc.de.confmetrics.hamburg.ccc.de.confmetrics.htpasswd.j2

1
collections/requirements.yaml
View file

@ -1,3 +1,4 @@
---
collections:
- community.general
- grafana.grafana.alloy

81
inventories/chaosknoten/host_vars/grafana.yaml
View file

@ -20,6 +20,7 @@ certbot__acme_account_email_address: le-admin@hamburg.ccc.de
certbot__certificate_domains:
- "grafana.hamburg.ccc.de"
- "loki.hamburg.ccc.de"
- "metrics.hamburg.ccc.de"
certbot__new_cert_commands:
- "systemctl reload nginx.service"
@ -30,6 +31,8 @@ nginx__deploy_htpasswds: true
nginx__htpasswds:
- name: loki
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/loki.htpasswd.j2') }}"
- name: metrics
content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2') }}"
nginx__configurations:
- name: redirectv6
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/redirect.conf') }}"
@ -37,3 +40,81 @@ nginx__configurations:
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/grafana.hamburg.ccc.de.conf') }}"
- name: loki.hamburg.ccc.de
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}"
- name: metrics.hamburg.ccc.de
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
alloy_config: |
prometheus.remote_write "default" {
endpoint {
url = "https://metrics.hamburg.ccc.de/api/v1/write"
basic_auth {
username = "chaos"
password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/metrics_chaos', create=false, missing='error') }}"
}
}
}
loki.write "default" {
endpoint {
url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
basic_auth {
username = "chaos"
password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/loki_chaos', create=false, missing='error') }}"
}
}
}
loki.relabel "journal" {
forward_to = []
rule {
source_labels = ["__journal__systemd_unit"]
target_label = "systemd_unit"
}
rule {
source_labels = ["__journal__hostname"]
target_label = "instance"
}
rule {
source_labels = ["__journal__transport"]
target_label = "systemd_transport"
}
rule {
source_labels = ["__journal_syslog_identifier"]
target_label = "syslog_identifier"
}
rule {
source_labels = ["__journal_priority_keyword"]
target_label = "level"
}
}
loki.source.journal "read_journal" {
forward_to = [loki.write.default.receiver]
relabel_rules = loki.relabel.journal.rules
format_as_json = true
labels = {component = "loki.source.journal", host = "grafana", org = "ccchh"}
}
logging {
level = "info"
}
prometheus.exporter.unix "local_system" { }
prometheus.relabel "default" {
forward_to = [prometheus.remote_write.default.receiver]
rule {
replacement = "org"
target_label = "ccchh"
}
rule {
target_label = "host"
replacement = "grafana"
}
}
prometheus.scrape "scrape_metrics" {
targets = prometheus.exporter.unix.local_system.targets
forward_to = [prometheus.relabel.default.receiver]
scrape_interval = "15s"
}

3
inventories/chaosknoten/hosts.yaml
View file

@ -177,3 +177,6 @@ proxmox_vm_template_hosts:
ansible_pull_hosts:
hosts:
netbox:
alloy_hosts:
hosts:
grafana:

8
playbooks/deploy.yaml
View file

@ -70,5 +70,13 @@
- "o=Docker,n=${distro_codename}"
- "o=nginx,n=${distro_codename}"
- name: Ensure Alloy is installed and Setup on alloy_hosts
hosts: alloy_hosts
become: true
tasks:
- name: Setup Alloy
ansible.builtin.include_role:
name: grafana.grafana.alloy
- name: Run ensure_eh22_styleguide_dir Playbook
ansible.builtin.import_playbook: ensure_eh22_styleguide_dir.yaml

3
resources/chaosknoten/grafana/docker_compose/compose.yaml.j2
View file

@ -6,6 +6,7 @@ services:
container_name: prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--web.enable-remote-write-receiver'
ports:
- 9090:9090
restart: unless-stopped
@ -54,7 +55,6 @@ services:
volumes:
- /dev/null:/etc/prometheus/pve.yml
loki:
image: grafana/loki:3
container_name: loki
@ -71,3 +71,4 @@ volumes:
prom_data: {}
alertmanager_data: {}
loki_data: {}
mimir_data: {}

16
resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf
View file

@ -3,13 +3,16 @@ server {
allow 172.31.17.128/25;
allow 212.12.51.128/28;
allow 2a00:14b0:42:100::/56;
allow 2a00:14b0:4200:3380::/64;
# Z9
allow 2a07:c480:0:100::/56;
allow 2a07:c481:1::/48;
deny all;
listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:9099 ssl http2;
listen 172.31.17.145:9099 ssl http2;
listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:50051 ssl;
listen 172.31.17.145:50051 ssl;
http2 on;
server_name loki.hamburg.ccc.de;
@ -27,7 +30,7 @@ server {
# This is https in any case.
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Scope-OrgID $remote_user;
grpc_pass grpc://localhost:19009;
grpc_pass grpc://localhost:19099;
}
}
@ -36,13 +39,15 @@ server {
allow 172.31.17.128/25;
allow 212.12.51.128/28;
allow 2a00:14b0:42:100::/56;
allow 2a00:14b0:4200:3380::/64;
# Z9
allow 2a07:c480:0:100::/56;
allow 2a07:c481:1::/48;
deny all;
listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:3100 ssl http2;
listen 172.31.17.145:3100 ssl http2;
listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl;
listen 172.31.17.145:443 ssl;
http2 on;
server_name loki.hamburg.ccc.de;
@ -62,7 +67,6 @@ server {
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Port 3100;
# This is https in any case.
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Scope-OrgID $remote_user;

55
resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf Normal file
View file

@ -0,0 +1,55 @@
server {
# Wieske
allow 172.31.17.128/25;
allow 212.12.51.128/28;
allow 2a00:14b0:42:100::/56;
allow 2a00:14b0:4200:3380::/64;
# Z9
allow 2a07:c480:0:100::/56;
allow 2a07:c481:1::/48;
deny all;
listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl;
listen 172.31.17.145:443 ssl;
http2 on;
server_name metrics.hamburg.ccc.de;
client_body_buffer_size 32k;
ssl_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/metrics.hamburg.ccc.de/privkey.pem;
# verify chain of trust of OCSP response using Root CA and Intermediate certs
ssl_trusted_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/chain.pem;
# HSTS (ngx_http_headers_module is required) (63072000 seconds)
add_header Strict-Transport-Security "max-age=63072000" always;
auth_basic "metrics";
auth_basic_user_file metrics.htpasswd;
location /api/v1/write {
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Port 3100;
# This is https in any case.
proxy_set_header X-Forwarded-Proto https;
proxy_pass http://127.0.0.1:9090;
}
location /ready {
rewrite ^ /-/ready break;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# This is https in any case.
proxy_set_header X-Forwarded-Proto https;
proxy_pass http://127.0.0.1:9090;
}
}

1
resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2 Normal file
View file

@ -0,0 +1 @@
chaos:{{ lookup("community.general.passwordstore", "noc/vm-secrets/chaosknoten/grafana/metrics_chaos_basic_auth", create=false, missing="error") }}