CCCHH/ansible-infra
CCCHH/ansible-infra
2
2
Fork 2
Code Issues 7 Pull requests 4 Wiki Activity Actions

rollout Alloy to replace prometheus_node_exporter
Some checks failed
/ Ansible Lint (push) Failing after 41s
Details
/ Ansible Lint (pull_request) Failing after 41s
Details

Browse source 
With the new network we need to deploy a push based solution in order to get metrics into prometheus
This commit is contained in:
chris 2026-01-25 20:03:13 +01:00
commit 10388c8333
Signed by: c6ristian
SSH key fingerprint: SHA256:B3m+yzpaxGXSEcDBpPHfvza/DNC0wuX+CKMeGq8wgak
12 changed files with 69 additions and 205 deletions
Download patch file Download diff file Expand all files Collapse all files

7
inventories/chaosknoten/group_vars/all.sops.yaml
View file

@ -1,4 +1,5 @@
msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str]
metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str]
sops:
age:
- recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3
@ -163,8 +164,8 @@ sops:
SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn
mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-10-13T23:45:06Z"
mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str]
lastmodified: "2026-01-25T18:06:26Z"
mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str]
pgp:
- created_at: "2025-10-20T19:03:07Z"
enc: |-
@ -360,4 +361,4 @@ sops:
-----END PGP MESSAGE-----
fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
unencrypted_suffix: _unencrypted
version: 3.10.2
version: 3.11.0

42
inventories/chaosknoten/host_vars/grafana.yaml
View file

@ -53,16 +53,7 @@ nginx__configurations:
- name: metrics.hamburg.ccc.de
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}"
alloy_config: |
prometheus.remote_write "default" {
endpoint {
url = "https://metrics.hamburg.ccc.de/api/v1/write"
basic_auth {
username = "chaos"
password = "{{ secret__metrics_chaos }}"
}
}
}
alloy_config_additional: |
loki.write "default" {
endpoint {
url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
@ -98,9 +89,9 @@ alloy_config: |
}
rule {
source_labels = ["__journal__hostname"]
target_label = "host"
target_label = "instance"
regex = "([^:]+)"
replacement = "${1}.hamburg.ccc.de"
replacement = "${1}.hosts.hamburg.ccc.de"
action = "replace"
}
}
@ -111,30 +102,3 @@ alloy_config: |
format_as_json = true
labels = {component = "loki.source.journal", org = "ccchh"}
}
logging {
level = "info"
}
prometheus.exporter.unix "local_system" {
enable_collectors = ["systemd"]
}
prometheus.relabel "default" {
forward_to = [prometheus.remote_write.default.receiver]
rule {
target_label = "org"
replacement = "ccchh"
}
rule {
source_labels = ["instance"]
target_label = "host"
regex = "([^:]+)"
replacement = "${1}.hamburg.ccc.de"
action = "replace"
}
}
prometheus.scrape "scrape_metrics" {
targets = prometheus.exporter.unix.local_system.targets
forward_to = [prometheus.relabel.default.receiver]
}

6
inventories/chaosknoten/host_vars/ntfy.sops.yaml
View file

@ -1,5 +1,3 @@
secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str]
secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str]
secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str]
ntfy:
user:
@ -18,8 +16,8 @@ sops:
bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM
sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-10-20T19:01:39Z"
mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str]
lastmodified: "2026-01-25T18:41:48Z"
mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str]
pgp:
- created_at: "2025-10-20T19:03:04Z"
enc: |-

86
inventories/chaosknoten/host_vars/ntfy.yaml
View file

@ -15,90 +15,8 @@ nginx__configurations:
- name: ntfy.hamburg.ccc.de
content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}"
alloy_config: |
prometheus.remote_write "default" {
endpoint {
url = "https://metrics.hamburg.ccc.de/api/v1/write"
basic_auth {
username = "chaos"
password = "{{ secret__metrics_chaos }}"
}
}
}
loki.write "default" {
endpoint {
url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
basic_auth {
username = "chaos"
password = "{{ secret__loki_chaos }}"
}
}
}
loki.relabel "journal" {
forward_to = []
rule {
source_labels = ["__journal__systemd_unit"]
target_label = "systemd_unit"
}
rule {
source_labels = ["__journal__hostname"]
target_label = "instance"
}
rule {
source_labels = ["__journal__transport"]
target_label = "systemd_transport"
}
rule {
source_labels = ["__journal_syslog_identifier"]
target_label = "syslog_identifier"
}
rule {
source_labels = ["__journal_priority_keyword"]
target_label = "level"
}
rule {
source_labels = ["__journal__hostname"]
target_label = "host"
regex = "([^:]+)"
replacement = "${1}.hamburg.ccc.de"
action = "replace"
}
}
loki.source.journal "read_journal" {
forward_to = [loki.write.default.receiver]
relabel_rules = loki.relabel.journal.rules
format_as_json = true
labels = {component = "loki.source.journal", org = "ccchh"}
}
prometheus.exporter.unix "local_system" {
enable_collectors = ["systemd"]
}
prometheus.relabel "default" {
forward_to = [prometheus.remote_write.default.receiver]
rule {
target_label = "org"
replacement = "ccchh"
}
rule {
source_labels = ["instance"]
target_label = "host"
regex = "([^:]+)"
replacement = "${1}.hamburg.ccc.de"
action = "replace"
}
}
prometheus.scrape "unix_metrics" {
targets = prometheus.exporter.unix.local_system.targets
forward_to = [prometheus.relabel.default.receiver]
}
alloy_config_additional: |
prometheus.scrape "ntfy_metrics" {
targets = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}]
forward_to = [prometheus.relabel.default.receiver]
forward_to = [prometheus.relabel.chaosknoten_common.receiver]
}

16
inventories/chaosknoten/hosts.yaml
View file

@ -166,11 +166,10 @@ certbot_hosts:
ntfy:
sunders:
spaceapiccc:
prometheus_node_exporter_hosts:
alloy_hosts:
hosts:
ccchoir:
eh22-wiki:
tickets:
keycloak:
netbox:
onlyoffice:
@ -178,6 +177,15 @@ prometheus_node_exporter_hosts:
pretalx:
wiki:
zammad:
grafana:
ntfy:
tickets:
renovate:
cloud:
public-reverse-proxy:
router:
sunders:
spaceapiccc:
infrastructure_authorized_keys_hosts:
hosts:
ccchoir:
@ -208,10 +216,6 @@ netbox_hosts:
proxmox_vm_template_hosts:
hosts:
chaosknoten:
alloy_hosts:
hosts:
grafana:
ntfy:
ansible_pull_hosts:
hosts:
netbox:

11
playbooks/deploy.yaml
View file

@ -64,11 +64,6 @@
roles:
- nginx
- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts
hosts: prometheus_node_exporter_hosts
roles:
- prometheus_node_exporter
- name: Configure unattended upgrades for all non-hypervisors
hosts: all:!hypervisors
become: true
@ -83,10 +78,8 @@
- name: Ensure Alloy is installed and Setup on alloy_hosts
hosts: alloy_hosts
become: true
tasks:
- name: Setup Alloy
ansible.builtin.include_role:
name: grafana.grafana.alloy
roles:
- alloy
- name: Ensure ansible_pull deployment on ansible_pull_hosts
hosts: ansible_pull_hosts

35
resources/chaosknoten/grafana/docker_compose/prometheus.yml
View file

@ -82,41 +82,6 @@ scrape_configs:
target_label: instance
- target_label: __address__
replacement: pve-exporter:9221
- job_name: hosts
static_configs:
# Wieske Chaosknoten VMs
- labels:
org: ccchh
site: wieske
type: virtual_machine
hypervisor: chaosknoten
targets:
- netbox-intern.hamburg.ccc.de:9100
- matrix-intern.hamburg.ccc.de:9100
- public-web-static-intern.hamburg.ccc.de:9100
- git-intern.hamburg.ccc.de:9100
- forgejo-actions-runner-intern.hamburg.ccc.de:9100
- eh22-wiki-intern.hamburg.ccc.de:9100
- mjolnir-intern.hamburg.ccc.de:9100
- woodpecker-intern.hamburg.ccc.de:9100
- penpot-intern.hamburg.ccc.de:9100
- jitsi.hamburg.ccc.de:9100
- onlyoffice-intern.hamburg.ccc.de:9100
- ccchoir-intern.hamburg.ccc.de:9100
- tickets-intern.hamburg.ccc.de:9100
- keycloak-intern.hamburg.ccc.de:9100
- onlyoffice-intern.hamburg.ccc.de:9100
- pad-intern.hamburg.ccc.de:9100
- wiki-intern.hamburg.ccc.de:9100
- zammad-intern.hamburg.ccc.de:9100
- pretalx-intern.hamburg.ccc.de:9100
- labels:
org: ccchh
site: wieske
type: physical_machine
targets:
- chaosknoten.hamburg.ccc.de:9100
storage:
tsdb:

1
resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf
View file

@ -9,7 +9,6 @@ server {
allow 2a00:14b0:4200:3380::/64;
allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing
# Z9
allow 2a07:c480:0:100::/56;
allow 2a07:c481:1::/48;
# fuxnoc
allow 2a07:c481:0:1::/64;

44
roles/alloy/defaults/main.yaml Normal file
View file

@ -0,0 +1,44 @@
alloy_config_default: |
prometheus.remote_write "default" {
endpoint {
url = "https://metrics.hamburg.ccc.de/api/v1/write"
basic_auth {
username = "chaos"
password = "{{ metrics__chaos_password }}"
}
}
}
prometheus.relabel "chaosknoten_common" {
forward_to = [prometheus.remote_write.default.receiver]
rule {
target_label = "org"
replacement = "ccchh"
}
rule {
target_label = "site"
replacement = "wieske"
}
rule {
source_labels = ["instance"]
target_label = "instance"
regex = "([^:]+)"
replacement = "${1}.hosts.hamburg.ccc.de"
action = "replace"
}
}
logging {
level = "info"
}
prometheus.exporter.unix "local_system" {
enable_collectors = ["systemd"]
}
prometheus.scrape "scrape_metrics" {
targets = prometheus.exporter.unix.local_system.targets
forward_to = [prometheus.relabel.chaosknoten_common.receiver]
}
alloy_config_additional: ""

2
roles/alloy/tasks/main.yaml
View file

@ -45,4 +45,6 @@
- name: Setup Alloy
ansible.builtin.import_role:
name: grafana.grafana.alloy
vars:
alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}"
become: true

10
roles/prometheus_node_exporter/meta/main.yaml
View file

@ -1,10 +0,0 @@
---
dependencies:
- role: distribution_check
vars:
distribution_check__distribution_support_spec:
- name: Debian
major_versions:
- "11"
- "12"
- "13"

14
roles/prometheus_node_exporter/tasks/main.yaml
View file

@ -1,14 +0,0 @@
- name: make sure the `prometheus-node-exporter` package is installed
ansible.builtin.apt:
name: prometheus-node-exporter
state: present
allow_change_held_packages: true
update_cache: true
become: true
- name: make sure `prometheus-node-exporter.service` is started and ansibled
ansible.builtin.systemd:
name: prometheus-node-exporter.service
state: started
enabled: true
become: true