diff --git a/inventories/chaosknoten/group_vars/all.sops.yaml b/inventories/chaosknoten/group_vars/all.sops.yaml index ebc53b7..0c5762d 100644 --- a/inventories/chaosknoten/group_vars/all.sops.yaml +++ b/inventories/chaosknoten/group_vars/all.sops.yaml @@ -1,4 +1,5 @@ msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str] +metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str] sops: age: - recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3 @@ -163,8 +164,8 @@ sops: SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-13T23:45:06Z" - mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str] + lastmodified: "2026-01-25T18:06:26Z" + mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str] pgp: - created_at: "2025-10-20T19:03:07Z" enc: |- @@ -360,4 +361,4 @@ sops: -----END PGP MESSAGE----- fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 unencrypted_suffix: _unencrypted - version: 3.10.2 + version: 3.11.0 diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index ecc942c..b87a198 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -53,16 +53,7 @@ nginx__configurations: - name: metrics.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } +alloy_config_additional: | loki.write "default" { endpoint { url = "https://loki.hamburg.ccc.de/loki/api/v1/push" @@ -98,9 +89,9 @@ alloy_config: | } rule { source_labels = ["__journal__hostname"] - target_label = "host" + target_label = "instance" regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" + replacement = "${1}.hosts.hamburg.ccc.de" action = "replace" } } @@ -111,30 +102,3 @@ alloy_config: | format_as_json = true labels = {component = "loki.source.journal", org = "ccchh"} } - - logging { - level = "info" - } - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "scrape_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } diff --git a/inventories/chaosknoten/host_vars/ntfy.sops.yaml b/inventories/chaosknoten/host_vars/ntfy.sops.yaml index e860cca..1328d66 100644 --- a/inventories/chaosknoten/host_vars/ntfy.sops.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.sops.yaml @@ -1,5 +1,3 @@ -secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str] -secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str] secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str] ntfy: user: @@ -18,8 +16,8 @@ sops: bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-20T19:01:39Z" - mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str] + lastmodified: "2026-01-25T18:41:48Z" + mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str] pgp: - created_at: "2025-10-20T19:03:04Z" enc: |- diff --git a/inventories/chaosknoten/host_vars/ntfy.yaml b/inventories/chaosknoten/host_vars/ntfy.yaml index cab4e76..2d68bfa 100644 --- a/inventories/chaosknoten/host_vars/ntfy.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.yaml @@ -15,90 +15,8 @@ nginx__configurations: - name: ntfy.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } - loki.write "default" { - endpoint { - url = "https://loki.hamburg.ccc.de/loki/api/v1/push" - basic_auth { - username = "chaos" - password = "{{ secret__loki_chaos }}" - } - } - } - - loki.relabel "journal" { - forward_to = [] - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "systemd_unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "instance" - } - rule { - source_labels = ["__journal__transport"] - target_label = "systemd_transport" - } - rule { - source_labels = ["__journal_syslog_identifier"] - target_label = "syslog_identifier" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - loki.source.journal "read_journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - format_as_json = true - labels = {component = "loki.source.journal", org = "ccchh"} - } - - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "unix_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } - +alloy_config_additional: | prometheus.scrape "ntfy_metrics" { targets = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}] - forward_to = [prometheus.relabel.default.receiver] + forward_to = [prometheus.relabel.chaosknoten_common.receiver] } diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 6246119..c238cec 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -166,11 +166,10 @@ certbot_hosts: ntfy: sunders: spaceapiccc: -prometheus_node_exporter_hosts: +alloy_hosts: hosts: ccchoir: eh22-wiki: - tickets: keycloak: netbox: onlyoffice: @@ -178,6 +177,15 @@ prometheus_node_exporter_hosts: pretalx: wiki: zammad: + grafana: + ntfy: + tickets: + renovate: + cloud: + public-reverse-proxy: + router: + sunders: + spaceapiccc: infrastructure_authorized_keys_hosts: hosts: ccchoir: @@ -208,10 +216,6 @@ netbox_hosts: proxmox_vm_template_hosts: hosts: chaosknoten: -alloy_hosts: - hosts: - grafana: - ntfy: ansible_pull_hosts: hosts: netbox: diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index f416b91..fe0cf78 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -64,11 +64,6 @@ roles: - nginx -- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts - hosts: prometheus_node_exporter_hosts - roles: - - prometheus_node_exporter - - name: Configure unattended upgrades for all non-hypervisors hosts: all:!hypervisors become: true @@ -83,10 +78,8 @@ - name: Ensure Alloy is installed and Setup on alloy_hosts hosts: alloy_hosts become: true - tasks: - - name: Setup Alloy - ansible.builtin.include_role: - name: grafana.grafana.alloy + roles: + - alloy - name: Ensure ansible_pull deployment on ansible_pull_hosts hosts: ansible_pull_hosts diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus.yml b/resources/chaosknoten/grafana/docker_compose/prometheus.yml index fd59034..7f94ab0 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus.yml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus.yml @@ -82,41 +82,6 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: pve-exporter:9221 - - job_name: hosts - static_configs: - # Wieske Chaosknoten VMs - - labels: - org: ccchh - site: wieske - type: virtual_machine - hypervisor: chaosknoten - targets: - - netbox-intern.hamburg.ccc.de:9100 - - matrix-intern.hamburg.ccc.de:9100 - - public-web-static-intern.hamburg.ccc.de:9100 - - git-intern.hamburg.ccc.de:9100 - - forgejo-actions-runner-intern.hamburg.ccc.de:9100 - - eh22-wiki-intern.hamburg.ccc.de:9100 - - mjolnir-intern.hamburg.ccc.de:9100 - - woodpecker-intern.hamburg.ccc.de:9100 - - penpot-intern.hamburg.ccc.de:9100 - - jitsi.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - ccchoir-intern.hamburg.ccc.de:9100 - - tickets-intern.hamburg.ccc.de:9100 - - keycloak-intern.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - pad-intern.hamburg.ccc.de:9100 - - wiki-intern.hamburg.ccc.de:9100 - - zammad-intern.hamburg.ccc.de:9100 - - pretalx-intern.hamburg.ccc.de:9100 - - labels: - org: ccchh - site: wieske - type: physical_machine - targets: - - chaosknoten.hamburg.ccc.de:9100 - storage: tsdb: diff --git a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf index ef2fe07..195b99d 100644 --- a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf +++ b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf @@ -9,7 +9,6 @@ server { allow 2a00:14b0:4200:3380::/64; allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing # Z9 - allow 2a07:c480:0:100::/56; allow 2a07:c481:1::/48; # fuxnoc allow 2a07:c481:0:1::/64; diff --git a/roles/alloy/defaults/main.yaml b/roles/alloy/defaults/main.yaml new file mode 100644 index 0000000..92bc960 --- /dev/null +++ b/roles/alloy/defaults/main.yaml @@ -0,0 +1,44 @@ +alloy_config_default: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ metrics__chaos_password }}" + } + } + } + + prometheus.relabel "chaosknoten_common" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + target_label = "org" + replacement = "ccchh" + } + rule { + target_label = "site" + replacement = "wieske" + } + rule { + source_labels = ["instance"] + target_label = "instance" + regex = "([^:]+)" + replacement = "${1}.hosts.hamburg.ccc.de" + action = "replace" + } + } + + logging { + level = "info" + } + + prometheus.exporter.unix "local_system" { + enable_collectors = ["systemd"] + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.chaosknoten_common.receiver] + } + +alloy_config_additional: "" diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml index 2bc308a..5e3cd64 100644 --- a/roles/alloy/tasks/main.yaml +++ b/roles/alloy/tasks/main.yaml @@ -45,4 +45,6 @@ - name: Setup Alloy ansible.builtin.import_role: name: grafana.grafana.alloy + vars: + alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}" become: true diff --git a/roles/prometheus_node_exporter/meta/main.yaml b/roles/prometheus_node_exporter/meta/main.yaml deleted file mode 100644 index 78bb770..0000000 --- a/roles/prometheus_node_exporter/meta/main.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -dependencies: - - role: distribution_check - vars: - distribution_check__distribution_support_spec: - - name: Debian - major_versions: - - "11" - - "12" - - "13" diff --git a/roles/prometheus_node_exporter/tasks/main.yaml b/roles/prometheus_node_exporter/tasks/main.yaml deleted file mode 100644 index c138f18..0000000 --- a/roles/prometheus_node_exporter/tasks/main.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: make sure the `prometheus-node-exporter` package is installed - ansible.builtin.apt: - name: prometheus-node-exporter - state: present - allow_change_held_packages: true - update_cache: true - become: true - -- name: make sure `prometheus-node-exporter.service` is started and ansibled - ansible.builtin.systemd: - name: prometheus-node-exporter.service - state: started - enabled: true - become: true