From 1523b15952987cc23f87a7845d480be9bf89327c Mon Sep 17 00:00:00 2001 From: June Date: Sun, 25 Jan 2026 21:11:07 +0100 Subject: [PATCH 1/8] docs: improve formatting and wording a bit to make things clearer --- docs/create-new-web-service-vm.md | 55 +++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/docs/create-new-web-service-vm.md b/docs/create-new-web-service-vm.md index 1f5874d..da8293b 100644 --- a/docs/create-new-web-service-vm.md +++ b/docs/create-new-web-service-vm.md @@ -5,36 +5,47 @@ Let's assume that you want to add a new web service `example.hamburg.ccc.de` whi ## IP, DNS, VM 1. Allocate a fresh [IPv6 in Netbox in the 2a00:14b0:42:102::/64 net](https://netbox.hamburg.ccc.de/ipam/prefixes/47/ip-addresses/). This will be the management address for the VM. -2. Add an entry `example.hosts.hamburg.ccc.de` with that AAAA to the name server (`ns-intern`). -3. Add an entry `example.hamburg.ccc.de` as a CNAME for `public-reverse-proxy` to the same zone. -4. Commit and reload the zone. -5. Create a new VM on chaosknoten, for example by cloning the Debian template 9023. Give it the name `example`. -6. Edit the ethernet interface to be connected to `vmbr0`, VLAN tag `2`. -7. Configure the IPv6 address in the Cloud-Init section. Leave IPv4 set to DHCP. -8. Make sure the VM is started at boot (options). -9. Adjust any other VM parameters as needed. -10. Boot the VM. -11. Add the [VM to Netbox](https://netbox.hamburg.ccc.de/virtualization/virtual-machines/). Make sure to enter the VM ID. -12. Add an Ethernet interface to the VM; we typically use `eth0` as a name. -13. Add IP for that interface, then choose "Assign IP" and search for the IP you've created. Make it the primary IP of that interface. +2. On `ns-intern`: + 1. Add an entry `example.hosts.hamburg.ccc.de` as an AAAA pointing to the allocated IP. + 2. Add an entry `example.hamburg.ccc.de` as a CNAME for `public-reverse-proxy` to the same zone. + 3. Commit and reload the zone. +3. On Chaosknoten: + 1. Create a new VM, for example by cloning the Debian template 9023. + Give it the name `example`. + 2. Edit the ethernet interface to be connected to `vmbr0`, VLAN tag `2`. + 3. Configure the IPv6 address in the Cloud-Init section. Leave IPv4 set to DHCP. + 4. Make sure the VM is started at boot (options). + 5. Adjust any other VM parameters as needed. + 6. Boot the VM. +4. Add the [VM to Netbox](https://netbox.hamburg.ccc.de/virtualization/virtual-machines/). + - Make sure to enter the VM ID. + - Add an Ethernet interface to the VM; we typically use `eth0` as a name. + - Add IP for that interface, then choose "Assign IP" and search for the IP you've created. Make it the primary IP of that interface. ## Ansible Basics As the first step, we need to make the host known to Ansible. 1. In `.sops.yaml`, add an entry for the host. Follow the other entries there. - 1. `keys.hosts.chaosknoten.age` needs an age public key (the private key is needed in the host-specific YAML) + 1. `keys.hosts.chaosknoten.age` needs an age public key (must be generated; the private key gets added later in the host-specific YAML) 2. `creation_rules` needs an entry for the host, referencing the age key. 2. In `inventories/chaosknoten/hosts.yaml`: 1. Configure basic connection info: - ``` + ```yaml example: ansible_host: example.hosts.hamburg.ccc.de ansible_user: chaos ansible_ssh_common_args: -J ssh://chaos@router.hamburg.ccc.de ``` You typically will want to use router as a jump host so that you can run Ansible on an IPv4 only connection. - 2. Add the host to the desired roles. As a minimum, you'll want `base_config_hosts` and `infrastructure_authorized_keys_hosts`. For a typical web service based on Docker Compose, you'll want `docker_compose_hosts`, `nginx_hosts`, and `certbot_hosts`. + 2. Add the host to the desired roles. + 1. As a minimum, you'll want the following roles: + - `base_config_hosts` + - `infrastructure_authorized_keys_hosts` + 2. For a typical web service based on Docker Compose, you'll also want: + - `docker_compose_hosts` + - `nginx_hosts` + - `certbot_hosts`. 3. In the directorry `inventories/chaosknoten/host_var/`: 1. A file `inventories/chaosknoten/host_var/example.yaml` with the host/service specific configuration. 2. A file `inventories/chaosknoten/host_var/example.sops.yaml` with the encrypted secrets for the host/service. Run `sops inventories/chaosknoten/host_var/example.yaml` to edit/create that file. Entries here should generally be prefixed with `secret__` to make it easier to see where that variable is coming from in templates etc. @@ -69,12 +80,20 @@ Before you can run Ansible successfully, you will want to make sure you can conn * `ssh chaos@example.hosts.hamburg.ccc.de` * `ssh -J chaos@router.hamburg.ccc.de chaos@example.hosts.hamburg.ccc.de` -Then run Ansible for `public-reverse-proxy` to add the necessary entries: `ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts.yaml --limit public-reverse-proxy`. +Then run Ansible for `public-reverse-proxy` to add the necessary entries: -Finally run Ansible for the new host: `ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts.yaml --limit example` +```sh +ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts.yaml --limit public-reverse-proxy +``` + +Finally run Ansible for the new host: + +```sh +ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts.yaml --limit example +``` # Commit your changes Do not forget to commit your changes, whether it's a new host or you are making changes to an existing host. -And always `git pull` before you run Ansible so avoid reverting anything! \ No newline at end of file +And always `git pull` before you run Ansible so avoid reverting anything! From c6c02724487ab278aa9904483d66efb8d4872d64 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 25 Jan 2026 21:16:38 +0100 Subject: [PATCH 2/8] docs: add section on conf. monitoring with Gatus (status.hamburg.ccc.de) --- docs/create-new-web-service-vm.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/create-new-web-service-vm.md b/docs/create-new-web-service-vm.md index da8293b..57a1e81 100644 --- a/docs/create-new-web-service-vm.md +++ b/docs/create-new-web-service-vm.md @@ -97,3 +97,16 @@ ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts Do not forget to commit your changes, whether it's a new host or you are making changes to an existing host. And always `git pull` before you run Ansible so avoid reverting anything! + +# Monitoring + +## Gatus (`status.hamburg.ccc.de`) + +After you configured a new service or website, add it to our status and uptime monitoring. +Take a look at the configuration in `resources/external/status/docker_compose/config` and extend it to cover the newly added service or website. The configuration should probably happen in either `services-chaosknoten.yaml` or `websites.yaml`. Taking the existing configuration as a reference should give guidance on how to configure new checks. Additionally there's also the comprehensive [Gatus Documentation](https://github.com/TwiN/gatus?tab=readme-ov-file#table-of-contents). + +After you've added some checks, the configuration can be deployed using: + +```sh +ansible-playbook playbooks/deploy.yaml --inventory inventories/external --limit status +``` From 8f7990acc0240123d2d488c334ba2904060ece12 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 25 Jan 2026 21:17:28 +0100 Subject: [PATCH 3/8] docs: add ansible_pull_hosts to the desired minimum roles/groups --- docs/create-new-web-service-vm.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/create-new-web-service-vm.md b/docs/create-new-web-service-vm.md index 57a1e81..46ff19f 100644 --- a/docs/create-new-web-service-vm.md +++ b/docs/create-new-web-service-vm.md @@ -42,6 +42,7 @@ As the first step, we need to make the host known to Ansible. 1. As a minimum, you'll want the following roles: - `base_config_hosts` - `infrastructure_authorized_keys_hosts` + - `ansible_pull_hosts` 2. For a typical web service based on Docker Compose, you'll also want: - `docker_compose_hosts` - `nginx_hosts` From 11779ab21d742da75e4ab13e001be29ee74ccd52 Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 25 Jan 2026 21:41:20 +0100 Subject: [PATCH 4/8] grafana: get alertmanager to be more chill a bit of help to deal with alert fatigue --- .../docker_compose/alertmanager.yaml.j2 | 23 +++++-------------- .../grafana/docker_compose/compose.yaml.j2 | 4 ++-- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 index 51aeb63..10fb346 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 @@ -7,7 +7,7 @@ route: group_by: [ "alertname", "site", "type", "hypervisor" ] group_wait: 30s group_interval: 5m - repeat_interval: 6h + repeat_interval: 26h routes: - receiver: "null" matchers: @@ -16,49 +16,38 @@ route: matchers: - org = "ccchh" - severity = "critical", - repeat_interval: 18h + repeat_interval: 26h continue: true - receiver: ntfy-ccchh matchers: - org = "ccchh" - severity =~ "info|warning", - repeat_interval: 36h + repeat_interval: 52h continue: true - receiver: ntfy-fux-critical matchers: - org = "fux" - severity = "critical", - repeat_interval: 18h + repeat_interval: 26h continue: true - receiver: email-fux-critical matchers: - org = "fux" - severity = "critical", - repeat_interval: 36h + repeat_interval: 52h continue: true - receiver: ntfy-fux matchers: - org = "fux" - severity =~ "info|warning", - repeat_interval: 36h + repeat_interval: 52h continue: true - - receiver: ccchh-infrastructure-alerts - matchers: - - org = "ccchh" - - severity =~ "info|warning|critical" templates: - "/etc/alertmanager/templates/*.tmpl" receivers: - name: "null" - - name: "ccchh-infrastructure-alerts" - telegram_configs: - - send_resolved: true - bot_token: {{ secret__alertmanager_telegram_bot_token }} - chat_id: -1002434372415 - parse_mode: HTML - message: {{ "'{{ template \"alert-message.telegram.ccchh\" . }}'" }} - name: "ntfy-ccchh-critical" webhook_configs: diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 index 90952ee..c2c312c 100644 --- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 @@ -19,7 +19,7 @@ services: - prom_data:/prometheus alertmanager: - image: docker.io/prom/alertmanager:v0.30.0 + image: docker.io/prom/alertmanager:v0.30.1 container_name: alertmanager command: - '--config.file=/etc/alertmanager/alertmanager.yaml' @@ -59,7 +59,7 @@ services: - /dev/null:/etc/prometheus/pve.yml loki: - image: docker.io/grafana/loki:3.6.3 + image: docker.io/grafana/loki:3.6.4 container_name: loki ports: - 13100:3100 From 995dbb06e2ea547177b00566b64b4fde595456cb Mon Sep 17 00:00:00 2001 From: June Date: Sun, 4 Jan 2026 19:32:42 +0100 Subject: [PATCH 5/8] wip: alloy --- roles/alloy/tasks/main.yaml | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 roles/alloy/tasks/main.yaml diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml new file mode 100644 index 0000000..2bc308a --- /dev/null +++ b/roles/alloy/tasks/main.yaml @@ -0,0 +1,48 @@ +# https://github.com/grafana/grafana-ansible-collection/blob/main/roles/alloy/tasks/deploy.yml#L124 +- name: ensure alloy user exists + ansible.builtin.user: + name: alloy + system: true + append: true + create_home: false + state: present + +- name: ensure the `/etc/alloy/` config directory exists + ansible.builtin.file: + path: /etc/alloy + state: directory + mode: "0770" + owner: root + group: alloy + become: true + +- name: synchronize the additional configuration files directory, if present + when: alloy__additional_configs_dir is defined and alloy__additional_configs_dir != "" + block: + - name: ensure rsync is installed + ansible.builtin.apt: + name: rsync + become: true + + - name: synchronize the additional configuration files directory, if present + ansible.posix.synchronize: + src: "{{ alloy__additional_configs_dir }}" + dest: /etc/alloy/additional + delete: true + recursive: true + use_ssh_args: true + rsync_opts: + - "--chown=root:alloy" + become: true + +- name: delete the additional configuration files directory, if not present + when: alloy__additional_configs_dir is not defined or alloy__additional_configs_dir == "" + ansible.builtin.file: + path: /etc/alloy/additional + state: absent + become: true + +- name: Setup Alloy + ansible.builtin.import_role: + name: grafana.grafana.alloy + become: true From c7d51af5b428b7b1fe065c23fbfa1d1b6bd51980 Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 25 Jan 2026 20:03:13 +0100 Subject: [PATCH 6/8] rollout Alloy to replace prometheus_node_exporter With the new network we need to deploy a push based solution in order to get metrics into prometheus --- .../chaosknoten/group_vars/all.sops.yaml | 7 +- .../chaosknoten/host_vars/grafana.yaml | 42 +-------- .../chaosknoten/host_vars/ntfy.sops.yaml | 6 +- inventories/chaosknoten/host_vars/ntfy.yaml | 86 +------------------ inventories/chaosknoten/hosts.yaml | 16 ++-- playbooks/deploy.yaml | 11 +-- .../grafana/docker_compose/prometheus.yml | 35 -------- .../grafana/nginx/metrics.hamburg.ccc.de.conf | 1 - roles/alloy/defaults/main.yaml | 44 ++++++++++ roles/alloy/tasks/main.yaml | 2 + roles/prometheus_node_exporter/meta/main.yaml | 10 --- .../prometheus_node_exporter/tasks/main.yaml | 14 --- 12 files changed, 69 insertions(+), 205 deletions(-) create mode 100644 roles/alloy/defaults/main.yaml delete mode 100644 roles/prometheus_node_exporter/meta/main.yaml delete mode 100644 roles/prometheus_node_exporter/tasks/main.yaml diff --git a/inventories/chaosknoten/group_vars/all.sops.yaml b/inventories/chaosknoten/group_vars/all.sops.yaml index ebc53b7..0c5762d 100644 --- a/inventories/chaosknoten/group_vars/all.sops.yaml +++ b/inventories/chaosknoten/group_vars/all.sops.yaml @@ -1,4 +1,5 @@ msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str] +metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str] sops: age: - recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3 @@ -163,8 +164,8 @@ sops: SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-13T23:45:06Z" - mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str] + lastmodified: "2026-01-25T18:06:26Z" + mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str] pgp: - created_at: "2025-10-20T19:03:07Z" enc: |- @@ -360,4 +361,4 @@ sops: -----END PGP MESSAGE----- fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 unencrypted_suffix: _unencrypted - version: 3.10.2 + version: 3.11.0 diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index ecc942c..b87a198 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -53,16 +53,7 @@ nginx__configurations: - name: metrics.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } +alloy_config_additional: | loki.write "default" { endpoint { url = "https://loki.hamburg.ccc.de/loki/api/v1/push" @@ -98,9 +89,9 @@ alloy_config: | } rule { source_labels = ["__journal__hostname"] - target_label = "host" + target_label = "instance" regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" + replacement = "${1}.hosts.hamburg.ccc.de" action = "replace" } } @@ -111,30 +102,3 @@ alloy_config: | format_as_json = true labels = {component = "loki.source.journal", org = "ccchh"} } - - logging { - level = "info" - } - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "scrape_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } diff --git a/inventories/chaosknoten/host_vars/ntfy.sops.yaml b/inventories/chaosknoten/host_vars/ntfy.sops.yaml index e860cca..1328d66 100644 --- a/inventories/chaosknoten/host_vars/ntfy.sops.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.sops.yaml @@ -1,5 +1,3 @@ -secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str] -secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str] secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str] ntfy: user: @@ -18,8 +16,8 @@ sops: bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-20T19:01:39Z" - mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str] + lastmodified: "2026-01-25T18:41:48Z" + mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str] pgp: - created_at: "2025-10-20T19:03:04Z" enc: |- diff --git a/inventories/chaosknoten/host_vars/ntfy.yaml b/inventories/chaosknoten/host_vars/ntfy.yaml index cab4e76..2d68bfa 100644 --- a/inventories/chaosknoten/host_vars/ntfy.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.yaml @@ -15,90 +15,8 @@ nginx__configurations: - name: ntfy.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } - loki.write "default" { - endpoint { - url = "https://loki.hamburg.ccc.de/loki/api/v1/push" - basic_auth { - username = "chaos" - password = "{{ secret__loki_chaos }}" - } - } - } - - loki.relabel "journal" { - forward_to = [] - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "systemd_unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "instance" - } - rule { - source_labels = ["__journal__transport"] - target_label = "systemd_transport" - } - rule { - source_labels = ["__journal_syslog_identifier"] - target_label = "syslog_identifier" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - loki.source.journal "read_journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - format_as_json = true - labels = {component = "loki.source.journal", org = "ccchh"} - } - - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "unix_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } - +alloy_config_additional: | prometheus.scrape "ntfy_metrics" { targets = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}] - forward_to = [prometheus.relabel.default.receiver] + forward_to = [prometheus.relabel.chaosknoten_common.receiver] } diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 2a95ba8..5c114c9 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -166,11 +166,10 @@ certbot_hosts: ntfy: sunders: spaceapiccc: -prometheus_node_exporter_hosts: +alloy_hosts: hosts: ccchoir: eh22-wiki: - tickets: keycloak: netbox: onlyoffice: @@ -178,6 +177,15 @@ prometheus_node_exporter_hosts: pretalx: wiki: zammad: + grafana: + ntfy: + tickets: + renovate: + cloud: + public-reverse-proxy: + router: + sunders: + spaceapiccc: infrastructure_authorized_keys_hosts: hosts: ccchoir: @@ -208,10 +216,6 @@ netbox_hosts: proxmox_vm_template_hosts: hosts: chaosknoten: -alloy_hosts: - hosts: - grafana: - ntfy: ansible_pull_hosts: hosts: netbox: diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index f416b91..fe0cf78 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -64,11 +64,6 @@ roles: - nginx -- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts - hosts: prometheus_node_exporter_hosts - roles: - - prometheus_node_exporter - - name: Configure unattended upgrades for all non-hypervisors hosts: all:!hypervisors become: true @@ -83,10 +78,8 @@ - name: Ensure Alloy is installed and Setup on alloy_hosts hosts: alloy_hosts become: true - tasks: - - name: Setup Alloy - ansible.builtin.include_role: - name: grafana.grafana.alloy + roles: + - alloy - name: Ensure ansible_pull deployment on ansible_pull_hosts hosts: ansible_pull_hosts diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus.yml b/resources/chaosknoten/grafana/docker_compose/prometheus.yml index fd59034..7f94ab0 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus.yml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus.yml @@ -82,41 +82,6 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: pve-exporter:9221 - - job_name: hosts - static_configs: - # Wieske Chaosknoten VMs - - labels: - org: ccchh - site: wieske - type: virtual_machine - hypervisor: chaosknoten - targets: - - netbox-intern.hamburg.ccc.de:9100 - - matrix-intern.hamburg.ccc.de:9100 - - public-web-static-intern.hamburg.ccc.de:9100 - - git-intern.hamburg.ccc.de:9100 - - forgejo-actions-runner-intern.hamburg.ccc.de:9100 - - eh22-wiki-intern.hamburg.ccc.de:9100 - - mjolnir-intern.hamburg.ccc.de:9100 - - woodpecker-intern.hamburg.ccc.de:9100 - - penpot-intern.hamburg.ccc.de:9100 - - jitsi.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - ccchoir-intern.hamburg.ccc.de:9100 - - tickets-intern.hamburg.ccc.de:9100 - - keycloak-intern.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - pad-intern.hamburg.ccc.de:9100 - - wiki-intern.hamburg.ccc.de:9100 - - zammad-intern.hamburg.ccc.de:9100 - - pretalx-intern.hamburg.ccc.de:9100 - - labels: - org: ccchh - site: wieske - type: physical_machine - targets: - - chaosknoten.hamburg.ccc.de:9100 - storage: tsdb: diff --git a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf index ef2fe07..195b99d 100644 --- a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf +++ b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf @@ -9,7 +9,6 @@ server { allow 2a00:14b0:4200:3380::/64; allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing # Z9 - allow 2a07:c480:0:100::/56; allow 2a07:c481:1::/48; # fuxnoc allow 2a07:c481:0:1::/64; diff --git a/roles/alloy/defaults/main.yaml b/roles/alloy/defaults/main.yaml new file mode 100644 index 0000000..92bc960 --- /dev/null +++ b/roles/alloy/defaults/main.yaml @@ -0,0 +1,44 @@ +alloy_config_default: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ metrics__chaos_password }}" + } + } + } + + prometheus.relabel "chaosknoten_common" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + target_label = "org" + replacement = "ccchh" + } + rule { + target_label = "site" + replacement = "wieske" + } + rule { + source_labels = ["instance"] + target_label = "instance" + regex = "([^:]+)" + replacement = "${1}.hosts.hamburg.ccc.de" + action = "replace" + } + } + + logging { + level = "info" + } + + prometheus.exporter.unix "local_system" { + enable_collectors = ["systemd"] + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.chaosknoten_common.receiver] + } + +alloy_config_additional: "" diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml index 2bc308a..5e3cd64 100644 --- a/roles/alloy/tasks/main.yaml +++ b/roles/alloy/tasks/main.yaml @@ -45,4 +45,6 @@ - name: Setup Alloy ansible.builtin.import_role: name: grafana.grafana.alloy + vars: + alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}" become: true diff --git a/roles/prometheus_node_exporter/meta/main.yaml b/roles/prometheus_node_exporter/meta/main.yaml deleted file mode 100644 index 78bb770..0000000 --- a/roles/prometheus_node_exporter/meta/main.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -dependencies: - - role: distribution_check - vars: - distribution_check__distribution_support_spec: - - name: Debian - major_versions: - - "11" - - "12" - - "13" diff --git a/roles/prometheus_node_exporter/tasks/main.yaml b/roles/prometheus_node_exporter/tasks/main.yaml deleted file mode 100644 index c138f18..0000000 --- a/roles/prometheus_node_exporter/tasks/main.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: make sure the `prometheus-node-exporter` package is installed - ansible.builtin.apt: - name: prometheus-node-exporter - state: present - allow_change_held_packages: true - update_cache: true - become: true - -- name: make sure `prometheus-node-exporter.service` is started and ansibled - ansible.builtin.systemd: - name: prometheus-node-exporter.service - state: started - enabled: true - become: true From 5693989c384646d260ad8b37f85d45aa5d5827b8 Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 25 Jan 2026 20:58:14 +0100 Subject: [PATCH 7/8] add alloy to the z9 hosts and some cleanup --- .sops.yaml | 4 + inventories/chaosknoten/group_vars/all.yaml | 43 +++++ inventories/z9/group_vars/all.sops.yaml | 200 ++++++++++++++++++++ inventories/z9/group_vars/all.yaml | 43 +++++ inventories/z9/hosts.yaml | 6 + roles/alloy/defaults/main.yaml | 12 +- 6 files changed, 302 insertions(+), 6 deletions(-) create mode 100644 inventories/z9/group_vars/all.sops.yaml create mode 100644 inventories/z9/group_vars/all.yaml diff --git a/.sops.yaml b/.sops.yaml index bb83e39..0b9c245 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -51,6 +51,10 @@ creation_rules: *admin_gpg_keys age: *host_external_age_keys + - path_regex: inventories/z9/group_vars/all.* + key_groups: + - pgp: + *admin_gpg_keys ## host vars # chaosknoten hosts - path_regex: inventories/chaosknoten/host_vars/cloud.* diff --git a/inventories/chaosknoten/group_vars/all.yaml b/inventories/chaosknoten/group_vars/all.yaml index 60349e0..76147d8 100644 --- a/inventories/chaosknoten/group_vars/all.yaml +++ b/inventories/chaosknoten/group_vars/all.yaml @@ -14,3 +14,46 @@ msmtp__smtp_port: 465 msmtp__smtp_tls_method: smtps msmtp__smtp_user: any@hosts.hamburg.ccc.de msmtp__smtp_from: "{{ inventory_hostname }}@hosts.hamburg.ccc.de" + +alloy_config_default: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ metrics__chaos_password }}" + } + } + } + + prometheus.relabel "chaosknoten_common" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + target_label = "org" + replacement = "ccchh" + } + rule { + target_label = "site" + replacement = "wieske" + } + rule { + source_labels = ["instance"] + target_label = "instance" + regex = "([^:]+)" + replacement = "${1}.hosts.hamburg.ccc.de" + action = "replace" + } + } + + logging { + level = "info" + } + + prometheus.exporter.unix "local_system" { + enable_collectors = ["systemd"] + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.chaosknoten_common.receiver] + } diff --git a/inventories/z9/group_vars/all.sops.yaml b/inventories/z9/group_vars/all.sops.yaml new file mode 100644 index 0000000..7c25351 --- /dev/null +++ b/inventories/z9/group_vars/all.sops.yaml @@ -0,0 +1,200 @@ +metrics__chaos_password: ENC[AES256_GCM,data:seOU504dZ9K21+NK1MBf9isee2L2rueP6Bl0F66R,iv:ZtQ516gzJQSSgvOOAzPF9MuarXqHSLXy37/9z85KoQ8=,tag:dIal6OxPLli+7DbzhjNFsA==,type:str] +sops: + lastmodified: "2026-01-25T19:52:03Z" + mac: ENC[AES256_GCM,data:6JXc+K8fmANf22puWyllV5wVSxZSVnN+U7GM9lNhkxbUBM4AaIedIHOXz9zDaZh/nT6onrW2nhKNC00kWziaddOnBxBUCWUk7bDGea6qJMIk4GfyU0f/xX7mHpgYorF/KmQP1uvNNAryn7zeSfS8Vk27GFDPbBO3GvYlOZFUJD8=,iv:6X6uf9obhNix/qLrpiP3bw1CWM7dY+XAEdfhuTTmuVc=,tag:KJHK1Hc/uV8DOw/7txHfEw==,type:str] + pgp: + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAxK/JaB2/SdtAQ/9GKyJ+6SzK5xucxIxUKPRdsxirJwd6LHuIDkVTr7JTjfi + sXQZKVtQ7ZXbbVgZKURLtsdbhayZoU/8xYCQsX4vzDeAKc4bS6X25PLxs2oBKCYB + 2oWl/jhKSAtVjtgnPnxljiEGxkDKW2sKlfjdjMj9yOYyif35AoQ8pIr2Tg4U8Z9C + ofaWBejvqxgaIShXe4jio3SIiOLYwTnaYmkoSY3QEA3RjckzNmqRE4aX+csB27cI + Vt8aGrcNzM8gCfi8IM1ypLHLw7Fg0OntF91RAUExG9OZJm2rGZabUixxhOCf/ttk + UOq7Eq29xFr9mTzyoZC2zmaOt7O+PIu8FDOvkvCgNv89ewn00DjT7DYSXB0AnPRD + VahAi4VAjKU2RXXbfZArdCXJpCTM2OPnXBh8Bfx/7eTnu2O8EK8OFbWuOWja8Ogr + 7z9bgsoK4Uva6F3BQcLlZppKmkLk0P8detZihvwNbS55kkkdsA9LiyYEoHpasWpG + HW4dcQOqyuKwGjLE7FsqPtlxmWD6psCK3GdHzKGQR9fbXfUyD+c0DmPgPh6roFW8 + XzvRGw6YUrP7/wtvUH4zSLQbB6kqz6nO88isPoLpClyQ/3Khj9QLljCDQB+yRBIu + p3a2HISwt4HQzuckk8W4yKIDdzf86dXVEMqUe4JTe+vW9PPobnUEXrPgRBNZYD7S + XgHOfGiWknFPa8s4KCHZK9sLB2joWAJTtQnk4cuaXoIgamiXB0qgiArc43PsjstE + N6kvVXrFVgQ9Xlrp8XDJHOsUYAy8admA8KNQF+XQ+KeHgQGKKX1RjbBFunIkaOc= + =1Rdp + -----END PGP MESSAGE----- + fp: EF643F59E008414882232C78FFA8331EEB7D6B70 + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA6EyPtWBEI+2AQ//QgGTp9DYSoNWI5n0+9gMUP7asTRdRl7T+xT88k7cCO9Z + lNi86qeqiGGkqTffARBJNaq+ut/D7EKc6ivp+ewfySimM5E3ape3C6rulHybE/j8 + 3EpP4VW+5yA4Nq+CbZvzQb60oXR8LnGzVX1gePWwyQozVzRS72/hnxyecQYsc1IJ + OTQPSfclZXJO3k6fK+BrfHsjJkOpHYoLAnI/9ty9JBSuwGfzI+ur2Jimn0Y783Ou + orZNzxrRKfkIPjdFnWGY3i31nW9tnkEXLdsdOHfOu8Ahtdi2NhwReSv5hMKPXbOw + lxhL/Y1bG4ChgAAFVG5QYZ69tuzSov8XP2Wv6auVA7HC3H+689fNeO0C6GhDcVgc + LBF1nK/zJq95uxlSNy5dpTSzKqwlwRzvLOCPByXc3pLcDDW9Zp194bS/iDGfnfqe + UUPK9e0gX8TYOeQhF3K+H6JMdFO/uYbiaeVZWmvOV6jSiii0CXoGe4oVZqcfcfA3 + RScUjLx0f2w4xQwU60ZmHsmvs2PmdBsNDPeQXrqeyAfgFReDoI1RLo+k+3khoAJE + LzzNFg6bVBx3rRazWoASlHYK8i6dTHpMBompPC+kmjorZnoqnTRX4bix0atsFY7g + vt8CVfqy8YKrVIGPZnDAsrZ3ecShIQFB6OfxgSb6nqN1K3NwFcjXWH05SJOfFR3S + XgGrU49/hKqHTmAGHbWoe54qkPj+WvRkeGccEnvtum8yrPpDpmYg+wyEm3JeQf1S + gCHS/j0pJS/CnnfgoUgkYCMokGvtSoTJgIE/2XTA/BFNRg0vc1Dgk/WonG33PDU= + =ev2Q + -----END PGP MESSAGE----- + fp: F155144FC925A1BEA1F8A2C59A2A4CD59BFDC5EC + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAz5uSgHG2iMJAQ//SxqxshraAR0pQf1lzhtZ5RHoNZJnaZKwic/pvJvIDUCA + 6zotOpu478rK4w8zWdX1gfjve+iu4BkaB16lZqvsV6lLq80dT3yfeil9ETFElKuA + womIEdAafq8W71eTffUZ3Klrg/WjDVjeDRRKqz8vv9pd9MQbYmDhrjRG/ySP5qgZ + +/apRAOrYnbtzgjlPAfLIGD65jvS3JRE3gbZfpzzkLB8P5M1JVOUf15FxAZ2tyhO + PZ3FYC2JbCzftp0Iiksq8sl42Fl0FzOTLFQb0GhQ60tJatFVWhG31NeXdRRuLnQU + 5bmanb2nJBroQJWM/8piG8npG8jhzRzeMTHboW5TezYAXBLxRQJct7pR9ZwDje2U + 5j9VkNyKQ+wMJ2vMiyshserEe6gjc2/E+XYDheAPrFPqwGNklb6OSemm4vWwd6GK + HNqDxA/C0du1b1vm9CTLgk7utbEpspnNQnZh32iifSfiQ3Zl7FwTxnA/2Bj0csQ/ + xrck7T2gzY39tOqXbq0QqIQA31BW4ukmxcAKn8pmJpguW0cBxDTaGNXQ4jo+8YtQ + MYYT4dR9S95MsOKWGREvMA0GMxzwbA2eMwZ7yUARCLVGD48MMiiDZcYqd03cnOO3 + hGj+vy0FbsVdknztBDeGttUYHOtjb+XO4gF4sHdpaWxdF7kVVknNUtciWa+Kw4LS + XgGqWekdWhsKZ+bPboinUPY9e5vkgLueSWrQ0aqi5Pte9lQ3pYPqT2U51fJG5G9/ + tYiofc0K7CB/qyxB7LpF5rtUla9oQQJd36xC0eO7laSapWiag2rzuIsMxR+4egA= + =a2qJ + -----END PGP MESSAGE----- + fp: 18DFCE01456DAB52EA38A6584EDC64F35FA1D6A5 + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAw5vwmoEJHQ1AQ//U6R7Cb1kfojFwzVy9ky+y7Puhk7Jyog+jLabWhurkV30 + RO4EeTXMEJ1gVU9qJDeHn7Fr1HbYr9cdTf1yw5Y23p+pBZA1wRTkxHctEk1KNDRW + 8QdCmQOu8jTDc5cq0F6d6lD6vJpfjaQez5cT7dN9Eqp0jUsmQCSLmqmXvbNEv3fW + n+o/VsvtaqMTjhlPUHvhe/d0/YWvkp4xPycDVW/oYh2KE3QUV5AKUJSOLuSIFENy + f0hjnoz8xmY7eA20uXvTWPgUU9J+KCSgmy87wM/WkM3kjKWOUkijDFCsCWSEIx4W + E2iY6N7yaYBkKfQ6s+f77xg+vc7g1plheK3pdyYkYvgfeqg27QFV/3m80f0gULS4 + bNrZKNRrMD+grgjB75cj14PRHGQcaZEouE7l2uCUNbR/hFIF1M2F91HAW61mVLv6 + ZNluofRYqHf+YWUO4KtJwpfgfh0gsCF3KaeHnAA/Xy9e+7KRgWbAbsDIQr40Nqm1 + Cbv/HqjHCeS7ylw3TmYcwFoGO1XoL/toSQQ4/y0JPMCae+MGslDm2o/1X1VqAnIZ + sdhcTKY8HJWxn8uc5MFG4Mr0PhMIXirhBBQLYXdVJ/tOj9yVU+gJZe3lv64uQgDZ + Ey1KESfJu98uwrPS3Dzy2YPbT7Gh58sOfHeaDoeZAu+YzQMOQ7V260vu0XXMgSfS + XgE3PiMBjbW4eypClEK6H5iSL4SjEm+NweQNkwGaxqLSsb7LuOtSkiEmf4mdQEnI + SI14d0nNv7ki0T59Ssmi65A49SXjvLzsCBE1DgeqVD8IwKCewma0dgkNErdyG4s= + =rV/a + -----END PGP MESSAGE----- + fp: 87AB00D45D37C9E9167B5A5A333448678B60E505 + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DerEtaFuTeewSAQdAdDLPgKw0A+eoKiYGIKxOFZHYVg0V4UmuIti0XC5RJCIw + IPu2/Y45X9L40RRhH8N9lazjLeJv5Lbo08hMlo/CgshZ0BJVot8mBAiH/R2DsVRC + 0l4BL6ctQ/xivjWQBBhy/DCYVtDRv8JXIEXNJgU/+UjkSE8Auh4NASANg9GTcBBn + lukzOBGYF9nH69fAkVtZbNL5+dFoPLDPUzZTU19D15J6IJkt+gKPSzjbtWaJqIsQ + =dGU4 + -----END PGP MESSAGE----- + fp: 057870A2C72CD82566A3EC983695F4FCBCAE4912 + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAxjNhCKPP69fAQ/+LQ3yEzhjYXoDkE9olsl0lVxQ9FdCbqDHFJZAIReI0jqg + WF+0GmoMuG4kFZu+ju3vCWpT5kH84SUxOFXyaXp1TCfcJ1zCUno93fVssOL/9Sma + vVPVIXpTqJqFBOWJNOe7wNjvQiDE4TxjGC/FXr+hOoLZXrf7gdNaUXxZOb+vPZih + t79eZhxALGwmwsMdZxkA8ERCmNJet1/wn7s5vUjwrDYRZL2zGf4yocSCjwGYHOCs + j+DcrYG7Cd5J+CS8rKu2Yh5KEAfMhgMxGjK0HKUVPk1cQxOgronnM1vrij30S4+9 + avNlOwAerg3RaFhXPj9UY7FGV+rZQY1CQKEWqr4AANkdDXb/LnLalwMBMcm+EDwT + zHxBhJ69QJmsZUP3Z5WQqxmyAux9+oodgehWKkY4sCR2huHuysbJNEStuI1HaTRj + ZJafiniHkFyQyTqc4wwJrRxkwJM6mVvcZdXuV7+QaEWr3FEF0t7tyEYUIRkUlJOQ + IUPDClDRLJnQGq11XT/QOlGfxET9fGoAkij1LrPqpvHxJ6IEGLMOPN4kw1yg02yO + u0HiB2wIUzKHJJ6vMR6zK3WY4MXCQISTZXpK7mILleRIIOWhw7C7gvlfuYkMT3fM + dXUQRhTblZXaeTxRuCUPqa0eGzac4UJBVoRAWXYiZWhdKxNJbyCMRQDcaOeho9/S + XgGENH9zFjI//pveCrlxx5BKDxTdqIn9R3iskbKbZRhVr+pU3IK4uCsUQlOBG3++ + zxQinHgNbqA22clcuRMZ1NeDrzDfBLndsWuSeyWaAA9qEG9XjmjCRRzPGACoDLs= + =dywj + -----END PGP MESSAGE----- + fp: F38C9D4228FC6F674E322D9C3326D914EB9B8F55 + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA1Hthzn+T1OoAQ/+P4h9d4LPElJbVr4L3cE+6/9mhLF6n1hre14cCTSxTe20 + HqpY7U9yE4G/1IfAE3pueOgc5FsLRPn8VnHAzy6ygt4/xXXzH6aACDiUtweqyExE + K39J9PLjczWb7XGbZc3YKoz5x0Q/93s+CdjK7piamb41bMRVbmffNtEceH8Gld9u + t8SI18F1yrRtXpK8FvzFLvA1jEt9sduPVq51bWG2LjMFaMrsm7hXt1ArUGsOoGN3 + 36E4VrVp6gp8BVa+apsUY3mHBi3hUE0h2tO4iEsi1qYr1OTn3v4Dn9oxKrYIIom2 + hHszqVSq0fnIqoKOZbyUe2AdXtnTGpQQRxCBvtIEBNB1FS/CKCe7ceXVBZujU2Kd + JD3Lg5uXgkolfyjFCPzOp292xvJ83i7QMoTuVEw14PSjux3jAa4K3wpKUvF80ja2 + ugGj3zTLAHdAV37lKO2WYZuMMJLKWKX1p9yKZqteJdiLQHH8f24dFZ2Vtoly/GKM + KzGJ1fimB6divQ8TOHVFAr1qDksk8zf1PBJ/IlWoBKv5IWwoikf42IOL/P2c/nk7 + 4pYwHrlk8y71Cgjw50K9/T/Ul0Ov6ay4FK+0vy+zbokSVczZKsrL4/Tc6s0S9ty6 + SVKm7yL+BSGgZWmDNesYoRzboBT2mSb1N45ThUaeW7AwMo3hDJPjEkaFtZN3bqzS + XgEIFYvxWH6OEIl+VZ/J5qxxmi3Cz6XVzTliCnoTFUoVxHyOxN6HX0Jn/qRqmmlN + mJX4OT1FJ3WnqOHQ5Cm4403bm2H79mGCBKYiXPQeO/bVBh0mTbeYKRr8bjsm+rc= + =lIEt + -----END PGP MESSAGE----- + fp: 5DA93D5C9D7320E1BD3522C79C78172B3551C9FD + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA46L6MuPqfJqAQ//Ue74vm+2K7chQcfdrY5GR0rPUKNiX9MMw9zgHpdyHlXa + FfL0NQbG8Rfc/6Yf7LH1sWMGED5Yvci3z+YkTWg8Fcv1pYQGj6Kul2L0aL4tvq/a + tdIJi+Ajp92jr+5jdae+GvJZedaHZYxykyeSe4/nk1j6k4u3TiVMBczk7z9701F+ + ZFtG1SBRcqjZ/Vx36B6s+10f9ft7TWCIJfeTimBJV3fFt7r698vTQ+S1/uxo5Ik0 + kQOtYDxsigBBeE0OX2ZBDyeGhfl/PZgN7GD7bgpjnfzDi+8kMXEMu4z88tOQulhk + qj+63irEydFCsMEC22XhLaqW8bjld0VAnkXv7DfoEbMt84XN6SejjDy6aXK9C0Qd + BIyQwTvsmgbInluw8Qu+GJPLLbY9qYjjuo5BbwUeBfiVxQaBYcm5lmPSKM0lq+Uq + fUYowpMS417L5kkp2yE/NmKOzi2ZuiFWMCpvPIvKea9zJxvEtIjohwtM86b4LH+j + 7yie9gWu0bhBw9keKtIbRmgbsilp8E5OUHXgOT0sNWTLenQDsWQ9dmgvtpeEb9ax + 6mw1QUpFz4CcHhuQixoI+q4y0SXcWxyN0U78U8igaELUtwpaRR7yf4VUJOEid+m1 + Rzu5jLCuhlLmmW03W6Y/Vl+n0QOyEl0uPCRiRgYeUzKiYw6NRYHPezbJnmNAeKnS + XgFCdzLc5Jl6OqJfy4V0yJucGq72oKK1wdJi74PqTNs44CaeEW8tDhxVWm367e5b + Ve88LJyDhOrMi57aKcAJ12HoL8pI5hambJ0qSs9RKpnQIJH7U60MBBTCBHN3H4k= + =az/q + -----END PGP MESSAGE----- + fp: 8996B62CBD159DCADD3B6DC08BB33A8ABCF7BC4A + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DQrf1tCqiJxoSAQdA04/sIHLMEFJO8wCB5+N5QWPzwyefW49JuNr/O2A+tTcw + Rm/CybmXPnSCx7p8QLruOG0tz8kM+YoSthSWlC9/B6TZgKLyrMOvx1U/fSNjKC4Q + 0l4BDFhVCnXKTQmfZtj5Qpwuj3H5fZ7QzKUQz542pvqN/fJVnc0Q4rQapKcU4AOx + JTdXpu6gP3FRGviA342GHJU0gq220vSzPu889dsdmtgNfAEQWPLVKKwjigDQN+SV + =2Eki + -----END PGP MESSAGE----- + fp: B71138A6A8964A3C3B8899857B4F70C356765BAB + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DzAGzViGx4qcSAQdAxLZsKVzF30df0Zk7Eg9u7fLJzApid00aEcZVxHQnZ0kw + 5SDeSOzzTue71lKcCyunbO1/e20jMrNVvYKQp0kKkNHpTWgjN0hW3vZt6zeLcrSo + 0lgBTOoJykoj24Y9WvIaQbae2K6M35drO2c7nhVmTzibUe7XEJ3C+vbUySdSTd+0 + WL1IjqZUGSUL4SUIW6kW0WFdSJ01O6vbXhw1gw7KwKMfBHgIUAzpENTW + =S45t + -----END PGP MESSAGE----- + fp: D2E9C0807BF681F5E164DAFC5EE1B61CD90954CD + - created_at: "2026-01-25T19:51:13Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA2pVdGTIrZI+AQ/8Dd/YnYUP9OA6qxJcerf9mCkQkC5PLehCPAOLOn4sNV5x + DnpdfAvgej1Vuy0CHK8//PAiEC7idLN+ictIdQgy0RaObp50tca44U2ssQOkmxcd + j5WpKKunHsKomksr7bRpwm/vtN4LoldQc7g1qaBlsaJE7iEOrB8I3n3fFhWD6xBG + TUzRe8r/M2/c25Agky9caILYvjm/etCsf/gZq3RwVvV48912JNqb+7o04vpj3MbO + AOsiEBCTNSZqN5XuRi/jCpQNe0p18M9irYkFWVe2be6Cb4wE2cdg904rC3K+v0QA + nwD6/bXWGI7WAF6nhvuiAS0vxmbvOePNI3KZ1CdEDeScqnAWUdkFuuAwmw0K7tHt + UJe/SlML6strnnjOGR6p3eeIjoDKtGBiqEjXYyEcXPVi8vFSd7muGcjLieyJUmfH + FVGA7bF+a6c4iTFSM2GNpANFV0qzZ/wa4aj9MqzOATTglQnr2LZJP7chnzoLyzx6 + 7cjTcWHsb3E+D7X37yF+mZAT6yvOoxaQNqTY6u1ZoY9NrGdJ1reudAlzg6k10cpf + O4Zww2Jjz5yEhvS9cTh8+bKOJYgKnbg/LLty/ade+rio4E0jn+a6VgRCqIMbGwgx + vf9ATU8S10/Es2cT6f99EpPgV0w9QCfhAGel/sjXk/zIT8rF2SbIlXf0/GK3yaXS + XgGrocZNe2RNZd3ZjsvtU6bBsPd9tekQLjC1vE6U/WXXPKapb6aOq2eL7Qb7QFu7 + XSGN+YA/c9OwmtJLP3y5mGBowa6vWT1Uf6NweamPYJBpNG27Bt5yLlnEnaDZokw= + =9ri7 + -----END PGP MESSAGE----- + fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/inventories/z9/group_vars/all.yaml b/inventories/z9/group_vars/all.yaml new file mode 100644 index 0000000..9a31251 --- /dev/null +++ b/inventories/z9/group_vars/all.yaml @@ -0,0 +1,43 @@ +alloy_config_default: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ metrics__chaos_password }}" + } + } + } + + prometheus.relabel "z9_common" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + target_label = "org" + replacement = "ccchh" + } + rule { + target_label = "site" + replacement = "z9" + } + rule { + source_labels = ["instance"] + target_label = "instance" + regex = "([^:]+)" + replacement = "${1}.z9.ccchh.net" + action = "replace" + } + } + + logging { + level = "info" + } + + prometheus.exporter.unix "local_system" { + enable_collectors = ["systemd"] + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.z9_common.receiver] + } + diff --git a/inventories/z9/hosts.yaml b/inventories/z9/hosts.yaml index 9f4a692..f88f106 100644 --- a/inventories/z9/hosts.yaml +++ b/inventories/z9/hosts.yaml @@ -49,5 +49,11 @@ ola_hosts: proxmox_vm_template_hosts: hosts: thinkcccore0: +alloy_hosts: + hosts: + authoritative-dns: + light: + yate: + dooris: ansible_pull_hosts: hosts: diff --git a/roles/alloy/defaults/main.yaml b/roles/alloy/defaults/main.yaml index 92bc960..09c99ee 100644 --- a/roles/alloy/defaults/main.yaml +++ b/roles/alloy/defaults/main.yaml @@ -4,26 +4,26 @@ alloy_config_default: | url = "https://metrics.hamburg.ccc.de/api/v1/write" basic_auth { username = "chaos" - password = "{{ metrics__chaos_password }}" + password = "chaos_password" } } } - prometheus.relabel "chaosknoten_common" { + prometheus.relabel "common" { forward_to = [prometheus.remote_write.default.receiver] rule { target_label = "org" - replacement = "ccchh" + replacement = "noorg" } rule { target_label = "site" - replacement = "wieske" + replacement = "nosite" } rule { source_labels = ["instance"] target_label = "instance" regex = "([^:]+)" - replacement = "${1}.hosts.hamburg.ccc.de" + replacement = "${1}.hosts.test" action = "replace" } } @@ -38,7 +38,7 @@ alloy_config_default: | prometheus.scrape "scrape_metrics" { targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.chaosknoten_common.receiver] + forward_to = [prometheus.relabel.common.receiver] } alloy_config_additional: "" From 5f4b9c6bede8becae16103a9bdf3386042d0b156 Mon Sep 17 00:00:00 2001 From: Renovate Date: Sun, 25 Jan 2026 20:45:50 +0000 Subject: [PATCH 8/8] Update docker.io/library/redis Docker tag to v8 --- resources/chaosknoten/tickets/docker_compose/compose.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/chaosknoten/tickets/docker_compose/compose.yaml.j2 b/resources/chaosknoten/tickets/docker_compose/compose.yaml.j2 index 938883b..6acbd2b 100644 --- a/resources/chaosknoten/tickets/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/tickets/docker_compose/compose.yaml.j2 @@ -13,7 +13,7 @@ services: restart: unless-stopped redis: - image: docker.io/library/redis:7.4.7 + image: docker.io/library/redis:8.4.0 ports: - "6379:6379" volumes: