From 2c1f95df9a22b0df3e8b037ddcf4b1d405979727 Mon Sep 17 00:00:00 2001 From: lilly Date: Thu, 23 Apr 2026 22:36:45 +0200 Subject: [PATCH 1/5] add documentation about how one commits DNS zones on ns-intern --- docs/create-new-web-service-vm.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/create-new-web-service-vm.md b/docs/create-new-web-service-vm.md index b91d9fa..b44a6e5 100644 --- a/docs/create-new-web-service-vm.md +++ b/docs/create-new-web-service-vm.md @@ -7,8 +7,9 @@ Let's assume that you want to add a new web service `example.hamburg.ccc.de` whi 1. Allocate a fresh [IPv6 in Netbox in the 2a00:14b0:42:102::/64 net](https://netbox.hamburg.ccc.de/ipam/prefixes/47/ip-addresses/). This will be the management address for the VM. 2. On `ns-intern`: 1. Add an entry `example.hosts.hamburg.ccc.de` as an AAAA pointing to the allocated IP. + To do this, edit the zonefile in `/etc/bind/master/hamburg.ccc.de.zone`. 2. Add an entry `example.hamburg.ccc.de` as a CNAME for `public-reverse-proxy` to the same zone. - 3. Commit and reload the zone. + 3. Commit and reload the zone by running `sudo make` while inside `/etc/bind/master`. 3. On Chaosknoten: 1. Create a new VM, for example by cloning the Debian template 9023. Give it the name `example`. From 0330c6b6caee303a794d708e011caae85b406456 Mon Sep 17 00:00:00 2001 From: lilly Date: Thu, 23 Apr 2026 22:36:45 +0200 Subject: [PATCH 2/5] reduce ansible grafana log verbosity by using loop_control labels --- roles/docker_compose/tasks/main.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/docker_compose/tasks/main.yaml b/roles/docker_compose/tasks/main.yaml index bf02828..3563102 100644 --- a/roles/docker_compose/tasks/main.yaml +++ b/roles/docker_compose/tasks/main.yaml @@ -45,6 +45,8 @@ ansible.builtin.set_fact: docker_compose__config_files_to_exist: "{{ docker_compose__config_files_to_exist + [ item.name ] }}" # noqa: jinja[spacing] loop: "{{ docker_compose__configuration_files }}" + loop_control: + label: "{{ item.name }}" - name: find configuration files to remove ansible.builtin.find: @@ -70,6 +72,8 @@ group: root become: true loop: "{{ docker_compose__configuration_files }}" + loop_control: + label: "{{ item.name }}" notify: docker compose restart - name: Sync files From 5242a14eed748f9ffd31e6787ca254ea531ca623 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Fri, 24 Apr 2026 17:42:51 +0200 Subject: [PATCH 3/5] Update to newest version --- resources/chaosknoten/pad/docker_compose/compose.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/chaosknoten/pad/docker_compose/compose.yaml.j2 b/resources/chaosknoten/pad/docker_compose/compose.yaml.j2 index db60eb9..1d35832 100644 --- a/resources/chaosknoten/pad/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/pad/docker_compose/compose.yaml.j2 @@ -13,7 +13,7 @@ services: restart: unless-stopped app: - image: quay.io/hedgedoc/hedgedoc:1.10.7 + image: quay.io/hedgedoc/hedgedoc:1.10.8 environment: - "CMD_DB_URL=postgres://hedgedoc:{{ secret__hedgedoc_db_password }}@database:5432/hedgedoc" - "CMD_DOMAIN=pad.hamburg.ccc.de" From b10d7d15929200a2be1b2e90b77125db710b1e1d Mon Sep 17 00:00:00 2001 From: lilly Date: Thu, 23 Apr 2026 22:36:45 +0200 Subject: [PATCH 4/5] deploy alerta on grafana.hosts.hamburg.ccc.de --- .../chaosknoten/host_vars/grafana.sops.yaml | 8 ++-- .../chaosknoten/host_vars/grafana.yaml | 5 +++ .../grafana/docker_compose/alertad.conf.j2 | 30 +++++++++++++ .../docker_compose/alertmanager.yaml.j2 | 8 ++++ .../grafana/docker_compose/compose.yaml.j2 | 21 +++++++++ .../grafana/nginx/alerta.hamburg.ccc.de.conf | 43 +++++++++++++++++++ .../nginx/acme_challenge.conf | 1 + .../public-reverse-proxy/nginx/nginx.conf | 1 + 8 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 resources/chaosknoten/grafana/docker_compose/alertad.conf.j2 create mode 100644 resources/chaosknoten/grafana/nginx/alerta.hamburg.ccc.de.conf diff --git a/inventories/chaosknoten/host_vars/grafana.sops.yaml b/inventories/chaosknoten/host_vars/grafana.sops.yaml index 5ae7949..720fb03 100644 --- a/inventories/chaosknoten/host_vars/grafana.sops.yaml +++ b/inventories/chaosknoten/host_vars/grafana.sops.yaml @@ -10,6 +10,8 @@ secret__metrics_fux: ENC[AES256_GCM,data:aV6zeZ/XsVlA3QepSfVd/cOr+tqFVhlAxRO9SHx secret__metrics_fux_basic_auth: ENC[AES256_GCM,data:YL+QLzZyyObzDcz+FcefViMrvdkVSwRhDsBx/AwoDX3RLHCDjg==,iv:GADdMa7FHMM1FnyPp8DUHElpXsJeqD+gN5Slw0R9bgs=,tag:KGCoEud2JLU5s1gurrbywg==,type:str] secret__ntfy_token: ENC[AES256_GCM,data:0tuPJVmxHcdDWOMIo0QQXgIEkJo+p9A5emH+kc+U5tw=,iv:NZcfiz3UFw2fMcMf+q1GRp4Fsxpxbptsx9n8wPR54z0=,tag:SJYFtXccCbPrXjECiKUOUA==,type:str] secret__alert_manager_email_password: ENC[AES256_GCM,data:AsBzn9KJEoMjcrUWiIhR7I/1jaaFEa+cl3gImOQVKrg=,iv:mtQnZqT0taap3+z/L/nMfUvQF3JlTKIdoljmzVr1R3c=,tag:mZrCB597p8LyB61I7ZvHNA==,type:str] +secret__alerta_secret_key: ENC[AES256_GCM,data:wMdKZ4j5iDYrFUsCK5KXGiznPZTtiHHyB02WlFGK976ypcA/1dh6b1cX1UbEI6i1us6bUu8oFQ97gG6NlVx/Xg==,iv:oZCgPBo2Us2agTxwU6nMqhYlEBHxG9D9GL7WE6XyEg0=,tag:4DVDVGwl92xHjatsek4NMQ==,type:str] +secret__alerta_oauth_client_secret: ENC[AES256_GCM,data:516IlJUnLNV4+5cpcJJOWSEhrjpI62mnzgOG3IFt6tw=,iv:pEqqKJyxJKM3z/CoeW6Sqo4fDyXE8fBxtjiFe/4teXM=,tag:brilNcC2W8sJ+uUiBLZaIA==,type:str] ansible_pull__age_private_key: ENC[AES256_GCM,data:OITNx5IxnrtyNQKrLoY7WKTLYeqjGXHAZ1bkwSrHzqUxXKMWOAGGHQkK5r+poOgGkiYSulNLA0NMgdBkkv7/YRQiu9Kt2di6pt4=,iv:CWGR9z1/KfCVoOOicqfWBu+XXGs6fTXMbk4De4izjcQ=,tag:Qum6h0k2YfWXPyunqFsQKw==,type:str] sops: age: @@ -22,8 +24,8 @@ sops: SWxKVmNXRDI2eHpIbThxVmYyS0lzTG8KMXMSLt9GPyqUVI40bRVGAllelsa8vujl FIoTVzjokV24wGXAV6RL8OltWWF3H9Lr44w8e/K0KnzmIWVhyihoIg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-14T23:38:52Z" - mac: ENC[AES256_GCM,data:bKdCkiO3OjBfwsNBOFVrFMti9fXtedYWhKuPFDgNQVYFblZji9UQP8XsX6kTSfzU1DxM+OR14RMgV5xssoB4f9q5tP5FgyhCcwWHCt1cg6lIQzIoOx3o68YmNf5X0C59Q/j5QGeGQiCNDV2eymDYcm/dV8eKEHD0EtmUGNFlOfQ=,iv:7jW4IFcebqx/0N0sqygV9SsvS+m+kQSlEGkixtIMy94=,tag:Eg3Z+blFGyG3lAXacwVPqQ==,type:str] + lastmodified: "2026-04-24T15:16:43Z" + mac: ENC[AES256_GCM,data:X9IdO7KOW2FJtP87SpaFFsJ2kkaePtZ7dP1vKm+uYRpsxROWjmHd4k8hQw3ebYh2wXVYQ/fo7JH1RBKuOqQIRVGsKdit4jzYfEBS1GGH/aRVF/5QL0jBXzdrKecZazytEK8iY4y7PzdHCZZ53ZrEv63VRnesJIFoEV6Ti/UVmD0=,iv:Rxn4zZ7g1MNZ84ISvxFEhNnepTn8X1bgO5tUfEJmd9A=,tag:QGBnhAWNAf4BLCeYwtIW0g==,type:str] pgp: - created_at: "2026-04-18T22:36:24Z" enc: |- @@ -216,4 +218,4 @@ sops: -----END PGP MESSAGE----- fp: 41FFAF3D519CF5C039FBD8414BCC213729AF0E49 unencrypted_suffix: _unencrypted - version: 3.10.2 + version: 3.12.1 diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index f259ad7..d60e7cb 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -24,12 +24,15 @@ docker_compose__configuration_files: content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux-critical.j2') }}" - name: ntfy-alertmanager-fux content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/ntfy-alertmanager-fux.j2') }}" + - name: alertad.conf + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/alertad.conf.j2') }}" certbot__acme_account_email_address: le-admin@hamburg.ccc.de certbot__certificate_domains: - "grafana.hamburg.ccc.de" - "loki.hamburg.ccc.de" - "metrics.hamburg.ccc.de" + - "alerta.hamburg.ccc.de" certbot__new_cert_commands: - "systemctl reload nginx.service" @@ -51,6 +54,8 @@ nginx__configurations: content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}" - name: metrics.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}" + - name: alerta.hamburg.ccc.de + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/alerta.hamburg.ccc.de.conf') }}" alloy_config_additional: | loki.write "default" { diff --git a/resources/chaosknoten/grafana/docker_compose/alertad.conf.j2 b/resources/chaosknoten/grafana/docker_compose/alertad.conf.j2 new file mode 100644 index 0000000..2b5b2af --- /dev/null +++ b/resources/chaosknoten/grafana/docker_compose/alertad.conf.j2 @@ -0,0 +1,30 @@ +# {{ ansible_managed }} +# https://docs.alerta.io/configuration.html#authentication-settings + +DEBUG = True +SECRET_KEY = "{{ secret__alerta_secret_key }}" +BASE_URL = "https://alerta.hamburg.ccc.de" +USE_PROXYFIX = True + +AUTH_REQUIRED = True +USER_DEFAULT_SCOPES = ['read', 'write'] +OAUTH2_CLIENT_ID = "alerta" +OAUTH2_CLIENT_SECRET = "{{ secret__alerta_oauth_client_secret }}" + +# AUTH_PROVIDER = "keycloak" +# KEYCLOAK_URL = "https://id.hamburg.ccc.de" +# KEYCLOAK_REALM = "ccchh" +# ALLOWED_KEYCLOAK_ROLES = "*" + +AUTH_PROVIDER = "openid" +OIDC_ISSUER_URL = "https://id.hamburg.ccc.de/realms/ccchh" +OIDC_VERIFY_TOKEN = True +ALLOWED_OIDC_ROLES = "*" + +# AUTH_REQUIRED = True +# ADMIN_USERS = ['admin@alerta.io', 'devops@example.com'] +# DEFAULT_ADMIN_ROLE = 'ops' +# ADMIN_ROLES = ['ops', 'devops', 'coolkids'] +# USER_DEFAULT_SCOPES = ['read', 'write:alerts'] +# CUSTOMER_VIEWS = True + diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 index 0689820..382d4ab 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 @@ -9,6 +9,9 @@ route: group_interval: 5m repeat_interval: 26h routes: + - receiver: alerta + matchers: {} + continue: true - receiver: "null" matchers: - sendAlert = "false" @@ -73,3 +76,8 @@ receivers: smarthost: "cow.hamburg.ccc.de:587" auth_username: "alert-manager@hamburg.ccc.de" auth_password: {{ secret__alert_manager_email_password }} + + - name: "alerta" + webhook_configs: + - url: "http://alerta.hamburg.ccc.de/webhooks/prometheus" + send_resolved: true diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 index 8c38500..5ad46a1 100644 --- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 @@ -105,9 +105,30 @@ services: - 8011:8011 restart: unless-stopped + alerta-mongodb: + image: docker.io/mongodb/mongodb-community-server:8.2-ubi9-slim + container_name: alerta-mongodb + volumes: + - alerta_mongodb_data:/data/db + ports: + - 27017:27017 + restart: unless-stopped + + alerta-web: + image: docker.io/alerta/alerta-web:9.1.0 + container_name: alerta-web + volumes: + - ./configs/alertad.conf:/app/alertad.conf:ro + environment: + - DATABASE_URL=mongodb://alerta-mongodb:27017/alerta + ports: + - 8012:8080 + restart: unless-stopped + volumes: graf_data: {} prom_data: {} alertmanager_data: {} loki_data: {} mimir_data: {} + alerta_mongodb_data: {} diff --git a/resources/chaosknoten/grafana/nginx/alerta.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/alerta.hamburg.ccc.de.conf new file mode 100644 index 0000000..7597c77 --- /dev/null +++ b/resources/chaosknoten/grafana/nginx/alerta.hamburg.ccc.de.conf @@ -0,0 +1,43 @@ +# partly generated 2022-01-08, Mozilla Guideline v5.6, nginx 1.17.7, OpenSSL 1.1.1k, intermediate configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.17.7&config=intermediate&openssl=1.1.1k&guideline=5.6 +server { + # Listen on a custom port for the proxy protocol. + listen [::]:8443 ssl proxy_protocol; + http2 on; + # Make use of the ngx_http_realip_module to set the $remote_addr and + # $remote_port to the client address and client port, when using proxy + # protocol. + # First set our proxy protocol proxy as trusted. + set_real_ip_from 2a00:14b0:4200:3000:125::1; + # Then tell the realip_module to get the addreses from the proxy protocol + # header. + real_ip_header proxy_protocol; + + server_name alerta.hamburg.ccc.de; + + ssl_certificate /etc/letsencrypt/live/alerta.hamburg.ccc.de/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/alerta.hamburg.ccc.de/privkey.pem; + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/letsencrypt/live/alerta.hamburg.ccc.de/chain.pem; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Port 443; + # This is https in any case. + proxy_set_header X-Forwarded-Proto https; + # Hide the X-Forwarded header. + proxy_hide_header X-Forwarded; + # Assume we are the only Reverse Proxy (well using Proxy Protocol, but that + # is transparent). + # Also provide "_hidden" for by, since it's not relevant. + proxy_set_header Forwarded "for=$remote_addr;proto=https;host=$host;by=_hidden"; + + location / { + proxy_pass http://127.0.0.1:8012/; + } +} diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf index 2bdf9cc..6cd53d3 100644 --- a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf +++ b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf @@ -13,6 +13,7 @@ map $host $upstream_acme_challenge_host { element.hamburg.ccc.de 172.31.17.151:31820; git.hamburg.ccc.de 172.31.17.154:31820; grafana.hamburg.ccc.de grafana.hosts.hamburg.ccc.de:31820; + alerta.hamburg.ccc.de grafana.hosts.hamburg.ccc.de:31820; hackertours.hamburg.ccc.de 172.31.17.151:31820; staging.hackertours.hamburg.ccc.de 172.31.17.151:31820; hamburg.ccc.de 172.31.17.151:31820; diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf index a7abf03..fb0d37a 100644 --- a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf +++ b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf @@ -28,6 +28,7 @@ stream { invite.hamburg.ccc.de keycloak.hosts.hamburg.ccc.de:8443; keycloak-admin.hamburg.ccc.de keycloak.hosts.hamburg.ccc.de:8443; grafana.hamburg.ccc.de grafana.hosts.hamburg.ccc.de:8443; + alerta.hamburg.ccc.de grafana.hosts.hamburg.ccc.de:8443; wiki.ccchh.net wiki.hosts.hamburg.ccc.de:8443; wiki.hamburg.ccc.de wiki.hosts.hamburg.ccc.de:8443; onlyoffice.hamburg.ccc.de onlyoffice.hosts.hamburg.ccc.de:8443; From 822c261a9ce3d5d669410848ef28786f4102531e Mon Sep 17 00:00:00 2001 From: Renovate Date: Sat, 25 Apr 2026 00:16:50 +0000 Subject: [PATCH 5/5] Update docker.io/grafana/grafana Docker tag to v13 --- resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 index 5ad46a1..9ac4b14 100644 --- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 @@ -32,7 +32,7 @@ services: - alertmanager_data:/alertmanager grafana: - image: docker.io/grafana/grafana:12.4.2 + image: docker.io/grafana/grafana:13.0.1 container_name: grafana ports: - 3000:3000