From 41d943a532186868f556a6ad026b38a5bb455731 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 4 Jan 2026 19:32:42 +0100 Subject: [PATCH 01/11] wip: alloy --- roles/alloy/tasks/main.yaml | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 roles/alloy/tasks/main.yaml diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml new file mode 100644 index 0000000..2bc308a --- /dev/null +++ b/roles/alloy/tasks/main.yaml @@ -0,0 +1,48 @@ +# https://github.com/grafana/grafana-ansible-collection/blob/main/roles/alloy/tasks/deploy.yml#L124 +- name: ensure alloy user exists + ansible.builtin.user: + name: alloy + system: true + append: true + create_home: false + state: present + +- name: ensure the `/etc/alloy/` config directory exists + ansible.builtin.file: + path: /etc/alloy + state: directory + mode: "0770" + owner: root + group: alloy + become: true + +- name: synchronize the additional configuration files directory, if present + when: alloy__additional_configs_dir is defined and alloy__additional_configs_dir != "" + block: + - name: ensure rsync is installed + ansible.builtin.apt: + name: rsync + become: true + + - name: synchronize the additional configuration files directory, if present + ansible.posix.synchronize: + src: "{{ alloy__additional_configs_dir }}" + dest: /etc/alloy/additional + delete: true + recursive: true + use_ssh_args: true + rsync_opts: + - "--chown=root:alloy" + become: true + +- name: delete the additional configuration files directory, if not present + when: alloy__additional_configs_dir is not defined or alloy__additional_configs_dir == "" + ansible.builtin.file: + path: /etc/alloy/additional + state: absent + become: true + +- name: Setup Alloy + ansible.builtin.import_role: + name: grafana.grafana.alloy + become: true From b492472179f8f56a6f46dd0aad0fdc036e99cdde Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 12:12:30 +0100 Subject: [PATCH 02/11] Explain how to add age key for ansible pull --- ...ng_up_secrets_using_sops_for_a_new_host.md | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/setting_up_secrets_using_sops_for_a_new_host.md b/docs/setting_up_secrets_using_sops_for_a_new_host.md index c88315f..df7b024 100644 --- a/docs/setting_up_secrets_using_sops_for_a_new_host.md +++ b/docs/setting_up_secrets_using_sops_for_a_new_host.md @@ -2,19 +2,29 @@ Because we're using the `community.sops.sops` vars plugin, the SOPS-encrypted secrets get stored in the inventory. -1. Add a new creation rule for the hosts `host_vars` file in the sops config at `.sops.yaml`. - It should probably hold all admin keys. +1. Create a new age key for Ansible pull on the host. + ``` + age-keygen + ``` + Then add an entry to `keys.hosts.chaosknoten.age` +2. Add a new creation rule for the hosts `host_vars` file in the sops config at `.sops.yaml`. + It should probably hold all admin keys plus the host entry. You can use existing creation rules as a reference. -2. Create a SOPS secrets file in the `host_vars` subdirectory of the relevant inventory. +3. Create a SOPS secrets file in the `host_vars` subdirectory of the relevant inventory. The name of the file should be in the format `[HOSTNAME].sops.yaml` to get picked up by the vars plugin and to match the previously created creation rule. This can be accomplished with a command similar to this: ``` sops inventories/[chaosknoten|z9]/host_vars/[HOSTNAME].secrets.yaml ``` -3. With the editor now open, add the secrets you want to store. +4. With the editor now open, add the secrets you want to store. Because we're using the `community.sops.sops` vars plugin, the stored secrets will be exposed as Ansible variables. Also note that SOPS only encrypts the values, not the keys. When now creating entries, try to adhere to the following variable naming convention: + - Make sure to put the prive age key in here under `ansible_pull__age_private_key`. - Prefix variable names with `secret__`, if they are intended to be used in a template file or similar. (e.g. `secret__netbox_secret_key: secret_value`) - Otherwise, if the variable is directly consumed by a role or similar, directly set the variable. (e.g. `netbox__db_password: secret_value`) -4. Now that the secrets are stored, they are exposed as variables and can simply be used like any other variable. +5. Now that the secrets are stored, they are exposed as variables and can simply be used like any other variable. + +## GPG Keys + +In order to edit encrypted files, you need all the GPG public keys imported into your GPG Keychain. You should be able to find the necessary public keys in https://git.hamburg.ccc.de/CCCHH/password-store. From 664b9115b8d4e01ae774bf33e041855ed10de585 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 13:01:52 +0100 Subject: [PATCH 03/11] Fix warning --- roles/docker/tasks/main/01_repo_setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/tasks/main/01_repo_setup.yaml b/roles/docker/tasks/main/01_repo_setup.yaml index aa77521..63bdb91 100644 --- a/roles/docker/tasks/main/01_repo_setup.yaml +++ b/roles/docker/tasks/main/01_repo_setup.yaml @@ -9,7 +9,7 @@ - name: Ensure Docker APT repository is added ansible.builtin.apt_repository: - repo: "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable" + repo: "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/docker.asc] https://download.docker.com/linux/debian {{ ansible_facts['distribution_release'] }} stable" filename: docker state: present become: true From f887de25c521e52375ae828cab59dd6a17f3b258 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 13:26:20 +0100 Subject: [PATCH 04/11] make building and pulling configurable --- roles/docker_compose/defaults/main.yaml | 2 ++ roles/docker_compose/tasks/main.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/docker_compose/defaults/main.yaml b/roles/docker_compose/defaults/main.yaml index 76831d6..621ee7b 100644 --- a/roles/docker_compose/defaults/main.yaml +++ b/roles/docker_compose/defaults/main.yaml @@ -1 +1,3 @@ +docker_compose__build: always docker_compose__configuration_files: [ ] +docker_compose__pull: always diff --git a/roles/docker_compose/tasks/main.yaml b/roles/docker_compose/tasks/main.yaml index bea3f4f..a706ab2 100644 --- a/roles/docker_compose/tasks/main.yaml +++ b/roles/docker_compose/tasks/main.yaml @@ -79,8 +79,8 @@ community.docker.docker_compose_v2: project_src: /ansible_docker_compose state: present - build: always - pull: always + build: "{{ docker_compose__build }}" + pull: "{{ docker_compose__pull }}" remove_orphans: true become: true From 0c782caee7a9e68e9b5247f681cf7222b1703ca7 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 14:03:34 +0100 Subject: [PATCH 05/11] Explain what all needs to be added for a new host --- docs/create-new-web-service-vm.md | 79 +++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/create-new-web-service-vm.md diff --git a/docs/create-new-web-service-vm.md b/docs/create-new-web-service-vm.md new file mode 100644 index 0000000..76eca21 --- /dev/null +++ b/docs/create-new-web-service-vm.md @@ -0,0 +1,79 @@ +# How to create all necessary entries for new (web service) VM + +Let's assume that you want to add a new web service `example.hamburg.ccc.de` which is going to be hosted on the VM `example` on chaosknoten. These are the steps that you need to take to create the VM and add it to the Ansible repo. + +## IP, DNS, VM + +1. Allocate a fresh [IPv6 in Netbox in the 2a00:14b0:42:102::/64 net](https://netbox.hamburg.ccc.de/ipam/prefixes/47/ip-addresses/). This will be the management address for the VM. +2. Add an entry `example.hosts.hamburg.ccc.de` with that AAAA to the name server (`ns-intern`). +3. Add an entry `example.hamburg.ccc.de` as a CNAME for `public-reverse-proxy` to the same zone. +4. Commit and reload the zone. +5. Create a new VM on chaosknoten, for example by cloning the Debian template 9023. Give it the name `example`. +6. Edit the ethernet interface to be connected to `vmbr0`, VLAN tag `2`. +7. Configure the IPv6 address in the Cloud-Init section. Leave IPv4 set to DHCP. +8. Make sure the VM is started at boot (options). +9. Adjust any other VM parameters as needed. +10. Boot the VM. +11. Add the [VM to Netbox](https://netbox.hamburg.ccc.de/virtualization/virtual-machines/). Make sure to enter the VM ID. +12. Add an Ethernet interface to the VM; we typically use `eth0` as a name. +13. Add IP for that interface, then choose "Assign IP" and search for the IP you've created. Make it the primary IP of that interface. + +## Ansible Basics + +As the first step, we need to make the host known to Ansible. + +1. In `.sops.yaml`, add an entry for the host. Follow the other entries there. + 1. `keys.hosts.chaosknoten.age` needs an age public key (the private key is needed in the host-specific YAML) + 2. `creation_rules` needs an entry for the host, referencing the age key. +2. In `inventories/chaosknoten/hosts.yaml`: + 1. Configure basic connection info: + ``` + example: + ansible_host: example.hosts.hamburg.ccc.de + ansible_user: chaos + ansible_ssh_common_args: -J ssh://chaos@router.hamburg.ccc.de + ``` + You typically will want to use router as a jump host so that you can run Ansible on an IPv4 only connection. + 2. Add the host to the desired roles. As a minimum, you'll want `base_config_hosts` and `infrastructure_authorized_keys_hosts`. For a typical web service based on Docker Compose, you'll want `docker_compose_hosts`, `nginx_hosts`, and `certbot_hosts`. + 3. In the directorry `inventories/chaosknoten/host_var/`: + 1. A file `inventories/chaosknoten/host_var/example.yaml` with the host/service specific configuration. + 2. A file `inventories/chaosknoten/host_var/example.sops.yaml` with the encrypted secrets for the host/service. Run `sops inventories/chaosknoten/host_var/example.yaml` to edit/create that file. Entries there should generally be prefixed with `secret__` to make it easier to see where that variable is coming from in templates etc. + +## Service-specific config + +From here, we go into the details of the web service that you want to configure. For a typical web service with Docker Compose, you will likely want to configure the following. + +Make `inventories/chaosknoten/host_var/example.yaml` look like this: +```yaml +certbot__version_spec: "" +certbot__acme_account_email_address: le-admin@hamburg.ccc.de +certbot__certificate_domains: + - "example.hamburg.ccc.de" +certbot__new_cert_commands: + - "systemctl reload nginx.service" + +docker_compose__compose_file_content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/example/docker_compose/compose.yaml.j2') }}" + +nginx__version_spec: "" +nginx__configurations: + - name: example.hamburg.ccc.de + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/spaceapiccc/nginx/example.hamburg.ccc.de.conf') }}" +``` + +This will create `compose.yaml` from the template `resources/chaosknoten/example/docker_compose/compose.yaml.j2'`, and the nginx config from `resources/chaosknoten/spaceapiccc/nginx/example.hamburg.ccc.de.conf`. Of course, depending on your service, you might need additional entries. See the other hosts and the roles for more info. + +## First Ansible run + +Before you can run Ansible successfully, you will want to make sure you can connect to the VM, and that the host key has been added to your known hosts: +* `ssh chaos@example.hosts.hamburg.ccc.de` +* `ssh -J chaos@router.hamburg.ccc.de chaos@example.hosts.hamburg.ccc.de` + +Then run Ansible for `public-reverse-proxy` to add the necessary entries: `ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts.yaml --limit public-reverse-proxy`. + +Finally run Ansible for the new host: `ansible-playbook playbooks/deploy.yaml --inventory inventories/chaosknoten/hosts.yaml --limit example` + +# Commit your changes + +Do not forget to commit your changes, whether it's a new host or you are making changes to an existing host. + +And always `git pull` before you run Ansible so avoid reverting anything! \ No newline at end of file From cee1fe970aa629d5b2f9bf58de2a8ed18b1726c9 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 14:03:54 +0100 Subject: [PATCH 06/11] Add spaceapiccc as a replacement for erfafoo --- .sops.yaml | 7 + .../host_vars/spaceapiccc.sops.yaml | 215 ++++++++++++++++++ .../chaosknoten/host_vars/spaceapiccc.yaml | 16 ++ inventories/chaosknoten/hosts.yaml | 9 + .../nginx/acme_challenge.conf | 2 + .../public-reverse-proxy/nginx/nginx.conf | 2 + .../docker_compose/compose.yaml.j2 | 39 ++++ .../nginx/spaceapiccc.hamburg.ccc.de.conf | 42 ++++ 8 files changed, 332 insertions(+) create mode 100644 inventories/chaosknoten/host_vars/spaceapiccc.sops.yaml create mode 100644 inventories/chaosknoten/host_vars/spaceapiccc.yaml create mode 100644 resources/chaosknoten/spaceapiccc/docker_compose/compose.yaml.j2 create mode 100644 resources/chaosknoten/spaceapiccc/nginx/spaceapiccc.hamburg.ccc.de.conf diff --git a/.sops.yaml b/.sops.yaml index 3b728e2..bb83e39 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -33,6 +33,7 @@ keys: - &host_public_reverse_proxy_ansible_pull_age_key age1p7pxgq5kwcpdkhkh3qq4pvnltrdk4gwf60hdhv8ka0mdxmgnjepqyleyen - &host_zammad_ansible_pull_age_key age1sv7uhpnk9d3u3je9zzvlux0kd83f627aclpamnz2h3ksg599838qjgrvqs - &host_ntfy_ansible_pull_age_key age1dkecypmfuj0tcm2cz8vnvq5drpu2ddhgnfkzxvscs7m4e79gpseqyhr9pg + - &host_spaceapiccc_ansible_pull_age_key age1mdtnk78aeqnwqadjqje5pfha04wu92d3ecchyqajjmy434kwq98qksq2wa external: age: &host_external_age_keys - &host_status_ansible_pull_age_key age1yl9ts8k6ceymaxjs72r5puetes5mtuzxuger7qgme9qkagfrm9hqzxx9qr @@ -160,6 +161,12 @@ creation_rules: *admin_gpg_keys age: - *host_public_reverse_proxy_ansible_pull_age_key + - path_regex: inventories/chaosknoten/host_vars/spaceapiccc.* + key_groups: + - pgp: + *admin_gpg_keys + age: + - *host_spaceapiccc_ansible_pull_age_key # external hosts - path_regex: inventories/external/host_vars/status.* key_groups: diff --git a/inventories/chaosknoten/host_vars/spaceapiccc.sops.yaml b/inventories/chaosknoten/host_vars/spaceapiccc.sops.yaml new file mode 100644 index 0000000..4f06e92 --- /dev/null +++ b/inventories/chaosknoten/host_vars/spaceapiccc.sops.yaml @@ -0,0 +1,215 @@ +ansible_pull__age_private_key: ENC[AES256_GCM,data:ZQJCVOcc2UTH/3tZRZEZAig2A7Vc/zBBz5IY+gKYMYpIKhLZN9S/OGrRdCc8VbXkN7pmZhzDL531PapI54cmFeCKr2yFJMlfXdE=,iv:1ilb+njcqgYVdownNiMNcAcG/TNpyRnLtAjEUGsCsl0=,tag:Od7kvNn8ZBl1LUnMyFwxpA==,type:str] +secret__spaceapiccc__shared_secret: ENC[AES256_GCM,data:0foffl4HF1SeL9rE3g==,iv:GzRTZAmr7zSBs1W+Vhyv6sMGhPnSy/SUZOSO39lzWHk=,tag:8IAS6Lt9vfpsJQwQfcunXg==,type:str] +secret__spaceapiccc__doku_ccc_de__username: ENC[AES256_GCM,data:fbrZROQz8Fzg/vI=,iv:LaR5UmkS3IhtroJp3C3xNF4ja7IhIiPRzGBHAfQbQGw=,tag:/VCNMKkw5qRbnRNHDnPj/w==,type:str] +secret__spaceapiccc__doku_ccc_de__password: ENC[AES256_GCM,data:mwkjOjRT7gOv,iv:wBzSeLzSWWe0j3LJesN/wnZ0tmUmXMVkRIBnp00qRhg=,tag:JSsbq1+qs2yA9BM2LouG1w==,type:str] +sops: + age: + - recipient: age1mdtnk78aeqnwqadjqje5pfha04wu92d3ecchyqajjmy434kwq98qksq2wa + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCY1Z0Slg4UmpQQUhGKzJX + S0ROZ2owdmNVRUFzbDhjWEJpNkxGQnF1RFFVClgrZDlZRDNCbllWeElEWFN4Uy95 + YXNzUGptcU9adjdJQVphSS9NQ1NaVTQKLS0tIEtQUlIyTURXK2lDbWtmMXU2OWtx + TnNtQjVpMUIzZjgzQnZicHV6OXE3ZlUKtChQKJlUmTV42FEpO2S1sTAI2+K/mro+ + C3cvwiqydpOlbH6tulcP6HSeDVExAAMeDZMfjebg/5cfq7Yfh6xa5Q== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-01-25T11:18:43Z" + mac: ENC[AES256_GCM,data:4s5GiYhU/+kieEGUY9bS5W0MAQ/AUS3TbvLezSypH8Div5HRoM7YfMeqgLq4jC+TjUL9d+ZfusjAmsOEG9PjHbIH051gg8U5TvB38wzmw3RpJxnpDtmiFrRh9QbXl+Fz8V/Oigf6hhXbgu01zZpZY9jy6YLNtUZc6AoqAQh27us=,iv:YUS/vGXcbgQPM1CKcK8YjOH5+KPlzBXcOtx3jmUblqA=,tag:jYzqaMfHv4Tyv2NelSSVvQ==,type:str] + pgp: + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAxK/JaB2/SdtARAAoP0ZuYWL+Z9vrnMN+ISg6/yx8Z3Oq2GufmYMowk/nQ7A + wk+xQQcywn7zLCweaTNtNb8CXtAcInnLhXZNRjviOecyAexZdFxcX+SIiT9x32aZ + xk2M3Bgnrtf9GQMV9q/mr7fgn+iaILyRjWTQMjUYFGuA5Hu7PNICxZZtA1y6p3G8 + iBDROt1vZS2M6WorA5n3FGSwCRFUCqWnRsBR+AkR0vjb/0xEmS4YpDZCdsqWVITq + fBxDZntznqQpmlTH9AxJV48QlfYMLAYFV7seHxp5VSjgDxaPJD4QIiNZMOylRa/y + 9hx1S5VN8KIfT9eW5piOeyNikE3Wv7hdwd4zOQ/ObESADh/QWFN582Smk+fxf76Q + /KlP7BM8JW7afjkvTHXg7cvc1qo9+GilWcWX9pK04v9bZtXTbO6H+uOhydlSmtUe + FGoHgQsMi52S4vHTFF1A8o76pvpQAIYNC2Zif2zZYq9ERvbLeAcgoIoo7bQihttc + lY8ZOqxQj9KbkFNbyLTlyekebNhfa512XjJij14YkYUVU2Y65kxtimZ3WpwKvLO2 + JcDWHOJduhUC+21TGTq6QFo1LNhpowyC447eybi8T0/WxMCBms/fhW+m4Mkt4bRi + ByjgQe8makgLqw2/EUlFl1qyF4zU0zjn+97pISvg0YBfQYhPIb5k8AWWkUF4mHHU + aAEJAhDMVlvoC4bopmVlgoCrCejX5wb+ULW9hle6S69440PVK4uN94Ral+NSH99o + CU4gmqngD9N6sw8SBp8lFFUzjhoqfcNwJ9cv8T9PIPgHLriPnRqwPsy4dHSYSsv1 + wWY4KUeOqk6Y + =Wm7O + -----END PGP MESSAGE----- + fp: EF643F59E008414882232C78FFA8331EEB7D6B70 + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA6EyPtWBEI+2ARAAh2b51c4cFL0wOPTuQtxjthkEZGVv0sQC19PiDOWAy/zi + 457Ix+QPA31Wmun4uGQF8E+vJC9StDXvOuEku2639wK7Gx8UVHSJM+QhFt+f9tiI + df5mVRPz4R1tVMU6P/f2rTOqqQyugR2pi3wCcwntnZplEuL/Gxw2SI4gGAq9B1Kb + FVVdMkJOxhx33QWFhIEOqLLfMU+gdvGPtRaDPkMA5KJD5FDO0xYzgd+5j6wKLsdb + rY7MVvaP3HWbmsMOpJD+8zo3ONBeaG3OwdhhF8KgbHxGP/49r25WwI5YWqXI44K9 + xIQvtBJFTLaisO3q5rTOZgqKEvWAAX3e82cY3tCUG4aDyKEeF8dOqQ9GbI+KWaKh + kqTFDz3gh9sWI3Ex2/JHxq4xGJE433x4ArxHgSmXxfKWfc9zhiDuhtE1GBfEWP8t + a+07FWvsG8TCbS8pzFu40z/6we2O/VGXnZBa+vlc/9YPyLBN+zmAH3+jfhgYzV22 + oF0HPQTzLdd6FoUx771ETTOqDgwg2H8Lqv+cC5MjPgxUPyScP4G7t0r9TMSydxFv + 85Yo7ZWiBjo5TgdiU7agCCLKYct1C1R+9M20uRyrttDBhrVSjDlsIKmuStIdI7jk + k/PPLjxUKf5osTw8KKsSLvHTxt0G+rRzt38HgOCsOPBSoE6zlMTn79rgy+Ipm7fS + XgHPPTT78/y2Xvx3QGx9C2X9YqPDGhs12uzQ7HdcRlUu3Ay9akrSiV99CKCFb6ZZ + lDzOZrWvuWHcOLLqykhK3x8uhieMmwsM5WCNopr1j7i74b8UlVCmItXFXCaTRqg= + =ytkN + -----END PGP MESSAGE----- + fp: F155144FC925A1BEA1F8A2C59A2A4CD59BFDC5EC + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAz5uSgHG2iMJARAA7Wenq30iYLUH7qTgwPJyIyPz0blUzqEpEeDyVjfLVxee + VzXUfrxL8ybD+1JNISQNogDRP+gi4Sa/kTwAwEudqg9nv8DTff2l+Ge7YRifTgoO + tK1yjPKpl/iH33s2tIRRPI9DJ38NKtIN7pFrZ9Icyinyx8O+Tx0U/rVOs+4I4i0K + eIhsjG2tD6z5AvDkTqJ70S16LWdlr+hrHfEFmZ9NDbesoVj6YlDjx8yXr6UAdBAd + nx4aVjy2vygBJFZHN3iqitD6pnBvFC6QM1SZTRfe1l0lb1NXiVbT42ir7hsQ1/Di + MKRw/GuD+5jwHWLAzFbmMeirLY1hw418AzMPmCUqg3xJxmm53v4abD/j6cnHaM8h + vkSEsO9iA9exDjM9RPqS5GXCGx3E2MdBzgBMZIdvRmEV8G7FTqBZAJZsElAA/wTl + WhCEB3iDqdTSuDUnEj2FHIrUGNG4IDKOm9mIexqkpdvF6ByXYHeOAVbeb0ByJmgO + 3QIYGsOYiWW2Uq1OCT2F+sP9ogn2GxInfMgPK7shFcUiXUbUKSnfBh4b5DbKPcJJ + wFtuJA4NbWgXbDPn0k2Lwbv33tMVuwQBRbCjseXD5JYUA+wEbNg341oNEl7gIBCp + oNyNJ0y2rkp8rxvf5mYLjk6VsMs0VO4vgRItg8oi78cZMmSrk2zdCda9yZA+JeHS + XgGnSemRkXBLcDcZMa1M178H/YTxispkRvsGyscxn7sjBRUgrFHnWM9j9P0GHtHE + RzBflQuBiG60jDb14l0SBEGDAm3Dp1bT5Up8attUJ0+03ta6E4G6iAR+fMXiBJA= + =LEoh + -----END PGP MESSAGE----- + fp: 18DFCE01456DAB52EA38A6584EDC64F35FA1D6A5 + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMAw5vwmoEJHQ1AQ/7Be0MaQ6HSb4N2DW+z2ALOuKSljRhSHLiLXt6bmhot+2Z + RRMsfsPGHWwDFzy5WWL6117ViPsxdFy88ZC8IhfT2ysf9d7IsNqBAj/W/a1kUXBg + b3PLPGXT3yHRitmRA0PxBWjmKBHuiKJgpj2AvKPBqmpJOpyWU8Yr0yu+fdPgHHmO + 9gMPwmoeDKCuVUQMtg78cxx3b9v3WzBXbx+VuhPepVPPUr7/iTWYnLWy8+s55hOV + A6qQS8f6JH9rhS7dqoSCMQ3wrqkSVzXhluhjfUXa/FW/EVp0g1r+lLMXHARA1Gtp + EGQS2SfwDB95xl6uLfqKblezzxt52yPvGp+hisAhgkCyoLonhL27fMTmtZ0+q9RX + FJoT2pPNTSP/zoLxfEJzsa9MgTCDKQL55215hTGHS2I/2ZeDtfINyc+/4LE/AhSc + 4OOdPSbgG7bIPkCepphBAccjbCVmPOQqaEOk5C9TfLbZREEBv0mQA7pzWVIsa6Gc + xep0qJGMSmRT5rmqs9pFFISAx57H7w91cRaEtwtGkg9/90+wTW2kIvnHMLXV/T6z + wxVG4RHn7eXlDdh9oz0ncpA1uh2A4fvEJN5dAbQHawiAUaOokm8cmv42LQ1zTF0x + 4EcZPQ1VAFzKsZE7/3TnCWoLPOUSNSOG+uJm2Gaps8P1DzIfgUAcSybaB+3cbGjS + XgEVALzLzyRrFB48McT/fU4l0dMiQ49OdFmWm5oWgOWDCCrHBomxPmWRQ5cUzVSV + wvgo/MrfGVOLrwinfeu/izoy9U0LxFcJtqiVLyxtUTARDlDcjv6OYWoRzvb0DzA= + =KudR + -----END PGP MESSAGE----- + fp: 87AB00D45D37C9E9167B5A5A333448678B60E505 + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DerEtaFuTeewSAQdAlSeQVBNgJ8WxD85XYmcCHmlNXIyIkAJPEu0coBpNpVQw + mGZKY6j0WkQSmHdCVAeh8/z6LOEgXMphP2jn0ZpZHiMu3FGNJJtWFloRKxOvOxr5 + 0l4BXq0oVpIYhcxeVxS1prF1F2EJf/OuRX8Zz9ngZuL7UlMoToBYHksPMaBfLlKB + iFcXPURafpmhvWpRaqD9CRqM3XRagm1nYPS6Zg8Yae9cfSmU7UnYMtJZwdMmJ+x4 + =gfNC + -----END PGP MESSAGE----- + fp: 057870A2C72CD82566A3EC983695F4FCBCAE4912 + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQILAxjNhCKPP69fAQ/4mdGngFM8WhiX5P5RFo679yRMp5iHtiPqD0V1dE1byyje + d7WzceQwOYfYq/UEEw2ruiqIPhUjHlzB/GQ6wqFbj0+1tm7+/X2B42tO7vkO9gQf + 2mvG0gCGB1iykMnfARQ6EH1s90oAHCBcPFamjBZ3oawS0sI34aSInQGqLl7Ss+O+ + UgoOc2fbhYmRriZW7Elyx+8DuQg4RZ6/oPs18mtwQdLfKB8dwrt1TQrJvBx7iPh4 + RQWrRf3id+C8EeysmWPtMotukh1FgvBtBFEXIL66wntJTDC65AlNU1c2xkgUTATI + rA6ucSoyROTGDOTAWhBdwA+yV9Tf2zw5hzu8G2vT1nFLU+DFQiuQWj6TNn1s5xzc + 63bQ9bFzY/0pKKB2T1TLdeU6xoSt9QoJukagFS86Tgh3NcoMi69dFSSlchldgeX2 + wiJwpUjl8DgeJFEXcQES1vbn+MNJHYZHSSAcZecQX5rauSj6EmTFTXxYg7Vp98D9 + S4lVnXl6P7OByxqRJyQUzBmSD21KYeVXs6O4hY4cAxKx+pXYXqlGMmSpQi4SqJKF + xyD0f7Iz1FjB1u3dpcJmf5/71wLkZWc9smKfJICLaFZzYKfbfrF32xbAPGRuTq50 + Fv5d3R1YJKA9afQUI3HT0PpCEOnsI44WPqgnoOPHyT032gruZt9geL7yM1sRj9Je + AfCwLc18oeiRWhnZLw/K1YMTnDACVhMMRufyoE7MEEixsV3xhuG54+5FIufERSO3 + aW2vmDt65mLjqGVcepqbEz/Ip4hfGeMOnPfNbNil79Hc6TV1SzTcPnem40QPAA== + =7Qbv + -----END PGP MESSAGE----- + fp: F38C9D4228FC6F674E322D9C3326D914EB9B8F55 + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA1Hthzn+T1OoAQ//QizKfdVMoIVzretcwqPNQPhXnKYbHNI/AHhpsK2AeOFw + N2pP+8itgzpoJ+l3qYc1s7HnUYqN69cVXNOkuB9+EKUmEoubj9oLJEJQdfr1apux + wrqgvIfeXuQZWp4E4aI/02ndyWzzedfVV3/qf+JC0ZColccmKFReSsMedz7dOmWK + BM2bieM1PajS65leCAO2VVFTrwayKiHWpURMUY8HvrMk8N6GQkXqe1XDdxXNJqFr + irXgWtBaKbl/KJgrxnT9HwlH9YkCebsyCi2sZKmJEqyIi78SOrhmWzeoTs5Mgg/M + EqZLWrGhOOD2/ineOxiDhFPOEDVjgoprghxei2Ef0i9pYITJmGMuB76KayMW3nbY + mEJgASKsWFN10zTiZK5DjxJoDEq4fyqtzFhYhRenwcvZqiklr2JudSzBWkKfx4Y/ + TOoLwwn93TQDLoIIEsOlLaWMBxm3LsAe4MAr2k9/gAkGGMzeOiTRISHJeFtaNRPe + xPv2hJBKqAJJkWu5nlcn5FEtAqdG8hPRPqEZWDyWRmQDlk0Rx286UFIS+BKSfwvo + Ak52YxruVlkwxn4lRJ8yCrIneZocLFlBgTNoqbr0uYSHkg6XHwzniN+qGRHxjrm8 + hDYcnVeAnLCDGEwPpMcx7KYVtLeEcr2Tm5btAlHugpQ1pNrUuZ3Lf47AdneMSY7S + XgE32gbAcEaZVQRl1fnehRIwqqNIuFDxjhFpDYpvX1Rep2NEUtEaxd50aqMh3PKm + XE6ZBkKbhSylRnOs8dgVZK3nqEe1xDsdcx5hFAoyyhs1QhWVT/MHUtfuB2PBcjo= + =T4dN + -----END PGP MESSAGE----- + fp: 5DA93D5C9D7320E1BD3522C79C78172B3551C9FD + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA46L6MuPqfJqAQ//SZac3kFkPkHZ4CveGECwnJLKA/UJO/XoV44mjiQDtY4Q + tFJ+YauR7GHK3CYMvpx8uWejiW6PzMkuqVuKwk5QMBsRA7q/6SmQeLUNNIPx8AAm + s1Lo1Cdjv5Ku8AnR7gAJ9w3O+qM635xo7zgtvEv5qJuPrwbqy8kstvS2fnxg9Zb3 + Dl4J+Wp1kRs8hHsFIkECKPqKNB0LfP57s63Vwd5tI2TltDMlMkvKvjgsQSPhUqQl + z0AIPT+zON37P4EW5buJ5NKvojYZ/QyzoqJ+Zb+2jn3uyMRDo4lqaT+uiVDcmB6w + jg2yBGKgU5XGAU5NyCSldBGW3yQfNHAEjTPHWIvcplfUOUQ2mKIV31c3ci8cBWa5 + zfA4K2UOFPSHSraohaT770Ani/qvm5XH9HvAA2HOI50LuIh4t8cWGocbW1f5PfvZ + gMIuA27UfWWD10tz+J3qvz2RGcfBPV+3BS8BJUh2SRC80ba8nDM/VSuQUkxQA1go + AHogKohH7v5vIPEN6ggRxZ3yCroQ3zfdABekrP8sfKXU652/vhw5MFPtqp8ow5hU + uJ3S3lCoKQCKE8tc+288WuJXIGaYG4LKhaVlFWFqQDib+0jfm8RfwqqxV5vis7np + mbPMIyl/MTAeevsQC2yqbHeZ+nDXhrb8b4lfWCnn5jpNwZFpP+RZpJT6XxFbONTS + XgGQowdDlIEa1Hs1klR8lPOScW3VyhWbTyfWkhg4cI6js21/0MMsC22myhjxjZKU + rCn8k0mgZw+HyB9qfm3eM4fYXHs+CXQM22eBQK+IK2VvzT9jbpSBIoJEDW0B47c= + =PbAZ + -----END PGP MESSAGE----- + fp: 8996B62CBD159DCADD3B6DC08BB33A8ABCF7BC4A + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DQrf1tCqiJxoSAQdArbiHTkrjSYBSPIIgSNnEoAWkU43Zn8/6rtksEivhPVgw + ik9/LvTH3VUSS1pDtLNoJq3wfE8aCoGTVXHjCtaEQqp7PJ9c83afZuT0/jSs20vo + 0l4Bbp+AopvK8wlLakYZM0rbXzJw7LyW7hyA3wSN/gL0MwT8sW6hb08BB3+zRY+f + dQGtPMDNZ0aJ8nzJ/WLVxi4GdC3pAWxqw/1AX0SwwMb0PEf9kdYSgnrmYQsqx9KU + =Cbzj + -----END PGP MESSAGE----- + fp: B71138A6A8964A3C3B8899857B4F70C356765BAB + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hF4DzAGzViGx4qcSAQdAQKsWq8NPJbW2SBhKhlgkW1gzYnx9baL8spEk1Wv31Asw + fuq75JZ/m8yR6+jnchE8ikuWrVQ1IRwyQBB2qlaArrdwnVpkF5HG/ggpDy4l5UYK + 0lgBhuKG36g1P7G0incMXR+S+UswYQhzm+19LqoB247HvZZoyIT4m0k7XndHBpUw + fzQyFTKdwQpmWyQWsbkW/ycvxkKyKcEce6xkga0e8UbB8w1fJ0P6gErz + =g5Ck + -----END PGP MESSAGE----- + fp: D2E9C0807BF681F5E164DAFC5EE1B61CD90954CD + - created_at: "2026-01-25T11:17:03Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQIMA2pVdGTIrZI+AQ//X0eMLW5Con7f2J4S15RwQX/uMc+p0tabrfSYAT8cg1oR + X8qyFgBWL4EK/VAcgS+Loe6cOCO8pDv7R81nn18wg2D6hVN3BOcotgLtLpqWEdMz + FguVIc++/Nh5+s+H1oDxqfwO6LbcuewBvvNS9xvBUtHBMuoAGVO0mUu7jpxrg+4k + dh2QeA/YWc4hGly/lO6eOhq61arAY4tukqs1K4JRY7z1vZYb2658HamfruLcRP1j + kM6yvJ9bgrg3hIEPG48lWX3SATRpKDP4ukyTYMFPN5rePUu67rnkwCvXwvBzWV4v + fvjmDZ4U2AD6Ihn5Be3ThZyQivZJPmxBlgit6uQOdu08Q5/S0DDWSS/MnbRnElQt + caQMnIcSbwLJfum2/0AS/dcl6f36vOl5t9eiy3nnrgufFEUcAMgJ2bJk8+6nPRli + MImBTXLMor97XD4DS+xyQ8NjYzf8XxEDduCzWA/EQborLkkaXj5J9ZmQSKDfv6bb + wcGfxt0+JGEPmOuOD/BwZHhEcd6eV8k3cM6k4oQ3k9cMGele+dtSkrlkyFKnnBNV + NrZVBE5j62sgnUUgKCesbKPfauETE5Z+R2uvOK5Y0gqjTfaw8hV1YF2q+x2qRWig + 6NjHheUtjigCgF61OK4x1a5WDJmVeuAe03JnwKYMujN4H5Oi9YMhSX65lX1+fhrU + aAEJAhCV01dJAuYksyvp+F5Dx62eKZj7gL/MHL3zHw97WbONvI7ApC3/Q7fkupYm + oPfYKQD5ov77V3u+Y8nVOoYM+Hb4thFQdEV01r90g9WUj8LrXvxd08j3GwAnzDMG + xU5hdDPzz/jT + =zb8A + -----END PGP MESSAGE----- + fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 + unencrypted_suffix: _unencrypted + version: 3.11.0 diff --git a/inventories/chaosknoten/host_vars/spaceapiccc.yaml b/inventories/chaosknoten/host_vars/spaceapiccc.yaml new file mode 100644 index 0000000..377b909 --- /dev/null +++ b/inventories/chaosknoten/host_vars/spaceapiccc.yaml @@ -0,0 +1,16 @@ +docker_compose__compose_file_content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/spaceapiccc/docker_compose/compose.yaml.j2') }}" +docker_compose__build: never +docker_compose__pull: never + +certbot__version_spec: "" +certbot__acme_account_email_address: le-admin@hamburg.ccc.de +certbot__certificate_domains: + # - "spaceapi.ccc.de" # after DNS has been adjusted + - "spaceapiccc.hamburg.ccc.de" +certbot__new_cert_commands: + - "systemctl reload nginx.service" + +nginx__version_spec: "" +nginx__configurations: + - name: spaceapiccc.hamburg.ccc.de + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/spaceapiccc/nginx/spaceapiccc.hamburg.ccc.de.conf') }}" diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 93c61be..6246119 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -74,6 +74,10 @@ all: ansible_host: renovate.hosts.hamburg.ccc.de ansible_user: chaos ansible_ssh_common_args: -J ssh://chaos@router.hamburg.ccc.de + spaceapiccc: + ansible_host: spaceapiccc.hosts.hamburg.ccc.de + ansible_user: chaos + ansible_ssh_common_args: -J ssh://chaos@router.hamburg.ccc.de hypervisors: hosts: chaosknoten: @@ -98,6 +102,7 @@ base_config_hosts: ntfy: sunders: renovate: + spaceapiccc: systemd_networkd_hosts: hosts: router: @@ -117,6 +122,7 @@ docker_compose_hosts: zammad: ntfy: sunders: + spaceapiccc: nextcloud_hosts: hosts: cloud: @@ -138,6 +144,7 @@ nginx_hosts: zammad: ntfy: sunders: + spaceapiccc: public_reverse_proxy_hosts: hosts: public-reverse-proxy: @@ -158,6 +165,7 @@ certbot_hosts: zammad: ntfy: sunders: + spaceapiccc: prometheus_node_exporter_hosts: hosts: ccchoir: @@ -189,6 +197,7 @@ infrastructure_authorized_keys_hosts: ntfy: sunders: renovate: + spaceapiccc: wiki_hosts: hosts: eh22-wiki: diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf index 82e596a..0a6a70d 100644 --- a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf +++ b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf @@ -78,6 +78,8 @@ map $host $upstream_acme_challenge_host { cryptoparty.hamburg.ccc.de 172.31.17.151:31820; staging.cryptoparty-hamburg.de 172.31.17.151:31820; staging.cryptoparty.hamburg.ccc.de 172.31.17.151:31820; + spaceapiccc.hamburg.ccc.de spaceapiccc.hosts.hamburg.ccc.de:31820; + spaceapi.ccc.de spaceapiccc.hosts.hamburg.ccc.de:31820; default ""; } diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf index 489dda5..2ad5dc4 100644 --- a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf +++ b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf @@ -96,6 +96,8 @@ stream { cryptoparty.hamburg.ccc.de 172.31.17.151:8443; staging.cryptoparty-hamburg.de 172.31.17.151:8443; staging.cryptoparty.hamburg.ccc.de 172.31.17.151:8443; + spaceapiccc.hamburg.ccc.de spaceapiccc.hosts.hamburg.ccc.de:8443; + spaceapi.ccc.de spaceapiccc.hosts.hamburg.ccc.de:8443; } server { diff --git a/resources/chaosknoten/spaceapiccc/docker_compose/compose.yaml.j2 b/resources/chaosknoten/spaceapiccc/docker_compose/compose.yaml.j2 new file mode 100644 index 0000000..67e4b58 --- /dev/null +++ b/resources/chaosknoten/spaceapiccc/docker_compose/compose.yaml.j2 @@ -0,0 +1,39 @@ +--- +services: + frontend: + #build: ./frontend + networks: + spaceapi-network: + ipv4_address: 172.16.238.10 + image: gidsi/spaceapi-ccc-frontend:saved_from_old_host + restart: always + expose: + - "80" + depends_on: + - backend + backend: + #build: ./backend + networks: + - spaceapi-network + image: gidsi/spaceapi-ccc-backend:saved_from_old_host + restart: always + environment: + SHARED_SECRET: "{{ secret__spaceapiccc__shared_secret }}" + DOKU_WIKI_USER: "{{ secret__spaceapiccc__doku_ccc_de__username }}" + DOKU_WIKI_PASSWORD: "{{ secret__spaceapiccc__doku_ccc_de__password }}" + depends_on: + - database + database: + image: mongo:saved_from_old_host + networks: + - spaceapi-network + restart: always + volumes: + - ./data/database:/data/db + +networks: + spaceapi-network: + ipam: + driver: default + config: + - subnet: 172.16.238.0/24 diff --git a/resources/chaosknoten/spaceapiccc/nginx/spaceapiccc.hamburg.ccc.de.conf b/resources/chaosknoten/spaceapiccc/nginx/spaceapiccc.hamburg.ccc.de.conf new file mode 100644 index 0000000..f060996 --- /dev/null +++ b/resources/chaosknoten/spaceapiccc/nginx/spaceapiccc.hamburg.ccc.de.conf @@ -0,0 +1,42 @@ +# partly generated 2022-01-08, Mozilla Guideline v5.6, nginx 1.17.7, OpenSSL 1.1.1k, intermediate configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.17.7&config=intermediate&openssl=1.1.1k&guideline=5.6 +server { + # Listen on a custom port for the proxy protocol. + listen [::]:8443 ssl http2 proxy_protocol; + # Make use of the ngx_http_realip_module to set the $remote_addr and + # $remote_port to the client address and client port, when using proxy + # protocol. + # First set our proxy protocol proxy as trusted. + set_real_ip_from 2a00:14b0:4200:3000:125::1; + # Then tell the realip_module to get the addreses from the proxy protocol + # header. + real_ip_header proxy_protocol; + + server_name spaceapi.ccc.de spaceapiccc.hamburg.ccc.de; + + ssl_certificate /etc/letsencrypt/live/spaceapiccc.hamburg.ccc.de/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/spaceapiccc.hamburg.ccc.de/privkey.pem; + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/letsencrypt/live/spaceapiccc.hamburg.ccc.de/chain.pem; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Port 443; + # This is https in any case. + proxy_set_header X-Forwarded-Proto https; + # Hide the X-Forwarded header. + proxy_hide_header X-Forwarded; + # Assume we are the only Reverse Proxy (well using Proxy Protocol, but that + # is transparent). + # Also provide "_hidden" for by, since it's not relevant. + proxy_set_header Forwarded "for=$remote_addr;proto=https;host=$host;by=_hidden"; + + location / { + proxy_pass http://172.16.238.10/; + } +} From d35f1cc779b9ed4a23ddc8e376c393cf1c33f2e7 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 15:31:42 +0100 Subject: [PATCH 07/11] GPG must be installed for the docker role to be able to add the repo --- roles/base_config/tasks/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/base_config/tasks/main.yaml b/roles/base_config/tasks/main.yaml index 7f0281e..ab737b7 100644 --- a/roles/base_config/tasks/main.yaml +++ b/roles/base_config/tasks/main.yaml @@ -30,4 +30,5 @@ - dnsutils - usbutils - kitty + - gpg become: true From c285694aaadd49374745c9749d54b2ece5c7afaa Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 25 Jan 2026 15:47:41 +0100 Subject: [PATCH 08/11] Add age private key --- docs/create-new-web-service-vm.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/create-new-web-service-vm.md b/docs/create-new-web-service-vm.md index 76eca21..1f5874d 100644 --- a/docs/create-new-web-service-vm.md +++ b/docs/create-new-web-service-vm.md @@ -37,7 +37,8 @@ As the first step, we need to make the host known to Ansible. 2. Add the host to the desired roles. As a minimum, you'll want `base_config_hosts` and `infrastructure_authorized_keys_hosts`. For a typical web service based on Docker Compose, you'll want `docker_compose_hosts`, `nginx_hosts`, and `certbot_hosts`. 3. In the directorry `inventories/chaosknoten/host_var/`: 1. A file `inventories/chaosknoten/host_var/example.yaml` with the host/service specific configuration. - 2. A file `inventories/chaosknoten/host_var/example.sops.yaml` with the encrypted secrets for the host/service. Run `sops inventories/chaosknoten/host_var/example.yaml` to edit/create that file. Entries there should generally be prefixed with `secret__` to make it easier to see where that variable is coming from in templates etc. + 2. A file `inventories/chaosknoten/host_var/example.sops.yaml` with the encrypted secrets for the host/service. Run `sops inventories/chaosknoten/host_var/example.yaml` to edit/create that file. Entries here should generally be prefixed with `secret__` to make it easier to see where that variable is coming from in templates etc. + * Add an entry `ansible_pull__age_private_key` with the age private key you generated above. ## Service-specific config From 5e23b86b95db573093a4e3c363160e814c63f8ae Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 25 Jan 2026 20:03:13 +0100 Subject: [PATCH 09/11] rollout Alloy to replace prometheus_node_exporter With the new network we need to deploy a push based solution in order to get metrics into prometheus --- .../chaosknoten/group_vars/all.sops.yaml | 7 +- .../chaosknoten/host_vars/grafana.yaml | 42 +-------- .../chaosknoten/host_vars/ntfy.sops.yaml | 6 +- inventories/chaosknoten/host_vars/ntfy.yaml | 86 +------------------ inventories/chaosknoten/hosts.yaml | 15 ++-- playbooks/deploy.yaml | 11 +-- .../grafana/docker_compose/prometheus.yml | 35 -------- .../grafana/nginx/metrics.hamburg.ccc.de.conf | 1 - roles/alloy/defaults/main.yaml | 44 ++++++++++ roles/alloy/tasks/main.yaml | 2 + roles/prometheus_node_exporter/meta/main.yaml | 10 --- .../prometheus_node_exporter/tasks/main.yaml | 14 --- 12 files changed, 68 insertions(+), 205 deletions(-) create mode 100644 roles/alloy/defaults/main.yaml delete mode 100644 roles/prometheus_node_exporter/meta/main.yaml delete mode 100644 roles/prometheus_node_exporter/tasks/main.yaml diff --git a/inventories/chaosknoten/group_vars/all.sops.yaml b/inventories/chaosknoten/group_vars/all.sops.yaml index ebc53b7..0c5762d 100644 --- a/inventories/chaosknoten/group_vars/all.sops.yaml +++ b/inventories/chaosknoten/group_vars/all.sops.yaml @@ -1,4 +1,5 @@ msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str] +metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str] sops: age: - recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3 @@ -163,8 +164,8 @@ sops: SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-13T23:45:06Z" - mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str] + lastmodified: "2026-01-25T18:06:26Z" + mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str] pgp: - created_at: "2025-10-20T19:03:07Z" enc: |- @@ -360,4 +361,4 @@ sops: -----END PGP MESSAGE----- fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 unencrypted_suffix: _unencrypted - version: 3.10.2 + version: 3.11.0 diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index ecc942c..b87a198 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -53,16 +53,7 @@ nginx__configurations: - name: metrics.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } +alloy_config_additional: | loki.write "default" { endpoint { url = "https://loki.hamburg.ccc.de/loki/api/v1/push" @@ -98,9 +89,9 @@ alloy_config: | } rule { source_labels = ["__journal__hostname"] - target_label = "host" + target_label = "instance" regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" + replacement = "${1}.hosts.hamburg.ccc.de" action = "replace" } } @@ -111,30 +102,3 @@ alloy_config: | format_as_json = true labels = {component = "loki.source.journal", org = "ccchh"} } - - logging { - level = "info" - } - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "scrape_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } diff --git a/inventories/chaosknoten/host_vars/ntfy.sops.yaml b/inventories/chaosknoten/host_vars/ntfy.sops.yaml index e860cca..1328d66 100644 --- a/inventories/chaosknoten/host_vars/ntfy.sops.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.sops.yaml @@ -1,5 +1,3 @@ -secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str] -secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str] secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str] ntfy: user: @@ -18,8 +16,8 @@ sops: bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-20T19:01:39Z" - mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str] + lastmodified: "2026-01-25T18:41:48Z" + mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str] pgp: - created_at: "2025-10-20T19:03:04Z" enc: |- diff --git a/inventories/chaosknoten/host_vars/ntfy.yaml b/inventories/chaosknoten/host_vars/ntfy.yaml index cab4e76..2d68bfa 100644 --- a/inventories/chaosknoten/host_vars/ntfy.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.yaml @@ -15,90 +15,8 @@ nginx__configurations: - name: ntfy.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } - loki.write "default" { - endpoint { - url = "https://loki.hamburg.ccc.de/loki/api/v1/push" - basic_auth { - username = "chaos" - password = "{{ secret__loki_chaos }}" - } - } - } - - loki.relabel "journal" { - forward_to = [] - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "systemd_unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "instance" - } - rule { - source_labels = ["__journal__transport"] - target_label = "systemd_transport" - } - rule { - source_labels = ["__journal_syslog_identifier"] - target_label = "syslog_identifier" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - loki.source.journal "read_journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - format_as_json = true - labels = {component = "loki.source.journal", org = "ccchh"} - } - - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "unix_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } - +alloy_config_additional: | prometheus.scrape "ntfy_metrics" { targets = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}] - forward_to = [prometheus.relabel.default.receiver] + forward_to = [prometheus.relabel.chaosknoten_common.receiver] } diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 93c61be..1251806 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -158,11 +158,10 @@ certbot_hosts: zammad: ntfy: sunders: -prometheus_node_exporter_hosts: +alloy_hosts: hosts: ccchoir: eh22-wiki: - tickets: keycloak: netbox: onlyoffice: @@ -170,6 +169,14 @@ prometheus_node_exporter_hosts: pretalx: wiki: zammad: + grafana: + ntfy: + tickets: + renovate: + cloud: + public-reverse-proxy: + router: + sunders: infrastructure_authorized_keys_hosts: hosts: ccchoir: @@ -199,10 +206,6 @@ netbox_hosts: proxmox_vm_template_hosts: hosts: chaosknoten: -alloy_hosts: - hosts: - grafana: - ntfy: ansible_pull_hosts: hosts: netbox: diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index f416b91..fe0cf78 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -64,11 +64,6 @@ roles: - nginx -- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts - hosts: prometheus_node_exporter_hosts - roles: - - prometheus_node_exporter - - name: Configure unattended upgrades for all non-hypervisors hosts: all:!hypervisors become: true @@ -83,10 +78,8 @@ - name: Ensure Alloy is installed and Setup on alloy_hosts hosts: alloy_hosts become: true - tasks: - - name: Setup Alloy - ansible.builtin.include_role: - name: grafana.grafana.alloy + roles: + - alloy - name: Ensure ansible_pull deployment on ansible_pull_hosts hosts: ansible_pull_hosts diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus.yml b/resources/chaosknoten/grafana/docker_compose/prometheus.yml index fd59034..7f94ab0 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus.yml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus.yml @@ -82,41 +82,6 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: pve-exporter:9221 - - job_name: hosts - static_configs: - # Wieske Chaosknoten VMs - - labels: - org: ccchh - site: wieske - type: virtual_machine - hypervisor: chaosknoten - targets: - - netbox-intern.hamburg.ccc.de:9100 - - matrix-intern.hamburg.ccc.de:9100 - - public-web-static-intern.hamburg.ccc.de:9100 - - git-intern.hamburg.ccc.de:9100 - - forgejo-actions-runner-intern.hamburg.ccc.de:9100 - - eh22-wiki-intern.hamburg.ccc.de:9100 - - mjolnir-intern.hamburg.ccc.de:9100 - - woodpecker-intern.hamburg.ccc.de:9100 - - penpot-intern.hamburg.ccc.de:9100 - - jitsi.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - ccchoir-intern.hamburg.ccc.de:9100 - - tickets-intern.hamburg.ccc.de:9100 - - keycloak-intern.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - pad-intern.hamburg.ccc.de:9100 - - wiki-intern.hamburg.ccc.de:9100 - - zammad-intern.hamburg.ccc.de:9100 - - pretalx-intern.hamburg.ccc.de:9100 - - labels: - org: ccchh - site: wieske - type: physical_machine - targets: - - chaosknoten.hamburg.ccc.de:9100 - storage: tsdb: diff --git a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf index ef2fe07..195b99d 100644 --- a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf +++ b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf @@ -9,7 +9,6 @@ server { allow 2a00:14b0:4200:3380::/64; allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing # Z9 - allow 2a07:c480:0:100::/56; allow 2a07:c481:1::/48; # fuxnoc allow 2a07:c481:0:1::/64; diff --git a/roles/alloy/defaults/main.yaml b/roles/alloy/defaults/main.yaml new file mode 100644 index 0000000..92bc960 --- /dev/null +++ b/roles/alloy/defaults/main.yaml @@ -0,0 +1,44 @@ +alloy_config_default: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ metrics__chaos_password }}" + } + } + } + + prometheus.relabel "chaosknoten_common" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + target_label = "org" + replacement = "ccchh" + } + rule { + target_label = "site" + replacement = "wieske" + } + rule { + source_labels = ["instance"] + target_label = "instance" + regex = "([^:]+)" + replacement = "${1}.hosts.hamburg.ccc.de" + action = "replace" + } + } + + logging { + level = "info" + } + + prometheus.exporter.unix "local_system" { + enable_collectors = ["systemd"] + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.chaosknoten_common.receiver] + } + +alloy_config_additional: "" diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml index 2bc308a..5e3cd64 100644 --- a/roles/alloy/tasks/main.yaml +++ b/roles/alloy/tasks/main.yaml @@ -45,4 +45,6 @@ - name: Setup Alloy ansible.builtin.import_role: name: grafana.grafana.alloy + vars: + alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}" become: true diff --git a/roles/prometheus_node_exporter/meta/main.yaml b/roles/prometheus_node_exporter/meta/main.yaml deleted file mode 100644 index 78bb770..0000000 --- a/roles/prometheus_node_exporter/meta/main.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -dependencies: - - role: distribution_check - vars: - distribution_check__distribution_support_spec: - - name: Debian - major_versions: - - "11" - - "12" - - "13" diff --git a/roles/prometheus_node_exporter/tasks/main.yaml b/roles/prometheus_node_exporter/tasks/main.yaml deleted file mode 100644 index c138f18..0000000 --- a/roles/prometheus_node_exporter/tasks/main.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: make sure the `prometheus-node-exporter` package is installed - ansible.builtin.apt: - name: prometheus-node-exporter - state: present - allow_change_held_packages: true - update_cache: true - become: true - -- name: make sure `prometheus-node-exporter.service` is started and ansibled - ansible.builtin.systemd: - name: prometheus-node-exporter.service - state: started - enabled: true - become: true From 29af58ff94b2447e8f0c164a9b982fee055c6763 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 4 Jan 2026 19:32:42 +0100 Subject: [PATCH 10/11] wip: alloy --- roles/alloy/tasks/main.yaml | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 roles/alloy/tasks/main.yaml diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml new file mode 100644 index 0000000..2bc308a --- /dev/null +++ b/roles/alloy/tasks/main.yaml @@ -0,0 +1,48 @@ +# https://github.com/grafana/grafana-ansible-collection/blob/main/roles/alloy/tasks/deploy.yml#L124 +- name: ensure alloy user exists + ansible.builtin.user: + name: alloy + system: true + append: true + create_home: false + state: present + +- name: ensure the `/etc/alloy/` config directory exists + ansible.builtin.file: + path: /etc/alloy + state: directory + mode: "0770" + owner: root + group: alloy + become: true + +- name: synchronize the additional configuration files directory, if present + when: alloy__additional_configs_dir is defined and alloy__additional_configs_dir != "" + block: + - name: ensure rsync is installed + ansible.builtin.apt: + name: rsync + become: true + + - name: synchronize the additional configuration files directory, if present + ansible.posix.synchronize: + src: "{{ alloy__additional_configs_dir }}" + dest: /etc/alloy/additional + delete: true + recursive: true + use_ssh_args: true + rsync_opts: + - "--chown=root:alloy" + become: true + +- name: delete the additional configuration files directory, if not present + when: alloy__additional_configs_dir is not defined or alloy__additional_configs_dir == "" + ansible.builtin.file: + path: /etc/alloy/additional + state: absent + become: true + +- name: Setup Alloy + ansible.builtin.import_role: + name: grafana.grafana.alloy + become: true From 5ff4de7e00f1da535ebb419a29bbad46b84e0458 Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 25 Jan 2026 20:03:13 +0100 Subject: [PATCH 11/11] rollout Alloy to replace prometheus_node_exporter With the new network we need to deploy a push based solution in order to get metrics into prometheus --- .../chaosknoten/group_vars/all.sops.yaml | 7 +- .../chaosknoten/host_vars/grafana.yaml | 42 +-------- .../chaosknoten/host_vars/ntfy.sops.yaml | 6 +- inventories/chaosknoten/host_vars/ntfy.yaml | 86 +------------------ inventories/chaosknoten/hosts.yaml | 15 ++-- playbooks/deploy.yaml | 11 +-- .../grafana/docker_compose/prometheus.yml | 35 -------- .../grafana/nginx/metrics.hamburg.ccc.de.conf | 1 - roles/alloy/defaults/main.yaml | 44 ++++++++++ roles/alloy/tasks/main.yaml | 2 + roles/prometheus_node_exporter/meta/main.yaml | 10 --- .../prometheus_node_exporter/tasks/main.yaml | 14 --- 12 files changed, 68 insertions(+), 205 deletions(-) create mode 100644 roles/alloy/defaults/main.yaml delete mode 100644 roles/prometheus_node_exporter/meta/main.yaml delete mode 100644 roles/prometheus_node_exporter/tasks/main.yaml diff --git a/inventories/chaosknoten/group_vars/all.sops.yaml b/inventories/chaosknoten/group_vars/all.sops.yaml index ebc53b7..0c5762d 100644 --- a/inventories/chaosknoten/group_vars/all.sops.yaml +++ b/inventories/chaosknoten/group_vars/all.sops.yaml @@ -1,4 +1,5 @@ msmtp__smtp_password: ENC[AES256_GCM,data:xcBVBTb6mfr5Ubyfga9ibKWKhrfrEEaDWD98vIbX8fl8lQ4YTovg8Ax1HTK4UQ6AkJGHq2A0D5B67KUTlp9eLw==,iv:TOp1G1LktRPj/KMCRU5CXBUsgKOqGssUvvk5oY0QnPM=,tag:SVBdDQy+fM0xeEToappP+A==,type:str] +metrics__chaos_password: ENC[AES256_GCM,data:al234VSAH7oxka8X0hTvEJKVLD6O/WCrCKfVLLvm,iv:+TmA+0hXMV4OxvK7RH2g1dIzm88Lpm3zevxSZxK23QQ=,tag:txCVr5SEW3dVHgNFInR94g==,type:str] sops: age: - recipient: age1ss82zwqkj438re78355p886r89csqrrfmkfp8lrrf8v23nza492qza4ey3 @@ -163,8 +164,8 @@ sops: SnUrSUlvMXhnY3JrbER0TkxBcGJucmsKdBDkRY5FUtOo8zQ0QtfPFGJn0O2Fg5xn mSloxLaFwdXAR9L1QfUdsW+9Vgez4s5bxMJtn8hkwqIfyJc25FEEcA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-13T23:45:06Z" - mac: ENC[AES256_GCM,data:QxH4lnNyCAAEJhzbgCrq7QeLs+OAtYgwQP4oFm93NE4Fbz7/Hz2dvL/2SopOdW7nYVeb1scuG1ra+yvgzuQDhg4lcgt9eBJoBiynM3qiHBs+FtcSJoKs16I/ACAadQwClALb4E0xxwKFJI8ewMZu5BAxi5EhYbgNfnKCIbhvgWo=,iv:LRa2vX0HUBugeEAVeOqXbPsMQrfrCpyzGUGjK6+VaQc=,tag:/sfhJM8V1IYBh94ZS/TDxQ==,type:str] + lastmodified: "2026-01-25T18:06:26Z" + mac: ENC[AES256_GCM,data:plHNLOgGWwNWbakKG6X5EOxwERE3rvYO4EOAzY/sz+uM7cZBEnqU5LZwjlD8B75hgRHqpnDBF0JbHgsEwVxfJJRL1phkeMJFOapQMjZVWMz6j7eb1hOwpdktd+bpuimy4XCD1aOxOoInKpFSK33usxLfyqSxjFDM5+i6D22qBTs=,iv:/iOIfNuSIDsa/UKLP0d63tpOrYMFO3Bk1qPssY0AzuI=,tag:k+824MXD+r0lNUcuvisudw==,type:str] pgp: - created_at: "2025-10-20T19:03:07Z" enc: |- @@ -360,4 +361,4 @@ sops: -----END PGP MESSAGE----- fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533 unencrypted_suffix: _unencrypted - version: 3.10.2 + version: 3.11.0 diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index ecc942c..b87a198 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -53,16 +53,7 @@ nginx__configurations: - name: metrics.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } +alloy_config_additional: | loki.write "default" { endpoint { url = "https://loki.hamburg.ccc.de/loki/api/v1/push" @@ -98,9 +89,9 @@ alloy_config: | } rule { source_labels = ["__journal__hostname"] - target_label = "host" + target_label = "instance" regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" + replacement = "${1}.hosts.hamburg.ccc.de" action = "replace" } } @@ -111,30 +102,3 @@ alloy_config: | format_as_json = true labels = {component = "loki.source.journal", org = "ccchh"} } - - logging { - level = "info" - } - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "scrape_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } diff --git a/inventories/chaosknoten/host_vars/ntfy.sops.yaml b/inventories/chaosknoten/host_vars/ntfy.sops.yaml index e860cca..1328d66 100644 --- a/inventories/chaosknoten/host_vars/ntfy.sops.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.sops.yaml @@ -1,5 +1,3 @@ -secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str] -secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str] secret__ntfy_web_push_private_key: ENC[AES256_GCM,data:YqNEYa1Ln3NFpNoIuBUN1V/WRzod5HAtYueBJYHOwyM59cCaYhQR1S9aQg==,iv:t8bEs5ZAEe6pqbbOb0mpJdfgruX1P9Jd+sbNurGqkng=,tag:Cdy5HKkvb55V6AeRt+MVHg==,type:str] ntfy: user: @@ -18,8 +16,8 @@ sops: bUhGdEFwOEVxUzVZdERReVF6cmcxeDgKDlO+jacsYgWXqjoxAIKJiB8mCHZ8U7TM sGD3oaCi9x6Uvse7hq0BaUe/LaJt2tDaqve9nm3n06V93HNcR9/cdw== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-10-20T19:01:39Z" - mac: ENC[AES256_GCM,data:a87jRAGBIypZfYCILYCOM+H8KCVUBgb2/1sG05wDbPmLe9IfDT6rzlljbRFOUozq9xsqxpFLsPQx1wPVDi1lhaRT+5oE/NDgVH8aQCofA96DQd3SeB8fWn3LhYjOpmo9ZsFSemvGcXYk/SjVvoU9aN8KG4DHYCOOseGIBTa/a2Y=,iv:5Atem3ACdfdCPUp184cAf/EI9BEXQ1i719l+sIlOnUY=,tag:LWQCxrsZ3660UCcOjY4gMQ==,type:str] + lastmodified: "2026-01-25T18:41:48Z" + mac: ENC[AES256_GCM,data:2+628ZxPIto0AUhRExTB0UF/XKD7l0qz/NVncKbk+E5nZ5IRGwnhvY5DPiaDNWxskngaYhSYaQZTJTuvC1TuflCr8+IsZRYobj22mYEsrK2KWbozQvYsuooK2HdSWAkE2U5xKKodev2KqxMT+ZY0AIq8ifCo033ro6t0rnIEVQI=,iv:ncKxlhfZ+04rylNmMtOaWyonCJO4gbsuABMAJfVDDIQ=,tag:6c141UrWXNuGM5giTS7Ecw==,type:str] pgp: - created_at: "2025-10-20T19:03:04Z" enc: |- diff --git a/inventories/chaosknoten/host_vars/ntfy.yaml b/inventories/chaosknoten/host_vars/ntfy.yaml index cab4e76..2d68bfa 100644 --- a/inventories/chaosknoten/host_vars/ntfy.yaml +++ b/inventories/chaosknoten/host_vars/ntfy.yaml @@ -15,90 +15,8 @@ nginx__configurations: - name: ntfy.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}" -alloy_config: | - prometheus.remote_write "default" { - endpoint { - url = "https://metrics.hamburg.ccc.de/api/v1/write" - basic_auth { - username = "chaos" - password = "{{ secret__metrics_chaos }}" - } - } - } - loki.write "default" { - endpoint { - url = "https://loki.hamburg.ccc.de/loki/api/v1/push" - basic_auth { - username = "chaos" - password = "{{ secret__loki_chaos }}" - } - } - } - - loki.relabel "journal" { - forward_to = [] - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "systemd_unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "instance" - } - rule { - source_labels = ["__journal__transport"] - target_label = "systemd_transport" - } - rule { - source_labels = ["__journal_syslog_identifier"] - target_label = "syslog_identifier" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - loki.source.journal "read_journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - format_as_json = true - labels = {component = "loki.source.journal", org = "ccchh"} - } - - prometheus.exporter.unix "local_system" { - enable_collectors = ["systemd"] - } - - prometheus.relabel "default" { - forward_to = [prometheus.remote_write.default.receiver] - rule { - target_label = "org" - replacement = "ccchh" - } - rule { - source_labels = ["instance"] - target_label = "host" - regex = "([^:]+)" - replacement = "${1}.hamburg.ccc.de" - action = "replace" - } - } - - prometheus.scrape "unix_metrics" { - targets = prometheus.exporter.unix.local_system.targets - forward_to = [prometheus.relabel.default.receiver] - } - +alloy_config_additional: | prometheus.scrape "ntfy_metrics" { targets = [{"__address__" = "localhost:9586", job = "ntfy", instance = "ntfy", __scrape_interval__ = "120s"}] - forward_to = [prometheus.relabel.default.receiver] + forward_to = [prometheus.relabel.chaosknoten_common.receiver] } diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 6246119..d4d2565 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -166,11 +166,10 @@ certbot_hosts: ntfy: sunders: spaceapiccc: -prometheus_node_exporter_hosts: +alloy_hosts: hosts: ccchoir: eh22-wiki: - tickets: keycloak: netbox: onlyoffice: @@ -178,6 +177,14 @@ prometheus_node_exporter_hosts: pretalx: wiki: zammad: + grafana: + ntfy: + tickets: + renovate: + cloud: + public-reverse-proxy: + router: + sunders: infrastructure_authorized_keys_hosts: hosts: ccchoir: @@ -208,10 +215,6 @@ netbox_hosts: proxmox_vm_template_hosts: hosts: chaosknoten: -alloy_hosts: - hosts: - grafana: - ntfy: ansible_pull_hosts: hosts: netbox: diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index f416b91..fe0cf78 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -64,11 +64,6 @@ roles: - nginx -- name: Ensure prometheus_node_exporter deployment on prometheus_node_exporter_hosts - hosts: prometheus_node_exporter_hosts - roles: - - prometheus_node_exporter - - name: Configure unattended upgrades for all non-hypervisors hosts: all:!hypervisors become: true @@ -83,10 +78,8 @@ - name: Ensure Alloy is installed and Setup on alloy_hosts hosts: alloy_hosts become: true - tasks: - - name: Setup Alloy - ansible.builtin.include_role: - name: grafana.grafana.alloy + roles: + - alloy - name: Ensure ansible_pull deployment on ansible_pull_hosts hosts: ansible_pull_hosts diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus.yml b/resources/chaosknoten/grafana/docker_compose/prometheus.yml index fd59034..7f94ab0 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus.yml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus.yml @@ -82,41 +82,6 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: pve-exporter:9221 - - job_name: hosts - static_configs: - # Wieske Chaosknoten VMs - - labels: - org: ccchh - site: wieske - type: virtual_machine - hypervisor: chaosknoten - targets: - - netbox-intern.hamburg.ccc.de:9100 - - matrix-intern.hamburg.ccc.de:9100 - - public-web-static-intern.hamburg.ccc.de:9100 - - git-intern.hamburg.ccc.de:9100 - - forgejo-actions-runner-intern.hamburg.ccc.de:9100 - - eh22-wiki-intern.hamburg.ccc.de:9100 - - mjolnir-intern.hamburg.ccc.de:9100 - - woodpecker-intern.hamburg.ccc.de:9100 - - penpot-intern.hamburg.ccc.de:9100 - - jitsi.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - ccchoir-intern.hamburg.ccc.de:9100 - - tickets-intern.hamburg.ccc.de:9100 - - keycloak-intern.hamburg.ccc.de:9100 - - onlyoffice-intern.hamburg.ccc.de:9100 - - pad-intern.hamburg.ccc.de:9100 - - wiki-intern.hamburg.ccc.de:9100 - - zammad-intern.hamburg.ccc.de:9100 - - pretalx-intern.hamburg.ccc.de:9100 - - labels: - org: ccchh - site: wieske - type: physical_machine - targets: - - chaosknoten.hamburg.ccc.de:9100 - storage: tsdb: diff --git a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf index ef2fe07..195b99d 100644 --- a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf +++ b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf @@ -9,7 +9,6 @@ server { allow 2a00:14b0:4200:3380::/64; allow 2a00:14b0:f000:23::/64; #CCCHH v6 bei Wieske, geroutet über turing # Z9 - allow 2a07:c480:0:100::/56; allow 2a07:c481:1::/48; # fuxnoc allow 2a07:c481:0:1::/64; diff --git a/roles/alloy/defaults/main.yaml b/roles/alloy/defaults/main.yaml new file mode 100644 index 0000000..92bc960 --- /dev/null +++ b/roles/alloy/defaults/main.yaml @@ -0,0 +1,44 @@ +alloy_config_default: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ metrics__chaos_password }}" + } + } + } + + prometheus.relabel "chaosknoten_common" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + target_label = "org" + replacement = "ccchh" + } + rule { + target_label = "site" + replacement = "wieske" + } + rule { + source_labels = ["instance"] + target_label = "instance" + regex = "([^:]+)" + replacement = "${1}.hosts.hamburg.ccc.de" + action = "replace" + } + } + + logging { + level = "info" + } + + prometheus.exporter.unix "local_system" { + enable_collectors = ["systemd"] + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.chaosknoten_common.receiver] + } + +alloy_config_additional: "" diff --git a/roles/alloy/tasks/main.yaml b/roles/alloy/tasks/main.yaml index 2bc308a..5e3cd64 100644 --- a/roles/alloy/tasks/main.yaml +++ b/roles/alloy/tasks/main.yaml @@ -45,4 +45,6 @@ - name: Setup Alloy ansible.builtin.import_role: name: grafana.grafana.alloy + vars: + alloy_config: "{{ alloy_config_default ~ alloy_config_additional }}" become: true diff --git a/roles/prometheus_node_exporter/meta/main.yaml b/roles/prometheus_node_exporter/meta/main.yaml deleted file mode 100644 index 78bb770..0000000 --- a/roles/prometheus_node_exporter/meta/main.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -dependencies: - - role: distribution_check - vars: - distribution_check__distribution_support_spec: - - name: Debian - major_versions: - - "11" - - "12" - - "13" diff --git a/roles/prometheus_node_exporter/tasks/main.yaml b/roles/prometheus_node_exporter/tasks/main.yaml deleted file mode 100644 index c138f18..0000000 --- a/roles/prometheus_node_exporter/tasks/main.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: make sure the `prometheus-node-exporter` package is installed - ansible.builtin.apt: - name: prometheus-node-exporter - state: present - allow_change_held_packages: true - update_cache: true - become: true - -- name: make sure `prometheus-node-exporter.service` is started and ansibled - ansible.builtin.systemd: - name: prometheus-node-exporter.service - state: started - enabled: true - become: true