From 98972e39c4e7ce319aadef0e8758fb35bcbef138 Mon Sep 17 00:00:00 2001 From: c6ristian Date: Sat, 22 Feb 2025 22:56:40 +0100 Subject: [PATCH 01/14] keycloak(host): update to 26.1 & postgres to 15.12 --- resources/chaosknoten/keycloak/docker_compose/compose.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/chaosknoten/keycloak/docker_compose/compose.yaml.j2 b/resources/chaosknoten/keycloak/docker_compose/compose.yaml.j2 index 9509654..231f581 100644 --- a/resources/chaosknoten/keycloak/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/keycloak/docker_compose/compose.yaml.j2 @@ -22,7 +22,7 @@ services: keycloak: - image: git.hamburg.ccc.de/ccchh/oci-images/keycloak:26.0 + image: git.hamburg.ccc.de/ccchh/oci-images/keycloak:26.1 pull_policy: always restart: unless-stopped command: start --optimized @@ -46,7 +46,7 @@ services: - "8080:8080" db: - image: postgres:15.2 + image: postgres:15.12 restart: unless-stopped networks: - keycloak From 614eebadba380604be4908c8de9c69af35dec466 Mon Sep 17 00:00:00 2001 From: jtbx Date: Sun, 23 Feb 2025 18:49:19 +0100 Subject: [PATCH 02/14] WIP router(host): initial config --- inventories/chaosknoten/hosts.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index cae283d..c164b0b 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -55,6 +55,9 @@ all: public-reverse-proxy: ansible_host: public-reverse-proxy.hamburg.ccc.de ansible_user: chaos + router: + ansible_host: router.hamburg.ccc.de + ansible_user: chaos wiki: ansible_host: wiki-intern.hamburg.ccc.de ansible_user: chaos @@ -81,6 +84,7 @@ base_config_hosts: pad: pretalx: public-reverse-proxy: + router: tickets: wiki: zammad: @@ -161,6 +165,7 @@ infrastructure_authorized_keys_hosts: pad: pretalx: public-reverse-proxy: + router: wiki: zammad: wiki_hosts: From ca16e3d55fe86ba054d8f114c8d4858ebf3a793a Mon Sep 17 00:00:00 2001 From: June Date: Sun, 23 Feb 2025 22:41:06 +0100 Subject: [PATCH 03/14] dep._hypervisor(playb.): introduce play for setting up vm template gen. --- .../chaosknoten/host_vars/chaosknoten.yaml | 6 +++ inventories/chaosknoten/hosts.yaml | 3 ++ playbooks/deploy_hypervisor.yaml | 54 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 inventories/chaosknoten/host_vars/chaosknoten.yaml create mode 100644 playbooks/deploy_hypervisor.yaml diff --git a/inventories/chaosknoten/host_vars/chaosknoten.yaml b/inventories/chaosknoten/host_vars/chaosknoten.yaml new file mode 100644 index 0000000..1c8fa93 --- /dev/null +++ b/inventories/chaosknoten/host_vars/chaosknoten.yaml @@ -0,0 +1,6 @@ +# Used in deploy_hypervisor playbook. +hypervisor__template_vm_config: + - name: STORAGE + value: nvme0 + - name: BRIDGE + value: vmbr4 diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index cae283d..1f5b31b 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -171,3 +171,6 @@ netbox_hosts: hosts: eh22-netbox: netbox: +proxmox_vm_template_hosts: + hosts: + chaosknoten: diff --git a/playbooks/deploy_hypervisor.yaml b/playbooks/deploy_hypervisor.yaml new file mode 100644 index 0000000..0739d35 --- /dev/null +++ b/playbooks/deploy_hypervisor.yaml @@ -0,0 +1,54 @@ +- name: Ensure the VM template generation is set up + hosts: proxmox_vm_template_hosts + tasks: + - name: Ensure /usr/local/{lib,sbin} exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + become: true + loop: + - "/usr/local/lib/" + - "/usr/local/sbin/" + + - name: Ensure the pve-template-vm repo is present + ansible.builtin.git: + repo: https://git.hamburg.ccc.de/CCCHH/pve-template-vm.git + dest: /usr/local/lib/pve-template-vm + version: main + force: true + depth: 1 + single_branch: true + track_submodules: true + become: true + + # /usr/local/sbin as the script uses qm, which is also found in /usr/sbin. + - name: Ensure symlink to build-proxmox-template exists in /usr/local/sbin + ansible.builtin.file: + src: /usr/local/lib/pve-template-vm/build-proxmox-template + dest: /usr/local/sbin/build-proxmox-template + state: link + owner: root + group: root + mode: '0755' + become: true + + # This sets up a cron job running /usr/local/sbin/build-proxmox-template using the env vars defined in hypervisor__template_vm_config. + - name: Ensure cron job is present for building a fresh VM template every week on Friday 04:00 + ansible.builtin.cron: + name: "ansible build proxmox template" + cron_file: ansible_build_proxmox_template + minute: 0 + hour: 4 + weekday: 5 + user: root + job: "{% if hypervisor__template_vm_config is defined and hypervisor__template_vm_config | length > 0 %}\ + /usr/bin/env \ + {% for item in hypervisor__template_vm_config | default([]) %}\ + {{ item.name }}=\"{{ item.value }}\" \ + {% endfor %}\ + {% endif %}\ + /usr/local/sbin/build-proxmox-template" + become: true From fd13e5341b6285480b956ec784b61dd5bf2715e8 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 23 Feb 2025 23:23:51 +0100 Subject: [PATCH 04/14] add thinkcccore0 to inventory and enable VM template gen. setup on it --- inventories/z9/hosts.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/inventories/z9/hosts.yaml b/inventories/z9/hosts.yaml index 0dde922..74428f3 100644 --- a/inventories/z9/hosts.yaml +++ b/inventories/z9/hosts.yaml @@ -6,6 +6,12 @@ all: authoritative-dns: ansible_host: authoritative-dns.z9.ccchh.net ansible_user: chaos + thinkcccore0: + ansible_host: thinkcccore0.z9.ccchh.net + ansible_user: june +hypervisors: + hosts: + thinkcccore0: nginx_hosts: hosts: light: @@ -19,3 +25,6 @@ infrastructure_authorized_keys_hosts: hosts: light: authoritative-dns: +proxmox_vm_template_hosts: + hosts: + thinkcccore0: From 41ba73d7c36cf791e372cc212c8587db3bcdd607 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 23 Feb 2025 23:26:36 +0100 Subject: [PATCH 05/14] dep._hyperv.(playb.): add deps step to vm template generation setup play --- playbooks/deploy_hypervisor.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/playbooks/deploy_hypervisor.yaml b/playbooks/deploy_hypervisor.yaml index 0739d35..4d3200f 100644 --- a/playbooks/deploy_hypervisor.yaml +++ b/playbooks/deploy_hypervisor.yaml @@ -1,6 +1,13 @@ - name: Ensure the VM template generation is set up hosts: proxmox_vm_template_hosts tasks: + - name: Ensure dependencies are present + ansible.builtin.apt: + name: + - git + - libguestfs-tools + become: true + - name: Ensure /usr/local/{lib,sbin} exist ansible.builtin.file: path: "{{ item }}" From 6b80f5b52a23cca4fa24ebf20d054b30a7427a36 Mon Sep 17 00:00:00 2001 From: June Date: Sun, 23 Feb 2025 23:51:58 +0100 Subject: [PATCH 06/14] fix accidentally added personalized ansible_user by removing it --- inventories/z9/hosts.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/inventories/z9/hosts.yaml b/inventories/z9/hosts.yaml index 74428f3..13e2cc9 100644 --- a/inventories/z9/hosts.yaml +++ b/inventories/z9/hosts.yaml @@ -8,7 +8,6 @@ all: ansible_user: chaos thinkcccore0: ansible_host: thinkcccore0.z9.ccchh.net - ansible_user: june hypervisors: hosts: thinkcccore0: From 77e1d3bc3e4f8d35517ee476dfb8c4b075bda61f Mon Sep 17 00:00:00 2001 From: c6ristian Date: Wed, 26 Feb 2025 23:47:20 +0100 Subject: [PATCH 07/14] keycloak: allow access form new IPv6 subnet at z9 --- .../keycloak/nginx/keycloak-admin.hamburg.ccc.de.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/chaosknoten/keycloak/nginx/keycloak-admin.hamburg.ccc.de.conf b/resources/chaosknoten/keycloak/nginx/keycloak-admin.hamburg.ccc.de.conf index 372715d..2b0d919 100644 --- a/resources/chaosknoten/keycloak/nginx/keycloak-admin.hamburg.ccc.de.conf +++ b/resources/chaosknoten/keycloak/nginx/keycloak-admin.hamburg.ccc.de.conf @@ -43,6 +43,7 @@ server { allow 185.161.129.132/32; # z9 allow 2a07:c480:0:100::/56; # z9 + allow 2a07:c481:1::/48; # z9 new ipv6 allow 213.240.180.39/32; # stbe home allow 2a01:170:118b::1/64; # stbe home deny all; From 811b5832da0b2fc3fd70786c6776a100b321ef22 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Wed, 16 Apr 2025 16:27:02 +0200 Subject: [PATCH 08/14] Incread file upload limit to 1G (from 10MB) --- resources/chaosknoten/pretalx/docker_compose/compose.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/chaosknoten/pretalx/docker_compose/compose.yaml.j2 b/resources/chaosknoten/pretalx/docker_compose/compose.yaml.j2 index b210098..20dbd9c 100644 --- a/resources/chaosknoten/pretalx/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/pretalx/docker_compose/compose.yaml.j2 @@ -53,6 +53,7 @@ services: restart: unless-stopped environment: PRETALX_DATA_DIR: /data + PRETALX_FILE_UPLOAD_LIMIT: 1000 # MB PRETALX_FILESYSTEM_MEDIA: /public/media PRETALX_FILESYSTEM_STATIC: /public/static PRETALX_SITE_URL: https://pretalx.hamburg.ccc.de From 9c57fca87635e69b918098ec825c85ad6d8b5ccf Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Wed, 16 Apr 2025 16:27:21 +0200 Subject: [PATCH 09/14] Also supply a dmoain for user content --- .../chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf | 1 + resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf index 4e0e8e3..e2b89d9 100644 --- a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf +++ b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf @@ -71,6 +71,7 @@ map $host $upstream_acme_challenge_host { hydra.hamburg.ccc.de 172.31.17.163:31820; cfp.eh22.easterhegg.eu 172.31.17.157:31820; hub.eh22.easterhegg.eu eh22hub-intern.hamburg.ccc.de:31820; + hub-usercontent.eh22.easterhegg.eu eh22hub-intern.hamburg.ccc.de:31820; netbox.eh22.easterhegg.eu eh22-netbox-intern.hamburg.ccc.de:31820; default ""; } diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf index 4a7f84c..6560b75 100644 --- a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf +++ b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf @@ -89,6 +89,7 @@ stream { hydra.hamburg.ccc.de 172.31.17.163:8443; cfp.eh22.easterhegg.eu pretalx-intern.hamburg.ccc.de:8443; hub.eh22.easterhegg.eu eh22hub-intern.hamburg.ccc.de:8443; + hub-usercontent.eh22.easterhegg.eu eh22hub-intern.hamburg.ccc.de:8443; netbox.eh22.easterhegg.eu eh22-netbox-intern.hamburg.ccc.de:8443; } From 3548c1f4d6763b2a2f53201603a9d1917a7951a8 Mon Sep 17 00:00:00 2001 From: June Date: Fri, 25 Apr 2025 02:01:29 +0200 Subject: [PATCH 10/14] restart ssh service instead of rebooting as this should be fine Active connections should survive a restart of the service and testing also didn't show any issues. --- inventories/chaosknoten/hosts.yaml | 3 +++ roles/deploy_ssh_server_config/handlers/main.yaml | 6 ++++-- roles/deploy_ssh_server_config/tasks/main.yaml | 6 ++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 1f5b31b..92185ed 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -174,3 +174,6 @@ netbox_hosts: proxmox_vm_template_hosts: hosts: chaosknoten: +ansible_pull_hosts: + hosts: + netbox: diff --git a/roles/deploy_ssh_server_config/handlers/main.yaml b/roles/deploy_ssh_server_config/handlers/main.yaml index 001bbe4..721a348 100644 --- a/roles/deploy_ssh_server_config/handlers/main.yaml +++ b/roles/deploy_ssh_server_config/handlers/main.yaml @@ -1,3 +1,5 @@ -- name: reboot the system +- name: restart the ssh service + ansible.builtin.systemd: + name: ssh.service + state: restarted become: true - ansible.builtin.reboot: diff --git a/roles/deploy_ssh_server_config/tasks/main.yaml b/roles/deploy_ssh_server_config/tasks/main.yaml index f5d00f5..4350790 100644 --- a/roles/deploy_ssh_server_config/tasks/main.yaml +++ b/roles/deploy_ssh_server_config/tasks/main.yaml @@ -12,8 +12,7 @@ group: root src: sshd_config.j2 notify: - # Reboot instead of just restarting the ssh service, since I don't know how Ansible reacts, when it restarts the service it probably needs for the connection. - - reboot the system + - restart the ssh service - name: deactivate short moduli ansible.builtin.shell: @@ -32,5 +31,4 @@ changed_when: - '"ansible-changed" in result.stdout' notify: - # Reboot instead of just restarting the ssh service, since I don't know how Ansible reacts, when it restarts the service it probably needs for the connection. - - reboot the system + - restart the ssh service From 456117a7892dd82baa52f136a70b01f78742ac5d Mon Sep 17 00:00:00 2001 From: c6ristian Date: Mon, 28 Apr 2025 20:04:19 +0200 Subject: [PATCH 11/14] adding loki --- .../chaosknoten/host_vars/grafana.yaml | 13 ++++ .../grafana/docker_compose/compose.yaml.j2 | 12 ++++ .../docker_compose/grafana-datasource.yml | 12 ++++ .../grafana/docker_compose/loki.yaml | 52 ++++++++++++++ .../grafana/nginx/loki.hamburg.ccc.de.conf | 71 +++++++++++++++++++ .../grafana/nginx/loki.htpasswd.j2 | 1 + .../chaosknoten/grafana/nginx/redirect.conf | 14 ++++ roles/nginx/meta/argument_specs.yaml | 16 +++++ roles/nginx/tasks/main/04_config_deploy.yaml | 14 ++++ 9 files changed, 205 insertions(+) create mode 100644 resources/chaosknoten/grafana/docker_compose/loki.yaml create mode 100644 resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf create mode 100644 resources/chaosknoten/grafana/nginx/loki.htpasswd.j2 create mode 100644 resources/chaosknoten/grafana/nginx/redirect.conf diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index 87cd328..2e8cb71 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -12,15 +12,28 @@ docker_compose__configuration_files: content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml') }}" - name: alertmanager_alert_templates.tmpl content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl') }}" + - name: loki.yaml + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/loki.yaml') }}" certbot__version_spec: "" certbot__acme_account_email_address: le-admin@hamburg.ccc.de certbot__certificate_domains: - "grafana.hamburg.ccc.de" + - "loki.hamburg.ccc.de" + certbot__new_cert_commands: - "systemctl reload nginx.service" nginx__version_spec: "" +nginx__deploy_redirect_conf: false +nginx__deploy_htpasswds: true +nginx__htpasswds: + - name: loki + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/loki.htpasswd.j2') }}" nginx__configurations: + - name: redirectv6 + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/redirect.conf') }}" - name: grafana.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/grafana.hamburg.ccc.de.conf') }}" + - name: loki.hamburg.ccc.de + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}" diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 index 3e994dc..cbe6e81 100644 --- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 @@ -55,7 +55,19 @@ services: - /dev/null:/etc/prometheus/pve.yml + loki: + image: grafana/loki:3 + container_name: loki + ports: + - 13100:3100 + - 19099:9099 + restart: unless-stopped + volumes: + - ./configs/loki.yaml:/etc/loki/local-config.yaml + - loki_data:/var/loki + volumes: graf_data: {} prom_data: {} alertmanager_data: {} + loki_data: {} diff --git a/resources/chaosknoten/grafana/docker_compose/grafana-datasource.yml b/resources/chaosknoten/grafana/docker_compose/grafana-datasource.yml index 44999d4..632ad1c 100644 --- a/resources/chaosknoten/grafana/docker_compose/grafana-datasource.yml +++ b/resources/chaosknoten/grafana/docker_compose/grafana-datasource.yml @@ -7,3 +7,15 @@ datasources: isDefault: true access: proxy editable: true + - name: Loki + type: loki + url: http://loki:3100 + access: proxy + editable: true + jsonData: + timeout: 60 + maxLines: 3000 + httpHeaderName1: "X-Scope-OrgID" + secureJsonData: + httpHeaderValue1: "chaos" + diff --git a/resources/chaosknoten/grafana/docker_compose/loki.yaml b/resources/chaosknoten/grafana/docker_compose/loki.yaml new file mode 100644 index 0000000..daf214f --- /dev/null +++ b/resources/chaosknoten/grafana/docker_compose/loki.yaml @@ -0,0 +1,52 @@ +auth_enabled: true + +server: + http_listen_port: 3100 + grpc_listen_port: 9099 + log_level: warn + +limits_config: + retention_period: 14d + +common: + instance_addr: 127.0.0.1 + path_prefix: /var/loki + storage: + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +storage_config: + filesystem: + directory: /var/loki/chunks + index_queries_cache_config: + embedded_cache: + enabled: true + max_size_mb: 80 + ttl: 30m + +schema_config: + configs: + - from: 2025-04-28 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 80 + ttl: 30m + write_dedupe_cache_config: + embedded_cache: + enabled: true + max_size_mb: 80 + ttl: 30m diff --git a/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf new file mode 100644 index 0000000..f62dc4e --- /dev/null +++ b/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf @@ -0,0 +1,71 @@ +server { + # Wieske + allow 172.31.17.128/25; + allow 212.12.51.128/28; + allow 2a00:14b0:42:100::/56; + # Z9 + allow 2a07:c480:0:100::/56; + allow 2a07:c481:1::/48; + deny all; + + listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:9099 ssl http2; + listen 172.31.17.145:9099 ssl http2; + + server_name loki.hamburg.ccc.de; + + ssl_certificate /etc/letsencrypt/live/loki.hamburg.ccc.de/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/loki.hamburg.ccc.de/privkey.pem; + + auth_basic "loki"; + auth_basic_user_file loki.htpasswd; + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Port 9099; + # This is https in any case. + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Scope-OrgID $remote_user; + grpc_pass grpc://localhost:19009; + } +} + +server { + # Wieske + allow 172.31.17.128/25; + allow 212.12.51.128/28; + allow 2a00:14b0:42:100::/56; + # Z9 + allow 2a07:c480:0:100::/56; + allow 2a07:c481:1::/48; + deny all; + + listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:3100 ssl http2; + listen 172.31.17.145:3100 ssl http2; + + server_name loki.hamburg.ccc.de; + + ssl_certificate /etc/letsencrypt/live/loki.hamburg.ccc.de/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/loki.hamburg.ccc.de/privkey.pem; + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/letsencrypt/live/loki.hamburg.ccc.de/chain.pem; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + auth_basic "loki"; + auth_basic_user_file loki.htpasswd; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Port 3100; + # This is https in any case. + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Scope-OrgID $remote_user; + proxy_pass http://127.0.0.1:13100; + } +} diff --git a/resources/chaosknoten/grafana/nginx/loki.htpasswd.j2 b/resources/chaosknoten/grafana/nginx/loki.htpasswd.j2 new file mode 100644 index 0000000..0c2f8b7 --- /dev/null +++ b/resources/chaosknoten/grafana/nginx/loki.htpasswd.j2 @@ -0,0 +1 @@ +chaos:{{ lookup("community.general.passwordstore", "noc/vm-secrets/chaosknoten/grafana/loki_chaos_basic_auth", create=false, missing="error") }} diff --git a/resources/chaosknoten/grafana/nginx/redirect.conf b/resources/chaosknoten/grafana/nginx/redirect.conf new file mode 100644 index 0000000..28b265a --- /dev/null +++ b/resources/chaosknoten/grafana/nginx/redirect.conf @@ -0,0 +1,14 @@ +# partly generated 2022-01-08, Mozilla Guideline v5.6, nginx 1.17.7, OpenSSL 1.1.1k, intermediate configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.17.7&config=intermediate&openssl=1.1.1k&guideline=5.6 +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + return 301 https://$host$request_uri; + } + + location /.well-known/acme-challenge/ { + proxy_pass http://127.0.0.1:31820/.well-known/acme-challenge/; + } +} diff --git a/roles/nginx/meta/argument_specs.yaml b/roles/nginx/meta/argument_specs.yaml index 866cb81..f2cb1d7 100644 --- a/roles/nginx/meta/argument_specs.yaml +++ b/roles/nginx/meta/argument_specs.yaml @@ -34,3 +34,19 @@ argument_specs: type: str required: false default: "" + nginx__deploy_htpasswds: + type: bool + required: false + default: false + nginx__htpasswds: + type: list + elements: dict + required: false + default: [ ] + options: + name: + type: str + required: true + content: + type: str + required: true diff --git a/roles/nginx/tasks/main/04_config_deploy.yaml b/roles/nginx/tasks/main/04_config_deploy.yaml index 38dbfc1..7dba579 100644 --- a/roles/nginx/tasks/main/04_config_deploy.yaml +++ b/roles/nginx/tasks/main/04_config_deploy.yaml @@ -131,6 +131,20 @@ label: "{{ item.name }}" notify: Restart nginx +- name: Ensure all given htpasswd files are deployed + when: nginx__deploy_htpasswds + ansible.builtin.copy: + content: "{{ item.content }}" + dest: "/etc/nginx/{{ item.name }}.htpasswd" + mode: "0644" + owner: root + group: root + become: true + loop: "{{ nginx__htpasswds }}" + loop_control: + label: "{{ item.name }}" + notify: Restart nginx + - name: Add names with suffixes from `nginx__configurations` to `nginx__config_files_to_exist` fact ansible.builtin.set_fact: nginx__config_files_to_exist: "{{ nginx__config_files_to_exist + [ item.name + '.conf' ] }}" # noqa: jinja[spacing] From b0660deb71f1fbd62d71a1020a3a8b5eb1e97770 Mon Sep 17 00:00:00 2001 From: c6ristian Date: Mon, 28 Apr 2025 22:23:11 +0200 Subject: [PATCH 12/14] fix: nginx role add defaults for nginx__deploy_htpasswds and nginx__htpasswds --- roles/nginx/defaults/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/nginx/defaults/main.yaml b/roles/nginx/defaults/main.yaml index e4d4fb0..2e56dac 100644 --- a/roles/nginx/defaults/main.yaml +++ b/roles/nginx/defaults/main.yaml @@ -4,3 +4,5 @@ nginx__deploy_logging_conf: true nginx__configurations: [ ] nginx__use_custom_nginx_conf: false nginx__custom_nginx_conf: "" +nginx__deploy_htpasswds: false +nginx__htpasswds: [ ] From e21ff26f36999e9e357bddf8714056aff463cc8b Mon Sep 17 00:00:00 2001 From: c6ristian Date: Mon, 28 Apr 2025 23:02:13 +0200 Subject: [PATCH 13/14] fix: alertmanager the message template now just give out simple string if the list of alerts is to long --- .../alertmanager_alert_templates.tmpl | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl b/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl index 5318fb0..4651496 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl @@ -20,16 +20,25 @@ Links & Resources {{ define "alert-message.telegram.ccchh" }} -{{- if .Alerts.Firing }} -🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥 -{{ range .Alerts.Firing -}} -{{ template "alert-item.telegram.ccchh.internal" . }} -{{- end }} -{{- end }} -{{- if .Alerts.Resolved }} -✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅ -{{ range .Alerts.Resolved -}} -{{ template "alert-item.telegram.ccchh.internal" . }} -{{- end }} -{{- end }} + {{- if .Alerts.Firing }} + 🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥 + {{- if le (len .Alerts.Firing) 6 }} + {{- range .Alerts.Firing }} + {{ template "alert-item.telegram.ccchh.internal" . }} + {{- end }} + {{- else }} + There are too many alerts firing at once + {{- end }} + {{- end }} + + {{- if .Alerts.Resolved }} + ✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅ + {{- if le (len .Alerts.Resolved) 6 }} + {{- range .Alerts.Resolved }} + {{ template "alert-item.telegram.ccchh.internal" . }} + {{- end }} + {{- else }} + There are too many resolved alerts to list + {{- end }} + {{- end }} {{- end }} From e183f1a2c3b6516a640d61eeb9dc46038ea0c21b Mon Sep 17 00:00:00 2001 From: c6ristian Date: Wed, 30 Apr 2025 01:08:43 +0200 Subject: [PATCH 14/14] prometheus remote write with alloy using it --- collections/requirements.yaml | 1 + .../chaosknoten/host_vars/grafana.yaml | 81 +++++++++++++++++++ inventories/chaosknoten/hosts.yaml | 3 + playbooks/deploy.yaml | 8 ++ .../grafana/docker_compose/compose.yaml.j2 | 3 +- .../grafana/nginx/loki.hamburg.ccc.de.conf | 16 ++-- .../grafana/nginx/metrics.hamburg.ccc.de.conf | 55 +++++++++++++ .../grafana/nginx/metrics.htpasswd.j2 | 1 + 8 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf create mode 100644 resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2 diff --git a/collections/requirements.yaml b/collections/requirements.yaml index a24c121..cec061f 100644 --- a/collections/requirements.yaml +++ b/collections/requirements.yaml @@ -1,3 +1,4 @@ --- collections: - community.general + - grafana.grafana.alloy diff --git a/inventories/chaosknoten/host_vars/grafana.yaml b/inventories/chaosknoten/host_vars/grafana.yaml index 2e8cb71..c0f2ad9 100644 --- a/inventories/chaosknoten/host_vars/grafana.yaml +++ b/inventories/chaosknoten/host_vars/grafana.yaml @@ -20,6 +20,7 @@ certbot__acme_account_email_address: le-admin@hamburg.ccc.de certbot__certificate_domains: - "grafana.hamburg.ccc.de" - "loki.hamburg.ccc.de" + - "metrics.hamburg.ccc.de" certbot__new_cert_commands: - "systemctl reload nginx.service" @@ -30,6 +31,8 @@ nginx__deploy_htpasswds: true nginx__htpasswds: - name: loki content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/loki.htpasswd.j2') }}" + - name: metrics + content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2') }}" nginx__configurations: - name: redirectv6 content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/redirect.conf') }}" @@ -37,3 +40,81 @@ nginx__configurations: content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/grafana.hamburg.ccc.de.conf') }}" - name: loki.hamburg.ccc.de content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf') }}" + - name: metrics.hamburg.ccc.de + content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf') }}" + + +alloy_config: | + prometheus.remote_write "default" { + endpoint { + url = "https://metrics.hamburg.ccc.de/api/v1/write" + basic_auth { + username = "chaos" + password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/metrics_chaos', create=false, missing='error') }}" + } + } + } + loki.write "default" { + endpoint { + url = "https://loki.hamburg.ccc.de/loki/api/v1/push" + basic_auth { + username = "chaos" + password = "{{ lookup('community.general.passwordstore', 'noc/vm-secrets/chaosknoten/grafana/loki_chaos', create=false, missing='error') }}" + } + } + } + + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "systemd_unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "instance" + } + rule { + source_labels = ["__journal__transport"] + target_label = "systemd_transport" + } + rule { + source_labels = ["__journal_syslog_identifier"] + target_label = "syslog_identifier" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + loki.source.journal "read_journal" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + format_as_json = true + labels = {component = "loki.source.journal", host = "grafana", org = "ccchh"} + } + + logging { + level = "info" + } + prometheus.exporter.unix "local_system" { } + + prometheus.relabel "default" { + forward_to = [prometheus.remote_write.default.receiver] + rule { + replacement = "org" + target_label = "ccchh" + } + rule { + target_label = "host" + replacement = "grafana" + } + } + + prometheus.scrape "scrape_metrics" { + targets = prometheus.exporter.unix.local_system.targets + forward_to = [prometheus.relabel.default.receiver] + scrape_interval = "15s" + } diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml index 92185ed..ed0d042 100644 --- a/inventories/chaosknoten/hosts.yaml +++ b/inventories/chaosknoten/hosts.yaml @@ -177,3 +177,6 @@ proxmox_vm_template_hosts: ansible_pull_hosts: hosts: netbox: +alloy_hosts: + hosts: + grafana: diff --git a/playbooks/deploy.yaml b/playbooks/deploy.yaml index d7dcdac..952aeec 100644 --- a/playbooks/deploy.yaml +++ b/playbooks/deploy.yaml @@ -70,5 +70,13 @@ - "o=Docker,n=${distro_codename}" - "o=nginx,n=${distro_codename}" +- name: Ensure Alloy is installed and Setup on alloy_hosts + hosts: alloy_hosts + become: true + tasks: + - name: Setup Alloy + ansible.builtin.include_role: + name: grafana.grafana.alloy + - name: Run ensure_eh22_styleguide_dir Playbook ansible.builtin.import_playbook: ensure_eh22_styleguide_dir.yaml diff --git a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 index cbe6e81..3ef0a0c 100644 --- a/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/compose.yaml.j2 @@ -6,6 +6,7 @@ services: container_name: prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' + - '--web.enable-remote-write-receiver' ports: - 9090:9090 restart: unless-stopped @@ -54,7 +55,6 @@ services: volumes: - /dev/null:/etc/prometheus/pve.yml - loki: image: grafana/loki:3 container_name: loki @@ -71,3 +71,4 @@ volumes: prom_data: {} alertmanager_data: {} loki_data: {} + mimir_data: {} diff --git a/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf index f62dc4e..cb51782 100644 --- a/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf +++ b/resources/chaosknoten/grafana/nginx/loki.hamburg.ccc.de.conf @@ -3,13 +3,16 @@ server { allow 172.31.17.128/25; allow 212.12.51.128/28; allow 2a00:14b0:42:100::/56; + allow 2a00:14b0:4200:3380::/64; # Z9 allow 2a07:c480:0:100::/56; allow 2a07:c481:1::/48; + deny all; - listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:9099 ssl http2; - listen 172.31.17.145:9099 ssl http2; + listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:50051 ssl; + listen 172.31.17.145:50051 ssl; + http2 on; server_name loki.hamburg.ccc.de; @@ -27,7 +30,7 @@ server { # This is https in any case. proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Scope-OrgID $remote_user; - grpc_pass grpc://localhost:19009; + grpc_pass grpc://localhost:19099; } } @@ -36,13 +39,15 @@ server { allow 172.31.17.128/25; allow 212.12.51.128/28; allow 2a00:14b0:42:100::/56; + allow 2a00:14b0:4200:3380::/64; # Z9 allow 2a07:c480:0:100::/56; allow 2a07:c481:1::/48; deny all; - listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:3100 ssl http2; - listen 172.31.17.145:3100 ssl http2; + listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl; + listen 172.31.17.145:443 ssl; + http2 on; server_name loki.hamburg.ccc.de; @@ -62,7 +67,6 @@ server { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Port 3100; # This is https in any case. proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Scope-OrgID $remote_user; diff --git a/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf new file mode 100644 index 0000000..a431821 --- /dev/null +++ b/resources/chaosknoten/grafana/nginx/metrics.hamburg.ccc.de.conf @@ -0,0 +1,55 @@ +server { + # Wieske + allow 172.31.17.128/25; + allow 212.12.51.128/28; + allow 2a00:14b0:42:100::/56; + allow 2a00:14b0:4200:3380::/64; + # Z9 + allow 2a07:c480:0:100::/56; + allow 2a07:c481:1::/48; + deny all; + + listen [2a00:14b0:4200:3380:0000:5a5f:1dbc:6a39]:443 ssl; + listen 172.31.17.145:443 ssl; + http2 on; + + server_name metrics.hamburg.ccc.de; + + client_body_buffer_size 32k; + + ssl_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/metrics.hamburg.ccc.de/privkey.pem; + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/letsencrypt/live/metrics.hamburg.ccc.de/chain.pem; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + auth_basic "metrics"; + auth_basic_user_file metrics.htpasswd; + + location /api/v1/write { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Port 3100; + # This is https in any case. + proxy_set_header X-Forwarded-Proto https; + + proxy_pass http://127.0.0.1:9090; + } + + location /ready { + rewrite ^ /-/ready break; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # This is https in any case. + proxy_set_header X-Forwarded-Proto https; + + proxy_pass http://127.0.0.1:9090; + } +} diff --git a/resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2 b/resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2 new file mode 100644 index 0000000..1dfcd6c --- /dev/null +++ b/resources/chaosknoten/grafana/nginx/metrics.htpasswd.j2 @@ -0,0 +1 @@ +chaos:{{ lookup("community.general.passwordstore", "noc/vm-secrets/chaosknoten/grafana/metrics_chaos_basic_auth", create=false, missing="error") }}