From e616c55edbcc95b915b35273051a0fb0ac91a3f8 Mon Sep 17 00:00:00 2001 From: June Date: Tue, 14 Oct 2025 02:51:09 +0200 Subject: [PATCH 1/4] WIP: ansible_pull(role): add failure notifications --- inventories/chaosknoten/group_vars/all.yaml | 2 ++ roles/ansible_pull/README.md | 1 + roles/ansible_pull/meta/argument_specs.yaml | 3 +++ roles/ansible_pull/meta/main.yaml | 3 +++ roles/ansible_pull/tasks/main.yaml | 18 +++++++++++++++--- .../ansible-pull-failure-notify.service.j2 | 9 +++++++++ .../ansible-pull-failure-notify.sh.j2 | 17 +++++++++++++++++ .../templates/ansible-pull.service.j2 | 1 + 8 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 roles/ansible_pull/meta/main.yaml create mode 100644 roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 create mode 100644 roles/ansible_pull/templates/ansible-pull-failure-notify.sh.j2 diff --git a/inventories/chaosknoten/group_vars/all.yaml b/inventories/chaosknoten/group_vars/all.yaml index 3612ebc..fab32d2 100644 --- a/inventories/chaosknoten/group_vars/all.yaml +++ b/inventories/chaosknoten/group_vars/all.yaml @@ -3,7 +3,9 @@ ansible_pull__repo_url: https://git.hamburg.ccc.de/CCCHH/ansible-infra.git ansible_pull__inventory: inventories/chaosknoten ansible_pull__playbook: playbooks/maintenance.yaml +ansible_pull__checkout: ansible_pull_notify ansible_pull__timer_on_calendar: "*-*-* 04:00:00 Europe/Berlin" +ansible_pull__failure_notification_address: june+test@jsts.xyz ansible_pull__timer_randomized_delay_sec: 30min # msmtp diff --git a/roles/ansible_pull/README.md b/roles/ansible_pull/README.md index f31c552..cf90e60 100644 --- a/roles/ansible_pull/README.md +++ b/roles/ansible_pull/README.md @@ -13,6 +13,7 @@ Should work on Debian-based distributions. - `ansible_pull__inventory`: The inventory to use. - `ansible_pull__playbook`: The playbook to run. - `ansible_pull__timer_on_calendar`: When to run the playbook. This is the argument to a systemd timers OnCalendar. See the systemd.time man page for reference. +- `ansible_pull__failure_notification_address`: The address to send the failure notification to. ## Optional Arguments diff --git a/roles/ansible_pull/meta/argument_specs.yaml b/roles/ansible_pull/meta/argument_specs.yaml index e5c88af..682fdcd 100644 --- a/roles/ansible_pull/meta/argument_specs.yaml +++ b/roles/ansible_pull/meta/argument_specs.yaml @@ -16,6 +16,9 @@ argument_specs: ansible_pull__timer_on_calendar: type: str required: true + ansible_pull__failure_notification_address: + type: str + required: true ansible_pull__user: type: str required: false diff --git a/roles/ansible_pull/meta/main.yaml b/roles/ansible_pull/meta/main.yaml new file mode 100644 index 0000000..25aaf90 --- /dev/null +++ b/roles/ansible_pull/meta/main.yaml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: msmtp diff --git a/roles/ansible_pull/tasks/main.yaml b/roles/ansible_pull/tasks/main.yaml index 53fc219..eff8cb0 100644 --- a/roles/ansible_pull/tasks/main.yaml +++ b/roles/ansible_pull/tasks/main.yaml @@ -15,6 +15,15 @@ virtualenv: /usr/local/lib/ansible_pull_venv become: true +- name: ensure ansible-pull-failure-notify script installation exists + ansible.builtin.template: + src: ansible-pull-failure-notify.sh.j2 + dest: /usr/local/sbin/ansible-pull-failure-notify.sh + owner: root + group: root + mode: "0755" + become: true + - name: ensure secrets directory exists ansible.builtin.file: path: /etc/ansible_pull_secrets @@ -33,14 +42,17 @@ group: "{{ ansible_pull__user }}" become: true -- name: ensure systemd service exists +- name: ensure systemd services exists ansible.builtin.template: - src: ansible-pull.service.j2 - dest: /etc/systemd/system/ansible-pull.service + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" owner: root group: root mode: "0644" become: true + loop: + - ansible-pull.service + - ansible-pull-failure-notify.service notify: - systemd daemon reload diff --git a/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 b/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 new file mode 100644 index 0000000..3c95bd1 --- /dev/null +++ b/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=ansible-pull failure notifier +After=ansible-pull.service +Wants=ansible-pull.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ansible-pull-failure-notify.sh +User=root diff --git a/roles/ansible_pull/templates/ansible-pull-failure-notify.sh.j2 b/roles/ansible_pull/templates/ansible-pull-failure-notify.sh.j2 new file mode 100644 index 0000000..fe7dbc8 --- /dev/null +++ b/roles/ansible_pull/templates/ansible-pull-failure-notify.sh.j2 @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Ideally we would use --invocation instead of --since, but this isn't supported in the systemd version Debian 12 ships. +ANSIBLE_PULL_LOG=$(journalctl --unit=ansible-pull --identifier=ansible-pull --since=-6h --output=cat) + +MESSAGE="Subject: [{{ inventory_hostname }}] ansible-pull: execution failure + +An error occured during the ansible-pull execution. + +Logs: +""$ANSIBLE_PULL_LOG"" + +To view the logs yourself run: +journalctl --unit=ansible-pull --identifier=ansible-pull -e +" + +printf "$MESSAGE" | msmtp '{{ ansible_pull__failure_notification_address }}' diff --git a/roles/ansible_pull/templates/ansible-pull.service.j2 b/roles/ansible_pull/templates/ansible-pull.service.j2 index 588741c..87051a3 100644 --- a/roles/ansible_pull/templates/ansible-pull.service.j2 +++ b/roles/ansible_pull/templates/ansible-pull.service.j2 @@ -16,3 +16,4 @@ ExecStart=/usr/local/lib/ansible_pull_venv/bin/ansible-pull \ User={{ ansible_pull__user }} # Reboot, if /var/run/reboot-required or /var/run/ansible-reboot-required exist. ExecStartPost=/usr/bin/bash -c 'if [ -e /var/run/reboot-required ] || [ -e /var/run/ansible-reboot-required ]; then sudo systemctl reboot; fi' +OnFailure=ansible-pull-failure-notify.service From baca447578aa6cb96f9a96decb60ca1d843f6d86 Mon Sep 17 00:00:00 2001 From: June Date: Tue, 14 Oct 2025 02:52:34 +0200 Subject: [PATCH 2/4] test failure --- inventories/chaosknoten/group_vars/all.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/inventories/chaosknoten/group_vars/all.yaml b/inventories/chaosknoten/group_vars/all.yaml index fab32d2..e81690b 100644 --- a/inventories/chaosknoten/group_vars/all.yaml +++ b/inventories/chaosknoten/group_vars/all.yaml @@ -1,7 +1,6 @@ # ansible_pull # ansible_pull__age_private_key needs to be defined per host (probably HOST.sops.yaml). ansible_pull__repo_url: https://git.hamburg.ccc.de/CCCHH/ansible-infra.git -ansible_pull__inventory: inventories/chaosknoten ansible_pull__playbook: playbooks/maintenance.yaml ansible_pull__checkout: ansible_pull_notify ansible_pull__timer_on_calendar: "*-*-* 04:00:00 Europe/Berlin" From c0c101fef3eb239278b22467159f80f4ca3f7854 Mon Sep 17 00:00:00 2001 From: June Date: Tue, 14 Oct 2025 02:57:28 +0200 Subject: [PATCH 3/4] fix onfailure --- roles/ansible_pull/templates/ansible-pull.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ansible_pull/templates/ansible-pull.service.j2 b/roles/ansible_pull/templates/ansible-pull.service.j2 index 87051a3..0f80907 100644 --- a/roles/ansible_pull/templates/ansible-pull.service.j2 +++ b/roles/ansible_pull/templates/ansible-pull.service.j2 @@ -2,6 +2,7 @@ Description=ansible-pull for configuration and maintenance After=network-online.target Wants=network-online.target +OnFailure=ansible-pull-failure-notify.service [Service] Type=oneshot @@ -16,4 +17,3 @@ ExecStart=/usr/local/lib/ansible_pull_venv/bin/ansible-pull \ User={{ ansible_pull__user }} # Reboot, if /var/run/reboot-required or /var/run/ansible-reboot-required exist. ExecStartPost=/usr/bin/bash -c 'if [ -e /var/run/reboot-required ] || [ -e /var/run/ansible-reboot-required ]; then sudo systemctl reboot; fi' -OnFailure=ansible-pull-failure-notify.service From 2f696c41544e99cdb3781560c4156170ccc85886 Mon Sep 17 00:00:00 2001 From: June Date: Tue, 14 Oct 2025 03:25:06 +0200 Subject: [PATCH 4/4] fix notify unit --- .../templates/ansible-pull-failure-notify.service.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 b/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 index 3c95bd1..fa5d471 100644 --- a/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 +++ b/roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2 @@ -1,7 +1,5 @@ [Unit] Description=ansible-pull failure notifier -After=ansible-pull.service -Wants=ansible-pull.service [Service] Type=oneshot