WIP: ansible_pull(role): add failure notifications

inventories/chaosknoten/group_vars/all.yaml

@@ -3,7 +3,9 @@
 ansible_pull__repo_url: https://git.hamburg.ccc.de/CCCHH/ansible-infra.git
 ansible_pull__inventory: inventories/chaosknoten
 ansible_pull__playbook: playbooks/maintenance.yaml
+ansible_pull__checkout: ansible_pull_notify
 ansible_pull__timer_on_calendar: "*-*-* 04:00:00 Europe/Berlin"
+ansible_pull__failure_notification_address: june+test@jsts.xyz
 ansible_pull__timer_randomized_delay_sec: 30min
 
 # msmtp

roles/ansible_pull/README.md

@@ -13,6 +13,7 @@ Should work on Debian-based distributions.
 - `ansible_pull__inventory`: The inventory to use.
 - `ansible_pull__playbook`: The playbook to run.
 - `ansible_pull__timer_on_calendar`: When to run the playbook. This is the argument to a systemd timers OnCalendar. See the systemd.time man page for reference.
+- `ansible_pull__failure_notification_address`: The address to send the failure notification to.
 
 ## Optional Arguments
 

roles/ansible_pull/meta/argument_specs.yaml

@@ -16,6 +16,9 @@ argument_specs:
       ansible_pull__timer_on_calendar:
         type: str
         required: true
+      ansible_pull__failure_notification_address:
+        type: str
+        required: true
       ansible_pull__user:
         type: str
         required: false

roles/ansible_pull/meta/main.yaml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: msmtp

roles/ansible_pull/tasks/main.yaml

@@ -15,6 +15,15 @@
     virtualenv: /usr/local/lib/ansible_pull_venv
   become: true
 
+- name: ensure ansible-pull-failure-notify script installation exists
+  ansible.builtin.template:
+    src: ansible-pull-failure-notify.sh.j2
+    dest: /usr/local/sbin/ansible-pull-failure-notify.sh
+    owner: root
+    group: root
+    mode: "0755"
+  become: true
+
 - name: ensure secrets directory exists
   ansible.builtin.file:
     path: /etc/ansible_pull_secrets
@@ -33,14 +42,17 @@
     group: "{{ ansible_pull__user }}"
   become: true
 
-- name: ensure systemd service exists
+- name: ensure systemd services exists
   ansible.builtin.template:
-    src: ansible-pull.service.j2
+    src: "{{ item }}.j2"
-    dest: /etc/systemd/system/ansible-pull.service
+    dest: "/etc/systemd/system/{{ item }}"
     owner: root
     group: root
     mode: "0644"
   become: true
+  loop:
+    - ansible-pull.service
+    - ansible-pull-failure-notify.service
   notify:
     - systemd daemon reload
 

roles/ansible_pull/templates/ansible-pull-failure-notify.service.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=ansible-pull failure notifier
+After=ansible-pull.service
+Wants=ansible-pull.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/ansible-pull-failure-notify.sh
+User=root

roles/ansible_pull/templates/ansible-pull-failure-notify.sh.j2

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Ideally we would use --invocation instead of --since, but this isn't supported in the systemd version Debian 12 ships.
+ANSIBLE_PULL_LOG=$(journalctl --unit=ansible-pull --identifier=ansible-pull --since=-6h --output=cat)
+
+MESSAGE="Subject: [{{ inventory_hostname }}] ansible-pull: execution failure
+
+An error occured during the ansible-pull execution.
+
+Logs:
+""$ANSIBLE_PULL_LOG""
+
+To view the logs yourself run:
+journalctl --unit=ansible-pull --identifier=ansible-pull -e
+"
+
+printf "$MESSAGE" | msmtp '{{ ansible_pull__failure_notification_address }}'

roles/ansible_pull/templates/ansible-pull.service.j2

@@ -16,3 +16,4 @@ ExecStart=/usr/local/lib/ansible_pull_venv/bin/ansible-pull \
 User={{ ansible_pull__user }}
 # Reboot, if /var/run/reboot-required or /var/run/ansible-reboot-required exist.
 ExecStartPost=/usr/bin/bash -c 'if [ -e /var/run/reboot-required ] || [ -e /var/run/ansible-reboot-required ]; then sudo systemctl reboot; fi'
+OnFailure=ansible-pull-failure-notify.service