netbox(host): setup ansible_pull for host and define common config

Define common ansible_pull configuration for chaosknoten inventory hosts
and setup ansible_pull for NetBox host.

inventories/chaosknoten/host_vars/grafana.yaml

@@ -10,6 +10,8 @@ docker_compose__configuration_files:
     content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2') }}"
   - name: prometheus_alerts.rules.yaml
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml') }}"
+  - name: prometheus_alerts-fux.rules.yaml
+    content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/prometheus_alerts-fux.rules.yaml') }}"
   - name: alertmanager_alert_templates.tmpl
     content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl') }}"
   - name: loki.yaml

inventories/chaosknoten/hosts.yaml

@@ -63,6 +63,10 @@ all:
       ansible_host: ntfy-intern.hamburg.ccc.de
       ansible_user: chaos
       ansible_ssh_common_args: -J ssh://chaos@public-reverse-proxy.hamburg.ccc.de
+    sunders:
+      ansible_host: sunders-intern.hamburg.ccc.de
+      ansible_user: chaos
+      ansible_ssh_common_args: -J ssh://chaos@public-reverse-proxy.hamburg.ccc.de
 hypervisors:
   hosts:
     chaosknoten:
@@ -84,6 +88,7 @@ base_config_hosts:
     wiki:
     zammad:
     ntfy:
+    sunders:
 docker_compose_hosts:
   hosts:
     ccchoir:
@@ -163,6 +168,7 @@ infrastructure_authorized_keys_hosts:
     wiki:
     zammad:
     ntfy:
+    sunders:
 wiki_hosts:
   hosts:
     eh22-wiki:

inventories/z9/host_vars/waybackproxy.yaml

@@ -0,0 +1,7 @@
+docker_compose__compose_file_content: "{{ lookup('ansible.builtin.template', 'resources/z9/waybackproxy/docker_compose/compose.yaml.j2') }}"
+docker_compose__configuration_files: [ ]
+
+nginx__version_spec: ""
+nginx__configurations:
+  - name: waybackproxy.ccchh.net
+    content: "{{ lookup('ansible.builtin.file', 'resources/z9/waybackproxy/nginx/waybackproxy.ccchh.net.conf') }}"

inventories/z9/hosts.yaml

@@ -11,6 +11,9 @@ all:
       ansible_user: chaos
     thinkcccore0:
       ansible_host: thinkcccore0.z9.ccchh.net
+    waybackproxy:
+      ansible_host: waybackproxy.ccchh.net
+      ansible_user: chaos
     yate:
       ansible_host: yate.ccchh.net
       ansible_user: chaos
@@ -20,6 +23,7 @@ certbot_hosts:
 docker_compose_hosts:
   hosts:
     dooris:
+    waybackproxy:
     yate:
 foobazdmx_hosts:
   hosts:
@@ -32,11 +36,13 @@ infrastructure_authorized_keys_hosts:
     dooris:
     light:
     authoritative-dns:
+    waybackproxy:
     yate:
 nginx_hosts:
   hosts:
     dooris:
     light:
+    waybackproxy:
 ola_hosts:
   hosts:
     light:

resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2

@@ -79,7 +79,7 @@ receivers:
   - name: "email-fux-critical"
     email_configs:
       - send_resolved: true
-        to: "stb@lassitu.de"
+        to: "stb@lassitu.de,fux@zimdahl.org"
         from: "alert-manager@hamburg.ccc.de"
         smarthost: "cow.hamburg.ccc.de:587"
         auth_username: "alert-manager@hamburg.ccc.de"

resources/chaosknoten/grafana/docker_compose/compose.yaml.j2

@@ -14,6 +14,7 @@ services:
     volumes:
       - ./configs/prometheus.yml:/etc/prometheus/prometheus.yml
       - ./configs/prometheus_alerts.rules.yaml:/etc/prometheus/rules/alerts.rules.yaml
+      - ./configs/prometheus_alerts-fux.rules.yaml:/etc/prometheus/rules/alerts-fux.rules.yaml
       - prom_data:/prometheus
 
   alertmanager:

resources/chaosknoten/grafana/docker_compose/grafana-datasource.yml

@@ -18,4 +18,3 @@ datasources:
       httpHeaderName1: "X-Scope-OrgID"
     secureJsonData:
       httpHeaderValue1: "chaos"
-

resources/chaosknoten/grafana/docker_compose/prometheus_alerts-fux.rules.yaml

@@ -0,0 +1,41 @@
+groups:
+  - name: Fux-Generic
+    rules:
+      - alert: HostJobFlaky
+        expr: group by(instance, job) (changes(up{org="fux"}[24h]) > 7)
+        for: 0m
+        labels:
+          severity: info
+          org: fux
+        annotations:
+          summary: Job {{ $labels.job }} flaky on (instance {{ $labels.instance }})
+          description: "The job {{ $labels.job }} on target: {{ $labels.instance }} has been flaky over the last 24 hours."
+  - name: Fux-SNMP
+    rules:
+      - alert: SnmpTargetMissing
+        expr: up{job=~".*snmp.*", org="fux"} == 0
+        for: 15m
+        labels:
+          severity: critical
+          org: fux
+        annotations:
+          summary: SNMP target missing (instance {{ $labels.instance }})
+          description: "SNMP target: {{ $labels.instance }} has disappeared for more the 15 min."
+  - name: Fux-DHCP
+    rules:
+      - alert: DhcpFuxSharedFailed
+        expr: script_success{script="check_dhcp_fux_shared"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: DHCP for Fux Shared stoped working
+          description: "No DHCP lease for the Fux Shared range was received \n V"
+      - alert: DhcpFuxAdminFailed
+        expr: script_success{script_success="check_dhcp_fux_admin"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: DHCP for Fux Admin stoped working
+          description: "No DHCP lease for the Fux Admin range was received"

resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml

@@ -410,7 +410,7 @@ groups:
           summary: Prometheus job missing (instance {{ $labels.instance }})
           description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}"
       - alert: PrometheusTargetMissing
-        expr: up == 0
+        expr: up{job!~"snmp|noc_room_temp"} == 0
         for: 0m
         labels:
           severity: critical
@@ -418,7 +418,7 @@ groups:
           summary: Prometheus target missing (instance {{ $labels.instance }})
           description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}"
       - alert: PrometheusAllTargetsMissing
-        expr: sum by (job) (up) == 0
+        expr: sum by (job) (up{job!~"snmp|noc_room_temp"}) == 0
         for: 0m
         labels:
           severity: critical

resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf

@@ -17,6 +17,8 @@ map $host $upstream_acme_challenge_host {
     invite.hamburg.ccc.de 172.31.17.144:31820;
     keycloak-admin.hamburg.ccc.de 172.31.17.144:31820;
     matrix.hamburg.ccc.de 172.31.17.150:31820;
+    mas.hamburg.ccc.de 172.31.17.150:31820;
+    element-admin.hamburg.ccc.de 172.31.17.151:31820;
     netbox.hamburg.ccc.de 172.31.17.167:31820;
     onlyoffice.hamburg.ccc.de 172.31.17.147:31820;
     pad.hamburg.ccc.de 172.31.17.141:31820;

resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf

@@ -34,6 +34,8 @@ stream {
         staging.hackertours.hamburg.ccc.de 172.31.17.151:8443;
         netbox.hamburg.ccc.de 172.31.17.167:8443;
         matrix.hamburg.ccc.de 172.31.17.150:8443;
+        mas.hamburg.ccc.de 172.31.17.150:8443;
+        element-admin.hamburg.ccc.de 172.31.17.151:8443;
         element.hamburg.ccc.de 172.31.17.151:8443;
         branding-resources.hamburg.ccc.de 172.31.17.151:8443;
         www.hamburg.ccc.de 172.31.17.151:8443;

resources/z9/waybackproxy/docker_compose/compose.yaml.j2

@@ -0,0 +1,10 @@
+services:
+  # https://github.com/richardg867/WaybackProxy
+  waybackproxy:
+    image: cttynul/waybackproxy:latest
+    environment:
+      DATE: 19990101
+      DATE_TOLERANCE: 730
+    ports:
+      - "1999:8888"
+    restart: unless-stopped

resources/z9/waybackproxy/nginx/waybackproxy.ccchh.net.conf

@@ -0,0 +1,5 @@
+# TODO: set up caching proxy
+
+# server {
+#     listen 1999
+# }