Compare commits
3 commits
89d9122420
...
73e77bde70
| Author | SHA1 | Date | |
|---|---|---|---|
|
73e77bde70 |
|||
|
6b19f69135 |
|||
|
b0347d64bf |
8 changed files with 33 additions and 32 deletions
|
|
@ -196,7 +196,6 @@ matrix-intern IN A 172.31.17.150
|
||||||
; have this for compatibility (like references in CI)
|
; have this for compatibility (like references in CI)
|
||||||
public-web-static-intern IN AAAA 2a00:14b0:42:102::17
|
public-web-static-intern IN AAAA 2a00:14b0:42:102::17
|
||||||
git-intern IN A 172.31.17.154
|
git-intern IN A 172.31.17.154
|
||||||
woodpecker-intern IN A 172.31.17.160
|
|
||||||
penpot-intern IN A 172.31.17.162
|
penpot-intern IN A 172.31.17.162
|
||||||
forgejo-runner-builder IN A 172.31.17.202
|
forgejo-runner-builder IN A 172.31.17.202
|
||||||
renovate-forgejo IN A 172.31.17.163
|
renovate-forgejo IN A 172.31.17.163
|
||||||
|
|
@ -275,7 +274,6 @@ matrix IN CNAME public-reverse-proxy
|
||||||
mas IN CNAME public-reverse-proxy
|
mas IN CNAME public-reverse-proxy
|
||||||
element-admin IN CNAME public-reverse-proxy
|
element-admin IN CNAME public-reverse-proxy
|
||||||
netbox IN CNAME public-reverse-proxy
|
netbox IN CNAME public-reverse-proxy
|
||||||
woodpecker IN CNAME public-reverse-proxy
|
|
||||||
onlyoffice IN CNAME public-reverse-proxy
|
onlyoffice IN CNAME public-reverse-proxy
|
||||||
pad IN CNAME public-reverse-proxy
|
pad IN CNAME public-reverse-proxy
|
||||||
pretalx IN CNAME public-reverse-proxy
|
pretalx IN CNAME public-reverse-proxy
|
||||||
|
|
|
||||||
|
|
@ -129,7 +129,7 @@ groups:
|
||||||
# General high disk read and write rate alerts.
|
# General high disk read and write rate alerts.
|
||||||
# Excluding: hypervisor hosts, CI hosts
|
# Excluding: hypervisor hosts, CI hosts
|
||||||
- alert: HostUnusualDiskReadRate
|
- alert: HostUnusualDiskReadRate
|
||||||
expr: (sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename=~".+", nodename!="forgejo-actions-runner", nodename!="woodpecker", nodename!="chaosknoten"}
|
expr: (sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename=~".+", nodename!="forgejo-actions-runner", nodename!="chaosknoten"}
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
|
@ -137,7 +137,7 @@ groups:
|
||||||
summary: Host unusual disk read rate (instance {{ $labels.instance }})
|
summary: Host unusual disk read rate (instance {{ $labels.instance }})
|
||||||
description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}"
|
description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}"
|
||||||
- alert: HostUnusualDiskWriteRate
|
- alert: HostUnusualDiskWriteRate
|
||||||
expr: (sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename=~".+", nodename!="forgejo-actions-runner", nodename!="woodpecker", nodename!="chaosknoten"}
|
expr: (sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename=~".+", nodename!="forgejo-actions-runner", nodename!="chaosknoten"}
|
||||||
for: 2m
|
for: 2m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
|
@ -147,7 +147,7 @@ groups:
|
||||||
# CI hosts high disk read and write alerts.
|
# CI hosts high disk read and write alerts.
|
||||||
# Longer intervals to account for disk intensive CI tasks.
|
# Longer intervals to account for disk intensive CI tasks.
|
||||||
- alert: CIHostUnusualDiskReadRate
|
- alert: CIHostUnusualDiskReadRate
|
||||||
expr: (sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename="forgejo-actions-runner", nodename="woodpecker"}
|
expr: (sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename="forgejo-actions-runner"}
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
|
@ -155,7 +155,7 @@ groups:
|
||||||
summary: CI host unusual disk read rate for 10 min (instance {{ $labels.instance }})
|
summary: CI host unusual disk read rate for 10 min (instance {{ $labels.instance }})
|
||||||
description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}"
|
description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}"
|
||||||
- alert: VirtualHostUnusualDiskWriteRate
|
- alert: VirtualHostUnusualDiskWriteRate
|
||||||
expr: (sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename="forgejo-actions-runner", nodename="woodpecker"}
|
expr: (sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename="forgejo-actions-runner"}
|
||||||
for: 4m
|
for: 4m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,6 @@ map $host $upstream_acme_challenge_host {
|
||||||
eh20.hamburg.ccc.de public-web-static.hosts.hamburg.ccc.de:31820;
|
eh20.hamburg.ccc.de public-web-static.hosts.hamburg.ccc.de:31820;
|
||||||
hacker.tours public-web-static.hosts.hamburg.ccc.de:31820;
|
hacker.tours public-web-static.hosts.hamburg.ccc.de:31820;
|
||||||
staging.hacker.tours public-web-static.hosts.hamburg.ccc.de:31820;
|
staging.hacker.tours public-web-static.hosts.hamburg.ccc.de:31820;
|
||||||
woodpecker.hamburg.ccc.de 172.31.17.160:31820;
|
|
||||||
design.hamburg.ccc.de 172.31.17.162:31820;
|
design.hamburg.ccc.de 172.31.17.162:31820;
|
||||||
hydra.hamburg.ccc.de 172.31.17.163:31820;
|
hydra.hamburg.ccc.de 172.31.17.163:31820;
|
||||||
ntfy.hamburg.ccc.de ntfy.hosts.hamburg.ccc.de:31820;
|
ntfy.hamburg.ccc.de ntfy.hosts.hamburg.ccc.de:31820;
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,6 @@ stream {
|
||||||
eh20.hamburg.ccc.de public-web-static.hosts.hamburg.ccc.de:8443;
|
eh20.hamburg.ccc.de public-web-static.hosts.hamburg.ccc.de:8443;
|
||||||
hacker.tours public-web-static.hosts.hamburg.ccc.de:8443;
|
hacker.tours public-web-static.hosts.hamburg.ccc.de:8443;
|
||||||
staging.hacker.tours public-web-static.hosts.hamburg.ccc.de:8443;
|
staging.hacker.tours public-web-static.hosts.hamburg.ccc.de:8443;
|
||||||
woodpecker.hamburg.ccc.de 172.31.17.160:8443;
|
|
||||||
design.hamburg.ccc.de 172.31.17.162:8443;
|
design.hamburg.ccc.de 172.31.17.162:8443;
|
||||||
hydra.hamburg.ccc.de 172.31.17.163:8443;
|
hydra.hamburg.ccc.de 172.31.17.163:8443;
|
||||||
cfp.eh22.easterhegg.eu pretalx.hosts.hamburg.ccc.de:8443;
|
cfp.eh22.easterhegg.eu pretalx.hosts.hamburg.ccc.de:8443;
|
||||||
|
|
|
||||||
|
|
@ -294,14 +294,6 @@ endpoints:
|
||||||
- "[CERTIFICATE_EXPIRATION] > 48h"
|
- "[CERTIFICATE_EXPIRATION] > 48h"
|
||||||
- "[BODY] == pat(*CCCHH Wiki*)"
|
- "[BODY] == pat(*CCCHH Wiki*)"
|
||||||
|
|
||||||
- name: Woodpecker
|
|
||||||
url: "https://woodpecker.hamburg.ccc.de/"
|
|
||||||
<<: *services_chaosknoten_defaults
|
|
||||||
conditions:
|
|
||||||
- "[STATUS] == 200"
|
|
||||||
- "[CERTIFICATE_EXPIRATION] > 48h"
|
|
||||||
- "[BODY] == pat(*Woodpecker*)"
|
|
||||||
|
|
||||||
- name: Zammad
|
- name: Zammad
|
||||||
url: "https://zammad.hamburg.ccc.de/"
|
url: "https://zammad.hamburg.ccc.de/"
|
||||||
<<: *services_chaosknoten_defaults
|
<<: *services_chaosknoten_defaults
|
||||||
|
|
|
||||||
7
roles/renovate/files/renovate-cleanup.service
Normal file
7
roles/renovate/files/renovate-cleanup.service
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
[Unit]
|
||||||
|
Description=renovate cleanup (delete docker volume)
|
||||||
|
Conflicts=renovate.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/bin/docker volume rm renovate
|
||||||
9
roles/renovate/files/renovate-cleanup.timer
Normal file
9
roles/renovate/files/renovate-cleanup.timer
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=renovate cleanup (delete docker volume) running daily
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# @daily with 10 minute offset
|
||||||
|
OnCalendar=*-*-* 00:10
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
@ -16,31 +16,28 @@
|
||||||
mode: "0640"
|
mode: "0640"
|
||||||
become: true
|
become: true
|
||||||
|
|
||||||
- name: ensure systemd service exists
|
- name: ensure systemd services and timers exist
|
||||||
ansible.builtin.copy:
|
ansible.builtin.copy:
|
||||||
src: renovate.service
|
src: "{{ item }}"
|
||||||
dest: /etc/systemd/system/renovate.service
|
dest: "/etc/systemd/system/{{ item }}"
|
||||||
owner: root
|
owner: root
|
||||||
group: root
|
group: root
|
||||||
mode: "0644"
|
mode: "0644"
|
||||||
become: true
|
become: true
|
||||||
|
loop:
|
||||||
|
- renovate.service
|
||||||
|
- renovate-cleanup.service
|
||||||
|
- renovate.timer
|
||||||
|
- renovate-cleanup.timer
|
||||||
notify:
|
notify:
|
||||||
- systemd daemon reload
|
- systemd daemon reload
|
||||||
|
|
||||||
- name: ensure systemd timer exists
|
- name: ensure systemd timers are started and enabled
|
||||||
ansible.builtin.copy:
|
|
||||||
src: renovate.timer
|
|
||||||
dest: /etc/systemd/system/renovate.timer
|
|
||||||
owner: root
|
|
||||||
group: root
|
|
||||||
mode: "0644"
|
|
||||||
become: true
|
|
||||||
notify:
|
|
||||||
- systemd daemon reload
|
|
||||||
|
|
||||||
- name: ensure systemd timer is started and enabled
|
|
||||||
ansible.builtin.systemd_service:
|
ansible.builtin.systemd_service:
|
||||||
name: renovate.timer
|
name: "{{ item }}"
|
||||||
state: started
|
state: started
|
||||||
enabled: true
|
enabled: true
|
||||||
|
loop:
|
||||||
|
- renovate.timer
|
||||||
|
- renovate-cleanup.timer
|
||||||
become: true
|
become: true
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue