diff --git a/.sops.yaml b/.sops.yaml
index 5bce7ef..d77d8fd 100644
--- a/.sops.yaml
+++ b/.sops.yaml
@@ -162,6 +162,21 @@ creation_rules:
- *admin_gpg_c6ristian
- *admin_gpg_lilly
- *admin_gpg_langoor
+ - path_regex: inventories/chaosknoten/host_vars/ntfy.*
+ key_groups:
+ - pgp:
+ - *admin_gpg_djerun
+ - *admin_gpg_stb
+ - *admin_gpg_jtbx
+ - *admin_gpg_yuri
+ - *admin_gpg_june
+ - *admin_gpg_haegar
+ - *admin_gpg_dario
+ - *admin_gpg_echtnurich
+ - *admin_gpg_max
+ - *admin_gpg_c6ristian
+ - *admin_gpg_lilly
+ - *admin_gpg_langoor
- path_regex: inventories/z9/host_vars/dooris.*
key_groups:
- pgp:
diff --git a/inventories/chaosknoten/host_vars/ntfy.sops.yaml b/inventories/chaosknoten/host_vars/ntfy.sops.yaml
new file mode 100644
index 0000000..2cb9b2a
--- /dev/null
+++ b/inventories/chaosknoten/host_vars/ntfy.sops.yaml
@@ -0,0 +1,235 @@
+secret__loki_chaos: ENC[AES256_GCM,data:LWFTOyER+m021ogmXYBrcr/2fUe3XuZhs5ho0KbM,iv:808LWnSUAPeclhsIgOyR6SutTvJGOu7mrGaVayo7v8M=,tag:f2WCPyUESfMiGDQ4Km5Dyw==,type:str]
+secret__metrics_chaos: ENC[AES256_GCM,data:lAepzCI4pwkF8KiGYzGnC4dPASdHDn+LfbJTFSvt,iv:EUW+CGeYUqhY4G1kb2bbU16j9iLwABHfRCdn2vac5gY=,tag:IcyscB9lZuZgC04XTxDb5w==,type:str]
+ntfy:
+ user:
+ admin: ENC[AES256_GCM,data:kwGLrQXBiqKRoHkStGzYiC0fbcGgQHdZrrk9NyZtcZcI4nrKTGx1sxrHOMI=,iv:ACrBFMOP6rkfshOgB+a32TFWH1OKhQaoHcYgwHx+tao=,tag:2QTWmH/vAzIWAjaOHOkrXg==,type:str]
+ uwrite: ENC[AES256_GCM,data:Jijz+zCPpzSaIEo0xhicKlMhWSewJNJ9GXJGYuohq1E=,iv:gnjEX3N0txcBIkJm5bOs4JfKVsdi5URgoMAmquCMqKQ=,tag:Fip0hA52NeaMODb9XxjInQ==,type:str]
+ uread: ENC[AES256_GCM,data:ZODLyYx15c/rPzKexoLURwA=,iv:WqUrXexY/RBAseUwiLPBVYpA5zqJeYBW8mmcvPvjtyI=,tag:SjB4OaTgIaVKHDe4JjDN3Q==,type:str]
+sops:
+ lastmodified: "2025-06-02T16:34:49Z"
+ mac: ENC[AES256_GCM,data:C74LONrD83loeeJpdtwd4qW9tB+hJM5B3/gJ+uNNYh0exBjmXd9bxE17gL0nLxLW8U8iHk5vUDYj55EYtrfL5YABogYKuhBSvibxrjo5ejr0UsO3ecGD6Bd9JIjoW1lv7hIAnEUqy1J25PxklO06gTGjUB61IxDQh2Ner1Cunps=,iv:0ZOZeF7pg4Pi6pD305BlJl7V46BOc5l7Eg0oHYlYK8s=,tag:GtAfyAwqWrZs1IYKhbzN0A==,type:str]
+ pgp:
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMAxK/JaB2/SdtAQ/+Irbhincv0agRseJ3U03cW+YNHa4suynF5eSew3BsnY6h
+ +EevEAN2uz4JIRVSmXjBeNFPv3VtN1h5kxzmWXNHmZwFH4nNR+0w9a7zfUEa2E2W
+ 2THwlZFZIPVgxRZIA1ntr88a97Bxy+M+gJDuazOq77YvNCAWLi46Iim4MxuHGqsT
+ jTJ6uSe039gKiKQapeS8PpXPNTfs0ORq+OHkN1NWtJ/FbePZquqfPYfdG3csLJIB
+ 2O0To8jX5qKYZi9Z8Vx1EUMB2C0rT7tcteBAKs2KqYq5peWAK0JJefAuDbL0Fdb3
+ GOXnRcXKopLlLkCI8P9JZ60oW0HyyjaeuF2dvoErdqGSZEhH/RSkfYnTPoM3x03+
+ XwH6qBVFVlj3y9IRUJt9FAt634CHnFpTKGEZ7gEiNHazrIUiqF0VOEzI8zHELVdq
+ Yrx3daWBJLhMJAkv1Tgk4S0OSeK5BbJDa+UhjVgkbBjOJEvT0J0CXzaR6JVJqKNm
+ 3mGBJtc7CVBMQGX7RQZ4r6J3a1vhElMycNZCy+4hTYZ9+KCtY1wPRjleYDfgoK0E
+ 8WnsZ06phqEmmSThzB7bbCpf/5SQcxoWWUpdV22poHOEc/W0XoCy7zYXsoM2r7hP
+ JW6k/MTznJD3QnI0kOrfS44T51xkdapBUz9lFsh07nRKhi9TJJB8JXxNbCnbMhnS
+ XgF8vGN8Qulz2ljp6IM+LhoMPADm3hrQtEkJrXQxz1dpkZE4XHUk/tvgsDx8Kxco
+ z7/LzohXg/4MrvKtA8q4sl9oOMpv4B0H9pSMzdURk2vmgd96U4egiYpjXwqwBnY=
+ =3Fho
+ -----END PGP MESSAGE-----
+ fp: EF643F59E008414882232C78FFA8331EEB7D6B70
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMA6EyPtWBEI+2AQ//Rh8YA6DUIBi6mjhixAd2eNCLPlQ5w/hRj991Q9uVCaBR
+ 55JWyQQBbondn/1MEVb2PlaHH22+HPAbv4p33FD4pbimz5W0taBw3T6CmDdx1V+E
+ UmitZIRNdoirbe4ChFToUjZ31RQbS5pdxW3ATSJKn1pmR1/g5sBq5SThenm1nwvU
+ ahV71QfUrs7oqJAYHqmPIipbR1PP1QSVfyDNGUx6gIYxWS7dQPtcNkVhS1fdCl8b
+ Utg1MW/pCqQuw9nRsI+2rSEtYfYqiap5Mv31Ihznfvu/cH+uyeBeT8Xmr4/9qmvA
+ 5WXJA/0qwd3S2+l6vcxBFgyoj9yFAYorTU200OBa1HBZGjQY+V9h9I4amYrj2SRC
+ 1wgsNgFxuhUQaEDhPlD8kdSts8QY/ApYwJyHnpCW1FuzgMPY2w6CfDjr0Hv4JCtw
+ /Iuy5zbh3cNbgV8jlVn3J4v3yMtEZnsh7rEb+EbPuZmpTuZ8AIG+NqIiW/SBfELW
+ qSHN/Iv1zIl0BmcV2qAKfrsox4QIOESM/77ISrwOLQoPd01qefNsTp8PExtt+yzn
+ 9MXNv0CHmpDA6u1ruIpub969T04tHu3oekZpM327glpCf5SoKVo+fYmEwB8IhIkW
+ NcNaQIeZ1P8jSjHM6XUAUfOHzzRMy0jqQVaz9kD/kHXCMfCJT5KfvKeSaJhCy7/S
+ XgEtCHT6VloJ2X9VxL695k5ugfyTsDYYDgteKuSD68cPbj2MnYS8uKD3VQh9/I/d
+ 5OJN8fsvpkpQIltUh3DeCgRv7AF03Zdou7amrTl5MEaNBZxX5mBJrA/qOw3XAWg=
+ =mRNR
+ -----END PGP MESSAGE-----
+ fp: F155144FC925A1BEA1F8A2C59A2A4CD59BFDC5EC
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMAz5uSgHG2iMJAQ//RXqm63AC3eWRV1cNDulWgCqZzThW1f/4o4xelGYxLQe0
+ cJuSqJmZoHsAItQ1GBIhyd/a+lcNt6Ym100RLlL6f5nPnHyk2pJNv/dPOpbs1b8b
+ +ulq2QBQEvvrzukmzXcqMGrjvJrzINB7U2L1uPBe0CTircMUR5J444LgOHC3VGnt
+ twBBgI5NQFcoZLADt8j73KEjfYzPJeaqHudhgU59h+cgPz+6N/v1fkG0vSQuzBuw
+ Tm+fk52t5X5qLWLyqrLtb4W8LdYN9D9TieRRlzjunYL8mISJikCQfpHroJkJWDjH
+ k4gaeVErauCOJWQ6Gp6aiYBtMehsHCh/8stGcnOgtyBpPh7o9FTTGcVR6j+qpijL
+ QYsjYfaH5aOU4JoUO5vq8wsBiVcOsP65CqeVFFLlvAVqZxPNzq3iBkBaWECLBfYy
+ QtIFRnRRznZQvTR0hjC0cw7vOpBGNwAcqnjPv9hQLPzdZyU2ViJjhwq/16alER9V
+ N2xFl6eKt/Mau5ZlX62lbq9eJLmR2Bqb+sL4rdMfRfl259kvGilBkCM7SMfkWnOq
+ z0do1+9FRzo8IC57WvYemzAS/pBfFH8o0Ey+PRSys03WC4YPW9XDnjSpRKEPpO8u
+ DbdhuKoVb19tAERzpZZKN2Rzuv68IpQ1vhEEP1BbsApoS0vlYIxcPSAVmSC1o7vS
+ XgE7yntjkVO+C8ciByubK1DGHZ/G5eXB/zkYQKj1w+bAmTJQ26DtHJa5/o7cXkk+
+ Ja3Qrc5Yp+W5MIV70+FHsDXNarpXSJbSPNf4nPKWsdFZGkauHks0o58T6D74LqQ=
+ =wHLh
+ -----END PGP MESSAGE-----
+ fp: 18DFCE01456DAB52EA38A6584EDC64F35FA1D6A5
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMAw5vwmoEJHQ1AQ/+OAK/CpxVtW9HoLP1wJR589/JMdqJZqOxkySgAlm+z6RY
+ 4knRz/0f5wdHSQyuvAYnq/M1K9BsBD34dFiqyvdTa0+G+bJUIkHDLkSTqM8IvGMb
+ 48sCbGwW4Ghmxn5mjK3MwuGbGKxVujJWqwaRmOp6lgtRJdpKReFD68vtwTHa2qhh
+ ixnABbOTyN08Bf9pJ9cgoAQaVOcSja0E+yuPRlHUvM2hjbGNndbaiTtfq1hFn5qJ
+ VoakC+u6tcKEp31Y4plN1NTRf/ywZ8oMmT4TIf3kvFGwx/XKx2miIB9cUSMw/ojU
+ GrGNXjh4vfEaT0iIRtZ+H8FfuGnjFkU6qodLEIKlVmng8MU7ETGLErHjyNEJf2JT
+ OMnaajJxq8jXaY2SDoHsKETMgON1uwDDKW6NOBhaK+fW79W6z27uGnsN055vMTpV
+ kh1YJixyI3wIkr6bbfNHBdr6C8Tb4sY20zghvkQYBA2xCZSLOT0a5lX7GBTUp0uY
+ +hgxdfyQJi0P+4QPam28/b18lOZ25LC69YX8AtczQ4vHhIM+jQ+bzoNSoMpwcSm4
+ vZSSmMB0tX8W5O6yo6A/XLoktzyuzvMfZ2v3/6LbIWK0FKJzy5G9A9/xwnbCRulB
+ BJf+xzfwWt92pW7n3yVgjO+o48J1c2b71qAaMtukhPLNFSozgHlqv4vy5BD72pnS
+ XgGNEavqMxIRuRQtyDeeV0W5gdGCY/XUAjYxh4Ly51XJVCL1yZptYiFaWMuYEB3F
+ G3unTkE+YedYk2g/Wt4pR9lcgRLW4zRlOCtzwiE6JbAkp5NsQ6Tn/Q0UD1sTRsw=
+ =Y1YG
+ -----END PGP MESSAGE-----
+ fp: 87AB00D45D37C9E9167B5A5A333448678B60E505
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMA4HMJd/cQYrVARAAxtzsDYAMwB8WAUx0U3RnEkBHEeyMqNvLCgzz0oU73B0v
+ eUWzHUYrTYMyYxRMKO8vqKULBPhYOKbns0hzL8s6YjCnT08XwNXtYiuKm90FVQcz
+ 4ARslyObb+0ayyfx9dd9+6aFCgyftgAZpctWCEWPhBLUIsKcsd/q+Q5hSNfhwp+1
+ IAfruNkBaCFD95A3apfsVd3E/clzXBXcNa9d2k7Te3LCduhD5Su9QUgqDvf5Je8o
+ WS1+Q8gih/+xTNR0avBfAZuSq24cqKyPg49KNRvfWq7drEZYYfUOdIMOJVZiBuRJ
+ y4HjNGgX+NIl/BDu4SpFQVFhDmv+kgIM0JxXF6p3Ap4hZAYicWRnn0StVJ5kaB6O
+ 7l58NTu9aX7eLR4W2NuYLTwmssnA/hJd8i42YSYYD05siQIKICxkaLSTVztqf1vS
+ N4RNNZNle6gkBvceRkb+8FgzPmLL8BFPkUiAFJOr5BDShbXwN/UocBgVKIRsuQah
+ mIJ5uu++9oy5jaR/eeff5QcRxtpCasi/86qW9igCSOqKuHWOMz0RWJCRaJmhWY/m
+ 5gvz0nNCqbnPOXwvbNiuAmFmhmhYs8AvEvqMPJR3DHUSy5U1Bqpx+Oeu4qK16alr
+ HxjnyyEoGLkTSfk22vN7wQZD+loQJlL9U8swQmZD+Y3pyPInCYrZotOwMBo6XazS
+ XgFRaZJlP0gC3tN83H0b1oC0eXBMagmEVkyhxMBwXCrGxl9BrcF7KGxP5GU7uqGm
+ nV0GU1UIJZAS2qrdf456Ou01E/5QbpTHac25/W7ZlPOhibqWbT9wV+ICYZfSMU0=
+ =07bf
+ -----END PGP MESSAGE-----
+ fp: 91213ABAA73B0B73D3C02B5B4E5F372D17BBE67C
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMAxjNhCKPP69fAQ/5AcTObI2/IVj3lxv7G+p65eqtuexRmMCn/dsLOR3MBLkB
+ Pw6JFRUIsRAgDlpD0YI7CrqB3pisej5LemUmvB9vK9H+6IALSB5eKEMd/6MXiqlV
+ HDUw/pmZUP+X16GAsXDwvMNT1RQQuEnigTzaIo8ydDWdsgAMOs7JZ7KcF/k62x1k
+ UCqCnEZhxyKopNOtbLuVhpW8R1DnRIenm8v3tB85neVTXPBRcG8fJ5y3zqRwpIPX
+ pXUT2QI1fD6P+djMNJPFPcQdf1zz1xj02OuQQnKX68qh/VW4QJSF5e0firXSZ37n
+ dpsfQ7ROU6PfnvcXFZTPoR6b8oUgo7TxwOy4ERPqXbuM1UZm5zr0hj42IYQz1AZm
+ LlcB/AIs2MJDXgv7B2aLryZQGipBMmsASNbqyTVU+cA7f0km3hyta83RZsOw6MsX
+ wQjTQhx/lnCx3/dOJevEwBE6YgybKJAVIqscNAagAFuCtlbq5RjVYKRA3nRBGgjK
+ hDFQ0yWWl2UHYC4aIl05SIsoL2KVXEzIT1qayy4sGR/L3YmUx1OcZLiBZOvCRBYw
+ v/DX/Poz7C9g2jEPC9SV7IHXF7J1SI6aTOWcxrqpXVY45vbIW2qLQC/uJz3GTOaR
+ Om361FwXnJAYeCjOxIZXSlBy6JLEgBSjA+F9dDtwuTz3Bay1IhdNJ3Z55zzVI5rS
+ XgGJHreDweUIhIhoGBMiEuKb+d6UCQ9F6oiBulvO3zYTpqJNM2U10xllF5MEztWe
+ 96Bai8OAPTkIR5UT2cpjodlye7+SvAabxvnUDdUqoL6+2jMtECUD5/VRzLEkrfU=
+ =w6pZ
+ -----END PGP MESSAGE-----
+ fp: F38C9D4228FC6F674E322D9C3326D914EB9B8F55
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMA1Hthzn+T1OoAQ/+Mj7CiCY2fpytnZIrwXUaSSTvEl4TkuJrgN10NXdhEiuB
+ MsIubs9q/dGvG+GLBTNIuRJzzQespRC0z7t38ylGNMvaLODUGpy7XvfDF6aiSzCG
+ hrGcWGPwWue2HnoyPBy4ObaZq+aB7FrGrNgxVS5p5sd7ovj/UKDu75G3DNXuQ9C6
+ AYgzETIGU6wtnJvp0EhqHQTaJ88dus+kiGpLVhMxDfGPhCAwOQ/2SYwI8R/uJTEh
+ qTCkNOYms5vV+DVGXCO1kfgqeQjgRj5vnMq0+2m3Twvfrj+EVNnRh2jrJbYypqRA
+ 6rtRGUFQFrr7b0rugaB+H3FIRffjrFy56rnW6iMwwcvbsEpAx3K56hm347d+vH+8
+ AcuaD955skQ8WnopbBYzLHmajRZZgK74JwY4bmEILeg1s0+gZy7xTRWsYQQZfvTR
+ 45Cq4wVR88QDNG23vVscABZIeV9WocSiCGlayo+LN+dOZdGpkhjnq76Qw/jfzd9A
+ h5UvMVsnHcvJMw1zo73cbdHlI6IS5oCuTLsVy/w62Ts6oTD2KsQSMyZ1E8QYQts5
+ ugZ7T1mRcHaB5LE8+hSIRi4Ck01gZUtApAdIXGwu76bSgspGfvINqOmuWpOd8+K4
+ uqXW0Wu5yEfYE+ypAmUY6sxfilXOV89PmJcIv56imZNEEnr9aK+u7rjqfX+41izS
+ XgHJhO78PVLoawWZ5x4tSw/Tjd3qabdr5dx4bQriUW1ghRJEt+X/2uDvYyMEQaxH
+ mM2c4FHpM/IyG3Td89JpHcbwVxktAm0fwjVswdILyyIz4bzht8+QsJHN+msL9OQ=
+ =xDlD
+ -----END PGP MESSAGE-----
+ fp: 5DA93D5C9D7320E1BD3522C79C78172B3551C9FD
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMA46L6MuPqfJqAQ//XakvJ2IaEP1Ynaw0qYQYOEyIiQp8SJk9KReYHDpDNuqP
+ emdLwZDZSVP/QqpnPC0diJkZaTM7yaSYxRmiXTnFd0r8bEYLCL1A6GBFXIFlh+8M
+ PgOff8TRbUrLmgEtyrkv1PMjf8rX0A4iSPiuNFFL3ew7m/MBkITiPYq+8YcE8yTz
+ vgtNyuYfi59TbKai2fcas4IX3bF0HeGrhAkys0aa2iFlH/lJj4yd7NqTAsOsDbO4
+ 1eplhf+IM8Rv0WND3UZCBNk29Em7S4yllFJpH4E9xS9noWqTEyMQ1qXeoq04BSry
+ dQ0evD1d7+gLacmV5+HQo5p80OhMSgYqrClGUJBO6eNsfE/hSc24MDjAB3rs6xFb
+ wGvzMWekWqosN0eXmU8Iy38bFeT8CWbAvCA9BJomwfDMbgE6MOjNo4PURZYQ0EMf
+ oMSRcTku3vTVidOumQS2a9qanNQW1dLTVigQvHnByNTRjPxneo3IZFIvqBqYdt1e
+ UbEDbjlDBQzqLt1vPEHSoX7FlMT49HZUY49yLwp/VMUGrDscApdLYqLRp9gbgf1Q
+ gHkh60sGLUQgUQZ65L1BRJgIm3NFhkJAtONQnJq2iY5f/1ZPHlAQVqrBN9a7Hp01
+ efrdHCvNMDvoIZXTpC+y7cnvnmN4fGXaXA3Z1dJsmai36Ak83hgtMhC7s75FMtXS
+ XgGlZQUDAnkpily0mS/ZQ4IMLW2yzcBH1BkHsuHEmFWij344+6f1TlrhObMuFD+V
+ 2E+A3Uux4SSl2RbpIfEcvZptVeVB17wutOuHrVXrn1sOm2+cT/k+Ousrrfrm4v0=
+ =j38o
+ -----END PGP MESSAGE-----
+ fp: 8996B62CBD159DCADD3B6DC08BB33A8ABCF7BC4A
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMA4EEKdYEzV0pAQ/9EYMqHVt60BlFDSZXR+J0/hfnxutbvta0CPkAUslJIQS5
+ XiPcUeptVEmyLUz66bw17m1R4j4miDW8o+3JVQH3oU4YYQPUFHcY/kkSVU8yuWp5
+ e8KkSkVTOcUaAyiPNTY7YswOjWcHKs3B81eSJBAKGiS2y2SakK78fZMan5x6vUJd
+ s4O57hxZPrRXrps08zEiTC+uI8/Wl+5VvoSfllOAqwaohJpEOzt2A74aBz3cit9T
+ yBwHb8nhaZ17RYZ8DJtGyeekMlgM7vj6IGWUbxb38C+kJlY/15MDIKKWEApZ2/m2
+ VXwUR0aJcqD/oLFOnQO/fKTQM6QGnrgAQFF8Z6X2pZqIU9W6vxNHTGEzt6cn3igS
+ 0Wvp0hRQEkfyYx94xPGm36/GM4Zqhz+W2YRo+z121/OO5PWBtMxLUT39/PKBDROw
+ BU/QLPl+l2nnLg80KQqcUw60HUXZIpR1p6KEQrmK7+jrDPIx45S1NI1RmNiMEv6y
+ h35boU1/0YymYKkt6nFyz/GvqD4qviCLimz6/21a606TaIx8LqZaLmZ3YdXk7yqD
+ XcHweJ1EBbhHkLYYCZsG4tNfJj9hBgVimOjjiCnr0lkzxKAPGdVghmPdwFLlYXIO
+ V+tAi9KKPK6SRdVBuCpzHZyg6JLiFGmUsmL/piSY5hXrvv8p4oQp/TI4S4Yblv7S
+ XgHt0Xy2jfYFUPedR0BMta5TqvaNjDh1qxAZepzbWRwiDjHiQ4gsAvjytUmiceIf
+ KJDhKQqUuaNYt7cBsNF9PgtSkD/ZuF4oTRFVqM6tr/JroxjSrGjg39T6lNtGo8o=
+ =v83W
+ -----END PGP MESSAGE-----
+ fp: 9DFA033E3DAEBAD7FDD71B056C7AAA54BE05F7BA
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hF4DQrf1tCqiJxoSAQdASHcTIysPla95JELBmv3+guJ1Ehx4VGq/zp8NFcU/nG8w
+ +o04dzI96ZV58cNGG0GZOpoq83q0XbspKpnzMnJyNtlbsMpVXhVZgrneUYY4EwnB
+ 0l4BNnrT5pIFX8+6dP7EytxWU2s1UTppVYgwELpWnWItZk+W0EgiK5f3V+x28nh6
+ psaXJSFsGOJaBJsitMv/GDyyOu7y+PKSKooY12GujdK4cgu5SZbzeq3iYcKAyQ8a
+ =TEyd
+ -----END PGP MESSAGE-----
+ fp: B71138A6A8964A3C3B8899857B4F70C356765BAB
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hF4DzAGzViGx4qcSAQdA/Y9Zxvac8WQmVo0KgX7LTs9c8GgtxOEMMUJ8QxP7RREw
+ NHIIMCpoidBtkB0RrLvObu23W4HO8/j4zrKV3dBmi3Z/6cdxbLMp3Kl6OK68UcCS
+ 0lgBLF455STDbzpSuZA7fMgeexxpB6rctYJt1EbVZ4Gq5CMdXEilccr+wsAqA19N
+ NFrV1QL5nlk9/qxU6X4DUaLcJP3/MAUga3ODsBq/5goVMjyQddDpprQZ
+ =p6Oh
+ -----END PGP MESSAGE-----
+ fp: D2E9C0807BF681F5E164DAFC5EE1B61CD90954CD
+ - created_at: "2025-06-01T21:41:02Z"
+ enc: |-
+ -----BEGIN PGP MESSAGE-----
+
+ hQIMA2pVdGTIrZI+AQ//YkQT0gnE4CS9iPm7kB9H6zQ6655S3vspQ/ftbVkjDgbG
+ XUlCIZOqBWcY2M+JDCSHknUsj44F8Y3COlC8c2nSKO9sFDheaDPiSMqtJxXjbuee
+ gdpbvc9pjsnIdWP2HDgOTsAtX+/qjh4OACWVjqaJI6H+mDA2EaOpt/cp00G41v7e
+ XwTbvGgeW0nwxwPSS1UzHr5oVjwBlKdZXVqjuZT3tzi+YzqbSfQ1uWwWpS8flVDL
+ yCPTaD9OpYPq16ztNJoviF6+6eyTwQVfmJHq/3DlZrmhIIcd0wsx6HOt2g4RjW4d
+ T1mAuHkGkAbxcEU5TiHzIBMCAEHEH2s4TCs7VtdG2pdjm/Fq7oz2aIsVdwI7dg/k
+ wbOGoWDvbY8YqiWD1o6RDyhDySCkuewwsi58UTDFTC7V7CJWnTapMLcqenoNOzUJ
+ E+aM/kH8zHdTXpqpOeYwtKWX4FqE6UHYJkWhI7F4KzhyQ57N+98PRoPEfXoukjjb
+ JsBWBuJg0pwNrz7aRurCMvYpW29AXuL8WbceUxwZgB0P6ztGKdnU8NLhOZj2DkE/
+ OLz28t9HtpbAfOZ1cxMrNp0log0hJFXD7g4cRX2F/zWuVKuWn0vUvhQot2GuAuw8
+ DRG0DJGSQEHhyNjtNuLufGR6FETeC2CNnpeXxXZhqik1kXwSB/AompaKZbjJGb3S
+ XgHkuxjOS/a9iREdy+vW/evtGnh1uMUa5/phMU3VGKiCp5ozfuwaQ5gvVMrE80b9
+ loGh0l/S66CyIOO1eXBlqkH5FxsMcvVAHB1u8uEZ3T9Y9yh0ontnc3LDWUpPxls=
+ =2DaK
+ -----END PGP MESSAGE-----
+ fp: 878FEA3CB6A6F6E7CD80ECBE28506E3585F9F533
+ unencrypted_suffix: _unencrypted
+ version: 3.10.2
diff --git a/inventories/chaosknoten/host_vars/ntfy.yaml b/inventories/chaosknoten/host_vars/ntfy.yaml
new file mode 100644
index 0000000..6d0e0eb
--- /dev/null
+++ b/inventories/chaosknoten/host_vars/ntfy.yaml
@@ -0,0 +1,90 @@
+docker_compose__compose_file_content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/ntfy/docker_compose/compose.yaml.j2') }}"
+docker_compose__configuration_files:
+ - name: server.yml
+ content: "{{ lookup('ansible.builtin.template', 'resources/chaosknoten/ntfy/docker_compose/server.yaml') }}"
+
+certbot__version_spec: ""
+certbot__acme_account_email_address: le-admin@hamburg.ccc.de
+certbot__certificate_domains:
+ - "ntfy.hamburg.ccc.de"
+certbot__new_cert_commands:
+ - "systemctl reload nginx.service"
+
+nginx__version_spec: ""
+nginx__configurations:
+ - name: ntfy.hamburg.ccc.de
+ content: "{{ lookup('ansible.builtin.file', 'resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf') }}"
+
+alloy_config: |
+ prometheus.remote_write "default" {
+ endpoint {
+ url = "https://metrics.hamburg.ccc.de/api/v1/write"
+ basic_auth {
+ username = "chaos"
+ password = "{{ secret__metrics_chaos }}"
+ }
+ }
+ }
+ loki.write "default" {
+ endpoint {
+ url = "https://loki.hamburg.ccc.de/loki/api/v1/push"
+ basic_auth {
+ username = "chaos"
+ password = "{{ secret__loki_chaos }}"
+ }
+ }
+ }
+
+ loki.relabel "journal" {
+ forward_to = []
+
+ rule {
+ source_labels = ["__journal__systemd_unit"]
+ target_label = "systemd_unit"
+ }
+ rule {
+ source_labels = ["__journal__hostname"]
+ target_label = "instance"
+ }
+ rule {
+ source_labels = ["__journal__transport"]
+ target_label = "systemd_transport"
+ }
+ rule {
+ source_labels = ["__journal_syslog_identifier"]
+ target_label = "syslog_identifier"
+ }
+ rule {
+ source_labels = ["__journal_priority_keyword"]
+ target_label = "level"
+ }
+ }
+
+ loki.source.journal "read_journal" {
+ forward_to = [loki.write.default.receiver]
+ relabel_rules = loki.relabel.journal.rules
+ format_as_json = true
+ labels = {component = "loki.source.journal", org = "ccchh"}
+ }
+
+ prometheus.exporter.unix "local_system" { }
+
+ prometheus.relabel "default" {
+ forward_to = [prometheus.remote_write.default.receiver]
+ rule {
+ target_label = "org"
+ replacement = "ccchh"
+ }
+ rule {
+ source_labels = ["instance"]
+ target_label = "host"
+ regex = "([^:]+)"
+ replacement = "${1}.hamburg.ccc.det"
+ action = "replace"
+ }
+ }
+
+ prometheus.scrape "scrape_metrics" {
+ targets = prometheus.exporter.unix.local_system.targets
+ forward_to = [prometheus.relabel.default.receiver]
+ }
diff --git a/inventories/chaosknoten/hosts.yaml b/inventories/chaosknoten/hosts.yaml
index 2450ca8..93ea984 100644
--- a/inventories/chaosknoten/hosts.yaml
+++ b/inventories/chaosknoten/hosts.yaml
@@ -59,6 +59,10 @@ all:
ansible_host: zammad-intern.hamburg.ccc.de
ansible_user: chaos
ansible_ssh_common_args: -J ssh://chaos@public-reverse-proxy.hamburg.ccc.de
+ ntfy:
+ ansible_host: ntfy-intern.hamburg.ccc.de
+ ansible_user: chaos
+ ansible_ssh_common_args: -J ssh://chaos@public-reverse-proxy.hamburg.ccc.de
hypervisors:
hosts:
chaosknoten:
@@ -79,6 +83,7 @@ base_config_hosts:
tickets:
wiki:
zammad:
+ ntfy:
docker_compose_hosts:
hosts:
ccchoir:
@@ -90,6 +95,7 @@ docker_compose_hosts:
pad:
pretalx:
zammad:
+ ntfy:
nextcloud_hosts:
hosts:
cloud:
@@ -109,6 +115,7 @@ nginx_hosts:
public-reverse-proxy:
wiki:
zammad:
+ ntfy:
public_reverse_proxy_hosts:
hosts:
public-reverse-proxy:
@@ -127,6 +134,7 @@ certbot_hosts:
pretalx:
wiki:
zammad:
+ ntfy:
prometheus_node_exporter_hosts:
hosts:
ccchoir:
@@ -154,6 +162,7 @@ infrastructure_authorized_keys_hosts:
public-reverse-proxy:
wiki:
zammad:
+ ntfy:
wiki_hosts:
hosts:
eh22-wiki:
@@ -170,3 +179,4 @@ ansible_pull_hosts:
alloy_hosts:
hosts:
grafana:
+ ntfy:
diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2
index 6d550ed..2219d3b 100644
--- a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2
+++ b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2
@@ -3,13 +3,15 @@
# - https://github.com/prometheus/alertmanager/blob/48a99764a1fc9279fc828de83e7a03ae2219abc7/doc/examples/simple.yml
route:
- group_by: ["alertname", "site", "type", "hypervisor"]
-
+ receiver: 'ccchh-infrastructure-alerts'
+ group_by: [ "alertname", "site", "type", "hypervisor" ]
group_wait: 30s
group_interval: 5m
repeat_interval: 3h
-
- receiver: ccchh-infrastructure-alerts
+ routes:
+ - matchers:
+ - org="ccchh"
+ receiver: 'ccchh-infrastructure-alerts'
{# Disable these for now, but might be interesting in the future.
diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl b/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl
index 4651496..3e97e6e 100644
--- a/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl
+++ b/resources/chaosknoten/grafana/docker_compose/alertmanager_alert_templates.tmpl
@@ -22,7 +22,7 @@ Links & Resources
{{ define "alert-message.telegram.ccchh" }}
{{- if .Alerts.Firing }}
🔥{{ len .Alerts.Firing }} Alert(/s) Firing 🔥
- {{- if le (len .Alerts.Firing) 6 }}
+ {{- if le (len .Alerts.Firing) 5 }}
{{- range .Alerts.Firing }}
{{ template "alert-item.telegram.ccchh.internal" . }}
{{- end }}
@@ -33,7 +33,7 @@ Links & Resources
{{- if .Alerts.Resolved }}
✅{{ len .Alerts.Resolved }} Alert(/s) Resolved ✅
- {{- if le (len .Alerts.Resolved) 6 }}
+ {{- if le (len .Alerts.Resolved) 5 }}
{{- range .Alerts.Resolved }}
{{ template "alert-item.telegram.ccchh.internal" . }}
{{- end }}
diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus.yml b/resources/chaosknoten/grafana/docker_compose/prometheus.yml
index cf7f594..769cdc8 100644
--- a/resources/chaosknoten/grafana/docker_compose/prometheus.yml
+++ b/resources/chaosknoten/grafana/docker_compose/prometheus.yml
@@ -22,6 +22,8 @@ scrape_configs:
static_configs:
- targets:
- localhost:9090
+ labels:
+ org: ccchh
- job_name: alertmanager
honor_timestamps: true
metrics_path: /metrics
@@ -29,6 +31,8 @@ scrape_configs:
static_configs:
- targets:
- alertmanager:9093
+ labels:
+ org: ccchh
- job_name: mumble
honor_timestamps: true
scrape_interval: 5s
@@ -38,6 +42,8 @@ scrape_configs:
static_configs:
- targets:
- mumble.hamburg.ccc.de:443
+ labels:
+ org: ccchh
- job_name: opnsense-ccchh
honor_timestamps: true
metrics_path: /metrics
@@ -45,6 +51,8 @@ scrape_configs:
static_configs:
- targets:
- 185.161.129.132:9100
+ labels:
+ org: ccchh
- job_name: jitsi
honor_timestamps: true
scrape_interval: 5s
@@ -54,10 +62,14 @@ scrape_configs:
static_configs:
- targets:
- jitsi.hamburg.ccc.de:9888 # Jitsi Video Bridge
+ labels:
+ org: ccchh
- job_name: 'pve'
static_configs:
- targets:
- 212.12.48.126 # chaosknoten
+ labels:
+ org: ccchh
metrics_path: /pve
params:
module: [ default ]
@@ -74,6 +86,7 @@ scrape_configs:
static_configs:
# Wieske Chaosknoten VMs
- labels:
+ org: ccchh
site: wieske
type: virtual_machine
hypervisor: chaosknoten
@@ -98,6 +111,7 @@ scrape_configs:
- zammad-intern.hamburg.ccc.de:9100
- pretalx-intern.hamburg.ccc.de:9100
- labels:
+ org: ccchh
site: wieske
type: physical_machine
targets:
diff --git a/resources/chaosknoten/ntfy/docker_compose/compose.yaml.j2 b/resources/chaosknoten/ntfy/docker_compose/compose.yaml.j2
new file mode 100644
index 0000000..818e17d
--- /dev/null
+++ b/resources/chaosknoten/ntfy/docker_compose/compose.yaml.j2
@@ -0,0 +1,23 @@
+---
+services:
+ ntfy:
+ image: binwiederhier/ntfy
+ container_name: ntfy
+ command:
+ - serve
+ volumes:
+ - ntfy_cache:/var/cache/ntfy
+ - ntfy_var:/var/lib/ntfy
+ - ./configs/server.yml:/etc/ntfy/server.yml
+ ports:
+ - 2586:2586
+ healthcheck: # optional: remember to adapt the host:port to your environment
+ test: ["CMD-SHELL", "wget -q --tries=1 http://localhost:2586/v1/health -O - | grep -Eo '\"healthy\"\\s*:\\s*true' || exit 1"]
+ interval: 60s
+ timeout: 10s
+ retries: 3
+ start_period: 40s
+ restart: unless-stopped
+volumes:
+ ntfy_cache: {}
+ ntfy_var: {}
diff --git a/resources/chaosknoten/ntfy/docker_compose/server.yaml b/resources/chaosknoten/ntfy/docker_compose/server.yaml
new file mode 100644
index 0000000..a58e931
--- /dev/null
+++ b/resources/chaosknoten/ntfy/docker_compose/server.yaml
@@ -0,0 +1,9 @@
+base-url: "https://ntfy.hamburg.ccc.de"
+default-host: "https://ntfy.hamburg.ccc.de"
+listen-http: ":2586"
+behind-proxy: true
+keepalive-interval: "45s"
+cache-file: "/var/cache/ntfy/cache.db"
+attachment-cache-dir: "/var/cache/ntfy/attachments"
+auth-default-access: "deny-all"
+auth-file: "/var/lib/ntfy/user.db"
diff --git a/resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf b/resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf
new file mode 100644
index 0000000..96aea75
--- /dev/null
+++ b/resources/chaosknoten/ntfy/nginx/ntfy.hamburg.ccc.de.conf
@@ -0,0 +1,43 @@
+# partly generated 2022-01-08, Mozilla Guideline v5.6, nginx 1.17.7, OpenSSL 1.1.1k, intermediate configuration
+# https://ssl-config.mozilla.org/#server=nginx&version=1.17.7&config=intermediate&openssl=1.1.1k&guideline=5.6
+server {
+ # Listen on a custom port for the proxy protocol.
+ listen 8443 ssl proxy_protocol;
+ http2 on;
+ # Make use of the ngx_http_realip_module to set the $remote_addr and
+ # $remote_port to the client address and client port, when using proxy
+ # protocol.
+ # First set our proxy protocol proxy as trusted.
+ set_real_ip_from 172.31.17.140;
+ # Then tell the realip_module to get the addreses from the proxy protocol
+ # header.
+ real_ip_header proxy_protocol;
+
+ server_name ntfy.hamburg.ccc.de;
+
+ ssl_certificate /etc/letsencrypt/live/ntfy.hamburg.ccc.de/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/ntfy.hamburg.ccc.de/privkey.pem;
+ # verify chain of trust of OCSP response using Root CA and Intermediate certs
+ ssl_trusted_certificate /etc/letsencrypt/live/ntfy.hamburg.ccc.de/chain.pem;
+
+ # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+ add_header Strict-Transport-Security "max-age=63072000" always;
+
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Port 443;
+ # This is https in any case.
+ proxy_set_header X-Forwarded-Proto https;
+
+ proxy_set_header Upgrade $http_upgrade;
+
+ location / {
+ proxy_pass http://127.0.0.1:2586;
+ proxy_buffering off;
+ proxy_request_buffering off;
+ proxy_redirect off;
+ client_max_body_size 0; # Stream request body to backend
+ }
+}
diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf
index 992161c..1b998fc 100644
--- a/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf
+++ b/resources/chaosknoten/public-reverse-proxy/nginx/acme_challenge.conf
@@ -70,6 +70,7 @@ map $host $upstream_acme_challenge_host {
design.hamburg.ccc.de 172.31.17.162:31820;
hydra.hamburg.ccc.de 172.31.17.163:31820;
cfp.eh22.easterhegg.eu 172.31.17.157:31820;
+ ntfy.hamburg.ccc.de 172.31.17.149:31820;
default "";
}
diff --git a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf
index a97d0a2..37f62a1 100644
--- a/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf
+++ b/resources/chaosknoten/public-reverse-proxy/nginx/nginx.conf
@@ -88,6 +88,7 @@ stream {
design.hamburg.ccc.de 172.31.17.162:8443;
hydra.hamburg.ccc.de 172.31.17.163:8443;
cfp.eh22.easterhegg.eu pretalx-intern.hamburg.ccc.de:8443;
+ ntfy.hamburg.ccc.de 172.31.17.149:8443;
}
server {