diff --git a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 index ec60c01..799701a 100644 --- a/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 +++ b/resources/chaosknoten/grafana/docker_compose/alertmanager.yaml.j2 @@ -13,10 +13,12 @@ route: - org = "ccchh" - severity = "critical", receiver: ntfy-ccchh-critical + repeat_interval: 18h - matchers: - org = "ccchh" - severity =~ "info|warning", receiver: ntfy-ccchh + repeat_interval: 36h - matchers: - org = "fux" - severity = "critical", @@ -31,6 +33,7 @@ route: - org = "ccchh" - severity =~ "info|warning|critical" receiver: ccchh-infrastructure-alerts + repeat_interval: 6h templates: - "/etc/alertmanager/templates/*.tmpl" diff --git a/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml b/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml index 4aac2dc..bd0a400 100644 --- a/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml +++ b/resources/chaosknoten/grafana/docker_compose/prometheus_alerts.rules.yaml @@ -438,6 +438,7 @@ groups: for: 0m labels: severity: warning + org: ccchh annotations: summary: Prometheus too many restarts (instance {{ $labels.instance }}) description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.\n VALUE = {{ $value }}" @@ -446,6 +447,7 @@ groups: for: 0m labels: severity: warning + org: ccchh annotations: summary: Prometheus AlertManager job missing (instance {{ $labels.instance }}) description: "A Prometheus AlertManager job has disappeared\n VALUE = {{ $value }}" @@ -454,6 +456,7 @@ groups: for: 0m labels: severity: warning + org: ccchh annotations: summary: Prometheus AlertManager configuration reload failure (instance {{ $labels.instance }}) description: "AlertManager configuration reload error\n VALUE = {{ $value }}" @@ -462,6 +465,7 @@ groups: for: 0m labels: severity: warning + org: ccchh annotations: summary: Prometheus AlertManager config not synced (instance {{ $labels.instance }}) description: "Configurations of AlertManager cluster instances are out of sync\n VALUE = {{ $value }}" @@ -479,6 +483,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus not connected to alertmanager (instance {{ $labels.instance }}) description: "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}" @@ -487,6 +492,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus rule evaluation failures (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n VALUE = {{ $value }}" @@ -495,6 +501,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus template text expansion failures (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} template text expansion failures\n VALUE = {{ $value }}" @@ -503,6 +510,7 @@ groups: for: 5m labels: severity: warning + org: ccchh annotations: summary: Prometheus rule evaluation slow (instance {{ $labels.instance }}) description: "Prometheus rule evaluation took more time than the scheduled interval. It indicates a slower storage backend access or too complex query.\n VALUE = {{ $value }}" @@ -519,6 +527,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus AlertManager notification failing (instance {{ $labels.instance }}) description: "Alertmanager is failing sending notifications\n VALUE = {{ $value }}" @@ -527,6 +536,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus target empty (instance {{ $labels.instance }}) description: "Prometheus has no target in service discovery\n VALUE = {{ $value }}" @@ -535,6 +545,7 @@ groups: for: 5m labels: severity: warning + org: ccchh annotations: summary: Prometheus target scraping slow (instance {{ $labels.instance }}) description: "Prometheus is scraping exporters slowly since it exceeded the requested interval time. Your Prometheus server is under-provisioned.\n VALUE = {{ $value }}" @@ -575,6 +586,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus TSDB compactions failed (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} TSDB compactions failures\n VALUE = {{ $value }}" @@ -583,6 +595,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus TSDB head truncations failed (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} TSDB head truncation failures\n VALUE = {{ $value }}" @@ -591,6 +604,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus TSDB reload failures (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} TSDB reload failures\n VALUE = {{ $value }}" @@ -599,6 +613,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus TSDB WAL corruptions (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} TSDB WAL corruptions\n VALUE = {{ $value }}" @@ -607,6 +622,7 @@ groups: for: 0m labels: severity: critical + org: ccchh annotations: summary: Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }}) description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures\n VALUE = {{ $value }}" @@ -615,6 +631,7 @@ groups: for: 0m labels: severity: warning + org: ccchh annotations: summary: Prometheus timeseries cardinality (instance {{ $labels.instance }}) description: "The \"{{ $labels.name }}\" timeseries cardinality is getting very high: {{ $value }}\n VALUE = {{ $value }}"