tmp

chris-here-bitte/templates/Caddyfile

@@ -0,0 +1,25 @@
+# {{ ansible_managed }}
+
+grafana.{{ monitoring_server_base_domain }} {
+  reverse_proxy localhost:3000
+}
+
+loki.{{ monitoring_server_base_domain }} {
+  basicauth {
+    {% for username, pw in (lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]') | from_yaml).items() -%}
+    {{ username }} {{ pw | password_hash(hashtype="bcrypt") }}
+    {% endfor %}
+  }
+
+  reverse_proxy localhost:3100
+}
+
+mimir.{{ monitoring_server_base_domain }} {
+  basicauth {
+    {% for username, pw in (lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]') | from_yaml).items() -%}
+    {{ username }} {{ pw | password_hash(hashtype="bcrypt") }}
+    {% endfor %}
+  }
+
+  reverse_proxy localhost:9009
+}

chris-here-bitte/templates/alloy.override.conf

@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+[Service]
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE

chris-here-bitte/templates/alloy/blackbox-exporter-config.yaml

@@ -0,0 +1,24 @@
+# {{ ansible_managed }}
+modules:
+  https4_2xx:
+    prober: http
+    http:
+      valid_status_codes: [ ]  # Defaults to 2xx
+      method: GET
+      follow_redirects: true
+      fail_if_ssl: false
+      fail_if_not_ssl: true
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false
+
+  tcp4_connect:
+    prober: tcp
+    tcp:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false
+
+  icmp4:
+    prober: icmp
+    icmp:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false

chris-here-bitte/templates/alloy/blackbox-exporter.alloy

@@ -0,0 +1,102 @@
+// {{ ansible_managed }}
+// For a full configuration reference, see https://grafana.com/docs/alloy
+
+discovery.http "netbox_devices" {
+  url = "{{ alloy_netbox_uri | mandatory }}/api/plugins/prometheus-sd/devices/"
+  http_headers = {
+    "Authorization" = [ "Token {{ alloy_netbox_api_key | mandatory }}" ],
+  }
+}
+
+// ICMP probe definitions
+discovery.relabel "icmp_probes_netbox" {
+  targets = discovery.http.netbox_devices.targets
+
+  // drop devices which don't have an IP address
+  rule {
+      source_labels = [ "__meta_netbox_primary_ip4" ]
+      regex = ".+"
+      action = "keep"
+  }
+
+  // configure target metadata from discovery
+  rule {
+    source_labels = [ "__meta_netbox_primary_ip4" ]
+    target_label = "address"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_name" ]
+    target_label = "name"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_location" ]
+    target_label = "location"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_role" ]
+    target_label = "netbox_role"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_status" ]
+    target_label = "netbox_status"
+  }
+
+  // set probe module to icmp4 to execute pings via ipv4
+  rule {
+    target_label = "module"
+    replacement = "icmp4"
+  }
+}
+
+
+discovery.file "icmp_probes" {
+  files = [ "/etc/alloy/icmp_probes.json" ]
+}
+
+
+prometheus.exporter.blackbox "icmp" {
+  config_file = "/etc/alloy/blackbox-exporter-config.yaml"
+  targets = discovery.file.icmp_probes.targets
+}
+
+
+
+// HTTPS probe definitions
+prometheus.exporter.blackbox "web" {
+  config_file = "/etc/alloy/blackbox-exporter-config.yaml"
+
+  target {
+    name = "eurofurence.org"
+    address = "https://www.eurofurence.org/EF29/"
+    module = "https4_2xx"
+    labels = { probed_instance = "eurofurence.org" }
+  }
+}
+
+
+
+// scraping config
+discovery.relabel "blackbox_probes" {
+  targets = array.concat(
+    prometheus.exporter.blackbox.icmp.targets,
+    prometheus.exporter.blackbox.web.targets,
+  )
+
+  // add "probe_module" label based on which probe module was used
+  rule {
+    source_labels = [ "__param_module" ]
+    target_label = "probe_module"
+  }
+  // add "probe_target" label based on what was probed
+  rule {
+    source_labels = [ "__param_target" ]
+    target_label = "probe_target"
+  }
+}
+
+prometheus.scrape "blackbox_probes" {
+  targets = discovery.relabel.blackbox_probes.output
+  forward_to = [ prometheus.remote_write.ef_mimir.receiver ]
+  scrape_interval = "30s"
+  scrape_timeout = "30s"
+}

chris-here-bitte/templates/alloy/icmp_probes.json

@@ -0,0 +1,34 @@
+[
+  {% for i in query("netbox.netbox.nb_lookup", "devices", api_endpoint="https://netbox.ef.scientress.de/") -%}
+  {% if i.value.role.slug not in ["patchpanel", "accesspoint"] -%}
+  {
+    "targets": [ "" ],
+    "labels": {
+      "name": "{{ i.value.name }}",
+      "probed_instance": "{{ i.value.name }}",
+      "location": "{{ i.value.location.name | default("unknown") }}",
+      "netbox_type": "device",
+      "netbox_role": "{{ i.value.role.slug | default("undefined") }}",
+      "netbox_status": "{{ i.value.status.value }}",
+      "module": "icmp4",
+      "address": "{{ (i.value.primary_ip4.address | default("192.0.2.0/24")).rsplit("/", maxsplit=1) | first }}"
+    }
+  },
+  {% endif %}
+  {% endfor %}
+
+  {% for i in query("netbox.netbox.nb_lookup", "virtual-machines", api_endpoint="https://netbox.ef.scientress.de/") -%}
+  {
+    "targets": [ "" ],
+    "labels": {
+      "name": "{{ i.value.name }}",
+      "probed_instance": "{{ i.value.name }}",
+      "netbox_type": "virtual-machine",
+      "netbox_role": "{{ i.value.role.slug | default("undefined") }}",
+      "netbox_status": "{{ i.value.status.value }}",
+      "module": "icmp4",
+      "address": "{{ (i.value.primary_ip4.address | default("192.0.2.0/24")).rsplit("/", maxsplit=1) | first }}"
+    }
+  }{% if not loop.last %},{% endif %}
+  {% endfor %}
+]

chris-here-bitte/templates/alloy/syslog.alloy

@@ -0,0 +1,19 @@
+// {{ ansible_managed }}
+// For a full configuration reference, see https://grafana.com/docs/alloy
+
+otelcol.receiver.syslog "ef_syslog" {
+  location = "Europe/Berlin"
+
+  tcp {
+    listen_address = "{{ primary_ip4 }}:514"
+  }
+  udp {
+    listen_address = "{{ primary_ip4 }}:514"
+  }
+
+  output {
+    logs = [ otelcol.processor.batch.default.input ]
+    metrics = [ otelcol.processor.batch.default.input ]
+    traces = [ otelcol.processor.batch.default.input ]
+  }
+}

chris-here-bitte/templates/grafana.container

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+
+[Container]
+Image=docker.io/grafana/grafana:{{ monitoring_server_grafana_version }}
+Exec=--config=/etc/grafana/grafana.ini
+Environment=GF_PLUGINS_PREINSTALL=grafana-polystat-panel
+Network=host
+UserNS=host
+User={{ user_id }}
+Group={{ group_id }}
+Volume=/etc/grafana/:/etc/grafana/:ro
+Volume=/var/lib/grafana/:/var/lib/grafana
+
+[Install]
+WantedBy=default.target

chris-here-bitte/templates/grafana.ini

@@ -0,0 +1,32 @@
+# {{ ansible_managed}}
+instance_name = monitoring.ef29.internal
+
+[paths]
+data = /var/lib/grafana
+provisioning = /etc/grafana/provisioning
+
+[log]
+mode = console
+
+[database]
+type = sqlite3
+path = /var/lib/grafana/db.sqlite3
+wal = true
+
+[analytics]
+enabled = false
+reporting_enabled = false
+
+[security]
+admin_user = admin
+admin_password = {{ lookup("community.sops.sops", "passwords.enc.yaml", extract='["grafana_users"]["admin"]') }}
+secret_key = {{ lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["grafana"]["secret_key"]') }}
+disable_gravatar = true
+
+[users]
+allow_sign_up = false
+auto_assign_org_role = Admin
+verify_email_enabled = false
+
+[news]
+news_feed_enabled = false

chris-here-bitte/templates/grafana_provisioning/datasources/local-loki.yaml

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+# https://grafana.com/docs/grafana/latest/administration/provisioning/
+apiVersion: 1
+prune: true
+datasources:
+  - name: Loki
+    version: 6
+    type: loki
+    url: https://loki.{{ monitoring_server_base_domain }}
+    basicAuth: true
+    basicAuthUser: remote-write
+    jsonData:
+      maxLines: 1000
+    secureJsonData:
+      basicAuthPassword: {{ lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]["remote-write"]') }}

chris-here-bitte/templates/grafana_provisioning/datasources/local-mimir.yaml

@@ -0,0 +1,19 @@
+# {{ ansible_managed }}
+# https://grafana.com/docs/grafana/latest/administration/provisioning/
+apiVersion: 1
+prune: true
+datasources:
+  - name: Mimir
+    version: 9
+    type: prometheus
+    url: https://mimir.{{ monitoring_server_base_domain }}/prometheus
+    basicAuth: true
+    basicAuthUser: remote-write
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+      prometheusType: Mimir
+      prometheusVersion: "2.9.1"  # well, >2.9.1 but that is assumed with this value -.-
+      defaultEditor: code
+    secureJsonData:
+      basicAuthPassword: {{ lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]["remote-write"]') }}

chris-here-bitte/templates/loki-config.yaml

@@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /mnt/storage
+  storage:
+    filesystem:
+      chunks_directory: /mnt/storage/chunks
+      rules_directory: /mnt/storage/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: false
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+
+analytics:
+  reporting_enabled: false

chris-here-bitte/templates/loki.container

@@ -0,0 +1,14 @@
+# {{ ansible_managed }}
+
+[Container]
+Image=docker.io/grafana/loki:{{ monitoring_server_loki_version }}
+Exec=-config.file=/mnt/config/config.yaml
+Network=host
+UserNS=host
+User={{ user_id }}
+Group={{ group_id }}
+Volume=/etc/loki/:/mnt/config/:ro
+Volume=/var/lib/loki/:/mnt/storage/
+
+[Install]
+WantedBy=default.target

chris-here-bitte/templates/mimir-config.yaml

@@ -0,0 +1,52 @@
+multitenancy_enabled: false
+
+blocks_storage:
+  backend: filesystem
+  bucket_store:
+    sync_dir: /mnt/storage/tsdb-sync
+  filesystem:
+    dir: /mnt/storage/data/tsdb
+  tsdb:
+    dir: /mnt/storage/tsdb
+
+compactor:
+  data_dir: /mnt/storage/compactor
+  sharding_ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+distributor:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+ingester:
+  ring:
+    replication_factor: 1
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+ruler:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+ruler_storage:
+  backend: filesystem
+  filesystem:
+    dir: /mnt/mimir/rules
+
+server:
+  http_listen_port: 9009
+  log_level: info
+
+store_gateway:
+  sharding_ring:
+    replication_factor: 1
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory

chris-here-bitte/templates/mimir.container

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+
+[Container]
+Image=docker.io/grafana/mimir:{{ monitoring_server_mimir_version }}
+Exec=--config.file=/mnt/config/config.yaml
+Network=host
+UserNS=host
+User={{ user_id }}
+Group={{ group_id }}
+WorkingDir=/mnt/storage/
+Volume=/etc/mimir/:/mnt/config/:ro
+Volume=/var/lib/mimir/:/mnt/storage/
+
+[Install]
+WantedBy=default.target