tmp

chris-here-bitte/defaults/main.yml

@@ -0,0 +1,5 @@
+monitoring_server_loki_version: "3.4.1"
+monitoring_server_mimir_version: "2.17.0"
+monitoring_server_grafana_version: "12.1.1"
+
+monitoring_server_base_domain: "monitoring.ef29.aut-sys.de"

chris-here-bitte/handlers/main.yaml

@@ -0,0 +1,44 @@
+- tags: [ monitoring_server, monitoring, loki, grafana_loki]
+  name: loki.restarted
+  become: true
+  systemd:
+    name: "loki.service"
+    state: restarted
+
+- tags: [ monitoring_server, monitoring, mimir, grafana_mimir]
+  name: mimir.restarted
+  become: true
+  systemd:
+    name: "mimir.service"
+    state: restarted
+
+- tags: [ monitoring_server, monitoring, grafana ]
+  name: grafana.restarted
+  become: true
+  systemd:
+    name: "grafana.service"
+    state: restarted
+
+- tags: [ monitoring_server, monitoring, reverse_proxy, webserver, caddy ]
+  name: caddy.reloaded
+  become: true
+  systemd:
+    name: "caddy.service"
+    state: reloaded
+
+- tags: [ monitoring, monitoring_server, grafana_alloy, alloy ]
+  name: alloy.reloaded
+  become: true
+  systemd:
+    name: alloy.service
+    state: reloaded
+
+- tags: [ monitoring, monitoring_server, grafana_alloy, alloy ]
+  name: alloy.restarted
+  become: true
+  systemd:
+    name: alloy.service
+    state: restarted
+    daemon_reload: true
+
+

chris-here-bitte/tasks/add_alloy_configs.yaml

@@ -0,0 +1,28 @@
+- tags: [ grafana_alloy, alloy ]
+  block:
+
+    - name: Deploy extra grafana-alloy config files
+      become: true
+      notify: alloy.reloaded
+      with_filetree: "templates/alloy/"
+      loop_control:
+        label: "{{ item.path }}"
+      template:
+        src: "alloy/{{ item.path }}"
+        dest: "/etc/alloy/{{ item.path }}"
+        owner: alloy
+        group: alloy
+        mode: "u=r,g=r,o="
+
+    - name: Add alloy systemd override directory
+      become: true
+      file:
+        path: /etc/systemd/system/alloy.service.d/
+        state: directory
+
+    - name: Configure systemd service override
+      notify: alloy.restarted
+      become: true
+      template:
+        src: alloy.override.conf
+        dest: /etc/systemd/system/alloy.service.d/override.conf

chris-here-bitte/tasks/install_container_runtime.yaml

@@ -0,0 +1,7 @@
+- tags: [ container_runtime, podman, container ]
+  block:
+
+    - name: Install podman
+      become: true
+      package:
+        name: [ podman ]

chris-here-bitte/tasks/main.yaml

@@ -0,0 +1,8 @@
+- tags: [ monitoring_server, monitoring ]
+  block:
+    - import_tasks: install_container_runtime.yaml
+    - import_tasks: setup_loki.yaml
+    - import_tasks: setup_mimir.yaml
+    - import_tasks: setup_grafana.yaml
+    - import_tasks: setup_reverse_proxy.yaml
+    - import_tasks: add_alloy_configs.yaml

chris-here-bitte/tasks/setup_grafana.yaml

@@ -0,0 +1,90 @@
+- tags: [ grafana ]
+  block:
+
+    - name: Create grafana group
+      become: true
+      register: grafana_group
+      group:
+        name: grafana
+
+    - name: Create grafana user
+      become: true
+      register: grafana_user
+      user:
+        name: grafana
+        group: grafana
+        system: true
+
+    - name: Create grafana configuration directory
+      become: true
+      file:
+        path: /etc/grafana
+        state: directory
+        owner: "{{ grafana_user.name }}"
+        group: "{{ grafana_group.name }}"
+
+    - name: Create grafana state directory
+      become: true
+      file:
+        path: /var/lib/grafana
+        state: directory
+        owner: "{{ grafana_user.name }}"
+        group: "{{ grafana_group.name }}"
+
+    - name: Deploy grafana configuration
+      become: true
+      notify: grafana.restarted
+      template:
+        src: grafana.ini
+        dest: /etc/grafana/grafana.ini
+        owner: "{{ grafana_user.name }}"
+        group: "{{ grafana_group.name }}"
+        mode: "u=rwX,g=rX,o="
+
+    - name: Create grafana provisioning directories
+      become: true
+      with_filetree: "templates/grafana_provisioning"
+      when: "item.state == 'directory'"
+      loop_control:
+        label: "{{ item.path }}"
+      file:
+        path: "/etc/grafana/provisioning/{{ item.path }}"
+        state: directory
+        owner: "{{ grafana_user.name }}"
+        group: "{{ grafana_group.name }}"
+        mode: "u=rwX,g=rX,o="
+
+    - name: Create grafana provisioning files
+      become: true
+      with_filetree: "templates/grafana_provisioning"
+      when: "item.state == 'file'"
+      notify: grafana.restarted
+      loop_control:
+        label: "{{ item.path }}"
+      template:
+        src: "grafana_provisioning/{{ item.path }}"
+        dest: "/etc/grafana/provisioning/{{ item.path }}"
+        owner: "{{ grafana_user.name }}"
+        group: "{{ grafana_group.name }}"
+        mode: "u=rw,g=r,o="
+
+    - name: Configure grafana quadlet
+      become: true
+      notify: grafana.restarted
+      register: grafana_quadlet
+      vars:
+        user_id: "{{ grafana_user.uid }}"
+        group_id: "{{ grafana_group.gid }}"
+      template:
+        src: grafana.container
+        dest: /etc/containers/systemd/grafana.container
+        owner: "{{ grafana_user.name }}"
+        group: "{{ grafana_group.name }}"
+
+    - name: Ensure grafana is running and enabled
+      become: true
+      systemd:
+        name: "grafana.service"
+        state: started
+        enabled: true
+        daemon_reload: "{{ grafana_quadlet.changed }}"

chris-here-bitte/tasks/setup_loki.yaml

@@ -0,0 +1,60 @@
+- tags: [ loki, grafana_loki ]
+  block:
+
+    - name: Create loki group
+      become: true
+      register: loki_group
+      group:
+        name: loki
+
+    - name: Create loki user
+      become: true
+      register: loki_user
+      user:
+        name: loki
+        group: loki
+        system: true
+
+    - name: Create loki configuration directory
+      become: true
+      file:
+        path: /etc/loki
+        state: directory
+        owner: "{{ loki_user.name }}"
+        group: "{{ loki_group.name }}"
+
+    - name: Create loki state directory
+      become: true
+      file:
+        path: /var/lib/loki
+        state: directory
+        owner: "{{ loki_user.name }}"
+        group: "{{ loki_group.name }}"
+
+    - name: Deploy loki configuration
+      become: true
+      notify: loki.restarted
+      template:
+        src: loki-config.yaml
+        dest: /etc/loki/config.yaml
+        owner: "{{ loki_user.name }}"
+        group: "{{ loki_group.name }}"
+
+    - name: Configure loki quadlet
+      become: true
+      notify: loki.restarted
+      register: loki_quadlet
+      vars:
+        user_id: "{{ loki_user.uid }}"
+        group_id: "{{ loki_group.gid }}"
+      template:
+        src: loki.container
+        dest: /etc/containers/systemd/loki.container
+
+    - name: Ensure loki is running and enabled
+      become: true
+      systemd:
+        name: "loki.service"
+        state: started
+        enabled: true
+        daemon_reload: "{{ loki_quadlet.changed }}"

chris-here-bitte/tasks/setup_mimir.yaml

@@ -0,0 +1,60 @@
+- tags: [ mimir, grafana_mimir ]
+  block:
+
+    - name: Create mimir group
+      become: true
+      register: mimir_group
+      group:
+        name: mimir
+
+    - name: Create mimir user
+      become: true
+      register: mimir_user
+      user:
+        name: mimir
+        group: mimir
+        system: true
+
+    - name: Create mimir configuration directory
+      become: true
+      file:
+        path: /etc/mimir
+        state: directory
+        owner: "{{ mimir_user.name }}"
+        group: "{{ mimir_group.name }}"
+
+    - name: Create mimir state directory
+      become: true
+      file:
+        path: /var/lib/mimir
+        state: directory
+        owner: "{{ mimir_user.name }}"
+        group: "{{ mimir_group.name }}"
+
+    - name: Deploy mimir configuration
+      become: true
+      notify: mimir.restarted
+      template:
+        src: mimir-config.yaml
+        dest: /etc/mimir/config.yaml
+        owner: "{{ mimir_user.name }}"
+        group: "{{ mimir_group.name }}"
+
+    - name: Configure mimir quadlet
+      become: true
+      notify: mimir.restarted
+      register: mimir_quadlet
+      vars:
+        user_id: "{{ mimir_user.uid }}"
+        group_id: "{{ mimir_group.gid }}"
+      template:
+        src: mimir.container
+        dest: /etc/containers/systemd/mimir.container
+
+    - name: Ensure mimir is running and enabled
+      become: true
+      systemd:
+        name: "mimir.service"
+        state: started
+        enabled: true
+        daemon_reload: "{{ mimir_quadlet.changed }}"

chris-here-bitte/tasks/setup_reverse_proxy.yaml

@@ -0,0 +1,21 @@
+- tags: [ reverse_proxy, webserver, caddy ]
+  block:
+
+    - name: Install caddy
+      become: true
+      package:
+        name: [ caddy ]
+
+    - name: Configure caddy
+      become: true
+      notify: caddy.reloaded
+      template:
+        src: Caddyfile
+        dest: /etc/caddy/Caddyfile
+
+    - name: Ensure caddy is running and enabled
+      become: true
+      systemd:
+        name: caddy.service
+        state: started
+        enabled: true

chris-here-bitte/templates/Caddyfile

@@ -0,0 +1,25 @@
+# {{ ansible_managed }}
+
+grafana.{{ monitoring_server_base_domain }} {
+  reverse_proxy localhost:3000
+}
+
+loki.{{ monitoring_server_base_domain }} {
+  basicauth {
+    {% for username, pw in (lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]') | from_yaml).items() -%}
+    {{ username }} {{ pw | password_hash(hashtype="bcrypt") }}
+    {% endfor %}
+  }
+
+  reverse_proxy localhost:3100
+}
+
+mimir.{{ monitoring_server_base_domain }} {
+  basicauth {
+    {% for username, pw in (lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]') | from_yaml).items() -%}
+    {{ username }} {{ pw | password_hash(hashtype="bcrypt") }}
+    {% endfor %}
+  }
+
+  reverse_proxy localhost:9009
+}

chris-here-bitte/templates/alloy.override.conf

@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+[Service]
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE

chris-here-bitte/templates/alloy/blackbox-exporter-config.yaml

@@ -0,0 +1,24 @@
+# {{ ansible_managed }}
+modules:
+  https4_2xx:
+    prober: http
+    http:
+      valid_status_codes: [ ]  # Defaults to 2xx
+      method: GET
+      follow_redirects: true
+      fail_if_ssl: false
+      fail_if_not_ssl: true
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false
+
+  tcp4_connect:
+    prober: tcp
+    tcp:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false
+
+  icmp4:
+    prober: icmp
+    icmp:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false

chris-here-bitte/templates/alloy/blackbox-exporter.alloy

@@ -0,0 +1,102 @@
+// {{ ansible_managed }}
+// For a full configuration reference, see https://grafana.com/docs/alloy
+
+discovery.http "netbox_devices" {
+  url = "{{ alloy_netbox_uri | mandatory }}/api/plugins/prometheus-sd/devices/"
+  http_headers = {
+    "Authorization" = [ "Token {{ alloy_netbox_api_key | mandatory }}" ],
+  }
+}
+
+// ICMP probe definitions
+discovery.relabel "icmp_probes_netbox" {
+  targets = discovery.http.netbox_devices.targets
+
+  // drop devices which don't have an IP address
+  rule {
+      source_labels = [ "__meta_netbox_primary_ip4" ]
+      regex = ".+"
+      action = "keep"
+  }
+
+  // configure target metadata from discovery
+  rule {
+    source_labels = [ "__meta_netbox_primary_ip4" ]
+    target_label = "address"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_name" ]
+    target_label = "name"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_location" ]
+    target_label = "location"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_role" ]
+    target_label = "netbox_role"
+  }
+  rule {
+    source_labels = [ "__meta_netbox_status" ]
+    target_label = "netbox_status"
+  }
+
+  // set probe module to icmp4 to execute pings via ipv4
+  rule {
+    target_label = "module"
+    replacement = "icmp4"
+  }
+}
+
+
+discovery.file "icmp_probes" {
+  files = [ "/etc/alloy/icmp_probes.json" ]
+}
+
+
+prometheus.exporter.blackbox "icmp" {
+  config_file = "/etc/alloy/blackbox-exporter-config.yaml"
+  targets = discovery.file.icmp_probes.targets
+}
+
+
+
+// HTTPS probe definitions
+prometheus.exporter.blackbox "web" {
+  config_file = "/etc/alloy/blackbox-exporter-config.yaml"
+
+  target {
+    name = "eurofurence.org"
+    address = "https://www.eurofurence.org/EF29/"
+    module = "https4_2xx"
+    labels = { probed_instance = "eurofurence.org" }
+  }
+}
+
+
+
+// scraping config
+discovery.relabel "blackbox_probes" {
+  targets = array.concat(
+    prometheus.exporter.blackbox.icmp.targets,
+    prometheus.exporter.blackbox.web.targets,
+  )
+
+  // add "probe_module" label based on which probe module was used
+  rule {
+    source_labels = [ "__param_module" ]
+    target_label = "probe_module"
+  }
+  // add "probe_target" label based on what was probed
+  rule {
+    source_labels = [ "__param_target" ]
+    target_label = "probe_target"
+  }
+}
+
+prometheus.scrape "blackbox_probes" {
+  targets = discovery.relabel.blackbox_probes.output
+  forward_to = [ prometheus.remote_write.ef_mimir.receiver ]
+  scrape_interval = "30s"
+  scrape_timeout = "30s"
+}

chris-here-bitte/templates/alloy/icmp_probes.json

@@ -0,0 +1,34 @@
+[
+  {% for i in query("netbox.netbox.nb_lookup", "devices", api_endpoint="https://netbox.ef.scientress.de/") -%}
+  {% if i.value.role.slug not in ["patchpanel", "accesspoint"] -%}
+  {
+    "targets": [ "" ],
+    "labels": {
+      "name": "{{ i.value.name }}",
+      "probed_instance": "{{ i.value.name }}",
+      "location": "{{ i.value.location.name | default("unknown") }}",
+      "netbox_type": "device",
+      "netbox_role": "{{ i.value.role.slug | default("undefined") }}",
+      "netbox_status": "{{ i.value.status.value }}",
+      "module": "icmp4",
+      "address": "{{ (i.value.primary_ip4.address | default("192.0.2.0/24")).rsplit("/", maxsplit=1) | first }}"
+    }
+  },
+  {% endif %}
+  {% endfor %}
+
+  {% for i in query("netbox.netbox.nb_lookup", "virtual-machines", api_endpoint="https://netbox.ef.scientress.de/") -%}
+  {
+    "targets": [ "" ],
+    "labels": {
+      "name": "{{ i.value.name }}",
+      "probed_instance": "{{ i.value.name }}",
+      "netbox_type": "virtual-machine",
+      "netbox_role": "{{ i.value.role.slug | default("undefined") }}",
+      "netbox_status": "{{ i.value.status.value }}",
+      "module": "icmp4",
+      "address": "{{ (i.value.primary_ip4.address | default("192.0.2.0/24")).rsplit("/", maxsplit=1) | first }}"
+    }
+  }{% if not loop.last %},{% endif %}
+  {% endfor %}
+]

chris-here-bitte/templates/alloy/syslog.alloy

@@ -0,0 +1,19 @@
+// {{ ansible_managed }}
+// For a full configuration reference, see https://grafana.com/docs/alloy
+
+otelcol.receiver.syslog "ef_syslog" {
+  location = "Europe/Berlin"
+
+  tcp {
+    listen_address = "{{ primary_ip4 }}:514"
+  }
+  udp {
+    listen_address = "{{ primary_ip4 }}:514"
+  }
+
+  output {
+    logs = [ otelcol.processor.batch.default.input ]
+    metrics = [ otelcol.processor.batch.default.input ]
+    traces = [ otelcol.processor.batch.default.input ]
+  }
+}

chris-here-bitte/templates/grafana.container

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+
+[Container]
+Image=docker.io/grafana/grafana:{{ monitoring_server_grafana_version }}
+Exec=--config=/etc/grafana/grafana.ini
+Environment=GF_PLUGINS_PREINSTALL=grafana-polystat-panel
+Network=host
+UserNS=host
+User={{ user_id }}
+Group={{ group_id }}
+Volume=/etc/grafana/:/etc/grafana/:ro
+Volume=/var/lib/grafana/:/var/lib/grafana
+
+[Install]
+WantedBy=default.target

chris-here-bitte/templates/grafana.ini

@@ -0,0 +1,32 @@
+# {{ ansible_managed}}
+instance_name = monitoring.ef29.internal
+
+[paths]
+data = /var/lib/grafana
+provisioning = /etc/grafana/provisioning
+
+[log]
+mode = console
+
+[database]
+type = sqlite3
+path = /var/lib/grafana/db.sqlite3
+wal = true
+
+[analytics]
+enabled = false
+reporting_enabled = false
+
+[security]
+admin_user = admin
+admin_password = {{ lookup("community.sops.sops", "passwords.enc.yaml", extract='["grafana_users"]["admin"]') }}
+secret_key = {{ lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["grafana"]["secret_key"]') }}
+disable_gravatar = true
+
+[users]
+allow_sign_up = false
+auto_assign_org_role = Admin
+verify_email_enabled = false
+
+[news]
+news_feed_enabled = false

chris-here-bitte/templates/grafana_provisioning/datasources/local-loki.yaml

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+# https://grafana.com/docs/grafana/latest/administration/provisioning/
+apiVersion: 1
+prune: true
+datasources:
+  - name: Loki
+    version: 6
+    type: loki
+    url: https://loki.{{ monitoring_server_base_domain }}
+    basicAuth: true
+    basicAuthUser: remote-write
+    jsonData:
+      maxLines: 1000
+    secureJsonData:
+      basicAuthPassword: {{ lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]["remote-write"]') }}

chris-here-bitte/templates/grafana_provisioning/datasources/local-mimir.yaml

@@ -0,0 +1,19 @@
+# {{ ansible_managed }}
+# https://grafana.com/docs/grafana/latest/administration/provisioning/
+apiVersion: 1
+prune: true
+datasources:
+  - name: Mimir
+    version: 9
+    type: prometheus
+    url: https://mimir.{{ monitoring_server_base_domain }}/prometheus
+    basicAuth: true
+    basicAuthUser: remote-write
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+      prometheusType: Mimir
+      prometheusVersion: "2.9.1"  # well, >2.9.1 but that is assumed with this value -.-
+      defaultEditor: code
+    secureJsonData:
+      basicAuthPassword: {{ lookup("community.sops.sops", "monitoring_secrets.enc.yaml", extract='["caddy_basic_auth"]["remote-write"]') }}

chris-here-bitte/templates/loki-config.yaml

@@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /mnt/storage
+  storage:
+    filesystem:
+      chunks_directory: /mnt/storage/chunks
+      rules_directory: /mnt/storage/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: false
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+
+analytics:
+  reporting_enabled: false

chris-here-bitte/templates/loki.container

@@ -0,0 +1,14 @@
+# {{ ansible_managed }}
+
+[Container]
+Image=docker.io/grafana/loki:{{ monitoring_server_loki_version }}
+Exec=-config.file=/mnt/config/config.yaml
+Network=host
+UserNS=host
+User={{ user_id }}
+Group={{ group_id }}
+Volume=/etc/loki/:/mnt/config/:ro
+Volume=/var/lib/loki/:/mnt/storage/
+
+[Install]
+WantedBy=default.target

chris-here-bitte/templates/mimir-config.yaml

@@ -0,0 +1,52 @@
+multitenancy_enabled: false
+
+blocks_storage:
+  backend: filesystem
+  bucket_store:
+    sync_dir: /mnt/storage/tsdb-sync
+  filesystem:
+    dir: /mnt/storage/data/tsdb
+  tsdb:
+    dir: /mnt/storage/tsdb
+
+compactor:
+  data_dir: /mnt/storage/compactor
+  sharding_ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+distributor:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+ingester:
+  ring:
+    replication_factor: 1
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+ruler:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+ruler_storage:
+  backend: filesystem
+  filesystem:
+    dir: /mnt/mimir/rules
+
+server:
+  http_listen_port: 9009
+  log_level: info
+
+store_gateway:
+  sharding_ring:
+    replication_factor: 1
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory

chris-here-bitte/templates/mimir.container

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+
+[Container]
+Image=docker.io/grafana/mimir:{{ monitoring_server_mimir_version }}
+Exec=--config.file=/mnt/config/config.yaml
+Network=host
+UserNS=host
+User={{ user_id }}
+Group={{ group_id }}
+WorkingDir=/mnt/storage/
+Volume=/etc/mimir/:/mnt/config/:ro
+Volume=/var/lib/mimir/:/mnt/storage/
+
+[Install]
+WantedBy=default.target