add mimir and loki

data/zones/noc.eh22.intern.zone

@@ -4,6 +4,8 @@ $ORIGIN noc.eh22.intern.
 auth-dns 0 IN A 10.20.25.3
 monitoring 0 IN A 10.20.25.4
 grafana 0 IN A 10.20.25.4
+loki 0 IN A 10.20.25.4
+mimir 0 IN A 10.20.25.4
 proxmox 0 IN A 10.20.25.1
 resolv-dns 0 IN A 10.20.25.5
 sketchy-router 0 IN A 10.20.25.2

secrets/passwords.yaml

@@ -6,6 +6,8 @@ services:
         root: ENC[AES256_GCM,data:RVv1d/nB9pgcERkujSasoLY+cR3OO3NWxw==,iv:EHkUDxP6XB2JWeDtno2rcVvBQdJ/jmG5HjRjPppfS0A=,tag:obzij0BkGLJoXfUbqWLRjw==,type:str]
     grafana:
         admin_password: ENC[AES256_GCM,data:EimHeXiWzrzDVHnqmfAs+6/jsNp0cyVRJQu8U7drsP4=,iv:WmpPZstgTru8AHg5VeKRhfFdc0r5J9OWhCHdCzw/g+E=,tag:uftQ1kgj8LAuFB+MLSqnJw==,type:str]
+    loki:
+        basic_auth: ENC[AES256_GCM,data:nr03H2jHSwn49EorFmapJ6MZ,iv:TNao6XZJPW+BXpeB0BOvV9Rty6N4sgpVWiwCDSKrVcQ=,tag:YmCkIUwfBCcASGIxlT4pGg==,type:str]
 hardware:
     proxmox_server:
         ipmi:
@@ -47,8 +49,8 @@ sops:
             SHhiSU5PZmpUakZvQVNtYk5nUk1tZjAKyHND2LZuuBciy7toDLrAH47kyWcGAN7c
             ORrD03DBoEV7mjBY86Hl3SaLKHxlBXsB93OOWqeZrvHlbki+qn/OZA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-03-07T23:08:48Z"
-    mac: ENC[AES256_GCM,data:lUivE03Wq9mRDLwVpazQFrc0XxqXhK0pFLYvU+Y/dMB+z7LJ1Y9S9uGmaZwApwv3FTiSiCjBqVse4ok2FXokvxAPoCnJ5tGw7gq93XY/e/MBXDO40C9ltc1ilmsueCX7f8ZDjg2LfH2LRLYltVhyAekpaiaY9Cv5EUOU635xRp4=,iv:QH9ot5PiWQ+IuOdA6Hv3PuHgw5BnN1PsZe0032IJjjI=,tag:5HU0UNHVm4AxcyUxBbRuGQ==,type:str]
+    lastmodified: "2025-03-10T21:45:17Z"
+    mac: ENC[AES256_GCM,data:8Qa+pqO3pm2geqpTTvOUyR8QEMh1KS7YorGn3P+Zt+Ts9bD4Dn/HkuYnE+vjmtAotl8OnSdGf6gaGEMSC2O7mHz6pbplsYWrho45i6ZkpLh7gDG+UR7hp5pjC1K6oX3wIGzfr1z3VyGRt8dnAaEk01tTIztoP5dFSzZrFeliUis=,iv:mgjqoJfkChkUkpeBQgUxdonalmtfI6fzO+TttS8LShk=,tag:tT57gxysO4Np/BZaKUDVug==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.4

systems/monitoring.noc.eh22.intern/grafana.nix

@@ -23,6 +23,27 @@
           user = "grafana";
           host = "/run/postgresql";
         };
+        feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
+        security.angular_support_enabled = false;
+      };
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Mimir";
+            type = "prometheus";
+            uid = "mimir";
+            access = "proxy";
+            url = "http://127.0.0.1:9009/prometheus";
+            isDefault = true;
+          }
+          {
+            name = "Loki";
+            type = "loki";
+            uid = "loki";
+            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+          }
+        ];
       };
     };
 
@@ -39,6 +60,7 @@
 
     nginx = {
       enable = true;
+      recommendedProxySettings = true;
       virtualHosts.${config.services.grafana.settings.server.domain} = {
         locations."/" = {
           proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";

systems/monitoring.noc.eh22.intern/loki.nix

@@ -0,0 +1,70 @@
+{
+  pkgs,
+  config,
+  ...
+}:
+{
+  services = {
+    loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+        server.http_listen_port = 3100;
+
+        common = {
+          ring.kvstore.store = "inmemory";
+          ring.instance_addr = "127.0.0.1";
+          replication_factor = 1;
+        };
+
+        storage_config = {
+          filesystem.directory = "/var/lib/loki/chunks";
+          tsdb_shipper = {
+            active_index_directory = "tsdb_active_index";
+            cache_location = "tsdb_cache";
+          };
+        };
+
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          compaction_interval = "10m";
+          retention_enabled = true;
+          retention_delete_delay = "1s";
+          retention_delete_worker_count = 150;
+          delete_request_store = "filesystem";
+        };
+
+        limits_config.retention_period = "1y";
+
+        schema_config = {
+          configs = [
+            {
+              from = "2025-03-01";
+              store = "tsdb";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+      };
+    };
+
+    nginx = {
+      upstreams.loki = {
+        servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = { };
+        extraConfig = "keepalive 20;";
+      };
+
+      virtualHosts."loki.noc.eh22.intern" = {
+        locations."/loki/api/v1/push" = {
+          proxyPass = "http://loki";
+          #basicAuthFile = config.sops.secrets."services/loki/basic_auth".path;
+        };
+      };
+    };
+  };
+}

systems/monitoring.noc.eh22.intern/mimir.nix

@@ -0,0 +1,46 @@
+{
+  pkgs,
+  config,
+  ...
+}:
+{
+  services.mimir = {
+    enable = true;
+    configuration = {
+      multitenancy_enabled = false;
+      target = "all,alertmanager";
+
+      blocks_storage = {
+        backend = "filesystem";
+      };
+
+      server = {
+        http_listen_port = 9009;
+        log_level = "warn";
+        grpc_listen_port = 9096;
+      };
+
+      ingester.ring.replication_factor = 1;
+
+      limits = {
+        ingestion_rate = 1000000; # can't set to unlimited :(
+        out_of_order_time_window = "12h";
+        max_global_series_per_user = 0; # unlimited
+        max_label_value_length = 10000; # we have pgscv queries that are LONG
+      };
+    };
+  };
+
+  services.nginx = {
+    upstreams.mimir = {
+      servers."127.0.0.1:${toString config.services.mimir.configuration.server.http_listen_port}" = { };
+      extraConfig = "keepalive 20;";
+    };
+
+    virtualHosts."mimir.noc.eh22.intern" = {
+      locations."/api/v1/push" = {
+        proxyPass = "http://mimir";
+      };
+    };
+  };
+}

systems/monitoring.noc.eh22.intern/system.nix

@@ -12,6 +12,9 @@
       group = "grafana";
       restartUnits = [ "grafana.service" ];
     };
+    secrets."services/loki/basic_auth" = {
+      mode = "0777";
+    };
   };
 
   # configure static IP address
@@ -29,8 +32,58 @@
 
   networking.firewall.allowedTCPPorts = [ 80 ];
 
+  services.alloy = {
+    enable = true;
+    configPath = "/etc/alloy/config.alloy";
+  };
+
+  environment.etc."alloy/config.alloy" = {
+    text = ''
+      prometheus.remote_write "default" {
+        endpoint {
+          url = "http://127.0.0.1:9009/api/v1/push"
+        }
+      }
+      loki.write "default" {
+        endpoint {
+          url = "http://127.0.0.1:3100/loki/api/v1/push" 
+        }
+      }
+
+      loki.relabel "journal" {
+        forward_to = []
+
+        rule {
+          source_labels = ["__journal__systemd_unit"]
+          target_label  = "unit"
+        }
+      }
+
+      loki.source.journal "read_journal"  {
+        forward_to    = [loki.write.default.receiver]
+        relabel_rules = loki.relabel.journal.rules
+        labels        = {component = "loki.source.journal", host = "${config.networking.hostName}"}
+      }
+
+      logging {
+        level = "info"
+        format = "logfmt"
+      }
+
+      prometheus.exporter.unix "local_system" { }
+
+      prometheus.scrape "scrape_metrics" {
+        targets         = prometheus.exporter.unix.local_system.targets
+        forward_to      = [prometheus.remote_write.default.receiver]
+        scrape_interval = "10s"
+      }
+    '';
+  };
+
   imports = [
     ./grafana.nix
+    ./mimir.nix
+    ./loki.nix
   ];
 
   # DO NOT CHANGE