add mimir and loki

systems/monitoring.noc.eh22.intern/grafana.nix

@@ -23,6 +23,27 @@
           user = "grafana";
           host = "/run/postgresql";
         };
+        feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
+        security.angular_support_enabled = false;
+      };
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Mimir";
+            type = "prometheus";
+            uid = "mimir";
+            access = "proxy";
+            url = "http://127.0.0.1:9009/prometheus";
+            isDefault = true;
+          }
+          {
+            name = "Loki";
+            type = "loki";
+            uid = "loki";
+            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+          }
+        ];
       };
     };
 
@@ -39,6 +60,7 @@
 
     nginx = {
       enable = true;
+      recommendedProxySettings = true;
       virtualHosts.${config.services.grafana.settings.server.domain} = {
         locations."/" = {
           proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";

systems/monitoring.noc.eh22.intern/loki.nix

@@ -0,0 +1,70 @@
+{
+  pkgs,
+  config,
+  ...
+}:
+{
+  services = {
+    loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+        server.http_listen_port = 3100;
+
+        common = {
+          ring.kvstore.store = "inmemory";
+          ring.instance_addr = "127.0.0.1";
+          replication_factor = 1;
+        };
+
+        storage_config = {
+          filesystem.directory = "/var/lib/loki/chunks";
+          tsdb_shipper = {
+            active_index_directory = "tsdb_active_index";
+            cache_location = "tsdb_cache";
+          };
+        };
+
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          compaction_interval = "10m";
+          retention_enabled = true;
+          retention_delete_delay = "1s";
+          retention_delete_worker_count = 150;
+          delete_request_store = "filesystem";
+        };
+
+        limits_config.retention_period = "1y";
+
+        schema_config = {
+          configs = [
+            {
+              from = "2025-03-01";
+              store = "tsdb";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+      };
+    };
+
+    nginx = {
+      upstreams.loki = {
+        servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = { };
+        extraConfig = "keepalive 20;";
+      };
+
+      virtualHosts."loki.noc.eh22.intern" = {
+        locations."/loki/api/v1/push" = {
+          proxyPass = "http://loki";
+          #basicAuthFile = config.sops.secrets."services/loki/basic_auth".path;
+        };
+      };
+    };
+  };
+}

systems/monitoring.noc.eh22.intern/mimir.nix

@@ -0,0 +1,46 @@
+{
+  pkgs,
+  config,
+  ...
+}:
+{
+  services.mimir = {
+    enable = true;
+    configuration = {
+      multitenancy_enabled = false;
+      target = "all,alertmanager";
+
+      blocks_storage = {
+        backend = "filesystem";
+      };
+
+      server = {
+        http_listen_port = 9009;
+        log_level = "warn";
+        grpc_listen_port = 9096;
+      };
+
+      ingester.ring.replication_factor = 1;
+
+      limits = {
+        ingestion_rate = 1000000; # can't set to unlimited :(
+        out_of_order_time_window = "12h";
+        max_global_series_per_user = 0; # unlimited
+        max_label_value_length = 10000; # we have pgscv queries that are LONG
+      };
+    };
+  };
+
+  services.nginx = {
+    upstreams.mimir = {
+      servers."127.0.0.1:${toString config.services.mimir.configuration.server.http_listen_port}" = { };
+      extraConfig = "keepalive 20;";
+    };
+
+    virtualHosts."mimir.noc.eh22.intern" = {
+      locations."/api/v1/push" = {
+        proxyPass = "http://mimir";
+      };
+    };
+  };
+}

systems/monitoring.noc.eh22.intern/system.nix

@@ -12,6 +12,9 @@
       group = "grafana";
       restartUnits = [ "grafana.service" ];
     };
+    secrets."services/loki/basic_auth" = {
+      mode = "0777";
+    };
   };
 
   # configure static IP address
@@ -29,8 +32,58 @@
 
   networking.firewall.allowedTCPPorts = [ 80 ];
 
+  services.alloy = {
+    enable = true;
+    configPath = "/etc/alloy/config.alloy";
+  };
+
+  environment.etc."alloy/config.alloy" = {
+    text = ''
+      prometheus.remote_write "default" {
+        endpoint {
+          url = "http://127.0.0.1:9009/api/v1/push"
+        }
+      }
+      loki.write "default" {
+        endpoint {
+          url = "http://127.0.0.1:3100/loki/api/v1/push" 
+        }
+      }
+
+      loki.relabel "journal" {
+        forward_to = []
+
+        rule {
+          source_labels = ["__journal__systemd_unit"]
+          target_label  = "unit"
+        }
+      }
+
+      loki.source.journal "read_journal"  {
+        forward_to    = [loki.write.default.receiver]
+        relabel_rules = loki.relabel.journal.rules
+        labels        = {component = "loki.source.journal", host = "${config.networking.hostName}"}
+      }
+
+      logging {
+        level = "info"
+        format = "logfmt"
+      }
+
+      prometheus.exporter.unix "local_system" { }
+
+      prometheus.scrape "scrape_metrics" {
+        targets         = prometheus.exporter.unix.local_system.targets
+        forward_to      = [prometheus.remote_write.default.receiver]
+        scrape_interval = "10s"
+      }
+    '';
+  };
+
   imports = [
     ./grafana.nix
+    ./mimir.nix
+    ./loki.nix
   ];
 
   # DO NOT CHANGE