better logging

modules/alloy.nix

@@ -58,7 +58,7 @@
         }
         rule {
           source_labels = ["__journal__hostname"]
-          target_label = "systemd_hostname"
+          target_label = "instance"
         }
         rule {
           source_labels = ["__journal__transport"]
@@ -68,11 +68,16 @@
           source_labels = ["__journal_syslog_identifier"]
           target_label = "syslog_identifier"
         }
+        rule {
+          source_labels = ["__journal_priority_keyword"]
+          target_label  = "level"
+        }
       }
 
       loki.source.journal "read_journal"  {
         forward_to    = [loki.write.default.receiver]
         relabel_rules = loki.relabel.journal.rules
+        format_as_json = true
         labels        = {component = "loki.source.journal", host = "${config.networking.hostName}"}
       }
 

systems/monitoring.noc.eh22.intern/loki.nix

@@ -9,7 +9,11 @@
       enable = true;
       configuration = {
         auth_enabled = false;
-        server.http_listen_port = 3100;
+
+        server = {
+          http_listen_port = 3100;
+          log_level = "warn";
+        };
 
         common = {
           ring.kvstore.store = "inmemory";
@@ -34,7 +38,16 @@
           delete_request_store = "filesystem";
         };
 
-        limits_config.retention_period = "1y";
+        ruler = {
+          storage = {
+            type = "local";
+            local.directory = "/var/lib/loki/rules";
+          };
+          rule_path = "/var/lib/loki/ruler";
+          enable_api = true;
+          ring.kvstore.store = "inmemory";
+          alertmanager_url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/";
+        };
 
         schema_config = {
           configs = [

systems/monitoring.noc.eh22.intern/system.nix

@@ -11,18 +11,21 @@
       owner = "grafana";
       group = "grafana";
       restartUnits = [ "grafana.service" ];
+      sopsFile = ../../secrets/passwords.yaml;
     };
     secrets."services/loki/nginx" = {
       mode = "0440";
       owner = "nginx";
       group = "nginx";
       restartUnits = [ "nginx.service" ];
+      sopsFile = ../../secrets/passwords.yaml;
     };
     secrets."services/mimir/nginx" = {
       mode = "0440";
       owner = "nginx";
       group = "nginx";
       restartUnits = [ "nginx.service" ];
+      sopsFile = ../../secrets/passwords.yaml;
     };
   };