better logging

modules/alloy.nix

@@ -58,7 +58,7 @@
         }
         rule {
           source_labels = ["__journal__hostname"]
-          target_label = "systemd_hostname"
+          target_label = "instance"
         }
         rule {
           source_labels = ["__journal__transport"]
@@ -68,11 +68,16 @@
           source_labels = ["__journal_syslog_identifier"]
           target_label = "syslog_identifier"
         }
+        rule {
+          source_labels = ["__journal_priority_keyword"]
+          target_label  = "level"
+        }
       }
 
       loki.source.journal "read_journal"  {
         forward_to    = [loki.write.default.receiver]
         relabel_rules = loki.relabel.journal.rules
+        format_as_json = true
         labels        = {component = "loki.source.journal", host = "${config.networking.hostName}"}
       }
 

systems/monitoring.noc.eh22.intern/system.nix

@@ -11,18 +11,21 @@
       owner = "grafana";
       group = "grafana";
       restartUnits = [ "grafana.service" ];
+      sopsFile = ../../secrets/passwords.yaml;
     };
     secrets."services/loki/nginx" = {
       mode = "0440";
       owner = "nginx";
       group = "nginx";
       restartUnits = [ "nginx.service" ];
+      sopsFile = ../../secrets/passwords.yaml;
     };
     secrets."services/mimir/nginx" = {
       mode = "0440";
       owner = "nginx";
       group = "nginx";
       restartUnits = [ "nginx.service" ];
+      sopsFile = ../../secrets/passwords.yaml;
     };
   };