From c2b6c5ddcb4e073ea6a1c71bb644323c5ffa0ce6 Mon Sep 17 00:00:00 2001
From: c6ristian <c6ristian@christian.moe>
Date: Sat, 15 Mar 2025 01:38:57 +0100
Subject: [PATCH] better logging
---
modules/alloy.nix | 7 ++++++-
systems/monitoring.noc.eh22.intern/loki.nix | 17 +++++++++++++++--
systems/monitoring.noc.eh22.intern/system.nix | 3 +++
3 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/modules/alloy.nix b/modules/alloy.nix
index 19d2c52..e601180 100644
--- a/modules/alloy.nix
+++ b/modules/alloy.nix
@@ -58,7 +58,7 @@
}
rule {
source_labels = ["__journal__hostname"]
- target_label = "systemd_hostname"
+ target_label = "instance"
}
rule {
source_labels = ["__journal__transport"]
@@ -68,11 +68,16 @@
source_labels = ["__journal_syslog_identifier"]
target_label = "syslog_identifier"
}
+ rule {
+ source_labels = ["__journal_priority_keyword"]
+ target_label = "level"
+ }
}
loki.source.journal "read_journal" {
forward_to = [loki.write.default.receiver]
relabel_rules = loki.relabel.journal.rules
+ format_as_json = true
labels = {component = "loki.source.journal", host = "${config.networking.hostName}"}
}
diff --git a/systems/monitoring.noc.eh22.intern/loki.nix b/systems/monitoring.noc.eh22.intern/loki.nix
index a740d4b..56f4195 100644
--- a/systems/monitoring.noc.eh22.intern/loki.nix
+++ b/systems/monitoring.noc.eh22.intern/loki.nix
@@ -9,7 +9,11 @@
enable = true;
configuration = {
auth_enabled = false;
- server.http_listen_port = 3100;
+
+ server = {
+ http_listen_port = 3100;
+ log_level = "warn";
+ };
common = {
ring.kvstore.store = "inmemory";
@@ -34,7 +38,16 @@
delete_request_store = "filesystem";
};
- limits_config.retention_period = "1y";
+ ruler = {
+ storage = {
+ type = "local";
+ local.directory = "/var/lib/loki/rules";
+ };
+ rule_path = "/var/lib/loki/ruler";
+ enable_api = true;
+ ring.kvstore.store = "inmemory";
+ alertmanager_url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/";
+ };
schema_config = {
configs = [
diff --git a/systems/monitoring.noc.eh22.intern/system.nix b/systems/monitoring.noc.eh22.intern/system.nix
index 7c1c8ae..cb182a6 100644
--- a/systems/monitoring.noc.eh22.intern/system.nix
+++ b/systems/monitoring.noc.eh22.intern/system.nix
@@ -11,18 +11,21 @@
owner = "grafana";
group = "grafana";
restartUnits = [ "grafana.service" ];
+ sopsFile = ../../secrets/passwords.yaml;
};
secrets."services/loki/nginx" = {
mode = "0440";
owner = "nginx";
group = "nginx";
restartUnits = [ "nginx.service" ];
+ sopsFile = ../../secrets/passwords.yaml;
};
secrets."services/mimir/nginx" = {
mode = "0440";
owner = "nginx";
group = "nginx";
restartUnits = [ "nginx.service" ];
+ sopsFile = ../../secrets/passwords.yaml;
};
};