diff --git a/data/zones/noc.eh22.intern.zone b/data/zones/noc.eh22.intern.zone
index afbd71f..140579b 100644
--- a/data/zones/noc.eh22.intern.zone
+++ b/data/zones/noc.eh22.intern.zone
@@ -4,6 +4,8 @@ $ORIGIN noc.eh22.intern.
auth-dns 0 IN A 10.20.25.3
monitoring 0 IN A 10.20.25.4
grafana 0 IN A 10.20.25.4
+loki 0 IN A 10.20.25.4
+mimir 0 IN A 10.20.25.4
proxmox 0 IN A 10.20.25.1
resolv-dns 0 IN A 10.20.25.5
sketchy-router 0 IN A 10.20.25.2
diff --git a/secrets/passwords.yaml b/secrets/passwords.yaml
index 57dea97..3453f35 100644
--- a/secrets/passwords.yaml
+++ b/secrets/passwords.yaml
@@ -6,6 +6,8 @@ services:
root: ENC[AES256_GCM,data:RVv1d/nB9pgcERkujSasoLY+cR3OO3NWxw==,iv:EHkUDxP6XB2JWeDtno2rcVvBQdJ/jmG5HjRjPppfS0A=,tag:obzij0BkGLJoXfUbqWLRjw==,type:str]
grafana:
admin_password: ENC[AES256_GCM,data:EimHeXiWzrzDVHnqmfAs+6/jsNp0cyVRJQu8U7drsP4=,iv:WmpPZstgTru8AHg5VeKRhfFdc0r5J9OWhCHdCzw/g+E=,tag:uftQ1kgj8LAuFB+MLSqnJw==,type:str]
+ loki:
+ basic_auth: ENC[AES256_GCM,data:nr03H2jHSwn49EorFmapJ6MZ,iv:TNao6XZJPW+BXpeB0BOvV9Rty6N4sgpVWiwCDSKrVcQ=,tag:YmCkIUwfBCcASGIxlT4pGg==,type:str]
hardware:
proxmox_server:
ipmi:
@@ -47,8 +49,8 @@ sops:
SHhiSU5PZmpUakZvQVNtYk5nUk1tZjAKyHND2LZuuBciy7toDLrAH47kyWcGAN7c
ORrD03DBoEV7mjBY86Hl3SaLKHxlBXsB93OOWqeZrvHlbki+qn/OZA==
-----END AGE ENCRYPTED FILE-----
- lastmodified: "2025-03-07T23:08:48Z"
- mac: ENC[AES256_GCM,data:lUivE03Wq9mRDLwVpazQFrc0XxqXhK0pFLYvU+Y/dMB+z7LJ1Y9S9uGmaZwApwv3FTiSiCjBqVse4ok2FXokvxAPoCnJ5tGw7gq93XY/e/MBXDO40C9ltc1ilmsueCX7f8ZDjg2LfH2LRLYltVhyAekpaiaY9Cv5EUOU635xRp4=,iv:QH9ot5PiWQ+IuOdA6Hv3PuHgw5BnN1PsZe0032IJjjI=,tag:5HU0UNHVm4AxcyUxBbRuGQ==,type:str]
+ lastmodified: "2025-03-10T21:45:17Z"
+ mac: ENC[AES256_GCM,data:8Qa+pqO3pm2geqpTTvOUyR8QEMh1KS7YorGn3P+Zt+Ts9bD4Dn/HkuYnE+vjmtAotl8OnSdGf6gaGEMSC2O7mHz6pbplsYWrho45i6ZkpLh7gDG+UR7hp5pjC1K6oX3wIGzfr1z3VyGRt8dnAaEk01tTIztoP5dFSzZrFeliUis=,iv:mgjqoJfkChkUkpeBQgUxdonalmtfI6fzO+TttS8LShk=,tag:tT57gxysO4Np/BZaKUDVug==,type:str]
pgp: []
unencrypted_suffix: _unencrypted
version: 3.9.4
diff --git a/systems/monitoring.noc.eh22.intern/grafana.nix b/systems/monitoring.noc.eh22.intern/grafana.nix
index 7a670e1..49c2e05 100644
--- a/systems/monitoring.noc.eh22.intern/grafana.nix
+++ b/systems/monitoring.noc.eh22.intern/grafana.nix
@@ -23,6 +23,27 @@
user = "grafana";
host = "/run/postgresql";
};
+ feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
+ security.angular_support_enabled = false;
+ };
+ provision = {
+ enable = true;
+ datasources.settings.datasources = [
+ {
+ name = "Mimir";
+ type = "prometheus";
+ uid = "mimir";
+ access = "proxy";
+ url = "http://127.0.0.1:9009/prometheus";
+ isDefault = true;
+ }
+ {
+ name = "Loki";
+ type = "loki";
+ uid = "loki";
+ url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+ }
+ ];
};
};
@@ -39,6 +60,7 @@
nginx = {
enable = true;
+ recommendedProxySettings = true;
virtualHosts.${config.services.grafana.settings.server.domain} = {
locations."/" = {
proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
diff --git a/systems/monitoring.noc.eh22.intern/loki.nix b/systems/monitoring.noc.eh22.intern/loki.nix
new file mode 100644
index 0000000..00fb4d4
--- /dev/null
+++ b/systems/monitoring.noc.eh22.intern/loki.nix
@@ -0,0 +1,70 @@
+{
+ pkgs,
+ config,
+ ...
+}:
+{
+ services = {
+ loki = {
+ enable = true;
+ configuration = {
+ auth_enabled = false;
+ server.http_listen_port = 3100;
+
+ common = {
+ ring.kvstore.store = "inmemory";
+ ring.instance_addr = "127.0.0.1";
+ replication_factor = 1;
+ };
+
+ storage_config = {
+ filesystem.directory = "/var/lib/loki/chunks";
+ tsdb_shipper = {
+ active_index_directory = "tsdb_active_index";
+ cache_location = "tsdb_cache";
+ };
+ };
+
+ compactor = {
+ working_directory = "/var/lib/loki/compactor";
+ compaction_interval = "10m";
+ retention_enabled = true;
+ retention_delete_delay = "1s";
+ retention_delete_worker_count = 150;
+ delete_request_store = "filesystem";
+ };
+
+ limits_config.retention_period = "1y";
+
+ schema_config = {
+ configs = [
+ {
+ from = "2025-03-01";
+ store = "tsdb";
+ object_store = "filesystem";
+ schema = "v13";
+ index = {
+ prefix = "index_";
+ period = "24h";
+ };
+ }
+ ];
+ };
+ };
+ };
+
+ nginx = {
+ upstreams.loki = {
+ servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = { };
+ extraConfig = "keepalive 20;";
+ };
+
+ virtualHosts."loki.noc.eh22.intern" = {
+ locations."/loki/api/v1/push" = {
+ proxyPass = "http://loki";
+ #basicAuthFile = config.sops.secrets."services/loki/basic_auth".path;
+ };
+ };
+ };
+ };
+}
diff --git a/systems/monitoring.noc.eh22.intern/mimir.nix b/systems/monitoring.noc.eh22.intern/mimir.nix
new file mode 100644
index 0000000..259dbe6
--- /dev/null
+++ b/systems/monitoring.noc.eh22.intern/mimir.nix
@@ -0,0 +1,46 @@
+{
+ pkgs,
+ config,
+ ...
+}:
+{
+ services.mimir = {
+ enable = true;
+ configuration = {
+ multitenancy_enabled = false;
+ target = "all,alertmanager";
+
+ blocks_storage = {
+ backend = "filesystem";
+ };
+
+ server = {
+ http_listen_port = 9009;
+ log_level = "warn";
+ grpc_listen_port = 9096;
+ };
+
+ ingester.ring.replication_factor = 1;
+
+ limits = {
+ ingestion_rate = 1000000; # can't set to unlimited :(
+ out_of_order_time_window = "12h";
+ max_global_series_per_user = 0; # unlimited
+ max_label_value_length = 10000; # we have pgscv queries that are LONG
+ };
+ };
+ };
+
+ services.nginx = {
+ upstreams.mimir = {
+ servers."127.0.0.1:${toString config.services.mimir.configuration.server.http_listen_port}" = { };
+ extraConfig = "keepalive 20;";
+ };
+
+ virtualHosts."mimir.noc.eh22.intern" = {
+ locations."/api/v1/push" = {
+ proxyPass = "http://mimir";
+ };
+ };
+ };
+}
diff --git a/systems/monitoring.noc.eh22.intern/system.nix b/systems/monitoring.noc.eh22.intern/system.nix
index a134aa1..dd9aa54 100644
--- a/systems/monitoring.noc.eh22.intern/system.nix
+++ b/systems/monitoring.noc.eh22.intern/system.nix
@@ -12,6 +12,9 @@
group = "grafana";
restartUnits = [ "grafana.service" ];
};
+ secrets."services/loki/basic_auth" = {
+ mode = "0777";
+ };
};
# configure static IP address
@@ -29,8 +32,58 @@
networking.firewall.allowedTCPPorts = [ 80 ];
+ services.alloy = {
+ enable = true;
+ configPath = "/etc/alloy/config.alloy";
+ };
+
+ environment.etc."alloy/config.alloy" = {
+ text = ''
+ prometheus.remote_write "default" {
+ endpoint {
+ url = "http://127.0.0.1:9009/api/v1/push"
+ }
+ }
+ loki.write "default" {
+ endpoint {
+ url = "http://127.0.0.1:3100/loki/api/v1/push"
+ }
+ }
+
+ loki.relabel "journal" {
+ forward_to = []
+
+ rule {
+ source_labels = ["__journal__systemd_unit"]
+ target_label = "unit"
+ }
+ }
+
+ loki.source.journal "read_journal" {
+ forward_to = [loki.write.default.receiver]
+ relabel_rules = loki.relabel.journal.rules
+ labels = {component = "loki.source.journal", host = "${config.networking.hostName}"}
+ }
+
+ logging {
+ level = "info"
+ format = "logfmt"
+ }
+
+ prometheus.exporter.unix "local_system" { }
+
+ prometheus.scrape "scrape_metrics" {
+ targets = prometheus.exporter.unix.local_system.targets
+ forward_to = [prometheus.remote_write.default.receiver]
+ scrape_interval = "10s"
+ }
+ '';
+ };
+
imports = [
./grafana.nix
+ ./mimir.nix
+ ./loki.nix
];
# DO NOT CHANGE