nuc: enable loki and promtail

hosts/falkenstein/modules/monitoring/default.nix

@@ -1,5 +1,9 @@
 { config, ... }:
 {
+  age.secrets."maxmind" = {
+    file = ../../../../secrets/shared/maxmind.age;
+  };
+  users.users."promtail".extraGroups = [ "caddy" ];
   services.prometheus = {
     exporters = {
       node = {
@@ -11,6 +15,62 @@
       };
     };
   };
+  services.geoipupdate = {
+    enable = true;
+    settings = {
+      AccountID = 1018346;
+      LicenseKey = config.age.secrets."maxmind".path;
+      EditionIDs = [
+        "GeoLite2-ASN"
+        "GeoLite2-City"
+        "GeoLite2-Country"
+      ];
+      DatabaseDirectory = "/var/lib/GeoIP";
+    };
+  };
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3031;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "http://nuc.vpn.rfive.de:3030/loki/api/v1/push";
+      }];
+      scrape_configs = [
+        {
+          job_name = "caddy_access_log";
+          static_configs = [
+            {
+              targets = [ "localhost" ];
+              labels = {
+                job = "caddy_access_log";
+                agent = "caddy-promtail";
+                __path__ = "/var/log/caddy/*.log";
+              };
+            }
+          ];
+          pipeline_stages = [
+            {
+              json.expressions.remote_ip = "request.remote_ip";
+            }
+            {
+              geoip = {
+                db = "/var/lib/GeoIP/GeoLite2-City.mmdb";
+                source = "remote_ip";
+                db_type = "city";
+              };
+            }
+          ];
+
+        }
+      ];
+    };
+  };
   networking.firewall.allowedTCPPorts = [
     config.services.prometheus.exporters.node.port
     config.services.prometheus.exporters.postfix.port

hosts/nuc/modules/monitoring/default.nix

@@ -8,10 +8,10 @@ in
     owner = "grafana";
   };
   age.secrets."maxmind" = {
-    file = ../../../../secrets/nuc/maxmind.age;
-    owner = "grafana";
+    file = ../../../../secrets/shared/maxmind.age;
   };
-  users.users."promtail".extraGroups = [ "caddy" ];
+  users.users."promtail".extraGroups = [ "caddy" "systemd-journal" ];
+  networking.firewall.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ];
   # grafana configuration
 
   # todo: move to own file
@@ -48,6 +48,7 @@ in
         user = "grafana";
         host = "/run/postgresql";
       };
+      auth.disable_login_form = true;
       "auth.generic_oauth" = {
         enabled = true;
         name = "Authentik";
@@ -109,6 +110,30 @@ in
           targets = [ "falkenstein.vpn.rfive.de:${toString config.services.prometheus.exporters.postfix.port}" ];
         }];
       }
+      {
+        job_name = "synapse";
+        static_configs = [{
+          targets = [ "matrix.rfive.de:8008" ];
+        }];
+        metrics_path = "/synapse/metrics";
+        scrape_interval = "15s";
+      }
+      {
+        job_name = "rspamd";
+        static_configs = [{
+          targets = [ "falkenstein.vpn.rfive.de:11334" ];
+        }];
+      }
+      {
+        job_name = "caddy";
+        static_configs = [{
+          targets = [
+            "falkenstein.vpn.rfive.de:2018"
+            "nuc.vpn.rfive.de:2018"
+          ];
+        }];
+        scrape_interval = "15s";
+      }
     ];
   };
   services.loki = {
@@ -205,6 +230,41 @@ in
         url = "http://nuc.vpn.rfive.de:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
       }];
       scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            json = false;
+            max_age = "12h";
+            path = "/var/log/journal";
+            labels.job = "systemd-journal";
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+            {
+              source_labels = [ "__journal__hostname" ];
+              target_label = "host";
+            }
+            {
+              source_labels = [ "__journal_priority_keyword" ];
+              target_label = "level";
+            }
+            {
+              source_labels = [ "__journal_syslog_identifier" ];
+              target_label = "syslog_identifier";
+            }
+          ];
+          pipeline_stages = [
+            {
+              match = {
+                selector = ''{unit="promtail.servicel"}'';
+                action = "drop";
+              };
+            }
+          ];
+        }
         {
           job_name = "caddy_access_log";
           static_configs = [
@@ -219,6 +279,13 @@ in
             }
           ];
           pipeline_stages = [
+            {
+              # remove :443 from matrix or rspamd logs
+              replace = {
+                expression = ".*(de:443).*";
+                replace = "de";
+              };
+            }
             {
               json.expressions.remote_ip = "request.remote_ip";
             }
@@ -236,7 +303,6 @@ in
     };
   };
 
-
   # nginx reverse proxy
   services.caddy.virtualHosts.${domain}.extraConfig = ''
     reverse_proxy 127.0.0.1:${toString config.services.grafana.settings.server.http_port}

secrets.nix

@@ -26,7 +26,6 @@ in
   "secrets/nuc/authentik/ldap.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/grafana/oidc.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/cache.age".publicKeys = [ rouven nuc ];
-  "secrets/nuc/maxmind.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/borg/passphrase.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/borg/key.age".publicKeys = [ rouven nuc ];
 
@@ -37,4 +36,7 @@ in
   "secrets/falkenstein/wireguard/dorm/preshared.age".publicKeys = [ rouven falkenstein ];
   "secrets/falkenstein/borg/passphrase.age".publicKeys = [ rouven falkenstein ];
   "secrets/falkenstein/borg/key.age".publicKeys = [ rouven falkenstein ];
+
+  #shared
+  "secrets/shared/maxmind.age".publicKeys = [ rouven nuc falkenstein ];
 }

secrets/shared/maxmind.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 uWbAHQ hL+MYiYI/53SAw5Ue9L2E/W1sCwENhTqBReBwlRn6g0
+laaky6yfLkEPofvdZwu64WyVqPcxTt8Lng/uhBHaKjs
+-> ssh-ed25519 2TRdXg dXERMyE1LqPxbAKn24SHruqrgKUTSIOLjy66nxiJSiE
+lMGTDVxDUSu7r9Lp7mTfCzuTiUONv/K9b6y4mRlLLj8
+-> ssh-ed25519 slrRig Q7EcsiO/jsscDk9hHhtkHVxQ+NRO6O9SSQu4dfCPXG8
+LGCdVmGbMASuGGGuVrom+1ijafq0Sk0PDnyhOv2O2A0
+--- YeAR7BXc2heRrnvLa9YDGRIgI/3EQ3MfIJEZAJen8pY
+Mü$¦
óNù~KI ÀJÑÏ•èUæ¦.1q¶Y„‹-€"ë/_Øëý 2^“-Dÿ¯¬­Å4ã£/b+
+ô›V^MX_ç®
ñ±