more monitoring

flake.lock

@@ -312,11 +312,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716457508,
+        "lastModified": 1717097707,
-        "narHash": "sha256-ZxzffLuWRyuMrkVVq7wastNUqeO0HJL9xqfY1QsYaqo=",
+        "narHash": "sha256-HC5vJ3oYsjwsCaSbkIPv80e4ebJpNvFKQTBOGlHvjLs=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "850cb322046ef1a268449cf1ceda5fd24d930b05",
+        "rev": "0eb314b4f0ba337e88123e0b1e57ef58346aafd9",
         "type": "github"
       },
       "original": {
@@ -460,11 +460,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716170277,
+        "lastModified": 1716772633,
-        "narHash": "sha256-fCAiox/TuzWGVaAz16PxrR4Jtf9lN5dwWL2W74DS0yI=",
+        "narHash": "sha256-Idcye44UW+EgjbjCoklf2IDF+XrehV6CVYvxR1omst4=",
         "owner": "nix-community",
         "repo": "nix-index-database",
-        "rev": "e0638db3db43b582512a7de8c0f8363a162842b9",
+        "rev": "ff80cb4a11bb87f3ce8459be6f16a25ac86eb2ac",
         "type": "github"
       },
       "original": {
@@ -475,11 +475,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1716509168,
+        "lastModified": 1716948383,
-        "narHash": "sha256-4zSIhSRRIoEBwjbPm3YiGtbd8HDWzFxJjw5DYSDy1n8=",
+        "narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bfb7a882678e518398ce9a31a881538679f6f092",
+        "rev": "ad57eef4ef0659193044870c731987a6df5cf56b",
         "type": "github"
       },
       "original": {
@@ -623,11 +623,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716449531,
+        "lastModified": 1717103025,
-        "narHash": "sha256-T/BycXsf5MZM+uqemM2/CzaZSjInKrjJc8MOOAOLKiw=",
+        "narHash": "sha256-bn/YPVgu6YmHnKhwMfwIFe7USGvIOC5ge4Ps6o47Tr8=",
         "owner": "~rouven",
         "repo": "purge",
-        "rev": "4b8353adb065c41d4ca6debba011eb8c1561ce80",
+        "rev": "4f8f075eeaafc90737216031eb644792a4652ead",
         "type": "sourcehut"
       },
       "original": {

hosts/falkenstein/modules/caddy/default.nix

@@ -20,6 +20,14 @@ in
     enable = true;
     email = "ca@${config.networking.domain}";
     logFormat = "format console";
+    globalConfig = ''
+      servers {
+        metrics
+      }
+    '';
+    virtualHosts.":2018".extraConfig = ''
+      metrics
+    '';
     virtualHosts."${config.networking.domain}".extraConfig = ''
       file_server browse
       root * /srv/web/${config.networking.domain}
@@ -28,6 +36,6 @@ in
     '';
   };
   systemd.services.caddy.environment.XDG_DATA_HOME = "/var/lib";
-  networking.firewall.allowedTCPPorts = [ 80 443 ];
+  networking.firewall.allowedTCPPorts = [ 80 443 2018 ];
   networking.firewall.allowedUDPPorts = [ 443 ];
 }

hosts/falkenstein/modules/mail/rspamd.nix

@@ -8,6 +8,7 @@
       locals = {
         "worker-controller.inc".text = ''
           secure_ip = "0.0.0.0/0";
+          bind_socket = "0.0.0.0:11334";
         '';
         "redis.conf".text = ''
           read_servers = "/run/redis-rspamd/redis.sock";
@@ -54,6 +55,7 @@
       reverse_proxy 127.0.0.1:11334
     '';
   };
+  networking.firewall.allowedTCPPorts = [ 11334 ];
   systemd = {
     services.rspamd-dmarc-report = {
       description = "rspamd dmarc reporter";

hosts/falkenstein/modules/monitoring/default.nix

@@ -3,7 +3,7 @@
   age.secrets."maxmind" = {
     file = ../../../../secrets/shared/maxmind.age;
   };
-  users.users."promtail".extraGroups = [ "caddy" ];
+  users.users."promtail".extraGroups = [ "caddy" "systemd-journal" ];
   services.prometheus = {
     exporters = {
       node = {
@@ -42,6 +42,41 @@
         url = "http://nuc.vpn.rfive.de:3030/loki/api/v1/push";
       }];
       scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            json = false;
+            max_age = "12h";
+            path = "/var/log/journal";
+            labels.job = "systemd-journal";
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+            {
+              source_labels = [ "__journal__hostname" ];
+              target_label = "host";
+            }
+            {
+              source_labels = [ "__journal_priority_keyword" ];
+              target_label = "level";
+            }
+            {
+              source_labels = [ "__journal_syslog_identifier" ];
+              target_label = "syslog_identifier";
+            }
+          ];
+          pipeline_stages = [
+            {
+              match = {
+                selector = ''{unit="promtail.servicel"}'';
+                action = "drop";
+              };
+            }
+          ];
+        }
         {
           job_name = "caddy_access_log";
           static_configs = [
@@ -55,6 +90,13 @@
             }
           ];
           pipeline_stages = [
+            {
+              # remove :443 from matrix or rspamd logs
+              replace = {
+                expression = ".*(de:443).*";
+                replace = "de";
+              };
+            }
             {
               json.expressions.remote_ip = "request.remote_ip";
             }

hosts/falkenstein/modules/trucksimulatorbot/default.nix

@@ -35,5 +35,6 @@ in
       uri strip_prefix /images
       reverse_proxy unix//run/trucksimulator/images.sock
     }
+    reverse_proxy unix//run/trucksimulator/app.sock
   '';
 }

hosts/fujitsu/modules/monitoring/default.nix

@@ -1,5 +1,6 @@
 { config, ... }:
 {
+  users.users."promtail".extraGroups = [ "caddy" "systemd-journal" ];
   services.prometheus = {
     exporters = {
       node = {
@@ -8,6 +9,58 @@
       };
     };
   };
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3031;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "http://nuc.vpn.rfive.de:3030/loki/api/v1/push";
+      }];
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            json = false;
+            max_age = "12h";
+            path = "/var/log/journal";
+            labels.job = "systemd-journal";
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+            {
+              source_labels = [ "__journal__hostname" ];
+              target_label = "host";
+            }
+            {
+              source_labels = [ "__journal_priority_keyword" ];
+              target_label = "level";
+            }
+            {
+              source_labels = [ "__journal_syslog_identifier" ];
+              target_label = "syslog_identifier";
+            }
+          ];
+          pipeline_stages = [
+            {
+              match = {
+                selector = ''{unit="promtail.servicel"}'';
+                action = "drop";
+              };
+            }
+          ];
+        }
+      ];
+    };
+  };
   networking.firewall.allowedTCPPorts = [
     config.services.prometheus.exporters.node.port
   ];

hosts/nuc/modules/caddy/default.nix

@@ -4,6 +4,14 @@
     enable = true;
     email = "ca@${config.networking.domain}";
     logFormat = "format console";
+    globalConfig = ''
+      servers {
+        metrics
+      }
+    '';
+    virtualHosts.":2018".extraConfig = ''
+      metrics
+    '';
   };
   systemd.services.caddy.environment.XDG_DATA_HOME = "/var/lib";
   networking.firewall.allowedTCPPorts = [ 80 443 ];