more monitoring

flake.lock

@@ -312,11 +312,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717097707,
+        "lastModified": 1717525419,
-        "narHash": "sha256-HC5vJ3oYsjwsCaSbkIPv80e4ebJpNvFKQTBOGlHvjLs=",
+        "narHash": "sha256-5z2422pzWnPXHgq2ms8lcCfttM0dz+hg+x1pCcNkAws=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "0eb314b4f0ba337e88123e0b1e57ef58346aafd9",
+        "rev": "a7117efb3725e6197dd95424136f79147aa35e5b",
         "type": "github"
       },
       "original": {
@@ -460,11 +460,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716772633,
+        "lastModified": 1717297675,
-        "narHash": "sha256-Idcye44UW+EgjbjCoklf2IDF+XrehV6CVYvxR1omst4=",
+        "narHash": "sha256-43UmlS1Ifx17y93/Vc258U7bOlAAIZbu8dsGDHOIIr0=",
         "owner": "nix-community",
         "repo": "nix-index-database",
-        "rev": "ff80cb4a11bb87f3ce8459be6f16a25ac86eb2ac",
+        "rev": "972a52bee3991ae1f1899e6452e0d7c01ee566d9",
         "type": "github"
       },
       "original": {
@@ -475,11 +475,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1716948383,
+        "lastModified": 1717602782,
-        "narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=",
+        "narHash": "sha256-pL9jeus5QpX5R+9rsp3hhZ+uplVHscNJh8n8VpqscM0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ad57eef4ef0659193044870c731987a6df5cf56b",
+        "rev": "e8057b67ebf307f01bdcc8fba94d94f75039d1f6",
         "type": "github"
       },
       "original": {

hosts/falkenstein/modules/caddy/default.nix

@@ -25,9 +25,14 @@ in
         metrics
       }
     '';
-    virtualHosts.":2018".extraConfig = ''
+    virtualHosts.":2018" = {
-      metrics
+      extraConfig = ''
-    '';
+        metrics
+      '';
+      logFormat = ''
+        output discard
+      '';
+    };
     virtualHosts."${config.networking.domain}".extraConfig = ''
       file_server browse
       root * /srv/web/${config.networking.domain}

hosts/falkenstein/modules/monitoring/default.nix

@@ -86,6 +86,7 @@
                 job = "caddy_access_log";
                 agent = "caddy-promtail";
                 __path__ = "/var/log/caddy/*.log";
+                host = "falkenstein";
               };
             }
           ];
@@ -107,6 +108,19 @@
                 db_type = "city";
               };
             }
+            {
+              geoip = {
+                db = "/var/lib/GeoIP/GeoLite2-ASN.mmdb";
+                source = "remote_ip";
+                db_type = "asn";
+              };
+            }
+            {
+              labeldrop = [
+                "geoip_subdivision_code"
+                "geoip_continent_code"
+              ];
+            }
           ];
 
         }

hosts/falkenstein/modules/monitoring/dmarc.nix

@@ -1,5 +1,7 @@
-{ config, ... }:
+{ config, lib, ... }:
 {
+  # required for elasticsearch
+  nixpkgs.config.allowUnfree = true;
   age.secrets.dmarc = {
     file = ../../../../secrets/falkenstein/dmarc.age;
   };
@@ -7,7 +9,8 @@
     description = "DMARC Report recipient";
     isNormalUser = true;
   };
-  networking.firewall.allowedTCPPorts = [ config.services.elasticsearch.tcp_port ];
+  networking.firewall.allowedTCPPorts = [ 9200 ];
+  services.elasticsearch.listenAddress = "0.0.0.0";
   services.parsedmarc = {
     enable = true;
     provision = {
@@ -16,7 +19,7 @@
         datasource = false;
       };
       localMail.enable = false;
-      elasticsearch = false;
+      elasticsearch = true;
       geoIp = false;
     };
     settings = {
@@ -28,8 +31,6 @@
           _secret = config.age.secrets.dmarc.path;
         };
       };
-      opensearch.hosts = "localhost:9200";
     };
   };
-  services.opensearch.enable = true;
 }

hosts/nuc/modules/caddy/default.nix

@@ -9,9 +9,14 @@
         metrics
       }
     '';
-    virtualHosts.":2018".extraConfig = ''
+    virtualHosts.":2018" = {
-      metrics
+      extraConfig = ''
-    '';
+        metrics
+      '';
+      logFormat = ''
+        output discard
+      '';
+    };
   };
   systemd.services.caddy.environment.XDG_DATA_HOME = "/var/lib";
   networking.firewall.allowedTCPPorts = [ 80 443 ];

hosts/nuc/modules/matrix/default.nix

@@ -40,9 +40,10 @@ in
 
       settings = {
         server_name = config.networking.domain;
+        enable_metrics = true;
 
         listeners = [{
-          bind_addresses = [ "127.0.0.1" "::1" ];
+          bind_addresses = [ "0.0.0.0" "::1" ];
           port = 8008;
           tls = false;
           type = "http";

hosts/nuc/modules/monitoring/default.nix

@@ -85,6 +85,9 @@ in
   services.prometheus = {
     enable = true;
     port = 9001;
+    ruleFiles = [
+      ./synapse-v2.rules
+    ];
     exporters = {
       node = {
         enable = true;
@@ -107,9 +110,9 @@ in
       {
         job_name = "synapse";
         static_configs = [{
-          targets = [ "matrix.rfive.de:8008" ];
+          targets = [ "nuc.vpn.rfive.de:8008" ];
         }];
-        metrics_path = "/synapse/metrics";
+        metrics_path = "/_synapse/metrics";
         scrape_interval = "15s";
       }
       {
@@ -118,6 +121,12 @@ in
           targets = [ "falkenstein.vpn.rfive.de:11334" ];
         }];
       }
+      {
+        job_name = "authentik";
+        static_configs = [{
+          targets = [ "nuc.vpn.rfive.de:9300" ];
+        }];
+      }
       {
         job_name = "caddy";
         static_configs = [{
@@ -269,6 +278,7 @@ in
                 # host = "matrix.rfive.de";
                 agent = "caddy-promtail";
                 __path__ = "/var/log/caddy/*.log";
+                host = "nuc";
               };
             }
           ];
@@ -290,6 +300,19 @@ in
                 db_type = "city";
               };
             }
+            {
+              geoip = {
+                db = "/var/lib/GeoIP/GeoLite2-ASN.mmdb";
+                source = "remote_ip";
+                db_type = "asn";
+              };
+            }
+            {
+              labeldrop = [
+                "geoip_subdivision_code"
+                "geoip_continent_code"
+              ];
+            }
           ];
 
         }

hosts/nuc/modules/monitoring/synapse-v2.rules

@@ -0,0 +1,74 @@
+groups:
+- name: synapse
+  rules:
+
+  ###
+  ### Prometheus Console Only
+  ### The following rules are only needed if you use the Prometheus Console
+  ### in contrib/prometheus/consoles/synapse.html
+  ###
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_client_sent_edus_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+  - record: 'synapse_federation_client_sent'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_client_sent_queries) by (job)'
+
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_server_received_edus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_server_received_pdus_total + 0'
+  - record: 'synapse_federation_server_received'
+    labels:
+      type: "Query"
+    expr: 'sum(synapse_federation_server_received_queries) by (job)'
+
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "EDU"
+    expr: 'synapse_federation_transaction_queue_pending_edus + 0'
+  - record: 'synapse_federation_transaction_queue_pending'
+    labels:
+      type: "PDU"
+    expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
+  ###
+  ### End of 'Prometheus Console Only' rules block
+  ###
+
+
+  ###
+  ### Grafana Only
+  ### The following rules are only needed if you use the Grafana dashboard
+  ### in contrib/grafana/synapse.json
+  ###
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
+    labels:
+      type: remote
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
+    labels:
+      type: local
+  - record: synapse_storage_events_persisted_by_source_type
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
+    labels:
+      type: bridges
+
+  - record: synapse_storage_events_persisted_by_event_type
+    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
+
+  - record: synapse_storage_events_persisted_by_origin
+    expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
+  ###
+  ### End of 'Grafana Only' rules block
+  ###

users/rouven/modules/packages.nix

@@ -36,7 +36,7 @@
 
     # cryptography
     yubikey-manager
-    python311Packages.pyhanko # broken, TODO fix
+    # python311Packages.pyhanko # broken, TODO fix
     bitwarden-cli
 
     # misc