nuc: configure loki

pain

hosts/falkenstein/modules/logging/default.nix

@@ -19,11 +19,11 @@
       weekly
       rotate 156
     }
-    "/var/log/caddy/*.log" {
-      compress
-      delaycompress
-      weekly
-      rotate 26
-    }
   '';
+  # "/var/log/caddy/*.log" {
+  #   compress
+  #   delaycompress
+  #   weekly
+  #   rotate 26
+  # }
 }

hosts/nuc/modules/matrix/default.nix

@@ -42,12 +42,13 @@ in
         server_name = config.networking.domain;
 
         listeners = [{
-          path = "/run/matrix-synapse/server.sock";
-          mode = "666";
+          bind_addresses = [ "127.0.0.1" "::1" ];
+          port = 8008;
+          tls = false;
           type = "http";
           x_forwarded = true;
           resources = [{
-            names = [ "client" "federation" ];
+            names = [ "client" "federation" "metrics" ];
             compress = false;
           }];
         }];
@@ -69,7 +70,7 @@ in
         "${domain}".extraConfig = ''
           reverse_proxy /client/* unix//run/matrix-sliding-sync/server.sock
           reverse_proxy /_matrix/client/unstable/org.matrix.msc3575/sync* unix//run/matrix-sliding-sync/server.sock
-          reverse_proxy unix//run/matrix-synapse/server.sock
+          reverse_proxy 127.0.0.1:8008
         '';
 
         # element

hosts/nuc/modules/monitoring/default.nix

@@ -1,4 +1,4 @@
-{ config, ... }:
+{ config, pkgs, ... }:
 let
   domain = "monitoring.${config.networking.domain}";
 in
@@ -7,9 +7,35 @@ in
     file = ../../../../secrets/nuc/grafana/oidc.age;
     owner = "grafana";
   };
+  age.secrets."maxmind" = {
+    file = ../../../../secrets/nuc/maxmind.age;
+    owner = "grafana";
+  };
+  users.users."promtail".extraGroups = [ "caddy" ];
   # grafana configuration
+
+  # todo: move to own file
+  services.geoipupdate = {
+    enable = true;
+    settings = {
+      AccountID = 1018346;
+      LicenseKey = config.age.secrets."maxmind".path;
+      EditionIDs = [
+        "GeoLite2-ASN"
+        "GeoLite2-City"
+        "GeoLite2-Country"
+      ];
+      DatabaseDirectory = "/var/lib/GeoIP";
+    };
+  };
+
+
   services.grafana = {
     enable = true;
+    declarativePlugins = with pkgs.grafanaPlugins; [
+      grafana-worldmap-panel
+      grafana-piechart-panel
+    ];
     settings = {
       server = {
         inherit domain;
@@ -63,10 +89,6 @@ in
         enable = true;
         enabledCollectors = [ "systemd" ];
       };
-      # postfix = {
-      #   enable = true;
-      #   port = 9003;
-      # };
     };
     scrapeConfigs = [
       {
@@ -89,6 +111,131 @@ in
       }
     ];
   };
+  services.loki = {
+    enable = true;
+    # copied from https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
+    configuration = {
+      server.http_listen_port = 3030;
+      auth_enabled = false;
+      common = {
+        path_prefix = "/tmp/loki";
+      };
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 999999;
+        chunk_retain_period = "30s";
+        # max_transfer_retries = 0;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2022-06-06";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+          # shared_store = "filesystem";
+        };
+
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      # chunk_store_config = {
+      #   max_look_back_period = "0s";
+      # };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki";
+        # shared_store = "filesystem";
+        compactor_ring = {
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+      };
+    };
+  };
+  # also copied
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3031;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "http://nuc.vpn.rfive.de:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+      }];
+      scrape_configs = [
+        {
+          job_name = "caddy_access_log";
+          static_configs = [
+            {
+              targets = [ "localhost" ];
+              labels = {
+                job = "caddy_access_log";
+                # host = "matrix.rfive.de";
+                agent = "caddy-promtail";
+                __path__ = "/var/log/caddy/*.log";
+              };
+            }
+          ];
+          pipeline_stages = [
+            {
+              json.expressions.remote_ip = "request.remote_ip";
+            }
+            {
+              geoip = {
+                db = "/var/lib/GeoIP/GeoLite2-City.mmdb";
+                source = "remote_ip";
+                db_type = "city";
+              };
+            }
+          ];
+
+        }
+      ];
+    };
+  };
+
 
   # nginx reverse proxy
   services.caddy.virtualHosts.${domain}.extraConfig = ''