nuc: configure loki

pain

flake.lock

@@ -12,11 +12,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1715290355,
-        "narHash": "sha256-2T7CHTqBXJJ3ZC6R/4TXTcKoXWHcvubKNj9SfomURnw=",
+        "lastModified": 1716561646,
+        "narHash": "sha256-UIGtLO89RxKt7RF2iEgPikSdU53r6v/6WYB0RW3k89I=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "8d37c5bdeade12b6479c85acd133063ab53187a0",
+        "rev": "c2fc0762bbe8feb06a2e59a364fa81b3a57671c9",
         "type": "github"
       },
       "original": {
@@ -475,11 +475,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1716330097,
-        "narHash": "sha256-8BO3B7e3BiyIDsaKA0tY8O88rClYRTjvAp66y+VBUeU=",
+        "lastModified": 1716509168,
+        "narHash": "sha256-4zSIhSRRIoEBwjbPm3YiGtbd8HDWzFxJjw5DYSDy1n8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "5710852ba686cc1fd0d3b8e22b3117d43ba374c2",
+        "rev": "bfb7a882678e518398ce9a31a881538679f6f092",
         "type": "github"
       },
       "original": {
@@ -537,22 +537,6 @@
         "type": "github"
       }
     },
-    "nixpkgs-systemd-256": {
-      "locked": {
-        "lastModified": 1714430104,
-        "narHash": "sha256-TGCTDeE8lEm/HC92Ev5ql2vx6Z4iUOwF2vsmLn/UjkM=",
-        "owner": "nikstur",
-        "repo": "nixpkgs",
-        "rev": "12215c110b0f3a652953d215e827fd4b56e0f536",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nikstur",
-        "ref": "systemd-256",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "pfersel": {
       "inputs": {
         "nixpkgs": [
@@ -663,7 +647,6 @@
         "nix-colors": "nix-colors",
         "nix-index-database": "nix-index-database",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-systemd-256": "nixpkgs-systemd-256",
         "pfersel": "pfersel",
         "purge": "purge",
         "trucksimulatorbot": "trucksimulatorbot"

flake.nix

@@ -3,7 +3,6 @@
   inputs = {
 
     nixpkgs.url = "nixpkgs/nixos-unstable";
-    nixpkgs-systemd-256.url = "github:nikstur/nixpkgs/systemd-256";
 
     nix-index-database = {
       url = "github:nix-community/nix-index-database";
@@ -56,7 +55,6 @@
   outputs =
     { self
     , nixpkgs
-    , nixpkgs-systemd-256
     , home-manager
     , dns
     , nix-index-database

hosts/falkenstein/modules/logging/default.nix

@@ -19,11 +19,11 @@
       weekly
       rotate 156
     }
-    "/var/log/caddy/*.log" {
-      compress
-      delaycompress
-      weekly
-      rotate 26
-    }
   '';
+  # "/var/log/caddy/*.log" {
+  #   compress
+  #   delaycompress
+  #   weekly
+  #   rotate 26
+  # }
 }

hosts/nuc/modules/matrix/default.nix

@@ -42,12 +42,13 @@ in
         server_name = config.networking.domain;
 
         listeners = [{
-          path = "/run/matrix-synapse/server.sock";
-          mode = "666";
+          bind_addresses = [ "127.0.0.1" "::1" ];
+          port = 8008;
+          tls = false;
           type = "http";
           x_forwarded = true;
           resources = [{
-            names = [ "client" "federation" ];
+            names = [ "client" "federation" "metrics" ];
             compress = false;
           }];
         }];
@@ -69,7 +70,7 @@ in
         "${domain}".extraConfig = ''
           reverse_proxy /client/* unix//run/matrix-sliding-sync/server.sock
           reverse_proxy /_matrix/client/unstable/org.matrix.msc3575/sync* unix//run/matrix-sliding-sync/server.sock
-          reverse_proxy unix//run/matrix-synapse/server.sock
+          reverse_proxy 127.0.0.1:8008
         '';
 
         # element

hosts/nuc/modules/monitoring/default.nix

@@ -1,4 +1,4 @@
-{ config, ... }:
+{ config, pkgs, ... }:
 let
   domain = "monitoring.${config.networking.domain}";
 in
@@ -7,9 +7,35 @@ in
     file = ../../../../secrets/nuc/grafana/oidc.age;
     owner = "grafana";
   };
+  age.secrets."maxmind" = {
+    file = ../../../../secrets/nuc/maxmind.age;
+    owner = "grafana";
+  };
+  users.users."promtail".extraGroups = [ "caddy" ];
   # grafana configuration
+
+  # todo: move to own file
+  services.geoipupdate = {
+    enable = true;
+    settings = {
+      AccountID = 1018346;
+      LicenseKey = config.age.secrets."maxmind".path;
+      EditionIDs = [
+        "GeoLite2-ASN"
+        "GeoLite2-City"
+        "GeoLite2-Country"
+      ];
+      DatabaseDirectory = "/var/lib/GeoIP";
+    };
+  };
+
+
   services.grafana = {
     enable = true;
+    declarativePlugins = with pkgs.grafanaPlugins; [
+      grafana-worldmap-panel
+      grafana-piechart-panel
+    ];
     settings = {
       server = {
         inherit domain;
@@ -63,10 +89,6 @@ in
         enable = true;
         enabledCollectors = [ "systemd" ];
       };
-      # postfix = {
-      #   enable = true;
-      #   port = 9003;
-      # };
     };
     scrapeConfigs = [
       {
@@ -89,6 +111,131 @@ in
       }
     ];
   };
+  services.loki = {
+    enable = true;
+    # copied from https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
+    configuration = {
+      server.http_listen_port = 3030;
+      auth_enabled = false;
+      common = {
+        path_prefix = "/tmp/loki";
+      };
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 999999;
+        chunk_retain_period = "30s";
+        # max_transfer_retries = 0;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2022-06-06";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+          # shared_store = "filesystem";
+        };
+
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      # chunk_store_config = {
+      #   max_look_back_period = "0s";
+      # };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki";
+        # shared_store = "filesystem";
+        compactor_ring = {
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+      };
+    };
+  };
+  # also copied
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3031;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "http://nuc.vpn.rfive.de:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+      }];
+      scrape_configs = [
+        {
+          job_name = "caddy_access_log";
+          static_configs = [
+            {
+              targets = [ "localhost" ];
+              labels = {
+                job = "caddy_access_log";
+                # host = "matrix.rfive.de";
+                agent = "caddy-promtail";
+                __path__ = "/var/log/caddy/*.log";
+              };
+            }
+          ];
+          pipeline_stages = [
+            {
+              json.expressions.remote_ip = "request.remote_ip";
+            }
+            {
+              geoip = {
+                db = "/var/lib/GeoIP/GeoLite2-City.mmdb";
+                source = "remote_ip";
+                db_type = "city";
+              };
+            }
+          ];
+
+        }
+      ];
+    };
+  };
+
 
   # nginx reverse proxy
   services.caddy.virtualHosts.${domain}.extraConfig = ''

overlays/default.nix

@@ -3,6 +3,7 @@ let
   inherit (prev) callPackage;
   inherit (prev) fetchFromGitHub;
   inherit (prev) fetchPypi;
+  inherit (prev) fetchpatch;
   inherit (prev) makeWrapper;
   inherit (prev) python3Packages;
 in
@@ -149,6 +150,4 @@ in
       hash = "sha256-m1kVaO3N7XC1vtnxXX9kMiEFPmZuoopRUYgA7gQzP8w=";
     };
   });
-
-
 }

secrets.nix

@@ -26,6 +26,7 @@ in
   "secrets/nuc/authentik/ldap.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/grafana/oidc.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/cache.age".publicKeys = [ rouven nuc ];
+  "secrets/nuc/maxmind.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/borg/passphrase.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/borg/key.age".publicKeys = [ rouven nuc ];