nixos-config/hosts/falkenstein/modules/monitoring/default.nix

{ config, ... }:
{
  age.secrets."maxmind" = {
    file = ../../../../secrets/shared/maxmind.age;
  };
  users.users."promtail".extraGroups = [ "caddy" "systemd-journal" ];
  services.prometheus = {
    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
      };
    };
  };
  services.geoipupdate = {
    enable = true;
    settings = {
      AccountID = 1018346;
      LicenseKey = config.age.secrets."maxmind".path;
      EditionIDs = [
        "GeoLite2-ASN"
        "GeoLite2-City"
        "GeoLite2-Country"
      ];
      DatabaseDirectory = "/var/lib/GeoIP";
    };
  };
  services.promtail = {
    enable = true;
    configuration = {
      server = {
        http_listen_port = 3031;
        grpc_listen_port = 0;
      };
      positions = {
        filename = "/tmp/positions.yaml";
      };
      clients = [{
        url = "http://nuc.vpn.rfive.de:3030/loki/api/v1/push";
      }];
      scrape_configs = [
        {
          job_name = "journal";
          journal = {
            json = false;
            max_age = "12h";
            path = "/var/log/journal";
            labels.job = "systemd-journal";
          };
          relabel_configs = [
            {
              source_labels = [ "__journal__systemd_unit" ];
              target_label = "unit";
            }
            {
              source_labels = [ "__journal__hostname" ];
              target_label = "host";
            }
            {
              source_labels = [ "__journal_priority_keyword" ];
              target_label = "level";
            }
            {
              source_labels = [ "__journal_syslog_identifier" ];
              target_label = "syslog_identifier";
            }
          ];
          pipeline_stages = [
            {
              match = {
                selector = ''{unit="promtail.servicel"}'';
                action = "drop";
              };
            }
          ];
        }
        {
          job_name = "caddy_access_log";
          static_configs = [
            {
              targets = [ "localhost" ];
              labels = {
                job = "caddy_access_log";
                agent = "caddy-promtail";
                __path__ = "/var/log/caddy/*.log";
              };
            }
          ];
          pipeline_stages = [
            {
              # remove :443 from matrix or rspamd logs
              replace = {
                expression = ".*(de:443).*";
                replace = "de";
              };
            }
            {
              json.expressions.remote_ip = "request.remote_ip";
            }
            {
              geoip = {
                db = "/var/lib/GeoIP/GeoLite2-City.mmdb";
                source = "remote_ip";
                db_type = "city";
              };
            }
          ];

        }
      ];
    };
  };
  networking.firewall.allowedTCPPorts = [
    config.services.prometheus.exporters.node.port
  ];
}