nixos-config/hosts/nuc/modules/monitoring/default.nix

{ config, pkgs, ... }:
let
  domain = "monitoring.${config.networking.domain}";
in
{
  age.secrets."grafana/oidc_secret" = {
    file = ../../../../secrets/nuc/grafana/oidc.age;
    owner = "grafana";
  };
  age.secrets."maxmind" = {
    file = ../../../../secrets/shared/maxmind.age;
  };
  users.users."promtail".extraGroups = [ "caddy" "systemd-journal" ];
  networking.firewall.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ];
  # grafana configuration

  # todo: move to own file
  services.geoipupdate = {
    enable = true;
    settings = {
      AccountID = 1018346;
      LicenseKey = config.age.secrets."maxmind".path;
      EditionIDs = [
        "GeoLite2-ASN"
        "GeoLite2-City"
        "GeoLite2-Country"
      ];
      DatabaseDirectory = "/var/lib/GeoIP";
    };
  };


  services.grafana = {
    enable = true;
    declarativePlugins = with pkgs.grafanaPlugins; [
      grafana-worldmap-panel
      grafana-piechart-panel
    ];
    settings = {
      server = {
        inherit domain;
        http_addr = "127.0.0.1";
        http_port = 2342;
        root_url = "https://${domain}";
      };
      database = {
        type = "postgres";
        user = "grafana";
        host = "/run/postgresql";
      };
      auth.disable_login_form = true;
      "auth.generic_oauth" = {
        enabled = true;
        name = "Authentik";
        allow_sign_up = true;
        client_id = "grafana";
        client_secret = "$__file{${config.age.secrets."grafana/oidc_secret".path}}";
        scopes = "openid email profile offline_access roles";

        email_attribute_path = "email";
        login_attribute_path = "username";
        name_attribute_path = "full_name";

        auth_url = "https://auth.rfive.de/application/o/authorize/";
        token_url = "https://auth.rfive.de/application/o/token/";
        api_url = "https://auth.rfive.de/application/o/userinfo/";
        role_attribute_path = "contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'";

      };

    };
  };

  services.postgresql = {
    enable = true;
    ensureUsers = [
      {
        name = "grafana";
        ensureDBOwnership = true;
      }
    ];
    ensureDatabases = [ "grafana" ];
  };

  services.prometheus = {
    enable = true;
    port = 9001;
    retentionTime = "1y";
    ruleFiles = [
      ./synapse-v2.rules
    ];
    exporters = {
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
      };
      # json = {
      #   enable = true;
      #   configFile = pkgs.writeText "json-exporter.yml" ''
      #     ---
      #     modules:
      #       pegelstand:
      #         metrics:
      #           - name: pegelstand_elbe_dresden
      #             path: '{ $.pegel }'
      #             type: value
      #             help: Pegelstand in Dresden
      #   '';
      # };
    };
    scrapeConfigs = [
      {
        job_name = "node";
        static_configs = [{
          targets = [
            "nuc.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
            "falkenstein.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
            "cudy.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
            "fujitsu.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
          ];
        }];
        scrape_interval = "15s";
      }
      {
        job_name = "synapse";
        static_configs = [{
          targets = [ "nuc.vpn.rfive.de:8008" ];
        }];
        metrics_path = "/_synapse/metrics";
        scrape_interval = "15s";
      }
      {
        job_name = "rspamd";
        static_configs = [{
          targets = [ "falkenstein.vpn.rfive.de:11334" ];
        }];
      }
      {
        job_name = "authentik";
        static_configs = [{
          targets = [ "nuc.vpn.rfive.de:9300" ];
        }];
      }
      # {
      #   job_name = "pegel_dresden";
      #   metrics_path = "/probe";
      #   params = {
      #     module = [ "pegelstand" ];
      #     target = [
      #       "https://api.stramke.com/wasserstand/sachsen/Dresden"
      #     ];
      #   };
      #   static_configs = [{
      #     targets = [ "nuc.vpn.rfive.de:7979" ];
      #   }];
      #   scrape_interval = "5m";
      # }
      {
        job_name = "caddy";
        static_configs = [{
          targets = [
            "falkenstein.vpn.rfive.de:2018"
            "nuc.vpn.rfive.de:2018"
          ];
        }];
        scrape_interval = "15s";
      }
    ];
  };
  services.loki = {
    enable = true;
    # copied from https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
    configuration = {
      server.http_listen_port = 3030;
      auth_enabled = false;
      common = {
        path_prefix = "/tmp/loki";
      };

      ingester = {
        lifecycler = {
          address = "127.0.0.1";
          ring = {
            kvstore = {
              store = "inmemory";
            };
            replication_factor = 1;
          };
        };
        chunk_idle_period = "1h";
        max_chunk_age = "1h";
        chunk_target_size = 999999;
        chunk_retain_period = "30s";
        # max_transfer_retries = 0;
      };

      schema_config = {
        configs = [{
          from = "2022-06-06";
          store = "tsdb";
          object_store = "filesystem";
          schema = "v13";
          index = {
            prefix = "index_";
            period = "24h";
          };
        }];
      };

      storage_config = {
        boltdb_shipper = {
          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
          cache_location = "/var/lib/loki/boltdb-shipper-cache";
          cache_ttl = "24h";
          # shared_store = "filesystem";
        };

        filesystem = {
          directory = "/var/lib/loki/chunks";
        };
      };

      limits_config = {
        reject_old_samples = true;
        reject_old_samples_max_age = "168h";
      };

      # chunk_store_config = {
      #   max_look_back_period = "0s";
      # };

      table_manager = {
        retention_deletes_enabled = false;
        retention_period = "0s";
      };

      compactor = {
        working_directory = "/var/lib/loki";
        # shared_store = "filesystem";
        compactor_ring = {
          kvstore = {
            store = "inmemory";
          };
        };
      };
    };
  };
  # also copied

  services.promtail = {
    enable = true;
    configuration = {
      server = {
        http_listen_port = 3031;
        grpc_listen_port = 0;
      };
      positions = {
        filename = "/tmp/positions.yaml";
      };
      clients = [{
        url = "http://nuc.vpn.rfive.de:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
      }];
      scrape_configs = [
        {
          job_name = "journal";
          journal = {
            json = false;
            max_age = "12h";
            path = "/var/log/journal";
            labels.job = "systemd-journal";
          };
          relabel_configs = [
            {
              source_labels = [ "__journal__systemd_unit" ];
              target_label = "unit";
            }
            {
              source_labels = [ "__journal__hostname" ];
              target_label = "host";
            }
            {
              source_labels = [ "__journal_priority_keyword" ];
              target_label = "level";
            }
            {
              source_labels = [ "__journal_syslog_identifier" ];
              target_label = "syslog_identifier";
            }
          ];
          pipeline_stages = [
            {
              match = {
                selector = ''{unit="promtail.servicel"}'';
                action = "drop";
              };
            }
          ];
        }
        {
          job_name = "caddy_access_log";
          static_configs = [
            {
              targets = [ "localhost" ];
              labels = {
                job = "caddy_access_log";
                # host = "matrix.rfive.de";
                agent = "caddy-promtail";
                __path__ = "/var/log/caddy/*.log";
                host = "nuc";
              };
            }
          ];
          pipeline_stages = [
            {
              # remove :443 from matrix or rspamd logs
              replace = {
                expression = ".*(de:443).*";
                replace = "de";
              };
            }
            {
              json.expressions.remote_ip = "request.remote_ip";
            }
            {
              geoip = {
                db = "/var/lib/GeoIP/GeoLite2-City.mmdb";
                source = "remote_ip";
                db_type = "city";
              };
            }
            {
              geoip = {
                db = "/var/lib/GeoIP/GeoLite2-ASN.mmdb";
                source = "remote_ip";
                db_type = "asn";
              };
            }
            {
              labeldrop = [
                "geoip_subdivision_code"
                "geoip_continent_code"
              ];
            }
          ];

        }
      ];
    };
  };

  # nginx reverse proxy
  services.caddy.virtualHosts.${domain}.extraConfig = ''
    reverse_proxy 127.0.0.1:${toString config.services.grafana.settings.server.http_port}
  '';
}