From f829430b35484a4cc72a4d65f3e7c733c12c0158 Mon Sep 17 00:00:00 2001 From: Rouven Seifert Date: Fri, 31 May 2024 14:48:41 +0200 Subject: [PATCH] nuc: enable loki and promtail --- .../modules/monitoring/default.nix | 60 ++++++++++++++ hosts/nuc/modules/monitoring/default.nix | 74 +++++++++++++++++- secrets.nix | 4 +- secrets/nuc/maxmind.age | Bin 363 -> 0 bytes secrets/shared/maxmind.age | 10 +++ 5 files changed, 143 insertions(+), 5 deletions(-) delete mode 100644 secrets/nuc/maxmind.age create mode 100644 secrets/shared/maxmind.age diff --git a/hosts/falkenstein/modules/monitoring/default.nix b/hosts/falkenstein/modules/monitoring/default.nix index ced57e6..0937e35 100644 --- a/hosts/falkenstein/modules/monitoring/default.nix +++ b/hosts/falkenstein/modules/monitoring/default.nix @@ -1,5 +1,9 @@ { config, ... }: { + age.secrets."maxmind" = { + file = ../../../../secrets/shared/maxmind.age; + }; + users.users."promtail".extraGroups = [ "caddy" ]; services.prometheus = { exporters = { node = { @@ -11,6 +15,62 @@ }; }; }; + services.geoipupdate = { + enable = true; + settings = { + AccountID = 1018346; + LicenseKey = config.age.secrets."maxmind".path; + EditionIDs = [ + "GeoLite2-ASN" + "GeoLite2-City" + "GeoLite2-Country" + ]; + DatabaseDirectory = "/var/lib/GeoIP"; + }; + }; + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 3031; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; + }; + clients = [{ + url = "http://nuc.vpn.rfive.de:3030/loki/api/v1/push"; + }]; + scrape_configs = [ + { + job_name = "caddy_access_log"; + static_configs = [ + { + targets = [ "localhost" ]; + labels = { + job = "caddy_access_log"; + agent = "caddy-promtail"; + __path__ = "/var/log/caddy/*.log"; + }; + } + ]; + pipeline_stages = [ + { + json.expressions.remote_ip = "request.remote_ip"; + } + { + geoip = { + db = "/var/lib/GeoIP/GeoLite2-City.mmdb"; + source = "remote_ip"; + db_type = "city"; + }; + } + ]; + + } + ]; + }; + }; networking.firewall.allowedTCPPorts = [ config.services.prometheus.exporters.node.port config.services.prometheus.exporters.postfix.port diff --git a/hosts/nuc/modules/monitoring/default.nix b/hosts/nuc/modules/monitoring/default.nix index 83211c5..ba3aed7 100644 --- a/hosts/nuc/modules/monitoring/default.nix +++ b/hosts/nuc/modules/monitoring/default.nix @@ -8,10 +8,10 @@ in owner = "grafana"; }; age.secrets."maxmind" = { - file = ../../../../secrets/nuc/maxmind.age; - owner = "grafana"; + file = ../../../../secrets/shared/maxmind.age; }; - users.users."promtail".extraGroups = [ "caddy" ]; + users.users."promtail".extraGroups = [ "caddy" "systemd-journal" ]; + networking.firewall.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ]; # grafana configuration # todo: move to own file @@ -48,6 +48,7 @@ in user = "grafana"; host = "/run/postgresql"; }; + auth.disable_login_form = true; "auth.generic_oauth" = { enabled = true; name = "Authentik"; @@ -109,6 +110,30 @@ in targets = [ "falkenstein.vpn.rfive.de:${toString config.services.prometheus.exporters.postfix.port}" ]; }]; } + { + job_name = "synapse"; + static_configs = [{ + targets = [ "matrix.rfive.de:8008" ]; + }]; + metrics_path = "/synapse/metrics"; + scrape_interval = "15s"; + } + { + job_name = "rspamd"; + static_configs = [{ + targets = [ "falkenstein.vpn.rfive.de:11334" ]; + }]; + } + { + job_name = "caddy"; + static_configs = [{ + targets = [ + "falkenstein.vpn.rfive.de:2018" + "nuc.vpn.rfive.de:2018" + ]; + }]; + scrape_interval = "15s"; + } ]; }; services.loki = { @@ -205,6 +230,41 @@ in url = "http://nuc.vpn.rfive.de:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push"; }]; scrape_configs = [ + { + job_name = "journal"; + journal = { + json = false; + max_age = "12h"; + path = "/var/log/journal"; + labels.job = "systemd-journal"; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + { + source_labels = [ "__journal__hostname" ]; + target_label = "host"; + } + { + source_labels = [ "__journal_priority_keyword" ]; + target_label = "level"; + } + { + source_labels = [ "__journal_syslog_identifier" ]; + target_label = "syslog_identifier"; + } + ]; + pipeline_stages = [ + { + match = { + selector = ''{unit="promtail.servicel"}''; + action = "drop"; + }; + } + ]; + } { job_name = "caddy_access_log"; static_configs = [ @@ -219,6 +279,13 @@ in } ]; pipeline_stages = [ + { + # remove :443 from matrix or rspamd logs + replace = { + expression = ".*(de:443).*"; + replace = "de"; + }; + } { json.expressions.remote_ip = "request.remote_ip"; } @@ -236,7 +303,6 @@ in }; }; - # nginx reverse proxy services.caddy.virtualHosts.${domain}.extraConfig = '' reverse_proxy 127.0.0.1:${toString config.services.grafana.settings.server.http_port} diff --git a/secrets.nix b/secrets.nix index d068991..055be04 100644 --- a/secrets.nix +++ b/secrets.nix @@ -26,7 +26,6 @@ in "secrets/nuc/authentik/ldap.age".publicKeys = [ rouven nuc ]; "secrets/nuc/grafana/oidc.age".publicKeys = [ rouven nuc ]; "secrets/nuc/cache.age".publicKeys = [ rouven nuc ]; - "secrets/nuc/maxmind.age".publicKeys = [ rouven nuc ]; "secrets/nuc/borg/passphrase.age".publicKeys = [ rouven nuc ]; "secrets/nuc/borg/key.age".publicKeys = [ rouven nuc ]; @@ -37,4 +36,7 @@ in "secrets/falkenstein/wireguard/dorm/preshared.age".publicKeys = [ rouven falkenstein ]; "secrets/falkenstein/borg/passphrase.age".publicKeys = [ rouven falkenstein ]; "secrets/falkenstein/borg/key.age".publicKeys = [ rouven falkenstein ]; + + #shared + "secrets/shared/maxmind.age".publicKeys = [ rouven nuc falkenstein ]; } diff --git a/secrets/nuc/maxmind.age b/secrets/nuc/maxmind.age deleted file mode 100644 index 8dd1762cb899a20d8afc8f3576284b60ccad7d64..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 363 zcmZ9_yH3JT002-|8x}_gV;41wwa}(O7n94Q(3Voln{F+Xme(!Z!nH6MWB3K*AjHAJ z%~fYjG|Wy;B>D+14jSD(KX44&qAb`zlfkI%!Xp>iG!7HgG3j{@Wi=U&3-jc-YJ!YL zmimkr%{6366=fmIMx*9HYC3$f=yMJfOcXOyToth)2mvvS=)TcWfur$b%!nyy>4sGs z)Ky8ak?zMkFeXoF^XFW0R~uD&3`2m9YE>Sl>#gm;*L?9FkUrm1o9yiL_po@L-`d?+Ue4F(v&V;<+8 ssh-ed25519 uWbAHQ hL+MYiYI/53SAw5Ue9L2E/W1sCwENhTqBReBwlRn6g0 +laaky6yfLkEPofvdZwu64WyVqPcxTt8Lng/uhBHaKjs +-> ssh-ed25519 2TRdXg dXERMyE1LqPxbAKn24SHruqrgKUTSIOLjy66nxiJSiE +lMGTDVxDUSu7r9Lp7mTfCzuTiUONv/K9b6y4mRlLLj8 +-> ssh-ed25519 slrRig Q7EcsiO/jsscDk9hHhtkHVxQ+NRO6O9SSQu4dfCPXG8 +LGCdVmGbMASuGGGuVrom+1ijafq0Sk0PDnyhOv2O2A0 +--- YeAR7BXc2heRrnvLa9YDGRIgI/3EQ3MfIJEZAJen8pY +Mü$¦óNù~KI ÀJÑÏ•èUæ¦.1q¶Y„‹-€"ë/_Øëý 2^“-Dÿ¯¬­Å4ã£/b+ +ô›V^MX_ç® ñ± \ No newline at end of file