grafana: init

hosts/falkenstein/default.nix

@@ -10,6 +10,7 @@
       ./modules/dns
       ./modules/fail2ban
       ./modules/mail
+      ./modules/monitoring
       ./modules/networks
       ./modules/pfersel
       ./modules/purge

hosts/falkenstein/modules/monitoring/default.nix

@@ -0,0 +1,18 @@
+{ config, ... }:
+{
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+      };
+      postfix = {
+        enable = true;
+      };
+    };
+  };
+  networking.firewall.allowedTCPPorts = [
+    config.services.prometheus.exporters.node.port
+    config.services.prometheus.exporters.postfix.port
+  ];
+}

hosts/fujitsu/default.nix

@@ -3,6 +3,7 @@
   imports = [
     ./hardware-configuration.nix
     ./modules/networks
+    ./modules/monitoring
   ];
 
   boot.loader.grub.enable = true;

hosts/fujitsu/modules/monitoring/default.nix

@@ -0,0 +1,14 @@
+{ config, ... }:
+{
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+      };
+    };
+  };
+  networking.firewall.allowedTCPPorts = [
+    config.services.prometheus.exporters.node.port
+  ];
+}

hosts/nuc/default.nix

@@ -12,6 +12,7 @@
       ./modules/cache
       ./modules/matrix
       ./modules/mautrix-telegram
+      ./modules/monitoring
       ./modules/seafile
       ./modules/torrent
       ./modules/vaultwarden

hosts/nuc/modules/monitoring/default.nix

@@ -0,0 +1,97 @@
+{ config, ... }:
+let
+  domain = "monitoring.${config.networking.domain}";
+in
+{
+  age.secrets."grafana/oidc_secret" = {
+    file = ../../../../secrets/nuc/grafana/oidc.age;
+    owner = "grafana";
+  };
+  # grafana configuration
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        inherit domain;
+        http_addr = "127.0.0.1";
+        http_port = 2342;
+        root_url = "https://${domain}";
+      };
+      database = {
+        type = "postgres";
+        user = "grafana";
+        host = "/run/postgresql";
+      };
+      "auth.generic_oauth" = {
+        enabled = true;
+        name = "Authentik";
+        allow_sign_up = true;
+        client_id = "grafana";
+        client_secret = "$__file{${config.age.secrets."grafana/oidc_secret".path}}";
+        scopes = "openid email profile offline_access roles";
+
+        email_attribute_path = "email";
+        login_attribute_path = "username";
+        name_attribute_path = "full_name";
+
+        auth_url = "https://auth.rfive.de/application/o/authorize/";
+        token_url = "https://auth.rfive.de/application/o/token/";
+        api_url = "https://auth.rfive.de/application/o/userinfo/";
+        role_attribute_path = "contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'";
+
+      };
+
+    };
+  };
+
+  services.postgresql = {
+    enable = true;
+    ensureUsers = [
+      {
+        name = "grafana";
+        ensureDBOwnership = true;
+      }
+    ];
+    ensureDatabases = [ "grafana" ];
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+      };
+      # postfix = {
+      #   enable = true;
+      #   port = 9003;
+      # };
+    };
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [{
+          targets = [
+            "nuc.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
+            "falkenstein.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
+            "cudy.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
+            "fujitsu.vpn.rfive.de:${toString config.services.prometheus.exporters.node.port}"
+          ];
+        }];
+        scrape_interval = "15s";
+      }
+      {
+        job_name = "postfix";
+        static_configs = [{
+          targets = [ "falkenstein.vpn.rfive.de:${toString config.services.prometheus.exporters.postfix.port}" ];
+        }];
+      }
+    ];
+  };
+
+  # nginx reverse proxy
+  services.caddy.virtualHosts.${domain}.extraConfig = ''
+    reverse_proxy 127.0.0.1:${toString config.services.grafana.settings.server.http_port}
+  '';
+}

hosts/nuc/modules/networks/default.nix

@@ -35,8 +35,7 @@
       }];
       networkConfig = {
         DNS = [
-          "9.9.9.9"
+          "192.168.42.1"
-          "149.112.112.112"
         ];
         LLDP = true;
         EmitLLDP = "nearest-bridge";

overlays/default.nix

@@ -63,8 +63,7 @@ in
   adguardian-term = callPackage ../pkgs/adguardian-term { };
 
   # upstream package is broken and can't be fixed by overriding attrs. so I just completely redo it in here
-  seahub = (python3Packages.buildPythonApplication
+  seahub = (python3Packages.buildPythonApplication rec {
-    rec {
     pname = "seahub";
     version = "11.0.1";
     format = "other";
@@ -138,5 +137,18 @@ in
       pythonPath = python.pkgs.makePythonPath propagatedBuildInputs;
     };
   });
+  # (hopefully) fix systemd journal reading
+  prometheus-postfix-exporter = prev.prometheus-postfix-exporter.overrideAttrs (_old: {
+    patches = [
+      ./prometheus-postfix-exporter/0001-cleanup-also-catch-milter-reject.patch
+    ];
+    src = fetchFromGitHub {
+      owner = "adangel";
+      repo = "postfix_exporter";
+      rev = "414ac12ee63415eede46cb3084d755a6da6fba23";
+      hash = "sha256-m1kVaO3N7XC1vtnxXX9kMiEFPmZuoopRUYgA7gQzP8w=";
+    };
+  });
+
 
 }

overlays/prometheus-postfix-exporter/0001-cleanup-also-catch-milter-reject.patch

@@ -0,0 +1,25 @@
+From f4c5dd5628c873981b2d6d6b8f3bbf036b9fd724 Mon Sep 17 00:00:00 2001
+From: Rouven Seifert <rouven.seifert@ifsr.de>
+Date: Thu, 2 May 2024 11:20:27 +0200
+Subject: [PATCH] cleanup: also catch milter-reject
+
+---
+ postfix_exporter.go | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/postfix_exporter.go b/postfix_exporter.go
+index f20d99c..676d767 100644
+--- a/postfix_exporter.go
++++ b/postfix_exporter.go
+@@ -335,6 +335,8 @@ func (e *PostfixExporter) CollectFromLogLine(line string) {
+ 				e.cleanupProcesses.Inc()
+ 			} else if strings.Contains(remainder, ": reject: ") {
+ 				e.cleanupRejects.Inc()
++			} else if strings.Contains(remainder, ": milter-reject: ") {
++				e.cleanupRejects.Inc()
+ 			} else {
+ 				e.addToUnsupportedLine(line, subprocess, level)
+ 			}
+-- 
+2.44.0
+

secrets.nix

@@ -24,6 +24,7 @@ in
   "secrets/nuc/keycloak/db.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/authentik/core.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/authentik/ldap.age".publicKeys = [ rouven nuc ];
+  "secrets/nuc/grafana/oidc.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/cache.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/borg/passphrase.age".publicKeys = [ rouven nuc ];
   "secrets/nuc/borg/key.age".publicKeys = [ rouven nuc ];