more monitoring

This commit is contained in:
Rouven Seifert 2024-06-06 20:50:33 +02:00
parent 5089f62112
commit 53eb192ea8
Signed by: rouven.seifert
GPG key ID: B95E8FE6B11C4D09
9 changed files with 147 additions and 24 deletions

View file

@ -312,11 +312,11 @@
]
},
"locked": {
"lastModified": 1717097707,
"narHash": "sha256-HC5vJ3oYsjwsCaSbkIPv80e4ebJpNvFKQTBOGlHvjLs=",
"lastModified": 1717525419,
"narHash": "sha256-5z2422pzWnPXHgq2ms8lcCfttM0dz+hg+x1pCcNkAws=",
"owner": "nix-community",
"repo": "home-manager",
"rev": "0eb314b4f0ba337e88123e0b1e57ef58346aafd9",
"rev": "a7117efb3725e6197dd95424136f79147aa35e5b",
"type": "github"
},
"original": {
@ -460,11 +460,11 @@
]
},
"locked": {
"lastModified": 1716772633,
"narHash": "sha256-Idcye44UW+EgjbjCoklf2IDF+XrehV6CVYvxR1omst4=",
"lastModified": 1717297675,
"narHash": "sha256-43UmlS1Ifx17y93/Vc258U7bOlAAIZbu8dsGDHOIIr0=",
"owner": "nix-community",
"repo": "nix-index-database",
"rev": "ff80cb4a11bb87f3ce8459be6f16a25ac86eb2ac",
"rev": "972a52bee3991ae1f1899e6452e0d7c01ee566d9",
"type": "github"
},
"original": {
@ -475,11 +475,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1716948383,
"narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=",
"lastModified": 1717602782,
"narHash": "sha256-pL9jeus5QpX5R+9rsp3hhZ+uplVHscNJh8n8VpqscM0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ad57eef4ef0659193044870c731987a6df5cf56b",
"rev": "e8057b67ebf307f01bdcc8fba94d94f75039d1f6",
"type": "github"
},
"original": {

View file

@ -25,9 +25,14 @@ in
metrics
}
'';
virtualHosts.":2018".extraConfig = ''
metrics
'';
virtualHosts.":2018" = {
extraConfig = ''
metrics
'';
logFormat = ''
output discard
'';
};
virtualHosts."${config.networking.domain}".extraConfig = ''
file_server browse
root * /srv/web/${config.networking.domain}

View file

@ -86,6 +86,7 @@
job = "caddy_access_log";
agent = "caddy-promtail";
__path__ = "/var/log/caddy/*.log";
host = "falkenstein";
};
}
];
@ -107,6 +108,19 @@
db_type = "city";
};
}
{
geoip = {
db = "/var/lib/GeoIP/GeoLite2-ASN.mmdb";
source = "remote_ip";
db_type = "asn";
};
}
{
labeldrop = [
"geoip_subdivision_code"
"geoip_continent_code"
];
}
];
}

View file

@ -1,5 +1,7 @@
{ config, ... }:
{ config, lib, ... }:
{
# required for elasticsearch
nixpkgs.config.allowUnfree = true;
age.secrets.dmarc = {
file = ../../../../secrets/falkenstein/dmarc.age;
};
@ -7,7 +9,8 @@
description = "DMARC Report recipient";
isNormalUser = true;
};
networking.firewall.allowedTCPPorts = [ config.services.elasticsearch.tcp_port ];
networking.firewall.allowedTCPPorts = [ 9200 ];
services.elasticsearch.listenAddress = "0.0.0.0";
services.parsedmarc = {
enable = true;
provision = {
@ -16,7 +19,7 @@
datasource = false;
};
localMail.enable = false;
elasticsearch = false;
elasticsearch = true;
geoIp = false;
};
settings = {
@ -28,8 +31,6 @@
_secret = config.age.secrets.dmarc.path;
};
};
opensearch.hosts = "localhost:9200";
};
};
services.opensearch.enable = true;
}

View file

@ -9,9 +9,14 @@
metrics
}
'';
virtualHosts.":2018".extraConfig = ''
metrics
'';
virtualHosts.":2018" = {
extraConfig = ''
metrics
'';
logFormat = ''
output discard
'';
};
};
systemd.services.caddy.environment.XDG_DATA_HOME = "/var/lib";
networking.firewall.allowedTCPPorts = [ 80 443 ];

View file

@ -40,9 +40,10 @@ in
settings = {
server_name = config.networking.domain;
enable_metrics = true;
listeners = [{
bind_addresses = [ "127.0.0.1" "::1" ];
bind_addresses = [ "0.0.0.0" "::1" ];
port = 8008;
tls = false;
type = "http";

View file

@ -85,6 +85,9 @@ in
services.prometheus = {
enable = true;
port = 9001;
ruleFiles = [
./synapse-v2.rules
];
exporters = {
node = {
enable = true;
@ -107,9 +110,9 @@ in
{
job_name = "synapse";
static_configs = [{
targets = [ "matrix.rfive.de:8008" ];
targets = [ "nuc.vpn.rfive.de:8008" ];
}];
metrics_path = "/synapse/metrics";
metrics_path = "/_synapse/metrics";
scrape_interval = "15s";
}
{
@ -118,6 +121,12 @@ in
targets = [ "falkenstein.vpn.rfive.de:11334" ];
}];
}
{
job_name = "authentik";
static_configs = [{
targets = [ "nuc.vpn.rfive.de:9300" ];
}];
}
{
job_name = "caddy";
static_configs = [{
@ -269,6 +278,7 @@ in
# host = "matrix.rfive.de";
agent = "caddy-promtail";
__path__ = "/var/log/caddy/*.log";
host = "nuc";
};
}
];
@ -290,6 +300,19 @@ in
db_type = "city";
};
}
{
geoip = {
db = "/var/lib/GeoIP/GeoLite2-ASN.mmdb";
source = "remote_ip";
db_type = "asn";
};
}
{
labeldrop = [
"geoip_subdivision_code"
"geoip_continent_code"
];
}
];
}

View file

@ -0,0 +1,74 @@
groups:
- name: synapse
rules:
###
### Prometheus Console Only
### The following rules are only needed if you use the Prometheus Console
### in contrib/prometheus/consoles/synapse.html
###
- record: 'synapse_federation_client_sent'
labels:
type: "EDU"
expr: 'synapse_federation_client_sent_edus_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "PDU"
expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "Query"
expr: 'sum(synapse_federation_client_sent_queries) by (job)'
- record: 'synapse_federation_server_received'
labels:
type: "EDU"
expr: 'synapse_federation_server_received_edus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "PDU"
expr: 'synapse_federation_server_received_pdus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "Query"
expr: 'sum(synapse_federation_server_received_queries) by (job)'
- record: 'synapse_federation_transaction_queue_pending'
labels:
type: "EDU"
expr: 'synapse_federation_transaction_queue_pending_edus + 0'
- record: 'synapse_federation_transaction_queue_pending'
labels:
type: "PDU"
expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
###
### End of 'Prometheus Console Only' rules block
###
###
### Grafana Only
### The following rules are only needed if you use the Grafana dashboard
### in contrib/grafana/synapse.json
###
- record: synapse_storage_events_persisted_by_source_type
expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
labels:
type: remote
- record: synapse_storage_events_persisted_by_source_type
expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
labels:
type: local
- record: synapse_storage_events_persisted_by_source_type
expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
labels:
type: bridges
- record: synapse_storage_events_persisted_by_event_type
expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
- record: synapse_storage_events_persisted_by_origin
expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
###
### End of 'Grafana Only' rules block
###

View file

@ -36,7 +36,7 @@
# cryptography
yubikey-manager
python311Packages.pyhanko # broken, TODO fix
# python311Packages.pyhanko # broken, TODO fix
bitwarden-cli
# misc