From 85bf54d7350c3478db16de9808f8948946912ba1 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Fri, 14 Feb 2025 23:04:43 -0500 Subject: [PATCH 1/5] nixos/victorialogs: init --- modules/nixos/custom/default.nix | 1 + modules/nixos/custom/victorialogs.nix | 129 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 modules/nixos/custom/victorialogs.nix diff --git a/modules/nixos/custom/default.nix b/modules/nixos/custom/default.nix index 8b9df32a..4b14136d 100644 --- a/modules/nixos/custom/default.nix +++ b/modules/nixos/custom/default.nix @@ -5,5 +5,6 @@ ./nvd-diff.nix ./nvk.nix ./remote-builders.nix + ./victorialogs.nix ]; } diff --git a/modules/nixos/custom/victorialogs.nix b/modules/nixos/custom/victorialogs.nix new file mode 100644 index 00000000..ab6be3a4 --- /dev/null +++ b/modules/nixos/custom/victorialogs.nix @@ -0,0 +1,129 @@ +# From https://github.com/NixOS/nixpkgs/pull/376834 +{ + config, + pkgs, + lib, + ... +}: + +let + inherit (lib) + getBin + hasPrefix + literalExpression + mkBefore + mkEnableOption + mkIf + mkOption + mkPackageOption + optionalString + types + ; + + cfg = config.borealis.victorialogs; + + startCLIList = [ + "${cfg.package}/bin/victoria-logs" + "-storageDataPath=/var/lib/${cfg.stateDir}" + "-httpListenAddr=${cfg.listenAddress}" + ] ++ cfg.extraOptions; +in + +{ + options.borealis.victorialogs = { + enable = mkEnableOption "VictoriaLogs is an open source user-friendly database for logs from VictoriaMetrics"; + package = mkPackageOption pkgs "victoriametrics" { }; + listenAddress = lib.mkOption { + default = "127.0.0.1:9428"; + type = types.str; + description = '' + TCP address to listen for incoming http requests. + ''; + }; + stateDir = mkOption { + type = types.str; + default = "victorialogs"; + description = '' + Directory below `/var/lib` to store VictoriaLogs data. + This directory will be created automatically using systemd's StateDirectory mechanism. + ''; + }; + extraOptions = mkOption { + type = types.listOf types.str; + default = [ ]; + example = literalExpression '' + [ + "-httpAuth.username=username" + "-httpAuth.password=file:///abs/path/to/file" + "-loggerLevel=WARN" + ] + ''; + description = '' + Extra options to pass to VictoriaLogs. See {command}`victoria-logs -help` for + possible options. + ''; + }; + }; + config = mkIf cfg.enable { + systemd.services.victorialogs = { + description = "VictoriaLogs logs database"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + startLimitBurst = 5; + + serviceConfig = { + ExecStart = lib.escapeShellArgs startCLIList; + DynamicUser = true; + RestartSec = 1; + Restart = "on-failure"; + RuntimeDirectory = "victorialogs"; + RuntimeDirectoryMode = "0700"; + StateDirectory = cfg.stateDir; + StateDirectoryMode = "0700"; + + # Hardening + DeviceAllow = [ "/dev/null rw" ]; + DevicePolicy = "strict"; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "full"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + ]; + }; + + postStart = + let + bindAddr = (optionalString (hasPrefix ":" cfg.listenAddress) "127.0.0.1") + cfg.listenAddress; + in + mkBefore '' + until ${getBin pkgs.curl}/bin/curl -s -o /dev/null http://${bindAddr}/ping; do + sleep 1; + done + ''; + }; + }; +} From 161fe02d15b9781cf1e4d688861d546a434330e9 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Fri, 14 Feb 2025 23:55:18 -0500 Subject: [PATCH 2/5] nixos: add grafana + prom/vm mixins --- modules/nixos/mixins/default.nix | 3 ++ modules/nixos/mixins/grafana.nix | 68 +++++++++++++++++++++++++ modules/nixos/mixins/journal-upload.nix | 7 +++ modules/nixos/mixins/node-exporter.nix | 11 ++++ 4 files changed, 89 insertions(+) create mode 100644 modules/nixos/mixins/grafana.nix create mode 100644 modules/nixos/mixins/journal-upload.nix create mode 100644 modules/nixos/mixins/node-exporter.nix diff --git a/modules/nixos/mixins/default.nix b/modules/nixos/mixins/default.nix index 8e77f34d..f4027760 100644 --- a/modules/nixos/mixins/default.nix +++ b/modules/nixos/mixins/default.nix @@ -7,11 +7,14 @@ ./comin.nix ./forgejo.nix ./gnome.nix + ./grafana.nix ./home-manager.nix + ./journal-upload.nix ./kanidm.nix ./lanzaboote.nix ./nginx.nix ./niri.nix + ./node-exporter.nix ./nvidia.nix ./pipewire.nix ./plasma.nix diff --git a/modules/nixos/mixins/grafana.nix b/modules/nixos/mixins/grafana.nix new file mode 100644 index 00000000..3385107f --- /dev/null +++ b/modules/nixos/mixins/grafana.nix @@ -0,0 +1,68 @@ +{ + config, + lib, + secretsDir, + ... +}: + +{ + config = lib.mkMerge [ + { + services.grafana = { + settings = { + analytics = { + feedback_links_enabled = false; + reporting_enabled = false; + }; + + "auth.anonymous".enable = true; + + server = { + http_port = 6000; + + domain = lib.mkDefault ("grafana." + config.networking.domain); + enable_gzip = true; + enforce_domain = true; + root_url = "https://" + config.services.grafana.settings.server.domain + "/"; + }; + }; + }; + } + + (lib.mkIf config.services.kanidm.enableServer { + services.grafana = { + settings = { + "auth.basic".enabled = false; + + "auth.generic_oauth" = { + enabled = true; + + name = "Kanidm"; + client_id = "grafana"; + client_secret = "$__file{${config.age.secrets.grafanaKanidm.path}}"; + scopes = "openid,profile,email,groups"; + auth_url = config.services.kanidm.serverSettings.origin + "/ui/oauth2"; + token_url = config.services.kanidm.serverSettings.origin + "/oauth2/token"; + api_url = config.services.kanidm.serverSettings.origin + "/oauth2/openid/grafana/userinfo"; + use_pkce = true; + use_refresh_token = true; + + allow_assign_grafana_admin = true; + allow_sign_up = true; + groups_attribute_path = "groups"; + login_attribute_path = "preferred_username"; + role_attribute_path = "contains(grafana_role[*], 'GrafanaAdmin') && 'GrafanaAdmin' || contains(grafana_role[*], 'Admin') && 'Admin' || contains(grafana_role[*], 'Editor') && 'Editor' || 'Viewer'"; + }; + }; + }; + }) + + (lib.mkIf (config.services.grafana.enable && config.services.kanidm.enableServer) { + age.secrets.grafanaKanidm = { + file = secretsDir + "/grafanaKanidmSecret.age"; + owner = config.users.users.grafana.name; + group = config.users.groups.grafana.name; + }; + }) + ]; +} diff --git a/modules/nixos/mixins/journal-upload.nix b/modules/nixos/mixins/journal-upload.nix new file mode 100644 index 00000000..4d780c91 --- /dev/null +++ b/modules/nixos/mixins/journal-upload.nix @@ -0,0 +1,7 @@ +{ + services.journald.upload = { + settings = { + Upload.URL = "http://atlas:9428/insert/journald"; + }; + }; +} diff --git a/modules/nixos/mixins/node-exporter.nix b/modules/nixos/mixins/node-exporter.nix new file mode 100644 index 00000000..752ff1d9 --- /dev/null +++ b/modules/nixos/mixins/node-exporter.nix @@ -0,0 +1,11 @@ +{ lib, ... }: + +{ + services.prometheus.exporters.node = { + openFirewall = lib.mkDefault true; + + enabledCollectors = [ + "systemd" + ]; + }; +} From 4e15c8af6fb19809371f7100d7b5cdc2dbaf6d00 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Fri, 14 Feb 2025 23:56:03 -0500 Subject: [PATCH 3/5] atlas: host grafana + vm --- secrets/atlas/grafanaKanidmSecret.age | 7 ++++ systems/atlas/default.nix | 2 ++ systems/atlas/grafana.nix | 18 ++++++++++ systems/atlas/victoria-metrics.nix | 52 +++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 secrets/atlas/grafanaKanidmSecret.age create mode 100644 systems/atlas/grafana.nix create mode 100644 systems/atlas/victoria-metrics.nix diff --git a/secrets/atlas/grafanaKanidmSecret.age b/secrets/atlas/grafanaKanidmSecret.age new file mode 100644 index 00000000..586f788a --- /dev/null +++ b/secrets/atlas/grafanaKanidmSecret.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> X25519 WN8YTDxF+JY8a2W10sd4nFDPEOzQhVFWmMK0TC7BRCw +2/au+k29lSsMZxMIj2+yGzJRt8PO9KnQ0snh/b8vkno +-> X25519 RVgT9JDOzVBhsWE+tw1uWyvz3ECyAexSArI4avMyTFs +iD/aHD+/w1dvyERSgqy8nu3KXC2I0xmWoMlJzLFBdNw +--- PW47hPTW9nkWD4CBGgGBVgL2mc5/Lu7qDmchRt1NJ2U +nÜrGæ{O E¹4ç6{ʇ„B8½ÛjU|W/1?>40ŒßV¶?½ùKyðP#&N$½­2—2P»¥³"…ñÅØ¢³Ïë1&†ù;ão9 \ No newline at end of file diff --git a/systems/atlas/default.nix b/systems/atlas/default.nix index 66ee4766..61be6488 100644 --- a/systems/atlas/default.nix +++ b/systems/atlas/default.nix @@ -4,10 +4,12 @@ (modulesPath + "/profiles/minimal.nix") ./hardware-configuration.nix ./forgejo.nix + ./grafana.nix ./kanidm.nix ./miniflux.nix ./moyai.nix ./nixpkgs-tracker-bot.nix + ./victoria-metrics.nix inputs.self.nixosModules.default ]; diff --git a/systems/atlas/grafana.nix b/systems/atlas/grafana.nix new file mode 100644 index 00000000..c6a84ab4 --- /dev/null +++ b/systems/atlas/grafana.nix @@ -0,0 +1,18 @@ +{ config, ... }: + +{ + services = { + grafana = { + enable = true; + }; + + nginx.virtualHosts = { + "grafana.getchoo.com" = { + locations."/" = { + proxyPass = "http://${config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}"; + proxyWebsockets = true; + }; + }; + }; + }; +} diff --git a/systems/atlas/victoria-metrics.nix b/systems/atlas/victoria-metrics.nix new file mode 100644 index 00000000..25f0e57b --- /dev/null +++ b/systems/atlas/victoria-metrics.nix @@ -0,0 +1,52 @@ +{ + lib, + inputs, + ... +}: + +let + usesNodeExporter = system: system.config.services.prometheus.exporters.node.enable; + + nodeExporterFrom = + system: + "http://${system.config.networking.hostName}:${toString system.config.services.prometheus.exporters.node.port}"; + + toNodeStaticConfig = system: { + targets = [ (nodeExporterFrom system) ]; + labels.type = "node"; + }; + + remoteNodes = lib.mapAttrsToList (lib.const toNodeStaticConfig) ( + lib.filterAttrs (lib.const usesNodeExporter) inputs.self.nixosConfigurations + ); +in + +{ + borealis = { + victorialogs = { + enable = true; + }; + }; + + services = { + journald.upload.enable = true; + + prometheus.exporters.node.enable = true; + + victoriametrics = { + enable = true; + + retentionPeriod = "7d"; + + prometheusConfig = { + scrape_configs = [ + { + job_name = "node-exporter"; + metrics_path = "/metrics"; + static_configs = remoteNodes; + } + ]; + }; + }; + }; +} From 003507d30ed7aecc8a76556093ecd050ad04fc08 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Sat, 15 Feb 2025 00:11:43 -0500 Subject: [PATCH 4/5] atlas: collect more metrics for local services --- modules/nixos/mixins/forgejo.nix | 8 +++-- systems/atlas/miniflux.nix | 27 +++++++++-------- systems/atlas/victoria-metrics.nix | 47 ++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/modules/nixos/mixins/forgejo.nix b/modules/nixos/mixins/forgejo.nix index a897bfbd..1e575ca0 100644 --- a/modules/nixos/mixins/forgejo.nix +++ b/modules/nixos/mixins/forgejo.nix @@ -21,13 +21,17 @@ in settings = { server = { - PROTOCOL = "http+unix"; + PROTOCOL = "http"; DOMAIN = lib.mkDefault ("git." + config.networking.domain); ROOT_URL = "https://" + forgejoCfg.settings.server.DOMAIN + "/"; DISABLE_SSH = lib.mkDefault true; }; + metrics = { + ENABLED = true; + }; + oauth2_client = { ENABLE_AUTO_REGISTRATION = lib.mkDefault true; }; @@ -50,7 +54,7 @@ in (lib.mkIf forgejoCfg.enable { services.nginx.virtualHosts.${forgejoCfg.settings.server.DOMAIN} = { locations."/" = { - proxyPass = "http://unix:${forgejoCfg.settings.server.HTTP_ADDR}"; + proxyPass = "http://${forgejoCfg.settings.server.HTTP_ADDR}:${toString forgejoCfg.settings.server.HTTP_PORT}"; }; }; diff --git a/systems/atlas/miniflux.nix b/systems/atlas/miniflux.nix index 5fd22f24..14c9fc52 100644 --- a/systems/atlas/miniflux.nix +++ b/systems/atlas/miniflux.nix @@ -1,6 +1,5 @@ { config, - lib, secretsDir, ... }: @@ -14,28 +13,32 @@ adminCredentialsFile = config.age.secrets.miniflux.path; config = { BASE_URL = "https://miniflux.${config.networking.domain}"; + LISTEN_ADDR = "localhost:7000"; + METRICS_COLLECTOR = 1; }; }; nginx.virtualHosts = { "miniflux.getchoo.com" = { locations."/" = { - proxyPass = "http://unix:${lib.head config.systemd.sockets.miniflux.listenStreams}"; + proxyPass = "http://${config.services.miniflux.config.LISTEN_ADDR}"; }; }; }; }; - # Create the socket manually to ensure NGINX has permission for the socket's parent directory - # ...since for some reason Miniflux will not give it the same `0777` permission as the socket itself - systemd = { - services.miniflux = { - requires = [ "miniflux.socket" ]; - }; + /* + # Create the socket manually to ensure NGINX has permission for the socket's parent directory + # ...since for some reason Miniflux will not give it the same `0777` permission as the socket itself + systemd = { + services.miniflux = { + requires = [ "miniflux.socket" ]; + }; - sockets.miniflux = { - wantedBy = [ "sockets.target" ]; - listenStreams = [ "/run/miniflux.sock" ]; + sockets.miniflux = { + wantedBy = [ "sockets.target" ]; + listenStreams = [ "/run/miniflux.sock" ]; + }; }; - }; + */ } diff --git a/systems/atlas/victoria-metrics.nix b/systems/atlas/victoria-metrics.nix index 25f0e57b..fba00ae6 100644 --- a/systems/atlas/victoria-metrics.nix +++ b/systems/atlas/victoria-metrics.nix @@ -1,4 +1,5 @@ { + config, lib, inputs, ... @@ -40,11 +41,57 @@ in prometheusConfig = { scrape_configs = [ + { + job_name = "forgejo"; + metrics_path = "/metrics"; + static_configs = [ + { + targets = [ + "http://${config.services.forgejo.settings.server.HTTP_ADDR}:${toString config.services.forgejo.settings.server.HTTP_PORT}" + ]; + labels.type = "forgejo"; + } + ]; + } + + { + job_name = "miniflux"; + metrics_path = "/metrics"; + static_configs = [ + { + targets = [ "http://${config.services.miniflux.config.LISTEN_ADDR}" ]; + labels.type = "miniflux"; + } + ]; + } + { job_name = "node-exporter"; metrics_path = "/metrics"; static_configs = remoteNodes; } + + { + job_name = "victoria-logs"; + metrics_path = "/metrics"; + static_configs = [ + { + targets = [ "localhost:9428" ]; + labels.type = "victoria-logs"; + } + ]; + } + + { + job_name = "victoria-metrics"; + metrics_path = "/metrics"; + static_configs = [ + { + targets = [ "localhost${config.services.victoriametrics.listenAddress}" ]; + labels.type = "victoria-logs"; + } + ]; + } ]; }; }; From 2c12e17d28a5b97388fc8c76d10ef9e463637491 Mon Sep 17 00:00:00 2001 From: Seth Flynn Date: Sat, 15 Feb 2025 01:10:15 -0500 Subject: [PATCH 5/5] nixos/grafana: allow anons to view --- modules/nixos/mixins/grafana.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/nixos/mixins/grafana.nix b/modules/nixos/mixins/grafana.nix index 3385107f..fde4535d 100644 --- a/modules/nixos/mixins/grafana.nix +++ b/modules/nixos/mixins/grafana.nix @@ -15,7 +15,10 @@ reporting_enabled = false; }; - "auth.anonymous".enable = true; + "auth.anonymous" = { + enable = true; + org_role = "Viewer"; + }; server = { http_port = 6000;