add prometheus + grafana

flake.lock

@@ -42,11 +42,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1707510244,
+        "lastModified": 1708021300,
-        "narHash": "sha256-lZpCFv1fmj6/7WsTRxiffko/dOoQ4/5z7z58yuD/MdI=",
+        "narHash": "sha256-MEi5P0+tlvyZoor7+AP5WcnIRe7Dg6/8BjLzOp+EDaQ=",
         "owner": "AtaraxiaSjel",
         "repo": "nur",
-        "rev": "0f4911bdd7c8f3c4dc2328c8c1420122233a6e02",
+        "rev": "c9a54074f026ebf486c88b93ee5502e5abef30e7",
         "type": "github"
       },
       "original": {

machines/Home-Hypervisor/default.nix

@@ -21,8 +21,8 @@ in {
     customProfiles.hoyolab
     customProfiles.inpx-web
     customProfiles.it-tools
-    # customProfiles.joplin-server
     customProfiles.media-stack
+    customProfiles.metrics
     customProfiles.minio
     customProfiles.nginx
     customProfiles.ocis

machines/Home-Hypervisor/dns-mapping.nix

@@ -25,7 +25,7 @@
     { name = "restic.ataraxiadev.com"; type = "A"; value = "100.64.0.3"; }
     { name = "s3.ataraxiadev.com"; type = "A"; value = "100.64.0.3"; }
     { name = "sonarr.ataraxiadev.com"; type = "A"; value = "100.64.0.3"; }
-    { name = "stats.ataraxiadev.com"; type = "A"; value = "100.64.0.1"; }
+    { name = "stats.ataraxiadev.com"; type = "A"; value = "100.64.0.3"; }
     { name = "tools.ataraxiadev.com"; type = "A"; value = "100.64.0.3"; }
     { name = "turn.ataraxiadev.com"; type = "A"; value = "100.64.0.1"; }
     { name = "vault.ataraxiadev.com"; type = "A"; value = "100.64.0.3"; }
@@ -57,7 +57,7 @@
     { name = "restic.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::3"; }
     { name = "s3.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::3"; }
     { name = "sonarr.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::3"; }
-    { name = "stats.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::1"; }
+    { name = "stats.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::3"; }
     { name = "tools.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::3"; }
     { name = "turn.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::1"; }
     { name = "vault.ataraxiadev.com"; type = "AAAA"; value = "fd7a:115c:a1e0::3"; }
@@ -68,7 +68,6 @@
     # TODO: Fix dns resolution in blocky for unmapped subdomains of ataraxiadev.com
     "/element.ataraxiadev.com/83.138.55.118"
     "/matrix.ataraxiadev.com/83.138.55.118"
-    "/stats.ataraxiadev.com/83.138.55.118"
     "/turn.ataraxiadev.com/83.138.55.118"
 
     "/api.ataraxiadev.com/192.168.0.10"
@@ -92,6 +91,7 @@
     "/radarr.ataraxiadev.com/192.168.0.10"
     "/restic.ataraxiadev.com/192.168.0.10"
     "/s3.ataraxiadev.com/192.168.0.10"
+    "/stats.ataraxiadev.com/192.168.0.10"
     "/sonarr.ataraxiadev.com/192.168.0.10"
     "/tools.ataraxiadev.com/192.168.0.10"
     "/vault.ataraxiadev.com/192.168.0.10"

profiles/servers/blocky.nix

@@ -19,7 +19,6 @@ in {
     bindMounts."/tmp/blocky-authkey".hostPath = "/tmp/blocky-authkey";
     config = { config, lib, ... }:
     let
-      grafanaPort = config.services.grafana.settings.server.http_port;
       blockyPort = config.services.blocky.settings.ports.dns;
       blockyHttpPort = config.services.blocky.settings.ports.http;
     in {
@@ -31,7 +30,7 @@ in {
         useHostResolvConf = false;
         firewall = {
           enable = true;
-          allowedTCPPorts = [ blockyPort grafanaPort ];
+          allowedTCPPorts = [ blockyPort blockyHttpPort ];
           allowedUDPPorts = [ blockyPort ];
         };
         hosts = {
@@ -94,72 +93,12 @@ in {
           };
           ports = {
             dns = 53;
-            http = "127.0.0.1:4000";
+            http = 4000;
           };
           prometheus.enable = true;
           queryLog.type = "console";
         };
       };
-      services.prometheus = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        globalConfig.scrape_interval = "15s";
-        globalConfig.evaluation_interval = "15s";
-        scrapeConfigs = [{
-          job_name = "blocky";
-          static_configs = [{
-            targets = [ blockyHttpPort ];
-          }];
-        }];
-      };
-      services.grafana = {
-        enable = true;
-        settings = {
-          analytics.reporting_enabled = false;
-          server = {
-            domain = "${nodeAddress}:${toString grafanaPort}";
-            http_addr = nodeAddress;
-            enable_gzip = true;
-          };
-          panels.disable_sanitize_html = true;
-        };
-        provision = {
-          enable = true;
-          datasources.settings = {
-            datasources = [{
-              name = "Prometheus";
-              type = "prometheus";
-              access = "proxy";
-              orgId = 1;
-              uid = "Y4SSG429DWCGDQ3R";
-              url = "http://127.0.0.1:${toString config.services.prometheus.port}";
-              isDefault = true;
-              jsonData = {
-                graphiteVersion = "1.1";
-                tlsAuth = false;
-                tlsAuthWithCACert = false;
-              };
-              version = 1;
-              editable = true;
-            }];
-          };
-          dashboards = {
-            settings = {
-              providers = [{
-                name = "My Dashboards";
-                options.path = "/etc/grafana-dashboards";
-              }];
-            };
-          };
-        };
-      };
-      environment.etc = {
-        "grafana-dashboards/blocky_rev3.json" = {
-          source = ../../misc/grafana_blocky_rev3.json;
-          group = "grafana";
-          user = "grafana";
-        };
-      };
       system.stateVersion = "23.11";
     };
   };

profiles/servers/metrics/dashboards/default.nix

@@ -0,0 +1,15 @@
+{ pkgs, blockyUrl, prometheusUid, ... }:
+let
+  template =
+    name: file: tmpl:
+    (pkgs.runCommandLocal name { } ''
+      mkdir $out
+      cp ${pkgs.substituteAll ({ src = file; } // tmpl)} $out/${name}.json
+    '');
+in
+pkgs.symlinkJoin {
+  name = "grafana-dashboards";
+  paths = [
+    (template "grafana-blocky" ./blocky.json { inherit blockyUrl prometheusUid; })
+  ];
+}

profiles/servers/metrics/default.nix

@@ -0,0 +1,144 @@
+{ config, pkgs, inputs, ... }:
+let
+  promStateDir = "prometheus2";
+  grafanaDataDir = "grafana";
+  prometheusUid = "d8e758af-3f6b-4891-a855-1efe6cdec658";
+  blockyUrl = "192.168.0.5:4000";
+  prometheusPort = 9001;
+  grafanaPort = 9002;
+
+  secretCfg = {
+    sopsFile = inputs.self.secretsDir + /home-hypervisor/metrics.yaml;
+    owner = "grafana";
+  };
+in
+{
+  imports = [ inputs.ataraxiasjel-nur.nixosModules.prometheus-exporters ];
+  sops.secrets.grafana-oidc-id = secretCfg;
+  sops.secrets.grafana-oidc-secret = secretCfg;
+
+  services.prometheus = {
+    enable = true;
+    stateDir = promStateDir;
+    listenAddress = "127.0.0.1";
+    port = prometheusPort;
+    globalConfig.scrape_interval = "15s";
+    globalConfig.evaluation_interval = "15s";
+    exporters = {
+      podman = {
+        enable = true;
+        enabledCollectors = [ "enable-all" ];
+        port = 9012;
+      };
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9010;
+      };
+      zfs = {
+        enable = true;
+        port = 9011;
+      };
+    };
+    scrapeConfigs = [
+      {
+        job_name = "blocky";
+        static_configs = [ { targets = [ blockyUrl ]; } ];
+      }
+      {
+        job_name = "podman";
+        static_configs = [
+          { targets = [ "localhost:${toString config.services.prometheus.exporters.podman.port}" ]; }
+        ];
+      }
+      {
+        job_name = "node";
+        static_configs = [
+          { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
+        ];
+      }
+      {
+        job_name = "zfs";
+        static_configs = [
+          { targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ]; }
+        ];
+      }
+    ];
+  };
+  services.grafana = {
+    enable = true;
+    dataDir = "/var/lib/${grafanaDataDir}";
+    declarativePlugins = with pkgs.grafanaPlugins; [ grafana-piechart-panel ];
+    settings = {
+      auth = {
+        signout_redirect_url = "https://auth.ataraxiadev.com/application/o/grafana/end-session/";
+        oauth_auto_login = true;
+      };
+      "auth.generic_oauth" = {
+        name = "authentik";
+        enabled = "true";
+        client_id = "$__file{${config.sops.secrets.grafana-oidc-id.path}}";
+        client_secret = "$__file{${config.sops.secrets.grafana-oidc-secret.path}}";
+        scopes = "openid email profile";
+        auth_url = "https://auth.ataraxiadev.com/application/o/authorize/";
+        token_url = "https://auth.ataraxiadev.com/application/o/token/";
+        api_url = "https://auth.ataraxiadev.com/application/o/userinfo/";
+        role_attribute_path = "contains(groups, 'grafanaAdmins') && 'Admin' || contains(groups, 'grafanaEditors') && 'Editor' || 'Viewer'";
+      };
+      users.auto_assign_org = true;
+      users.auto_assign_org_id = 1;
+      analytics.reporting_enabled = false;
+      server = {
+        domain = "stats.ataraxiadev.com";
+        http_addr = "127.0.0.1";
+        http_port = grafanaPort;
+        root_url = "https://%(domain)s/";
+        enable_gzip = true;
+      };
+      panels.disable_sanitize_html = true;
+    };
+    provision = {
+      enable = true;
+      datasources.settings = {
+        datasources = [
+          {
+            name = "Prometheus ${config.networking.hostName}";
+            type = "prometheus";
+            access = "proxy";
+            orgId = 1;
+            uid = prometheusUid;
+            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+            isDefault = true;
+            jsonData = {
+              httpMethod = "POST";
+              manageAlerts = true;
+              prometheusType = "Prometheus";
+              prometheusVersion = config.services.prometheus.package.version;
+              cacheLevel = "High";
+            };
+            editable = false;
+          }
+        ];
+      };
+      dashboards = {
+        settings = {
+          providers = [
+            {
+              name = "Dashboards";
+              # folder = "Services";
+              options.path = import ./dashboards {
+                inherit pkgs prometheusUid;
+                blockyUrl = "http://${blockyUrl}";
+              };
+            }
+          ];
+        };
+      };
+    };
+  };
+
+  persist.state.directories = [
+    "/var/lib/${promStateDir}"
+    "/var/lib/${grafanaDataDir}"
+  ];
+}

profiles/servers/nginx.nix

@@ -248,6 +248,17 @@ in {
           proxy_request_buffering off;
         '';
       } // default;
+      "stats.ataraxiadev.com" = {
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:9002";
+          extraConfig = proxySettings;
+        };
+        locations."/api/live/" = {
+          proxyPass = "http://127.0.0.1:9002";
+          proxyWebsockets = true;
+          extraConfig = proxySettings;
+        };
+      } // default;
       "tools.ataraxiadev.com" = default // authentik {
         proxyPass = "http://127.0.0.1:8070";
       };

secrets/home-hypervisor/metrics.yaml

@@ -0,0 +1,48 @@
+grafana-oidc-id: ENC[AES256_GCM,data:MwMBTYxnKGC4F9g2gHSbjspM2XthDjJf1bCVOLrZGETAiTbsooB97Q==,iv:MCTHCfwuWuFl/ZwTL4U1qB5rw8RUGC7X3tDV1Vc+AWA=,tag:UCfk3JBW5DYK8O0G9dhuQw==,type:str]
+grafana-oidc-secret: ENC[AES256_GCM,data:hiIW0ZbvM/+pJqCSJPnbVGrDuMRE5k3ySw4xFLju3RlxzcJjZWNQMdoQ8k419Z3lc5EcXe37/G1OcctahpZJOaw+X0q85duIJfUop9uh4gMc/s4Rn5FN6YTCA+AVB36ww7m/2QLH6ZplxMrF+nP5O84uCrWSg5ic8rUwnjJ3UMo=,iv:hdbTVO0jrGeSt1a6wtSoXKhKhouOkE0PP6hDIKi1HbM=,tag:f/+m31yw3RyPsrvoW9Um8A==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-02-15T18:51:33Z"
+    mac: ENC[AES256_GCM,data:+fEdaehzrXcnBfdb1A54E50iAy3gJqzwFqiYEUqXqgjSRLB+UnIjX7l6Ej2fyyNhts47GkB2Y0QRYTHxiQtLqEFebnnW1pzbpFj78jEouuzuBcgsZwBXZaeym40ObqrCvM42hPEkAPShBhFXmt0PABsXkP+BfVrFAN8wgQztfCY=,iv:8SKPI7EM52f+015MAFhVtezf7X3fZsK3jDBRrTtyjaA=,tag:jItSjgA+U01Ch1AtePFSHQ==,type:str]
+    pgp:
+        - created_at: "2024-02-15T18:51:00Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQEMAwcagTG/Fm6AAQf6A1t92UHbKKWnVKok+iBIi/UEKkMn8HY4sOnRqSxuY4Ey
+            uxj6B89+Uu8WuMs+ZjvY1CB3LZjSy/cCPcX7PBWAajkpTCY6lZWsScegCMufJpD5
+            urYLiNTLmmrY3Chvz2cpzjdIkD2u9O2LPbGmVE0f/X73Umb70G1dC4YEmo3VqovD
+            6wyZyKo/zVSJVlZHCurK0HXaI9Y+1QIKXumFyZc45Vf5xt745KGYN1zGGKTOPOka
+            QLi5hUp+YD8hNpg+SGC27GHHhVKRb6cvBSHMPK1CuSFxHzI4ZWwe0Ng9LLrsY5Rc
+            UD9zbZv5S3fHoPZJYk2ZMjfFF5wwsRo/6H9tQ8BAANJWAaqBLGXDETz0ob/yQy5C
+            S1M6wnz1Aagcn8AACXUEo5BfAcw9Bmcm8Rux42x6dxeOuKN6FDTGJWH8WDRL4TEy
+            JJhw4cy3n6UlC+cqhsp8bKPrMgQtLOE=
+            =vWWH
+            -----END PGP MESSAGE-----
+          fp: ad382d058c964607b7bbf01b071a8131bf166e80
+        - created_at: "2024-02-15T18:51:00Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA187ia82lSDGAQ//U8WpV7UnqQEJWVGZWs+r7sW5KKNpZtpkQ6yfgLCEm8+G
+            zb6ytSGapUkS7TrTnH5PZQS+5ZvFl+e4LdeBkwJZsXlk2kZ2CCu2ppOzBhimNc7H
+            1kwUhAtJzX2J/YPEFSSigLPcCq12JLRC3DEMp+NTjQz7DcVNgbXHyr++rY2j0X7g
+            LUmnscGhWkiWcxosqhvdvWrokQxc0Hh6pSq48lBXxs0AC8PwEwBmNxu7Nq+QCN69
+            iI5dcoxeL3jHGrWekZZOXbPuv/b6hs2FCpt7gW4QqmNLmGDT+/mK4YG6hK9Obj0V
+            Oc3aaogchwXUCHDWJeV9MWx6BWGDB+74g6Z3lyEj7GD87eTf4B7FBL0B694eMiQu
+            kfKgACRNp+LjOuR422+IKO9N21xhWY85eROYvEIHvQZQSlYX6GHpNMiDYGwu4tz7
+            A3qmUpYc7/HjsSNtLzn8W33uuXZtvS2p+Q5t7SKTSssN5E2Ns4XcEfT4X+VEuwmh
+            756efNV1XJfVtSFP7Tw2LRz0vDXxqjFcjUcHsnHuilAcndrfYPwFowjCVfvMLIRE
+            /ZMZhSMKMvFlejAZ8nFX3z2Pe995mOXlyzwjWUeXeRrR2J4qoSID3c7PP825iiX5
+            K1kwrI64pg6SGnsf4YAzXJVVpI/8RX+/okFc7kyqPk3iHu0jhrRsIwWzXQy9R03S
+            VgGUxCN1JM+2CZXE3VHx1Ba7nj4Pa2Sk2lSm8BNgzOsKzqifokpvsTHpHnj/MLdb
+            +Litx6XW5Ao2DwJYPVPzEZdB8INcflRquT5C2K/6Dh+x7GNbi5+D
+            =tczC
+            -----END PGP MESSAGE-----
+          fp: a32018133c7afbfd05d5b2795f3b89af369520c6
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1