pyroscope cooking

nix/modules/default/default.nix

@@ -179,5 +179,51 @@ in
     };
   };
 
+  services.alloy = {
+    enable = true;
+  };
+  systemd.services.alloy.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+  };
+  environment.etc."alloy/config.alloy".text = ''
+    discovery.process "all" {
+      discover_config {
+        cgroup_path = true
+        container_id = true
+      }
+    }
+
+    discovery.relabel "alloy" {
+      targets = discovery.process.all.targets
+      // Filter needed processes
+      rule {
+          source_labels = ["__meta_process_cgroup_path"]
+          regex = "0::/system.slice/.*.service"
+          action = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_process_cgroup_path"]
+        regex = "0::/system.slice/(.*.service)"
+        target_label = "service_name"
+        action = "replace"
+      }
+    }
+
+    pyroscope.ebpf "instance" {
+      forward_to     = [pyroscope.write.endpoint.receiver]
+      targets = discovery.relabel.alloy.output
+    }
+
+    pyroscope.write "endpoint" {
+      endpoint {
+        url = "http://vps3.local:4040"
+      }
+      external_labels = {
+        "instance" = env("HOSTNAME"),
+      }
+    }
+  '';
+
   deployment.tags = networkingConfig."${name}".tags;
 }

nix/modules/prometheus/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, networkingConfig, ... }: {
+{ config, lib, networkingConfig, pkgs, ... }: {
   services.prometheus = {
     enable = true;
     globalConfig = { };
@@ -83,12 +83,21 @@
             access = "proxy";
             url = "http://vps3.local:3100";
           }
+          {
+            name = "pyroscope";
+            type = "grafana-pyroscope-datasource";
+            access = "proxy";
+            url = "http://vps3.local:4040";
+          }
         ];
       };
     };
   };
 
-  networking.firewall.interfaces.wg0.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ];
+  networking.firewall.interfaces.wg0.allowedTCPPorts = [
+    config.services.loki.configuration.server.http_listen_port
+    4040 # pyroscope
+  ];
   age.secrets.loki_env.file = ../../secrets/loki_env.age;
   systemd.services.loki.serviceConfig.EnvironmentFile = config.age.secrets.loki_env.path;
   services.loki = {
@@ -142,4 +151,56 @@
     mkdir -p /var/lib/loki/{index,cache}
     chown ${config.services.loki.user}:${config.services.loki.group} -R /var/lib/loki
   '';
+
+
+  age.secrets.pyroscope_s3_secret = {
+    file = ../../secrets/pyroscope_s3_secret.age;
+    owner = config.users.users.pyroscope.name;
+  };
+
+  systemd.services.pyroscope =
+    let
+      pyroscope = builtins.fetchTarball {
+        url = "https://github.com/grafana/pyroscope/releases/download/v1.14.0/pyroscope_1.14.0_linux_amd64.tar.gz";
+        sha256 = "sha256:005539bp2a2kac8ff6vz77g0niav81rggha1bsfx454fw4dyli4y";
+      };
+      pyroscopeConfig = {
+        analytics.reporting_enabled = false;
+        server = {
+          grpc_listen_port = 9084; # random port
+        };
+        storage = {
+          backend = "s3";
+          s3 = {
+            bucket_name = "pyroscope";
+            region = "garage";
+            endpoint = "localhost:3900";
+            insecure = true;
+            access_key_id = "\${ACCESS_KEY_ID}";
+            secret_access_key = "\${ACCESS_SECRET_KEY}";
+          };
+        };
+      };
+    in
+    {
+      description = "pyroscope, the continuous profiling database";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      serviceConfig = {
+        Restart = "always";
+        User = config.users.users.pyroscope.name;
+        Group = config.users.users.pyroscope.group;
+        ExecStart = "${pyroscope}/pyroscope -config.expand-env=true -config.file ${pkgs.writeText "config.yml" (builtins.toJSON pyroscopeConfig)}";
+        EnvironmentFile = config.age.secrets.pyroscope_s3_secret.path;
+        WorkingDirectory = "/var/lib/pyroscope";
+      };
+    };
+
+  users.users.pyroscope = {
+    group = "pyroscope";
+    isSystemUser = true;
+    home = "/var/lib/pyroscope";
+    createHome = true;
+  };
+  users.groups.pyroscope = { };
 }