pyroscope cooking

nix/modules/default/default.nix

@@ -179,5 +179,51 @@ in
     };
   };
 
+  services.alloy = {
+    enable = true;
+  };
+  systemd.services.alloy.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+  };
+  environment.etc."alloy/config.alloy".text = ''
+    discovery.process "all" {
+      discover_config {
+        cgroup_path = true
+        container_id = true
+      }
+    }
+
+    discovery.relabel "alloy" {
+      targets = discovery.process.all.targets
+      // Filter needed processes
+      rule {
+          source_labels = ["__meta_process_cgroup_path"]
+          regex = "0::/system.slice/.*.service"
+          action = "keep"
+      }
+
+      rule {
+        source_labels = ["__meta_process_cgroup_path"]
+        regex = "0::/system.slice/(.*.service)"
+        target_label = "service_name"
+        action = "replace"
+      }
+    }
+
+    pyroscope.ebpf "instance" {
+      forward_to     = [pyroscope.write.endpoint.receiver]
+      targets = discovery.relabel.alloy.output
+    }
+
+    pyroscope.write "endpoint" {
+      endpoint {
+        url = "http://vps3.local:4040"
+      }
+      external_labels = {
+        "instance" = env("HOSTNAME"),
+      }
+    }
+  '';
+
   deployment.tags = networkingConfig."${name}".tags;
 }

nix/modules/prometheus/default.nix

@@ -1,4 +1,4 @@
-{ config, lib, networkingConfig, ... }: {
+{ config, lib, networkingConfig, pkgs, ... }: {
   services.prometheus = {
     enable = true;
     globalConfig = { };
@@ -83,12 +83,21 @@
             access = "proxy";
             url = "http://vps3.local:3100";
           }
+          {
+            name = "pyroscope";
+            type = "grafana-pyroscope-datasource";
+            access = "proxy";
+            url = "http://vps3.local:4040";
+          }
         ];
       };
     };
   };
 
-  networking.firewall.interfaces.wg0.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ];
+  networking.firewall.interfaces.wg0.allowedTCPPorts = [
+    config.services.loki.configuration.server.http_listen_port
+    4040 # pyroscope
+  ];
   age.secrets.loki_env.file = ../../secrets/loki_env.age;
   systemd.services.loki.serviceConfig.EnvironmentFile = config.age.secrets.loki_env.path;
   services.loki = {
@@ -142,4 +151,56 @@
     mkdir -p /var/lib/loki/{index,cache}
     chown ${config.services.loki.user}:${config.services.loki.group} -R /var/lib/loki
   '';
+
+
+  age.secrets.pyroscope_s3_secret = {
+    file = ../../secrets/pyroscope_s3_secret.age;
+    owner = config.users.users.pyroscope.name;
+  };
+
+  systemd.services.pyroscope =
+    let
+      pyroscope = builtins.fetchTarball {
+        url = "https://github.com/grafana/pyroscope/releases/download/v1.14.0/pyroscope_1.14.0_linux_amd64.tar.gz";
+        sha256 = "sha256:005539bp2a2kac8ff6vz77g0niav81rggha1bsfx454fw4dyli4y";
+      };
+      pyroscopeConfig = {
+        analytics.reporting_enabled = false;
+        server = {
+          grpc_listen_port = 9084; # random port
+        };
+        storage = {
+          backend = "s3";
+          s3 = {
+            bucket_name = "pyroscope";
+            region = "garage";
+            endpoint = "localhost:3900";
+            insecure = true;
+            access_key_id = "\${ACCESS_KEY_ID}";
+            secret_access_key = "\${ACCESS_SECRET_KEY}";
+          };
+        };
+      };
+    in
+    {
+      description = "pyroscope, the continuous profiling database";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      serviceConfig = {
+        Restart = "always";
+        User = config.users.users.pyroscope.name;
+        Group = config.users.users.pyroscope.group;
+        ExecStart = "${pyroscope}/pyroscope -config.expand-env=true -config.file ${pkgs.writeText "config.yml" (builtins.toJSON pyroscopeConfig)}";
+        EnvironmentFile = config.age.secrets.pyroscope_s3_secret.path;
+        WorkingDirectory = "/var/lib/pyroscope";
+      };
+    };
+
+  users.users.pyroscope = {
+    group = "pyroscope";
+    isSystemUser = true;
+    home = "/var/lib/pyroscope";
+    createHome = true;
+  };
+  users.groups.pyroscope = { };
 }

nix/secrets/docker_registry_password.age

@@ -1,5 +1,6 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg UtoSFhZQ2PW1y3ifXgSdQQswoi5kdRg2gvczlEateC4
-ir2FpFkYo17MGBy+C4thM4lit7vn2CiBi09DcTb6ubs
---- YvRhsfFzedjeKssmOTzHvKkvIG0zXVVCIJsRNc/LTVg
-:Ë €KîÞ$é†Prm;Û·ûÎªæ ¹Œö+é	ÚqE@<40>Àv]’¢Ôòm =Í™'Sm
+-> ssh-ed25519 qM6TYg EuWuwScs5A1DFpXrWif52YGxrscegIPGS7ZpoVS+7QI
+NsTXnjIRz7G8oNAS9MLRPkDSS7LNogLBK9hJBaC3qXE
+--- 4dEpELTzVN+A2cdD25L0A1iU8YKoB+249GEn1lxZzU8
+£Å©Ô(Ù'¥<>;CÇ5QOà´œ‰\]W('ÌÏNZ?¤†µ2f^iáBþ¸«³cÒ½‘ó
+²

nix/secrets/forgejo_s3_key_secret.age

@@ -1,7 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg GNYf0FjEDEqCe09mS9Hl7OIIjvhKTu8urwUPtY+yyB0
-xmAtm4n3s0rfq3S5OKFEG2k/noXFTKMt8hiW5QrD9SU
---- HGBYxXQGM254m2YP5twgjgDme80f0uOL2m4uKy19ZBs
-ÖÂ(
-Õ×åÇÄT
-‚®à±Öì{’ÙõF“ü-\ƒ6{mítÏæÊMÑ-óX{‡%bQd]E³’Éàü]i¸úãË}F»2¸$7¤ö#k4“;8ZžGþ_‘oÛ
–¼
+-> ssh-ed25519 qM6TYg 87cGJ52WhU1xh3a4QSpDtOrfmonf9YE7NtfZD5UvRQQ
+bvxaucwpWGnlDHNMjG5gJEGRVn1jCg6/RxYpFvlaQio
+--- umLbtzmXAZRbYIVbYjyHuorrxgxR1cctk5RYwm1JnYM
+•Kúi4ítÔÃ"pL&¥ñÅM·ýh%qóF#$Í¥×NA~†L"H
B¦±c7~úÜž/§wŘh<4PcN?è.ÄBOÑü¡0*<2A>X<EFBFBD>ö­ô_ɵ—?ˆ¿ñL!О”#

nix/secrets/garage_secrets.age

@@ -1,13 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg F9aj1EmsmRSXt1m3a41zpuwFmDBOuuaIrHkqP7PTVno
-tVs8Oxa9gV/HdUf0hN/JLuWhbrXI9BXIrsh5HnsKBQI
--> ssh-ed25519 pP9cdg dQdPm3OfbWl5Y8kJxmsUZ4rwpUo8w3+P3CHCiXw9VCw
-9yWbGgzgBz9GICAgYiOyPtMjDk/tBb4vsOveTuYP9bw
--> ssh-ed25519 XzACZQ 4lldtotM16DN/75dRX3QEmOzfIEySHcNOlFWqymI+Rs
-oOaD7dZu0xC0R7CrVpfwoBU7eSgaWyJmAZ4WptCQdes
--> ssh-ed25519 51bcvA k9eq2Tc3A9MztsdTvt3sDYUj/usYBJMp9IJQZAR67Ac
-ezccfIhPZaHKsVcUrxJL7u3jSA/kCTqLmWuQfxrFQBo
--> ssh-ed25519 vT7ExA BOCylq1RqaburnXxfsl3xqAmGSJnIxVhXK8H2xeFynk
-OWhqsbJgHWlo3hsRZVQgEaArK32OI25N4Poi2qJ9wQs
---- bBQkNfDI0onJOyxOJIN3Yl2jkK5iRgYbK67RWsipXOE
-3‡ýåA9â¯ÒAÕînÛ¯t•y®ßÚCj-îž{ÏŇâ)ô6¬DfØOÆQ¹Ü}'_n†øÈã‡>UPêNæDRŸÀÁª¨ûÊÆþ-<2D>¾„…éÂ"‡´úÛâšÙ?À>)E0<7F>™‡v(~7 eÍC¾O\UJJüŽ$SÂ8èá`€F«˜ÄíšQ§0uÙ3õmH•Ž~P÷Ž£ŒÅLqfõ~ºi¸Æn]‘=rSre#²wGŒ ³¥@ß|X#éØ÷’Â
+-> ssh-ed25519 qM6TYg +ZtvDoar2xUhBMyQFbxnxkwdF21rM5YUCZZ6CEknHj8
+wThjMiqj5tMscA3abmnhIVMAzfudkcIso7a17//FcDU
+-> ssh-ed25519 pP9cdg svRAEsI47g14ZF2YwJoxAn7zASjOHBLNEYkk8ljG00g
+WtN98vYIJ5Z8GQoaabeYlhY88OywHxp/nslKjXeRcVI
+-> ssh-ed25519 XzACZQ 6Pb4Xab6o/AyuXfLW/Ii1ogeFfT8WvUjRu2NCeMlIVA
+krn+6YjMq/FxrC6Pi4+4CA6Hd/HPI597SKFStO7hEgs
+-> ssh-ed25519 51bcvA gmx/B0WeJxVMkxa+b+LalVYDr4CP7Uv16QcrAAvOiC4
+1dzMXakXJuwx97ClerylljgKb3BlB6kiX8cpRWFOpQs
+-> ssh-ed25519 vT7ExA RNqHGxsZh8kMGy1B4GfHSE2HsJTjrjdYktVZAVSz8m0
+s4KvnACKfmNIgDz/HOUDS+z1hafjavonAntovcOr7F8
+--- 4TNQG6H8EqkGMtPysmKmJoYqZkRsNdlmGLCwsrkmQbg
+œÐ<EFBFBD>rÀž„“ÛP¿¡”žÑo`S§MÏIŠ/† «Žßù´ŠÝqÍšç•ø–yÿòýï?‡¾b^k¤<6B>¶<EFBFBD> Òš8¿Ì×*:ÞŠGA¤¥ôþ
Ðvb‰?îÙÆwí¢W›Kwî\E¾½$‚èŸEÊ eÞdÔ<64>ÞiSs/2gG"ÿµ_¯ü*Mî%ðܲzäíÿL{¦Dãž|¶õJP>¬ï‰ö´ÑsÅk’GCåÒ Ià¼\†júrÔ@N¦)¦.g{£

nix/secrets/openolat_db_password.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg 0lWcSdSricBNu8i0oMnNe0gOsoDrY9DfPvmCIS63ohc
-fY0M+k7xXU5nlLTSbJQF7iDevujQVxZ2lLca9CiBTaI
---- 5ObZSaeWsTlkqKq5D8vWKsrY8WCku2ndSlrjBKRtQE8
-ÕóˆI™þye$Q˜÷|<7C>îÂÏÂ<C38F>h'Q1Q¥·1éCõÕòÞ€mQ:ØQ.¿ýÎS¼îžE¨=cm…‘
¥äŠ@ß-¤9Öj®Œó7fǺFÚTÜ"<22>oâù"|¼0€DÎ’¾‰å,£™‘ÊöWÅ‹m*ß̬õ~5â'ç›{ jÝlu„¿Á”u Òßy+„¢ö
+-> ssh-ed25519 qM6TYg nFtXAacXhJboR+gfX7x/jtJd8ApDl3tbPezVOKZYeAE
+Ca6eA7o/b3DV+fs2RqZzyDbgu8H8rQyFmMMr28f8uOU
+--- EG+F8eM/wVjXvWKS7leaOAZJnjlkT+JRJKcnAC0iWKM
+—¼:,åL­¢Éó ß¾9I©˜BÒ£Šwú%–hè¯Ó§‡Åàû<C2A0>àb·é²þžcbˆˆ‚ŽŽ~Iç<49>¸¦&¿\NÇÉmÖTÙ´4ŽOàò0jÍ2[ßo…Œ+å÷©Üò(ˆ:|虜|9cz»µ„­ÿ·ƒµ¬µE<C2B5>ñ¦uuaæßļ§
E

nix/secrets/secrets.nix

@@ -25,6 +25,7 @@ in
   "killua_env.age".publicKeys = [ vps1 ];
   "forgejo_s3_key_secret.age".publicKeys = [ vps1 ];
   "upload_files_s3_secret.age".publicKeys = [ vps1 ];
+  "pyroscope_s3_secret.age".publicKeys = [ vps3 ];
   "wg_private_dns1.age".publicKeys = [ dns1 ];
   "wg_private_dns2.age".publicKeys = [ dns2 ];
   "wg_private_vps1.age".publicKeys = [ vps1 ];

nix/secrets/upload_files_s3_secret.age

@@ -1,5 +1,7 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg Tq8qyikECRKhPhMbKFDd+YZIGkx9uj3vOWk7QRHEkn8
-wDbkM7KZWGDF3mECEa1MPPTC5F7uxe8nGtIZZkVCWU0
---- hpRMWveZaPAIS44Jr6rRGHMOQfRi7nFpN0nxHU6fPOQ
-t4Ľ`Ćňł:,P»ůĘŤµN„?‘Äł\¬Č±é›µż­uAH_Ä?PgÎ#¨ 	łT+ŻęRÁ-ëČşXĆ,!YeZF m»d–˘Ŕ‚“şřĄ\4ÍbDAk×Lžk¤1RzĘÜő<C39C>6xo(Â8ÄgzVÍ+s|ł‚ .ĂT;<3B>O¶¨M6<0B>z‹¶A¨’Q<E28099>ĄöV®™ˇź~„ôާmŰN«ŠXĐI
qk˛ËtÔ#óÄJ–yşrSđuĘ?ÉîÚ–NŐ<>bř!KsyÜ Ď, AyfWÄŃŔ##"Ë`˘™ćnPÎX,$z1đŽ(PČÍ
ŤÄ"yű€<C5B1>|ČsđŤTźÂýxĺBFtl!6ۉ¤ěÔ0Ůos*.H/Üoëä5ŹŰ­ĐlçŔ
+-> ssh-ed25519 qM6TYg 188H/PE0b8YiAgejIznCnBKRt9z4J0il43y3O0Nk2lQ
+nu/Wbgn9dT/0TtiO+6bTUEu9F+ra/2rQKCfUK09JeN4
+--- 1YJ9Cg60UshpVEzIiqWerLqu8Bch++0Cw22tbtPJxS0
+½¿2™%ÏW».9ÆMS—»9ˆ¨e 
>Z×ü¥v²uÂò1M~<7E>ÓͪåÖý(<3w2(¼ÂZv”Ìíùãl r
‡½[8R@ª1u–ûuØ؉RûÍĸY¯!Ä_®ñZXž¦«
+:
âr_ ó
+ºÞê8ûöËf½rJžÊL×âq2s¹RŽ6!*ÿ'å½›VkÎ^î>*’·Ãº.¹®DN¶#®µÛßs"Oú,ùJ½µD´`Šàħ\¦i_-þÛKgi6CD"©u:"‡ü8²Ò51üÈãœíXÆ$©¦~«Ä|õðf¤¾8þ¾%Šáå
)tzBl«:¡óBÂó÷Ž4Îu@â(bG¾ÉüCÌú×ëFH?

nix/secrets/wg_private_dns1.age

@@ -1,5 +1,6 @@
 age-encryption.org/v1
--> ssh-ed25519 LZU5Eg C3IfbvL4t0pOHEb3Bc54+r6DZESgN6K6zPDhBlDumXk
-UwOtrqp8I90Vux6L7CsV5K+2SDFB8LBiyLO8ud7IsQU
---- 2tIecoG70broXFTtgjCUMcvk2RdKqpe5tihO6meI8DY
-泓、kﾕﾚｳ&
ﾐ`!ﾍ藪_致`<60>╋ﾏ-｡ﾔEp^<5E>ザ#:ｩ壌]ﾑ優mｻﾋy<EFBE8B>厥^O†+t8ｧ€<EFBDA7>.ｺ鉷奘; ｮ
+-> ssh-ed25519 LZU5Eg px1Nln+wkhpIG4xFeLEOhO3Dkknv+7GG9+JZdppLljg
+zL60ZG9CHQrSnLfN+gd4kV9V3kBdiGOJ4t0DmkDI9FI
+--- LIBuGo8IeBk/RyWcg1ordV6IR895iV05uz/P1iWx/+0
+%‹&×>´µüS&¢;
+</šAè·Uº¥Ú"Bÿ%/)	Kôv•Õ`ƒ‰…Š……D<E280A6>ƒ&uÁͼwŸ}y‘ºð¢D‘ùZ_Ȳ<¢yÝ

nix/secrets/wg_private_dns2.age

@@ -1,6 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 5bWSnQ wqkRMdob+7G2mTNKySF2kiGhOKt4GLN/ne+4lM3pIwA
-Iz2Brik6I6YHjVxQcoDL0UTJOWcjuiErf5kCeWpnaV0
---- 1ZkP0GiP78eGKl8te1w+o5I5kEbyPaiJFq7WGH4k1LE
-á61zIìTÂU/ò5ã'|Œ½ûÊÌþìhÕ>z±ñ¶Ýr^É‹wanog´lùX„º,kܶG
-cÊõ¸æPÇ!Rh×»fW¥éhä §
+-> ssh-ed25519 5bWSnQ rj5NgCPBbNq/15YXa6DQXKwTU9Hpm7ocLiuINgLmhCQ
+KOEgfF4MQopCZkzl2oi3obi/KBmykqzsCbHZQpC4rLY
+--- +JacdysLbQkZq1H/lLD3oPzBi3orjxix5WYlv/lprHw
+uT&É^ÀãV[pÂ6:W<>Á<EFBFBD> {k<>VÞ<56>"j„6Ò‚äŽßän¨;^éúE$ˆì„ø4BP¼–`'FX¿Q	ÖÕ}oÿ¦Çiʸ]

nix/secrets/wg_private_vps2.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 pP9cdg GI2CXAYTJWUqmab/Fnl/cFZVCCBxYZX/snQ+w0aPjSk
-8D6TxN4VYH14GQJ/XhUqyfKNLjM8f3LDmykLAvtl+IM
---- 6ru8v60LKlJjpy2PnmcwBdV09KMEh+neITYyuFscSIQ
-F Đ™ży#<ﯗ֋mŕß<C595>žş¦ĽQ2Ő^T2Lâ9µ]LÄžµh
ž[bĽrߤ!ďłjEnS?ľějCRŕ%„Ósź;mś<6D>\R
+-> ssh-ed25519 pP9cdg HiT/+dwaX8i9h5SeKF8AmEWQHFNs6nmyO089M/1Rjxo
+RjSM4+UBIrM+G9eckuH8+0Te9mhs3zXENPoFGJcHtAQ
+--- nDLsXpiyO2KyB/KDgO288tCfyvL8eCDhESLaRqBNlVg
+íŠZ±Û¡ª¼ñqÈþbB­G¾¢A¦HÁû*»_cÊ0`Öp*<2A>XY8ÂÌN<C38C>ÛÔGäÄœÛ5§‚¬Iãý
³­XÙ…Æ—h,çQ*FR%

nix/secrets/wg_private_vps3.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 XzACZQ pOD3jNWIufLkEVtkFJu6W0QjdzPJTK+t1MwgACv1zXU
-EJQ+9xPw6MnB6nJW6nDBUlzfHyY9XlfBIQlgje+FVE4
---- BmTwJED+mJ/Qr0WFDELozwR2BgGDkHDcR2I9eSxuVn8
-KêÃ~alNh.€½ù «kiAÛF*Ã/MY±ñ¡Zd†¬p”AÉ+-²Ù¬A<C2AC>¹Ü¢*S¥Z¶ï„ Nê­F­fˆ‚bô3tª×rûÇy
+-> ssh-ed25519 XzACZQ kJVyFsqcqh/THJgijrdhNFFelSpD2w4VGZEvNsQSwlM
+DzC/l6rFyE9JF+IoA3pQv/3PihXnIlWQHje6++9rJsA
+--- kRjNUG+4tPhNCcoAO47gwj04gmBXZXBPqgvH81KNn/E
+‹ý#¦n>fa^!ÄŚ§+sFđě”ÚŔŤŤ™ś<>ĄB.u{IďÖ9NďVů–§äĎ<C3A4>˘ixéé<C3A9>ŇňŇŽżŇY	öć“Í.Je

nix/secrets/wg_private_vps4.age

@@ -1,6 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 51bcvA mzB9FcwUgPczK4/Rd2DZvCYoQfjT4qE+Z7HE9yHjgGU
-sPDlr+YNhvbjYagyJb/kua9dWeG9tSt6KNjKh+/p+ps
---- uZVoWpqKjapTtWRGpc7cUoifwOVFfd5DU+9pQpwruuo
-F
v6¾œÚ‹Ëï,ø»K¶¶Ó†(ÍÝè‰kÙ~Y4Á.`z(]w2²MV
"¼%À³JUÚ$ô•È«ÁǸCïG
-_:F¸Ý§ S
+-> ssh-ed25519 51bcvA bcFC2L95dfbs7WLewqcQeH62fgba42DbYrFu126AWFg
+vfS/79EjlhWxp4cdOJv0WRWp8BIGefIvkNHrr5wcC7Y
+--- F8/9B/N6d0SjEHRHuiuCsqRGgRr2XoZdFRT1UKbi7+0
+9²å¤èlx<EFBFBD>"uV×ÔW#IE”Ûs¤Þjì89®
d©žÑD<C391>kTgöÊ´?<<3C>°2
5·qïØ'ë¬ )yÖ‚¢má}