improve the backup situation

nix/apps/does-it-build/default.nix

@@ -39,12 +39,10 @@ in
     };
   };
 
-  services.custom-backup.jobs = [
-    {
-      app = "does-it-build";
-      file = "/var/lib/does-it-build/db.sqlite";
-    }
-  ];
+  services.custom-backup-restic.jobs = [{
+    app = "does-it-build";
+    path = "/var/lib/does-it-build/db.sqlite";
+  }];
 
   users.users.does-it-build = {
     isSystemUser = true;

nix/apps/forgejo/default.nix

@@ -1,4 +1,4 @@
-{ config, ... }: {
+{ config, lib, pkgs, ... }: {
   age.secrets.forgejo_s3_key_secret.file = ../../secrets/forgejo_s3_key_secret.age;
 
 
@@ -51,8 +51,23 @@
     '';
   };
 
-  services.custom-backup.jobs = [{
+  services.custom-backup-restic.jobs = [{
     app = "forgejo";
-    file = "/var/lib/forgejo/data/forgejo.db";
+    # this is a mess. do not question it. it is a beautiful mess.
+    dynamicFilesFrom = "${lib.getExe pkgs.sudo} --user=forgejo ${lib.getExe (pkgs.writeShellApplication {
+      name = "backup-forgejo.sh";
+      runtimeInputs = [ pkgs.unzip ];
+      text = ''
+        rm -rf /tmp/forgejo-backup
+        mkdir -p /tmp/forgejo-backup
+        {
+          cd /tmp/forgejo-backup
+          ${lib.getExe config.services.forgejo.package} dump -c ${config.services.forgejo.customDir}/conf/app.ini
+          unzip forgejo-dump-* >/dev/null
+          rm forgejo-dump-*
+        } >&2
+        echo /tmp/forgejo-backup
+      '';
+    })}";
   }];
 }

nix/apps/killua/default.nix

@@ -25,10 +25,10 @@ in
     };
   };
 
-  services.custom-backup.jobs = [
+  services.custom-backup-restic.jobs = [
     {
       app = "killua";
-      file = "${dataDir}/trivia_questions.json";
+      path = dataDir;
     }
   ];
 

nix/hive.nix

@@ -178,6 +178,7 @@
       ./modules/podman
       ./modules/registry
       ./modules/backup
+      ./modules/restic
 
       # apps
       ./apps/website
@@ -227,6 +228,7 @@
       ./modules/wg-mesh
       ./modules/garage
       ./modules/backup
+      ./modules/restic
 
       # apps
       ./apps/website

nix/modules/restic/default.nix

@@ -0,0 +1,74 @@
+{ config, lib, ... }: with lib;
+let
+  jobOptions = { ... }: {
+    options = {
+      app = mkOption {
+        type = types.str;
+        description = "The app name, used as the directory in the bucket";
+      };
+      environmentFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+      };
+      path = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+      };
+      dynamicFilesFrom = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+      };
+      pgDump = mkOption {
+        type = types.nullOr (types.submodule ({ ... }: {
+          options = {
+            containerName = mkOption {
+              type = types.str;
+            };
+            dbName = mkOption {
+              type = types.str;
+            };
+            userName = mkOption {
+              type = types.str;
+            };
+          };
+        }));
+        default = null;
+      };
+      #mongo_dump = { };
+    };
+  };
+in
+{
+  options.services.custom-backup-restic = {
+    jobs = mkOption {
+      default = [ ];
+      type = types.listOf (types.submodule jobOptions);
+      description = "Backup jobs to execute";
+    };
+  };
+
+  config = {
+    age.secrets.restic_backup.file = ../../secrets/restic_backup.age;
+    age.secrets.generic_backup_password.file = ../../secrets/generic_backup_password.age;
+
+    services.restic.backups =
+      builtins.listToAttrs (map
+        (job: {
+          name = job.app;
+          value = {
+            paths = if job.path != null then [ job.path ] else null;
+            dynamicFilesFrom = job.dynamicFilesFrom;
+            initialize = true;
+            timerConfig = {
+              OnCalendar = "00:00";
+              RandomizedDelaySec = "5h";
+            };
+            passwordFile = config.age.secrets.generic_backup_password.path;
+            repository = "s3:http://localhost:3900/backups-restic/${job.app}";
+            environmentFile = config.age.secrets.restic_backup.path;
+          };
+        })
+        config.services.custom-backup-restic.jobs);
+  };
+}
+

nix/secrets/docker_registry_password.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg 8JUZfDdX9AEAdOITGWTvE7DRH7VPUqsM9T7u7AhExlQ
-UJhh3gVlfXc6ij/DFdd3a4I2QfZu8mZ0CrLaotxQ+Ck
---- eUV5GMuqhgxWHxZZ5Ee3QobSg42A3ja8h1nuxAeWt1Y
-DóѬvúZË_ò<5F>%ŠðÅC ŠFþ._é
:ñâ§R®™»1Uj²«ší€O«PÇ+ƒ}
+-> ssh-ed25519 qM6TYg kxQujT+O6ZGlzTONdS/18DUVoxNapwtxitQo8GKr2hc
+b7KjCjuvhmWcqNB6BvNruL17Ww6yWkVKjjm/MGd+jlE
+--- q3EzroLr8b0T2gKQ4xUR67YOLSwFP1V8UxAnKY0PP24
+ﾟl貊0嬌1ｽCXq<58>謎{T姻tg傅ﾘﾟｭﾍ<EFBDAD>ﾛ捕詑<E68D95>ｦ<EFBFBD>s5<13>hxk-

nix/secrets/forgejo_s3_key_secret.age

@@ -1,8 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg UP4wmNgpJ9JErCdgk4oCAjwVw8w8MOE9IRdZfDADYgQ
-xXd++OFcjJvkyYzow0WAVy0n2AV/0MZUXy+MYbIwZxo
---- 6aQIiK6E6tp6wXkaRdJcMfVYHh5zFzSmL9r2iU60wEo
-¥°Sps}z¢ç
-½Ò
-ûœØÊÞ@\Hž_)úŠ”…£äUÊûÛ½åœ<C3A5>*ôâ
-¾Jœ48¡i]š¨D¼èÓð+_…uÉù‹Ð,Ζ2‰m³âj(˜…¶’.ÓÙ¯lA
+-> ssh-ed25519 qM6TYg yxVVZ7LOgN9NiKsl1+dN7Rp6Rsf0zlqb25Y6w43styk
+gQ5g7TL8+lyGp0SxdcoRg0nTpu1w6WbZZK0ERyqRpkc
+--- 5uKpMbkW4zZ035mNXCuty+64IZ360gly/ezxnwtRX/0
+šË<EFBFBD>ô±ëSϸ>q!ŁźěOß·żźŤ”ÉT››CU˙ż¸ŁŇşž<C59F>Ď[ń*󅬟«’\wźÝ‹tcÇ×Őíg?
|B±ŘĹë;’»"*îd<C3AE>Wţ<57>ţÔvŔ/vĚnqe

nix/secrets/garage_secrets.age

@@ -1,14 +1,14 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg P1fCQKmzsmKh0JnB185cO8KhfJ1Nyf9c4Ld90TdMg1w
-quzrkpAmOStHDT1CUwE7hMBf5NyoGZ/tF0vUatVGrl8
--> ssh-ed25519 91VHug C6sFv6lpxgOQNnABrtZEwv82n71wzKo67dQ4hE0LaW0
-dkHAjqWF201gRr9DivqUfycT3/lkrhDJajUnxVBaws4
--> ssh-ed25519 XzACZQ 7OpgE7ZvoFOd380AkrS4wZZQtpiXwKG9xq+w5EzzxD0
-w4aGa7AUwTvcl6DEfwJhFp4uOD1gyQG+MZkgUCHN5KA
--> ssh-ed25519 51bcvA pYE+ZDrXfjSf0cKBZwo0OZ4BXQvTZhDll9/fn11Mj3I
-l/2Z0lN2irj216+5EEIDvRd2sGWFCWWnqOXqGUtRR1w
--> ssh-ed25519 vT7ExA 9WGSmvdL9I0Hv6aPPQgUSEXmUBfJfv6Zbi3ywG2BBTQ
-qTsDY/NN/RVJcznCjoGC0ABYhWaorzEBB206X214HRo
---- yUwt0ca0wolODUhl1JwYyiF4PoLzd7N0KkEef9de4YQ
-iš@>jĽP<C4BD>N+<2B>s6
Ú‰’€VČ+8eL‹Ě"˙Őł,WŁéŢ1‡ů©”Š~é(“71‚ĺNwĽ2UŚŔ1u<31>©ŃOÝ<4F>8c›‹÷I<C3B7>§am ¤c•çČ˙_9~8
ĺ€IĹ€Ž<E282AC>® ˙Ă–~.Đš(¶ŹTŃÖ…Rľq…Îď<X…#wĄĎő!?A}
-KT|ޤĆMečç‘ý±¨DMëš˙Îjó
~ëV 9«´íCo?tő]©›ĘĚ`€IęWK
+-> ssh-ed25519 qM6TYg L6XQVJyw/T2kwH2iOa3dAxwxlAInVTKXek5QHUKsU3g
+Mm8VXH3CITbrJTqBcjdMHGm1k7Kztd4irHqWnF0yKWM
+-> ssh-ed25519 91VHug 2PoHH6c5lVj6hfTH6+2NLzJcpXh20LgYgUHYrpY9Wg4
+iSaKgUoh0eSjeoiZ7mggn7LWV9C9xsM2foZrpJZY/ok
+-> ssh-ed25519 XzACZQ xyYG90gHM9XHAxIv2cFU/WvZsdLo6prAPQgRKADP0UY
+XdiJz1Zmh3S+IvaOCZBqrF8DSWBrrqePjy+ZiEwuaSA
+-> ssh-ed25519 51bcvA IYyt435x3NPiTKDKCVb8dIK5naOSyU/Wh7dOLQ6SRAM
+M0s39vyjDY24Mlb16UHy9lFEseaJxzZhtCvszKxwVSA
+-> ssh-ed25519 vT7ExA ZBvUjCU8vYFddKgNfnuROuiDnDXhitG6eR6ek4+1R1E
+fiyuqamkakZ4t1MosIUqZaR0WEh7XVAwJiwH8lz3bsc
+--- zvHNHd6bOMd0f3eIrl1qzyRh6zUe7G8mdrraYGtrnXU
+¿Ÿ%iFr Ñ™Ù`|)Ñ+é¢ÏŽúÜ<C3BA>ÿšS—”®ŽGÎNP²ß>ƒ?E{Ö¦<C396>mda‡dä<64>j`Ôˆ¾îþVÙü uÅn¢§­ÅZ¥Ö»X±ÂïUæ—½}ú¾>Ðj_´Dü_;	5ŠOpaò;	
+Ÿ¾Þ~4bðwZ'7¬µ0æ^Œ2Žã+w­Z0j;<3B>4ÕyTdp·•1'9Ù ²7Zësbøó¿ß.JÁ*(§¦y”Wö
_‡P¡‹g`S^Gäàrƒ

nix/secrets/generic_backup_password.age

@@ -0,0 +1,14 @@
+age-encryption.org/v1
+-> ssh-ed25519 qM6TYg IBVFRlOVLHcuS6xa7UVGA1z9NTBtNwGbt94c/yTB8wE
+T+VtsTngND9kAd6DAtksXN4xYs+E8JZSxDeOm+G23tc
+-> ssh-ed25519 91VHug nUkRwHgpn2i56NNY0VAuG+r3CX1rjt1M0ZVKj+ijwGo
+ea8Ry6JIJlPOObY+v2Q5MkdcZqCeDLAOxC583WY38Hg
+-> ssh-ed25519 XzACZQ 7f+8YcecMvwnOgwxjRMUUUm9Sp4cyKpIZWWMDrrCtzg
+Bqhd2kpuTg3Xchme5wHfg4zkuikeM4H9GdOZVUv+HZk
+-> ssh-ed25519 51bcvA DUk4CsGXhdj4uIqzYpoGmtHs5dnjIBUb0c9zj1DEum4
+hGe3j5Ycn/WVV5wgg+vZuh2KhnamHACkHrDWcVgkSjo
+-> ssh-ed25519 vT7ExA Zf67OkbMvOpgABZDuXw3U94KqX32VG8nnjo3Xmkbih0
+5K5fnBxkQDaYwuMPhyNU5ZrZLjkgknG7dzMzyuANMuU
+--- Jon4j4/xeZqS/6KsWszsVOoVOgJgsPEKxmtC7PcocCA
+Ú솳—µ’~Š…
+¢íNŒ+jKþ߬Ÿ/á]Ó !䂶Œ¢.7\«–„k~<÷ñfÃCÖT.ªO
ŸêÅ¥*aÁ
•Û

nix/secrets/grafana_admin_password.age

@@ -1,5 +1,6 @@
 age-encryption.org/v1
--> ssh-ed25519 XzACZQ lWHvBQNaeM4hMI6u36HvYCqLS3G/ScLbwOThzdGSzSA
-PGPpaoY0V84v6CRutJk+K6M9BM7XaRwp2awPsB/Db6k
---- hlo1/uWQVHBmhfMRLPiA+9H0TGKYF/+gxUzzdAM4rYU
-ä+@96YĘđďą‹~ť I—Ř]Očh­Mş6T{3€XˇČĚý(q‘ZśËVR]CLp®O-ĂÇÖ>`/-Ł
÷Üń.2ˇĚŮYMÓjăĺťŮýyÝUČ°î^GŚn¤ŮçS
+-> ssh-ed25519 XzACZQ OeTS5wU4ac+Qh7s1PXbdFH3LDlRW1LV+qFtoVGI47XQ
+JsixYPLzpnF45ODQH7nuVowXzwbNQi8lWx1Bp2YFVWc
+--- MEG4bfGwoFRm9HizYdqtK7KApYhYH+QjAIEp7CpLznA
+CŒ¢µÍ/wC
+F‘<EFBFBD>zÙ?ŸMÀõókÙr‰ Žx£N¸©'NTzùà¼WŽÈb¹åº{›ÞóÕéAj3X6m¹Ý²²J@í¼OI—{u<15>ßý
”Ï?¹ A,CÃdûý^

nix/secrets/registry_htpasswd.age

@@ -1,5 +1,6 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg qhB01I5HcTnTHTJTEYLEtJi416tlC1EMD4yjoBIK7gw
-h8CcWgY/GslHI1FbXi5k5QXFs7YbM7wr7JWtez6ct84
---- oLyc6wK3Bgl/zxjpZJYWoGAxWnXx4LN/+iT+r8RPOco
-\,Ý	µĘf­‡¬#
`¾‡ünSÊK×ÕÌPVv”Vʵ<÷ œÙ-:Ÿçxg™´“屄þM(Yòiu€¹Šcìþl^‹aí°pਬ%^ô¸÷+"‘w»ªV‚<56>
+-> ssh-ed25519 qM6TYg amvNJk2G0JJHgGOwAWCtYIJgylqBAYDSXTKNYKwb4mo
+aBXr7jN0/VUDTxCGvn+obz3JIU0boKcm4BbwFAidm1o
+--- B6s8naj3JiQdjBDdwzY+PqW01QZFgKMpKSOVXHDfbHw
+˛Tą1ôÄQđ˛qÎýŇ”Ś?)W“Ĺ<E2809C>‰ť€˙ŘÚ<C598><C39A>‡4ę¬zŃc"CĎ+ĹŚHÓmQ)ö·ËÔ‹ˇ#Ś1HăßýÂnc«¬^ĆaÜűGş›±.™µ'Â
+$ bc´Ě

nix/secrets/secrets.nix

@@ -26,6 +26,8 @@ in
   "forgejo_s3_key_secret.age".publicKeys = [ vps1 ];
   "upload_files_s3_secret.age".publicKeys = [ vps1 ];
   "pyroscope_s3_secret.age".publicKeys = [ vps3 ];
+  "restic_backup.age".publicKeys = [ vps1 vps2 vps3 vps4 vps5 ];
+  "generic_backup_password.age".publicKeys = [ vps1 vps2 vps3 vps4 vps5 ];
   "wg_private_dns1.age".publicKeys = [ dns1 ];
   "wg_private_dns2.age".publicKeys = [ dns2 ];
   "wg_private_vps1.age".publicKeys = [ vps1 ];

nix/secrets/wg_private_dns1.age

@@ -1,6 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 LZU5Eg PttdTzbbxLbUw+V+mCZjRREsWuIHhGeVvIKkPCAkvQQ
-3adTE4nRuPaMYo3wslgO2kND5dVYv5NOKYpi129kRrE
---- eKBS0RAp1BiY55dJ2vdJZdkJIA6wk/OA+JA2aTvLAFE
-†F‡]Y@JÈÞU”M„“—ȯø4æ9£1®÷ 4<C2A0>§cböÏ““B9»^š.¯Ñ\™ù~È’èg‚kS’Gz÷Þ
-ìÌôµ
+-> ssh-ed25519 LZU5Eg o+MPatbYPM3sZq0MCqvvxlvKMQwlbajHURPQ+0g0qm8
+UUurAYkPWXCaow746EV4dAQ+qTJnHIehcorUmanBc+o
+--- BV+bxd0OIc3J4uT39al2odyn8ScDpq58SiwnW5pvRj4
+òçT7W
 í|õfJÞÜ%"cõôäqÁ{TãP~f<>v,;Ñ:å…<C3A5>¾êŒ-ÓÏšÛ4þ€a† æ-¯uÌ\L
ƒ_-¼VHâøûš³½%

nix/secrets/wg_private_vps3.age

@@ -1,6 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 XzACZQ 4f3Sm/Xpuu+lgnR+C5sLxrsADC4KjAwRCvb91zrPlg4
-Iok5RHD15fZmRWIay0nHzy1rtZjgt3Pbq23z6n6Zr78
---- 4pu7oE7I2dV3Gd3r+cbezJWZULNS4n98B//0D+Vj55U
-Ýâćß%3<>
'=Ú¨@x^­-`ę[
-ÖŮJÚ_•;·WęLEʨ—@-ĆőS‡„ŢAyĘ›©Ś›	eöuÚqÎŐzâLĄâÇřQ‚ß
+-> ssh-ed25519 XzACZQ lm64+fQEWa9hF98cV/x1U3Mz+6zuM23dAV3XkwE7iz4
+7Rgqd13DThp/JLryCe5xTdXwDujaTj4viR2CBTdXYLs
+--- pwebssA2O2VjzPFRAQ0/65+qiiF/MijCIIXexwH5mgk
+\ ‹fóËæÅv×̤ä[§ýÚŸÆIŒ´†[—5á÷*×·90²'ý4Âôî+áV<C3A1>;L›‹~jÌÂà¦
‹úœ†;ÝÒSÁª2y·b

nix/secrets/wg_private_vps5.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 vT7ExA cGTbCRJ9dO5DMMYVZyMGswdyx6q114yInksFITtJR1U
-EmFSgbzljek+luv2MncANyEoCRlCxrQN1OOrn5ejf6U
---- Sap30+9H/NG4GrkJTxjXUI0rxIugDMB5JIlh0PgSPhk
-£–Xó@~ô¶<C3B4>îŽL¬_®äA²m3 ñýƒûÜXS)g‚rÇ<>ínwH<77>ñž%lƒ¦µ
J²’€ŸcDŸü¨´Ï£öÎ5ãPK«S¦‡
+-> ssh-ed25519 vT7ExA G9mqOZiAvq+ot4OUevoxvNPIkgWgS8KqMY76uGsxeGs
+AMEwoZoFc+axirDc5q+FM3e76IedkxblC3vVqUjmPL8
+--- oXGSsFKfJRPvcU1X3zHN7M6vd0IxBpNowyh4sPesq3A
+¢¡i3<><18>¥Ÿûôc—ÿòØMÄTN0—‰}r"Зs˜Œ§ö~þ<>OrP˜®ÃîFP`Q•˜¯<º%å:7ø3ç‚