backup static files to garage

newinfra/nix/apps/cargo-bisect-rustc-service/default.nix

@@ -21,6 +21,13 @@ in
     };
   };
 
+  services.custom-backup.jobs = [
+    {
+      app = "cargo-bisect-rustc-service";
+      file = "/var/lib/cargo-bisect-rustc-service/db.sqlite";
+    }
+  ];
+
   system.activationScripts.makeCargoBisectRustcServiceDir = lib.stringAfter [ "var" ] ''
     mkdir -p /var/lib/cargo-bisect-rustc-service/
     chmod ugo+w /var/lib/cargo-bisect-rustc-service/

newinfra/nix/apps/uptime/default.nix

@@ -29,6 +29,13 @@
     };
   };
 
+  services.custom-backup.jobs = [
+    {
+      app = "uptime";
+      file = "/var/lib/uptime/uptime.db";
+    }
+  ];
+
   system.activationScripts.makeUptimeDir = lib.stringAfter [ "var" ] ''
     mkdir -p /var/lib/uptime/
   '';

newinfra/nix/apps/uptime/uptime.json

@@ -24,15 +24,15 @@
     },
     {
       "name": "bisect-rustc.nilstrieb.dev",
-      "url": "https://bisect-rustc.nilstrieb.dev"
+      "url": "https://bisect-rustc.noratrieb.dev"
     },
     {
       "name": "hugo-chat.nilstrieb.dev",
-      "url": "https://hugo-chat.nilstrieb.dev"
+      "url": "https://hugo-chat.noratrieb.dev"
     },
     {
       "name": "api.hugo-chat.nilstrieb.dev",
-      "url": "https://api.hugo-chat.nilstrieb.dev/api/v2/rooms"
+      "url": "https://api.hugo-chat.noratrieb.dev/api/v2/rooms"
     },
     {
       "name": "cors-school.nilstrieb.dev",

newinfra/nix/hive.nix

@@ -161,6 +161,7 @@
       ./modules/garage
       ./modules/podman
       ./modules/registry
+      ./modules/backup
 
       # apps
       ./apps/widetom
@@ -234,13 +235,22 @@
     
     '';
   };
-  vps5 = { name, nodes, modulesPath, config, ... }: {
+  vps5 = { name, nodes, modulesPath, config, pkgs, ... }: {
     imports = [
       (modulesPath + "/profiles/qemu-guest.nix")
       ./modules/contabo
       ./modules/ingress
       ./modules/wg-mesh
       ./modules/garage
+      ./modules/backup
+    ];
+
+    services.custom-backup.jobs = [
+      {
+        app = "testapp";
+        file = "/etc/hosts";
+        environmentFile = pkgs.writeText "env" "MyEnv=true\n";
+      }
     ];
 
     deployment.tags = [ "eu" "apps" "wg" ];

newinfra/nix/modules/backup/backup.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+time="$(date --iso-8601=s --utc)"
+echo "Starting backup procedure with time=$time"
+
+dir=$(mktemp -d)
+echo "Setting workdir to $dir"
+cd "$dir"
+# Delete the temporary directory afterwards.
+# Yes, this variable should expand now.
+# shellcheck disable=SC2064
+trap "rm -rf $dir" EXIT
+
+echo "Logging into garage"
+export MC_CONFIG_DIR="$dir"
+mc alias set garage "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" --api S3v4
+
+mc ls garage/backups
+
+files=$(jq -c '.files[]' "$CONFIG_FILE") 
+
+IFS=$'\n'
+for file_config in $files; do
+    filepath=$(echo "$file_config" | jq -r ".file")
+    app=$(echo "$file_config" | jq -r ".app")
+
+    echo "Backing up app $app FILE $filepath..."
+    tmppath="$dir/file"
+    xz < "$filepath" > "$tmppath"
+
+    echo "Uplading file"
+    mc put "$tmppath" "garage/$S3_BUCKET/$app/$time/$(basename "$filepath").xz"
+    echo "Uploaded file"
+done

newinfra/nix/modules/backup/default.nix

@@ -0,0 +1,64 @@
+{ config, lib, pkgs, ... }: with lib;
+let
+  jobOptions = { ... }: {
+    options = {
+      app = mkOption {
+        type = types.string;
+        description = "The app name, used as the directory in the bucket";
+      };
+      environmentFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+      };
+      file = mkOption {
+        type = types.string;
+      };
+      #pg_dump = { };
+      #mongo_dump = { };
+    };
+  };
+in
+{
+  options.services.custom-backup = {
+    jobs = mkOption {
+      default = [ ];
+      type = types.listOf (types.submodule jobOptions);
+      description = "Backup jobs to execute";
+    };
+  };
+
+  config =
+    let
+      cfg = config.services.custom-backup;
+      backupConfig = {
+        files = builtins.map (job: { app = job.app; file = job.file; })
+          (builtins.filter (job: job.file != null) cfg.jobs);
+      };
+      backupScript = pkgs.writeShellApplication {
+        name = "backup";
+        runtimeInputs = with pkgs; [ jq minio-client getent xz ];
+        text = builtins.readFile ./backup.sh;
+      };
+    in
+    {
+      age.secrets.backup_s3_secret.file = ../../secrets/backup_s3_secret.age;
+
+      systemd.services.custom-backup = {
+        startAt = "daily";
+        serviceConfig = {
+          DynamicUser = true;
+          ExecStart = "${backupScript}/bin/backup";
+          Environment = [
+            "CONFIG_FILE=${pkgs.writeText "backup-config.json" (builtins.toJSON backupConfig)}"
+            "S3_BUCKET=backups"
+            "S3_ENDPOINT=http://localhost:3900"
+          ];
+          EnvironmentFile = (builtins.filter (file: file != null)
+            (builtins.map (job: job.environmentFile) cfg.jobs)) ++ [
+            config.age.secrets.backup_s3_secret.path
+          ];
+        };
+      };
+    };
+}
+

newinfra/nix/modules/garage/README.md

@@ -21,9 +21,15 @@
     - key `docker-registry` RW
 - `loki`
     - key `loki` RW
+- `backups`
+    - key `backups` RW
 
 ## keys
 
 - `caddy`: `GK25e33d4ba20d54231e513b80`
 - `docker-registry`: `GK48011ee5b5ccbaf4233c0e40`
 - `loki`: `GK84ffae2a0728abff0f96667b`
+- `backups`: `GK8cb8454a6f650326562bff2f`
+
+- `admin`: `GKaead6cf5340e54a4a19d9490`
+    - RW permissions on ~every bucket

newinfra/nix/modules/garage/default.nix

@@ -1,6 +1,10 @@
 { config, pkgs, name, ... }: {
   age.secrets.garage_secrets.file = ../../secrets/garage_secrets.age;
 
+  environment.systemPackages = with pkgs; [
+    minio-client
+  ];
+
   networking.firewall.interfaces.wg0.allowedTCPPorts = [
     3901 # RPC
     3903 # admin for metrics

newinfra/nix/secrets/docker_registry_password.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg NvguOs7htIflYp6bh6oiiH7Cp2l/0Mf4mcf/4b8ReQg
-BngCQfbilctBfNKjE+TkEhE3Bk2pkIlc1UYdAFISP/g
---- 3hA+KfCqIAvwuL+mr4PFW9hVlpsc+t0uwG8I8Uc8JXY
-JŸÄ@¡zI­´9ì5/Š`…Ÿ…VfWߤæßp#™0;DÔ 
Ëدw'¬r²ÞgËÁ+n
+-> ssh-ed25519 qM6TYg 1HgGuuBWZKvGpR755SyGybRGIq26JR8qb4x4hywwWU8
+6e0gmCgL6CttzzzZ73oUYzpCcvhArAdFJGycwacFaIY
+--- tfUAHcZONQZuZIXtumjCh1Crawf+BSl7djHSHC3WvJ8
+Sùs¢ƒ.²QeMçœ~Kàâ!ÙŸ•\‹‡ˆC¥¨<C2A5>qêy¥^)<29>S¸ìGøjaòŽaÕ™

newinfra/nix/secrets/secrets.nix

@@ -18,6 +18,8 @@ in
   "registry_s3_key_secret.age".publicKeys = [ vps1 ];
   "grafana_admin_password.age".publicKeys = [ vps3 ];
   "loki_env.age".publicKeys = [ vps3 ];
+  "backup_s3_secret.age".publicKeys = [ vps1 vps3 vps4 vps5 ];
+  "s3_mc_admin_client.age".publicKeys = [ vps1 vps3 vps4 vps5 ];
   "wg_private_dns1.age".publicKeys = [ dns1 ];
   "wg_private_dns2.age".publicKeys = [ dns2 ];
   "wg_private_vps1.age".publicKeys = [ vps1 ];

newinfra/nix/secrets/wg_private_dns2.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 5bWSnQ gkB+3cKWYoBYV0eDyBD5g5UWsSs94DaQSka42g9EsA8
-B8Hm9IrecZNv75DWfPJPPGJSP8AkBcOgy070foI8gPQ
---- d6F58mcEIAHmIpaAr1sgMaQUQAqa6Tr7DHq5hFgetOc
-¢±…Lmb&¶šËÉ<C38B>d¼1×¥Ø^{¹;÷¦”µtHå ¨5Mª©ç½Xö—Þ1¿cæ¹[”Õb<>®ø[õÎß ¯¬ë篈jzßû(
+-> ssh-ed25519 5bWSnQ yYpbqupe6d0ZiH4CxnkHx6clUSI6VOAwiFicoeghIi8
+Q1rxBbAhYeZfi5uSNW7/kE/sn15ZpDSxC/P8/SuekWQ
+--- CsY6lrPSTBryg9t7U1FfnoAYoz0pDRhRpkTy+bsJrZc
+E¤›'¯½fŽìå0Ö7TÉBcôœã¯¦<14>±‚-߬½&‚K¬“pdç~¯ØG„üÄÁµm8: )L¡pyÎòœ x˲ûÑ<ˆÿ