diff --git a/newinfra/nix/apps/cargo-bisect-rustc-service/default.nix b/newinfra/nix/apps/cargo-bisect-rustc-service/default.nix index 2553d92..c9ae3d8 100644 --- a/newinfra/nix/apps/cargo-bisect-rustc-service/default.nix +++ b/newinfra/nix/apps/cargo-bisect-rustc-service/default.nix @@ -21,6 +21,13 @@ in }; }; + services.custom-backup.jobs = [ + { + app = "cargo-bisect-rustc-service"; + file = "/var/lib/cargo-bisect-rustc-service/db.sqlite"; + } + ]; + system.activationScripts.makeCargoBisectRustcServiceDir = lib.stringAfter [ "var" ] '' mkdir -p /var/lib/cargo-bisect-rustc-service/ chmod ugo+w /var/lib/cargo-bisect-rustc-service/ diff --git a/newinfra/nix/apps/uptime/default.nix b/newinfra/nix/apps/uptime/default.nix index b836a43..7552789 100644 --- a/newinfra/nix/apps/uptime/default.nix +++ b/newinfra/nix/apps/uptime/default.nix @@ -29,6 +29,13 @@ }; }; + services.custom-backup.jobs = [ + { + app = "uptime"; + file = "/var/lib/uptime/uptime.db"; + } + ]; + system.activationScripts.makeUptimeDir = lib.stringAfter [ "var" ] '' mkdir -p /var/lib/uptime/ ''; diff --git a/newinfra/nix/apps/uptime/uptime.json b/newinfra/nix/apps/uptime/uptime.json index 955289c..af235e3 100644 --- a/newinfra/nix/apps/uptime/uptime.json +++ b/newinfra/nix/apps/uptime/uptime.json @@ -24,15 +24,15 @@ }, { "name": "bisect-rustc.nilstrieb.dev", - "url": "https://bisect-rustc.nilstrieb.dev" + "url": "https://bisect-rustc.noratrieb.dev" }, { "name": "hugo-chat.nilstrieb.dev", - "url": "https://hugo-chat.nilstrieb.dev" + "url": "https://hugo-chat.noratrieb.dev" }, { "name": "api.hugo-chat.nilstrieb.dev", - "url": "https://api.hugo-chat.nilstrieb.dev/api/v2/rooms" + "url": "https://api.hugo-chat.noratrieb.dev/api/v2/rooms" }, { "name": "cors-school.nilstrieb.dev", diff --git a/newinfra/nix/hive.nix b/newinfra/nix/hive.nix index 3de4891..d95358d 100644 --- a/newinfra/nix/hive.nix +++ b/newinfra/nix/hive.nix @@ -161,6 +161,7 @@ ./modules/garage ./modules/podman ./modules/registry + ./modules/backup # apps ./apps/widetom @@ -234,13 +235,22 @@ ''; }; - vps5 = { name, nodes, modulesPath, config, ... }: { + vps5 = { name, nodes, modulesPath, config, pkgs, ... }: { imports = [ (modulesPath + "/profiles/qemu-guest.nix") ./modules/contabo ./modules/ingress ./modules/wg-mesh ./modules/garage + ./modules/backup + ]; + + services.custom-backup.jobs = [ + { + app = "testapp"; + file = "/etc/hosts"; + environmentFile = pkgs.writeText "env" "MyEnv=true\n"; + } ]; deployment.tags = [ "eu" "apps" "wg" ]; diff --git a/newinfra/nix/modules/backup/backup.sh b/newinfra/nix/modules/backup/backup.sh new file mode 100755 index 0000000..88d3eb6 --- /dev/null +++ b/newinfra/nix/modules/backup/backup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +time="$(date --iso-8601=s --utc)" +echo "Starting backup procedure with time=$time" + +dir=$(mktemp -d) +echo "Setting workdir to $dir" +cd "$dir" +# Delete the temporary directory afterwards. +# Yes, this variable should expand now. +# shellcheck disable=SC2064 +trap "rm -rf $dir" EXIT + +echo "Logging into garage" +export MC_CONFIG_DIR="$dir" +mc alias set garage "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" --api S3v4 + +mc ls garage/backups + +files=$(jq -c '.files[]' "$CONFIG_FILE") + +IFS=$'\n' +for file_config in $files; do + filepath=$(echo "$file_config" | jq -r ".file") + app=$(echo "$file_config" | jq -r ".app") + + echo "Backing up app $app FILE $filepath..." + tmppath="$dir/file" + xz < "$filepath" > "$tmppath" + + echo "Uplading file" + mc put "$tmppath" "garage/$S3_BUCKET/$app/$time/$(basename "$filepath").xz" + echo "Uploaded file" +done diff --git a/newinfra/nix/modules/backup/default.nix b/newinfra/nix/modules/backup/default.nix new file mode 100644 index 0000000..32424d4 --- /dev/null +++ b/newinfra/nix/modules/backup/default.nix @@ -0,0 +1,64 @@ +{ config, lib, pkgs, ... }: with lib; +let + jobOptions = { ... }: { + options = { + app = mkOption { + type = types.string; + description = "The app name, used as the directory in the bucket"; + }; + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + }; + file = mkOption { + type = types.string; + }; + #pg_dump = { }; + #mongo_dump = { }; + }; + }; +in +{ + options.services.custom-backup = { + jobs = mkOption { + default = [ ]; + type = types.listOf (types.submodule jobOptions); + description = "Backup jobs to execute"; + }; + }; + + config = + let + cfg = config.services.custom-backup; + backupConfig = { + files = builtins.map (job: { app = job.app; file = job.file; }) + (builtins.filter (job: job.file != null) cfg.jobs); + }; + backupScript = pkgs.writeShellApplication { + name = "backup"; + runtimeInputs = with pkgs; [ jq minio-client getent xz ]; + text = builtins.readFile ./backup.sh; + }; + in + { + age.secrets.backup_s3_secret.file = ../../secrets/backup_s3_secret.age; + + systemd.services.custom-backup = { + startAt = "daily"; + serviceConfig = { + DynamicUser = true; + ExecStart = "${backupScript}/bin/backup"; + Environment = [ + "CONFIG_FILE=${pkgs.writeText "backup-config.json" (builtins.toJSON backupConfig)}" + "S3_BUCKET=backups" + "S3_ENDPOINT=http://localhost:3900" + ]; + EnvironmentFile = (builtins.filter (file: file != null) + (builtins.map (job: job.environmentFile) cfg.jobs)) ++ [ + config.age.secrets.backup_s3_secret.path + ]; + }; + }; + }; +} + diff --git a/newinfra/nix/modules/garage/README.md b/newinfra/nix/modules/garage/README.md index 95f82f8..d9d3b85 100644 --- a/newinfra/nix/modules/garage/README.md +++ b/newinfra/nix/modules/garage/README.md @@ -21,9 +21,15 @@ - key `docker-registry` RW - `loki` - key `loki` RW +- `backups` + - key `backups` RW ## keys - `caddy`: `GK25e33d4ba20d54231e513b80` - `docker-registry`: `GK48011ee5b5ccbaf4233c0e40` - `loki`: `GK84ffae2a0728abff0f96667b` +- `backups`: `GK8cb8454a6f650326562bff2f` + +- `admin`: `GKaead6cf5340e54a4a19d9490` + - RW permissions on ~every bucket diff --git a/newinfra/nix/modules/garage/default.nix b/newinfra/nix/modules/garage/default.nix index 13c616a..864260b 100644 --- a/newinfra/nix/modules/garage/default.nix +++ b/newinfra/nix/modules/garage/default.nix @@ -1,6 +1,10 @@ { config, pkgs, name, ... }: { age.secrets.garage_secrets.file = ../../secrets/garage_secrets.age; + environment.systemPackages = with pkgs; [ + minio-client + ]; + networking.firewall.interfaces.wg0.allowedTCPPorts = [ 3901 # RPC 3903 # admin for metrics diff --git a/newinfra/nix/secrets/backup_s3_secret.age b/newinfra/nix/secrets/backup_s3_secret.age new file mode 100644 index 0000000..003ef35 Binary files /dev/null and b/newinfra/nix/secrets/backup_s3_secret.age differ diff --git a/newinfra/nix/secrets/caddy_s3_key_secret.age b/newinfra/nix/secrets/caddy_s3_key_secret.age index c1fc9f7..086a912 100644 Binary files a/newinfra/nix/secrets/caddy_s3_key_secret.age and b/newinfra/nix/secrets/caddy_s3_key_secret.age differ diff --git a/newinfra/nix/secrets/docker_registry_password.age b/newinfra/nix/secrets/docker_registry_password.age index 26aee99..fa31a33 100644 --- a/newinfra/nix/secrets/docker_registry_password.age +++ b/newinfra/nix/secrets/docker_registry_password.age @@ -1,5 +1,5 @@ age-encryption.org/v1 --> ssh-ed25519 qM6TYg NvguOs7htIflYp6bh6oiiH7Cp2l/0Mf4mcf/4b8ReQg -BngCQfbilctBfNKjE+TkEhE3Bk2pkIlc1UYdAFISP/g ---- 3hA+KfCqIAvwuL+mr4PFW9hVlpsc+t0uwG8I8Uc8JXY -J@zI95/`VfWߤp#0;D دw'rg+n \ No newline at end of file +-> ssh-ed25519 qM6TYg 1HgGuuBWZKvGpR755SyGybRGIq26JR8qb4x4hywwWU8 +6e0gmCgL6CttzzzZ73oUYzpCcvhArAdFJGycwacFaIY +--- tfUAHcZONQZuZIXtumjCh1Crawf+BSl7djHSHC3WvJ8 +Ss.QeM~K!ٟ\Cqy^)SGjaaՙ \ No newline at end of file diff --git a/newinfra/nix/secrets/garage_secrets.age b/newinfra/nix/secrets/garage_secrets.age index 1c7e29b..4aa6674 100644 Binary files a/newinfra/nix/secrets/garage_secrets.age and b/newinfra/nix/secrets/garage_secrets.age differ diff --git a/newinfra/nix/secrets/grafana_admin_password.age b/newinfra/nix/secrets/grafana_admin_password.age index f43a25d..cbba5bf 100644 Binary files a/newinfra/nix/secrets/grafana_admin_password.age and b/newinfra/nix/secrets/grafana_admin_password.age differ diff --git a/newinfra/nix/secrets/hugochat_db_password.age b/newinfra/nix/secrets/hugochat_db_password.age index 4e1e5c8..a7f95a0 100644 Binary files a/newinfra/nix/secrets/hugochat_db_password.age and b/newinfra/nix/secrets/hugochat_db_password.age differ diff --git a/newinfra/nix/secrets/loki_env.age b/newinfra/nix/secrets/loki_env.age index 65ec96e..ac87033 100644 Binary files a/newinfra/nix/secrets/loki_env.age and b/newinfra/nix/secrets/loki_env.age differ diff --git a/newinfra/nix/secrets/minio_env_file.age b/newinfra/nix/secrets/minio_env_file.age index 9ccf0e2..52e7b35 100644 Binary files a/newinfra/nix/secrets/minio_env_file.age and b/newinfra/nix/secrets/minio_env_file.age differ diff --git a/newinfra/nix/secrets/registry_htpasswd.age b/newinfra/nix/secrets/registry_htpasswd.age index 2bf8a3f..65f648b 100644 Binary files a/newinfra/nix/secrets/registry_htpasswd.age and b/newinfra/nix/secrets/registry_htpasswd.age differ diff --git a/newinfra/nix/secrets/registry_s3_key_secret.age b/newinfra/nix/secrets/registry_s3_key_secret.age index 57df63e..00f42ca 100644 Binary files a/newinfra/nix/secrets/registry_s3_key_secret.age and b/newinfra/nix/secrets/registry_s3_key_secret.age differ diff --git a/newinfra/nix/secrets/s3_mc_admin_client.age b/newinfra/nix/secrets/s3_mc_admin_client.age new file mode 100644 index 0000000..42ef966 Binary files /dev/null and b/newinfra/nix/secrets/s3_mc_admin_client.age differ diff --git a/newinfra/nix/secrets/secrets.nix b/newinfra/nix/secrets/secrets.nix index edf17a0..dfb89b6 100644 --- a/newinfra/nix/secrets/secrets.nix +++ b/newinfra/nix/secrets/secrets.nix @@ -18,6 +18,8 @@ in "registry_s3_key_secret.age".publicKeys = [ vps1 ]; "grafana_admin_password.age".publicKeys = [ vps3 ]; "loki_env.age".publicKeys = [ vps3 ]; + "backup_s3_secret.age".publicKeys = [ vps1 vps3 vps4 vps5 ]; + "s3_mc_admin_client.age".publicKeys = [ vps1 vps3 vps4 vps5 ]; "wg_private_dns1.age".publicKeys = [ dns1 ]; "wg_private_dns2.age".publicKeys = [ dns2 ]; "wg_private_vps1.age".publicKeys = [ vps1 ]; diff --git a/newinfra/nix/secrets/wg_private_dns1.age b/newinfra/nix/secrets/wg_private_dns1.age index 44c6e6e..3dc6f81 100644 Binary files a/newinfra/nix/secrets/wg_private_dns1.age and b/newinfra/nix/secrets/wg_private_dns1.age differ diff --git a/newinfra/nix/secrets/wg_private_dns2.age b/newinfra/nix/secrets/wg_private_dns2.age index 86fce00..5b3d149 100644 --- a/newinfra/nix/secrets/wg_private_dns2.age +++ b/newinfra/nix/secrets/wg_private_dns2.age @@ -1,5 +1,5 @@ age-encryption.org/v1 --> ssh-ed25519 5bWSnQ gkB+3cKWYoBYV0eDyBD5g5UWsSs94DaQSka42g9EsA8 -B8Hm9IrecZNv75DWfPJPPGJSP8AkBcOgy070foI8gPQ ---- d6F58mcEIAHmIpaAr1sgMaQUQAqa6Tr7DHq5hFgetOc -Lmb&Ɂd1ץ^{;tH堨5MX1c[ b[ߠ 篈jz( \ No newline at end of file +-> ssh-ed25519 5bWSnQ yYpbqupe6d0ZiH4CxnkHx6clUSI6VOAwiFicoeghIi8 +Q1rxBbAhYeZfi5uSNW7/kE/sn15ZpDSxC/P8/SuekWQ +--- CsY6lrPSTBryg9t7U1FfnoAYoz0pDRhRpkTy+bsJrZc +E' f07TBc㯦-߬&Kpd~Gm8: )Lpy x˲ < \ No newline at end of file diff --git a/newinfra/nix/secrets/wg_private_vps1.age b/newinfra/nix/secrets/wg_private_vps1.age index 368b053..44225da 100644 Binary files a/newinfra/nix/secrets/wg_private_vps1.age and b/newinfra/nix/secrets/wg_private_vps1.age differ diff --git a/newinfra/nix/secrets/wg_private_vps3.age b/newinfra/nix/secrets/wg_private_vps3.age index ab1a7d7..d7388a4 100644 Binary files a/newinfra/nix/secrets/wg_private_vps3.age and b/newinfra/nix/secrets/wg_private_vps3.age differ diff --git a/newinfra/nix/secrets/wg_private_vps4.age b/newinfra/nix/secrets/wg_private_vps4.age index b8f355f..fda3f23 100644 Binary files a/newinfra/nix/secrets/wg_private_vps4.age and b/newinfra/nix/secrets/wg_private_vps4.age differ diff --git a/newinfra/nix/secrets/wg_private_vps5.age b/newinfra/nix/secrets/wg_private_vps5.age index 6a84e45..6faebf3 100644 Binary files a/newinfra/nix/secrets/wg_private_vps5.age and b/newinfra/nix/secrets/wg_private_vps5.age differ diff --git a/newinfra/nix/secrets/widetom_bot_token.age b/newinfra/nix/secrets/widetom_bot_token.age index 0638e3b..0cf4c28 100644 Binary files a/newinfra/nix/secrets/widetom_bot_token.age and b/newinfra/nix/secrets/widetom_bot_token.age differ diff --git a/newinfra/nix/secrets/widetom_config_toml.age b/newinfra/nix/secrets/widetom_config_toml.age index dc3698c..a4a749e 100644 Binary files a/newinfra/nix/secrets/widetom_config_toml.age and b/newinfra/nix/secrets/widetom_config_toml.age differ diff --git a/newinfra/secrets-git-crypt/backup_s3_secret b/newinfra/secrets-git-crypt/backup_s3_secret new file mode 100644 index 0000000..1a810a5 Binary files /dev/null and b/newinfra/secrets-git-crypt/backup_s3_secret differ diff --git a/newinfra/secrets-git-crypt/s3_mc_admin_client b/newinfra/secrets-git-crypt/s3_mc_admin_client new file mode 100644 index 0000000..637a251 Binary files /dev/null and b/newinfra/secrets-git-crypt/s3_mc_admin_client differ