high availability caddy

this was a fucking nightmare.
setting up the s3 storage plugin was painful, since caddy plugins are not nicely supported by the nix build yet.
rip.
oh well, i got it working.

and it WORKS

newinfra/nix/modules/ingress/base.Caddyfile

@@ -0,0 +1,17 @@
+{
+	email noratrieb@proton.me
+	auto_https disable_redirects
+
+	storage s3 {
+		host "localhost:3900"
+		bucket "caddy-store"
+		# access_id ENV S3_ACCESS_ID
+		# secret_key ENV S3_SECRET_KEY
+
+		insecure true
+	}
+}
+
+http:// {
+	respond "This is an HTTPS-only server, silly you. Go to https:// instead." 418
+}

newinfra/nix/modules/ingress/caddy-build.nix

@@ -0,0 +1,116 @@
+# Copied from https://github.com/NixOS/nixpkgs/pull/259275 and updated.
+
+{ lib
+, buildGoModule
+, fetchFromGitHub
+, gnused
+, nixosTests
+, caddy
+, stdenv
+, testers
+, installShellFiles
+, externalPlugins ? [ ]
+, vendorHash ? "sha256-1Api8bBZJ1/oYk4ZGIiwWCSraLzK9L+hsKXkFtk6iVM="
+}:
+
+let
+  attrsToModules = attrs:
+    builtins.map ({ name, repo, version }: "${repo}") attrs;
+  attrsToSources = attrs:
+    builtins.map ({ name, repo, version }: "${repo}@${version}") attrs;
+in
+buildGoModule rec {
+  pname = "caddy";
+  version = "2.8.4";
+
+  dist = fetchFromGitHub {
+    owner = "caddyserver";
+    repo = "dist";
+    rev = "v${version}";
+    hash = "sha256-O4s7PhSUTXoNEIi+zYASx8AgClMC5rs7se863G6w+l0=";
+  };
+
+  src = fetchFromGitHub {
+    owner = "caddyserver";
+    repo = "caddy";
+    rev = "v${version}";
+    hash = "sha256-CBfyqtWp3gYsYwaIxbfXO3AYaBiM7LutLC7uZgYXfkQ=";
+  };
+
+  inherit vendorHash;
+
+  subPackages = [ "cmd/caddy" ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X github.com/caddyserver/caddy/v2.CustomVersion=${version}"
+  ];
+
+  # matches upstream since v2.8.0
+  tags = [ "nobadger" ];
+
+  nativeBuildInputs = [ gnused installShellFiles ];
+
+  modBuildPhase = ''
+    for module in ${builtins.toString (attrsToModules externalPlugins)}; do
+      sed -i "/standard/a _ \"$module\"" ./cmd/caddy/main.go
+    done
+    for plugin in ${builtins.toString (attrsToSources externalPlugins)}; do
+      go get $plugin
+    done
+
+    go generate
+    go mod vendor
+  '';
+
+  modInstallPhase = ''
+    mv -t vendor go.mod go.sum
+    cp -r --reflink=auto vendor "$out"
+  '';
+
+  preBuild = ''
+    chmod -R u+w vendor
+    [ -f vendor/go.mod ] && mv -t . vendor/go.{mod,sum}
+    go generate
+
+    for module in ${builtins.toString (attrsToModules externalPlugins)}; do
+      sed -i "/standard/a _ \"$module\"" ./cmd/caddy/main.go
+    done
+  '';
+
+  postInstall = ''
+    install -Dm644 ${dist}/init/caddy.service ${dist}/init/caddy-api.service -t $out/lib/systemd/system
+
+    substituteInPlace $out/lib/systemd/system/caddy.service \
+      --replace-fail "/usr/bin/caddy" "$out/bin/caddy"
+    substituteInPlace $out/lib/systemd/system/caddy-api.service \
+      --replace-fail "/usr/bin/caddy" "$out/bin/caddy"
+  '' + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
+    # Generating man pages and completions fail on cross-compilation
+    # https://github.com/NixOS/nixpkgs/issues/308283
+
+    $out/bin/caddy manpage --directory manpages
+    installManPage manpages/*
+
+    installShellCompletion --cmd caddy \
+      --bash <($out/bin/caddy completion bash) \
+      --fish <($out/bin/caddy completion fish) \
+      --zsh <($out/bin/caddy completion zsh)
+  '';
+  passthru.tests = {
+    inherit (nixosTests) caddy;
+    version = testers.testVersion {
+      command = "${caddy}/bin/caddy version";
+      package = caddy;
+    };
+  };
+
+  meta = with lib; {
+    homepage = "https://caddyserver.com";
+    description = "Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS";
+    license = licenses.asl20;
+    mainProgram = "caddy";
+    maintainers = with maintainers; [ Br1ght0ne emilylange techknowlogick ];
+  };
+}

newinfra/nix/modules/ingress/default.nix

@@ -1,7 +1,20 @@
-{ pkgs, nixpkgs-unstable, config, lib, name, website, slides, blog, ... }:
+{ pkgs, config, lib, name, website, slides, blog, ... }:
 
-let caddy = nixpkgs-unstable.caddy; in
+let
+  caddy = pkgs.callPackage ./caddy-build.nix {
+    externalPlugins = [
+      {
+        name = "certmagic-s3";
+        repo = "github.com/noratrieb-mirrors/certmagic-s3";
+        version = "e48519f95173e982767cbb881d49335b6a00a599";
+      }
+    ];
+    vendorHash = "sha256-KP9bYitM/Pocw4DxOXPVBigWh4IykNf8yKJiBlTFZmI=";
+  };
+in
 {
+  environment.systemPackages = [ caddy ];
+
   networking.firewall = {
     allowedTCPPorts = [
       80 # HTTP
@@ -12,22 +25,17 @@ let caddy = nixpkgs-unstable.caddy; in
     ];
   };
 
+  age.secrets.caddy_s3_key_secret.file = ../../secrets/caddy_s3_key_secret.age;
+
+  systemd.services.caddy.serviceConfig.EnvironmentFile = config.age.secrets.caddy_s3_key_secret.path;
   services.caddy = {
     enable = true;
     package = caddy;
     configFile = pkgs.writeTextFile {
       name = "Caddyfile";
       text = (
+        builtins.readFile ./base.Caddyfile +
         ''
-          {
-              email nilstrieb@proton.me
-              auto_https disable_redirects
-          }
-
-          http:// {
-            respond "This is an HTTPS-only server, silly you. Go to https:// instead." 418
-          }
-
           ${config.networking.hostName}.infra.noratrieb.dev {
               encode zstd gzip
               header -Last-Modified
@@ -37,29 +45,33 @@ let caddy = nixpkgs-unstable.caddy; in
                 inherit pkgs lib;
               }}
               file_server {
-                etag_file_extensions .sha256
-                precompressed zstd gzip br
+                  etag_file_extensions .sha256
+                  precompressed zstd gzip br
               }
           }
 
           ${
-            if name == "vps1" then
-            builtins.readFile ./Caddyfile + ''
-              noratrieb.dev {
-                  encode zstd gzip
-                  header -Last-Modified
-                  root * ${import ./caddy-static-prepare {
-                    name = "website";
-                    src = website { inherit pkgs slides blog; };
-                    inherit pkgs lib;
-                  }}
-                  file_server {
+            if name == "vps1" || name == "vps3" || name == "vps4" then ''
+            noratrieb.dev {
+                encode zstd gzip
+                header -Last-Modified
+                root * ${import ./caddy-static-prepare {
+                  name = "website";
+                  src = website { inherit pkgs slides blog; };
+                  inherit pkgs lib;
+                }}
+                file_server {
                     etag_file_extensions .sha256
                     precompressed zstd gzip br
-                  }
-              }
+                }
+            }
             '' else ""
           }
+
+          ${
+            if name == "vps1" then
+            builtins.readFile ./Caddyfile else ""
+          }
         ''
       );
       checkPhase = ''