high availability caddy

this was a fucking nightmare.
setting up the s3 storage plugin was painful, since caddy plugins are not nicely supported by the nix build yet.
rip.
oh well, i got it working.

and it WORKS

newinfra/nix/hive.nix

@@ -29,7 +29,7 @@
           wg = {
             privateIP = "10.0.0.1";
             publicKey = "5tg3w/TiCuCeKIBJCd6lHUeNjGEA76abT1OXnhNVyFQ=";
-            peers = [ "vps3" "vps4" ];
+            peers = [ "vps3" "vps4" "vps5" ];
           };
         };
         vps3 = {
@@ -142,6 +142,7 @@
       ./modules/contabo
       ./modules/wg-mesh
       ./modules/ingress
+      ./modules/garage
       ./modules/podman
 
       # apps

newinfra/nix/modules/dns/noratrieb.dev.nix

@@ -12,10 +12,15 @@ let
       vps2 = {
         A = [ "184.174.32.252" ];
       };
+
+      combine = hosts: {
+        A = lib.lists.flatten (map (host: if builtins.hasAttr "A" host then host.A else [ ]) hosts);
+        AAAA = lib.lists.flatten (map (host: if builtins.hasAttr "AAAA" host then host.AAAA else [ ]) hosts);
+      };
     in
     with hostsToDns;
-    # vps1 contains root noratrieb.dev
-    vps1 // {
+    # vps{1,3,4} contains root noratrieb.dev
+    combine [ vps1 vps3 vps4 ] // {
       SOA = {
         nameServer = "ns1.noratrieb.dev.";
         adminEmail = "void@noratrieb.dev";

newinfra/nix/modules/garage/README.md

@@ -8,6 +8,15 @@
 
 | name | disk space | identifier | zone  |
 | ---- | ---------- | ---------- | ----- |
+| vps3 | 70GB       | cabe      | co-du |
 | vps3 | 100GB      | 020bd      | co-ka |
 | vps4 | 30GB       | 41e40      | he-nu |
 | vps5 | 100GB      | 848d8      | co-du |
+
+## buckets
+
+- `caddy-store`: Store for Caddy webservers
+
+## keys
+
+- `caddy`: `GK25e33d4ba20d54231e513b80`

newinfra/nix/modules/ingress/base.Caddyfile

@@ -0,0 +1,17 @@
+{
+	email noratrieb@proton.me
+	auto_https disable_redirects
+
+	storage s3 {
+		host "localhost:3900"
+		bucket "caddy-store"
+		# access_id ENV S3_ACCESS_ID
+		# secret_key ENV S3_SECRET_KEY
+
+		insecure true
+	}
+}
+
+http:// {
+	respond "This is an HTTPS-only server, silly you. Go to https:// instead." 418
+}

newinfra/nix/modules/ingress/caddy-build.nix

@@ -0,0 +1,116 @@
+# Copied from https://github.com/NixOS/nixpkgs/pull/259275 and updated.
+
+{ lib
+, buildGoModule
+, fetchFromGitHub
+, gnused
+, nixosTests
+, caddy
+, stdenv
+, testers
+, installShellFiles
+, externalPlugins ? [ ]
+, vendorHash ? "sha256-1Api8bBZJ1/oYk4ZGIiwWCSraLzK9L+hsKXkFtk6iVM="
+}:
+
+let
+  attrsToModules = attrs:
+    builtins.map ({ name, repo, version }: "${repo}") attrs;
+  attrsToSources = attrs:
+    builtins.map ({ name, repo, version }: "${repo}@${version}") attrs;
+in
+buildGoModule rec {
+  pname = "caddy";
+  version = "2.8.4";
+
+  dist = fetchFromGitHub {
+    owner = "caddyserver";
+    repo = "dist";
+    rev = "v${version}";
+    hash = "sha256-O4s7PhSUTXoNEIi+zYASx8AgClMC5rs7se863G6w+l0=";
+  };
+
+  src = fetchFromGitHub {
+    owner = "caddyserver";
+    repo = "caddy";
+    rev = "v${version}";
+    hash = "sha256-CBfyqtWp3gYsYwaIxbfXO3AYaBiM7LutLC7uZgYXfkQ=";
+  };
+
+  inherit vendorHash;
+
+  subPackages = [ "cmd/caddy" ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X github.com/caddyserver/caddy/v2.CustomVersion=${version}"
+  ];
+
+  # matches upstream since v2.8.0
+  tags = [ "nobadger" ];
+
+  nativeBuildInputs = [ gnused installShellFiles ];
+
+  modBuildPhase = ''
+    for module in ${builtins.toString (attrsToModules externalPlugins)}; do
+      sed -i "/standard/a _ \"$module\"" ./cmd/caddy/main.go
+    done
+    for plugin in ${builtins.toString (attrsToSources externalPlugins)}; do
+      go get $plugin
+    done
+
+    go generate
+    go mod vendor
+  '';
+
+  modInstallPhase = ''
+    mv -t vendor go.mod go.sum
+    cp -r --reflink=auto vendor "$out"
+  '';
+
+  preBuild = ''
+    chmod -R u+w vendor
+    [ -f vendor/go.mod ] && mv -t . vendor/go.{mod,sum}
+    go generate
+
+    for module in ${builtins.toString (attrsToModules externalPlugins)}; do
+      sed -i "/standard/a _ \"$module\"" ./cmd/caddy/main.go
+    done
+  '';
+
+  postInstall = ''
+    install -Dm644 ${dist}/init/caddy.service ${dist}/init/caddy-api.service -t $out/lib/systemd/system
+
+    substituteInPlace $out/lib/systemd/system/caddy.service \
+      --replace-fail "/usr/bin/caddy" "$out/bin/caddy"
+    substituteInPlace $out/lib/systemd/system/caddy-api.service \
+      --replace-fail "/usr/bin/caddy" "$out/bin/caddy"
+  '' + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
+    # Generating man pages and completions fail on cross-compilation
+    # https://github.com/NixOS/nixpkgs/issues/308283
+
+    $out/bin/caddy manpage --directory manpages
+    installManPage manpages/*
+
+    installShellCompletion --cmd caddy \
+      --bash <($out/bin/caddy completion bash) \
+      --fish <($out/bin/caddy completion fish) \
+      --zsh <($out/bin/caddy completion zsh)
+  '';
+  passthru.tests = {
+    inherit (nixosTests) caddy;
+    version = testers.testVersion {
+      command = "${caddy}/bin/caddy version";
+      package = caddy;
+    };
+  };
+
+  meta = with lib; {
+    homepage = "https://caddyserver.com";
+    description = "Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS";
+    license = licenses.asl20;
+    mainProgram = "caddy";
+    maintainers = with maintainers; [ Br1ght0ne emilylange techknowlogick ];
+  };
+}

newinfra/nix/modules/ingress/default.nix

@@ -1,7 +1,20 @@
-{ pkgs, nixpkgs-unstable, config, lib, name, website, slides, blog, ... }:
+{ pkgs, config, lib, name, website, slides, blog, ... }:
 
-let caddy = nixpkgs-unstable.caddy; in
+let
+  caddy = pkgs.callPackage ./caddy-build.nix {
+    externalPlugins = [
+      {
+        name = "certmagic-s3";
+        repo = "github.com/noratrieb-mirrors/certmagic-s3";
+        version = "e48519f95173e982767cbb881d49335b6a00a599";
+      }
+    ];
+    vendorHash = "sha256-KP9bYitM/Pocw4DxOXPVBigWh4IykNf8yKJiBlTFZmI=";
+  };
+in
 {
+  environment.systemPackages = [ caddy ];
+
   networking.firewall = {
     allowedTCPPorts = [
       80 # HTTP
@@ -12,22 +25,17 @@ let caddy = nixpkgs-unstable.caddy; in
     ];
   };
 
+  age.secrets.caddy_s3_key_secret.file = ../../secrets/caddy_s3_key_secret.age;
+
+  systemd.services.caddy.serviceConfig.EnvironmentFile = config.age.secrets.caddy_s3_key_secret.path;
   services.caddy = {
     enable = true;
     package = caddy;
     configFile = pkgs.writeTextFile {
       name = "Caddyfile";
       text = (
+        builtins.readFile ./base.Caddyfile +
         ''
-          {
-              email nilstrieb@proton.me
-              auto_https disable_redirects
-          }
-
-          http:// {
-            respond "This is an HTTPS-only server, silly you. Go to https:// instead." 418
-          }
-
           ${config.networking.hostName}.infra.noratrieb.dev {
               encode zstd gzip
               header -Last-Modified
@@ -37,29 +45,33 @@ let caddy = nixpkgs-unstable.caddy; in
                 inherit pkgs lib;
               }}
               file_server {
-                etag_file_extensions .sha256
-                precompressed zstd gzip br
+                  etag_file_extensions .sha256
+                  precompressed zstd gzip br
               }
           }
 
           ${
-            if name == "vps1" then
-            builtins.readFile ./Caddyfile + ''
-              noratrieb.dev {
-                  encode zstd gzip
-                  header -Last-Modified
-                  root * ${import ./caddy-static-prepare {
-                    name = "website";
-                    src = website { inherit pkgs slides blog; };
-                    inherit pkgs lib;
-                  }}
-                  file_server {
+            if name == "vps1" || name == "vps3" || name == "vps4" then ''
+            noratrieb.dev {
+                encode zstd gzip
+                header -Last-Modified
+                root * ${import ./caddy-static-prepare {
+                  name = "website";
+                  src = website { inherit pkgs slides blog; };
+                  inherit pkgs lib;
+                }}
+                file_server {
                     etag_file_extensions .sha256
                     precompressed zstd gzip br
-                  }
-              }
+                }
+            }
             '' else ""
           }
+
+          ${
+            if name == "vps1" then
+            builtins.readFile ./Caddyfile else ""
+          }
         ''
       );
       checkPhase = ''

newinfra/nix/secrets/minio_env_file.age

@@ -1,8 +1,7 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg 5dnvDdUqtCoJo4RYNTpIw4ckH0um6E50ZcOs08ZZLig
-6hwjJMNit300VZhBeID7kdV8NJBbZg/3bq8vHO8R9i8
--> ssh-ed25519 XzACZQ g+vhR2tRcQHZ/8Ud0oDxA0a+d1+dyNMVi21GbNRXrx0
-6kYx9RQYfuqvEfm3qpSp9oKXktJccqQoYEvI+SEhEqg
---- 1Z17ztzvaWForsjjcF72J0/bdSmZCqEuScxEQ22V6xo
-GÑ}ÜÅ7ô[EÛÉSÝŒ¥aO®O²N¦òlTGN¶_Ï
-!ãû¼H@•ÆѬƒM©¡6š^$Û²o@™Ú…u›M–Y;(8‚ÿ++Ï<>å­ºMƒ€l÷—Q¢aþmµG•‚|<7C>ÔYÖÍ
+-> ssh-ed25519 qM6TYg Jtt9cLPGha9Qs5gEuKSwU3E1bNMhrjlHtnj/I3dKqW0
+0iDfPorED8lq0Rc5LVDNWID7l2F+AnmeEr7Yik/OC44
+-> ssh-ed25519 XzACZQ Q9WpNGn/k35J0/LzGAlcf1ktN2/VG3nZdpfMbJXAnWw
+bl2Pasbxmb6LNbWiZrEVBQ99gYYC5Md6kdvIt4VAf7k
+--- +B0f8ilJGkB7Qj+BdzeKfW6HRl9yzMd+iT4sOAmJI5Y
+<EFBFBD>\ÒÖñÈ<C3B1>'ËZtbJ7úAL££²â›£&Á•‹ØC+LM¹nhІŠ]Rº·;	Ô†JHK»O¯7å B’»¤“¡ß\(ÓQmÎ’åËU>r³4"Šà¥èXh¹ó…Ð<C390>ãIñd§cE6G_oN©

newinfra/nix/secrets/secrets.nix

@@ -11,6 +11,7 @@ in
   "hugochat_db_password.age".publicKeys = [ vps1 ];
   "minio_env_file.age".publicKeys = [ vps1 vps3 ];
   "garage_secrets.age".publicKeys = [ vps1 vps3 vps4 vps5 ];
+  "caddy_s3_key_secret.age".publicKeys = [ vps1 vps3 vps4 vps5 ];
   "wg_private_vps1.age".publicKeys = [ vps1 ];
   "wg_private_vps3.age".publicKeys = [ vps3 ];
   "wg_private_vps4.age".publicKeys = [ vps4 ];

newinfra/nix/secrets/wg_private_vps1.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 qM6TYg Q5X+l2POBANoYyo8HNMy89MLtpodzzN9prnQY71mSTE
-X3MJesW3kfHCfCyvaWm22mOI8vSgP7JWlLugCXtiy+U
---- ZH3UZFDfQwZ+DIF3yFADfBKEv2K6k9DTCh5wLVnyaTs
-‡‹ÙiÔ,ÿ¯±ª<C2B1>‘1æF¸Äßfë	[_+­õ[
!¢>)ep'ÆYøAïÂWgã ªÇ<­ê^¢Ê=ô(Bˆ®ú)~e±G”
+-> ssh-ed25519 qM6TYg xCaglRQkcl1+kGIVjPEn+NlnrBUvcWLSH7MMPLXK9kU
+78t/Z81+NaXQMW30EQH8WMhed6Lm77+atPTkBQbDMd0
+--- AsnraeejCWHj1iRI/1btRXI6tqdnBW4S+twfx35eNEI
+³6Â1ŒŽKqH\vé<17>lW¢IX{éåK;€#ÞS—â&ãg^.ÍÊKQþ8”a7V˜œ:<3A><>e)9åïÐŒ!ëÞèO

newinfra/nix/secrets/wg_private_vps3.age

@@ -1,5 +1,5 @@
 age-encryption.org/v1
--> ssh-ed25519 XzACZQ nsIkJQw/lrrXChkpFc87upQ4pbGefolI36wqMOWZGAE
-t49QoSdb2azGQlDBX5AyWMxCOt+ETpT7erp4WU5p2rQ
---- 4UbCHfpAfwiuRYsiN3HgdhbSLFBG05DxGCw55XT1IGg
-YÈ	ìŸÔÆŽ ÆŒ­¿½Æòþ˜€2ô×ÖôRsÝ	À‰ìQÈ4d…
IÑ.KpPFthù©ô±£–Á“a„
½É<C2BD>ÃõRX›
+-> ssh-ed25519 XzACZQ PAqPA1RpuXwjKCsn838qwsuRmuh8ES7BPiyCIFdhMmA
+QIAC+dfBMSZwzHwcQpO1IyDPKwTvr/iG35PkrFOyzwE
+--- zNejM9ypNWH1Bg1J1V4UCqMIyVP+gIV/mmgBaCfFCKk
+y<EFBFBD><1F><>2y<>v0<>W<EFBFBD><19>}q<><71>Ymh<6D><68><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Z<EFBFBD>{B|<7C>t7,@<40>6B_<42>V8<56>0i<30><69>a<EFBFBD>z9<11><><EFBFBD>@<40><><EFBFBD>j<EFBFBD><0F>)<29><>

newinfra/nix/secrets/wg_private_vps4.age

@@ -1,5 +1,6 @@
 age-encryption.org/v1
--> ssh-ed25519 51bcvA 9dYzUZSs/ilKHHRiuMgT6GEbtyBwWHAl8ycBcsvTQz0
-iq0ozCU1p1sekOH4qbxKxWezY2pyVM6LjhUuNpmTQx0
---- wjCRFJISrIrpgosh7ZBNM1qR78BPmhVBBwFpaQc10oA
-a~ÊueŠ<0E>?'iIl
ŸŸËC"âwÀ:Í\áR)	(.ŠÕ¥%*>†âp™Éýó’õ"Žu­ †Ÿy4’s<E28099>Ññ>”2¦¿—
+-> ssh-ed25519 51bcvA mJYJJnaKusYBpSL5qAokXISlrXkBZ0QPKZVPkiyKSnk
+IAsX5+UPxhap7ehB9za8Q9aEfeA0Ypd4Tw7XiU4f2eM
+--- VBlmFpr+g83UfZ4rftOkNzKL/ZxSxAi7/tBl4TMaln4
+mäľ’A†W˙Ďýť§ĺŁc€NWň·Ý-ü
—ÚŘäF6Ȇy…T=~<7E>ť–·k‚ަg%€˛Uš	;D­Ý˛Ëiľ‰
ŻĐ&[jÂ+®_
+Ç