compress and etag

newinfra/nix/hive.nix

@@ -8,6 +8,8 @@
     nixpkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/a1cc729dcbc31d9b0d11d86dc7436163548a9665.tar.gz"); # nixos-24.05 2024-07-26
 
     specialArgs = {
+      nixpkgs-unstable = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/d04953086551086b44b6f3c6b7eeb26294f207da.tar.gz") { }; # nixos-unstable 2024-08-05
+
       website = import (fetchTarball "https://github.com/Noratrieb/website/archive/dc4352b9f01c4780539bdd50249d8f552e541fd9.tar.gz");
       blog = fetchTarball "https://github.com/Noratrieb/nilstrieb.github.io/archive/8162ce0cff29f940507032be6b0692290d73594c.tar.gz";
       slides = fetchTarball "https://github.com/Noratrieb/slides/archive/0401f35c22b124b69447655f0c537badae9e223c.tar.gz";

newinfra/nix/modules/ingress/caddy-static-prepare/default.nix

@@ -0,0 +1,13 @@
+{ pkgs, lib, name, src ? null, ... }: pkgs.stdenv.mkDerivation {
+  inherit name src;
+
+  buildInputs = with pkgs; [ python311 python311Packages.zstandard python311Packages.brotli ];
+
+  buildPhase = ''
+    mkdir -p $out
+    cp -r $src/* $out/
+    chmod -R +w $out
+    ${lib.getExe pkgs.python311} ${./prepare.py} $out
+    chmod -R -w $out
+  '';
+}

newinfra/nix/modules/ingress/caddy-static-prepare/prepare.py

@@ -0,0 +1,60 @@
+import os
+import sys
+import gzip
+import brotli
+import zstandard
+import hashlib
+
+
+def usage():
+    print("usage: prepare.py [SRC]")
+
+
+def write_etag(path, content):
+    shasum = hashlib.sha256(content)
+    etag_path = path+".sha256"
+    with open(etag_path, "w") as f:
+        print(f"Writing ETag {etag_path}")
+        f.write(f'"{shasum.hexdigest()}"')
+
+
+def main():
+    if len(sys.argv) < 2:
+        usage()
+        exit(1)
+
+    src_dir = sys.argv[1]
+
+    for root, dirs, files in os.walk(src_dir):
+        for file in files:
+            path = os.path.join(root, file)
+
+            # Ignore etags
+            if path.endswith(".sha256") or path.endswith(".b3sum"):
+                continue
+
+            # Ignore already compressed files        
+            if path.endswith(".gz") or path.endswith(".zst") or path.endswith(".br"):
+                continue
+
+            with open(path, "rb") as f:
+                content = f.read()
+
+            compressions = [
+                (".gz", gzip),
+                (".zst", zstandard),
+                (".br", brotli),
+            ]
+
+            for ext, alg in compressions:
+                new_path = path+ext
+                with open(new_path, "wb") as out:
+                    print(f"Writing {new_path}")
+                    compressed = alg.compress(content)
+                    out.write(compressed)
+                    write_etag(new_path, compressed)
+
+            write_etag(path, content)
+
+if __name__ == "__main__":
+    main()

newinfra/nix/modules/ingress/default.nix

@@ -1,4 +1,7 @@
-{ pkgs, config, lib, name, website, slides, blog, ... }: {
+{ pkgs, nixpkgs-unstable, config, lib, name, website, slides, blog, ... }:
+
+let caddy = nixpkgs-unstable.caddy; in
+{
   networking.firewall.allowedTCPPorts = [
     80
     443
@@ -6,6 +9,7 @@
 
   services.caddy = {
     enable = true;
+    package = caddy;
     configFile = pkgs.writeTextFile {
       name = "Caddyfile";
       text = (
@@ -22,8 +26,15 @@
           ${config.networking.hostName}.infra.noratrieb.dev {
               encode zstd gzip
               header -Last-Modified
-              root * ${./debugging-page}
+              root * ${import ./caddy-static-prepare {
-              file_server
+                name = "debugging-page";
+                src = ./debugging-page;
+                inherit pkgs lib;
+              }}
+              file_server {
+                etag_file_extensions .sha256
+                precompressed zstd gzip
+              }
           }
 
           ${
@@ -32,15 +43,23 @@
               noratrieb.dev {
                   encode zstd gzip
                   header -Last-Modified
-                  root * ${website {inherit pkgs slides blog;}}
+                  root * ${import ./caddy-static-prepare {
-                  file_server
+                    name = "website";
+                    src = website { inherit pkgs slides blog; };
+                    inherit pkgs lib;
+                  }}
+                  file_server {
+                    etag_file_extensions .sha256
+                    precompressed zstd gzip
+                  }
               }
             '' else ""
           }
         ''
       );
       checkPhase = ''
-        ${lib.getExe pkgs.caddy} validate --adapter=caddyfile --config=$out
+        ${lib.getExe caddy} --version
+        ${lib.getExe caddy} validate --adapter=caddyfile --config=$out
       '';
     };
   };

shell.nix

@@ -12,6 +12,8 @@
     git-crypt
     opentofu
     wireguard-tools
+    python311Packages.zstandard
+    python311Packages.brotli
     (import (builtins.fetchTarball "https://github.com/ryantm/agenix/archive/de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6.tar.gz") { }).agenix
   ];
 }