From a0efca50e0c31b2bf43ba122c0d994bc67b3b498 Mon Sep 17 00:00:00 2001
From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com>
Date: Sun, 2 Jun 2024 20:48:17 +0200
Subject: [PATCH] UI test error annotations!

---
 src/context.ts                                |  82 +--
 src/error.test.ts                             |   2 +-
 src/error.ts                                  | 105 +++-
 src/index.ts                                  |   3 +-
 src/lexer.test.ts                             |  11 +-
 src/lexer.ts                                  |   5 +-
 src/loader.ts                                 |   2 +-
 src/options.ts                                |  82 +++
 src/typeck/infer.test.ts                      |   9 +-
 ui-harness/Cargo.lock                         | 466 ++----------------
 ui-harness/Cargo.toml                         |   5 +-
 ui-harness/src/bin/nilc-wrapper.rs            |  97 +++-
 ui-harness/src/main.rs                        |  29 +-
 ui-tests/asm/instr_not_string.nil             |   1 +
 ui-tests/asm/instr_not_string.stderr          |   2 +-
 ui-tests/asm/invalid_instr.nil                |   1 +
 ui-tests/asm/invalid_instr.stderr             |   2 +-
 ui-tests/asm/missing_locals.nil               |   1 +
 ui-tests/asm/missing_locals.stderr            |   2 +-
 ui-tests/asm/not_toplevel.nil                 |   2 +
 ui-tests/asm/not_toplevel.stderr              |   4 +-
 ui-tests/asm/wrong_imm.nil                    |   2 +
 ui-tests/asm/wrong_imm.stderr                 |   6 +-
 ui-tests/basic_recovery.nil                   |   5 +
 ui-tests/basic_recovery.stderr                |  16 +-
 ui-tests/functions/body_mismatched_parens.nil |   1 -
 .../functions/body_mismatched_parens.stderr   |   8 -
 ui-tests/functions/missing_args.nil           |   1 +
 ui-tests/functions/missing_args.stderr        |   2 +-
 ui-tests/lint/unused_vars.stderr              |   6 +-
 ui-tests/modules/nil_submodules.nil           |   1 +
 ui-tests/modules/nil_submodules.stderr        |   2 +-
 ui-tests/resolve/module_not_found.nil         |   3 +-
 ui-tests/resolve/module_not_found.stderr      |  12 +-
 .../type/generics/generics_on_primitive.nil   |   3 +-
 .../generics/generics_on_primitive.stderr     |  10 +-
 .../generics/generics_structs_in_args.stderr  |   6 +-
 ui-tests/type/generics/structs.stderr         |   6 +-
 ui-tests/type/generics/wrong_amount.nil       |  30 +-
 ui-tests/type/generics/wrong_amount.stderr    |  84 +---
 ui-tests/type/type_alias.stderr               |   2 +-
 ui-tests/type/type_assignments.stderr         |  12 +-
 42 files changed, 437 insertions(+), 694 deletions(-)
 create mode 100644 src/options.ts
 delete mode 100644 ui-tests/functions/body_mismatched_parens.nil
 delete mode 100644 ui-tests/functions/body_mismatched_parens.stderr

diff --git a/src/context.ts b/src/context.ts
index f25eb9f..23c7e3d 100644
--- a/src/context.ts
+++ b/src/context.ts
@@ -1,9 +1,7 @@
 import { Pkg, DepPkg, Final, Item, ItemId, Phase } from "./ast";
 import { ErrorHandler, Span } from "./error";
+import { Options } from "./options";
 import { Ids, unwrap } from "./utils";
-import fs from "fs";
-import path from "path";
-
 export type PkgLoader = (
   gcx: GlobalContext,
   name: string,
@@ -27,7 +25,7 @@ export class GlobalContext {
   public pkgId: Ids = new Ids();
 
   constructor(public opts: Options, public pkgLoader: PkgLoader) {
-    this.error = new ErrorHandler(opts.treatErrAsBug);
+    this.error = new ErrorHandler(opts);
   }
 
   public findItem<P extends Phase>(
@@ -76,79 +74,3 @@ export class GlobalContext {
     return mod;
   }
 }
-
-export type Options = {
-  input: string;
-  filename: string;
-  packageName: string;
-  debug: Set<string>;
-  noOutput: boolean;
-  noStd: boolean;
-  treatErrAsBug: boolean;
-};
-
-export function parseArgs(hardcodedInput: string): Options {
-  let filename: string;
-  let input: string;
-  let packageName: string;
-  let debug = new Set<string>();
-  let noOutput = false;
-  let noStd = false;
-  let treatErrAsBug = false;
-
-  if (process.argv.length > 2) {
-    filename = process.argv[2];
-    if (path.extname(filename) !== ".nil") {
-      console.error(process.argv);
-
-      console.error(
-        `error: filename must have \`.nil\` extension: \`${filename}\``,
-      );
-      process.exit(1);
-    }
-
-    input = fs.readFileSync(filename, { encoding: "utf-8" });
-    if (filename.endsWith(".mod.nil")) {
-      packageName = path.basename(filename, ".mod.nil");
-    } else {
-      packageName = path.basename(filename, ".nil");
-    }
-
-    const debugArg = process.argv.find((arg) => arg.startsWith("--debug="));
-    if (debugArg !== undefined) {
-      const debugs = debugArg.slice("--debug=".length);
-      debug = new Set(debugs.split(","));
-    }
-
-    if (process.argv.some((arg) => arg === "--no-output")) {
-      noOutput = true;
-    }
-    if (process.argv.some((arg) => arg === "--no-std")) {
-      noStd = true;
-    }
-    if (process.argv.some((arg) => arg === "--treat-err-as-bug")) {
-      treatErrAsBug = true;
-    }
-  } else {
-    filename = "<hardcoded>";
-    input = hardcodedInput;
-    packageName = "test";
-    debug = new Set([
-      "tokens",
-      "parsed",
-      "resolved",
-      "typecked",
-      "wasm-validate",
-    ]);
-  }
-
-  return {
-    filename,
-    input,
-    packageName,
-    debug,
-    noOutput,
-    noStd,
-    treatErrAsBug,
-  };
-}
diff --git a/src/error.test.ts b/src/error.test.ts
index ae2bfb9..cd298d5 100644
--- a/src/error.test.ts
+++ b/src/error.test.ts
@@ -3,7 +3,7 @@ import { lines } from "./error";
 it("should extract lines correctly", () => {
   const input = "AAA\nmeow\n:3\n\n";
 
-  const lineSpans = lines({ content: input });
+  const lineSpans = lines({ content: input, path: "" });
   const lineContents = lineSpans.map(({ start, end }) =>
     input.slice(start, end),
   );
diff --git a/src/error.ts b/src/error.ts
index 46cb8db..369aadb 100644
--- a/src/error.ts
+++ b/src/error.ts
@@ -1,10 +1,13 @@
 import chalk from "chalk";
+import { Options } from "./options";
 
 export type LoadedFile = {
-  path?: string;
+  path: string;
   content: string;
 };
 
+export type ErrorFormat = "text-render" | "json";
+
 export class Span {
   constructor(
     public start: number,
@@ -25,42 +28,40 @@ export class Span {
   }
 
   public static eof(file: LoadedFile): Span {
-    return new Span(Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, file);
+    return new Span(file.content.length, file.content.length, file);
   }
 
   public static startOfFile(file: LoadedFile): Span {
     return new Span(0, 1, file);
   }
 
-  public static DUMMY: Span = new Span(0, 0, { content: "" });
+  public static DUMMY: Span = new Span(0, 0, { content: "", path: "" });
 }
 
 export type Emitter = (string: string) => void;
 
 export class ErrorHandler {
   private errors: CompilerError[] = [];
-  private treatErrAsBug: boolean;
+  private opts: Options;
 
   constructor(
-    treatErrAsBug: boolean,
+    opts: Options,
     private emitter = (msg: string) => globalThis.console.error(msg),
   ) {
-    this.treatErrAsBug = treatErrAsBug;
+    this.opts = opts;
   }
 
   public emitError(err: CompilerError): ErrorEmitted {
-    if (this.treatErrAsBug) {
+    if (this.opts.treatErrAsBug) {
       throw new Error(`--treat-err-as-bug: ${err.msg}`);
     }
-    renderDiagnostic(this.emitter, err, (msg) => chalk.red(`error: ${msg}`));
+    emitDiagnostic(this.opts, this.emitter, err, "error");
     this.errors.push(err);
     return ERROR_EMITTED;
   }
 
   public warn(err: CompilerError): void {
-    renderDiagnostic(this.emitter, err, (msg) =>
-      chalk.yellow(`warning: ${msg}`),
-    );
+    emitDiagnostic(this.opts, this.emitter, err, "warning");
   }
 
   public hasErrors(): boolean {
@@ -85,27 +86,67 @@ export class CompilerError {
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const console = {};
 
+type JsonDiagnostic = {
+  kind: "error" | "warning";
+  message: string;
+  span: {
+    fileName: string;
+    byteStart: number;
+    byteEnd: number;
+    lineNumber: number;
+  };
+  rendered: string;
+};
+
+function emitDiagnostic(
+  opts: Options,
+  emitter: Emitter,
+  e: CompilerError,
+  kind: "error" | "warning",
+) {
+  switch (opts.errorFormat) {
+    case "json": {
+      let rendered = "";
+      renderDiagnostic((msg) => (rendered += `${msg}\n`), e, kind);
+
+      const [lineNumber] = spanToLine(e.span);
+      const object: JsonDiagnostic = {
+        kind: kind,
+        message: e.msg,
+        span: {
+          // TODO: this is *characters*, not bytes. lol.
+          byteStart: e.span.start,
+          byteEnd: e.span.start + e.span.end,
+          fileName: e.span.file.path,
+          lineNumber,
+        },
+        rendered,
+      };
+      emitter(JSON.stringify(object));
+
+      break;
+    }
+    case "text-render": {
+      renderDiagnostic(emitter, e, kind);
+      break;
+    }
+  }
+}
+
 function renderDiagnostic(
   emitter: Emitter,
   e: CompilerError,
-  render_msg: (msg: string) => string,
+  kind: "error" | "warning",
 ) {
   const { span } = e;
   const { content } = span.file;
 
-  const lineSpans = lines(span.file);
-  const line =
-    span.start === Number.MAX_SAFE_INTEGER
-      ? lineSpans[lineSpans.length - 1]
-      : lineSpans.find(
-          (line) => line.start <= span.start && line.end >= span.start,
-        );
-  if (!line) {
-    throw Error(`Span out of bounds: ${span.start}..${span.end}`);
+  const [lineNo, line] = spanToLine(span);
+  if (kind === "error") {
+    emitter(chalk.red(`error: ${e.msg}`));
+  } else {
+    emitter(chalk.yellow(`warning: ${e.msg}`));
   }
-  const lineIdx = lineSpans.indexOf(line);
-  const lineNo = lineIdx + 1;
-  emitter(render_msg(e.msg));
   emitter(` --> ${span.file.path ?? "<unknown>"}:${lineNo}`);
 
   emitter(`${lineNo} | ${spanToSnippet(content, line)}`);
@@ -124,6 +165,22 @@ function renderDiagnostic(
   );
 }
 
+function spanToLine(span: Span): [number, Span] {
+  const lineSpans = lines(span.file);
+  const line =
+    span.start === Number.MAX_SAFE_INTEGER
+      ? lineSpans[lineSpans.length - 1]
+      : lineSpans.find(
+          (line) => line.start <= span.start && line.end >= span.start,
+        );
+  if (!line) {
+    throw Error(`Span out of bounds: ${span.start}..${span.end}`);
+  }
+  const lineIdx = lineSpans.indexOf(line);
+  const lineNo = lineIdx + 1;
+  return [lineNo, line];
+}
+
 function spanToSnippet(input: string, span: Span): string {
   if (span.start === Number.MAX_SAFE_INTEGER) {
     return "";
diff --git a/src/index.ts b/src/index.ts
index 1d96ed2..bbea994 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -9,8 +9,9 @@ import { writeModuleWatToString } from "./wasm/wat";
 import fs from "fs";
 import { exec } from "child_process";
 import { Pkg, Built, Typecked } from "./ast";
-import { GlobalContext, parseArgs } from "./context";
+import { GlobalContext } from "./context";
 import { loadPkg } from "./loader";
+import { parseArgs } from "./options";
 
 const INPUT = `
 type A = struct { a: Int };
diff --git a/src/lexer.test.ts b/src/lexer.test.ts
index 3bcedd3..9acb0ce 100644
--- a/src/lexer.test.ts
+++ b/src/lexer.test.ts
@@ -1,10 +1,14 @@
 import { ErrorHandler, unreachable } from "./error";
 import { tokenize } from "./lexer";
+import { defaultOptions } from "./options";
 
 it("should tokenize an emtpy function", () => {
   const input = `function hello() = ;`;
 
-  const tokens = tokenize(new ErrorHandler(false), { content: input });
+  const tokens = tokenize(new ErrorHandler(defaultOptions()), {
+    content: input,
+    path: "",
+  });
   if (!tokens.ok) unreachable("lexer error");
 
   expect(tokens.tokens).toMatchInlineSnapshot(`
@@ -77,7 +81,10 @@ it("should tokenize an emtpy function", () => {
 it("should tokenize hello world", () => {
   const input = `print("hello world")`;
 
-  const tokens = tokenize(new ErrorHandler(false), { content: input });
+  const tokens = tokenize(new ErrorHandler(defaultOptions()), {
+    content: input,
+    path: "",
+  });
   if (!tokens.ok) unreachable("lexer error");
 
   expect(tokens.tokens).toMatchInlineSnapshot(`
diff --git a/src/lexer.ts b/src/lexer.ts
index e438e0e..9db100c 100644
--- a/src/lexer.ts
+++ b/src/lexer.ts
@@ -133,10 +133,13 @@ function tokenizeInner(file: LoadedFile): Token[] {
 
   finish: while (i < input.length) {
     const next = input[i];
+    if (next === undefined) {
+      throw new Error("index out of bounds");
+    }
     const span: Span = new Span(i, i + 1, file);
 
     if (next === "/" && input[i + 1] === "/") {
-      while (input[i] !== "\n") {
+      while (input[i] !== "\n" && input[i] !== undefined) {
         i++;
       }
 
diff --git a/src/loader.ts b/src/loader.ts
index 2f3fafa..92aa27d 100644
--- a/src/loader.ts
+++ b/src/loader.ts
@@ -77,7 +77,7 @@ function dummyErrorPkg(
     packageName,
     rootItems: [],
     itemsById: new ComplexMap(),
-    rootFile: { content: "<dummy>" },
+    rootFile: { content: "<dummy>", path: "" },
     fatalError: emitted,
     typeckResults: {
       main: undefined,
diff --git a/src/options.ts b/src/options.ts
new file mode 100644
index 0000000..337bdb0
--- /dev/null
+++ b/src/options.ts
@@ -0,0 +1,82 @@
+import { ErrorFormat } from "./error";
+import * as path from "node:path";
+import * as fs from "node:fs";
+
+export type Options = {
+  input: string;
+  filename: string;
+  packageName: string;
+  debug: Set<string>;
+  noOutput: boolean;
+  noStd: boolean;
+  treatErrAsBug: boolean;
+  errorFormat: ErrorFormat;
+};
+
+export function defaultOptions(): Options {
+  return {
+    filename: "",
+    input: "",
+    packageName: "main",
+    debug: new Set<string>(),
+    noOutput: false,
+    noStd: false,
+    treatErrAsBug: false,
+    errorFormat: "text-render",
+  };
+}
+
+export function parseArgs(hardcodedInput: string): Options {
+  const opts = defaultOptions();
+
+  if (process.argv.length > 2) {
+    opts.filename = process.argv[2];
+    if (path.extname(opts.filename) !== ".nil") {
+      console.error(process.argv);
+
+      console.error(
+        `error: filename must have \`.nil\` extension: \`${opts.filename}\``,
+      );
+      process.exit(1);
+    }
+
+    opts.input = fs.readFileSync(opts.filename, { encoding: "utf-8" });
+    if (opts.filename.endsWith(".mod.nil")) {
+      opts.packageName = path.basename(opts.filename, ".mod.nil");
+    } else {
+      opts.packageName = path.basename(opts.filename, ".nil");
+    }
+
+    const debugArg = process.argv.find((arg) => arg.startsWith("--debug="));
+    if (debugArg !== undefined) {
+      const debugs = debugArg.slice("--debug=".length);
+      opts.debug = new Set(debugs.split(","));
+    }
+
+    if (process.argv.some((arg) => arg === "--no-output")) {
+      opts.noOutput = true;
+    }
+    if (process.argv.some((arg) => arg === "--no-std")) {
+      opts.noStd = true;
+    }
+    if (process.argv.some((arg) => arg === "--treat-err-as-bug")) {
+      opts.treatErrAsBug = true;
+    }
+    if (process.argv.some((arg) => arg === "--error-format-json")) {
+      opts.errorFormat = "json";
+    }
+  } else {
+    opts.filename = "<hardcoded>";
+    opts.input = hardcodedInput;
+    opts.packageName = "test";
+    opts.debug = new Set([
+      "tokens",
+      "parsed",
+      "resolved",
+      "typecked",
+      "wasm-validate",
+    ]);
+  }
+
+  return opts;
+}
diff --git a/src/typeck/infer.test.ts b/src/typeck/infer.test.ts
index 15cca76..2aab857 100644
--- a/src/typeck/infer.test.ts
+++ b/src/typeck/infer.test.ts
@@ -1,13 +1,14 @@
 import { Emitter, ErrorHandler, Span } from "../error";
+import { defaultOptions } from "../options";
 import { TYS } from "../types";
 import { InferContext } from "./infer";
 
-const SPAN: Span = Span.startOfFile({ content: "" });
+const SPAN: Span = Span.startOfFile({ content: "", path: "" });
 
 const dummyEmitter: Emitter = () => {};
 
 it("should infer types across assignments", () => {
-  const infcx = new InferContext(new ErrorHandler(false, dummyEmitter));
+  const infcx = new InferContext(new ErrorHandler(defaultOptions(), dummyEmitter));
 
   const a = infcx.newVar();
   const b = infcx.newVar();
@@ -30,7 +31,7 @@ it("should infer types across assignments", () => {
 it("should conflict assignments to resolvable type vars", () => {
   let errorLines = 0;
   const emitter = () => (errorLines += 1);
-  const infcx = new InferContext(new ErrorHandler(false, emitter));
+  const infcx = new InferContext(new ErrorHandler(defaultOptions(), emitter));
 
   const a = infcx.newVar();
   const b = infcx.newVar();
@@ -46,7 +47,7 @@ it("should conflict assignments to resolvable type vars", () => {
 });
 
 it("should not cycle", () => {
-  const infcx = new InferContext(new ErrorHandler(false, dummyEmitter));
+  const infcx = new InferContext(new ErrorHandler(defaultOptions(), dummyEmitter));
 
   const a = infcx.newVar();
   const b = infcx.newVar();
diff --git a/ui-harness/Cargo.lock b/ui-harness/Cargo.lock
index bc76d41..7053f56 100644
--- a/ui-harness/Cargo.lock
+++ b/ui-harness/Cargo.lock
@@ -18,12 +18,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
-name = "aho-corasick"
-version = "1.1.3"
+name = "annotate-snippets"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+checksum = "5a5a59f105fb9635e9eebdc1e29d53e764fa5795b9cf899a638a53e61567ef61"
 dependencies = [
- "memchr",
+ "anstyle",
+ "unicode-width",
 ]
 
 [[package]]
@@ -35,72 +36,18 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "anstream"
-version = "0.6.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
-]
-
 [[package]]
 name = "anstyle"
 version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
 
-[[package]]
-name = "anstyle-parse"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
-dependencies = [
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
-dependencies = [
- "anstyle",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "anyhow"
 version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
 
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "backtrace"
 version = "0.3.71"
@@ -116,12 +63,6 @@ dependencies = [
  "rustc-demangle",
 ]
 
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
 [[package]]
 name = "bitflags"
 version = "2.5.0"
@@ -159,9 +100,9 @@ dependencies = [
 
 [[package]]
 name = "cargo_metadata"
-version = "0.15.4"
+version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a"
+checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
 dependencies = [
  "camino",
  "cargo-platform",
@@ -183,61 +124,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "ansi_term",
- "atty",
- "bitflags 1.3.2",
- "strsim 0.8.0",
- "textwrap",
- "unicode-width",
- "vec_map",
-]
-
-[[package]]
-name = "clap"
-version = "4.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0"
-dependencies = [
- "clap_builder",
- "clap_derive",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim 0.11.1",
-]
-
-[[package]]
-name = "clap_derive"
-version = "4.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
-
 [[package]]
 name = "color-eyre"
 version = "0.6.3"
@@ -265,12 +151,6 @@ dependencies = [
  "tracing-error",
 ]
 
-[[package]]
-name = "colorchoice"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
-
 [[package]]
 name = "colored"
 version = "2.1.0"
@@ -293,7 +173,7 @@ version = "0.15.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
 dependencies = [
- "encode_unicode 0.3.6",
+ "encode_unicode",
  "lazy_static",
  "libc",
  "unicode-width",
@@ -315,66 +195,12 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
-[[package]]
-name = "csv"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "dirs-next"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
-
-[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
-[[package]]
-name = "distance"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d9d8664cf849d7d0f3114a3a387d2f5e4303176d746d5a951aaddc66dfe9240"
-
 [[package]]
 name = "encode_unicode"
 version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 
-[[package]]
-name = "encode_unicode"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
-
 [[package]]
 name = "errno"
 version = "0.3.9"
@@ -401,53 +227,12 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
 
-[[package]]
-name = "getrandom"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
 [[package]]
 name = "gimli"
 version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
 
-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
-
 [[package]]
 name = "indenter"
 version = "0.3.3"
@@ -476,23 +261,6 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "is-terminal"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
-dependencies = [
- "hermit-abi 0.3.9",
- "libc",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "is_terminal_polyfill"
-version = "1.70.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
-
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -505,34 +273,24 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "levenshtein"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760"
+
 [[package]]
 name = "libc"
 version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
-[[package]]
-name = "libredox"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
-dependencies = [
- "bitflags 2.5.0",
- "libc",
-]
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
-[[package]]
-name = "log"
-version = "0.4.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
-
 [[package]]
 name = "memchr"
 version = "2.7.2"
@@ -604,46 +362,6 @@ checksum = "8ff1fec61082821f8236cf6c0c14e8172b62ce8a72a0eedc30d3b247bb68dc11"
 dependencies = [
  "ansi_term",
  "pad",
- "prettytable-rs",
- "structopt",
-]
-
-[[package]]
-name = "prettytable-rs"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
-dependencies = [
- "csv",
- "encode_unicode 1.0.0",
- "is-terminal",
- "lazy_static",
- "term",
- "unicode-width",
-]
-
-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
 ]
 
 [[package]]
@@ -664,25 +382,12 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "redox_users"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
-dependencies = [
- "getrandom",
- "libredox",
- "thiserror",
-]
-
 [[package]]
 name = "regex"
 version = "1.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
 dependencies = [
- "aho-corasick",
- "memchr",
  "regex-automata",
  "regex-syntax",
 ]
@@ -693,8 +398,6 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
- "aho-corasick",
- "memchr",
  "regex-syntax",
 ]
 
@@ -721,14 +424,14 @@ dependencies = [
 
 [[package]]
 name = "rustfix"
-version = "0.6.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecd2853d9e26988467753bd9912c3a126f642d05d229a4b53f5752ee36c56481"
+checksum = "5b338d50bbf36e891c7e40337c8d4cf654094a14d50c3583c6022793c01a259c"
 dependencies = [
- "anyhow",
- "log",
  "serde",
  "serde_json",
+ "thiserror",
+ "tracing",
 ]
 
 [[package]]
@@ -737,19 +440,13 @@ version = "0.38.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags",
  "errno",
  "libc",
  "linux-raw-sys",
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "rustversion"
-version = "1.0.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
-
 [[package]]
 name = "ryu"
 version = "1.0.18"
@@ -782,7 +479,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn",
 ]
 
 [[package]]
@@ -806,50 +503,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "strsim"
-version = "0.8.0"
+name = "spanned"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "structopt"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
+checksum = "ed14ba8b4b82241bd5daba2c49185d4a0581a0058355fe96537338f002b8605d"
 dependencies = [
- "clap 2.34.0",
- "lazy_static",
- "structopt-derive",
-]
-
-[[package]]
-name = "structopt-derive"
-version = "0.4.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
-dependencies = [
- "heck 0.3.3",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
+ "bstr",
+ "color-eyre",
 ]
 
 [[package]]
@@ -875,26 +535,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "term"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
-dependencies = [
- "dirs-next",
- "rustversion",
- "winapi",
-]
-
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.61"
@@ -912,7 +552,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn",
 ]
 
 [[package]]
@@ -932,9 +572,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
 dependencies = [
  "pin-project-lite",
+ "tracing-attributes",
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-attributes"
+version = "0.1.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tracing-core"
 version = "0.1.32"
@@ -970,33 +622,37 @@ dependencies = [
 name = "ui-tests"
 version = "0.1.0"
 dependencies = [
+ "cargo_metadata",
+ "serde",
+ "serde_json",
  "ui_test",
 ]
 
 [[package]]
 name = "ui_test"
-version = "0.12.1"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6a8ae5b9da03f8814fe1a6691e54095bc08b704255997c543d943b1c0006603"
+checksum = "29e5f4ffcbab82453958fbf59990e981b8e8a177dcd60c2bd8f9b52c3036a6e1"
 dependencies = [
+ "annotate-snippets",
  "anyhow",
  "bstr",
  "cargo-platform",
  "cargo_metadata",
- "clap 4.5.4",
  "color-eyre",
  "colored",
  "comma",
  "crossbeam-channel",
- "distance",
  "indicatif",
  "lazy_static",
+ "levenshtein",
  "prettydiff",
  "regex",
  "rustc_version",
  "rustfix",
  "serde",
  "serde_json",
+ "spanned",
  "tempfile",
 ]
 
@@ -1006,48 +662,18 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
 
-[[package]]
-name = "utf8parse"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
-
 [[package]]
 name = "valuable"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
 [[package]]
 name = "winapi"
 version = "0.3.9"
diff --git a/ui-harness/Cargo.toml b/ui-harness/Cargo.toml
index 1ab5878..840874d 100644
--- a/ui-harness/Cargo.toml
+++ b/ui-harness/Cargo.toml
@@ -6,4 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ui_test = "0.12.1"
+cargo_metadata = "0.18.1"
+serde = { version = "1.0.203", features = ["derive"] }
+serde_json = "1.0.117"
+ui_test = "0.23.0"
diff --git a/ui-harness/src/bin/nilc-wrapper.rs b/ui-harness/src/bin/nilc-wrapper.rs
index 59dac18..fde499f 100644
--- a/ui-harness/src/bin/nilc-wrapper.rs
+++ b/ui-harness/src/bin/nilc-wrapper.rs
@@ -1,3 +1,62 @@
+use std::io::Write;
+
+use cargo_metadata::diagnostic::DiagnosticLevel;
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize)]
+struct RiverdeltaError {
+    kind: cargo_metadata::diagnostic::DiagnosticLevel,
+    message: String,
+    span: RiverdeltaErrorSpan,
+    rendered: String,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct RiverdeltaErrorSpan {
+    line_number: u64,
+    file_name: String,
+    byte_start: u32,
+    byte_end: u32,
+}
+
+#[derive(Serialize)]
+pub struct RustcDiagnostic {
+    pub message: String,
+    pub level: DiagnosticLevel,
+    pub spans: Vec<RustcDiagnosticSpan>,
+    pub children: Vec<()>,
+    pub rendered: String,
+}
+
+#[derive(Serialize)]
+pub struct RustcDiagnosticSpan {
+    pub file_name: String,
+    /// The byte offset in the file where this diagnostic starts from.
+    pub byte_start: u32,
+    /// The byte offset in the file where this diagnostic ends.
+    pub byte_end: u32,
+    /// 1-based. The line in the file.
+    pub line_start: u64,
+    /// 1-based. The line in the file.
+    pub line_end: u64,
+    /// 1-based, character offset.
+    pub column_start: u64,
+    /// 1-based, character offset.
+    pub column_end: u64,
+    /// Is this a "primary" span -- meaning the point, or one of the points,
+    /// where the error occurred?
+    ///
+    /// There are rare cases where multiple spans are marked as primary,
+    /// e.g. "immutable borrow occurs here" and "mutable borrow ends here" can
+    /// be two separate spans both "primary". Top (parent) messages should
+    /// always have at least one primary span, unless it has 0 spans. Child
+    /// messages may have 0 or more primary spans.
+    pub is_primary: bool,
+    /// Source text from the start of line_start to the end of line_end.
+    pub text: Vec<()>,
+}
+
 fn main() {
     let mut args = std::env::args().skip(1).collect::<Vec<_>>();
 
@@ -21,15 +80,45 @@ fn main() {
 
     // We don't want out.wat polluting things.
     args.push("--no-output".into());
+    args.push("--error-format-json".into());
 
     let result = std::process::Command::new("node")
         .arg(".")
         .args(args)
-        .spawn()
-        .unwrap()
-        .wait()
+        .output()
         .unwrap();
 
-    let code = result.code().unwrap_or(1);
+    std::io::stdout().write_all(&result.stdout).unwrap();
+
+    let stderr = String::from_utf8(result.stderr).unwrap();
+    for line in stderr.lines() {
+        if line.starts_with("{") {
+            let err: RiverdeltaError = serde_json::from_str::<RiverdeltaError>(line).unwrap();
+
+            let rustc_error = RustcDiagnostic {
+                message: err.message,
+                level: err.kind,
+                spans: vec![RustcDiagnosticSpan {
+                    file_name: err.span.file_name,
+                    byte_start: err.span.byte_start,
+                    byte_end: err.span.byte_end,
+                    line_start: err.span.line_number,
+                    line_end: err.span.line_number,
+                    column_start: 1,
+                    column_end: 1,
+                    is_primary: true,
+                    text: vec![],
+                }],
+                children: vec![],
+                rendered: err.rendered,
+            };
+
+            eprintln!("{}", serde_json::to_string(&rustc_error).unwrap());
+        } else {
+            eprintln!("{line}");
+        }
+    }
+
+    let code = result.status.code().unwrap_or(1);
     std::process::exit(code);
 }
diff --git a/ui-harness/src/main.rs b/ui-harness/src/main.rs
index c6ee12d..e66cf19 100644
--- a/ui-harness/src/main.rs
+++ b/ui-harness/src/main.rs
@@ -1,9 +1,8 @@
 use ui_test::{
-    clap::Parser, default_filter_by_arg, default_per_file_config, status_emitter, Args,
-    CommandBuilder, Config, Mode, OutputConflictHandling,
+    default_per_file_config, status_emitter, Args, CommandBuilder, Config, OutputConflictHandling,
 };
 
-fn main() {
+fn main() -> ui_test::Result<()> {
     std::process::Command::new("cargo")
         .args(&[
             "build",
@@ -23,32 +22,28 @@ fn main() {
     let mut config = Config::rustc("./ui-tests");
     config.host = Some("wasm :3".into());
     config.program = CommandBuilder::cmd("ui-harness/target/debug/nilc-wrapper");
-    config.mode = Mode::Fail {
-        require_patterns: false,
-    };
 
-    let args = Args::parse();
+    let args = Args::parse_args(Args::default(), std::env::args().skip(1))?;
 
-    let text = if args.quiet {
-        status_emitter::Text::quiet()
-    } else {
-        status_emitter::Text::verbose()
-    };
+    config.with_args(&args);
 
     if !args.check && std::env::var_os("GITHUB_ACTIONS").is_none() {
         config.output_conflict_handling = OutputConflictHandling::Bless;
     }
 
     let result = ui_test::run_tests_generic(
-        config,
-        args,
-        |path, args| {
-            path.extension().is_some_and(|ext| ext == "nil") && default_filter_by_arg(path, args)
+        vec![config],
+        |path, config| {
+            if !path.extension().is_some_and(|ext| ext == "nil") {
+                return None;
+            }
+            Some(ui_test::default_any_file_filter(path, config))
         },
         default_per_file_config,
-        (text, status_emitter::Gha::<true> { name: "ui".into() }),
+        status_emitter::Text::quiet(),
     );
     if let Err(result) = result {
         println!("{:?}", result);
     }
+    Ok(())
 }
diff --git a/ui-tests/asm/instr_not_string.nil b/ui-tests/asm/instr_not_string.nil
index 0018d3d..ed30c6b 100644
--- a/ui-tests/asm/instr_not_string.nil
+++ b/ui-tests/asm/instr_not_string.nil
@@ -2,6 +2,7 @@ function a(_a: I32) =
     ___asm(
         __locals(),
         0,
+        //~^ ERROR: inline assembly instruction must be string literal with instruction
     );
 
 function main() = ;
diff --git a/ui-tests/asm/instr_not_string.stderr b/ui-tests/asm/instr_not_string.stderr
index 1a146bd..1f8659d 100644
--- a/ui-tests/asm/instr_not_string.stderr
+++ b/ui-tests/asm/instr_not_string.stderr
@@ -1,4 +1,4 @@
 error: inline assembly instruction must be string literal with instruction
- --> $DIR/instr_not_string.nil:4
+ --> ./ui-tests/asm/instr_not_string.nil:4
 4 |         0,
             ^
diff --git a/ui-tests/asm/invalid_instr.nil b/ui-tests/asm/invalid_instr.nil
index c383579..0ac98a9 100644
--- a/ui-tests/asm/invalid_instr.nil
+++ b/ui-tests/asm/invalid_instr.nil
@@ -2,6 +2,7 @@ function dropping(_a: I32) =
     ___asm(
         __locals(),
         "meow meow",
+        //~^ ERROR: unknown instruction: meow
     );
 
 function main() = ;
diff --git a/ui-tests/asm/invalid_instr.stderr b/ui-tests/asm/invalid_instr.stderr
index 7d6002c..5208517 100644
--- a/ui-tests/asm/invalid_instr.stderr
+++ b/ui-tests/asm/invalid_instr.stderr
@@ -1,4 +1,4 @@
 error: unknown instruction: meow
- --> $DIR/invalid_instr.nil:4
+ --> ./ui-tests/asm/invalid_instr.nil:4
 4 |         "meow meow",
             ^^^^^^^^^^^
diff --git a/ui-tests/asm/missing_locals.nil b/ui-tests/asm/missing_locals.nil
index 95a0937..deef865 100644
--- a/ui-tests/asm/missing_locals.nil
+++ b/ui-tests/asm/missing_locals.nil
@@ -1,5 +1,6 @@
 function dropping(_a: I32) =
     ___asm(
+        //~^ ERROR: inline assembly must have __locals() as first argument
         "local.get 0",
         "drop",
     );
diff --git a/ui-tests/asm/missing_locals.stderr b/ui-tests/asm/missing_locals.stderr
index 08be720..cfd25e1 100644
--- a/ui-tests/asm/missing_locals.stderr
+++ b/ui-tests/asm/missing_locals.stderr
@@ -1,4 +1,4 @@
 error: inline assembly must have __locals() as first argument
- --> $DIR/missing_locals.nil:2
+ --> ./ui-tests/asm/missing_locals.nil:2
 2 |     ___asm(
               ^
diff --git a/ui-tests/asm/not_toplevel.nil b/ui-tests/asm/not_toplevel.nil
index ddff7e1..f6a1c00 100644
--- a/ui-tests/asm/not_toplevel.nil
+++ b/ui-tests/asm/not_toplevel.nil
@@ -1,6 +1,8 @@
 function dropping(_a: I32) = (
     1;
     ___asm(__locals(), "drop");
+    //~^ ERROR: `___asm` cannot be used as a value
+    //~| ERROR: `__locals` cannot be used as a value
 );
 
 function main() = ;
diff --git a/ui-tests/asm/not_toplevel.stderr b/ui-tests/asm/not_toplevel.stderr
index b0bbfe1..57a1757 100644
--- a/ui-tests/asm/not_toplevel.stderr
+++ b/ui-tests/asm/not_toplevel.stderr
@@ -1,8 +1,8 @@
 error: `___asm` cannot be used as a value
- --> $DIR/not_toplevel.nil:3
+ --> ./ui-tests/asm/not_toplevel.nil:3
 3 |     ___asm(__locals(), "drop");
         ^^^^^^
 error: `__locals` cannot be used as a value
- --> $DIR/not_toplevel.nil:3
+ --> ./ui-tests/asm/not_toplevel.nil:3
 3 |     ___asm(__locals(), "drop");
                ^^^^^^^^
diff --git a/ui-tests/asm/wrong_imm.nil b/ui-tests/asm/wrong_imm.nil
index 1ed6d5a..48ede49 100644
--- a/ui-tests/asm/wrong_imm.nil
+++ b/ui-tests/asm/wrong_imm.nil
@@ -2,6 +2,7 @@ function a(_a: I32) =
     ___asm(
         __locals(),
         "local.get 0 0",
+        //~^ ERROR: mismatched immediate lengths, expected 1, got 2
         "drop",
     );
 
@@ -9,6 +10,7 @@ function b(_a: I32) =
     ___asm(
         __locals(),
         "local.get",
+        //~^ ERROR: mismatched immediate lengths, expected 1, got 0
         "drop",
     );
 
diff --git a/ui-tests/asm/wrong_imm.stderr b/ui-tests/asm/wrong_imm.stderr
index 2e7da70..c6b3d6b 100644
--- a/ui-tests/asm/wrong_imm.stderr
+++ b/ui-tests/asm/wrong_imm.stderr
@@ -1,8 +1,8 @@
 error: mismatched immediate lengths, expected 1, got 2
- --> $DIR/wrong_imm.nil:4
+ --> ./ui-tests/asm/wrong_imm.nil:4
 4 |         "local.get 0 0",
             ^^^^^^^^^^^^^^^
 error: mismatched immediate lengths, expected 1, got 0
- --> $DIR/wrong_imm.nil:11
-11 |         "local.get",
+ --> ./ui-tests/asm/wrong_imm.nil:12
+12 |         "local.get",
              ^^^^^^^^^^^
diff --git a/ui-tests/basic_recovery.nil b/ui-tests/basic_recovery.nil
index e6953c0..bb561e2 100644
--- a/ui-tests/basic_recovery.nil
+++ b/ui-tests/basic_recovery.nil
@@ -1,5 +1,10 @@
 function main() = (
   let a: Int = "";
+  //~^ ERROR: cannot assign String to Int
+  //~| WARN: unused variable: `a`
   let b: Int = "";
+  //~^ ERROR: cannot assign String to Int
+  //~| WARN: unused variable: `b`
   c;
+  //~^ ERROR: cannot find c
 );
diff --git a/ui-tests/basic_recovery.stderr b/ui-tests/basic_recovery.stderr
index aa671f2..6bed19e 100644
--- a/ui-tests/basic_recovery.stderr
+++ b/ui-tests/basic_recovery.stderr
@@ -1,20 +1,20 @@
 error: cannot find c
- --> $DIR/basic_recovery.nil:4
-4 |   c;
+ --> ./ui-tests/basic_recovery.nil:8
+8 |   c;
       ^
 error: cannot assign String to Int
- --> $DIR/basic_recovery.nil:2
+ --> ./ui-tests/basic_recovery.nil:2
 2 |   let a: Int = "";
           ^
 error: cannot assign String to Int
- --> $DIR/basic_recovery.nil:3
-3 |   let b: Int = "";
+ --> ./ui-tests/basic_recovery.nil:5
+5 |   let b: Int = "";
           ^
 warning: unused variable: `a`
- --> $DIR/basic_recovery.nil:2
+ --> ./ui-tests/basic_recovery.nil:2
 2 |   let a: Int = "";
           ^
 warning: unused variable: `b`
- --> $DIR/basic_recovery.nil:3
-3 |   let b: Int = "";
+ --> ./ui-tests/basic_recovery.nil:5
+5 |   let b: Int = "";
           ^
diff --git a/ui-tests/functions/body_mismatched_parens.nil b/ui-tests/functions/body_mismatched_parens.nil
deleted file mode 100644
index 810f57b..0000000
--- a/ui-tests/functions/body_mismatched_parens.nil
+++ /dev/null
@@ -1 +0,0 @@
-function main() = (
diff --git a/ui-tests/functions/body_mismatched_parens.stderr b/ui-tests/functions/body_mismatched_parens.stderr
deleted file mode 100644
index d45b176..0000000
--- a/ui-tests/functions/body_mismatched_parens.stderr
+++ /dev/null
@@ -1,8 +0,0 @@
-error: unexpected end of file
- --> $DIR/body_mismatched_parens.nil:2
-2 | 
-    ^
-error: `main` function not found
- --> $DIR/body_mismatched_parens.nil:1
-1 | function main() = (
-    ^
diff --git a/ui-tests/functions/missing_args.nil b/ui-tests/functions/missing_args.nil
index 857fa31..62c521a 100644
--- a/ui-tests/functions/missing_args.nil
+++ b/ui-tests/functions/missing_args.nil
@@ -1,5 +1,6 @@
 function main() = (
     x();
+    //~^ ERROR: missing argument of type Int
 );
 
 function x(_a: Int) = ;
\ No newline at end of file
diff --git a/ui-tests/functions/missing_args.stderr b/ui-tests/functions/missing_args.stderr
index cbee1c7..af48af4 100644
--- a/ui-tests/functions/missing_args.stderr
+++ b/ui-tests/functions/missing_args.stderr
@@ -1,4 +1,4 @@
 error: missing argument of type Int
- --> $DIR/missing_args.nil:2
+ --> ./ui-tests/functions/missing_args.nil:2
 2 |     x();
          ^
diff --git a/ui-tests/lint/unused_vars.stderr b/ui-tests/lint/unused_vars.stderr
index ad493fa..f950712 100644
--- a/ui-tests/lint/unused_vars.stderr
+++ b/ui-tests/lint/unused_vars.stderr
@@ -1,12 +1,12 @@
 warning: unused variable: `x`
- --> $DIR/unused_vars.nil:4
+ --> ./ui-tests/lint/unused_vars.nil:4
 4 |   let x = 0;
           ^
 warning: unused variable: `y`
- --> $DIR/unused_vars.nil:17
+ --> ./ui-tests/lint/unused_vars.nil:17
 17 |   let y = x;
            ^
 warning: unused function parameter: `p`
- --> $DIR/unused_vars.nil:22
+ --> ./ui-tests/lint/unused_vars.nil:22
 22 | function param(p: Int) = ;
                     ^
diff --git a/ui-tests/modules/nil_submodules.nil b/ui-tests/modules/nil_submodules.nil
index f7ab8db..05efd2d 100644
--- a/ui-tests/modules/nil_submodules.nil
+++ b/ui-tests/modules/nil_submodules.nil
@@ -1,3 +1,4 @@
 mod b;
+//~^ ERROR: .nil files cannot have submodules. use .mod.nil in a subdirectory
 
 function main() = ;
\ No newline at end of file
diff --git a/ui-tests/modules/nil_submodules.stderr b/ui-tests/modules/nil_submodules.stderr
index 9d20ce8..f118db2 100644
--- a/ui-tests/modules/nil_submodules.stderr
+++ b/ui-tests/modules/nil_submodules.stderr
@@ -1,4 +1,4 @@
 error: .nil files cannot have submodules. use .mod.nil in a subdirectory
- --> $DIR/nil_submodules.nil:1
+ --> ./ui-tests/modules/nil_submodules.nil:1
 1 | mod b;
         ^
diff --git a/ui-tests/resolve/module_not_found.nil b/ui-tests/resolve/module_not_found.nil
index 64deb9f..4ef5874 100644
--- a/ui-tests/resolve/module_not_found.nil
+++ b/ui-tests/resolve/module_not_found.nil
@@ -1,3 +1,4 @@
 function main() = (
-  let _false, l = list.new();
+  let _l = list.new();
+  //~^ ERROR: cannot find list
 );
diff --git a/ui-tests/resolve/module_not_found.stderr b/ui-tests/resolve/module_not_found.stderr
index 84e444b..a4d6fb4 100644
--- a/ui-tests/resolve/module_not_found.stderr
+++ b/ui-tests/resolve/module_not_found.stderr
@@ -1,8 +1,4 @@
-error: expected `=`, found `,`
- --> $DIR/module_not_found.nil:2
-2 |   let _false, l = list.new();
-                ^
-error: `main` function not found
- --> $DIR/module_not_found.nil:1
-1 | function main() = (
-    ^
+error: cannot find list
+ --> ./ui-tests/resolve/module_not_found.nil:2
+2 |   let _l = list.new();
+               ^^^^
diff --git a/ui-tests/type/generics/generics_on_primitive.nil b/ui-tests/type/generics/generics_on_primitive.nil
index 6bc109b..57664bb 100644
--- a/ui-tests/type/generics/generics_on_primitive.nil
+++ b/ui-tests/type/generics/generics_on_primitive.nil
@@ -1,3 +1,4 @@
 function main() = (
-    let a: I32[I32] = 0;
+    let _a: I32[I32] = 0;
+    //~^ ERROR: type I32 does not take any generic arguments but 1 were passed
 );
diff --git a/ui-tests/type/generics/generics_on_primitive.stderr b/ui-tests/type/generics/generics_on_primitive.stderr
index 887b9c2..9979fa8 100644
--- a/ui-tests/type/generics/generics_on_primitive.stderr
+++ b/ui-tests/type/generics/generics_on_primitive.stderr
@@ -1,8 +1,4 @@
 error: type I32 does not take any generic arguments but 1 were passed
- --> $DIR/generics_on_primitive.nil:2
-2 |     let a: I32[I32] = 0;
-               ^^^
-warning: unused variable: `a`
- --> $DIR/generics_on_primitive.nil:2
-2 |     let a: I32[I32] = 0;
-            ^
+ --> ./ui-tests/type/generics/generics_on_primitive.nil:2
+2 |     let _a: I32[I32] = 0;
+                ^^^
diff --git a/ui-tests/type/generics/generics_structs_in_args.stderr b/ui-tests/type/generics/generics_structs_in_args.stderr
index d70b550..59506d7 100644
--- a/ui-tests/type/generics/generics_structs_in_args.stderr
+++ b/ui-tests/type/generics/generics_structs_in_args.stderr
@@ -1,12 +1,12 @@
 warning: unused function parameter: `a`
- --> $DIR/generics_structs_in_args.nil:11
+ --> ./ui-tests/type/generics/generics_structs_in_args.nil:11
 11 | function test(a: A[I32], b: B[I32, Int, I32], c: C) = ;
                    ^
 warning: unused function parameter: `b`
- --> $DIR/generics_structs_in_args.nil:11
+ --> ./ui-tests/type/generics/generics_structs_in_args.nil:11
 11 | function test(a: A[I32], b: B[I32, Int, I32], c: C) = ;
                               ^
 warning: unused function parameter: `c`
- --> $DIR/generics_structs_in_args.nil:11
+ --> ./ui-tests/type/generics/generics_structs_in_args.nil:11
 11 | function test(a: A[I32], b: B[I32, Int, I32], c: C) = ;
                                                    ^
diff --git a/ui-tests/type/generics/structs.stderr b/ui-tests/type/generics/structs.stderr
index 3597880..e34dff5 100644
--- a/ui-tests/type/generics/structs.stderr
+++ b/ui-tests/type/generics/structs.stderr
@@ -1,12 +1,12 @@
 warning: unused function parameter: `a`
- --> $DIR/structs.nil:9
+ --> ./ui-tests/type/generics/structs.nil:9
 9 | function test(a: A[I32], b: B[I32, Int, I32], c: C) = ;
                   ^
 warning: unused function parameter: `b`
- --> $DIR/structs.nil:9
+ --> ./ui-tests/type/generics/structs.nil:9
 9 | function test(a: A[I32], b: B[I32, Int, I32], c: C) = ;
                              ^
 warning: unused function parameter: `c`
- --> $DIR/structs.nil:9
+ --> ./ui-tests/type/generics/structs.nil:9
 9 | function test(a: A[I32], b: B[I32, Int, I32], c: C) = ;
                                                   ^
diff --git a/ui-tests/type/generics/wrong_amount.nil b/ui-tests/type/generics/wrong_amount.nil
index 3a58db2..3859d31 100644
--- a/ui-tests/type/generics/wrong_amount.nil
+++ b/ui-tests/type/generics/wrong_amount.nil
@@ -6,20 +6,26 @@ type B[T, U, V] = struct {
 type C = ();
 
 function test(
-    a1: A,
-    a2: A[],
-    a3: A[I32],
-    a4: A[I32, I32],
+    _a1: A,
+    //~^ ERROR: missing generics for type A, expected 1, but only 0 were passed
+    _a2: A[],
+    //~^ ERROR: missing generics for type A, expected 1, but only 0 were passed
+    _a3: A[I32],
+    _a4: A[I32, I32],
 
-    b1: B,
-    b2: B[],
-    b3: B[Int, Int],
-    b4: B[Int, I32, Int],
-    b5: B[Int, Int, Int, Int],
+    _b1: B,
+    //~^ ERROR: missing generics for type B, expected 3, but only 0 were passed
+    _b2: B[],
+    //~^ ERROR: missing generics for type B, expected 3, but only 0 were passed
+    _b3: B[Int, Int],
+    //~^ ERROR: issing generics for type B, expected 3, but only 2 were passed
+    _b4: B[Int, I32, Int],
+    _b5: B[Int, Int, Int, Int],
 
-    c1: C,
-    c2: C[],
-    c3: C[I32],
+    _c1: C,
+    _c2: C[],
+    _c3: C[I32],
+    //~^ ERROR: type () does not take any generic arguments but 1 were passed
 ) = ;
 
 function main() = ;
diff --git a/ui-tests/type/generics/wrong_amount.stderr b/ui-tests/type/generics/wrong_amount.stderr
index 03675ee..f479ae8 100644
--- a/ui-tests/type/generics/wrong_amount.stderr
+++ b/ui-tests/type/generics/wrong_amount.stderr
@@ -1,72 +1,24 @@
 error: missing generics for type A, expected 1, but only 0 were passed
- --> $DIR/wrong_amount.nil:9
-9 |     a1: A,
-            ^
+ --> ./ui-tests/type/generics/wrong_amount.nil:9
+9 |     _a1: A,
+             ^
 error: missing generics for type A, expected 1, but only 0 were passed
- --> $DIR/wrong_amount.nil:10
-10 |     a2: A[],
-             ^
+ --> ./ui-tests/type/generics/wrong_amount.nil:11
+11 |     _a2: A[],
+              ^
 error: missing generics for type B, expected 3, but only 0 were passed
- --> $DIR/wrong_amount.nil:14
-14 |     b1: B,
-             ^
+ --> ./ui-tests/type/generics/wrong_amount.nil:16
+16 |     _b1: B,
+              ^
 error: missing generics for type B, expected 3, but only 0 were passed
- --> $DIR/wrong_amount.nil:15
-15 |     b2: B[],
-             ^
+ --> ./ui-tests/type/generics/wrong_amount.nil:18
+18 |     _b2: B[],
+              ^
 error: missing generics for type B, expected 3, but only 2 were passed
- --> $DIR/wrong_amount.nil:16
-16 |     b3: B[Int, Int],
-             ^
+ --> ./ui-tests/type/generics/wrong_amount.nil:20
+20 |     _b3: B[Int, Int],
+              ^
 error: type () does not take any generic arguments but 1 were passed
- --> $DIR/wrong_amount.nil:22
-22 |     c3: C[I32],
-             ^
-warning: unused function parameter: `a1`
- --> $DIR/wrong_amount.nil:9
-9 |     a1: A,
-        ^^
-warning: unused function parameter: `a2`
- --> $DIR/wrong_amount.nil:10
-10 |     a2: A[],
-         ^^
-warning: unused function parameter: `a3`
- --> $DIR/wrong_amount.nil:11
-11 |     a3: A[I32],
-         ^^
-warning: unused function parameter: `a4`
- --> $DIR/wrong_amount.nil:12
-12 |     a4: A[I32, I32],
-         ^^
-warning: unused function parameter: `b1`
- --> $DIR/wrong_amount.nil:14
-14 |     b1: B,
-         ^^
-warning: unused function parameter: `b2`
- --> $DIR/wrong_amount.nil:15
-15 |     b2: B[],
-         ^^
-warning: unused function parameter: `b3`
- --> $DIR/wrong_amount.nil:16
-16 |     b3: B[Int, Int],
-         ^^
-warning: unused function parameter: `b4`
- --> $DIR/wrong_amount.nil:17
-17 |     b4: B[Int, I32, Int],
-         ^^
-warning: unused function parameter: `b5`
- --> $DIR/wrong_amount.nil:18
-18 |     b5: B[Int, Int, Int, Int],
-         ^^
-warning: unused function parameter: `c1`
- --> $DIR/wrong_amount.nil:20
-20 |     c1: C,
-         ^^
-warning: unused function parameter: `c2`
- --> $DIR/wrong_amount.nil:21
-21 |     c2: C[],
-         ^^
-warning: unused function parameter: `c3`
- --> $DIR/wrong_amount.nil:22
-22 |     c3: C[I32],
-         ^^
+ --> ./ui-tests/type/generics/wrong_amount.nil:27
+27 |     _c3: C[I32],
+              ^
diff --git a/ui-tests/type/type_alias.stderr b/ui-tests/type/type_alias.stderr
index 46551de..a1d6c78 100644
--- a/ui-tests/type/type_alias.stderr
+++ b/ui-tests/type/type_alias.stderr
@@ -1,4 +1,4 @@
 warning: unused variable: `a`
- --> $DIR/type_alias.nil:6
+ --> ./ui-tests/type/type_alias.nil:6
 6 |     let a: A = (0, 0);
             ^
diff --git a/ui-tests/type/type_assignments.stderr b/ui-tests/type/type_assignments.stderr
index 60ecfea..becfc3d 100644
--- a/ui-tests/type/type_assignments.stderr
+++ b/ui-tests/type/type_assignments.stderr
@@ -1,24 +1,24 @@
 warning: unused variable: `a1`
- --> $DIR/type_assignments.nil:15
+ --> ./ui-tests/type/type_assignments.nil:15
 15 |     let a1: Int = a;
              ^^
 warning: unused variable: `b1`
- --> $DIR/type_assignments.nil:16
+ --> ./ui-tests/type/type_assignments.nil:16
 16 |     let b1: I32 = b;
              ^^
 warning: unused variable: `c1`
- --> $DIR/type_assignments.nil:17
+ --> ./ui-tests/type/type_assignments.nil:17
 17 |     let c1: String = c;
              ^^
 warning: unused variable: `d1`
- --> $DIR/type_assignments.nil:18
+ --> ./ui-tests/type/type_assignments.nil:18
 18 |     let d1: Bool = d;
              ^^
 warning: unused variable: `e1`
- --> $DIR/type_assignments.nil:19
+ --> ./ui-tests/type/type_assignments.nil:19
 19 |     let e1: CustomType = e;
              ^^
 warning: unused variable: `f1`
- --> $DIR/type_assignments.nil:20
+ --> ./ui-tests/type/type_assignments.nil:20
 20 |     let f1: (Int, I32) = f;
              ^^