refactor crate loading

.eslintrc.cjs

@@ -47,6 +47,11 @@ module.exports = {
     ],
     // No, I will use `type` instead of `interface`.
     "@typescript-eslint/consistent-type-definitions": ["error", "type"],
+
+    // This lint is horrible with noisy false positives every time there are typescript errors.
+    "@typescript-eslint/no-unsafe-return": "off",
+    "@typescript-eslint/no-unsafe-assignment": "off",
+
     // Useful extra lints that are not on by default:
     "@typescript-eslint/explicit-module-boundary-types": "warn",
     // This has caused several bugs before. Thanks eslint!

src/ast.ts

@@ -41,6 +41,8 @@ export type Typecked = {
   typeckResults: HasTypeckResults;
 };
 
+export type Final = Typecked;
+
 export type AnyPhase = {
   res: No | HasRes;
   defPath: No | HasDefPath;
@@ -57,6 +59,8 @@ export type Crate<P extends Phase> = {
   packageName: string;
 } & P["typeckResults"];
 
+export type DepCrate = Crate<Final>;
+
 export type Ident = {
   name: string;
   span: Span;

src/context.ts

@@ -0,0 +1,49 @@
+import { Crate, DepCrate, Final, Item, ItemId, Phase } from "./ast";
+import { DUMMY_SPAN, Span } from "./error";
+import { Ids, unwrap } from "./utils";
+
+export type CrateLoader = (
+  gcx: GlobalContext,
+  name: string,
+  span: Span
+) => DepCrate;
+
+/**
+ * The global context containing information about the _global compilation session_,
+ * like loaded crates.
+ * Notably, the global context is _not_ supposed to have information specific to the "local crate",
+ * because with the current compilation model, there is no "local crate" in a session.
+ *
+ * There is a "downstream"/"binary"/"final" crate with crateId=0, where `function main()` lives, but
+ * dependencies (which also use the same context) do not care about that.
+ */
+export class GlobalContext {
+  public depCrates: Crate<Final>[] = [];
+  public crateId: Ids = new Ids();
+
+  constructor(public crateLoader: CrateLoader) {}
+
+  public findItem<P extends Phase>(
+    id: ItemId,
+    localCrate: Crate<P>
+  ): Item<P | Final> {
+    const crate = unwrap(
+      [localCrate, ...this.depCrates].find((crate) => crate.id === id.crateId)
+    );
+
+    if (id.itemIdx === 0) {
+      // Return a synthetic module representing the crate root.
+      return {
+        kind: "mod",
+        node: {
+          contents: crate.rootItems,
+          name: crate.packageName,
+        },
+        span: DUMMY_SPAN,
+        id,
+      };
+    }
+
+    return unwrap(crate.itemsById.get(id));
+  }
+}

src/index.ts

@@ -9,8 +9,8 @@ import { writeModuleWatToString } from "./wasm/wat";
 import fs from "fs";
 import path from "path";
 import { exec } from "child_process";
-import { Crate, Built, Typecked } from "./ast";
+import { Crate, Built, Typecked, DepCrate } from "./ast";
-import { Ids } from "./utils";
+import { GlobalContext, CrateLoader } from "./context";
 
 const INPUT = `
 extern mod std;
@@ -105,6 +105,9 @@ function main() {
     process.exit(1);
   }
 
+  const gcx = new GlobalContext(loadCrate);
+  const mainCrate = gcx.crateId.next();
+
   withErrorPrinter(
     input,
     filename,
@@ -117,7 +120,7 @@ function main() {
         console.log(tokens);
       }
 
-      const ast: Crate<Built> = parse(packageName, tokens, 0);
+      const ast: Crate<Built> = parse(packageName, tokens, mainCrate);
       if (debug.has("ast")) {
         console.log("-----AST---------------");
 
@@ -131,7 +134,7 @@ function main() {
       if (debug.has("resolved")) {
         console.log("-----AST resolved------");
       }
-      const [resolved, crates] = resolve(ast, loadCrate);
+      const resolved = resolve(gcx, ast);
       if (debug.has("resolved")) {
         const resolvedPrinted = printAst(resolved);
         console.log(resolvedPrinted);
@@ -140,7 +143,7 @@ function main() {
       if (debug.has("typecked")) {
         console.log("-----AST typecked------");
       }
-      const typecked: Crate<Typecked> = typeck(resolved, crates);
+      const typecked: Crate<Typecked> = typeck(gcx, resolved);
       if (debug.has("typecked")) {
         const typeckPrinted = printAst(typecked);
         console.log(typeckPrinted);
@@ -149,7 +152,7 @@ function main() {
       if (debug.has("wat")) {
         console.log("-----wasm--------------");
       }
-      const wasmModule = lowerToWasm([typecked, ...crates]);
+      const wasmModule = lowerToWasm([typecked, ...gcx.depCrates]);
       const moduleStringColor = writeModuleWatToString(wasmModule, true);
       const moduleString = writeModuleWatToString(wasmModule);
 
@@ -186,42 +189,54 @@ function main() {
   );
 }
 
-function loadCrate(
+const loadCrate: CrateLoader = (
+  gcx: GlobalContext,
   name: string,
-  span: Span,
+  span: Span
-  crateId: Ids,
+): DepCrate => {
-  existingCrates: Crate<Typecked>[]
-): [Crate<Typecked>, Crate<Typecked>[]] {
   // We really, really want a good algorithm for finding crates.
   // But right now we just look for files in the CWD.
 
-  const existing = existingCrates.find((crate) => crate.packageName === name);
+  const existing = gcx.depCrates.find((crate) => crate.packageName === name);
   if (existing) {
-    return [existing, []];
+    return existing;
   }
 
-  const filename = `${name}.nil`;
+  const options = [`${name}.nil`, `${name}/${name}.mod.nil`];
-  let input: string;
+
+  let input: string | undefined = undefined;
+  let filename: string | undefined = undefined;
+  options.forEach((tryName) => {
     try {
-    input = fs.readFileSync(filename, { encoding: "utf-8" });
+      input = fs.readFileSync(tryName, { encoding: "utf-8" });
-  } catch (e) {
+      filename = tryName;
+    } catch (e) {}
+  });
+
+  if (input === undefined || filename === undefined) {
     throw new CompilerError(
-      `failed to load ${name}, could not fine \`${filename}\``,
+      `failed to load ${name}, could not find ${options.join(" or ")}`,
       span
     );
   }
 
+  const inputString: string = input;
+
   return withErrorPrinter(
-    input,
+    inputString,
     filename,
-    () => {
+    (): DepCrate => {
-      const tokens = tokenize(input);
+      const crateId = gcx.crateId.next();
-      const ast = parse(name, tokens, crateId.next());
+
-      const [resolved, crates] = resolve(ast, loadCrate);
+      const tokens = tokenize(inputString);
+      const ast = parse(name, tokens, crateId);
+      const resolved = resolve(gcx, ast);
       console.log(resolved);
 
-      const typecked = typeck(resolved, [...existingCrates, ...crates]);
+      const typecked = typeck(gcx, resolved);
-      return [typecked, crates];
+
+      gcx.depCrates.push(typecked);
+      return typecked;
     },
     () => {
       throw new CompilerError(
@@ -230,6 +245,6 @@ function loadCrate(
       );
     }
   );
-}
+};
 
 main();

src/lower.test.ts

@@ -23,7 +23,7 @@ it("should compute struct layout correctly", () => {
           },
           "types": [
             {
-              "offset": 0,
+              "offset": 4,
               "type": "i32",
             },
           ],

src/lower.ts

@@ -2,6 +2,7 @@ import {
   Crate,
   Expr,
   ExprBlock,
+  Final,
   Folder,
   FunctionDef,
   GlobalItem,
@@ -60,7 +61,7 @@ export type Context = {
   reservedHeapMemoryStart: number;
   funcIndices: ComplexMap<Resolution, FuncOrImport>;
   globalIndices: ComplexMap<Resolution, wasm.GlobalIdx>;
-  crates: Crate<Typecked>[];
+  crates: Crate<Final>[];
   relocations: Relocation[];
   knownDefPaths: ComplexMap<string[], ItemId>;
 };
@@ -154,7 +155,7 @@ function getKnownDefPaths(
   return knows;
 }
 
-export function lower(crates: Crate<Typecked>[]): wasm.Module {
+export function lower(crates: Crate<Final>[]): wasm.Module {
   const knownDefPaths = getKnownDefPaths(crates);
 
   const mod: wasm.Module = {

src/resolve.ts

@@ -17,37 +17,20 @@ import {
   superFoldItem,
   superFoldType,
   ExternItem,
-  Typecked,
-  findCrateItem,
 } from "./ast";
-import { CompilerError, Span, spanMerge } from "./error";
+import { GlobalContext } from "./context";
-import { ComplexMap, Ids, unwrap } from "./utils";
+import { CompilerError, spanMerge } from "./error";
+import { ComplexMap } from "./utils";
 
 const BUILTIN_SET = new Set<string>(BUILTINS);
 
-export type CrateLoader = (
-  name: string,
-  span: Span,
-  crateId: Ids,
-  existingCrates: Crate<Typecked>[]
-) => [Crate<Typecked>, Crate<Typecked>[]];
-
 type Context = {
   ast: Crate<Built>;
-  crates: Crate<Typecked>[];
+  gcx: GlobalContext;
   modContentsCache: ComplexMap<ItemId, Map<string, ItemId>>;
   newItemsById: ComplexMap<ItemId, Item<Resolved>>;
-  crateLoader: CrateLoader;
-  crateId: Ids;
 };
 
-function findItem(cx: Context, id: ItemId): Item<Built> {
-  const crate = unwrap(
-    [cx.ast, ...cx.crates].find((crate) => crate.id === id.crateId)
-  );
-  return findCrateItem(crate, id);
-}
-
 function resolveModItem(
   cx: Context,
   mod: ModItem<Built> | ExternItem,
@@ -64,15 +47,7 @@ function resolveModItem(
   if ("contents" in mod) {
     contents = new Map(mod.contents.map((item) => [item.node.name, item.id]));
   } else {
-    const [loadedCrate, itsDeps] = cx.crateLoader(
+    const loadedCrate = cx.gcx.crateLoader(cx.gcx, item.node.name, item.span);
-      item.node.name,
-      item.span,
-      cx.crateId,
-      cx.crates
-    );
-    cx.crates.push(loadedCrate);
-    cx.crates.push(...itsDeps);
-
     contents = new Map(
       loadedCrate.rootItems.map((item) => [item.node.name, item.id])
     );
@@ -83,30 +58,23 @@ function resolveModItem(
 }
 
 export function resolve(
-  ast: Crate<Built>,
+  gcx: GlobalContext,
-  crateLoader: CrateLoader
+  ast: Crate<Built>
-): [Crate<Resolved>, Crate<Typecked>[]] {
+): Crate<Resolved> {
-  const crateId = new Ids();
-  crateId.next(); // Local crate.
   const cx: Context = {
     ast,
-    crates: [],
+    gcx,
     modContentsCache: new ComplexMap(),
     newItemsById: new ComplexMap(),
-    crateLoader,
-    crateId,
   };
 
   const rootItems = resolveModule(cx, [ast.packageName], ast.rootItems);
-  return [
+  return {
-    {
     id: ast.id,
     itemsById: cx.newItemsById,
     rootItems,
     packageName: ast.packageName,
-    },
+  };
-    cx.crates,
-  ];
 }
 
 function resolveModule(
@@ -280,7 +248,7 @@ function resolveModule(
               lhs.kind === "ident" ? [lhs.value.name] : lhs.segments;
 
             if (res.kind === "item") {
-              const module = findItem(cx, res.id);
+              const module = cx.gcx.findItem(res.id, cx.ast);
 
               if (module.kind === "mod" || module.kind === "extern") {
                 if (typeof expr.field.value === "number") {

src/typeck.ts

@@ -31,9 +31,10 @@ import {
   findCrateItem,
   StructLiteralField,
 } from "./ast";
+import { GlobalContext } from "./context";
 import { CompilerError, Span } from "./error";
 import { printTy } from "./printer";
-import { ComplexMap, unwrap } from "./utils";
+import { ComplexMap } from "./utils";
 
 function mkTyFn(params: Ty[], returnTy: Ty): Ty {
   return { kind: "fn", params, returnTy };
@@ -123,17 +124,15 @@ function lowerAstTyBase(
 }
 
 export function typeck(
-  ast: Crate<Resolved>,
+  gcx: GlobalContext,
-  otherCrates: Crate<Typecked>[]
+  ast: Crate<Resolved>
 ): Crate<Typecked> {
   const itemTys = new ComplexMap<ItemId, Ty | null>();
 
   function typeOfItem(itemId: ItemId, cause: Span): Ty {
     if (itemId.crateId !== ast.id) {
-      const crate = unwrap(
+      const item = gcx.findItem(itemId, ast);
-        otherCrates.find((crate) => crate.id === itemId.crateId)
+
-      );
-      const item = findCrateItem(crate, itemId);
       switch (item.kind) {
         case "function":
         case "import":
@@ -239,23 +238,13 @@ export function typeck(
     );
   }
 
-  function findItem(itemId: ItemId): Item<Resolved> {
-    if (itemId.crateId === ast.id) {
-      return findCrateItem(ast, itemId);
-    }
-    return findCrateItem(
-      unwrap(otherCrates.find((crate) => crate.id === itemId.crateId)),
-      itemId
-    );
-  }
-
   const checker: Folder<Resolved, Typecked> = {
     ...mkDefaultFolder(),
     itemInner(item: Item<Resolved>): Item<Typecked> {
       switch (item.kind) {
         case "function": {
           const fnTy = typeOfItem(item.id, item.span) as TyFn;
-          const body = checkBody(item.node.body, fnTy, typeOfItem, findItem);
+          const body = checkBody(gcx, ast, item.node.body, fnTy, typeOfItem);
 
           const returnType = item.node.returnType && {
             ...item.node.returnType,
@@ -604,10 +593,11 @@ export class InferContext {
 }
 
 export function checkBody(
+  gcx: GlobalContext,
+  ast: Crate<Resolved>,
   body: Expr<Resolved>,
   fnTy: TyFn,
-  typeOfItem: (itemId: ItemId, cause: Span) => Ty,
+  typeOfItem: (itemId: ItemId, cause: Span) => Ty
-  findItem: (itemId: ItemId) => Item<Resolved>
 ): Expr<Typecked> {
   const localTys = [...fnTy.params];
   const loopState: { hasBreak: boolean; loopId: LoopId }[] = [];
@@ -697,7 +687,7 @@ export function checkBody(
                 case "local":
                   break;
                 case "item": {
-                  const item = findItem(res.id);
+                  const item = gcx.findItem(res.id, ast);
                   if (item.kind !== "global") {
                     throw new CompilerError("cannot assign to item", expr.span);
                   }