Noratrieb/riverdelta
1
0
Fork 0
mirror of https://github.com/Noratrieb/riverdelta.git synced 2026-01-14 16:35:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

start implememting crate loading

Browse source
This commit is contained in:
nora 2023-07-31 18:03:26 +02:00
parent a0599342ea
commit b3b0bc2e6f
12 changed files with 249 additions and 165 deletions
Download patch file Download diff file Expand all files Collapse all files

59
src/ast.ts
View file

@ -1,12 +1,6 @@
import { Span } from "./error";
import { LitIntType } from "./lexer";
export type Ast<P extends Phase> = {
rootItems: Item<P>[];
itemsById: Map<ItemId, Item<P>>;
packageName: string;
} & P["typeckResults"];
export type Phase = {
res: unknown;
defPath: unknown;
@ -45,6 +39,7 @@ export type Typecked = {
ty: HasTy;
typeckResults: HasTypeckResults;
};
export type AnyPhase = {
res: No | HasRes;
defPath: No | HasDefPath;
@ -52,6 +47,15 @@ export type AnyPhase = {
typeckResults: No | HasTypeckResults;
};
export type CrateId = number;
export type Crate<P extends Phase> = {
id: CrateId;
rootItems: Item<P>[];
itemsById: Map<ItemId, Item<P>>;
packageName: string;
} & P["typeckResults"];
export type Ident = {
name: string;
span: Span;
@ -80,6 +84,10 @@ export type ItemKind<P extends Phase> =
| {
kind: "mod";
node: ModItem<P>;
}
| {
kind: "extern";
node: ExternItem;
};
export type Item<P extends Phase> = ItemKind<P> & {
@ -123,17 +131,10 @@ export type ImportDef<P extends Phase> = {
export type ModItem<P extends Phase> = {
name: string;
modKind: ModItemKind<P>;
contents: Item<P>[];
};
export type ModItemKind<P extends Phase> =
| {
kind: "inline";
contents: Item<P>[];
}
| {
kind: "extern";
};
export type ExternItem = { name: string };
export type ExprEmpty = { kind: "empty" };
@ -486,7 +487,7 @@ export const TY_I32: Ty = { kind: "i32" };
export const TY_NEVER: Ty = { kind: "never" };
export type TypeckResults = {
main: Resolution;
main: Resolution | undefined;
};
// folders
@ -521,7 +522,7 @@ export function mkDefaultFolder<
newItemsById: new Map(),
item(item) {
const newItem = this.itemInner(item);
this.newItemsById.set(item.id, newItem);
this.newItemsById.set(newItem.id, newItem);
return newItem;
},
itemInner(_item) {
@ -534,14 +535,15 @@ export function mkDefaultFolder<
}
export function foldAst<From extends Phase, To extends Phase>(
ast: Ast<From>,
ast: Crate<From>,
folder: Folder<From, To>
): Ast<To> {
): Crate<To> {
if ((folder.item as any)[ITEM_DEFAULT] !== ITEM_DEFAULT) {
throw new Error("must not override `item` on folders");
}
return {
id: ast.id,
rootItems: ast.rootItems.map((item) => folder.item(item)),
itemsById: folder.newItemsById,
typeckResults: "typeckResults" in ast ? ast.typeckResults : undefined,
@ -603,29 +605,18 @@ export function superFoldItem<From extends Phase, To extends Phase>(
};
}
case "mod": {
let kind: ModItemKind<To>;
const { modKind: itemKind } = item.node;
switch (itemKind.kind) {
case "inline":
kind = {
kind: "inline",
contents: itemKind.contents.map((item) => folder.item(item)),
};
break;
case "extern":
kind = { kind: "extern" };
break;
}
return {
...item,
kind: "mod",
node: {
name: item.node.name,
modKind: kind,
contents: item.node.contents.map((item) => folder.item(item)),
},
};
}
case "extern": {
return { ...item, kind: "extern" };
}
}
}

8
src/error.ts
View file

@ -23,19 +23,19 @@ export class CompilerError extends Error {
}
}
export function withErrorHandler(input: string, f: () => void): void {
export function withErrorPrinter(input: string, filename: string, f: () => void): void {
try {
f();
} catch (e) {
if (e instanceof CompilerError) {
renderError(input, e);
renderError(input, filename, e);
} else {
throw e;
}
}
}
function renderError(input: string, e: CompilerError) {
function renderError(input: string, filename: string, e: CompilerError) {
const lineSpans = lines(input);
const line =
e.span.start === Number.MAX_SAFE_INTEGER
@ -49,6 +49,8 @@ function renderError(input: string, e: CompilerError) {
const lineIdx = lineSpans.indexOf(line);
const lineNo = lineIdx + 1;
console.error(`error: ${e.message}`);
console.error(` --> ${filename}:${lineNo}`);
console.error(`${lineNo} | ${spanToSnippet(input, line)}`);
const startRelLine =

69
src/index.ts
View file

@ -1,4 +1,4 @@
import { withErrorHandler } from "./error";
import { CompilerError, Span, withErrorPrinter } from "./error";
import { isValidIdent, tokenize } from "./lexer";
import { lower as lowerToWasm } from "./lower";
import { parse } from "./parser";
@ -9,23 +9,23 @@ import { writeModuleWatToString } from "./wasm/wat";
import fs from "fs";
import path from "path";
import { exec } from "child_process";
import { Ast, Built, Typecked } from "./ast";
import { Crate, Built, Typecked } from "./ast";
import { Ids } from "./utils";
const INPUT = `
function main() = (
prIntln(0);
);
extern mod std;
function prIntln(x: Int) = (
print("\n");
function main() = (
std.pow(10, 2);
);
`;
function main() {
let filename: string;
let input: string;
let packageName: string;
if (process.argv.length > 2) {
const filename = process.argv[2];
filename = process.argv[2];
if (path.extname(filename) !== ".nil") {
console.error(
`error: filename must have \`.nil\` extension: \`${filename}\``
@ -36,6 +36,7 @@ function main() {
input = fs.readFileSync(filename, { encoding: "utf-8" });
packageName = path.basename(filename, ".nil");
} else {
filename = "<hardcoded>";
input = INPUT;
packageName = "test";
}
@ -49,14 +50,14 @@ function main() {
console.log(`package name: '${packageName}'`);
withErrorHandler(input, () => {
withErrorPrinter(input, filename, () => {
const start = Date.now();
const tokens = tokenize(input);
console.log("-----TOKENS------------");
console.log(tokens);
const ast: Ast<Built> = parse(packageName, tokens);
const ast: Crate<Built> = parse(packageName, tokens, 0);
console.log("-----AST---------------");
console.dir(ast.rootItems, { depth: 50 });
@ -66,17 +67,17 @@ function main() {
console.log(printed);
console.log("-----AST resolved------");
const resolved = resolve(ast);
const [resolved, crates] = resolve(ast, loadCrate);
const resolvedPrinted = printAst(resolved);
console.log(resolvedPrinted);
console.log("-----AST typecked------");
const typecked: Ast<Typecked> = typeck(resolved);
const typecked: Crate<Typecked> = typeck(resolved);
const typeckPrinted = printAst(typecked);
console.log(typeckPrinted);
console.log("-----wasm--------------");
const wasmModule = lowerToWasm(typecked);
const wasmModule = lowerToWasm([typecked, ...crates]);
const moduleStringColor = writeModuleWatToString(wasmModule, true);
const moduleString = writeModuleWatToString(wasmModule);
@ -105,4 +106,46 @@ function main() {
});
}
function loadCrate(
name: string,
span: Span,
crateId: Ids,
existingCrates: Crate<Typecked>[]
): [Crate<Typecked>, Crate<Typecked>[]] {
// We really, really want a good algorithm for finding crates.
// But right now we just look for files in the CWD.
const existing = existingCrates.find((crate) => crate.packageName === name);
if (existing) {
return [existing, []];
}
const filename = `${name}.nil`;
let input;
try {
input = fs.readFileSync(filename, { encoding: "utf-8" });
} catch (e) {
throw new CompilerError(
`failed to load ${name}, could not fine \`${filename}\``,
span
);
}
try {
const tokens = tokenize(input);
const ast = parse(name, tokens, crateId.next());
const [resolved, crates] = resolve(ast, loadCrate);
const typecked = typeck(resolved);
return [typecked, crates];
} catch (e) {
withErrorPrinter(input, filename, () => {
throw e;
});
throw new CompilerError(
`failed to load crate ${name}: crate contains errors`,
span
);
}
}
main();

3
src/lexer.ts
View file

@ -230,10 +230,7 @@ export function tokenize(input: string): Token[] {
}
let type: LitIntType = "Int";
console.log(input[i + 2]);
if (input[i + 1] === "_" && isIdentStart(input[i + 2])) {
console.log("yes", input.slice(i + 2, i + 5));
if (input.slice(i + 2, i + 5) === "Int") {
i += 4;
type = "Int";

23
src/lower.ts
View file

@ -1,5 +1,5 @@
import {
Ast,
Crate,
Expr,
ExprBlock,
FunctionDef,
@ -39,7 +39,7 @@ export type Context = {
funcTypes: ComplexMap<wasm.FuncType, wasm.TypeIdx>;
reservedHeapMemoryStart: number;
funcIndices: ComplexMap<Resolution, FuncOrImport>;
ast: Ast<Typecked>;
crates: Crate<Typecked>[];
relocations: Relocation[];
};
@ -89,7 +89,7 @@ function appendData(cx: Context, newData: Uint8Array): number {
}
}
export function lower(ast: Ast<Typecked>): wasm.Module {
export function lower(crates: Crate<Typecked>[]): wasm.Module {
const mod: wasm.Module = {
types: [],
funcs: [],
@ -119,7 +119,7 @@ export function lower(ast: Ast<Typecked>): wasm.Module {
funcTypes: new ComplexMap(),
funcIndices: new ComplexMap(),
reservedHeapMemoryStart: 0,
ast,
crates,
relocations: [],
};
@ -135,15 +135,12 @@ export function lower(ast: Ast<Typecked>): wasm.Module {
break;
}
case "mod": {
if (item.node.modKind.kind === "inline") {
lowerMod(item.node.modKind.contents);
}
lowerMod(item.node.contents);
}
}
});
}
lowerMod(ast.rootItems);
crates.forEach((ast) => lowerMod(ast.rootItems));
const HEAP_ALIGN = 0x08;
cx.reservedHeapMemoryStart =
@ -151,7 +148,7 @@ export function lower(ast: Ast<Typecked>): wasm.Module {
? (mod.datas[0].init.length + (HEAP_ALIGN - 1)) & ~(HEAP_ALIGN - 1)
: 0;
addRt(cx, ast);
addRt(cx, crates);
// THE LINKER
const offset = cx.mod.imports.length;
@ -863,14 +860,16 @@ function todo(msg: string): never {
}
// Make the program runnable using wasi-preview-1
function addRt(cx: Context, ast: Ast<Typecked>) {
function addRt(cx: Context, crates: Crate<Typecked>[]) {
const { mod } = cx;
const crate0 = unwrap(crates.find((crate) => crate.id === 0));
const mainCall: wasm.Instr = { kind: "call", func: 9999999 };
cx.relocations.push({
kind: "funccall",
instr: mainCall,
res: ast.typeckResults.main,
res: unwrap(crate0.typeckResults.main),
});
const start: wasm.Func = {

30
src/parser.ts
View file

@ -1,7 +1,7 @@
import {
ARITH_FACTOR_KINDS,
ARITH_TERM_KINDS,
Ast,
Crate,
BinaryKind,
COMPARISON_KINDS,
mkDefaultFolder,
@ -27,6 +27,7 @@ import {
superFoldItem,
Built,
Parsed,
ExternItem,
} from "./ast";
import { CompilerError, Span, spanMerge } from "./error";
import { BaseToken, Token, TokenIdent, TokenLitString } from "./lexer";
@ -34,7 +35,11 @@ import { Ids } from "./utils";
type Parser<T> = (t: Token[]) => [Token[], T];
export function parse(packageName: string, t: Token[]): Ast<Built> {
export function parse(
packageName: string,
t: Token[],
crateId: number
): Crate<Built> {
const items: Item<Parsed>[] = [];
while (t.length > 0) {
@ -43,7 +48,7 @@ export function parse(packageName: string, t: Token[]): Ast<Built> {
items.push(item);
}
const ast = buildAst(packageName, items);
const ast: Crate<Built> = buildCrate(packageName, items, crateId);
validateAst(ast);
@ -137,14 +142,13 @@ function parseItem(t: Token[]): [Token[], Item<Parsed>] {
let name;
[t, name] = expectNext<TokenIdent>(t, "identifier");
const node: ModItem<Parsed> = {
const node: ExternItem = {
name: name.ident,
modKind: { kind: "extern" },
};
[t] = expectNext(t, ";");
return [t, { kind: "mod", node, span: name.span, id: 0 }];
return [t, { kind: "extern", node, span: name.span, id: 0 }];
} else if (tok.kind === "mod") {
let name;
[t, name] = expectNext<TokenIdent>(t, "identifier");
@ -165,7 +169,7 @@ function parseItem(t: Token[]): [Token[], Item<Parsed>] {
const node: ModItem<Parsed> = {
name: name.ident,
modKind: { kind: "inline", contents },
contents,
};
return [t, { kind: "mod", node, span: name.span, id: 0 }];
@ -658,7 +662,7 @@ function unexpectedToken(token: Token, expected: string): never {
throw new CompilerError(`unexpected token, expected ${expected}`, token.span);
}
function validateAst(ast: Ast<Built>) {
function validateAst(ast: Crate<Built>) {
const seenItemIds = new Set();
const validator: Folder<Built, Built> = {
@ -719,11 +723,16 @@ function validateAst(ast: Ast<Built>) {
foldAst(ast, validator);
}
function buildAst(packageName: string, rootItems: Item<Parsed>[]): Ast<Built> {
function buildCrate(
packageName: string,
rootItems: Item<Parsed>[],
crateId: number
): Crate<Built> {
const itemId = new Ids();
const loopId = new Ids();
const ast: Ast<Built> = {
const ast: Crate<Built> = {
id: crateId,
rootItems,
itemsById: new Map(),
packageName,
@ -733,7 +742,6 @@ function buildAst(packageName: string, rootItems: Item<Parsed>[]): Ast<Built> {
...mkDefaultFolder(),
itemInner(item: Item<Parsed>): Item<Built> {
const id = itemId.next();
ast.itemsById.set(id, item);
return { ...superFoldItem(item, this), id };
},
expr(expr: Expr<Parsed>): Expr<Built> {

16
src/printer.ts
View file

@ -1,6 +1,6 @@
import {
AnyPhase,
Ast,
Crate,
Expr,
FunctionDef,
IdentWithRes,
@ -15,7 +15,7 @@ import {
tyIsUnit,
} from "./ast";
export function printAst(ast: Ast<AnyPhase>): string {
export function printAst(ast: Crate<AnyPhase>): string {
return ast.rootItems.map(printItem).join("\n");
}
@ -39,6 +39,9 @@ function printItem(item: Item<AnyPhase>): string {
case "mod": {
return id + printMod(item.node);
}
case "extern": {
return id + `extern mod ${item.node.name};`;
}
}
}
@ -73,14 +76,7 @@ function printImportDef(def: ImportDef<AnyPhase>): string {
}
function printMod(mod: ModItem<AnyPhase>): string {
switch (mod.modKind.kind) {
case "inline":
return `mod ${mod.name} (\n${mod.modKind.contents
.map(printItem)
.join("\n ")});`;
case "extern":
return `extern mod ${mod.name};`;
}
return `mod ${mod.name} (\n${mod.contents.map(printItem).join("\n ")});`;
}
function printExpr(expr: Expr<AnyPhase>, indent: number): string {

122
src/resolve.ts
View file

@ -1,5 +1,5 @@
import {
Ast,
Crate,
BUILTINS,
Built,
BuiltinName,
@ -16,56 +16,89 @@ import {
superFoldExpr,
superFoldItem,
superFoldType,
ExternItem,
Typecked,
} from "./ast";
import { CompilerError, spanMerge, todo } from "./error";
import { unwrap } from "./utils";
import { CompilerError, Span, spanMerge } from "./error";
import { Ids, unwrap } from "./utils";
const BUILTIN_SET = new Set<string>(BUILTINS);
export type CrateLoader = (
name: string,
span: Span,
crateId: Ids,
existingCrates: Crate<Typecked>[]
) => [Crate<Typecked>, Crate<Typecked>[]];
type Context = {
ast: Ast<Built>;
ast: Crate<Built>;
crates: Crate<Typecked>[];
modContentsCache: Map<ItemId, Map<string, ItemId>>;
newItemsById: Map<ItemId, Item<Resolved>>;
crateLoader: CrateLoader;
crateId: Ids;
};
function resolveModItem(
cx: Context,
mod: ModItem<Built>,
modId: ItemId,
mod: ModItem<Built> | ExternItem,
item: Item<Built>,
name: string
): ItemId | undefined {
const cachedContents = cx.modContentsCache.get(modId);
const cachedContents = cx.modContentsCache.get(item.id);
if (cachedContents) {
return cachedContents.get(name);
}
switch (mod.modKind.kind) {
case "inline": {
const contents = new Map(
mod.modKind.contents.map((item) => [item.node.name, item.id])
);
cx.modContentsCache.set(modId, contents);
return contents.get(name);
}
case "extern": {
todo("extern mod items");
}
let contents: Map<string, ItemId>;
if ("contents" in mod) {
contents = new Map(mod.contents.map((item) => [item.node.name, item.id]));
} else {
const [loadedCrate, itsDeps] = cx.crateLoader(
item.node.name,
item.span,
cx.crateId,
cx.crates
);
cx.crates.push(loadedCrate);
cx.crates.push(...itsDeps);
contents = new Map(
loadedCrate.rootItems.map((item) => [item.node.name, item.id])
);
}
cx.modContentsCache.set(item.id, contents);
return contents.get(name);
}
export function resolve(ast: Ast<Built>): Ast<Resolved> {
export function resolve(
ast: Crate<Built>,
crateLoader: CrateLoader
): [Crate<Resolved>, Crate<Typecked>[]] {
const crateId = new Ids();
crateId.next(); // Local crate.
const cx: Context = {
ast,
crates: [],
modContentsCache: new Map(),
newItemsById: new Map(),
crateLoader,
crateId,
};
const rootItems = resolveModule(cx, [ast.packageName], ast.rootItems);
return {
itemsById: cx.newItemsById,
rootItems,
packageName: ast.packageName,
};
return [
{
id: ast.id,
itemsById: cx.newItemsById,
rootItems,
packageName: ast.packageName,
},
cx.crates,
];
}
function resolveModule(
@ -129,7 +162,7 @@ function resolveModule(
const resolver: Folder<Built, Resolved> = {
...mkDefaultFolder(),
itemInner(item) {
itemInner(item): Item<Resolved> {
const defPath = [...modName, item.node.name];
switch (item.kind) {
@ -162,20 +195,23 @@ function resolveModule(
};
}
case "mod": {
if (item.node.modKind.kind === "inline") {
const contents = resolveModule(
cx,
defPath,
item.node.modKind.contents
);
return {
...item,
kind: "mod",
node: { ...item.node, modKind: { kind: "inline", contents } },
defPath,
};
}
break;
const contents = resolveModule(cx, defPath, item.node.contents);
return {
...item,
kind: "mod",
node: { ...item.node, contents },
defPath,
};
}
case "extern": {
const node: ExternItem = {
...item.node,
};
return {
...item,
node,
defPath,
};
}
}
@ -230,7 +266,11 @@ function resolveModule(
if (res.kind === "item") {
const module = unwrap(cx.ast.itemsById.get(res.id));
if (module.kind === "mod") {
console.log("nested", module.kind, res.id, cx.ast.itemsById);
if (module.kind === "mod" || module.kind === "extern") {
console.log("resolve");
if (typeof expr.field.value === "number") {
throw new CompilerError(
"module contents cannot be indexed with a number",
@ -241,7 +281,7 @@ function resolveModule(
const pathResItem = resolveModItem(
cx,
module.node,
module.id,
module,
expr.field.value
);
if (pathResItem === undefined) {

72
src/typeck.ts
View file

@ -1,5 +1,5 @@
import {
Ast,
Crate,
BuiltinName,
COMPARISON_KINDS,
mkDefaultFolder,
@ -14,7 +14,6 @@ import {
ItemId,
LOGICAL_KINDS,
LoopId,
ModItemKind,
Resolution,
Resolved,
Ty,
@ -116,7 +115,7 @@ function lowerAstTyBase(
}
}
export function typeck(ast: Ast<Resolved>): Ast<Typecked> {
export function typeck(ast: Crate<Resolved>): Crate<Typecked> {
const itemTys = new Map<number, Ty | null>();
function typeOfItem(index: ItemId, cause: Span): Ty {
const item = unwrap(ast.itemsById.get(index));
@ -162,7 +161,13 @@ export function typeck(ast: Ast<Resolved>): Ast<Typecked> {
}
case "mod": {
throw new CompilerError(
`module ${item.node.name} is not a type`,
`module ${item.node.name} cannot be used as a type or value`,
cause
);
}
case "extern": {
throw new CompilerError(
`extern declaration ${item.node.name} cannot be used as a type or value`,
cause
);
}
@ -297,30 +302,20 @@ export function typeck(ast: Ast<Resolved>): Ast<Typecked> {
};
}
case "mod": {
switch (item.node.modKind.kind) {
case "inline": {
const modKind: ModItemKind<Typecked> = {
kind: "inline",
contents: item.node.modKind.contents.map((item) =>
this.item(item)
),
};
return {
...item,
node: {
...item.node,
modKind,
},
};
}
case "extern":
// Nothing to check.
return {
...item,
node: { ...item.node, modKind: { ...item.node.modKind } },
};
}
return {
...item,
node: {
...item.node,
contents: item.node.contents.map((item) => this.item(item)),
},
};
}
case "extern": {
// Nothing to check.
return {
...item,
node: { ...item.node },
};
}
}
},
@ -355,16 +350,19 @@ export function typeck(ast: Ast<Resolved>): Ast<Typecked> {
return false;
});
if (!main) {
throw new CompilerError(`\`main\` function not found`, {
start: 0,
end: 1,
});
}
if (ast.id === 0) {
// Only the final id=0 crate needs and cares about main.
if (!main) {
throw new CompilerError(`\`main\` function not found`, {
start: 0,
end: 1,
});
}
typecked.typeckResults = {
main: { kind: "item", id: main.id },
};
typecked.typeckResults = {
main: { kind: "item", id: main.id },
};
}
return typecked;
}

2
src/utils.ts
View file

@ -3,7 +3,7 @@ export function encodeUtf8(s: string): Uint8Array {
}
export class Ids {
nextId = 0;
private nextId = 0;
public next(): number {
return this.nextId++;