some stuff

jest.config.js

@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

package.json

@@ -4,14 +4,20 @@
   "description": "",
   "main": "target/index.js",
   "scripts": {
+    "dev": "node-dev --respawn src/index.ts",
     "build": "tsc",
     "fmt": "prettier -w .",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "jest"
   },
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "prettier": "^3.0.0",
+    "@types/jest": "^29.5.3",
+    "jest": "^29.6.1",
+    "node-dev": "^8.0.0",
+    "prettier": "^2.0.0",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.1",
     "typescript": "^5.1.6"
   }
 }

src/__snapshots__/lexer.test.ts.snap

@@ -0,0 +1,84 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`should tokenize an emtpy function 1`] = `
+[
+  {
+    "kind": "kw_function",
+    "span": {
+      "end": 8,
+      "start": 0,
+    },
+  },
+  {
+    "ident": "hello",
+    "kind": "identifier",
+    "span": {
+      "end": 14,
+      "start": 9,
+    },
+  },
+  {
+    "kind": "p_popen",
+    "span": {
+      "end": 15,
+      "start": 14,
+    },
+  },
+  {
+    "kind": "p_pclose",
+    "span": {
+      "end": 16,
+      "start": 15,
+    },
+  },
+  {
+    "kind": "p_bopen",
+    "span": {
+      "end": 18,
+      "start": 17,
+    },
+  },
+  {
+    "kind": "p_bclose",
+    "span": {
+      "end": 19,
+      "start": 18,
+    },
+  },
+]
+`;
+
+exports[`should tokenize hello world 1`] = `
+[
+  {
+    "ident": "print",
+    "kind": "identifier",
+    "span": {
+      "end": 5,
+      "start": 0,
+    },
+  },
+  {
+    "kind": "p_popen",
+    "span": {
+      "end": 6,
+      "start": 5,
+    },
+  },
+  {
+    "kind": "lit_string",
+    "span": {
+      "end": 19,
+      "start": 6,
+    },
+    "value": "hello world",
+  },
+  {
+    "kind": "p_pclose",
+    "span": {
+      "end": 20,
+      "start": 19,
+    },
+  },
+]
+`;

src/ast.ts

@@ -0,0 +1,33 @@
+import { Span } from "./error";
+
+export type ItemKind = {
+    kind: "function",
+    node: FunctionDef,
+};
+
+export type Item = ItemKind & {
+    span: Span,
+}
+
+export type FunctionDef = {
+    name: string,
+    args: FunctionArg[], 
+    body: Expr,
+}
+
+export type FunctionArg = {
+    name: string,
+    span: Span,
+}
+
+export type ExprKind = {
+    kind: "lit_string",
+    value: string,
+} | {
+    kind: "ident",
+    value: string,
+}
+
+export type Expr = ExprKind & {
+    span: Span,
+}

src/error.test.ts

@@ -0,0 +1,12 @@
+import { Span, lines } from "./error";
+
+it("should extract lines correctly", () => {
+  const input = "AAA\nmeow\n:3\n\n";
+
+  const lineSpans = lines(input);
+  const lineContents = lineSpans.map(({ start, end }) =>
+    input.slice(start, end)
+  );
+
+  expect(lineContents).toStrictEqual(["AAA", "meow", ":3", "", ""]);
+});

src/error.ts

@@ -0,0 +1,74 @@
+export type Span = {
+  start: number;
+  end: number;
+};
+
+export class CompilerError extends Error {
+  msg: string;
+  span: Span;
+
+  constructor(msg: string, span: Span) {
+    super(msg);
+    this.msg = msg;
+    this.span = span;
+  }
+}
+
+export function withErrorHandler(input: string, f: () => void) {
+  try {
+    f();
+  } catch (e) {
+    if (e instanceof CompilerError) {
+      renderError(input, e);
+    } else {
+      throw e;
+    }
+  }
+}
+
+function renderError(input: string, e: CompilerError) {
+  const lineSpans = lines(input);
+  const line =
+    e.span.start === Number.MAX_SAFE_INTEGER
+      ? lineSpans[lineSpans.length - 1]
+      : lineSpans.find(
+          (line) => line.start <= e.span.start && line.end >= e.span.start
+        );
+  if (!line) {
+    throw Error(`Span out of bounds: ${e.span.start}..${e.span.end}`);
+  }
+  const lineIdx = lineSpans.indexOf(line);
+  console.error(`error: ${e.message}`);
+
+  console.error(`${lineIdx} | ${spanToSnippet(input, line)}`);
+  const startRelLine =
+    e.span.start === Number.MAX_SAFE_INTEGER ? 0 : e.span.start - line.start;
+  console.error(
+    `${" ".repeat(String(lineIdx).length)}   ${" ".repeat(startRelLine)}^`
+  );
+}
+
+function spanToSnippet(input: string, span: Span): string {
+  if (span.start === Number.MAX_SAFE_INTEGER) {
+    return "";
+  }
+  return input.slice(span.start, span.end);
+}
+
+export function lines(input: string): Span[] {
+  const lines: Span[] = [{ start: 0, end: 0 }];
+
+  for (let i = 0; i < input.length; i++) {
+    if (input[i] === "\n") {
+      lines.push({ start: i + 1, end: i + 1 });
+    } else {
+      lines[lines.length - 1].end++;
+    }
+  }
+
+  return lines;
+}
+
+export function todo(msg: string, span: Span): never {
+  throw new CompilerError(`TODO: ${msg}`, span);
+}

src/index.ts

@@ -1,102 +1,19 @@
+import { withErrorHandler } from "./error";
+import { tokenize } from "./lexer";
+import { parse } from "./parser";
+
 const input = `
-function ok() {}
+function hello() {}
 `;
 
 function main() {
-  const tokens = tokenize(input);
+  withErrorHandler(input, () => {
-  console.log(tokens);
+    const tokens = tokenize(input);
-}
+    console.log(tokens);
 
-type Span = {
+    const ast = parse(tokens);
-  start: number;
+    console.log(ast);
-  len: number;
+  });
-};
-
-type DatalessToken =
-  | "kw_function"
-  | "p_popen"
-  | "p_pclose"
-  | "p_bopen"
-  | "p_bclose";
-
-type TokenKind =
-  | { kind: DatalessToken }
-  | { kind: "identifier"; ident: string };
-
-type Token = TokenKind & {
-  span: Span;
-};
-
-function tokenize(input: string): Token[] {
-  const tokens: Token[] = [];
-  let i = 0;
-
-  finish: while (i < input.length) {
-    const next = input[i];
-    const span: Span = { start: i, len: 1 };
-    switch (next) {
-      case undefined: {
-        break finish;
-      }
-      case "(": {
-        tokens.push({ kind: "p_popen", span });
-        break;
-      }
-      case ")": {
-        tokens.push({ kind: "p_pclose", span });
-        break;
-      }
-      case "{": {
-        tokens.push({ kind: "p_bopen", span });
-        break;
-      }
-      case "}": {
-        tokens.push({ kind: "p_bclose", span });
-        break;
-      }
-      default: {
-        if (isDigit(next)) {
-          throw new Error("digit");
-        } else if (isIdentStart(next)) {
-          while (isIdentContinue(input[i + 1])) {
-            span.len++;
-            i++;
-          }
-          const ident = input.slice(span.start, span.start + span.len);
-          tokens.push({ kind: "identifier", span, ident: ident });
-        } else if (isWhitespace(next)) {
-          // ignore
-        }
-      }
-    }
-
-    i++;
-  }
-
-  return tokens;
-}
-
-function isIdentStart(char: string): boolean {
-  return (
-    (char <= "Z" && char >= "A") || (char <= "z" && char >= "a") || char === "_"
-  );
-}
-
-function isIdentContinue(char: string): boolean {
-  return (
-    (char <= "Z" && char >= "A") ||
-    (char <= "z" && char >= "a") ||
-    char === "_" ||
-    isDigit(char)
-  );
-}
-
-function isDigit(char: string): boolean {
-  return !Number.isNaN(parseInt(char, 10));
-}
-
-function isWhitespace(char: string): boolean {
-  return char === " " || char === "\t" || char === "\n" || char === "\r";
 }
 
 main();

src/lexer.test.ts

@@ -0,0 +1,17 @@
+import { tokenize } from "./lexer";
+
+it('should tokenize an emtpy function', () => {
+    const input = `function hello() {}`;
+
+    const tokens = tokenize(input);
+
+    expect(tokens).toMatchSnapshot();
+});
+
+it('should tokenize hello world', () => {
+    const input = `print("hello world")`;
+
+    const tokens = tokenize(input);
+
+    expect(tokens).toMatchSnapshot();
+});

src/lexer.ts

@@ -0,0 +1,126 @@
+import { CompilerError, Span } from "./error";
+
+export type DatalessToken =
+  | "kw_function"
+  | "kw_let"
+  | "p_popen"
+  | "p_pclose"
+  | "p_bopen"
+  | "p_bclose"
+  | "p_semi";
+
+export type TokenKind =
+  | { kind: DatalessToken }
+  | { kind: "identifier"; ident: string }
+  | { kind: "lit_string"; value: string };
+
+export type Token = TokenKind & {
+  span: Span;
+};
+
+export function tokenize(input: string): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+
+  finish: while (i < input.length) {
+    const next = input[i];
+    const span: Span = { start: i, end: i + 1 };
+    switch (next) {
+      case undefined: {
+        break finish;
+      }
+      case "(": {
+        tokens.push({ kind: "p_popen", span });
+        break;
+      }
+      case ")": {
+        tokens.push({ kind: "p_pclose", span });
+        break;
+      }
+      case "{": {
+        tokens.push({ kind: "p_bopen", span });
+        break;
+      }
+      case "}": {
+        tokens.push({ kind: "p_bclose", span });
+        break;
+      }
+      case ";": {
+        tokens.push({ kind: "p_semi", span });
+        break;
+      }
+      case '"': {
+        while (true) {
+          const next = input[i + 1];
+          span.end++;
+          i++;
+          if (next === '"') {
+            break;
+          }
+          if (next === undefined) {
+            throw new CompilerError(`Unterminated string literal`, span);
+          }
+        }
+        const value = input.slice(span.start + 1, span.end - 1);
+        tokens.push({ kind: "lit_string", span, value });
+        break;
+      }
+      default: {
+        if (isDigit(next)) {
+          throw new Error("digit");
+        } else if (isIdentStart(next)) {
+          while (isIdentContinue(input[i + 1])) {
+            span.end++;
+            i++;
+          }
+          const ident = input.slice(span.start, span.end);
+          let kw = isKeyword(ident);
+          if (kw) {
+            tokens.push({ kind: kw, span });
+          } else {
+            tokens.push({ kind: "identifier", span, ident: ident });
+          }
+        } else if (isWhitespace(next)) {
+          // ignore
+        } else {
+          throw new CompilerError(`Invalid character: \`${next}\``, span);
+        }
+      }
+    }
+
+    i++;
+  }
+
+  return tokens;
+}
+
+function isIdentStart(char: string): boolean {
+  return (
+    (char <= "Z" && char >= "A") || (char <= "z" && char >= "a") || char === "_"
+  );
+}
+
+function isIdentContinue(char: string): boolean {
+  return (
+    (char <= "Z" && char >= "A") ||
+    (char <= "z" && char >= "a") ||
+    char === "_" ||
+    isDigit(char)
+  );
+}
+
+function isDigit(char: string): boolean {
+  return !Number.isNaN(parseInt(char, 10));
+}
+
+function isWhitespace(char: string): boolean {
+  return char === " " || char === "\t" || char === "\n" || char === "\r";
+}
+
+const keywords = new Map<string, DatalessToken>([
+  ["function", "kw_function"],
+  ["let", "kw_let"],
+]);
+function isKeyword(kw: string): DatalessToken | undefined {
+  return keywords.get(kw);
+}

src/parser.ts

@@ -0,0 +1,45 @@
+import { FunctionDef, Item } from "./ast";
+import { CompilerError, todo } from "./error";
+import { Token } from "./lexer";
+
+export function parse(t: Token[]): Item[] {
+    const items: Item[] = [];
+
+    while (t.length > 0) {
+        let item;
+        [t, item] = parseItem(t);
+        items.push(item);
+    }
+
+    return items;
+}
+
+function parseItem(t: Token[]): [Token[], Item] {
+    let next;
+    [t, next] = nextT(t);
+    if (next.kind === "kw_function") {
+
+        const def: FunctionDef = {
+            name: "",
+            args: [],
+            body: todo("todo", next.span)
+        }
+
+        return [t, {kind: "function", node: def, span: next.span}]
+    } else {
+        unexpectedToken(next);
+    }
+}
+
+function nextT(t: Token[]): [Token[], Token] {
+    const next = t[0];
+    if (!next) {
+        throw new CompilerError("unexpected end of file", {start: Number.MAX_SAFE_INTEGER, end: Number.MAX_SAFE_INTEGER})
+    }
+    const rest = t.slice(1);
+    return [rest, next];
+}
+
+function unexpectedToken(token: Token): never {
+    throw new CompilerError("unexpected token", token.span);
+}