Noratrieb/riverdelta
1
0
Fork 0
mirror of https://github.com/Noratrieb/riverdelta.git synced 2026-01-14 08:25:02 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

start lexer

Browse source
This commit is contained in:
nora 2023-07-23 10:48:57 +02:00
commit ef32e646d6
7 changed files with 338 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
/node_modules
/target
*.tsbuildinfo

26
flake.lock generated Normal file
View file

@ -0,0 +1,26 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1690100086,
"narHash": "sha256-ajup699tAa2zw3WdnMAjLnk9SQm0Q8sJHzqeg5td9TQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ffce3e381807c51a98cc62bd1afa251a0456248c",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

36
flake.nix Normal file
View file

@ -0,0 +1,36 @@
{
description = "Example JavaScript development environment for Zero to Nix";
# Flake inputs
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs"; # also valid: "nixpkgs"
};
# Flake outputs
outputs = { self, nixpkgs }:
let
# Systems supported
allSystems = [
"x86_64-linux" # 64-bit Intel/AMD Linux
"aarch64-linux" # 64-bit ARM Linux
"x86_64-darwin" # 64-bit Intel macOS
"aarch64-darwin" # 64-bit ARM macOS
];
# Helper to provide system-specific attributes
forAllSystems = f: nixpkgs.lib.genAttrs allSystems (system: f {
pkgs = import nixpkgs { inherit system; };
});
in
{
# Development environment output
devShells = forAllSystems ({ pkgs }: {
default = pkgs.mkShell {
# The Nix packages provided in the environment
packages = with pkgs; [
nodejs-18_x # Node.js 18, plus npm, npx, and corepack
];
};
});
};
}

45
package-lock.json generated Normal file
View file

@ -0,0 +1,45 @@
{
"name": "tscript",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "tscript",
"version": "1.0.0",
"license": "ISC",
"devDependencies": {
"prettier": "^3.0.0",
"typescript": "^5.1.6"
}
},
"node_modules/prettier": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.0.tgz",
"integrity": "sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g==",
"dev": true,
"bin": {
"prettier": "bin/prettier.cjs"
},
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/prettier/prettier?sponsor=1"
}
},
"node_modules/typescript": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz",
"integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
}
}
}

17
package.json Normal file
View file

@ -0,0 +1,17 @@
{
"name": "tscript",
"version": "1.0.0",
"description": "",
"main": "target/index.js",
"scripts": {
"build": "tsc",
"fmt": "prettier -w .",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"devDependencies": {
"prettier": "^3.0.0",
"typescript": "^5.1.6"
}
}

102
src/index.ts Normal file
View file

@ -0,0 +1,102 @@
const input = `
function ok() {}
`;
function main() {
const tokens = tokenize(input);
console.log(tokens);
}
type Span = {
start: number;
len: number;
};
type DatalessToken =
| "kw_function"
| "p_popen"
| "p_pclose"
| "p_bopen"
| "p_bclose";
type TokenKind =
| { kind: DatalessToken }
| { kind: "identifier"; ident: string };
type Token = TokenKind & {
span: Span;
};
function tokenize(input: string): Token[] {
const tokens: Token[] = [];
let i = 0;
finish: while (i < input.length) {
const next = input[i];
const span: Span = { start: i, len: 1 };
switch (next) {
case undefined: {
break finish;
}
case "(": {
tokens.push({ kind: "p_popen", span });
break;
}
case ")": {
tokens.push({ kind: "p_pclose", span });
break;
}
case "{": {
tokens.push({ kind: "p_bopen", span });
break;
}
case "}": {
tokens.push({ kind: "p_bclose", span });
break;
}
default: {
if (isDigit(next)) {
throw new Error("digit");
} else if (isIdentStart(next)) {
while (isIdentContinue(input[i + 1])) {
span.len++;
i++;
}
const ident = input.slice(span.start, span.start + span.len);
tokens.push({ kind: "identifier", span, ident: ident });
} else if (isWhitespace(next)) {
// ignore
}
}
}
i++;
}
return tokens;
}
function isIdentStart(char: string): boolean {
return (
(char <= "Z" && char >= "A") || (char <= "z" && char >= "a") || char === "_"
);
}
function isIdentContinue(char: string): boolean {
return (
(char <= "Z" && char >= "A") ||
(char <= "z" && char >= "a") ||
char === "_" ||
isDigit(char)
);
}
function isDigit(char: string): boolean {
return !Number.isNaN(parseInt(char, 10));
}
function isWhitespace(char: string): boolean {
return char === " " || char === "\t" || char === "\n" || char === "\r";
}
main();

109
tsconfig.json Normal file
View file

@ -0,0 +1,109 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
"incremental": true /* Save .tsbuildinfo files to allow for incremental compilation of projects. */,
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
"module": "commonjs" /* Specify what module code is generated. */,
"rootDir": "./src" /* Specify the root folder within your source files. */,
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
// "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
// "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
// "resolveJsonModule": true, /* Enable importing .json files. */
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "./target" /* Specify an output folder for all emitted files. */,
// "removeComments": true, /* Disable emitting comments. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": true /* Enable all strict type-checking options. */,
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
"noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
}
}