Noratrieb/Hunterlang
1
0
Fork 0
mirror of https://github.com/Noratrieb/Hunterlang.git synced 2026-01-14 12:35:01 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

parser rewrite wip

Browse source
This commit is contained in:
nora 2021-01-16 23:33:16 +01:00
parent d7c9aeb127
commit 7d72665c74
5 changed files with 192 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/main/java/com/github/nilstrieb/hunterlang/hllibrary/FunctionArgLookup.java
View file

@ -2,13 +2,10 @@ package com.github.nilstrieb.hunterlang.hllibrary;
public class FunctionArgLookup {
public static int argLookup(String lib, String name){
return switch (lib){
case "Leorio" -> switch (name){
case "say" -> 1;
case "listen" -> 0;
default -> 0;
};
public static int argLookup(String name) {
return switch (name) {
case "Leorio.say" -> 1;
case "Leorio.listen" -> 0;
default -> 0;
};
}

35
src/main/java/com/github/nilstrieb/hunterlang/lexer/LexToken.java
View file

@ -1,35 +1,52 @@
package com.github.nilstrieb.hunterlang.lexer;
import com.github.nilstrieb.hunterlang.hllibrary.FunctionArgLookup;
import com.github.nilstrieb.hunterlang.lib.ConsoleColors;
import com.github.nilstrieb.hunterlang.parser.ParseTreeBodyNode;
import com.github.nilstrieb.hunterlang.parser.ParseTreeNode;
public class LexToken {
WordType key;
String value;
private int argOverrideValue = -1;
public LexToken(WordType key, String value) {
this.key = key;
this.value = value;
if (key.expectsPostArg() == -1) {
if (key == WordType.LIBFUNCCALL) {
//a function. will only support lib functions
argOverrideValue = FunctionArgLookup.argLookup(value);
}
}
}
public LexToken(WordType key) {
this(key, "");
}
public int expectsPostArgCount() {
return key.expectsPostArg();
public boolean hasPostfix() {
if(argOverrideValue == -1){
return key.expectsPostArg() > 0;
} else {
return argOverrideValue > 0;
}
}
public boolean expectsPreArg() {
public boolean hasPrefix() {
return key.expectsPreArg() > 0;
}
public ParseTreeNode toNode(){
public ParseTreeNode toNode() {
return new ParseTreeNode(key, value);
}
@Override
public String toString() {
return ConsoleColors.PURPLE_BOLD + "T" + ConsoleColors.BLUE_BOLD + "{Key=" +
return ConsoleColors.PURPLE_BOLD + "T" + ConsoleColors.BLUE_BOLD + "{Key=" +
ConsoleColors.RED_BOLD_BRIGHT + key +
ConsoleColors.BLUE_BOLD + ", value=" +
ConsoleColors.YELLOW + value + ConsoleColors.BLUE_BOLD + "}" + ConsoleColors.RESET;
@ -42,4 +59,12 @@ public class LexToken {
public String getValue() {
return value;
}
public int postfixCount() {
if(argOverrideValue == -1){
return key.expectsPostArg();
} else {
return argOverrideValue;
}
}
}

25
src/main/java/com/github/nilstrieb/hunterlang/lexer/Lexer.java
View file

@ -13,7 +13,7 @@ public class Lexer {
private static final String MEMCALL = "killua";
public static final String ASSIGNMENT = "hunts";
public static final String IF = "Gon";
public static final String IF = "Gon wants";
public static final String WANTS = "wants";
public static final String ELSE = "got";
public static final String BOPEN = "{";
@ -38,6 +38,7 @@ public class Lexer {
private ArrayList<LexToken> temp;
public ArrayList<LexToken> lex(String code) {
System.out.println(ConsoleColors.GREEN_BACKGROUND + ConsoleColors.BLACK_BOLD + "------START LEXER------" + ConsoleColors.RESET);
tokens = new ArrayList<>();
temp = new ArrayList<>();
@ -124,8 +125,7 @@ public class Lexer {
else if (sub.matches(LIBFUNCCALL_REGEX)) {
String libName = sub.replaceAll(LIBFUNCCALL_REGEX, "$1");
String funcName = sub.replaceAll(LIBFUNCCALL_REGEX, "$2");
addToken(WordType.LIB, libName);
addToken(WordType.FUNCCALL, funcName);
addToken(WordType.LIBFUNCCALL, libName + "." + funcName);
j += libName.length(); //lib name
j += 1 + 4 + 1; // space + does + space
j += funcName.length() - 1; //func name
@ -137,6 +137,7 @@ public class Lexer {
temp.clear();
}
System.out.println(ConsoleColors.GREEN_BACKGROUND + ConsoleColors.BLACK_BOLD + "------STOP LEXER------" + ConsoleColors.RESET);
return tokens;
}
@ -152,11 +153,13 @@ public class Lexer {
Lexer l = new Lexer();
String assign = """
killua70 hunts 3 > 0
killua0 hunts 3
killua0 hunts -3.4 #hunts nothing
killua1 hunts "hallo"
#comment""";
String ifs = """
killua0 hunts 3 > 0
Gon wants false {
Leorio does say "false"
}
@ -165,19 +168,29 @@ public class Lexer {
} got {
Leorio does say "small killua"
}
Gon wants true {
Gon wants true {
1
}
}
""";
String sif = """
Gon wants true {
Leorio does say "false"
""";
String hierarchy = """
killua0 hunts 3 > 3
""";
ArrayList<LexToken> tokens = l.lex(hierarchy);
ArrayList<LexToken> tokens = l.lex(sif);
Parser p = new Parser();
try {
ArrayList<ParseTreeNode> nodes = p.parse(tokens);
for (ParseTreeNode node : nodes) {
System.out.println(node);
for (ParseTreeNode parseTreeNode : nodes) {
System.out.println(parseTreeNode);
}
} catch (ParseException e) {
e.printStackTrace();

6
src/main/java/com/github/nilstrieb/hunterlang/lexer/WordType.java
View file

@ -1,5 +1,7 @@
package com.github.nilstrieb.hunterlang.lexer;
import com.github.nilstrieb.hunterlang.hllibrary.FunctionArgLookup;
public enum WordType {
ASSIGNMENT(1, 1), //memory, value
MEMCALL(1), //adress
@ -14,15 +16,13 @@ public enum WordType {
GTHAN(1, 1),
LTHAN(1, 1),
EQUALS(1, 1),
FUNCCALL(-1), //args
LIB(1), //function
MINUS(1, 1),
PLUS(1, 1),
MULTIPLY(1, 1),
DIVIDE(1, 1),
MOD(1, 1),
NEGATIVE(1),
EMPTY;
EMPTY, LIBFUNCCALL(-1);
private int postArgAmount;
private int preArgAmount;

222
src/main/java/com/github/nilstrieb/hunterlang/parser/Parser.java
View file

@ -1,116 +1,166 @@
package com.github.nilstrieb.hunterlang.parser;
import com.github.nilstrieb.hunterlang.hllibrary.FunctionArgLookup;
import com.github.nilstrieb.hunterlang.lexer.LexToken;
import com.github.nilstrieb.hunterlang.lexer.WordType;
import com.github.nilstrieb.hunterlang.lib.ConsoleColors;
import com.sun.source.tree.LambdaExpressionTree;
import java.util.ArrayList;
import java.util.ListIterator;
public class Parser {
private ListIterator<LexToken> iterator;
private LexToken currentToken = null;
private LexToken prevToken = null;
private LexToken nextToken = null;
private ParseTreeNode prevNode;
public ArrayList<ParseTreeNode> parse(ArrayList<LexToken> tokens) throws ParseException {
System.out.println(ConsoleColors.GREEN_BACKGROUND + ConsoleColors.BLACK_BOLD + "------START PARSER------" + ConsoleColors.RESET);
ArrayList<ParseTreeNode> statements = new ArrayList<>();
iterator = tokens.listIterator();
currentToken = iterator.next();
if (iterator.hasNext()) {
nextToken = iterator.next();
ArrayList<ArrayList<LexToken>> split = splitStatements(tokens);
for (ArrayList<LexToken> lexTokens : split) {
System.out.println(lexTokens);
}
//goes through each statement. On pass -> one statement
do {
System.out.println(ConsoleColors.GREEN_BRIGHT + "NEXT STATEMENT");
ArrayList<ParseTreeNode> list = new ArrayList<>();
prevNode = new ParseTreeNode();
ParseTreeNode parent = null;
for (ArrayList<LexToken> statement : split) {
ListIterator<LexToken> iterator = statement.listIterator();
//start with first token
ParseTreeNode startNode = currentToken.toNode();
LexToken current = iterator.next();
//if it wants a pre argument theres an error
if (currentToken.expectsPreArg()) {
throw new ParseException("previous expression expected, doesn't exist");
switch (current.getKey()) {
case MEMCALL -> {
//assignment
int assignmentIndex = 0;
while (iterator.hasNext()) {
assignmentIndex++;
current = iterator.next();
if (current.getKey() == WordType.ASSIGNMENT) {
break;
}
}
parent = current.toNode();
parent.addChild(evaluate(new ArrayList<>(statement.subList(0, assignmentIndex))));
parent.addChild(evaluate(new ArrayList<>(statement.subList(assignmentIndex + 1, statement.size()))));
}
case IF, WANTS -> {
parent = new ParseTreeBodyNode(current.toNode());
ParseTreeBodyNode parentBN = (ParseTreeBodyNode) parent;
while (current.getKey() != WordType.BOPEN) {
current = iterator.next();
parent.addChild(current.toNode());
}
int brackets = 0;
while ((brackets != 0 || current.getKey() != WordType.BCLOSE) && iterator.hasNext()) {
current = iterator.next();
parentBN.addToken(current);
if (current.getKey() == WordType.BOPEN) {
brackets++;
} else if (current.getKey() == WordType.BCLOSE) {
brackets--;
}
}
parentBN.parse();
}
case ELSE -> {
}
case LIBFUNCCALL -> {
}
case BCLOSE -> {
}
default -> throw new ParseException("Unexpected value at start of statement: " + current.getKey());
}
//calculate the arguments of the start node, parsing the whole statement
doStatement(startNode, true);
System.out.println(prevNode);
statements.add(prevNode);
} while (iterator.hasNext());
list.add(parent);
}
System.out.println(ConsoleColors.GREEN_BACKGROUND + ConsoleColors.BLACK_BOLD + "------STOP PARSER------" + ConsoleColors.RESET);
return statements;
return list;
}
private void doStatement(ParseTreeNode node, boolean isParent) throws ParseException {
if (node.getKey() == WordType.MINUS && prevNode.getKey() != WordType.NUMBER) {
node.setKey(WordType.NEGATIVE);
}
/**
* Evalues an expression. For example something like killuakillua3+4 > 6
*
* @param tokens The tokens
* @return The parent node
*/
private ParseTreeNode evaluate(ArrayList<LexToken> tokens) throws ParseException {
ListIterator<LexToken> iterator = tokens.listIterator();
LexToken prev;
LexToken curr;
LexToken next;
ParseTreeNode currentParent;
if (node.expectsPreArg()) {
node.addChild(prevNode, true);
}
int expectedArg = node.expectsPostArgCount();
if (expectedArg == -1) {
if (node.getKey() == WordType.FUNCCALL) {
//a function. will only support lib functions
System.out.println(currentToken);
expectedArg = FunctionArgLookup.argLookup(prevToken.getValue(), currentToken.getValue());
} else {
ParseTreeBodyNode bNode = new ParseTreeBodyNode(node);
node = bNode;
fillBody(bNode);
}
}
if (isParent) {
prevNode = node;
}
nextToken();
if (expectedArg != 0) {
for (int i = 0; i < expectedArg; i++) {
ParseTreeNode child = currentToken.toNode();
doStatement(child, false);
node.addChild(child);
}
}
if (currentToken != null && currentToken.expectsPreArg()) {
doStatement(currentToken.toNode(), true);
}
}
private void fillBody(ParseTreeBodyNode node) throws ParseException {
int level = 0;
nextToken();
while (currentToken.getKey() != WordType.BCLOSE || level != 0) {
if (currentToken.getKey() == WordType.BOPEN) {
level++;
} else if (currentToken.getKey() == WordType.BCLOSE) {
level--;
}
node.addToken(currentToken);
nextToken();
}
node.parse();
}
private void nextToken() {
prevToken = currentToken;
currentToken = nextToken;
prev = iterator.next();
if (iterator.hasNext()) {
nextToken = iterator.next();
curr = iterator.next();
} else {
//1 token
return prev.toNode();
}
if (iterator.hasNext()) {
next = iterator.next();
} else {
//2 tokens
currentParent = prev.toNode();
currentParent.addChild(curr.toNode());
return currentParent;
}
if (curr.hasPrefix()) {
currentParent = curr.toNode();
currentParent.addChild(prev.toNode(), true);
currentParent.addChild(evaluate(new ArrayList<>(tokens.subList(2, tokens.size()))));
} else if (prev.postfixCount() == 1) {
currentParent = prev.toNode();
currentParent.addChild(evaluate(new ArrayList<>(tokens.subList(1, tokens.size()))));
} else {
throw new ParseException("more than one arg: " + prev.postfixCount() + " thrown on: " + prev + " all: " + tokens);
}
return currentParent;
}
private ArrayList<ArrayList<LexToken>> splitStatements(ArrayList<LexToken> tokens) throws ParseException {
System.out.println(ConsoleColors.YELLOW_BACKGROUND_BRIGHT + ConsoleColors.BLACK_BOLD + "SPLITTER" + ConsoleColors.RESET);
ArrayList<ArrayList<LexToken>> tokensSplit = new ArrayList<>();
tokensSplit.add(new ArrayList<>());
ListIterator<LexToken> iterator = tokens.listIterator();
if (tokens.size() < 2) {
throw new ParseException("code has to consist of at least 2 tokens");
}
LexToken prev;
LexToken curr = iterator.next();
tokensSplit.get(0).add(curr);
int i = 0;
int brackets = 0;
while (iterator.hasNext()) {
prev = curr;
curr = iterator.next();
if (curr.getKey() == WordType.BOPEN) {
brackets++;
}
if (brackets == 0 && !prev.hasPostfix() && !curr.hasPrefix()) {
//when there is no connection between two tokens, they are part of seperate statements
tokensSplit.add(new ArrayList<>());
i++;
}
tokensSplit.get(i).add(curr);
if (curr.getKey() == WordType.BCLOSE) {
brackets--;
}
}
return tokensSplit;
}
}