From 9d9c06f51b45db351266da25f62e63047c6f40f2 Mon Sep 17 00:00:00 2001 From: Nilstrieb Date: Sat, 16 Jan 2021 14:09:31 +0100 Subject: [PATCH] lexer working --- .idea/.gitignore | 5 + .idea/compiler.xml | 16 ++ .idea/discord.xml | 6 + .idea/jarRepositories.xml | 20 +++ .idea/misc.xml | 13 ++ .idea/modules.xml | 8 + .idea/uiDesigner.xml | 124 ++++++++++++++ .idea/vcs.xml | 6 + Hunterlang.iml | 15 ++ pom.xml | 16 ++ .../hunterlang/editor/EditorView.form | 21 +++ .../hunterlang/editor/EditorView.java | 8 + .../nilstrieb/hunterlang/lexer/LexToken.java | 21 +++ .../nilstrieb/hunterlang/lexer/LexedList.java | 27 +++ .../nilstrieb/hunterlang/lexer/Lexer.java | 155 ++++++++++++++++++ .../nilstrieb/hunterlang/lexer/WordType.java | 5 + .../hunterlang/lib/ConsoleColors.java | 76 +++++++++ .../nilstrieb/hunterlang/parser/Parser.java | 7 + 18 files changed, 549 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/compiler.xml create mode 100644 .idea/discord.xml create mode 100644 .idea/jarRepositories.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/uiDesigner.xml create mode 100644 .idea/vcs.xml create mode 100644 Hunterlang.iml create mode 100644 pom.xml create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.form create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.java create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/lexer/LexToken.java create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/lexer/LexedList.java create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/lexer/Lexer.java create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/lexer/WordType.java create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/lib/ConsoleColors.java create mode 100644 src/main/java/com/github/nilstrieb/hunterlang/parser/Parser.java diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..69ee9e0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +out\ +.class \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..d6610ae --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/discord.xml b/.idea/discord.xml new file mode 100644 index 0000000..cd711a0 --- /dev/null +++ b/.idea/discord.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 0000000..712ab9d --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..4219847 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,13 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..595b8ab --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..e96534f --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Hunterlang.iml b/Hunterlang.iml new file mode 100644 index 0000000..991bcc3 --- /dev/null +++ b/Hunterlang.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..6630406 --- /dev/null +++ b/pom.xml @@ -0,0 +1,16 @@ + + + 4.0.0 + + com.github.nilstrieb + Hunterlang + 1.0-SNAPSHOT + + + 15 + 15 + + + \ No newline at end of file diff --git a/src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.form b/src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.form new file mode 100644 index 0000000..43d331a --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.form @@ -0,0 +1,21 @@ + +
+ + + + + + + + + + + + + + + + + + +
diff --git a/src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.java b/src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.java new file mode 100644 index 0000000..6b996a2 --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/editor/EditorView.java @@ -0,0 +1,8 @@ +package com.github.nilstrieb.hunterlang.editor; + +import javax.swing.*; + +public class EditorView { + private JEditorPane editorPane1; + private JPanel panel1; +} diff --git a/src/main/java/com/github/nilstrieb/hunterlang/lexer/LexToken.java b/src/main/java/com/github/nilstrieb/hunterlang/lexer/LexToken.java new file mode 100644 index 0000000..3fa8c45 --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/lexer/LexToken.java @@ -0,0 +1,21 @@ +package com.github.nilstrieb.hunterlang.lexer; + +import com.github.nilstrieb.hunterlang.lib.ConsoleColors; + +public class LexToken { + WordType key; + String value; + + public LexToken(WordType key, String value) { + this.key = key; + this.value = value; + } + + @Override + public String toString() { + return ConsoleColors.BLUE_BOLD + "{Key=" + + ConsoleColors.RED_BOLD_BRIGHT + key + + ConsoleColors.BLUE_BOLD + ", value=" + + ConsoleColors.YELLOW + value + ConsoleColors.BLUE_BOLD + "}"; + } +} diff --git a/src/main/java/com/github/nilstrieb/hunterlang/lexer/LexedList.java b/src/main/java/com/github/nilstrieb/hunterlang/lexer/LexedList.java new file mode 100644 index 0000000..fe37092 --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/lexer/LexedList.java @@ -0,0 +1,27 @@ +package com.github.nilstrieb.hunterlang.lexer; + +import java.util.ArrayList; + +public class LexedList { + private ArrayList> list; + + public LexedList() { + list = new ArrayList<>(); + } + + public void newLine(){ + list.add(new ArrayList<>()); + } + + public void add(int line, WordType type, String value) { + list.get(line).add(new LexToken(type, value)); + } + + public void add(int line, WordType type){ + add(line, type, ""); + } + + public ArrayList get(int i) { + return list.get(i); + } +} diff --git a/src/main/java/com/github/nilstrieb/hunterlang/lexer/Lexer.java b/src/main/java/com/github/nilstrieb/hunterlang/lexer/Lexer.java new file mode 100644 index 0000000..82b0339 --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/lexer/Lexer.java @@ -0,0 +1,155 @@ +package com.github.nilstrieb.hunterlang.lexer; + +import com.github.nilstrieb.hunterlang.lib.ConsoleColors; + +import java.util.ArrayList; + +public class Lexer { + + public static final String COMMENT = "#"; + + private static final String MEMCALL = "killua"; + public static final String ASSIGNMENT = "hunts"; + public static final String IF = "Gon"; + public static final String WANTS = "wants"; + public static final String ELSE = "got"; + public static final String BOPEN = "{"; + public static final String BCLOSE = "}"; + + public static final String GTHAN = ">"; + public static final String LTHAN = "<"; + public static final String EQUALS = "="; + public static final String PLUS = "+"; + public static final String MINUS = "-"; + public static final String MULTIPLY = "*"; + public static final String DIVIDE = "/"; + public static final String MOD = "%"; + + public static final String LIBFUNCCALL_REGEX = "(\\w+) does (\\w+).*"; + + public static final String BOOL_REGEX = "(true|false).*"; + public static final String STRING_REGEX = "^\"(.*)\".*"; + public static final String NUMBER_REGEX = "^(-?\\d+.?\\d*).*"; + + private LexedList tokens; + + public LexedList lex(String code) { + tokens = new LexedList(); + ArrayList tempList = new ArrayList<>(); + String[] lines = code.split("\\r\\n|\\n"); + + for (int i = 0; i < lines.length; i++) { + tokens.newLine(); + String line = lines[i]; + for (int j = 0; j < line.length(); j++) { + String sub = line.substring(j); + + //COMMENT + + if (sub.startsWith(COMMENT)) { + break; + } + + //FIXED KEYWORDS + + else if (sub.startsWith(MEMCALL)) { + tokens.add(i, WordType.MEMCALL); + j += MEMCALL.length() - 1; + } else if (sub.startsWith(ASSIGNMENT)) { + tokens.add(i, WordType.ASSIGNMENT); + j += ASSIGNMENT.length() - 1; + } else if (sub.startsWith(IF)) { + tokens.add(i, WordType.IF); + j += IF.length() - 1; + } else if (sub.startsWith(WANTS)){ + tokens.add(i, WordType.WANTS); + j += WANTS.length() - 1; + } else if (sub.startsWith(ELSE)){ + tokens.add(i, WordType.ELSE); + j += ELSE.length() - 1; + } + + //CONDITIONS + else if (sub.startsWith(GTHAN)){ + tokens.add(i, WordType.GTHAN); + j += GTHAN.length() - 1; + } else if (sub.startsWith(LTHAN)){ + tokens.add(i, WordType.LTHAN); + j += LTHAN.length() - 1; + } else if (sub.startsWith(EQUALS)){ + tokens.add(i, WordType.EQUALS); + j += EQUALS.length() - 1; + } + + //OPERATORS + else if (sub.startsWith(PLUS)){ + tokens.add(i, WordType.PLUS); + } else if (sub.startsWith(MINUS)){ + tokens.add(i, WordType.MINUS); + } else if (sub.startsWith(MULTIPLY)){ + tokens.add(i, WordType.MULTIPLY); + }else if (sub.startsWith(DIVIDE)){ + tokens.add(i, WordType.DIVIDE); + } else if (sub.startsWith(MOD)){ + tokens.add(i, WordType.MOD); + } + + // BRACKETS + else if (sub.startsWith(BOPEN)){ + tokens.add(i, WordType.BOPEN); + } else if (sub.startsWith(BCLOSE)){ + tokens.add(i, WordType.BCLOSE); + } + + //VALUES + else if (sub.matches(STRING_REGEX)) { + String string = sub.replaceAll(STRING_REGEX, "$1"); + tokens.add(i, WordType.STRING, string); + j += string.length() + 1; + } else if (sub.matches(NUMBER_REGEX)) { + String number = sub.replaceAll(NUMBER_REGEX, "$1"); + tokens.add(i, WordType.NUMBER, number); + j += number.length() - 1; + } else if (sub.matches(BOOL_REGEX)){ + String bool = sub.replaceAll(BOOL_REGEX, "$1"); + tokens.add(i, WordType.BOOL, bool); + j += bool.length() - 1; + } + + //Calls + else if(sub.matches(LIBFUNCCALL_REGEX)){ + String libName = sub.replaceAll(LIBFUNCCALL_REGEX, "$1"); + String funcName = sub.replaceAll(LIBFUNCCALL_REGEX, "$2"); + tokens.add(i, WordType.LIB, libName); + tokens.add(i, WordType.FUNCCALL, funcName); + j += libName.length(); //lib name + j += 1 + 4 + 1; // space + does + space + j += funcName.length() - 1; //func name + } + } + System.out.println(ConsoleColors.GREEN_BOLD + line); + System.out.println(ConsoleColors.BLUE_BOLD + tokens.get(i)); + } + + return tokens; + } + + public static void main(String[] args) { + Lexer l = new Lexer(); + + LexedList tokens = l.lex(""" + killua0 hunts 3 + killua0 hunts -3.4 #hunts nothing + killua0 hunts 4 + killua1 hunts "hallo" + #comment + Gon wants false { + Leorio does say "false" + } wants killua0 > 3 { + Leorio does say "big killua" + } got { + Leorio does say "small killua" + } + """); + } +} diff --git a/src/main/java/com/github/nilstrieb/hunterlang/lexer/WordType.java b/src/main/java/com/github/nilstrieb/hunterlang/lexer/WordType.java new file mode 100644 index 0000000..2bf17c0 --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/lexer/WordType.java @@ -0,0 +1,5 @@ +package com.github.nilstrieb.hunterlang.lexer; + +public enum WordType { + ASSIGNMENT, MEMCALL, STRING, NUMBER, BOOL, IF, WANTS, ELSE, BOPEN, BCLOSE, GTHAN, LTHAN, EQUALS, FUNCNAME, FUNCARGS, FUNCRETURN, IOCALL, FUNCCALL, OPERATOR, LIB, MINUS, DEFAULT, PLUS, MULTIPLY, DIVIDE, MOD; +} diff --git a/src/main/java/com/github/nilstrieb/hunterlang/lib/ConsoleColors.java b/src/main/java/com/github/nilstrieb/hunterlang/lib/ConsoleColors.java new file mode 100644 index 0000000..0c910cd --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/lib/ConsoleColors.java @@ -0,0 +1,76 @@ +package com.github.nilstrieb.hunterlang.lib; + +public class ConsoleColors { + // Reset + public static final String RESET = "\033[0m"; // Text Reset + + // Regular Colors + public static final String BLACK = "\033[0;30m"; // BLACK + public static final String RED = "\033[0;31m"; // RED + public static final String GREEN = "\033[0;32m"; // GREEN + public static final String YELLOW = "\033[0;33m"; // YELLOW + public static final String BLUE = "\033[0;34m"; // BLUE + public static final String PURPLE = "\033[0;35m"; // PURPLE + public static final String CYAN = "\033[0;36m"; // CYAN + public static final String WHITE = "\033[0;37m"; // WHITE + + // Bold + public static final String BLACK_BOLD = "\033[1;30m"; // BLACK + public static final String RED_BOLD = "\033[1;31m"; // RED + public static final String GREEN_BOLD = "\033[1;32m"; // GREEN + public static final String YELLOW_BOLD = "\033[1;33m"; // YELLOW + public static final String BLUE_BOLD = "\033[1;34m"; // BLUE + public static final String PURPLE_BOLD = "\033[1;35m"; // PURPLE + public static final String CYAN_BOLD = "\033[1;36m"; // CYAN + public static final String WHITE_BOLD = "\033[1;37m"; // WHITE + + // Underline + public static final String BLACK_UNDERLINED = "\033[4;30m"; // BLACK + public static final String RED_UNDERLINED = "\033[4;31m"; // RED + public static final String GREEN_UNDERLINED = "\033[4;32m"; // GREEN + public static final String YELLOW_UNDERLINED = "\033[4;33m"; // YELLOW + public static final String BLUE_UNDERLINED = "\033[4;34m"; // BLUE + public static final String PURPLE_UNDERLINED = "\033[4;35m"; // PURPLE + public static final String CYAN_UNDERLINED = "\033[4;36m"; // CYAN + public static final String WHITE_UNDERLINED = "\033[4;37m"; // WHITE + + // Background + public static final String BLACK_BACKGROUND = "\033[40m"; // BLACK + public static final String RED_BACKGROUND = "\033[41m"; // RED + public static final String GREEN_BACKGROUND = "\033[42m"; // GREEN + public static final String YELLOW_BACKGROUND = "\033[43m"; // YELLOW + public static final String BLUE_BACKGROUND = "\033[44m"; // BLUE + public static final String PURPLE_BACKGROUND = "\033[45m"; // PURPLE + public static final String CYAN_BACKGROUND = "\033[46m"; // CYAN + public static final String WHITE_BACKGROUND = "\033[47m"; // WHITE + + // High Intensity + public static final String BLACK_BRIGHT = "\033[0;90m"; // BLACK + public static final String RED_BRIGHT = "\033[0;91m"; // RED + public static final String GREEN_BRIGHT = "\033[0;92m"; // GREEN + public static final String YELLOW_BRIGHT = "\033[0;93m"; // YELLOW + public static final String BLUE_BRIGHT = "\033[0;94m"; // BLUE + public static final String PURPLE_BRIGHT = "\033[0;95m"; // PURPLE + public static final String CYAN_BRIGHT = "\033[0;96m"; // CYAN + public static final String WHITE_BRIGHT = "\033[0;97m"; // WHITE + + // Bold High Intensity + public static final String BLACK_BOLD_BRIGHT = "\033[1;90m"; // BLACK + public static final String RED_BOLD_BRIGHT = "\033[1;91m"; // RED + public static final String GREEN_BOLD_BRIGHT = "\033[1;92m"; // GREEN + public static final String YELLOW_BOLD_BRIGHT = "\033[1;93m";// YELLOW + public static final String BLUE_BOLD_BRIGHT = "\033[1;94m"; // BLUE + public static final String PURPLE_BOLD_BRIGHT = "\033[1;95m";// PURPLE + public static final String CYAN_BOLD_BRIGHT = "\033[1;96m"; // CYAN + public static final String WHITE_BOLD_BRIGHT = "\033[1;97m"; // WHITE + + // High Intensity backgrounds + public static final String BLACK_BACKGROUND_BRIGHT = "\033[0;100m";// BLACK + public static final String RED_BACKGROUND_BRIGHT = "\033[0;101m";// RED + public static final String GREEN_BACKGROUND_BRIGHT = "\033[0;102m";// GREEN + public static final String YELLOW_BACKGROUND_BRIGHT = "\033[0;103m";// YELLOW + public static final String BLUE_BACKGROUND_BRIGHT = "\033[0;104m";// BLUE + public static final String PURPLE_BACKGROUND_BRIGHT = "\033[0;105m"; // PURPLE + public static final String CYAN_BACKGROUND_BRIGHT = "\033[0;106m"; // CYAN + public static final String WHITE_BACKGROUND_BRIGHT = "\033[0;107m"; // WHITE +} \ No newline at end of file diff --git a/src/main/java/com/github/nilstrieb/hunterlang/parser/Parser.java b/src/main/java/com/github/nilstrieb/hunterlang/parser/Parser.java new file mode 100644 index 0000000..83017e7 --- /dev/null +++ b/src/main/java/com/github/nilstrieb/hunterlang/parser/Parser.java @@ -0,0 +1,7 @@ +package com.github.nilstrieb.hunterlang.parser; + +public class Parser { +} + +class ParseTreeNode { +} \ No newline at end of file