From bc218efe0e03c13c6252c13071d0a436f4d3f660 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Fri, 31 Dec 2021 14:14:53 +0100 Subject: [PATCH] fix lex test --- src/gc.rs | 8 +- src/lex.rs | 205 ++++-------------- src/lib.rs | 2 +- src/parse.rs | 2 +- ...ia__lex__test__braces_brackets_parens.snap | 14 ++ ...st__braces_brackets_parens_whitespace.snap | 14 ++ .../dilaria__lex__test__comments.snap | 9 + .../dilaria__lex__test__countdown.snap | 26 +++ .../dilaria__lex__test__fancy_stuff.snap | 15 ++ ...x__test__greater_than_less_than_equal.snap | 14 ++ .../dilaria__lex__test__greeting.snap | 13 ++ ...dilaria__lex__test__keyword_and_ident.snap | 25 +++ .../dilaria__lex__test__keywords.snap | 23 ++ .../dilaria__lex__test__larger_numbers.snap | 19 ++ ...ia__lex__test__long_multiline_comment.snap | 11 + .../dilaria__lex__test__no_no_no.snap | 12 + ...laria__lex__test__not_quite_a_keyword.snap | 59 +++++ .../dilaria__lex__test__serious_program.snap | 69 ++++++ .../dilaria__lex__test__smiley_face.snap | 13 ++ src/snapshots/dilaria__lex__test__string.snap | 11 + .../dilaria__lex__test__strings.snap | 22 ++ ...terminate_multiline_comment_correctly.snap | 11 + 22 files changed, 437 insertions(+), 160 deletions(-) create mode 100644 src/snapshots/dilaria__lex__test__braces_brackets_parens.snap create mode 100644 src/snapshots/dilaria__lex__test__braces_brackets_parens_whitespace.snap create mode 100644 src/snapshots/dilaria__lex__test__comments.snap create mode 100644 src/snapshots/dilaria__lex__test__countdown.snap create mode 100644 src/snapshots/dilaria__lex__test__fancy_stuff.snap create mode 100644 src/snapshots/dilaria__lex__test__greater_than_less_than_equal.snap create mode 100644 src/snapshots/dilaria__lex__test__greeting.snap create mode 100644 src/snapshots/dilaria__lex__test__keyword_and_ident.snap create mode 100644 src/snapshots/dilaria__lex__test__keywords.snap create mode 100644 src/snapshots/dilaria__lex__test__larger_numbers.snap create mode 100644 src/snapshots/dilaria__lex__test__long_multiline_comment.snap create mode 100644 src/snapshots/dilaria__lex__test__no_no_no.snap create mode 100644 src/snapshots/dilaria__lex__test__not_quite_a_keyword.snap create mode 100644 src/snapshots/dilaria__lex__test__serious_program.snap create mode 100644 src/snapshots/dilaria__lex__test__smiley_face.snap create mode 100644 src/snapshots/dilaria__lex__test__string.snap create mode 100644 src/snapshots/dilaria__lex__test__strings.snap create mode 100644 src/snapshots/dilaria__lex__test__terminate_multiline_comment_correctly.snap diff --git a/src/gc.rs b/src/gc.rs index 41a4a28..1f178ea 100644 --- a/src/gc.rs +++ b/src/gc.rs @@ -42,7 +42,7 @@ impl Clone for Gc { impl Copy for Gc {} /// An interned String. Hashing and Equality are O(1) and just look at the pointer address -#[derive(Debug, Clone, Copy)] +#[derive(Clone, Copy)] pub struct Symbol { gc: Gc, } @@ -83,6 +83,12 @@ impl Deref for Symbol { } } +impl Debug for Symbol { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.as_str().fmt(f) + } +} + #[derive(Debug)] struct Object { kind: ObjectKind, diff --git a/src/lex.rs b/src/lex.rs index afaa035..8597999 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -353,92 +353,75 @@ fn is_valid_ident_start(char: char) -> bool { char.is_alphabetic() || char == '_' } -#[cfg(test_ignore)] +#[cfg(test)] mod test { use crate::lex::Lexer; - use crate::lex::TokenKind::{self, *}; + use crate::RtAlloc; type StdString = std::string::String; - fn lex_types(str: &str) -> Vec { - let lexer = Lexer::new(str); - lexer.map(|token| token.kind).collect::>() - } + fn lex_test(code: &str) { + // SAFETY: we only work in this tiny scope + let mut runtime = unsafe { RtAlloc::new() }; - fn lex_test(code: &str, expected: Vec) { - assert_eq!(lex_types(code), expected) + let lexer = Lexer::new(code, &mut runtime); + let tokens = lexer.map(|token| token.kind).collect::>(); + + insta::assert_debug_snapshot!(tokens); } #[test] fn smiley_face() { - lex_test(">>.<<", vec![Greater, Greater, Dot, Less, Less]) + lex_test(">>.<<") } #[test] fn greater_than_less_than_equal() { - lex_test( - ">= <= == < < >=", - vec![ - GreaterEqual, - LessEqual, - EqualEqual, - Less, - Less, - GreaterEqual, - ], - ) + lex_test(">= <= == < < >=") } #[test] fn no_no_no() { - lex_test("!= != = !=", vec![BangEqual, BangEqual, Equal, BangEqual]) + lex_test("!= != = !=") } #[test] fn braces_brackets_parens() { - lex_test( - "{([]]}", - vec![BraceO, ParenO, BracketO, BracketC, BracketC, BraceC], - ); + lex_test("{([]]}"); } #[test] fn braces_brackets_parens_whitespace() { lex_test( "{ ( [ ] ] - - }", - vec![BraceO, ParenO, BracketO, BracketC, BracketC, BraceC], + + }", ); } #[test] fn fancy_stuff() { - lex_test( - ". ,- * -, .", - vec![Dot, Comma, Minus, Asterisk, Minus, Comma, Dot], - ) + lex_test(". ,- * -, .") } #[test] fn comments() { - lex_test("fn # fn", vec![Fn]); + lex_test("fn # fn"); } #[test] fn long_multiline_comment() { lex_test( "fn ## hello i am something - - i span multiple lines - - will you love me? 🥺🥺🥺🥺🥺 - - pls :) o(*￣▽￣*)ブ - i like the indentation here ngl | sneak for -> ## for ## <- sneak for - ## and", - vec![Fn, For, And], + i span multiple lines + + will you love me? 🥺🥺🥺🥺🥺 + +pls :) o(*￣▽￣*)ブ + + i like the indentation here ngl | sneak for -> ## for ## <- sneak for + ## and", ) } @@ -446,102 +429,50 @@ mod test { fn terminate_multiline_comment_correctly() { lex_test( "fn ## # no not here :( ## let # ## <- this is commented out - # so no multiline comment - ## - - here it starts - # let # - # # and - ## or - ", - vec![Fn, Let, Or], + # so no multiline comment + ## + + here it starts + # let # + # # and + ## or + ", ) } #[test] fn greeting() { - lex_test("-.- /%", vec![Minus, Dot, Minus, Slash, Percent]) + lex_test("-.- /%") } #[test] fn countdown() { - lex_test( - "3 . . 2 . . 1 . . 0", - vec![ - Number(3.0), - Dot, - Dot, - Number(2.0), - Dot, - Dot, - Number(1.0), - Dot, - Dot, - Number(0.0), - ], - ) + lex_test("3 . . 2 . . 1 . . 0") } #[test] fn larger_numbers() { - lex_test( - "123456789, 123456789.1234, 64785903", - vec![ - Number(123456789.0), - Comma, - Number(123456789.1234), - Comma, - Number(64785903.0), - ], - ) + lex_test("123456789, 123456789.1234, 64785903") } #[test] fn string() { - lex_test(r#""uwu""#, vec![String("uwu".to_string())]) + lex_test(r#""uwu""#) } #[test] fn strings() { - lex_test( - r#"( "hi" "uwu" "\"uwu\"" "no \\ u" )"#, - vec![ - ParenO, - String("hi".to_string()), - String("uwu".to_string()), - String("\"uwu\"".to_string()), - String("no \\ u".to_string()), - ParenC, - ], - ) + lex_test(r#"( "hi" "uwu" "\"uwu\"" "no \\ u" )"#) } #[test] fn keywords() { - lex_test( - "let fn if else loop while break for true false null and not or print", - vec![ - Let, Fn, If, Else, Loop, While, Break, For, True, False, Null, And, Not, Or, Print, - ], - ) + lex_test("let fn if else loop while break for true false null and not or print") } #[test] fn keyword_and_ident() { - lex_test( - "let variable be a loop if false is true", - vec![ - Let, - Ident("variable"), - Ident("be"), - Ident("a"), - Loop, - If, - False, - Ident("is"), - True, - ], - ) + lex_test("let variable be a loop if false is true") } #[test] @@ -569,61 +500,21 @@ mod test { .iter() .map(|word| format!("{} ", word)) .collect::(); - let expected = words.map(TokenKind::Ident).to_vec(); - lex_test(&sentences, expected) + lex_test(&sentences) } #[test] fn serious_program() { lex_test( r#"let string = "hallol" - let number = 5 - let me out ._. - fn world() { - if number == 5 or true == false and not false { - println("Hello \\ World!") - } - }"#, - vec![ - Let, - Ident("string"), - Equal, - String("hallol".to_string()), - Let, - Ident("number"), - Equal, - Number(5.0), - Let, - Ident("me"), - Ident("out"), - Dot, - Ident("_"), - Dot, - Fn, - Ident("world"), - ParenO, - ParenC, - BraceO, - If, - Ident("number"), - EqualEqual, - Number(5.0), - Or, - True, - EqualEqual, - False, - And, - Not, - False, - BraceO, - Ident("println"), - ParenO, - String("Hello \\ World!".to_string()), - ParenC, - BraceC, - BraceC, - ], + let number = 5 + let me out ._. + fn world() { + if number == 5 or true == false and not false { + println("Hello \\ World!") + } + }"#, ) } } diff --git a/src/lib.rs b/src/lib.rs index 3b42f15..eeb89e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,8 +10,8 @@ mod parse; mod vm; use crate::ast::Program; - use crate::gc::RtAlloc; + pub use bumpalo::Bump; pub use lex::*; pub use parse::*; diff --git a/src/parse.rs b/src/parse.rs index a9790ab..e01f074 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -3,7 +3,7 @@ //! It's a handwritten recursive descent parser. It has an internal peekable iterator from where //! it gets its next tokens. Only a lookahead of one is required. -#[cfg(test)] +#[cfg(test_ignore_lol)] mod test; use crate::ast::*; diff --git a/src/snapshots/dilaria__lex__test__braces_brackets_parens.snap b/src/snapshots/dilaria__lex__test__braces_brackets_parens.snap new file mode 100644 index 0000000..cf4143d --- /dev/null +++ b/src/snapshots/dilaria__lex__test__braces_brackets_parens.snap @@ -0,0 +1,14 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + BraceO, + ParenO, + BracketO, + BracketC, + BracketC, + BraceC, +] diff --git a/src/snapshots/dilaria__lex__test__braces_brackets_parens_whitespace.snap b/src/snapshots/dilaria__lex__test__braces_brackets_parens_whitespace.snap new file mode 100644 index 0000000..cf4143d --- /dev/null +++ b/src/snapshots/dilaria__lex__test__braces_brackets_parens_whitespace.snap @@ -0,0 +1,14 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + BraceO, + ParenO, + BracketO, + BracketC, + BracketC, + BraceC, +] diff --git a/src/snapshots/dilaria__lex__test__comments.snap b/src/snapshots/dilaria__lex__test__comments.snap new file mode 100644 index 0000000..de832d6 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__comments.snap @@ -0,0 +1,9 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Fn, +] diff --git a/src/snapshots/dilaria__lex__test__countdown.snap b/src/snapshots/dilaria__lex__test__countdown.snap new file mode 100644 index 0000000..844908a --- /dev/null +++ b/src/snapshots/dilaria__lex__test__countdown.snap @@ -0,0 +1,26 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Number( + 3.0, + ), + Dot, + Dot, + Number( + 2.0, + ), + Dot, + Dot, + Number( + 1.0, + ), + Dot, + Dot, + Number( + 0.0, + ), +] diff --git a/src/snapshots/dilaria__lex__test__fancy_stuff.snap b/src/snapshots/dilaria__lex__test__fancy_stuff.snap new file mode 100644 index 0000000..3745b08 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__fancy_stuff.snap @@ -0,0 +1,15 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Dot, + Comma, + Minus, + Asterisk, + Minus, + Comma, + Dot, +] diff --git a/src/snapshots/dilaria__lex__test__greater_than_less_than_equal.snap b/src/snapshots/dilaria__lex__test__greater_than_less_than_equal.snap new file mode 100644 index 0000000..6713d36 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__greater_than_less_than_equal.snap @@ -0,0 +1,14 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + GreaterEqual, + LessEqual, + EqualEqual, + Less, + Less, + GreaterEqual, +] diff --git a/src/snapshots/dilaria__lex__test__greeting.snap b/src/snapshots/dilaria__lex__test__greeting.snap new file mode 100644 index 0000000..bbfa30e --- /dev/null +++ b/src/snapshots/dilaria__lex__test__greeting.snap @@ -0,0 +1,13 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Minus, + Dot, + Minus, + Slash, + Percent, +] diff --git a/src/snapshots/dilaria__lex__test__keyword_and_ident.snap b/src/snapshots/dilaria__lex__test__keyword_and_ident.snap new file mode 100644 index 0000000..e96a7b1 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__keyword_and_ident.snap @@ -0,0 +1,25 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Let, + Ident( + "variable", + ), + Ident( + "be", + ), + Ident( + "a", + ), + Loop, + If, + False, + Ident( + "is", + ), + True, +] diff --git a/src/snapshots/dilaria__lex__test__keywords.snap b/src/snapshots/dilaria__lex__test__keywords.snap new file mode 100644 index 0000000..9336078 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__keywords.snap @@ -0,0 +1,23 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Let, + Fn, + If, + Else, + Loop, + While, + Break, + For, + True, + False, + Null, + And, + Not, + Or, + Print, +] diff --git a/src/snapshots/dilaria__lex__test__larger_numbers.snap b/src/snapshots/dilaria__lex__test__larger_numbers.snap new file mode 100644 index 0000000..feb915a --- /dev/null +++ b/src/snapshots/dilaria__lex__test__larger_numbers.snap @@ -0,0 +1,19 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Number( + 123456789.0, + ), + Comma, + Number( + 123456789.1234, + ), + Comma, + Number( + 64785903.0, + ), +] diff --git a/src/snapshots/dilaria__lex__test__long_multiline_comment.snap b/src/snapshots/dilaria__lex__test__long_multiline_comment.snap new file mode 100644 index 0000000..9ccfb3e --- /dev/null +++ b/src/snapshots/dilaria__lex__test__long_multiline_comment.snap @@ -0,0 +1,11 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Fn, + For, + And, +] diff --git a/src/snapshots/dilaria__lex__test__no_no_no.snap b/src/snapshots/dilaria__lex__test__no_no_no.snap new file mode 100644 index 0000000..832b2a2 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__no_no_no.snap @@ -0,0 +1,12 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + BangEqual, + BangEqual, + Equal, + BangEqual, +] diff --git a/src/snapshots/dilaria__lex__test__not_quite_a_keyword.snap b/src/snapshots/dilaria__lex__test__not_quite_a_keyword.snap new file mode 100644 index 0000000..952a30c --- /dev/null +++ b/src/snapshots/dilaria__lex__test__not_quite_a_keyword.snap @@ -0,0 +1,59 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Ident( + "letter", + ), + Ident( + "fori", + ), + Ident( + "fnfn", + ), + Ident( + "iffy", + ), + Ident( + "bloop", + ), + Ident( + "loopy_yeah", + ), + Ident( + "whileTrue", + ), + Ident( + "truefalse", + ), + Ident( + "falsetrue", + ), + Ident( + "nullability", + ), + Ident( + "rot", + ), + Ident( + "ornot", + ), + Ident( + "nor", + ), + Ident( + "andnowQuestionMark", + ), + Ident( + "notOrAnd", + ), + Ident( + "breakMe", + ), + Ident( + "Ibreak", + ), +] diff --git a/src/snapshots/dilaria__lex__test__serious_program.snap b/src/snapshots/dilaria__lex__test__serious_program.snap new file mode 100644 index 0000000..a877ec8 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__serious_program.snap @@ -0,0 +1,69 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Let, + Ident( + "string", + ), + Equal, + String( + "hallol", + ), + Let, + Ident( + "number", + ), + Equal, + Number( + 5.0, + ), + Let, + Ident( + "me", + ), + Ident( + "out", + ), + Dot, + Ident( + "_", + ), + Dot, + Fn, + Ident( + "world", + ), + ParenO, + ParenC, + BraceO, + If, + Ident( + "number", + ), + EqualEqual, + Number( + 5.0, + ), + Or, + True, + EqualEqual, + False, + And, + Not, + False, + BraceO, + Ident( + "println", + ), + ParenO, + String( + "Hello \\ World!", + ), + ParenC, + BraceC, + BraceC, +] diff --git a/src/snapshots/dilaria__lex__test__smiley_face.snap b/src/snapshots/dilaria__lex__test__smiley_face.snap new file mode 100644 index 0000000..785267f --- /dev/null +++ b/src/snapshots/dilaria__lex__test__smiley_face.snap @@ -0,0 +1,13 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Greater, + Greater, + Dot, + Less, + Less, +] diff --git a/src/snapshots/dilaria__lex__test__string.snap b/src/snapshots/dilaria__lex__test__string.snap new file mode 100644 index 0000000..45d3b44 --- /dev/null +++ b/src/snapshots/dilaria__lex__test__string.snap @@ -0,0 +1,11 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + String( + "uwu", + ), +] diff --git a/src/snapshots/dilaria__lex__test__strings.snap b/src/snapshots/dilaria__lex__test__strings.snap new file mode 100644 index 0000000..9a9dbbd --- /dev/null +++ b/src/snapshots/dilaria__lex__test__strings.snap @@ -0,0 +1,22 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + ParenO, + String( + "hi", + ), + String( + "uwu", + ), + String( + "\"uwu\"", + ), + String( + "no \\ u", + ), + ParenC, +] diff --git a/src/snapshots/dilaria__lex__test__terminate_multiline_comment_correctly.snap b/src/snapshots/dilaria__lex__test__terminate_multiline_comment_correctly.snap new file mode 100644 index 0000000..a80c32a --- /dev/null +++ b/src/snapshots/dilaria__lex__test__terminate_multiline_comment_correctly.snap @@ -0,0 +1,11 @@ +--- +source: src/lex.rs +assertion_line: 370 +expression: tokens + +--- +[ + Fn, + Let, + Or, +]