From 7fc10f3b6c2046701118406276e5161f96bb1b87 Mon Sep 17 00:00:00 2001 From: nils <48135649+Nilstrieb@users.noreply.github.com> Date: Tue, 21 Jun 2022 13:29:20 +0200 Subject: [PATCH] tests that work! --- parser/src/pre/lexer.rs | 33 ++++-- ...arser__pre__lexer__tests__hello_world.snap | 100 ++++++++++++++++++ ...arser__pre__lexer__tests__identifiers.snap | 39 +++++++ ...parser__pre__lexer__tests__lex_test-2.snap | 39 +++++++ .../parser__pre__lexer__tests__lex_test.snap | 100 ++++++++++++++++++ 5 files changed, 301 insertions(+), 10 deletions(-) create mode 100644 parser/src/pre/snapshots/parser__pre__lexer__tests__hello_world.snap create mode 100644 parser/src/pre/snapshots/parser__pre__lexer__tests__identifiers.snap create mode 100644 parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test-2.snap create mode 100644 parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test.snap diff --git a/parser/src/pre/lexer.rs b/parser/src/pre/lexer.rs index 966c7cb..37c89b0 100644 --- a/parser/src/pre/lexer.rs +++ b/parser/src/pre/lexer.rs @@ -136,6 +136,7 @@ where let mut ident = vec![c]; while let Some((span, c)) = self.src.peek() { + println!("uwu {c}"); let (span, c) = (*span, *c); if c.is_c_identifier() { self.src.next(); @@ -257,14 +258,18 @@ where fn next(&mut self) -> Option { use PToken::Punctuator as TokP; - let (start_span, char1) = self.src.next()?; - let char2 = self.src.peek().map(|(_, c)| *c); - let char3 = self.src.peek_nth(2).map(|(_, c)| *c); + let mut start_span; let (token, end_span) = loop { + let (span, char1) = self.src.next()?; + start_span = span; + let char2 = self.src.peek().map(|(_, c)| *c); + let char3 = self.src.peek_nth(2).map(|(_, c)| *c); + match (char1, char2, char3) { // IDENTIFIER (c, _, _) if c.is_c_identifier_nondigit() => { + println!("AA"); break self.identifier(c, start_span); } // NUMBER @@ -354,20 +359,28 @@ pub fn preprocess_tokens( #[cfg(test)] mod tests { - fn lex_test(str: &str) { - let bytes = str.bytes().enumerate(); - let tokens = super::preprocess_tokens(bytes); - let tokens = tokens.collect::>(); - insta::assert_debug_snapshot!(tokens); + macro_rules! lex_test { + ($str:expr) => { + let bytes = $str.bytes().enumerate(); + let tokens = super::preprocess_tokens(bytes); + let tokens = tokens.collect::>(); + insta::assert_debug_snapshot!(tokens); + }; + } + + #[test] + fn identifiers() { + let src = r#"AAAA BBBB CCCC"#; + lex_test!(src); } #[test] fn hello_world() { - let src = r#"\ + let src = r#" int main() { puts("Hello, World!"); } "#; - lex_test(src); + lex_test!(src); } } diff --git a/parser/src/pre/snapshots/parser__pre__lexer__tests__hello_world.snap b/parser/src/pre/snapshots/parser__pre__lexer__tests__hello_world.snap new file mode 100644 index 0000000..c0a7a20 --- /dev/null +++ b/parser/src/pre/snapshots/parser__pre__lexer__tests__hello_world.snap @@ -0,0 +1,100 @@ +--- +source: parser/src/pre/lexer.rs +expression: tokens +--- +[ + ( + Identifier( + [ + 105, + 110, + 116, + ], + ), + 1..4, + ), + ( + Identifier( + [ + 109, + 97, + 105, + 110, + ], + ), + 5..9, + ), + ( + Punctuator( + ParenOpen, + ), + 9..10, + ), + ( + Punctuator( + ParenClose, + ), + 10..11, + ), + ( + Punctuator( + BraceOpen, + ), + 12..13, + ), + ( + Identifier( + [ + 112, + 117, + 116, + 115, + ], + ), + 18..22, + ), + ( + Punctuator( + ParenOpen, + ), + 22..23, + ), + ( + StringLiteral( + [ + 72, + 101, + 108, + 108, + 111, + 44, + 32, + 87, + 111, + 114, + 108, + 100, + 33, + ], + ), + 23..37, + ), + ( + Punctuator( + ParenClose, + ), + 38..39, + ), + ( + Punctuator( + Semicolon, + ), + 39..40, + ), + ( + Punctuator( + BraceClose, + ), + 41..42, + ), +] diff --git a/parser/src/pre/snapshots/parser__pre__lexer__tests__identifiers.snap b/parser/src/pre/snapshots/parser__pre__lexer__tests__identifiers.snap new file mode 100644 index 0000000..f0e69cd --- /dev/null +++ b/parser/src/pre/snapshots/parser__pre__lexer__tests__identifiers.snap @@ -0,0 +1,39 @@ +--- +source: parser/src/pre/lexer.rs +expression: tokens +--- +[ + ( + Identifier( + [ + 65, + 65, + 65, + 65, + ], + ), + 0..4, + ), + ( + Identifier( + [ + 66, + 66, + 66, + 66, + ], + ), + 5..9, + ), + ( + Identifier( + [ + 67, + 67, + 67, + 67, + ], + ), + 10..14, + ), +] diff --git a/parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test-2.snap b/parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test-2.snap new file mode 100644 index 0000000..f0e69cd --- /dev/null +++ b/parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test-2.snap @@ -0,0 +1,39 @@ +--- +source: parser/src/pre/lexer.rs +expression: tokens +--- +[ + ( + Identifier( + [ + 65, + 65, + 65, + 65, + ], + ), + 0..4, + ), + ( + Identifier( + [ + 66, + 66, + 66, + 66, + ], + ), + 5..9, + ), + ( + Identifier( + [ + 67, + 67, + 67, + 67, + ], + ), + 10..14, + ), +] diff --git a/parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test.snap b/parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test.snap new file mode 100644 index 0000000..c0a7a20 --- /dev/null +++ b/parser/src/pre/snapshots/parser__pre__lexer__tests__lex_test.snap @@ -0,0 +1,100 @@ +--- +source: parser/src/pre/lexer.rs +expression: tokens +--- +[ + ( + Identifier( + [ + 105, + 110, + 116, + ], + ), + 1..4, + ), + ( + Identifier( + [ + 109, + 97, + 105, + 110, + ], + ), + 5..9, + ), + ( + Punctuator( + ParenOpen, + ), + 9..10, + ), + ( + Punctuator( + ParenClose, + ), + 10..11, + ), + ( + Punctuator( + BraceOpen, + ), + 12..13, + ), + ( + Identifier( + [ + 112, + 117, + 116, + 115, + ], + ), + 18..22, + ), + ( + Punctuator( + ParenOpen, + ), + 22..23, + ), + ( + StringLiteral( + [ + 72, + 101, + 108, + 108, + 111, + 44, + 32, + 87, + 111, + 114, + 108, + 100, + 33, + ], + ), + 23..37, + ), + ( + Punctuator( + ParenClose, + ), + 38..39, + ), + ( + Punctuator( + Semicolon, + ), + 39..40, + ), + ( + Punctuator( + BraceClose, + ), + 41..42, + ), +]