diff --git a/parser/src/pre/lexer.rs b/parser/src/pre/lexer.rs index 6005380..f278295 100644 --- a/parser/src/pre/lexer.rs +++ b/parser/src/pre/lexer.rs @@ -349,6 +349,8 @@ where (b'.', Some(b'.'), Some(b'.')) => triple_punct!(self, DotDotDot), (b'<', Some(b'<'), Some(b'=')) => triple_punct!(self, LeftLeftChevronEq), (b'>', Some(b'>'), Some(b'=')) => triple_punct!(self, RightRightChevronEq), + // the pretty pretty unicode arrow → is obviously supported here, i hope this won't cause probelms + (b'\xE2', Some(b'\x86'), Some(b'\x92')) => triple_punct!(self, Arrow), // DOUBLE CHARACTER PUNCTUATION (b'<', Some(b':'), _) => double_punct!(self, BracketOpen), (b':', Some(b'>'), _) => double_punct!(self, BracketClose), @@ -457,4 +459,13 @@ int hello(const char* uwu) <% lex_test!(src); } + + #[test] + fn pretty_arrow() { + let src = r#" +struct_ptr→field; +"#; + + lex_test!(src); + } } diff --git a/parser/src/pre/snapshots/parser__pre__lexer__tests__pretty_arrow.snap b/parser/src/pre/snapshots/parser__pre__lexer__tests__pretty_arrow.snap new file mode 100644 index 0000000..6d91f93 --- /dev/null +++ b/parser/src/pre/snapshots/parser__pre__lexer__tests__pretty_arrow.snap @@ -0,0 +1,30 @@ +--- +source: parser/src/pre/lexer.rs +expression: tokens +--- +[ + ( + Identifier( + "struct_ptr", + ), + 1..11, + ), + ( + Punctuator( + Arrow, + ), + 11..14, + ), + ( + Identifier( + "field", + ), + 14..19, + ), + ( + Punctuator( + Semicolon, + ), + 19..20, + ), +]