diff --git a/src/lex.rs b/src/lex.rs index 5bb463e..72d2c54 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -12,24 +12,24 @@ use std::str::CharIndices; #[derive(Debug, Clone)] pub struct Token<'code> { pub span: Span, - pub kind: TokenType<'code>, + pub kind: TokenKind<'code>, } impl<'code> Token<'code> { - fn single_span(start: usize, kind: TokenType<'code>) -> Token<'code> { + fn single_span(start: usize, kind: TokenKind<'code>) -> Token<'code> { Self { span: Span::single(start), kind, } } - fn new(span: Span, kind: TokenType<'code>) -> Token<'code> { + fn new(span: Span, kind: TokenKind<'code>) -> Token<'code> { Self { span, kind } } } #[derive(Debug, Clone, PartialEq)] -pub enum TokenType<'code> { +pub enum TokenKind<'code> { // keywords Let, Print, @@ -124,8 +124,8 @@ impl<'code> Lexer<'code> { fn maybe_next_char<'a>( &mut self, expect_char: char, - true_type: TokenType<'a>, - false_type: TokenType<'a>, + true_type: TokenKind<'a>, + false_type: TokenKind<'a>, start: usize, ) -> Token<'a> { if self.expect(expect_char) { @@ -170,36 +170,36 @@ impl<'code> Iterator for Lexer<'code> { } } } - ';' => break Token::single_span(start, TokenType::Semi), - '+' => break Token::single_span(start, TokenType::Plus), - '-' => break Token::single_span(start, TokenType::Minus), - '*' => break Token::single_span(start, TokenType::Asterisk), - '/' => break Token::single_span(start, TokenType::Slash), - '%' => break Token::single_span(start, TokenType::Percent), - '{' => break Token::single_span(start, TokenType::BraceO), - '}' => break Token::single_span(start, TokenType::BraceC), - '[' => break Token::single_span(start, TokenType::BracketO), - ']' => break Token::single_span(start, TokenType::BracketC), - '(' => break Token::single_span(start, TokenType::ParenO), - ')' => break Token::single_span(start, TokenType::ParenC), - '.' => break Token::single_span(start, TokenType::Dot), - ',' => break Token::single_span(start, TokenType::Comma), + ';' => break Token::single_span(start, TokenKind::Semi), + '+' => break Token::single_span(start, TokenKind::Plus), + '-' => break Token::single_span(start, TokenKind::Minus), + '*' => break Token::single_span(start, TokenKind::Asterisk), + '/' => break Token::single_span(start, TokenKind::Slash), + '%' => break Token::single_span(start, TokenKind::Percent), + '{' => break Token::single_span(start, TokenKind::BraceO), + '}' => break Token::single_span(start, TokenKind::BraceC), + '[' => break Token::single_span(start, TokenKind::BracketO), + ']' => break Token::single_span(start, TokenKind::BracketC), + '(' => break Token::single_span(start, TokenKind::ParenO), + ')' => break Token::single_span(start, TokenKind::ParenC), + '.' => break Token::single_span(start, TokenKind::Dot), + ',' => break Token::single_span(start, TokenKind::Comma), '=' => { break self.maybe_next_char( '=', - TokenType::EqualEqual, - TokenType::Equal, + TokenKind::EqualEqual, + TokenKind::Equal, start, ); } '!' => { break if self.expect('=') { let _ = self.code.next(); // consume =; - Token::new(Span::start_len(start, start + 2), TokenType::BangEqual) + Token::new(Span::start_len(start, start + 2), TokenKind::BangEqual) } else { Token::new( Span::single(start), - TokenType::Error(CompilerError::with_note( + TokenKind::Error(CompilerError::with_note( Span::single(start), "Expected '=' after '!'".to_string(), "If you meant to use it for negation, use `not`".to_string(), @@ -210,13 +210,13 @@ impl<'code> Iterator for Lexer<'code> { '>' => { break self.maybe_next_char( '=', - TokenType::GreaterEqual, - TokenType::Greater, + TokenKind::GreaterEqual, + TokenKind::Greater, start, ); } '<' => { - break self.maybe_next_char('=', TokenType::LessEqual, TokenType::Less, start); + break self.maybe_next_char('=', TokenKind::LessEqual, TokenKind::Less, start); } '"' => { let mut buffer = String::new(); @@ -232,7 +232,7 @@ impl<'code> Iterator for Lexer<'code> { None => { return Some(Token::new( Span::single(start), - TokenType::Error(CompilerError::with_note( + TokenKind::Error(CompilerError::with_note( Span::single(start), // no not show the whole literal, this does not make sense "String literal not closed".to_string(), "Close the literal using '\"'".to_string(), @@ -241,7 +241,7 @@ impl<'code> Iterator for Lexer<'code> { } } }; - break Token::new(Span::start_end(start, end), TokenType::String(buffer)); + break Token::new(Span::start_end(start, end), TokenKind::String(buffer)); } char => { if char.is_ascii_digit() { @@ -265,15 +265,15 @@ impl<'code> Iterator for Lexer<'code> { let number = number_str.parse::(); break match number { Ok(number) if number.is_infinite() => { - Token::new(span, TokenType::Error(CompilerError::with_note( + Token::new(span, TokenKind::Error(CompilerError::with_note( span, "Number literal too long".to_string(), "A number literal cannot be larger than a 64 bit float can represent" .to_string(), ))) } - Ok(number) => Token::new(span, TokenType::Number(number)), - Err(err) => Token::new(span, TokenType::Error(CompilerError::with_note( + Ok(number) => Token::new(span, TokenKind::Number(number)), + Err(err) => Token::new(span, TokenKind::Error(CompilerError::with_note( span, "Invalid number".to_string(), err.to_string(), @@ -297,7 +297,7 @@ impl<'code> Iterator for Lexer<'code> { } else { break Token::new( Span::single(start), - TokenType::Error(CompilerError::with_note( + TokenKind::Error(CompilerError::with_note( Span::single(start), format!("Unexpected character: '{}'", char), "Character is not allowed outside of string literals and comments" @@ -313,25 +313,25 @@ impl<'code> Iterator for Lexer<'code> { } } -fn keyword_or_ident(name: &str) -> TokenType { +fn keyword_or_ident(name: &str) -> TokenKind { match name { - "loop" => TokenType::Loop, - "let" => TokenType::Let, - "fn" => TokenType::Fn, - "for" => TokenType::For, - "false" => TokenType::False, - "if" => TokenType::If, - "else" => TokenType::Else, - "while" => TokenType::While, - "break" => TokenType::Break, - "return" => TokenType::Return, - "true" => TokenType::True, - "null" => TokenType::Null, - "not" => TokenType::Not, - "and" => TokenType::And, - "or" => TokenType::Or, - "print" => TokenType::Print, - _ => TokenType::Ident(name), + "loop" => TokenKind::Loop, + "let" => TokenKind::Let, + "fn" => TokenKind::Fn, + "for" => TokenKind::For, + "false" => TokenKind::False, + "if" => TokenKind::If, + "else" => TokenKind::Else, + "while" => TokenKind::While, + "break" => TokenKind::Break, + "return" => TokenKind::Return, + "true" => TokenKind::True, + "null" => TokenKind::Null, + "not" => TokenKind::Not, + "and" => TokenKind::And, + "or" => TokenKind::Or, + "print" => TokenKind::Print, + _ => TokenKind::Ident(name), } } @@ -346,16 +346,16 @@ fn is_valid_ident_start(char: char) -> bool { #[cfg(test)] mod test { use crate::lex::Lexer; - use crate::lex::TokenType::{self, *}; + use crate::lex::TokenKind::{self, *}; type StdString = std::string::String; - fn lex_types(str: &str) -> Vec { + fn lex_types(str: &str) -> Vec { let lexer = Lexer::new(str); lexer.map(|token| token.kind).collect::>() } - fn lex_test(code: &str, expected: Vec) { + fn lex_test(code: &str, expected: Vec) { assert_eq!(lex_types(code), expected) } @@ -559,7 +559,7 @@ mod test { .iter() .map(|word| format!("{} ", word)) .collect::(); - let expected = words.map(TokenType::Ident).to_vec(); + let expected = words.map(TokenKind::Ident).to_vec(); lex_test(&sentences, expected) } diff --git a/src/lib.rs b/src/lib.rs index d0b16bb..f40facc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,13 +18,7 @@ pub fn run_program(program: &str) { let ast_alloc = Bump::new(); let lexer = lex::Lexer::new(program); - let ast = parse::parse( - lexer.map(|token| match &token.kind { - TokenType::Error(err) => Err(err.clone()), - _ => Ok(token), - }), - &ast_alloc, - ); + let ast = parse::parse(lexer, &ast_alloc); match ast { Ok(ast) => process_ast(program, ast), diff --git a/src/parse.rs b/src/parse.rs index 4f8d986..71283aa 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -3,7 +3,7 @@ mod test; use crate::ast::*; use crate::errors::{CompilerError, Span}; -use crate::lex::{Token, TokenType}; +use crate::lex::{Token, TokenKind}; use bumpalo::boxed::Box; use bumpalo::collections::Vec; use bumpalo::Bump; @@ -12,7 +12,7 @@ use std::iter::Peekable; #[derive(Debug)] struct Parser<'code, 'ast, I> where - I: Iterator, CompilerError>>, + I: Iterator>, I: 'code, { tokens: Peekable, @@ -23,7 +23,7 @@ where } pub fn parse<'ast, 'code>( - tokens: impl Iterator, CompilerError>> + 'code, + tokens: impl Iterator> + 'code, ast_bump: &'ast Bump, ) -> Result, CompilerError> { let mut parser = Parser { @@ -37,7 +37,7 @@ pub fn parse<'ast, 'code>( Ok(program) } -type ParseResult<'code, T> = Result>; +type ParseResult = Result; macro_rules! parse_bin_op { ($self: ident, $lhs: ident, $kind: expr, $function: ident) => {{ @@ -73,28 +73,33 @@ macro_rules! enter_parse { impl<'code, 'ast, I> Parser<'code, 'ast, I> where - I: Iterator, CompilerError>>, + I: Iterator>, I: 'code, { const MAX_DEPTH: usize = 100; - fn program(&mut self) -> ParseResult<'code, Program<'ast>> { + fn program(&mut self) -> ParseResult> { Ok(Program(self.statement_list()?)) } - fn too_nested_error(&mut self) -> ParseResult<'code, ()> { - let next_token = self.next()?; + fn too_nested_error(&mut self) -> ParseResult<()> { + let next_token = self.next(); match next_token { - Some(token) => Err(ParseErr::MaxDepth(token.span)), - None => Err(ParseErr::Eof("reached EOF while being nested to deeply")), + Some(token) => Err(CompilerError::new( + token.span, + "reached maximal nesting depth".to_string(), + )), + None => Err(CompilerError::eof( + "reached EOF while being nested to deeply", + )), } } - fn statement_list(&mut self) -> ParseResult<'code, Vec<'ast, Stmt<'ast>>> { + fn statement_list(&mut self) -> ParseResult>> { enter_parse!(self); let mut stmts = Vec::new_in(self.bump); let return_stmts = loop { - if let Some(TokenType::BraceC) | None = self.peek_kind()? { + if let Some(TokenKind::BraceC) | None = self.peek_kind() { break Ok(stmts); } let stmt = self.statement()?; @@ -104,12 +109,12 @@ where return_stmts } - fn block(&mut self) -> ParseResult<'code, Block<'ast>> { + fn block(&mut self) -> ParseResult> { enter_parse!(self); - let start_span = self.expect(TokenType::BraceO)?.span; + let start_span = self.expect(TokenKind::BraceO)?.span; let stmts = self.statement_list()?; - let end_span = self.expect(TokenType::BraceC)?.span; + let end_span = self.expect(TokenKind::BraceC)?.span; exit_parse!(self); @@ -119,19 +124,22 @@ where }) } - fn statement(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn statement(&mut self) -> ParseResult> { enter_parse!(self); - let stmt = match *self.peek_kind()?.ok_or(ParseErr::Eof("statement"))? { - TokenType::Let => self.declaration(), - TokenType::Fn => self.fn_decl(), - TokenType::If => Ok(Stmt::If(self.if_stmt()?)), - TokenType::Loop => self.loop_stmt(), - TokenType::While => self.while_stmt(), - TokenType::Break => self.break_stmt(), - TokenType::Return => self.return_stmt(), - TokenType::Print => self.print_stmt(), - TokenType::BraceO => Ok(Stmt::Block(self.block()?)), + let stmt = match *self + .peek_kind() + .ok_or_else(|| CompilerError::eof("statement"))? + { + TokenKind::Let => self.declaration(), + TokenKind::Fn => self.fn_decl(), + TokenKind::If => Ok(Stmt::If(self.if_stmt()?)), + TokenKind::Loop => self.loop_stmt(), + TokenKind::While => self.while_stmt(), + TokenKind::Break => self.break_stmt(), + TokenKind::Return => self.return_stmt(), + TokenKind::Print => self.print_stmt(), + TokenKind::BraceO => Ok(Stmt::Block(self.block()?)), _ => { let stmt = self.assignment()?; Ok(stmt) @@ -141,14 +149,14 @@ where stmt } - fn declaration(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn declaration(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::Let)?.span; + let keyword_span = self.expect(TokenKind::Let)?.span; let name = self.ident()?; - self.expect(TokenType::Equal)?; + self.expect(TokenKind::Equal)?; let init = self.expression()?; - self.expect(TokenType::Semi)?; + self.expect(TokenKind::Semi)?; exit_parse!(self); @@ -159,10 +167,10 @@ where })) } - fn fn_decl(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn fn_decl(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::Fn)?.span; + let keyword_span = self.expect(TokenKind::Fn)?.span; let name = self.ident()?; let args = self.fn_args()?; @@ -180,26 +188,26 @@ where })) } - fn fn_args(&mut self) -> ParseResult<'code, Vec<'ast, Ident>> { + fn fn_args(&mut self) -> ParseResult> { enter_parse!(self); - self.expect(TokenType::ParenO)?; - let params = self.parse_list(TokenType::ParenC, Self::ident)?; - self.expect(TokenType::ParenC)?; + self.expect(TokenKind::ParenO)?; + let params = self.parse_list(TokenKind::ParenC, Self::ident)?; + self.expect(TokenKind::ParenC)?; exit_parse!(self); Ok(params) } - fn if_stmt(&mut self) -> ParseResult<'code, IfStmt<'ast>> { + fn if_stmt(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::If)?.span; + let keyword_span = self.expect(TokenKind::If)?.span; let cond = self.expression()?; let body = self.block()?; - let else_part = if let Some(TokenType::Else) = self.peek_kind()? { + let else_part = if let Some(TokenKind::Else) = self.peek_kind() { Some(self.else_part()?) } else { None @@ -217,12 +225,12 @@ where }) } - fn else_part(&mut self) -> ParseResult<'code, ElsePart<'ast>> { + fn else_part(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::Else)?.span; + let keyword_span = self.expect(TokenKind::Else)?.span; - let else_part = if let Some(TokenType::If) = self.peek_kind()? { + let else_part = if let Some(TokenKind::If) = self.peek_kind() { let else_if_stmt = self.if_stmt()?; let else_span = keyword_span.extend(else_if_stmt.span); Ok(ElsePart::ElseIf(else_if_stmt, else_span)) @@ -237,10 +245,10 @@ where else_part } - fn loop_stmt(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn loop_stmt(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::Loop)?.span; + let keyword_span = self.expect(TokenKind::Loop)?.span; self.inside_loop_depth += 1; let block = self.block()?; @@ -253,10 +261,10 @@ where Ok(Stmt::Loop(block, keyword_span.extend(loop_span))) } - fn while_stmt(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn while_stmt(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::While)?.span; + let keyword_span = self.expect(TokenKind::While)?.span; let cond = self.expression()?; self.inside_loop_depth += 1; @@ -272,75 +280,79 @@ where })) } - fn break_stmt(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn break_stmt(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::Break)?.span; - let semi_span = self.expect(TokenType::Semi)?.span; + let keyword_span = self.expect(TokenKind::Break)?.span; + let semi_span = self.expect(TokenKind::Semi)?.span; exit_parse!(self); if self.inside_loop_depth == 0 { - Err(ParseErr::BreakOutsideLoop(keyword_span.extend(semi_span))) + Err(CompilerError::new( + keyword_span.extend(semi_span), + "break used outside of loop".to_string(), + )) } else { Ok(Stmt::Break(keyword_span.extend(semi_span))) } } - fn return_stmt(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn return_stmt(&mut self) -> ParseResult> { enter_parse!(self); - let keyword_span = self.expect(TokenType::Return)?.span; + let keyword_span = self.expect(TokenKind::Return)?.span; - let expr = if let Some(TokenType::Semi) = self.peek_kind()? { + let expr = if let Some(TokenKind::Semi) = self.peek_kind() { None } else { Some(self.expression()?) }; - let semi_span = self.expect(TokenType::Semi)?.span; + let semi_span = self.expect(TokenKind::Semi)?.span; exit_parse!(self); if self.inside_fn_depth == 0 { - Err(ParseErr::ReturnOutsideFunction( + Err(CompilerError::new( keyword_span.extend(semi_span), + "return used outside of function".to_string(), )) } else { Ok(Stmt::Return(expr, keyword_span.extend(semi_span))) } } - fn print_stmt(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn print_stmt(&mut self) -> ParseResult> { enter_parse!(self); - let print_span = self.expect(TokenType::Print)?.span; + let print_span = self.expect(TokenKind::Print)?.span; let expr = self.expression()?; - let semi_span = self.expect(TokenType::Semi)?.span; + let semi_span = self.expect(TokenKind::Semi)?.span; exit_parse!(self); Ok(Stmt::Print(expr, print_span.extend(semi_span))) } - fn assignment(&mut self) -> ParseResult<'code, Stmt<'ast>> { + fn assignment(&mut self) -> ParseResult> { enter_parse!(self); let expr = self.expression()?; - let stmt = if let Some(TokenType::Equal) = self.peek_kind()? { - let _ = self.expect(TokenType::Equal)?; + let stmt = if let Some(TokenKind::Equal) = self.peek_kind() { + let _ = self.expect(TokenKind::Equal)?; let init = self.expression()?; - let semi_span = self.expect(TokenType::Semi)?.span; + let semi_span = self.expect(TokenKind::Semi)?.span; Ok(Stmt::Assignment(Assignment { span: expr.span().extend(semi_span), lhs: expr, rhs: init, })) } else { - let _ = self.expect(TokenType::Semi)?; + let _ = self.expect(TokenKind::Semi)?; Ok(Stmt::Expr(expr)) }; @@ -348,19 +360,19 @@ where stmt } - fn expression(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn expression(&mut self) -> ParseResult> { enter_parse!(self); let return_expr = self.logical_or(); exit_parse!(self); return_expr } - fn logical_or(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn logical_or(&mut self) -> ParseResult> { enter_parse!(self); let lhs = self.logical_and()?; - let return_expr = match self.peek_kind()? { - Some(TokenType::Or) => parse_bin_op!(self, lhs, BinaryOpKind::Or, logical_or), + let return_expr = match self.peek_kind() { + Some(TokenKind::Or) => parse_bin_op!(self, lhs, BinaryOpKind::Or, logical_or), _ => Ok(lhs), }; @@ -368,12 +380,12 @@ where return_expr } - fn logical_and(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn logical_and(&mut self) -> ParseResult> { enter_parse!(self); let lhs = self.equality()?; - let return_expr = match self.peek_kind()? { - Some(TokenType::And) => parse_bin_op!(self, lhs, BinaryOpKind::And, logical_and), + let return_expr = match self.peek_kind() { + Some(TokenKind::And) => parse_bin_op!(self, lhs, BinaryOpKind::And, logical_and), _ => Ok(lhs), }; @@ -381,15 +393,15 @@ where return_expr } - fn equality(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn equality(&mut self) -> ParseResult> { enter_parse!(self); let lhs = self.comparison()?; - let return_expr = match self.peek_kind()? { - Some(TokenType::BangEqual) => { + let return_expr = match self.peek_kind() { + Some(TokenKind::BangEqual) => { parse_bin_op!(self, lhs, BinaryOpKind::NotEqual, comparison) } - Some(TokenType::EqualEqual) => { + Some(TokenKind::EqualEqual) => { parse_bin_op!(self, lhs, BinaryOpKind::Equal, comparison) } _ => Ok(lhs), @@ -398,17 +410,17 @@ where return_expr } - fn comparison(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn comparison(&mut self) -> ParseResult> { enter_parse!(self); let lhs = self.term()?; - let return_expr = match self.peek_kind()? { - Some(TokenType::Greater) => parse_bin_op!(self, lhs, BinaryOpKind::Greater, term), - Some(TokenType::GreaterEqual) => { + let return_expr = match self.peek_kind() { + Some(TokenKind::Greater) => parse_bin_op!(self, lhs, BinaryOpKind::Greater, term), + Some(TokenKind::GreaterEqual) => { parse_bin_op!(self, lhs, BinaryOpKind::GreaterEqual, term) } - Some(TokenType::Less) => parse_bin_op!(self, lhs, BinaryOpKind::Less, term), - Some(TokenType::LessEqual) => { + Some(TokenKind::Less) => parse_bin_op!(self, lhs, BinaryOpKind::Less, term), + Some(TokenKind::LessEqual) => { parse_bin_op!(self, lhs, BinaryOpKind::LessEqual, term) } _ => Ok(lhs), @@ -417,39 +429,39 @@ where return_expr } - fn term(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn term(&mut self) -> ParseResult> { enter_parse!(self); let lhs = self.factor()?; - let return_expr = match self.peek_kind()? { - Some(TokenType::Plus) => parse_bin_op!(self, lhs, BinaryOpKind::Add, term), - Some(TokenType::Minus) => parse_bin_op!(self, lhs, BinaryOpKind::Sub, term), + let return_expr = match self.peek_kind() { + Some(TokenKind::Plus) => parse_bin_op!(self, lhs, BinaryOpKind::Add, term), + Some(TokenKind::Minus) => parse_bin_op!(self, lhs, BinaryOpKind::Sub, term), _ => Ok(lhs), }; exit_parse!(self); return_expr } - fn factor(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn factor(&mut self) -> ParseResult> { enter_parse!(self); let lhs = self.unary()?; - let return_expr = match self.peek_kind()? { - Some(TokenType::Asterisk) => parse_bin_op!(self, lhs, BinaryOpKind::Mul, factor), - Some(TokenType::Slash) => parse_bin_op!(self, lhs, BinaryOpKind::Div, factor), - Some(TokenType::Percent) => parse_bin_op!(self, lhs, BinaryOpKind::Mod, factor), + let return_expr = match self.peek_kind() { + Some(TokenKind::Asterisk) => parse_bin_op!(self, lhs, BinaryOpKind::Mul, factor), + Some(TokenKind::Slash) => parse_bin_op!(self, lhs, BinaryOpKind::Div, factor), + Some(TokenKind::Percent) => parse_bin_op!(self, lhs, BinaryOpKind::Mod, factor), _ => Ok(lhs), }; exit_parse!(self); return_expr } - fn unary(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn unary(&mut self) -> ParseResult> { enter_parse!(self); - let return_expr = match self.peek_kind()? { - Some(TokenType::Not) => { - let unary_op_span = self.next()?.unwrap().span; + let return_expr = match self.peek_kind() { + Some(TokenKind::Not) => { + let unary_op_span = self.next().unwrap().span; let expr = self.call()?; Ok(Expr::UnaryOp(Box::new_in( UnaryOp { @@ -460,8 +472,8 @@ where self.bump, ))) } - Some(TokenType::Minus) => { - let unary_op_span = self.next()?.unwrap().span; + Some(TokenKind::Minus) => { + let unary_op_span = self.next().unwrap().span; let expr = self.call()?; Ok(Expr::UnaryOp(Box::new_in( UnaryOp { @@ -478,17 +490,17 @@ where return_expr } - fn call(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn call(&mut self) -> ParseResult> { enter_parse!(self); let mut expr = self.primary()?; loop { - expr = match self.peek_kind()? { - Some(TokenType::ParenO) => { - let open_span = self.expect(TokenType::ParenO)?.span; - let args = self.parse_list(TokenType::ParenC, Self::expression)?; - let close_span = self.expect(TokenType::ParenC)?.span; + expr = match self.peek_kind() { + Some(TokenKind::ParenO) => { + let open_span = self.expect(TokenKind::ParenO)?.span; + let args = self.parse_list(TokenKind::ParenC, Self::expression)?; + let close_span = self.expect(TokenKind::ParenC)?.span; Expr::Call(Box::new_in( Call { @@ -499,8 +511,8 @@ where self.bump, )) } - Some(TokenType::Dot) => { - let dot_span = self.expect(TokenType::Dot)?.span; + Some(TokenKind::Dot) => { + let dot_span = self.expect(TokenKind::Dot)?.span; let field = self.ident()?; Expr::Call(Box::new_in( @@ -521,73 +533,80 @@ where Ok(expr) } - fn primary(&mut self) -> ParseResult<'code, Expr<'ast>> { + fn primary(&mut self) -> ParseResult> { enter_parse!(self); - let next = self.next()?.ok_or(ParseErr::Eof("primary"))?; + let next = self.next().ok_or_else(|| CompilerError::eof("primary"))?; let return_expr = match next.kind { - TokenType::String(literal) => Ok(Expr::Literal(Literal::String(literal, next.span))), - TokenType::Number(literal) => Ok(Expr::Literal(Literal::Number(literal, next.span))), - TokenType::False => Ok(Expr::Literal(Literal::Boolean(false, next.span))), - TokenType::True => Ok(Expr::Literal(Literal::Boolean(true, next.span))), - TokenType::Null => Ok(Expr::Literal(Literal::Null(next.span))), - TokenType::BraceO => self.object_literal(next.span), - TokenType::BracketO => self.array_literal(next.span), - TokenType::ParenO => { + TokenKind::String(literal) => Ok(Expr::Literal(Literal::String(literal, next.span))), + TokenKind::Number(literal) => Ok(Expr::Literal(Literal::Number(literal, next.span))), + TokenKind::False => Ok(Expr::Literal(Literal::Boolean(false, next.span))), + TokenKind::True => Ok(Expr::Literal(Literal::Boolean(true, next.span))), + TokenKind::Null => Ok(Expr::Literal(Literal::Null(next.span))), + TokenKind::BraceO => self.object_literal(next.span), + TokenKind::BracketO => self.array_literal(next.span), + TokenKind::ParenO => { let expr = self.expression()?; - let _ = self.expect(TokenType::ParenC)?; + let _ = self.expect(TokenKind::ParenC)?; Ok(expr) } - TokenType::Ident(name) => { + TokenKind::Ident(name) => { let name_owned = name.to_owned(); Ok(Expr::Ident(Ident { sym: name_owned, span: next.span, })) } - _ => Err(ParseErr::InvalidTokenPrimary(next)), + TokenKind::Error(error) => Err(error), + _ => Err(CompilerError::new( + next.span, + format!("invalid token in expression: `{:?}`", next.kind), + )), }; exit_parse!(self); return_expr } - fn ident(&mut self) -> ParseResult<'code, Ident> { + fn ident(&mut self) -> ParseResult { enter_parse!(self); - let Token { kind, span } = self.next()?.ok_or(ParseErr::Eof("identifier"))?; + let Token { kind, span } = self + .next() + .ok_or_else(|| CompilerError::eof("identifier"))?; let return_expr = match kind { - TokenType::Ident(name) => { + TokenKind::Ident(name) => { let name_owned = name.to_owned(); Ok(Ident { sym: name_owned, span, }) } + TokenKind::Error(error) => Err(error), _ => { - return Err(ParseErr::MismatchedKind { - expected: TokenType::Ident(""), - actual: Token { span, kind }, - }) + return Err(CompilerError::new( + span, + format!("expected identifier, received `{:?}`", kind), + )) } }; exit_parse!(self); return_expr } - fn object_literal(&mut self, open_span: Span) -> ParseResult<'code, Expr<'ast>> { + fn object_literal(&mut self, open_span: Span) -> ParseResult> { enter_parse!(self); - let close_span = self.expect(TokenType::BraceC)?.span; + let close_span = self.expect(TokenKind::BraceC)?.span; exit_parse!(self); Ok(Expr::Literal(Literal::Object(open_span.extend(close_span)))) } - fn array_literal(&mut self, open_span: Span) -> ParseResult<'code, Expr<'ast>> { + fn array_literal(&mut self, open_span: Span) -> ParseResult> { enter_parse!(self); - let elements = self.parse_list(TokenType::BracketC, Self::expression)?; - let closing_bracket = self.expect(TokenType::BracketC)?; + let elements = self.parse_list(TokenKind::BracketC, Self::expression)?; + let closing_bracket = self.expect(TokenKind::BracketC)?; let return_expr = Ok(Expr::Literal(Literal::Array( elements, @@ -599,32 +618,34 @@ where fn parse_list( &mut self, - close: TokenType<'code>, + close: TokenKind<'code>, mut parser: F, - ) -> ParseResult<'code, Vec<'ast, T>> + ) -> ParseResult> where - F: FnMut(&mut Self) -> ParseResult<'code, T>, + F: FnMut(&mut Self) -> ParseResult, { enter_parse!(self); let mut elements = Vec::new_in(self.bump); - if self.peek_kind()? == Some(&close) { + if self.peek_kind() == Some(&close) { return Ok(elements); } let expr = parser(self)?; elements.push(expr); - while self - .peek_kind()? - .ok_or_else(|| ParseErr::EofExpecting(close.clone()))? - != &close + while self.peek_kind().ok_or_else(|| { + CompilerError::new( + Span::dummy(), + format!("reached EOF expecting `{:?}`", close.clone()), + ) + })? != &close { - self.expect(TokenType::Comma)?; + self.expect(TokenKind::Comma)?; // trailing comma support - if self.peek_kind()? == Some(&close) { + if self.peek_kind() == Some(&close) { break; } @@ -638,96 +659,46 @@ where // token helpers - fn next(&mut self) -> ParseResult<'code, Option>> { - match self.tokens.next() { - Some(Ok(t)) => Ok(Some(t)), - Some(Err(comp_err)) => Err(ParseErr::LexError(comp_err)), - None => Ok(None), - } + fn next(&mut self) -> Option> { + self.tokens.next() } - fn peek(&mut self) -> ParseResult<'code, Option<&Token<'code>>> { - match self.tokens.peek() { - Some(Ok(t)) => Ok(Some(t)), - Some(Err(comp_err)) => Err(ParseErr::LexError(comp_err.clone())), - None => Ok(None), - } + fn peek(&mut self) -> Option<&Token<'code>> { + self.tokens.peek() } - fn peek_kind(&mut self) -> ParseResult<'code, Option<&TokenType<'code>>> { - self.peek().map(|option| option.map(|token| &token.kind)) + fn peek_kind(&mut self) -> Option<&TokenKind<'code>> { + self.peek().map(|token| &token.kind) } - fn expect(&mut self, kind: TokenType<'code>) -> ParseResult<'code, Token> { - if let Some(token) = self.next()? { + fn expect(&mut self, kind: TokenKind<'code>) -> ParseResult { + if let Some(token) = self.next() { if token.kind == kind { Ok(token) + } else if let TokenKind::Error(err) = token.kind { + Err(err) } else { - Err(ParseErr::MismatchedKind { - expected: kind, - actual: token, - }) + Err(CompilerError::new( + token.span, + format!("expected `{:?}`, received `{:?}`", kind, token.kind), + )) } } else { - Err(ParseErr::EofExpecting(kind)) + Err(CompilerError::new( + Span::dummy(), + format!("reached EOF expecting `{:?}`", kind), + )) } } } -#[derive(Debug)] -pub enum ParseErr<'code> { - MaxDepth(Span), - BreakOutsideLoop(Span), - ReturnOutsideFunction(Span), - MismatchedKind { - expected: TokenType<'code>, - actual: Token<'code>, - }, - InvalidTokenPrimary(Token<'code>), - EofExpecting(TokenType<'code>), - Eof(&'static str), - LexError(CompilerError), -} - -// todo: remove this and ParseErr -impl From> for CompilerError { - fn from(error: ParseErr<'_>) -> Self { +impl CompilerError { + fn eof(message: &str) -> Self { Self { - span: match &error { - ParseErr::MismatchedKind { - actual: Token { span, .. }, - .. - } => *span, - ParseErr::InvalidTokenPrimary(Token { span, .. }) => *span, - ParseErr::EofExpecting(_) => Span::dummy(), - ParseErr::Eof(_) => Span::dummy(), - ParseErr::BreakOutsideLoop(span) => *span, - ParseErr::ReturnOutsideFunction(span) => *span, - ParseErr::MaxDepth(span) => *span, - ParseErr::LexError(err) => err.span, - }, - message: match &error { - ParseErr::MismatchedKind { expected, actual } => { - format!("expected `{:?}`, received `{:?}`", expected, actual.kind) - } - ParseErr::InvalidTokenPrimary(token) => { - format!("invalid token in expression: `{:?}`", token.kind) - } - ParseErr::EofExpecting(token) => { - format!("reached EOF searching for `{:?}`", token) - } - ParseErr::Eof(message) => { - format!("reached EOF while parsing `{}`", message) - } - ParseErr::BreakOutsideLoop(_) => "break used outside of loop".to_string(), - ParseErr::ReturnOutsideFunction(_) => "return used outside of function".to_string(), - ParseErr::MaxDepth(_) => "reached maximal nesting depth".to_string(), - ParseErr::LexError(err) => err.message.clone(), - }, - note: match error { - ParseErr::LexError(err) => err.note.clone(), - _ => None, - }, + // todo: don't + span: Span::dummy(), + message: format!("reached EOF while parsing `{}`", message), + note: None, } } } diff --git a/src/parse/test.rs b/src/parse/test.rs index eda013a..12d899a 100644 --- a/src/parse/test.rs +++ b/src/parse/test.rs @@ -1,4 +1,4 @@ -use crate::errors::{CompilerError, Span}; +use crate::errors::Span; use crate::parse::Parser; use bumpalo::Bump; use prelude::*; @@ -6,9 +6,9 @@ use prelude::*; mod prelude { pub(super) use super::{parser, test_literal_bin_op, test_number_literal, token}; pub(super) use crate::ast::{Expr, Stmt}; - pub(super) use crate::lex::TokenType::*; + pub(super) use crate::lex::TokenKind::*; pub type Token = crate::lex::Token<'static>; - pub type TokenType = crate::lex::TokenType<'static>; + pub type TokenType = crate::lex::TokenKind<'static>; pub(super) use bumpalo::Bump; } @@ -22,13 +22,8 @@ fn token(kind: TokenType) -> Token { fn parser<'ast>( tokens: std::vec::Vec, alloc: &'ast Bump, -) -> Parser<'static, 'ast, std::vec::IntoIter>> +) -> Parser<'static, 'ast, std::vec::IntoIter> where { - let tokens = tokens - .into_iter() - .map(Ok) - .collect::>>(); - Parser { tokens: tokens.into_iter().peekable(), depth: 0,