Noratrieb/ub
1
0
Fork 0
mirror of https://github.com/Noratrieb/ub.git synced 2026-01-14 16:45:05 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

I AM NORMAL AND CAN BE TRUSTED WITH PARSER STATE AND LIFETIMES

Browse source
This commit is contained in:
nora 2023-01-22 20:07:45 +01:00
parent c9b85152ed
commit 6c92f2b41f
13 changed files with 247 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

13
parser/src/ast.rs
View file

@ -1,5 +1,8 @@
use std::{ops::Range, path::PathBuf};
#[derive(Debug, Clone, PartialEq)]
pub struct NodeId(u32);
type Span = Range<usize>;
#[derive(Debug, Clone, PartialEq)]
@ -32,6 +35,7 @@ pub struct FnDecl {
pub name: String,
pub params: Vec<NameTyPair>,
pub ret_ty: Option<Ty>,
pub id: NodeId,
pub span: Span,
pub body: Vec<Stmt>,
}
@ -40,6 +44,7 @@ pub struct FnDecl {
pub struct NameTyPair {
pub name: String,
pub ty: Ty,
pub id: NodeId,
pub span: Span,
}
@ -47,6 +52,7 @@ pub struct NameTyPair {
pub struct StructDecl {
pub name: String,
pub fields: Vec<NameTyPair>,
pub id: NodeId,
pub span: Span,
}
@ -106,6 +112,7 @@ pub struct LoopStmt {
#[derive(Debug, Clone, PartialEq)]
pub struct Expr {
pub kind: ExprKind,
pub id: NodeId,
pub span: Span,
}
@ -183,3 +190,9 @@ pub enum Literal {
String(String, Span),
Integer(u64, Span),
}
impl NodeId {
pub(crate) fn new(id: u32) -> Self {
Self(id)
}
}

3
parser/src/lib.rs
View file

@ -28,8 +28,9 @@ fn main() {
let lexer = Token::lexer(src);
let len = lexer.source().len();
let state = parser::ParserState::default();
let (file, errors) = parser::parse(lexer.spanned(), len, "test_file".into());
let (file, errors) = parser::parse(lexer.spanned(), &state, len, "test_file".into());
if let Some(file) = file {
println!("AST: {file:#?}");

132
parser/src/parser.rs
View file

@ -1,11 +1,11 @@
use std::{ops::Range, path::PathBuf};
use std::{cell::Cell, ops::Range, path::PathBuf};
use chumsky::{prelude::*, Stream};
use crate::{
ast::{
Assignment, BinOp, BinOpKind, Call, ElsePart, Expr, ExprKind, File, FnDecl, IfStmt, Item,
Literal, NameTyPair, Stmt, StructDecl, Ty, TyKind, UnaryOp, UnaryOpKind, VarDecl,
Literal, NameTyPair, NodeId, Stmt, StructDecl, Ty, TyKind, UnaryOp, UnaryOpKind, VarDecl,
WhileStmt,
},
lexer::Token,
@ -14,6 +14,19 @@ use crate::{
type Error<'src> = Simple<Token<'src>>;
type Span = Range<usize>;
#[derive(Default)]
pub struct ParserState {
next_id: Cell<u32>,
}
impl ParserState {
pub fn next_id(&self) -> NodeId {
let next = self.next_id.get();
self.next_id.set(next + 1);
NodeId::new(next)
}
}
fn ident_parser<'src>() -> impl Parser<Token<'src>, String, Error = Error<'src>> + Clone {
let ident = select! {
Token::Ident(ident) => ident.to_owned(),
@ -51,7 +64,9 @@ fn ty_parser<'src>() -> impl Parser<Token<'src>, Ty, Error = Error<'src>> + Clon
})
}
fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> + Clone {
fn expr_parser<'src>(
state: &'src ParserState,
) -> impl Parser<Token<'src>, Expr, Error = Error<'src>> + Clone + 'src {
recursive(|expr| {
let literal = filter_map(|span: Span, token| match token {
Token::String(str) => Ok(Expr {
@ -59,11 +74,13 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
str[1..str.len() - 2].to_owned(),
span.clone(),
)),
id: state.next_id(),
span,
}),
// todo lol unwrap
Token::Integer(int) => Ok(Expr {
kind: ExprKind::Literal(Literal::Integer(int.parse().unwrap(), span.clone())),
id: state.next_id(),
span,
}),
_ => Err(Simple::expected_input_found(span, Vec::new(), Some(token))),
@ -83,12 +100,14 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
.delimited_by(just(Token::BracketO), just(Token::BracketC))
.map_with_span(|exprs: Vec<Expr>, span| Expr {
kind: ExprKind::Array(exprs),
id: state.next_id(),
span,
});
let atom = literal
.or(ident_parser().map_with_span(|name, span| Expr {
kind: ExprKind::Name(name),
id: state.next_id(),
span,
}))
.or(array)
@ -112,6 +131,7 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
callee: Box::new(callee),
args,
}),
id: state.next_id(),
span,
}
})
@ -134,6 +154,7 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
kind,
span: span.clone(),
}),
id: state.next_id(),
span,
}
})
@ -156,6 +177,7 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
rhs: Box::new(b),
span: span.clone(),
}),
id: state.next_id(),
span,
}
});
@ -176,6 +198,7 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
rhs: Box::new(b),
span: span.clone(),
}),
id: state.next_id(),
span,
}
})
@ -198,6 +221,7 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
rhs: Box::new(b),
span: span.clone(),
}),
id: state.next_id(),
span,
}
});
@ -205,12 +229,14 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
})
}
fn statement_parser<'src>() -> impl Parser<Token<'src>, Stmt, Error = Error<'src>> + Clone {
fn statement_parser<'src>(
state: &'src ParserState,
) -> impl Parser<Token<'src>, Stmt, Error = Error<'src>> + Clone {
recursive(|stmt| {
let var_decl = ty_parser()
.then(ident_parser())
.then_ignore(just(Token::Eq))
.then(expr_parser())
.then(expr_parser(state))
.then_ignore(just(Token::Semi))
.map(|((ty, name), rhs)| {
Stmt::VarDecl(VarDecl {
@ -222,9 +248,9 @@ fn statement_parser<'src>() -> impl Parser<Token<'src>, Stmt, Error = Error<'src
})
.boxed();
let assignment = expr_parser()
let assignment = expr_parser(state)
.then_ignore(just(Token::Eq))
.then(expr_parser())
.then(expr_parser(state))
.then_ignore(just(Token::Semi))
.map(|(place, rhs)| {
Stmt::Assignment(Assignment {
@ -240,14 +266,14 @@ fn statement_parser<'src>() -> impl Parser<Token<'src>, Stmt, Error = Error<'src
.delimited_by(just(Token::BraceO), just(Token::BraceC));
let while_loop = just(Token::While)
.ignore_then(expr_parser())
.ignore_then(expr_parser(state))
.then(block.clone())
.map_with_span(|(cond, body), span| Stmt::WhileStmt(WhileStmt { cond, body, span }))
.labelled("while loop");
let if_stmt = recursive(|if_stmt| {
just(Token::If)
.ignore_then(expr_parser())
.ignore_then(expr_parser(state))
.then(block.clone())
.then(
just(Token::Else)
@ -270,7 +296,9 @@ fn statement_parser<'src>() -> impl Parser<Token<'src>, Stmt, Error = Error<'src
var_decl
.or(assignment)
.or(expr_parser().then_ignore(just(Token::Semi)).map(Stmt::Expr))
.or(expr_parser(state)
.then_ignore(just(Token::Semi))
.map(Stmt::Expr))
.or(if_stmt)
.or(while_loop)
})
@ -278,18 +306,26 @@ fn statement_parser<'src>() -> impl Parser<Token<'src>, Stmt, Error = Error<'src
.boxed()
}
fn name_ty_pair_parser<'src>() -> impl Parser<Token<'src>, NameTyPair, Error = Error<'src>> + Clone
{
fn name_ty_pair_parser<'src>(
state: &'src ParserState,
) -> impl Parser<Token<'src>, NameTyPair, Error = Error<'src>> + Clone {
ident_parser()
.then_ignore(just(Token::Colon))
.then(ty_parser())
.map_with_span(|(name, ty), span| NameTyPair { name, ty, span })
.map_with_span(|(name, ty), span| NameTyPair {
name,
ty,
id: state.next_id(),
span,
})
}
fn struct_parser<'src>() -> impl Parser<Token<'src>, StructDecl, Error = Error<'src>> + Clone {
fn struct_parser<'src>(
state: &'src ParserState,
) -> impl Parser<Token<'src>, StructDecl, Error = Error<'src>> + Clone {
let name = just(Token::Struct).ignore_then(ident_parser());
let fields = name_ty_pair_parser()
let fields = name_ty_pair_parser(state)
.separated_by(just(Token::Comma))
.delimited_by(just(Token::BraceO), just(Token::BraceC));
@ -297,17 +333,20 @@ fn struct_parser<'src>() -> impl Parser<Token<'src>, StructDecl, Error = Error<'
.map(|(name, fields)| StructDecl {
name,
fields,
id: state.next_id(),
span: Default::default(),
})
.labelled("struct")
}
fn item_parser<'src>() -> impl Parser<Token<'src>, Item, Error = Error<'src>> + Clone {
fn item_parser<'src>(
state: &'src ParserState,
) -> impl Parser<Token<'src>, Item, Error = Error<'src>> + Clone {
// ---- function
let name = ident_parser();
let params = name_ty_pair_parser()
let params = name_ty_pair_parser(state)
.separated_by(just(Token::Comma))
.allow_trailing()
.delimited_by(just(Token::ParenO), just(Token::ParenC))
@ -319,7 +358,7 @@ fn item_parser<'src>() -> impl Parser<Token<'src>, Item, Error = Error<'src>> +
.then(params)
.then(ret_ty)
.then(
statement_parser()
statement_parser(state)
.repeated()
.delimited_by(just(Token::BraceO), just(Token::BraceC)),
)
@ -327,6 +366,7 @@ fn item_parser<'src>() -> impl Parser<Token<'src>, Item, Error = Error<'src>> +
name,
params,
ret_ty,
id: state.next_id(),
span,
body,
})
@ -336,14 +376,15 @@ fn item_parser<'src>() -> impl Parser<Token<'src>, Item, Error = Error<'src>> +
function
.map(Item::FnDecl)
.or(struct_parser().map(Item::StructDecl))
.or(struct_parser(state).map(Item::StructDecl))
.labelled("item")
}
fn file_parser<'src>(
file_name: PathBuf,
state: &'src ParserState,
) -> impl Parser<Token<'src>, File, Error = Error<'src>> + Clone {
item_parser()
item_parser(state)
.repeated()
.then_ignore(end())
.map(move |items| File {
@ -353,12 +394,17 @@ fn file_parser<'src>(
.labelled("file")
}
pub fn parse<'src, I>(lexer: I, len: usize, file_name: PathBuf) -> (Option<File>, Vec<Error<'src>>)
pub fn parse<'src, I>(
lexer: I,
state: &'src ParserState,
len: usize,
file_name: PathBuf,
) -> (Option<File>, Vec<Error<'src>>)
where
I: 'src,
I: Iterator<Item = (Token<'src>, Span)>,
{
file_parser(file_name).parse_recovery_verbose(Stream::from_iter(len..len + 1, lexer))
file_parser(file_name, state).parse_recovery_verbose(Stream::from_iter(len..len + 1, lexer))
}
#[cfg(test)]
@ -367,14 +413,16 @@ mod tests {
use logos::Logos;
use super::ParserState;
use crate::lexer::Token;
fn parse(src: &str) -> impl Debug + '_ {
fn parse<'src>(src: &'src str, state: &'src ParserState) -> impl Debug + 'src {
let lexer = Token::lexer(src);
let len = lexer.source().len();
super::parse(
lexer.spanned(),
state,
len,
PathBuf::from(module_path!().replace("::", "__")),
)
@ -382,18 +430,21 @@ mod tests {
#[test]
fn addition() {
let r = parse("fn main() { 1 + 4; }");
let state = ParserState::default();
let r = parse("fn main() { 1 + 4; }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn expression() {
let r = parse("fn main() { (4 / hallo()) + 5; }");
let state = ParserState::default();
let r = parse("fn main() { (4 / hallo()) + 5; }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn unary() {
let state = ParserState::default();
let r = parse(
"fn main() {
-(*5);
@ -401,49 +452,66 @@ mod tests {
2 + &8;
*6 * *8; // :)
}",
&state,
);
insta::assert_debug_snapshot!(r);
}
#[test]
fn function() {
let r = parse("fn foo() -> u64 { 1 + 5; }");
let state = ParserState::default();
let r = parse("fn foo() -> u64 { 1 + 5; }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn if_no_else() {
let r = parse("fn foo() -> u64 { if false {} }");
let state = ParserState::default();
let r = parse("fn foo() -> u64 { if false {} }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn if_else() {
let r = parse("fn foo() -> u64 { if false {} else {} }");
let state = ParserState::default();
let r = parse("fn foo() -> u64 { if false {} else {} }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn while_loop() {
let r = parse("fn foo() -> u64 { while false {} }");
let state = ParserState::default();
let r = parse("fn foo() -> u64 { while false {} }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn var_decl() {
let r = parse("fn foo() -> u64 { u64 hello = 5; }");
let state = ParserState::default();
let r = parse("fn foo() -> u64 { u64 hello = 5; }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn struct_() {
let r = parse("struct X { y: u64, x: u64 }");
let state = ParserState::default();
let r = parse("struct X { y: u64, x: u64 }", &state);
insta::assert_debug_snapshot!(r);
}
#[test]
fn types() {
let r = parse("fn types() -> ptr u64 { Test test = 2; ptr u64 int = 25; }");
let state = ParserState::default();
let r = parse(
"fn types() -> ptr u64 { Test test = 2; ptr u64 int = 25; }",
&state,
);
insta::assert_debug_snapshot!(r);
}
}

12
parser/src/snapshots/parser__parser__tests__addition.snap
View file

@ -12,6 +12,9 @@ expression: r
name: "main",
params: [],
ret_ty: None,
id: NodeId(
6,
),
span: 0..20,
body: [
Expr(
@ -26,6 +29,9 @@ expression: r
12..13,
),
),
id: NodeId(
3,
),
span: 12..13,
},
rhs: Expr {
@ -35,11 +41,17 @@ expression: r
16..17,
),
),
id: NodeId(
4,
),
span: 16..17,
},
span: 12..17,
},
),
id: NodeId(
5,
),
span: 12..17,
},
),

21
parser/src/snapshots/parser__parser__tests__expression.snap
View file

@ -12,6 +12,9 @@ expression: r
name: "main",
params: [],
ret_ty: None,
id: NodeId(
12,
),
span: 0..32,
body: [
Expr(
@ -30,6 +33,9 @@ expression: r
13..14,
),
),
id: NodeId(
6,
),
span: 13..14,
},
rhs: Expr {
@ -39,16 +45,25 @@ expression: r
kind: Name(
"hallo",
),
id: NodeId(
7,
),
span: 17..22,
},
args: [],
},
),
id: NodeId(
8,
),
span: 17..22,
},
span: 13..22,
},
),
id: NodeId(
9,
),
span: 13..22,
},
rhs: Expr {
@ -58,11 +73,17 @@ expression: r
28..29,
),
),
id: NodeId(
10,
),
span: 28..29,
},
span: 13..29,
},
),
id: NodeId(
11,
),
span: 13..29,
},
),

12
parser/src/snapshots/parser__parser__tests__function.snap
View file

@ -17,6 +17,9 @@ expression: r
kind: U64,
},
),
id: NodeId(
6,
),
span: 0..26,
body: [
Expr(
@ -31,6 +34,9 @@ expression: r
18..19,
),
),
id: NodeId(
3,
),
span: 18..19,
},
rhs: Expr {
@ -40,11 +46,17 @@ expression: r
22..23,
),
),
id: NodeId(
4,
),
span: 22..23,
},
span: 18..23,
},
),
id: NodeId(
5,
),
span: 18..23,
},
),

6
parser/src/snapshots/parser__parser__tests__if_else.snap
View file

@ -17,6 +17,9 @@ expression: r
kind: U64,
},
),
id: NodeId(
1,
),
span: 0..39,
body: [
IfStmt(
@ -25,6 +28,9 @@ expression: r
kind: Name(
"false",
),
id: NodeId(
0,
),
span: 21..26,
},
body: [],

6
parser/src/snapshots/parser__parser__tests__if_no_else.snap
View file

@ -17,6 +17,9 @@ expression: r
kind: U64,
},
),
id: NodeId(
1,
),
span: 0..31,
body: [
IfStmt(
@ -25,6 +28,9 @@ expression: r
kind: Name(
"false",
),
id: NodeId(
0,
),
span: 21..26,
},
body: [],

10
parser/src/snapshots/parser__parser__tests__struct_.snap
View file

@ -1,6 +1,5 @@
---
source: parser/src/parser.rs
assertion_line: 302
expression: r
---
(
@ -18,6 +17,9 @@ expression: r
span: 14..17,
kind: U64,
},
id: NodeId(
0,
),
span: 11..17,
},
NameTyPair {
@ -26,9 +28,15 @@ expression: r
span: 22..25,
kind: U64,
},
id: NodeId(
1,
),
span: 19..25,
},
],
id: NodeId(
2,
),
span: 0..0,
},
),

9
parser/src/snapshots/parser__parser__tests__types.snap
View file

@ -22,6 +22,9 @@ expression: r
),
},
),
id: NodeId(
2,
),
span: 0..58,
body: [
VarDecl(
@ -41,6 +44,9 @@ expression: r
36..37,
),
),
id: NodeId(
0,
),
span: 36..37,
},
),
@ -67,6 +73,9 @@ expression: r
53..55,
),
),
id: NodeId(
1,
),
span: 53..55,
},
),

45
parser/src/snapshots/parser__parser__tests__unary.snap
View file

@ -12,6 +12,9 @@ expression: r
name: "main",
params: [],
ret_ty: None,
id: NodeId(
28,
),
span: 0..63,
body: [
Expr(
@ -28,18 +31,27 @@ expression: r
19..20,
),
),
id: NodeId(
3,
),
span: 19..20,
},
kind: Deref,
span: 19..20,
},
),
id: NodeId(
4,
),
span: 19..20,
},
kind: Neg,
span: 19..20,
},
),
id: NodeId(
5,
),
span: 19..20,
},
),
@ -54,12 +66,18 @@ expression: r
28..29,
),
),
id: NodeId(
8,
),
span: 28..29,
},
kind: AddrOf,
span: 28..29,
},
),
id: NodeId(
9,
),
span: 28..29,
},
),
@ -75,6 +93,9 @@ expression: r
35..36,
),
),
id: NodeId(
14,
),
span: 35..36,
},
rhs: Expr {
@ -87,17 +108,26 @@ expression: r
40..41,
),
),
id: NodeId(
15,
),
span: 40..41,
},
kind: AddrOf,
span: 40..41,
},
),
id: NodeId(
16,
),
span: 40..41,
},
span: 35..41,
},
),
id: NodeId(
17,
),
span: 35..41,
},
),
@ -116,12 +146,18 @@ expression: r
48..49,
),
),
id: NodeId(
23,
),
span: 48..49,
},
kind: Deref,
span: 48..49,
},
),
id: NodeId(
24,
),
span: 48..49,
},
rhs: Expr {
@ -134,17 +170,26 @@ expression: r
53..54,
),
),
id: NodeId(
25,
),
span: 53..54,
},
kind: Deref,
span: 53..54,
},
),
id: NodeId(
26,
),
span: 53..54,
},
span: 48..54,
},
),
id: NodeId(
27,
),
span: 48..54,
},
),

6
parser/src/snapshots/parser__parser__tests__var_decl.snap
View file

@ -17,6 +17,9 @@ expression: r
kind: U64,
},
),
id: NodeId(
1,
),
span: 0..34,
body: [
VarDecl(
@ -34,6 +37,9 @@ expression: r
30..31,
),
),
id: NodeId(
0,
),
span: 30..31,
},
),

6
parser/src/snapshots/parser__parser__tests__while_loop.snap
View file

@ -17,6 +17,9 @@ expression: r
kind: U64,
},
),
id: NodeId(
1,
),
span: 0..34,
body: [
WhileStmt(
@ -25,6 +28,9 @@ expression: r
kind: Name(
"false",
),
id: NodeId(
0,
),
span: 24..29,
},
body: [],