Noratrieb/ub
1
0
Fork 0
mirror of https://github.com/Noratrieb/ub.git synced 2026-01-15 09:05:05 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

some parsing things

Browse source
This commit is contained in:
nora 2022-03-23 22:07:36 +01:00
parent 1bd5859f0c
commit 4aabcdfd76
8 changed files with 320 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

16
ub_parser/src/ast.rs
View file

@ -5,7 +5,7 @@ type Span = Range<usize>;
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct File { pub struct File {
pub name: PathBuf, pub name: PathBuf,
pub items: Vec<Stmt>, // todo make item pub items: Vec<Item>,
} }
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
@ -30,14 +30,14 @@ pub enum Item {
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct FnDecl { pub struct FnDecl {
pub name: String, pub name: String,
pub params: Vec<FnParam>, pub params: Vec<NameTyPair>,
pub ret_ty: Ty, pub ret_ty: Option<Ty>,
pub span: Span, pub span: Span,
pub body: Vec<Stmt>, pub body: Vec<Stmt>,
} }
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct FnParam { pub struct NameTyPair {
pub name: String, pub name: String,
pub ty: Ty, pub ty: Ty,
pub span: Span, pub span: Span,
@ -46,14 +46,8 @@ pub struct FnParam {
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct StructDecl { pub struct StructDecl {
pub name: String, pub name: String,
pub fields: Vec<NameTyPair>,
pub span: Span, pub span: Span,
pub fields: Vec<StructField>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct StructField {
pub name: String,
pub ty: Ty,
} }
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]

6
ub_parser/src/lexer.rs
View file

@ -58,6 +58,10 @@ pub enum Token<'a> {
AndAnd, AndAnd,
#[token("^")] #[token("^")]
Caret, Caret,
#[token("->")]
Arrow,
#[token(":")]
Colon,
// keywords // keywords
#[token("struct")] #[token("struct")]
@ -102,7 +106,7 @@ mod tests {
#[test] #[test]
fn punctuation() { fn punctuation() {
let tokens = lex_test("{} [] () .,; = == != >= <= < > + - * / | || & && ^"); let tokens = lex_test("{} [] () .,; = == != >= <= < > + - * / | || & && ^ -> :");
insta::assert_debug_snapshot!(tokens); insta::assert_debug_snapshot!(tokens);
} }

164
ub_parser/src/parser.rs
View file

@ -3,19 +3,35 @@ use std::{ops::Range, path::PathBuf};
use chumsky::{prelude::*, Stream}; use chumsky::{prelude::*, Stream};
use crate::{ use crate::{
ast::{BinOp, BinOpKind, Call, Expr, File, Literal, Stmt}, ast::{
Assignment, BinOp, BinOpKind, Call, Expr, File, FnDecl, Item, Literal, NameTyPair, Stmt,
StructDecl, Ty, TyKind, VarDecl,
},
lexer::Token, lexer::Token,
}; };
type Error<'src> = Simple<Token<'src>>; type Error<'src> = Simple<Token<'src>>;
type Span = Range<usize>; type Span = Range<usize>;
fn ident_parser<'src>() -> impl Parser<Token<'src>, &'src str, Error = Error<'src>> + Clone { fn ident_parser<'src>() -> impl Parser<Token<'src>, String, Error = Error<'src>> + Clone {
filter_map(|span, tok| match tok { filter_map(|span, tok| match tok {
Token::Ident(ident) => Ok(ident), Token::Ident(ident) => Ok(ident.to_owned()),
_ => Err(Simple::expected_input_found(span, Vec::new(), Some(tok))), _ => Err(Simple::expected_input_found(span, Vec::new(), Some(tok))),
}) })
.labelled("identifier") .labelled("identifier")
.boxed()
}
fn ty_parser<'src>() -> impl Parser<Token<'src>, Ty, Error = Error<'src>> + Clone {
filter_map(|span, token| {
let kind = match token {
Token::Ident("u64") => TyKind::U64,
_ => return Err(Simple::expected_input_found(span, Vec::new(), Some(token))),
};
Ok(Ty { span, kind })
})
.boxed()
} }
fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> + Clone { fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> + Clone {
@ -49,7 +65,8 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
.or(array) .or(array)
.or(expr .or(expr
.clone() .clone()
.delimited_by(just(Token::ParenO), just(Token::ParenC))); .delimited_by(just(Token::ParenO), just(Token::ParenC)))
.boxed();
let call = atom let call = atom
.then( .then(
@ -112,19 +129,114 @@ fn expr_parser<'src>() -> impl Parser<Token<'src>, Expr, Error = Error<'src>> +
}) })
}); });
compare compare.boxed()
})
}
fn statement_parser<'src>(
item: impl Parser<Token<'src>, Item, Error = Error<'src>> + Clone,
) -> impl Parser<Token<'src>, Stmt, Error = Error<'src>> + Clone {
let var_decl = ty_parser()
.then(ident_parser())
.then_ignore(just(Token::Eq))
.then(expr_parser())
.map(|((ty, name), rhs)| {
Stmt::VarDecl(VarDecl {
name,
ty,
rhs: Some(rhs),
span: Default::default(),
})
});
let assignment = expr_parser()
.then_ignore(just(Token::Eq))
.then(expr_parser())
.map(|(place, rhs)| {
Stmt::Assignment(Assignment {
place,
rhs,
span: Default::default(),
})
});
var_decl
.or(assignment)
.or(expr_parser().map(|expr| Stmt::Expr(expr)))
.or(item.clone().map(|item| Stmt::Item(item)))
.then_ignore(just(Token::Semi))
}
fn name_ty_pair_parser<'src>() -> impl Parser<Token<'src>, NameTyPair, Error = Error<'src>> + Clone
{
ident_parser()
.then_ignore(just(Token::Colon))
.then(ty_parser())
.map_with_span(|(name, ty), span| NameTyPair { name, ty, span })
}
fn function_parser<'src>(
item: impl Parser<Token<'src>, Item, Error = Error<'src>> + Clone,
) -> impl Parser<Token<'src>, FnDecl, Error = Error<'src>> + Clone {
let name = ident_parser();
let params = name_ty_pair_parser()
.separated_by(just(Token::Comma))
.allow_trailing()
.delimited_by(just(Token::ParenO), just(Token::ParenC))
.labelled("function arguments");
let ret_ty = just(Token::Arrow).ignore_then(ty_parser()).or_not();
just(Token::Fn)
.map_with_span(|_, span| span)
.then(name)
.then(params)
.then(ret_ty)
.then(
statement_parser(item.clone())
.repeated()
.delimited_by(just(Token::BraceO), just(Token::BraceC)),
)
.map(|((((fn_span, name), params), ret_ty), body)| FnDecl {
name,
params,
ret_ty,
span: fn_span,
body,
})
.labelled("function")
}
fn struct_parser<'src>() -> impl Parser<Token<'src>, StructDecl, Error = Error<'src>> + Clone {
let name = just(Token::Struct).ignore_then(ident_parser());
let fields = name_ty_pair_parser()
.separated_by(just(Token::Comma))
.delimited_by(just(Token::BraceO), just(Token::BraceC));
name.then(fields).map(|(name, fields)| StructDecl {
name,
fields,
span: Default::default(),
})
}
fn item_parser<'src>() -> impl Parser<Token<'src>, Item, Error = Error<'src>> + Clone {
recursive(|item| {
function_parser(item)
.map(Item::FnDecl)
.or(struct_parser().map(Item::StructDecl))
}) })
} }
fn file_parser<'src>( fn file_parser<'src>(
file_name: PathBuf, file_name: PathBuf,
) -> impl Parser<Token<'src>, File, Error = Error<'src>> + Clone { ) -> impl Parser<Token<'src>, File, Error = Error<'src>> + Clone {
expr_parser() item_parser().repeated().map(move |items| File {
.map(move |expr| File { name: file_name.clone(),
name: file_name.clone(), items,
items: vec![Stmt::Expr(expr)], })
})
.then_ignore(just(Token::Semi))
} }
pub fn parse<'src, I>(lexer: I, len: usize, file_name: PathBuf) -> (Option<File>, Vec<Error<'src>>) pub fn parse<'src, I>(lexer: I, len: usize, file_name: PathBuf) -> (Option<File>, Vec<Error<'src>>)
@ -132,7 +244,7 @@ where
I: 'src, I: 'src,
I: Iterator<Item = (Token<'src>, Span)>, I: Iterator<Item = (Token<'src>, Span)>,
{ {
file_parser(file_name).parse_recovery(Stream::from_iter(len..len + 1, lexer)) file_parser(file_name).parse_recovery_verbose(Stream::from_iter(len..len + 1, lexer))
} }
#[cfg(test)] #[cfg(test)]
@ -156,13 +268,37 @@ mod tests {
#[test] #[test]
fn addition() { fn addition() {
let r = parse("1 + 4;"); let r = parse("fn main() { 1 + 4; }");
insta::assert_debug_snapshot!(r); insta::assert_debug_snapshot!(r);
} }
#[test] #[test]
fn expression() { fn expression() {
let r = parse("(4 / hallo()) + 5;"); let r = parse("fn main() { (4 / hallo()) + 5; }");
insta::assert_debug_snapshot!(r)
}
#[test]
fn function() {
let r = parse("fn foo() -> u64 { 1 + 5; }");
insta::assert_debug_snapshot!(r)
}
#[test]
fn nested_function() {
let r = parse("fn foo() { fn foo2() {} fn foo3() {} }");
insta::assert_debug_snapshot!(r)
}
#[test]
fn nested_function2() {
let r = parse("fn foo() { fn foo2() {} 1 + 5; }");
insta::assert_debug_snapshot!(r)
}
#[test]
fn struct_() {
let r = parse("struct X { y: u64, x: u64 }");
insta::assert_debug_snapshot!(r) insta::assert_debug_snapshot!(r)
} }
} }

3
ub_parser/src/snapshots/ub_parser__lexer__tests__punctuation.snap
View file

@ -1,5 +1,6 @@
--- ---
source: ub_parser/src/lexer.rs source: ub_parser/src/lexer.rs
assertion_line: 110
expression: tokens expression: tokens
--- ---
[ [
@ -28,4 +29,6 @@ expression: tokens
And, And,
AndAnd, AndAnd,
Caret, Caret,
Arrow,
Colon,
] ]

45
ub_parser/src/snapshots/ub_parser__parser__tests__addition.snap
View file

@ -1,5 +1,6 @@
--- ---
source: ub_parser/src/parser.rs source: ub_parser/src/parser.rs
assertion_line: 272
expression: r expression: r
--- ---
( (
@ -7,25 +8,35 @@ expression: r
File { File {
name: "ub_parser__parser__tests", name: "ub_parser__parser__tests",
items: [ items: [
Expr( FnDecl(
BinOp( FnDecl {
BinOp { name: "main",
kind: Add, params: [],
lhs: Literal( ret_ty: None,
Integer( span: 0..2,
1, body: [
0..1, Expr(
BinOp(
BinOp {
kind: Add,
lhs: Literal(
Integer(
1,
12..13,
),
),
rhs: Literal(
Integer(
4,
16..17,
),
),
span: 0..0,
},
), ),
), ),
rhs: Literal( ],
Integer( },
4,
4..5,
),
),
span: 0..0,
},
),
), ),
], ],
}, },

69
ub_parser/src/snapshots/ub_parser__parser__tests__expression.snap
View file

@ -1,5 +1,6 @@
--- ---
source: ub_parser/src/parser.rs source: ub_parser/src/parser.rs
assertion_line: 278
expression: r expression: r
--- ---
( (
@ -7,39 +8,49 @@ expression: r
File { File {
name: "ub_parser__parser__tests", name: "ub_parser__parser__tests",
items: [ items: [
Expr( FnDecl(
BinOp( FnDecl {
BinOp { name: "main",
kind: Add, params: [],
lhs: BinOp( ret_ty: None,
BinOp { span: 0..2,
kind: Div, body: [
lhs: Literal( Expr(
Integer( BinOp(
4, BinOp {
1..2, kind: Add,
lhs: BinOp(
BinOp {
kind: Div,
lhs: Literal(
Integer(
4,
13..14,
),
),
rhs: Call(
Call {
callee: Name(
"hallo",
),
args: [],
},
),
span: 0..0,
},
), ),
), rhs: Literal(
rhs: Call( Integer(
Call { 5,
callee: Name( 28..29,
"hallo",
), ),
args: [], ),
}, span: 0..0,
), },
span: 0..0,
},
),
rhs: Literal(
Integer(
5,
16..17,
), ),
), ),
span: 0..0, ],
}, },
),
), ),
], ],
}, },

50
ub_parser/src/snapshots/ub_parser__parser__tests__function.snap Normal file
View file

@ -0,0 +1,50 @@
---
source: ub_parser/src/parser.rs
assertion_line: 284
expression: r
---
(
Some(
File {
name: "ub_parser__parser__tests",
items: [
FnDecl(
FnDecl {
name: "foo",
params: [],
ret_ty: Some(
Ty {
span: 12..15,
kind: U64,
},
),
span: 0..2,
body: [
Expr(
BinOp(
BinOp {
kind: Add,
lhs: Literal(
Integer(
1,
18..19,
),
),
rhs: Literal(
Integer(
5,
22..23,
),
),
span: 0..0,
},
),
),
],
},
),
],
},
),
[],
)

39
ub_parser/src/snapshots/ub_parser__parser__tests__struct_.snap Normal file
View file

@ -0,0 +1,39 @@
---
source: ub_parser/src/parser.rs
assertion_line: 290
expression: r
---
(
Some(
File {
name: "ub_parser__parser__tests",
items: [
StructDecl(
StructDecl {
name: "X",
fields: [
NameTyPair {
name: "y",
ty: Ty {
span: 14..17,
kind: U64,
},
span: 11..17,
},
NameTyPair {
name: "x",
ty: Ty {
span: 22..25,
kind: U64,
},
span: 19..25,
},
],
span: 0..0,
},
),
],
},
),
[],
)