lex things

parser/Cargo.toml

@@ -7,3 +7,6 @@ edition = "2021"
 
 [dependencies]
 peekmore = { version = "1.0.0", features = ["smallvec"] }
+
+[dev-dependencies]
+insta = "1.15.0"

parser/src/lib.rs

@@ -1,3 +1,6 @@
 #![allow(dead_code)]
 
-mod lexer;
+mod pre;
+mod token;
+
+pub type Span = std::ops::Range<usize>;

parser/src/pre/lexer.rs

@@ -7,8 +7,9 @@ use std::ops::Not;
 
 use peekmore::PeekMore;
 
-type Span = std::ops::Range<usize>;
+use crate::Span;
 
+#[derive(Debug)]
 pub enum PToken {
     HeaderName(Vec<u8>),
     Identifier(Vec<u8>),
@@ -20,20 +21,7 @@ pub enum PToken {
     Error,
 }
 
-pub enum Token {
-    Keyword(Keyword),
-    Identifier(),
-    Constant(),
-    StringLiteral(),
-    Punctuator(Punctuator),
-}
-
-pub enum Keyword {}
-
-pub enum Constant {
-    Integer(i64),
-}
-
+#[derive(Debug)]
 pub enum Punctuator {
     /// [   <:
     BracketOpen,
@@ -363,3 +351,23 @@ pub fn preprocess_tokens(
     };
     lexer
 }
+
+#[cfg(test)]
+mod tests {
+    fn lex_test(str: &str) {
+        let bytes = str.bytes().enumerate();
+        let tokens = super::preprocess_tokens(bytes);
+        let tokens = tokens.collect::<Vec<_>>();
+        insta::assert_debug_snapshot!(tokens);
+    }
+
+    #[test]
+    fn hello_world() {
+        let src = r#"\
+int main() {
+    puts("Hello, World!");
+}
+"#;
+    lex_test(src);
+    }
+}

parser/src/pre/mod.rs

@@ -0,0 +1,3 @@
+mod lexer;
+
+pub use lexer::{preprocess_tokens, PToken, Punctuator};

parser/src/token.rs

@@ -0,0 +1,21 @@
+use crate::{
+    pre::{PToken, Punctuator},
+    Span,
+};
+
+pub enum Token {
+    Keyword(Keyword),
+    Identifier(),
+    Constant(),
+    StringLiteral(),
+    Punctuator(Punctuator),
+    Error,
+}
+
+pub struct Keyword;
+
+fn from_pre_toks(
+    pre_toks: impl Iterator<Item = (PToken, Span)>,
+) -> impl IntoIterator<Item = (Token, Span)> {
+    pre_toks.map(|token| todo!())
+}