create preprocessor

parser/src/lib.rs

@@ -12,7 +12,7 @@ mod pre;
 mod pretty;
 mod token;
 
-pub use parser::Parser;
+pub use crate::parser::Parser;
 
 pub type Spanned<T> = (T, Span);
 

parser/src/parser/expr.rs

@@ -7,7 +7,7 @@ use crate::{
         ArithOpKind, Atom, BinaryOp, ComparisonKind, Expr, ExprBinary, ExprPostfix, ExprUnary,
         PostfixOp, UnaryOp,
     },
-    parser::{expect, Parser, ParserError, Result, eat},
+    parser::{eat, expect, Parser, ParserError, Result},
     pre::Punctuator as P,
     token::{Constant, Token as Tok},
     Span, Spanned,

parser/src/pre/lexer.rs

@@ -3,7 +3,7 @@
 //!
 //! Code might be bad. Possibly.
 
-use std::{fmt::Display, ops::Not};
+use std::{fmt::Display, iter::Enumerate, ops::Not, str::Bytes};
 
 use peekmore::PeekMore;
 
@@ -176,7 +176,7 @@ impl Display for Punctuator {
     }
 }
 
-struct PLexer<'src, I>
+pub struct PLexer<'src, I>
 where
     I: Iterator<Item = (usize, u8)>,
 {
@@ -184,6 +184,15 @@ where
     src: peekmore::PeekMoreIterator<I>,
 }
 
+impl<'src> PLexer<'src, Enumerate<Bytes<'src>>> {
+    pub fn new(src_str: &'src str) -> Self {
+        Self {
+            src_str,
+            src: src_str.bytes().enumerate().peekmore(),
+        }
+    }
+}
+
 impl<'src, I> PLexer<'src, I>
 where
     I: Iterator<Item = (usize, u8)>,
@@ -409,19 +418,11 @@ where
     }
 }
 
-pub fn preprocess_tokens(src: &str) -> impl Iterator<Item = (PToken<'_>, Span)> {
-    let lexer = PLexer {
-        src_str: src,
-        src: src.bytes().enumerate().peekmore(),
-    };
-    lexer
-}
-
 #[cfg(test)]
 mod tests {
     macro_rules! lex_test {
         ($str:expr) => {
-            let tokens = super::preprocess_tokens($str);
+            let tokens = super::super::preprocess_tokens($str);
             let tokens = tokens.collect::<Vec<_>>();
             insta::assert_debug_snapshot!(tokens);
         };

parser/src/pre/mod.rs

@@ -1,3 +1,29 @@
 mod lexer;
 
-pub use lexer::{preprocess_tokens, PToken, Punctuator};
+use lexer::PLexer;
+pub use lexer::{PToken, Punctuator};
+
+use crate::Span;
+
+pub struct Preprocessor<L> {
+    lexer: L,
+}
+
+impl<'src, L> Iterator for Preprocessor<L>
+where
+    L: Iterator<Item = (PToken<'src>, Span)>,
+{
+    type Item = (PToken<'src>, Span);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.lexer.next()
+    }
+}
+
+pub fn preprocess_tokens(src: &str) -> impl Iterator<Item = (PToken<'_>, Span)> {
+    let lexer = PLexer::new(src);
+
+    let preprocessor = Preprocessor { lexer };
+
+    preprocessor
+}