From cfb6ef500a86bb035ae27339fe00551bb800832f Mon Sep 17 00:00:00 2001
From: nils <48135649+Nilstrieb@users.noreply.github.com>
Date: Mon, 18 Jul 2022 16:13:21 +0200
Subject: [PATCH] create preprocessor

---
 codegen_x86/src/lib.rs    |  2 +-
 parser/src/lib.rs         |  2 +-
 parser/src/parser.rs      |  2 +-
 parser/src/parser/expr.rs |  2 +-
 parser/src/pre/lexer.rs   | 23 ++++++++++++-----------
 parser/src/pre/mod.rs     | 28 +++++++++++++++++++++++++++-
 6 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/codegen_x86/src/lib.rs b/codegen_x86/src/lib.rs
index 43e55db..fafa470 100644
--- a/codegen_x86/src/lib.rs
+++ b/codegen_x86/src/lib.rs
@@ -3,4 +3,4 @@
 
 pub fn generate() {
     println!("ud2");
-}
\ No newline at end of file
+}
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 82be820..c6f9e46 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -12,7 +12,7 @@ mod pre;
 mod pretty;
 mod token;
 
-pub use parser::Parser;
+pub use crate::parser::Parser;
 
 pub type Spanned<T> = (T, Span);
 
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index fe9fa1d..8406b97 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -552,7 +552,7 @@ where
 impl<'src, I> Iterator for Parser<'src, I>
 where
     I: Iterator<Item = (Tok<'src>, Span)>,
-     {
+{
     type Item = Result<Spanned<ExternalDecl>>;
 
     fn next(&mut self) -> Option<Self::Item> {
diff --git a/parser/src/parser/expr.rs b/parser/src/parser/expr.rs
index c521ba3..eb6bff1 100644
--- a/parser/src/parser/expr.rs
+++ b/parser/src/parser/expr.rs
@@ -7,7 +7,7 @@ use crate::{
         ArithOpKind, Atom, BinaryOp, ComparisonKind, Expr, ExprBinary, ExprPostfix, ExprUnary,
         PostfixOp, UnaryOp,
     },
-    parser::{expect, Parser, ParserError, Result, eat},
+    parser::{eat, expect, Parser, ParserError, Result},
     pre::Punctuator as P,
     token::{Constant, Token as Tok},
     Span, Spanned,
diff --git a/parser/src/pre/lexer.rs b/parser/src/pre/lexer.rs
index 3c83dff..2a6b259 100644
--- a/parser/src/pre/lexer.rs
+++ b/parser/src/pre/lexer.rs
@@ -3,7 +3,7 @@
 //!
 //! Code might be bad. Possibly.
 
-use std::{fmt::Display, ops::Not};
+use std::{fmt::Display, iter::Enumerate, ops::Not, str::Bytes};
 
 use peekmore::PeekMore;
 
@@ -176,7 +176,7 @@ impl Display for Punctuator {
     }
 }
 
-struct PLexer<'src, I>
+pub struct PLexer<'src, I>
 where
     I: Iterator<Item = (usize, u8)>,
 {
@@ -184,6 +184,15 @@ where
     src: peekmore::PeekMoreIterator<I>,
 }
 
+impl<'src> PLexer<'src, Enumerate<Bytes<'src>>> {
+    pub fn new(src_str: &'src str) -> Self {
+        Self {
+            src_str,
+            src: src_str.bytes().enumerate().peekmore(),
+        }
+    }
+}
+
 impl<'src, I> PLexer<'src, I>
 where
     I: Iterator<Item = (usize, u8)>,
@@ -409,19 +418,11 @@ where
     }
 }
 
-pub fn preprocess_tokens(src: &str) -> impl Iterator<Item = (PToken<'_>, Span)> {
-    let lexer = PLexer {
-        src_str: src,
-        src: src.bytes().enumerate().peekmore(),
-    };
-    lexer
-}
-
 #[cfg(test)]
 mod tests {
     macro_rules! lex_test {
         ($str:expr) => {
-            let tokens = super::preprocess_tokens($str);
+            let tokens = super::super::preprocess_tokens($str);
             let tokens = tokens.collect::<Vec<_>>();
             insta::assert_debug_snapshot!(tokens);
         };
diff --git a/parser/src/pre/mod.rs b/parser/src/pre/mod.rs
index f711db4..d03a8a6 100644
--- a/parser/src/pre/mod.rs
+++ b/parser/src/pre/mod.rs
@@ -1,3 +1,29 @@
 mod lexer;
 
-pub use lexer::{preprocess_tokens, PToken, Punctuator};
+use lexer::PLexer;
+pub use lexer::{PToken, Punctuator};
+
+use crate::Span;
+
+pub struct Preprocessor<L> {
+    lexer: L,
+}
+
+impl<'src, L> Iterator for Preprocessor<L>
+where
+    L: Iterator<Item = (PToken<'src>, Span)>,
+{
+    type Item = (PToken<'src>, Span);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.lexer.next()
+    }
+}
+
+pub fn preprocess_tokens(src: &str) -> impl Iterator<Item = (PToken<'_>, Span)> {
+    let lexer = PLexer::new(src);
+
+    let preprocessor = Preprocessor { lexer };
+
+    preprocessor
+}