lex things

parser/src/token.rs

@@ -0,0 +1,21 @@
+use crate::{
+    pre::{PToken, Punctuator},
+    Span,
+};
+
+pub enum Token {
+    Keyword(Keyword),
+    Identifier(),
+    Constant(),
+    StringLiteral(),
+    Punctuator(Punctuator),
+    Error,
+}
+
+pub struct Keyword;
+
+fn from_pre_toks(
+    pre_toks: impl Iterator<Item = (PToken, Span)>,
+) -> impl IntoIterator<Item = (Token, Span)> {
+    pre_toks.map(|token| todo!())
+}