Start migrating to tree sitter

Migrates all the lower level code to tree-sitter not touching passes yet.

.gitignore

@@ -5,3 +5,5 @@ target
 /invalid
 .direnv/
 g
+perf.data*
+*.svg

Cargo.lock

@@ -113,6 +113,9 @@ dependencies = [
  "tracing",
  "tracing-subscriber",
  "tracing-tree",
+ "tree-sitter",
+ "tree-sitter-edit",
+ "tree-sitter-rust",
  "walkdir",
 ]
 
@@ -855,6 +858,45 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "tree-sitter"
+version = "0.20.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d"
+dependencies = [
+ "cc",
+ "regex",
+]
+
+[[package]]
+name = "tree-sitter-edit"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed3213ee656e99748eca539913b5c90df3d52618d9a1714e0935013955c8031"
+dependencies = [
+ "tree-sitter",
+ "tree-sitter-traversal",
+]
+
+[[package]]
+name = "tree-sitter-rust"
+version = "0.20.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0832309b0b2b6d33760ce5c0e818cb47e1d72b468516bfe4134408926fa7594"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
+[[package]]
+name = "tree-sitter-traversal"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df8a158225e4a4d8505f071340bba9edd109b23f01b70540dccb7c799868f307"
+dependencies = [
+ "tree-sitter",
+]
+
 [[package]]
 name = "unicode-id"
 version = "0.3.4"

Cargo.toml

@@ -16,8 +16,12 @@ license = "MIT OR Apache-2.0"
 [profile.release]
 lto = "thin"
 
-[profile.dev]
+[profile.dev.package.proc-macro2]
-opt-level = 1
+opt-level = 3
+[profile.dev.package.syn]
+opt-level = 3
+[profile.dev.package.genemichaels]
+opt-level = 3
 
 [dependencies]
 anyhow = "1.0.65"
@@ -36,4 +40,7 @@ tempfile = "3.3.0"
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 tracing-tree = "0.2.2"
+tree-sitter = "0.20.10"
+tree-sitter-edit = "0.3.0"
+tree-sitter-rust = "0.20.4"
 walkdir = "2.3.2"

src/dylib_flag.rs

@@ -32,7 +32,7 @@ impl FromStr for RustFunction {
 fn wrap_func_body(func: &str) -> Result<String> {
     let closure = syn::parse_str::<syn::ExprClosure>(func).context("invalid rust syntax")?;
 
-    let syn_file = syn::parse_quote! {
+    let file = quote::quote! {
         #[repr(C)]
         pub struct __RawOutput {
             out_ptr: *const u8,
@@ -80,7 +80,7 @@ fn wrap_func_body(func: &str) -> Result<String> {
         }
     };
 
-    crate::formatting::format(syn_file)
+    Ok(file.to_string())
 }
 
 impl RustFunction {

src/formatting.rs

@@ -1,12 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::Context;
-use genemichaels::FormatConfig;
-
-pub fn format(file: syn::File) -> anyhow::Result<String> {
-    Ok(
-        genemichaels::format_ast(file, &FormatConfig::default(), HashMap::new())
-            .context("formatting source file")?
-            .rendered,
-    )
-}

src/lib.rs

@@ -9,7 +9,7 @@ use std::{
 
 mod build;
 mod dylib_flag;
-mod formatting;
+mod tree_sitter;
 mod passes;
 mod processor;
 

src/processor/files.rs

@@ -19,24 +19,26 @@ mod file {
     pub(crate) struct SourceFile {
         path: PathBuf,
         content_str: RefCell<String>,
-        content: RefCell<syn::File>,
+        content: RefCell<tree_sitter::Tree>,
     }
 
     impl SourceFile {
         pub(crate) fn open(path: PathBuf) -> Result<Self> {
             let string = std::fs::read_to_string(&path)
                 .with_context(|| format!("reading file {}", path.display()))?;
-            let content = syn::parse_file(&string)
+
-                .with_context(|| format!("parsing file {}", path.display()))?;
+            let content_ts = crate::tree_sitter::parse(&string)
+                .with_context(|| format!("parsing file {path:?}"))?;
+
             Ok(SourceFile {
                 path,
                 content_str: RefCell::new(string),
-                content: RefCell::new(content),
+                content: RefCell::new(content_ts),
             })
         }
 
-        pub(crate) fn write(&self, new: syn::File) -> Result<()> {
+        pub(crate) fn write(&self, new: tree_sitter::Tree) -> Result<()> {
-            let string = crate::formatting::format(new.clone())?;
+            let string = crate::tree_sitter::format(new, &*self.content_str.borrow())?;
             std::fs::write(&self.path, &string)
                 .with_context(|| format!("writing file {}", self.path.display()))?;
             *self.content_str.borrow_mut() = string;
@@ -96,17 +98,17 @@ pub(crate) struct FileChange<'a, 'b> {
     pub(crate) path: &'a Path,
     source_file: &'a SourceFile,
     before_content_str: String,
-    before_content: syn::File,
+    before_content: tree_sitter::Tree,
     changes: &'b mut Changes,
     has_written_change: bool,
 }
 
 impl FileChange<'_, '_> {
-    pub(crate) fn before_content(&self) -> (&str, &syn::File) {
+    pub(crate) fn before_content(&self) -> (&str, &tree_sitter::Tree) {
         (&self.before_content_str, &self.before_content)
     }
 
-    pub(crate) fn write(&mut self, new: syn::File) -> Result<()> {
+    pub(crate) fn write(&mut self, new: tree_sitter::Tree) -> Result<()> {
         self.has_written_change = true;
         self.source_file.write(new)?;
         Ok(())

src/processor/reaper.rs

@@ -82,7 +82,7 @@ impl Minimizer {
 
             let result =
                 rustfix::apply_suggestions(change.before_content().0, &desired_suggestions)?;
-            let result = syn::parse_file(&result).context("parsing file after rustfix")?;
+            let result = crate::tree_sitter::parse(&result).context("parsing file after rustfix")?;
             change.write(result)?;
 
             let after = self.build.build()?;

src/tree_sitter.rs

@@ -0,0 +1,34 @@
+use anyhow::{Context, Result};
+
+pub fn parse(source: &str) -> Result<tree_sitter::Tree> {
+    let mut parser = tree_sitter::Parser::new();
+    parser
+        .set_language(tree_sitter_rust::language())
+        .context("loading tree sitter rust grammar")?;
+    let content_ts = parser.parse(source, None).context("parsing file")?;
+    Ok(content_ts)
+}
+
+pub fn format(file: tree_sitter::Tree, source: &str) -> anyhow::Result<String> {
+    let mut s = Vec::new();
+    tree_sitter_edit::render(&mut s, &file, source.as_bytes(), &Editor);
+
+    Ok(String::from_utf8(s).unwrap())
+}
+
+struct Editor;
+
+impl tree_sitter_edit::Editor for Editor {
+    fn has_edit(&self, tree: &tree_sitter::Tree, node: &tree_sitter::Node<'_>) -> bool {
+        false
+    }
+
+    fn edit(
+        &self,
+        source: &[u8],
+        tree: &tree_sitter::Tree,
+        node: &tree_sitter::Node<'_>,
+    ) -> Vec<u8> {
+        unimplemented!()
+    }
+}