perf!: move to threadpool, fix excess memory usage

Cargo.toml

@@ -9,8 +9,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.69"
-crossbeam-channel = "0.5.7"
 neorg-dirman = "0.1.0"
-rusty_pool = "0.7.0"
+threadpool = "1.8.1"
 tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter" }
 tree-sitter-norg = { git = "https://github.com/nvim-neorg/tree-sitter-norg2" }

src/breeze.rs

@@ -1,23 +1,22 @@
 // TODO: Generalize to work with any language
 
 use anyhow::{anyhow, Result};
-use rusty_pool::Builder;
-use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
-use tree_sitter::{Tree, Language};
+use std::fs::File;
+use threadpool::Builder;
+use tree_sitter::{Parser, Tree};
 
 /// Parses a file and returns its [`Tree`].
 ///
 /// * `filepath`: The path of the file to read.
-fn parse_file(filepath: &std::path::PathBuf, language: Language) -> Result<Tree> {
+fn parse_file(filepath: &std::path::PathBuf, parser: &mut Parser) -> Result<Tree> {
     let mut file = File::open(filepath)?;
 
     let mut content = String::new();
     file.read_to_string(&mut content)?;
 
-    let mut parser = tree_sitter::Parser::new();
-    parser.set_language(language)?;
+    drop(file);
 
     parser.parse(content, None).ok_or_else(|| {
         anyhow!(format!(
@@ -27,35 +26,24 @@ fn parse_file(filepath: &std::path::PathBuf, language: Language) -> Result<Tree>
     })
 }
 
-pub fn parse_files(files: Vec<PathBuf>) -> Result<Vec<Tree>> {
-    let threadpool = Builder::new().name("neorg".into()).build();
-
-    let mut output: Vec<Option<Tree>> = vec![None; files.len()];
-    let file_count = files.len();
-
-    let (tx, rx) = crossbeam_channel::bounded(file_count);
+pub fn parse_files(files: Vec<PathBuf>, callback: &'static (dyn Fn(Tree) + Send + Sync)) -> Result<()> {
+    let threadpool = Builder::new().thread_name("neorg".into()).num_threads(8).build();
 
     let language = tree_sitter_norg::language();
 
-    for (i, file) in files.into_iter().enumerate() {
-        let tx_clone = tx.clone();
-
+    for file in files {
         threadpool.execute(move || {
-            let parsed = parse_file(&file, language.clone());
-            tx_clone.send((i, parsed)).unwrap();
+            let mut parser = Parser::new();
+            parser.set_language(language).unwrap();
+
+            let tree = parse_file(&file, &mut parser).unwrap();
+            callback(tree);
         });
     }
 
-    threadpool.shutdown_join();
-
-    for _ in 0..file_count {
-        match rx.recv()? {
-            (i, Ok(tree)) => output[i] = Some(tree),
-            (_, Err(err)) => return Err(err),
-        }
-    }
+    threadpool.join();
 
-    Ok(output.into_iter().flatten().collect())
+    Ok(())
 }
 
 #[cfg(test)]
@@ -67,7 +55,9 @@ mod tests {
     #[test]
     fn test_parse_file() {
         let filepath = PathBuf::from("test/example_workspace/file1.norg");
-        let tree = parse_file(&filepath, tree_sitter_norg::language()).unwrap();
+        let mut parser = Parser::new();
+        parser.set_language(tree_sitter_norg::language()).unwrap();
+        let tree = parse_file(&filepath, &mut parser).unwrap();
 
         assert!(tree.root_node().kind() == "document");
     }
@@ -79,9 +69,7 @@ mod tests {
             path: "test/example_workspace".into(),
         };
 
-        let trees = parse_files(workspace.files())
+        parse_files(workspace.files(), &|tree: Tree| assert!(tree.root_node().kind() == "document"))
             .expect("Unable to parse files in the current workspace!");
-
-        assert!(trees[0].root_node().kind() == "document");
     }
 }

src/c_functions.rs

@@ -1,54 +0,0 @@
-use neorg_dirman::c_functions::FileList;
-use std::ffi::CStr;
-use std::mem::ManuallyDrop;
-
-use crate::breeze;
-use tree_sitter::ffi::TSTree;
-
-pub unsafe extern "C" fn parse_files(files: *const FileList) -> *const TSTree {
-    let vec = unsafe { std::slice::from_raw_parts((*files).data, (*files).length) }.to_vec();
-    let paths = vec
-        .into_iter()
-        .map(|str| CStr::from_ptr(str).to_string_lossy().into_owned().into())
-        .collect();
-
-    let tree_vec = ManuallyDrop::new(
-        breeze::parse_files(paths)
-            .expect("Give me better error messages!")
-            .into_iter()
-            .map(|tree| *tree.into_raw())
-            .collect::<Vec<_>>(),
-    );
-
-    tree_vec.as_ptr()
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use neorg_dirman::c_functions::*;
-    use neorg_dirman::workspace::Workspace;
-
-    #[test]
-    fn test_parse_files() {
-        unsafe {
-            let workspace = Workspace {
-                name: "test".into(),
-                path: "test/example_workspace".into(),
-            };
-
-            let files = workspace_files(&workspace);
-
-            let tree = parse_files(files);
-            assert!(!tree.is_null());
-
-            destroy_files(files);
-        }
-    }
-}
-
-// pub extern "C" fn parse_workspace(workspace: *mut Workspace) {
-//     let vec = unsafe { std::slice::from_raw_parts((*files).data, (*files).length) }.to_vec();
-//
-//     let tree_vec = breeze::parse_files();
-// }