From e5644efc652520825c235aa239a4ab5519e23a7c Mon Sep 17 00:00:00 2001 From: Vhyrro Date: Sat, 11 Mar 2023 19:48:24 +0100 Subject: [PATCH] feat: initial commit --- .gitignore | 2 + Cargo.toml | 16 +++++ src/c_functions.rs | 0 src/lib.rs | 74 ++++++++++++++++++++++++ test/example_workspace/.hidden-file.norg | 1 + test/example_workspace/file1.norg | 50 ++++++++++++++++ test/example_workspace/file2.norg | 50 ++++++++++++++++ test/example_workspace/file3.norg | 50 ++++++++++++++++ test/example_workspace/file4.norg | 50 ++++++++++++++++ test/example_workspace/file5.norg | 50 ++++++++++++++++ test/example_workspace/file6.norg | 50 ++++++++++++++++ test/example_workspace/file7.norg | 50 ++++++++++++++++ 12 files changed, 443 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/c_functions.rs create mode 100644 src/lib.rs create mode 100644 test/example_workspace/.hidden-file.norg create mode 100644 test/example_workspace/file1.norg create mode 100644 test/example_workspace/file2.norg create mode 100644 test/example_workspace/file3.norg create mode 100644 test/example_workspace/file4.norg create mode 100644 test/example_workspace/file5.norg create mode 100644 test/example_workspace/file6.norg create mode 100644 test/example_workspace/file7.norg diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4fffb2f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b8148b9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "neorg-breeze" +description = "A library for parsing entire directories of Norg files in a multithreaded fashion." +repository = "https://github.com/nvim-neorg/neorg-breeze" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.69" +crossbeam-channel = "0.5.7" +neorg-dirman = "0.1.0" +rusty_pool = "0.7.0" +tree-sitter = "0.20.9" +tree-sitter-norg = "0.1.1" diff --git a/src/c_functions.rs b/src/c_functions.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..3bbbd10 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,74 @@ +use tree_sitter::Tree; +use rusty_pool::Builder; +use std::io::Read; +use std::fs::File; +use anyhow::{Result, anyhow}; + +pub mod c_functions; + +fn parse_file(filepath: &std::path::PathBuf) -> Result { + let mut file = File::open(filepath)?; + + let mut content = String::new(); + file.read_to_string(&mut content)?; + + let mut parser = tree_sitter::Parser::new(); + parser.set_language(tree_sitter_norg::language())?; + + parser.parse(content, None).ok_or_else(|| anyhow!(format!("Parsing for file {} timed out!", filepath.display()))) +} + +// TODO: Change Vec> to Result, (Err, output.filter(|x| x.is_some()))> +// +pub fn parse_files(workspace: neorg_dirman::workspace::Workspace) -> Vec> { + let files = workspace.files(); + + let threadpool = Builder::new().name("neorg".into()).build(); + + let mut output: Vec> = vec![None; files.len()]; + let file_count = files.len(); + + let (tx, rx) = crossbeam_channel::bounded(file_count); + + for (i, file) in files.into_iter().enumerate() { + let tx_clone = tx.clone(); + + threadpool.execute(move || { + let parsed = parse_file(&file).unwrap(); + tx_clone.send((i, parsed)).unwrap(); + }); + } + + for _ in 0..file_count { + let (i, tree) = rx.recv().unwrap(); + output[i] = Some(tree); + } + + output +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use neorg_dirman::workspace::Workspace; + + #[test] + fn test_parse_file() { + let filepath = PathBuf::from("test/example_workspace/file1.norg"); + let tree = parse_file(&filepath).unwrap(); + + assert!(tree.root_node().kind() == "document"); + } + + #[test] + fn test_parse_files() { + let workspace = Workspace { + name: "example workspace".into(), + path: "test/example_workspace".into(), + }; + let trees = parse_files(workspace); + + assert!(trees[0].is_some()); + } +} diff --git a/test/example_workspace/.hidden-file.norg b/test/example_workspace/.hidden-file.norg new file mode 100644 index 0000000..611e7f0 --- /dev/null +++ b/test/example_workspace/.hidden-file.norg @@ -0,0 +1 @@ +* This is hidden hehe diff --git a/test/example_workspace/file1.norg b/test/example_workspace/file1.norg new file mode 100644 index 0000000..22e53d0 --- /dev/null +++ b/test/example_workspace/file1.norg @@ -0,0 +1,50 @@ +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! +* File 1! diff --git a/test/example_workspace/file2.norg b/test/example_workspace/file2.norg new file mode 100644 index 0000000..1388621 --- /dev/null +++ b/test/example_workspace/file2.norg @@ -0,0 +1,50 @@ +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! +* File 2! diff --git a/test/example_workspace/file3.norg b/test/example_workspace/file3.norg new file mode 100644 index 0000000..18bcf40 --- /dev/null +++ b/test/example_workspace/file3.norg @@ -0,0 +1,50 @@ +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! +* File 3! diff --git a/test/example_workspace/file4.norg b/test/example_workspace/file4.norg new file mode 100644 index 0000000..78797b9 --- /dev/null +++ b/test/example_workspace/file4.norg @@ -0,0 +1,50 @@ +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! +* File 4! diff --git a/test/example_workspace/file5.norg b/test/example_workspace/file5.norg new file mode 100644 index 0000000..91d5832 --- /dev/null +++ b/test/example_workspace/file5.norg @@ -0,0 +1,50 @@ +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! +* File 5! diff --git a/test/example_workspace/file6.norg b/test/example_workspace/file6.norg new file mode 100644 index 0000000..b829003 --- /dev/null +++ b/test/example_workspace/file6.norg @@ -0,0 +1,50 @@ +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! +* File 6! diff --git a/test/example_workspace/file7.norg b/test/example_workspace/file7.norg new file mode 100644 index 0000000..e4bea74 --- /dev/null +++ b/test/example_workspace/file7.norg @@ -0,0 +1,50 @@ +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7! +* File 7!