From ef251e68e7cd7620d93a78920e212af2d93b2d52 Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Fri, 8 Dec 2023 12:10:07 +0000 Subject: [PATCH 01/29] chore(cli): use `ignore` and `globset` --- Cargo.lock | 58 +- Cargo.toml | 5 +- crates/biome_cli/Cargo.toml | 1 + crates/biome_cli/src/execute/process_file.rs | 1 + crates/biome_cli/src/execute/traverse.rs | 16 +- crates/biome_cli/tests/commands/check.rs | 2 +- crates/biome_cli/tests/configs.rs | 4 +- .../fs_error_dereferenced_symlink.snap | 8 + ...or_infinite_symlink_expansion_to_dirs.snap | 16 + .../fs_files_ignore_symlink.snap | 2 +- .../ignores_file_inside_directory.snap | 2 +- .../does_not_format_ignored_directories.snap | 4 +- .../fs_error_dereferenced_symlink.snap | 8 + ...or_infinite_symlink_expansion_to_dirs.snap | 16 + .../fs_files_ignore_symlink.snap | 14 +- crates/biome_diagnostics/Cargo.toml | 1 + crates/biome_diagnostics/src/adapters.rs | 33 +- crates/biome_fs/Cargo.toml | 1 + crates/biome_fs/src/fs.rs | 2 + crates/biome_fs/src/fs/memory.rs | 5 + crates/biome_fs/src/fs/os.rs | 28 + crates/biome_service/Cargo.toml | 1 + .../src/configuration/diagnostics.rs | 27 +- .../biome_service/src/matcher/LICENSE-APACHE | 201 ---- crates/biome_service/src/matcher/LICENSE-MIT | 25 - crates/biome_service/src/matcher/mod.rs | 110 +-- crates/biome_service/src/matcher/pattern.rs | 927 ------------------ crates/biome_service/src/settings.rs | 26 +- crates/biome_service/src/workspace/server.rs | 3 +- 29 files changed, 246 insertions(+), 1301 deletions(-) delete mode 100644 crates/biome_service/src/matcher/LICENSE-APACHE delete mode 100644 crates/biome_service/src/matcher/LICENSE-MIT delete mode 100644 crates/biome_service/src/matcher/pattern.rs diff --git a/Cargo.lock b/Cargo.lock index 48155a23afb0..cbabd49aa0db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -185,6 +185,7 @@ dependencies = [ "crossbeam", "dashmap", "hdrhistogram", + "ignore", "indexmap", "insta", "lazy_static", @@ -289,6 +290,7 @@ dependencies = [ "biome_text_size", "bitflags 2.3.1", "bpaf", + "ignore", "schemars", "serde", "serde_json", @@ -367,6 +369,7 @@ dependencies = [ "biome_console", "biome_diagnostics", "crossbeam", + "ignore", "indexmap", "parking_lot", "rayon", @@ -732,6 +735,7 @@ dependencies = [ "biome_text_edit", "bpaf", "dashmap", + "globset", "indexmap", "insta", "lazy_static", @@ -870,14 +874,14 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" dependencies = [ "lazy_static", "memchr", - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "bstr" -version = "1.3.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffdb39cb703212f3c11973452c2861b972f757b021158f3516ba10f2fa8b2c1" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" dependencies = [ "memchr", "serde", @@ -1555,15 +1559,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.10" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" dependencies = [ "aho-corasick", - "bstr 1.3.0", - "fnv", + "bstr 1.8.0", "log", - "regex", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -1909,14 +1913,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -2267,7 +2271,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.6.28", "rusty-fork", "tempfile", "unarray", @@ -2450,13 +2454,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2465,7 +2470,18 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.28", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.2", ] [[package]] @@ -2474,6 +2490,12 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "retain_mut" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index c847f366709b..d2cac8c70e68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,7 @@ bitflags = "2.3.1" bpaf = { version = "0.9.5", features = ["derive"] } countme = "3.0.1" dashmap = "5.4.0" +ignore = "0.4.20" indexmap = "1.9.3" insta = "1.29.0" lazy_static = "1.4.0" @@ -132,9 +133,9 @@ serde = { version = "1.0.163", features = ["derive"] } serde_json = "1.0.96" smallvec = { version = "1.10.0", features = ["union", "const_new"] } tracing = { version = "0.1.37", default-features = false, features = ["std"] } +unicode-bom = "2.0.3" # pinning to version 1.18 to avoid multiple versions of windows-sys as dependency -tokio = { version = "~1.18.5" } -unicode-bom = "2.0.3" +tokio = { version = "~1.18.5" } [profile.dev.package.biome_wasm] diff --git a/crates/biome_cli/Cargo.toml b/crates/biome_cli/Cargo.toml index 0979a35b104e..37f72c43d439 100644 --- a/crates/biome_cli/Cargo.toml +++ b/crates/biome_cli/Cargo.toml @@ -38,6 +38,7 @@ bpaf = { workspace = true, features = ["bright-color"] } crossbeam = "0.8.1" dashmap = { workspace = true } hdrhistogram = { version = "7.5.0", default-features = false } +ignore = { workspace = true } indexmap = { workspace = true } lazy_static = { workspace = true } rayon = "1.5.1" diff --git a/crates/biome_cli/src/execute/process_file.rs b/crates/biome_cli/src/execute/process_file.rs index c17531d4e722..74bc32d3b826 100644 --- a/crates/biome_cli/src/execute/process_file.rs +++ b/crates/biome_cli/src/execute/process_file.rs @@ -212,6 +212,7 @@ pub(crate) fn process_file(ctx: &TraversalOptions, path: &Path) -> FileResult { let shared_context = &SharedTraversalOptions::new(ctx); ctx.increment_processed(); + match ctx.execution.traversal_mode { TraversalMode::Lint { .. } => { // the unsupported case should be handled already at this point diff --git a/crates/biome_cli/src/execute/traverse.rs b/crates/biome_cli/src/execute/traverse.rs index dc2559bcd314..b2080cd8d4b5 100644 --- a/crates/biome_cli/src/execute/traverse.rs +++ b/crates/biome_cli/src/execute/traverse.rs @@ -90,7 +90,7 @@ pub(crate) fn traverse( let mut report = Report::default(); let duration = thread::scope(|s| { - thread::Builder::new() + let handler = thread::Builder::new() .name(String::from("biome::console")) .spawn_scoped(s, || { process_messages(ProcessMessagesOptions { @@ -112,7 +112,7 @@ pub(crate) fn traverse( // The traversal context is scoped to ensure all the channels it // contains are properly closed once the traversal finishes - traverse_inputs( + let elapsed = traverse_inputs( fs, inputs, &TraversalOptions { @@ -126,7 +126,12 @@ pub(crate) fn traverse( sender_reports, remaining_diagnostics: &remaining_diagnostics, }, - ) + ); + + // wait for the main thread to finish + handler.join().unwrap(); + + elapsed }); let count = processed.load(Ordering::Relaxed); @@ -244,11 +249,8 @@ fn init_thread_pool() { /// run it to completion, returning the duration of the process fn traverse_inputs(fs: &dyn FileSystem, inputs: Vec, ctx: &TraversalOptions) -> Duration { let start = Instant::now(); - fs.traversal(Box::new(move |scope: &dyn TraversalScope| { - for input in inputs { - scope.spawn(ctx, PathBuf::from(input)); - } + scope.traverse_paths(ctx, inputs.iter().map(PathBuf::from).collect()); })); start.elapsed() diff --git a/crates/biome_cli/tests/commands/check.rs b/crates/biome_cli/tests/commands/check.rs index 78f0547b77fe..96e1650df2bc 100644 --- a/crates/biome_cli/tests/commands/check.rs +++ b/crates/biome_cli/tests/commands/check.rs @@ -1940,7 +1940,7 @@ fn ignores_file_inside_directory() { let mut console = BufferConsole::default(); let git_ignore = r#" -ignored/ +ignored/** "#; let code1 = r#"array.map(sentence => sentence.split(' ')).flat();"#; diff --git a/crates/biome_cli/tests/configs.rs b/crates/biome_cli/tests/configs.rs index ad9ec7e8d506..af04e89375a4 100644 --- a/crates/biome_cli/tests/configs.rs +++ b/crates/biome_cli/tests/configs.rs @@ -218,8 +218,8 @@ pub const CONFIG_FORMATTER_IGNORED_DIRECTORIES: &str = r#"{ "formatter": { "ignore": [ "test1.js", - "./test2.js", - "./test3/**/*", + "test2.js", + "test3/**/*", "/test4/**/*", "test5/**/*", "**/test6/*.js", diff --git a/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_dereferenced_symlink.snap b/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_dereferenced_symlink.snap index bba65dc9b097..8349e01efd0b 100644 --- a/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_dereferenced_symlink.snap +++ b/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_dereferenced_symlink.snap @@ -15,6 +15,14 @@ internalError/io ━━━━━━━━━━━━━━━━━━━━━ # Emitted Messages +```block +flags/invalid FIXABLE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + × /check_rome_test_broken_symlink/broken_symlink: No such file or directory (os error 2) + + +``` + ```block /check_rome_test_broken_symlink/broken_symlink internalError/fs ━━━━━━━━━━━━━━━━━━━━ diff --git a/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_infinite_symlink_expansion_to_dirs.snap b/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_infinite_symlink_expansion_to_dirs.snap index 4adf3b22e4a7..85455167a614 100644 --- a/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_infinite_symlink_expansion_to_dirs.snap +++ b/crates/biome_cli/tests/snapshots/main_commands_check/fs_error_infinite_symlink_expansion_to_dirs.snap @@ -15,6 +15,22 @@ internalError/io ━━━━━━━━━━━━━━━━━━━━━ # Emitted Messages +```block +flags/invalid FIXABLE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + × File system loop found: /check_rome_test_infinite_symlink_expansion_to_dirs/foo/bar/symlink2/symlink1 points to an ancestor /check_rome_test_infinite_symlink_expansion_to_dirs/foo/bar + + +``` + +```block +flags/invalid FIXABLE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + × File system loop found: /check_rome_test_infinite_symlink_expansion_to_dirs/prefix/symlink1/symlink2 points to an ancestor /check_rome_test_infinite_symlink_expansion_to_dirs/prefix + + +``` + ```block Checked 0 file(s) in