refactor(turborepo): Improve package changes watcher performance

Cargo.toml

@@ -259,6 +259,7 @@ pretty_assertions = "1.3.0"
 proc-macro2 = "1.0.79"
 qstring = "0.7.2"
 quote = "1.0.23"
+radix_trie = "0.2.1"
 rand = "0.8.5"
 ratatui = "0.26.1"
 regex = "1.7.0"

crates/turborepo-filewatch/Cargo.toml

@@ -14,7 +14,7 @@ futures = { version = "0.3.26" }
 itertools = { workspace = true }
 nibble_vec = "0.1.0"
 notify = { workspace = true }
-radix_trie = "0.2.1"
+radix_trie = { workspace = true }
 thiserror = "1.0.38"
 tokio = { workspace = true, features = ["full", "time"] }
 tracing = "0.1.37"

crates/turborepo-filewatch/src/lib.rs

@@ -1,3 +1,25 @@
+//! File watching utilities for Turborepo. Includes a file watcher that is
+//! designed to work across multiple platforms, with consistent behavior and
+//! consistent ordering.
+//!
+//! Also includes watchers that take in file change events and produce derived
+//! data like changed packages or the workspaces in a repository.
+//!
+//! ## Watcher Implementation
+//! It's important to note that when implementing a watcher, you should aim to
+//! make file change event processing as fast as possible. There should be
+//! almost no slow code in the main event loop. Otherwise, the receiver will lag
+//! behind, and return a `Lagged` event.
+//!
+//! A common pattern that we use to avoid lag is having a separate event thread
+//! that processes events and accumulates them into a data structure, say a
+//! `Trie` for changed files, or a `HashSet` for changed packages. From there, a
+//! second thread is responsible for actually taking that accumulated data and
+//! processing it. This second thread can do slower tasks like executing a run
+//! or mapping files to changed packages. It can either be parked and awoken
+//! using `tokio::sync::Notify` or it can run periodically using
+//! `tokio::time::interval`.
+
 #![deny(clippy::all)]
 #![feature(assert_matches)]
 

crates/turborepo-lib/Cargo.toml

@@ -81,6 +81,7 @@ pprof = { version = "0.12.1", features = [
   "frame-pointer",
 ], optional = true }
 prost = "0.12.3"
+radix_trie = { workspace = true }
 rand = { workspace = true }
 rayon = "1.7.0"
 regex.workspace = true