From 592d799403919d9c2952c9d132a644c10abccbb2 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Mon, 22 Apr 2024 14:45:23 -0400 Subject: [PATCH 1/3] Using file hashing for package watching --- crates/turborepo-lib/src/daemon/server.rs | 6 +- .../src/package_changes_watcher.rs | 64 +++++++++++++++++-- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/crates/turborepo-lib/src/daemon/server.rs b/crates/turborepo-lib/src/daemon/server.rs index bc3b23964c339..675865d877ad3 100644 --- a/crates/turborepo-lib/src/daemon/server.rs +++ b/crates/turborepo-lib/src/daemon/server.rs @@ -136,7 +136,11 @@ impl FileWatching { scm, )); - let package_changes_watcher = Arc::new(PackageChangesWatcher::new(repo_root, recv.clone())); + let package_changes_watcher = Arc::new(PackageChangesWatcher::new( + repo_root, + recv.clone(), + hash_watcher.clone(), + )); Ok(FileWatching { watcher, diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index e268d9a472d9b..d9a6e41af148c 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -1,16 +1,25 @@ -use std::{cell::RefCell, collections::HashSet, ops::DerefMut}; +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + ops::DerefMut, + sync::Arc, +}; use ignore::gitignore::Gitignore; use notify::Event; use radix_trie::{Trie, TrieCommon}; use tokio::sync::{broadcast, oneshot, Mutex}; use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPath, AnchoredSystemPathBuf}; -use turborepo_filewatch::{NotifyError, OptionalWatch}; +use turborepo_filewatch::{ + hash_watcher::{HashSpec, HashWatcher}, + NotifyError, OptionalWatch, +}; use turborepo_repository::{ change_mapper::{ChangeMapper, GlobalDepsPackageChangeMapper, PackageChanges}, package_graph::{PackageGraph, PackageGraphBuilder, PackageName, WorkspacePackage}, package_json::PackageJson, }; +use turborepo_scm::package_deps::GitHashes; use crate::turbo_json::TurboJson; @@ -35,11 +44,17 @@ impl PackageChangesWatcher { pub fn new( repo_root: AbsoluteSystemPathBuf, file_events_lazy: OptionalWatch>>, + hash_watcher: Arc, ) -> Self { let (exit_tx, exit_rx) = oneshot::channel(); let (package_change_events_tx, package_change_events_rx) = broadcast::channel(CHANGE_EVENT_CHANNEL_CAPACITY); - let subscriber = Subscriber::new(repo_root, file_events_lazy, package_change_events_tx); + let subscriber = Subscriber::new( + repo_root, + file_events_lazy, + package_change_events_tx, + hash_watcher, + ); let _handle = tokio::spawn(subscriber.watch(exit_rx)); Self { @@ -80,6 +95,7 @@ struct Subscriber { changed_files: Mutex>, repo_root: AbsoluteSystemPathBuf, package_change_events_tx: broadcast::Sender, + hash_watcher: Arc, } // This is a workaround because `ignore` doesn't match against a path's @@ -127,12 +143,14 @@ impl Subscriber { repo_root: AbsoluteSystemPathBuf, file_events_lazy: OptionalWatch>>, package_change_events_tx: broadcast::Sender, + hash_watcher: Arc, ) -> Self { Subscriber { repo_root, file_events_lazy, changed_files: Default::default(), package_change_events_tx, + hash_watcher, } } @@ -172,6 +190,33 @@ impl Subscriber { )) } + async fn is_same_hash( + &self, + pkg: &WorkspacePackage, + package_file_hashes: &mut HashMap, + ) -> bool { + let Ok(hash) = self + .hash_watcher + .get_file_hashes(HashSpec { + package_path: pkg.path.clone(), + // TODO: Support inputs + inputs: None, + }) + .await + else { + return false; + }; + + let old_hash = package_file_hashes.get(&pkg.path).cloned(); + + if Some(&hash) != old_hash.as_ref() { + package_file_hashes.insert(pkg.path.clone(), hash); + return false; + } + + true + } + async fn watch(mut self, exit_rx: oneshot::Receiver<()>) { let Ok(mut file_events) = self.file_events_lazy.get().await.map(|r| r.resubscribe()) else { // if we get here, it means that file watching has not started, so we should @@ -222,6 +267,9 @@ impl Subscriber { else { return; }; + // We store the hash of the package's files. If the hash is already + // in here, we don't need to recompute it + let mut package_file_hashes = HashMap::new(); let mut change_mapper = match repo_state.get_change_mapper() { Some(change_mapper) => change_mapper, @@ -335,11 +383,13 @@ impl Subscriber { } for pkg in filtered_pkgs { - let _ = - self.package_change_events_tx - .send(PackageChangeEvent::Package { + if !self.is_same_hash(&pkg, &mut package_file_hashes).await { + let _ = self.package_change_events_tx.send( + PackageChangeEvent::Package { name: pkg.name.clone(), - }); + }, + ); + } } } Err(err) => { From 4ae9476753ecece70abb55fa70dd4912107b1ec4 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Tue, 7 May 2024 11:17:26 -0600 Subject: [PATCH 2/3] Add a log output that hashing works --- crates/turborepo-lib/src/package_changes_watcher.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index d9a6e41af148c..f2651f881be9d 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -214,6 +214,8 @@ impl Subscriber { return false; } + tracing::warn!("hashes are the same, no need to rerun"); + true } From 88d154930be6d1721c15be003694797ca144b543 Mon Sep 17 00:00:00 2001 From: nicholaslyang Date: Tue, 7 May 2024 13:49:01 -0600 Subject: [PATCH 3/3] Fix issue after rebase --- Cargo.lock | 2 ++ crates/turborepo-lib/src/package_changes_watcher.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98efa5323dff7..e6c1c484d41cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10859,6 +10859,7 @@ dependencies = [ "turbopack-static", "turbopack-swc-utils", "turbopack-test-utils", + "turbopack-trace-server", "turbopack-trace-utils", ] @@ -10920,6 +10921,7 @@ dependencies = [ "turbopack-node", "turbopack-nodejs", "turbopack-resolve", + "turbopack-trace-server", "turbopack-trace-utils", "webbrowser", ] diff --git a/crates/turborepo-lib/src/package_changes_watcher.rs b/crates/turborepo-lib/src/package_changes_watcher.rs index f2651f881be9d..3e05ed46cd08e 100644 --- a/crates/turborepo-lib/src/package_changes_watcher.rs +++ b/crates/turborepo-lib/src/package_changes_watcher.rs @@ -11,7 +11,7 @@ use radix_trie::{Trie, TrieCommon}; use tokio::sync::{broadcast, oneshot, Mutex}; use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPath, AnchoredSystemPathBuf}; use turborepo_filewatch::{ - hash_watcher::{HashSpec, HashWatcher}, + hash_watcher::{HashSpec, HashWatcher, InputGlobs}, NotifyError, OptionalWatch, }; use turborepo_repository::{ @@ -200,7 +200,7 @@ impl Subscriber { .get_file_hashes(HashSpec { package_path: pkg.path.clone(), // TODO: Support inputs - inputs: None, + inputs: InputGlobs::Default, }) .await else {