From bfc7fd786fe8aee05ee81ea50b752bbea9024656 Mon Sep 17 00:00:00 2001 From: Justin Ridgewell Date: Fri, 20 Oct 2023 16:05:25 -0700 Subject: [PATCH 1/2] Add caching to `chunk_content`'s graph walking --- .../turbopack-core/src/chunk/chunk_group.rs | 2 +- crates/turbopack-core/src/chunk/mod.rs | 307 ++++++++++-------- 2 files changed, 167 insertions(+), 142 deletions(-) diff --git a/crates/turbopack-core/src/chunk/chunk_group.rs b/crates/turbopack-core/src/chunk/chunk_group.rs index 16520e7f848ea..da457ad3f0a54 100644 --- a/crates/turbopack-core/src/chunk/chunk_group.rs +++ b/crates/turbopack-core/src/chunk/chunk_group.rs @@ -30,7 +30,7 @@ pub async fn make_chunk_group( forward_edges_inherit_async, local_back_edges_inherit_async, available_async_modules_back_edges_inherit_async, - } = chunk_content(chunking_context, entries, Value::new(availability_info)).await?; + } = chunk_content(chunking_context, entries, availability_info).await?; // Find all local chunk items that are self async let self_async_children = chunk_items diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index 4a60cdfc56747..34d20a8994170 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -25,7 +25,7 @@ use turbo_tasks::{ debug::ValueDebugFormat, graph::{AdjacencyMap, GraphTraversal, GraphTraversalResult, Visit, VisitControlFlow}, trace::TraceRawVcs, - ReadRef, TryFlatJoinIterExt, Upcast, Value, ValueToString, Vc, + ReadRef, TaskInput, TryFlatJoinIterExt, Upcast, ValueToString, Vc, }; use turbo_tasks_fs::FileSystemPath; use turbo_tasks_hash::DeterministicHash; @@ -198,12 +198,12 @@ pub struct ChunkContentResult { pub async fn chunk_content( chunking_context: Vc>, entries: impl IntoIterator>>, - availability_info: Value, + availability_info: AvailabilityInfo, ) -> Result { chunk_content_internal_parallel(chunking_context, entries, availability_info).await } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs, Debug)] enum InheritAsyncEdge { /// The chunk item is in the current chunk group and async module info need /// to be computed for it @@ -215,7 +215,7 @@ enum InheritAsyncEdge { AvailableAsyncModule, } -#[derive(Eq, PartialEq, Clone, Hash)] +#[derive(Eq, PartialEq, Clone, Hash, Serialize, Deserialize, TraceRawVcs, Debug)] enum ChunkContentGraphNode { // An asset not placed in the current chunk, but whose references we will // follow to find more graph nodes. @@ -241,167 +241,192 @@ enum ChunkContentGraphNode { }, } -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy, TaskInput)] +enum ChunkGraphNodeToReferences { + PassthroughModule(Vc>), + ChunkItem(Vc>), +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, TraceRawVcs)] +struct ChunkGraphEdge { + key: Option>>, + node: ChunkContentGraphNode, +} + +#[derive(Debug, Clone)] +#[turbo_tasks::value(transparent)] +struct ChunkGraphEdges(Vec); + +#[derive(Debug, Clone)] +#[turbo_tasks::value] struct ChunkContentContext { chunking_context: Vc>, - availability_info: Value, + availability_info: AvailabilityInfo, } -async fn reference_to_graph_nodes( - chunk_content_context: ChunkContentContext, - reference: Vc>, - parent: Option>>, -) -> Result>>, ChunkContentGraphNode)>> { - let Some(chunkable_module_reference) = - Vc::try_resolve_downcast::>(reference).await? - else { - return Ok(vec![( - None, - ChunkContentGraphNode::ExternalModuleReference(reference), - )]); - }; - - let Some(chunking_type) = *chunkable_module_reference.chunking_type().await? else { - return Ok(vec![( - None, - ChunkContentGraphNode::ExternalModuleReference(reference), - )]); +#[turbo_tasks::function] +async fn graph_node_to_referenced_nodes( + node: ChunkGraphNodeToReferences, + chunk_content_context: Vc, +) -> Result> { + let mut graph_nodes = vec![]; + let (parent, references) = match &node { + ChunkGraphNodeToReferences::PassthroughModule(module) => (None, module.references()), + ChunkGraphNodeToReferences::ChunkItem(item) => (Some(*item), item.references()), }; + let chunk_content_context = chunk_content_context.await?; + + let references = references.await?; + for &reference in &references { + let Some(chunkable_module_reference) = + Vc::try_resolve_downcast::>(reference).await? + else { + graph_nodes.push(ChunkGraphEdge { + key: None, + node: ChunkContentGraphNode::ExternalModuleReference(reference), + }); + continue; + }; - let modules = reference.resolve_reference().primary_modules().await?; + let Some(chunking_type) = *chunkable_module_reference.chunking_type().await? else { + graph_nodes.push(ChunkGraphEdge { + key: None, + node: ChunkContentGraphNode::ExternalModuleReference(reference), + }); + continue; + }; - let mut inherit_async_references = Vec::new(); + let modules = reference.resolve_reference().primary_modules().await?; - let mut graph_nodes = vec![]; + let mut inherit_async_references = Vec::new(); - for &module in &modules { - let module = module.resolve().await?; - - if Vc::try_resolve_sidecast::>(module) - .await? - .is_some() - { - graph_nodes.push(( - Some(module), - ChunkContentGraphNode::PassthroughModule { module }, - )); - continue; - } + for &module in &modules { + let module = module.resolve().await?; - let chunkable_module = - match Vc::try_resolve_sidecast::>(module).await? { - Some(chunkable_module) => chunkable_module, - _ => { - return Ok(vec![( - None, - ChunkContentGraphNode::ExternalModuleReference(reference), - )]); - } - }; + if Vc::try_resolve_sidecast::>(module) + .await? + .is_some() + { + graph_nodes.push(ChunkGraphEdge { + key: Some(module), + node: ChunkContentGraphNode::PassthroughModule { module }, + }); + continue; + } - match chunking_type { - ChunkingType::Parallel => { - let chunk_item = chunkable_module - .as_chunk_item(chunk_content_context.chunking_context) - .resolve() - .await?; - if let Some(available_chunk_items) = chunk_content_context - .availability_info - .available_chunk_items() - { - if available_chunk_items.get(chunk_item).await?.is_some() { + let chunkable_module = + match Vc::try_resolve_sidecast::>(module).await? { + Some(chunkable_module) => chunkable_module, + _ => { + graph_nodes.push(ChunkGraphEdge { + key: None, + node: ChunkContentGraphNode::ExternalModuleReference(reference), + }); continue; } - } + }; - graph_nodes.push(( - Some(module), - ChunkContentGraphNode::ChunkItem { - item: chunk_item, - ident: module.ident().to_string().await?, - }, - )); - } - ChunkingType::ParallelInheritAsync => { - let chunk_item = chunkable_module - .as_chunk_item(chunk_content_context.chunking_context) - .resolve() - .await?; - if let Some(available_chunk_items) = chunk_content_context - .availability_info - .available_chunk_items() - { - if let Some(info) = &*available_chunk_items.get(chunk_item).await? { - if info.is_async { - inherit_async_references - .push((chunk_item, InheritAsyncEdge::AvailableAsyncModule)); + match chunking_type { + ChunkingType::Parallel => { + let chunk_item = chunkable_module + .as_chunk_item(chunk_content_context.chunking_context) + .resolve() + .await?; + if let Some(available_chunk_items) = chunk_content_context + .availability_info + .available_chunk_items() + { + if available_chunk_items.get(chunk_item).await?.is_some() { + continue; } - continue; } + + graph_nodes.push(ChunkGraphEdge { + key: Some(module), + node: ChunkContentGraphNode::ChunkItem { + item: chunk_item, + ident: module.ident().to_string().await?, + }, + }); + } + ChunkingType::ParallelInheritAsync => { + let chunk_item = chunkable_module + .as_chunk_item(chunk_content_context.chunking_context) + .resolve() + .await?; + if let Some(available_chunk_items) = chunk_content_context + .availability_info + .available_chunk_items() + { + if let Some(info) = &*available_chunk_items.get(chunk_item).await? { + if info.is_async { + inherit_async_references + .push((chunk_item, InheritAsyncEdge::AvailableAsyncModule)); + } + continue; + } + } + inherit_async_references.push((chunk_item, InheritAsyncEdge::LocalModule)); + graph_nodes.push(ChunkGraphEdge { + key: Some(module), + node: ChunkContentGraphNode::ChunkItem { + item: chunk_item, + ident: module.ident().to_string().await?, + }, + }); + } + ChunkingType::Async => { + graph_nodes.push(ChunkGraphEdge { + key: None, + node: ChunkContentGraphNode::AsyncModule { + module: chunkable_module, + }, + }); } - inherit_async_references.push((chunk_item, InheritAsyncEdge::LocalModule)); - graph_nodes.push(( - Some(module), - ChunkContentGraphNode::ChunkItem { - item: chunk_item, - ident: module.ident().to_string().await?, - }, - )); - } - ChunkingType::Async => { - graph_nodes.push(( - None, - ChunkContentGraphNode::AsyncModule { - module: chunkable_module, - }, - )); } } - } - if !inherit_async_references.is_empty() { - if let Some(parent) = parent { - graph_nodes.push(( - None, - ChunkContentGraphNode::InheritAsyncInfo { - item: parent, - references: inherit_async_references, - }, - )) + if !inherit_async_references.is_empty() { + if let Some(parent) = parent { + graph_nodes.push(ChunkGraphEdge { + key: None, + node: ChunkContentGraphNode::InheritAsyncInfo { + item: parent, + references: inherit_async_references, + }, + }) + } } } - Ok(graph_nodes) + Ok(Vc::cell(graph_nodes)) } struct ChunkContentVisit { - chunk_content_context: ChunkContentContext, + chunk_content_context: Vc, processed_modules: HashSet>>, } -type ChunkItemToGraphNodesEdges = - impl Iterator>>, ChunkContentGraphNode)>; +type ChunkItemToGraphNodesEdges = impl Iterator; type ChunkItemToGraphNodesFuture = impl Future>; impl Visit for ChunkContentVisit { - type Edge = (Option>>, ChunkContentGraphNode); + type Edge = ChunkGraphEdge; type EdgesIntoIter = ChunkItemToGraphNodesEdges; type EdgesFuture = ChunkItemToGraphNodesFuture; - fn visit( - &mut self, - (option_key, node): (Option>>, ChunkContentGraphNode), - ) -> VisitControlFlow { - let Some(module) = option_key else { - return VisitControlFlow::Skip(node); + fn visit(&mut self, edge: ChunkGraphEdge) -> VisitControlFlow { + let ChunkGraphEdge { key, node } = edge; + let Some(module) = key else { + return VisitControlFlow::Skip(node.clone()); }; if !self.processed_modules.insert(module) { - return VisitControlFlow::Skip(node); + return VisitControlFlow::Skip(node.clone()); } - VisitControlFlow::Continue(node) + VisitControlFlow::Continue(node.clone()) } fn edges(&mut self, node: &ChunkContentGraphNode) -> Self::EdgesFuture { @@ -410,23 +435,22 @@ impl Visit for ChunkContentVisit { let chunk_content_context = self.chunk_content_context; async move { - let (references, parent) = match node { - ChunkContentGraphNode::PassthroughModule { module } => (module.references(), None), - ChunkContentGraphNode::ChunkItem { item, .. } => (item.references(), Some(item)), + let node = match node { + ChunkContentGraphNode::PassthroughModule { module } => { + ChunkGraphNodeToReferences::PassthroughModule(module) + } + ChunkContentGraphNode::ChunkItem { item, .. } => { + ChunkGraphNodeToReferences::ChunkItem(item) + } _ => { return Ok(vec![].into_iter()); } }; - Ok(references - .await? - .into_iter() - .map(|reference| { - reference_to_graph_nodes(chunk_content_context, *reference, parent) - }) - .try_flat_join() + let nodes = graph_node_to_referenced_nodes(node, chunk_content_context) .await? - .into_iter()) + .clone_value(); + Ok(nodes.into_iter()) } } @@ -442,7 +466,7 @@ impl Visit for ChunkContentVisit { async fn chunk_content_internal_parallel( chunking_context: Vc>, entries: impl IntoIterator>>, - availability_info: Value, + availability_info: AvailabilityInfo, ) -> Result { let root_edges = entries .into_iter() @@ -453,16 +477,16 @@ async fn chunk_content_internal_parallel( else { return Ok(None); }; - Ok(Some(( - Some(entry), - ChunkContentGraphNode::ChunkItem { + Ok(Some(ChunkGraphEdge { + key: Some(entry), + node: ChunkContentGraphNode::ChunkItem { item: chunkable_module .as_chunk_item(chunking_context) .resolve() .await?, ident: chunkable_module.ident().to_string().await?, }, - ))) + })) }) .try_flat_join() .await?; @@ -470,7 +494,8 @@ async fn chunk_content_internal_parallel( let chunk_content_context = ChunkContentContext { chunking_context, availability_info, - }; + } + .cell(); let visit = ChunkContentVisit { chunk_content_context, From 79ea7194cc1d27dd04593782706945a79ac25001 Mon Sep 17 00:00:00 2001 From: Justin Ridgewell Date: Fri, 20 Oct 2023 16:12:34 -0700 Subject: [PATCH 2/2] Remove old clone --- crates/turbopack-core/src/chunk/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index 34d20a8994170..35c8db0768d0a 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -419,14 +419,14 @@ impl Visit for ChunkContentVisit { fn visit(&mut self, edge: ChunkGraphEdge) -> VisitControlFlow { let ChunkGraphEdge { key, node } = edge; let Some(module) = key else { - return VisitControlFlow::Skip(node.clone()); + return VisitControlFlow::Skip(node); }; if !self.processed_modules.insert(module) { - return VisitControlFlow::Skip(node.clone()); + return VisitControlFlow::Skip(node); } - VisitControlFlow::Continue(node.clone()) + VisitControlFlow::Continue(node) } fn edges(&mut self, node: &ChunkContentGraphNode) -> Self::EdgesFuture {