Skip to content

Commit

Permalink
Parallelize chunk_content walk (vercel/turborepo#6245)
Browse files Browse the repository at this point in the history 
### Description

vercel/turborepo#6242 extracted `chunk_content`'s
module references -> graph node walking into a cacheable helper, and now
we can parallelize that work to make it even faster.

### Testing Instructions

<!--
  Give a quick description of steps to test your changes.
-->


Closes WEB-1822

---------

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>
  • Loading branch information
@jridgewell @sokra
jridgewell and sokra authored Oct 23, 2023
1 parent df5beda commit 911d179
Showing 1 changed file with 136 additions and 106 deletions.
242 changes: 136 additions & 106 deletions crates/turbopack-core/src/chunk/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use turbo_tasks::{
debug::ValueDebugFormat,
graph::{AdjacencyMap, GraphTraversal, GraphTraversalResult, Visit, VisitControlFlow},
trace::TraceRawVcs,
ReadRef, TaskInput, TryFlatJoinIterExt, Upcast, ValueToString, Vc,
ReadRef, TaskInput, TryFlatJoinIterExt, TryJoinIterExt, Upcast, ValueToString, Vc,
};
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::DeterministicHash;
Expand Down Expand Up @@ -269,135 +269,165 @@ async fn graph_node_to_referenced_nodes(
node: ChunkGraphNodeToReferences,
chunk_content_context: Vc<ChunkContentContext>,
) -> Result<Vc<ChunkGraphEdges>> {
let mut graph_nodes = vec![];
let (parent, references) = match &node {
ChunkGraphNodeToReferences::PassthroughModule(module) => (None, module.references()),
ChunkGraphNodeToReferences::ChunkItem(item) => (Some(*item), item.references()),
};
let chunk_content_context = chunk_content_context.await?;

let references = references.await?;
for &reference in &references {
let Some(chunkable_module_reference) =
Vc::try_resolve_downcast::<Box<dyn ChunkableModuleReference>>(reference).await?
else {
graph_nodes.push(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference),
});
continue;
};

let Some(chunking_type) = *chunkable_module_reference.chunking_type().await? else {
graph_nodes.push(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference),
});
continue;
};

let modules = reference.resolve_reference().primary_modules().await?;
let graph_nodes = references
.iter()
.map(|reference| async {
let reference = *reference;
let Some(chunkable_module_reference) =
Vc::try_resolve_downcast::<Box<dyn ChunkableModuleReference>>(reference).await?
else {
return Ok(vec![ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference),
}]);
};

let mut inherit_async_references = Vec::new();
let Some(chunking_type) = *chunkable_module_reference.chunking_type().await? else {
return Ok(vec![ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference),
}]);
};

for &module in &modules {
let module = module.resolve().await?;
let modules = reference.resolve_reference().primary_modules().await?;

if Vc::try_resolve_sidecast::<Box<dyn PassthroughModule>>(module)
.await?
.is_some()
{
graph_nodes.push(ChunkGraphEdge {
key: Some(module),
node: ChunkContentGraphNode::PassthroughModule { module },
});
continue;
}
let module_data = modules
.iter()
.map(|module| async {
let module = (*module).resolve().await?;

let chunkable_module =
match Vc::try_resolve_sidecast::<Box<dyn ChunkableModule>>(module).await? {
Some(chunkable_module) => chunkable_module,
_ => {
graph_nodes.push(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference),
});
continue;
}
};

match chunking_type {
ChunkingType::Parallel => {
let chunk_item = chunkable_module
.as_chunk_item(chunk_content_context.chunking_context)
.resolve()
.await?;
if let Some(available_chunk_items) = chunk_content_context
.availability_info
.available_chunk_items()
if Vc::try_resolve_sidecast::<Box<dyn PassthroughModule>>(module)
.await?
.is_some()
{
if available_chunk_items.get(chunk_item).await?.is_some() {
continue;
}
return Ok((
Some(ChunkGraphEdge {
key: Some(module),
node: ChunkContentGraphNode::PassthroughModule { module },
}),
None,
));
}

graph_nodes.push(ChunkGraphEdge {
key: Some(module),
node: ChunkContentGraphNode::ChunkItem {
item: chunk_item,
ident: module.ident().to_string().await?,
},
});
}
ChunkingType::ParallelInheritAsync => {
let chunk_item = chunkable_module
.as_chunk_item(chunk_content_context.chunking_context)
.resolve()
.await?;
if let Some(available_chunk_items) = chunk_content_context
.availability_info
.available_chunk_items()
{
if let Some(info) = &*available_chunk_items.get(chunk_item).await? {
if info.is_async {
inherit_async_references
.push((chunk_item, InheritAsyncEdge::AvailableAsyncModule));
let Some(chunkable_module) =
Vc::try_resolve_sidecast::<Box<dyn ChunkableModule>>(module).await?
else {
return Ok((
Some(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::ExternalModuleReference(reference),
}),
None,
));
};

match chunking_type {
ChunkingType::Parallel => {
let chunk_item = chunkable_module
.as_chunk_item(chunk_content_context.chunking_context)
.resolve()
.await?;
if let Some(available_chunk_items) = chunk_content_context
.availability_info
.available_chunk_items()
{
if available_chunk_items.get(chunk_item).await?.is_some() {
return Ok((None, None));
}
}
continue;

Ok((
Some(ChunkGraphEdge {
key: Some(module),
node: ChunkContentGraphNode::ChunkItem {
item: chunk_item,
ident: module.ident().to_string().await?,
},
}),
None,
))
}
ChunkingType::ParallelInheritAsync => {
let chunk_item = chunkable_module
.as_chunk_item(chunk_content_context.chunking_context)
.resolve()
.await?;
if let Some(available_chunk_items) = chunk_content_context
.availability_info
.available_chunk_items()
{
if let Some(info) = &*available_chunk_items.get(chunk_item).await? {
if info.is_async {
return Ok((
None,
Some((
chunk_item,
InheritAsyncEdge::AvailableAsyncModule,
)),
));
}
return Ok((None, None));
}
}
Ok((
Some(ChunkGraphEdge {
key: Some(module),
node: ChunkContentGraphNode::ChunkItem {
item: chunk_item,
ident: module.ident().to_string().await?,
},
}),
Some((chunk_item, InheritAsyncEdge::LocalModule)),
))
}
ChunkingType::Async => Ok((
Some(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::AsyncModule {
module: chunkable_module,
},
}),
None,
)),
}
inherit_async_references.push((chunk_item, InheritAsyncEdge::LocalModule));
graph_nodes.push(ChunkGraphEdge {
key: Some(module),
node: ChunkContentGraphNode::ChunkItem {
item: chunk_item,
ident: module.ident().to_string().await?,
},
});
})
.try_join()
.await?;

let mut graph_nodes = vec![];
let mut inherit_async_references = vec![];
for (n, iar) in module_data {
if let Some(n) = n {
graph_nodes.push(n);
}
if let Some(iar) = iar {
inherit_async_references.push(iar);
}
ChunkingType::Async => {
}

if !inherit_async_references.is_empty() {
if let Some(parent) = parent {
graph_nodes.push(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::AsyncModule {
module: chunkable_module,
node: ChunkContentGraphNode::InheritAsyncInfo {
item: parent,
references: inherit_async_references,
},
});
})
}
}
}

if !inherit_async_references.is_empty() {
if let Some(parent) = parent {
graph_nodes.push(ChunkGraphEdge {
key: None,
node: ChunkContentGraphNode::InheritAsyncInfo {
item: parent,
references: inherit_async_references,
},
})
}
}
}
Ok(graph_nodes)
})
.try_flat_join()
.await?;

Ok(Vc::cell(graph_nodes))
}
Expand Down

0 comments on commit 911d179

Please sign in to comment.