mirror of
https://github.com/hanzoai/next.js.git
synced 2026-09-24 17:00:51 +00:00
Turbopack: make available_modules an OperationVc and simplify merged module bookkeeping (#94055)
### What? Reworks how `chunk_group_content` exposes `available_modules` for downstream chunking, and consolidates the bookkeeping for merged modules in `ChunkGroup`. - Splits `ChunkGroupContent` into a `turbo_tasks::value` `ChunkGroupContentInner` (carrying `chunkable_items`, `batch_groups`, `async_modules`, `traced_modules`, and `available_modules`) plus a thin wrapper that pairs it with the resulting `AvailabilityInfo`. `chunk_group_content_operation` now runs entirely inside `turbo_tasks`, allowing `AvailableModules` to store its set as an `OperationVc<AvailableModulesSet>` and `.connect()` it on read. - Replaces the separate `included` set and `should_create_chunk_item_for` method on `MergedModuleInfo` with an `Option<ChunkableModule>` value type on the `replacements` map. Call sites now decide all three cases (replace / skip / keep) in one match via `should_replace_module`. ### Why? - `AvailableModules::snapshot` did not correctly derive its invalidation priority, which caused double invalidations: the snapshot was treated as a low-priority dependency even though it gated chunking work, so when its inputs changed turbo-tasks would invalidate it after dependent computations had already started, triggering them a second time. Exposing `available_modules` as an `OperationVc` makes `AvailableModules` inherit the correct invalidation priority from the operation that produces it, which removes the double-invalidation. - The previous `MergedModuleInfo` API had two parallel data structures (`replacements` and `included`) and a helper that combined them. Folding `included` into the value of `replacements` removes the duplication, makes the three states (replace, skip-because-merged, keep) explicit, and shrinks the call-site logic in `chunk_group.rs`. ### How? **`available_modules` as an `OperationVc`:** - New `ChunkGroupContentInner` is a `turbo_tasks::value` holding the chunkable items, batch groups, async/traced modules, and an `OperationVc<AvailableModulesSet>` for `available_modules`. - `ChunkGroupContent` becomes a plain struct wrapping `ResolvedVc<ChunkGroupContentInner>` together with the `AvailabilityInfo` computed for the group. - `chunk_group_content_operation` is an `OperationVc`-producing function so its inner `available_modules` can be returned as an `OperationVc<AvailableModulesSet>`. `AvailableModules` now stores that operation Vc and `.connect()`s it whenever the set is read, so consumers reach the set through the operation and inherit its priority. - Browser and Node.js chunking contexts are updated to thread the new shape through `make_chunk_group` and friends. **`MergedModuleInfo` simplification:** - `replacements` changes from `HashMap<ChunkableModule, ChunkableModule>` to `HashMap<ChunkableModule, Option<ChunkableModule>>`: `Some(m)` means "replace with `m`", `None` means "skip — already included in another merged module", absent means "keep as-is". - The old `included` set and `should_create_chunk_item_for` helper are removed; `should_replace_module` returns the three-way outcome and consumers match on it once. - Call sites in `chunk_group.rs` are restructured around the single match so the logic is no longer split across two lookups. ### Notes - No user-facing behavior change is intended — this is an internal refactor in `turbopack-core` (`chunk_group_content`, `available_modules`, `chunk_group`) and the browser/nodejs chunking contexts. - Existing chunking tests in `turbopack-core` exercise the new shape. <!-- NEXT_JS_LLM_PR -->
This commit is contained in:
@@ -763,13 +763,12 @@ impl ChunkingContext for BrowserChunkingContext {
|
||||
self: ResolvedVc<Self>,
|
||||
ident: Vc<AssetIdent>,
|
||||
chunk_group: ChunkGroup,
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
availability_info: AvailabilityInfo,
|
||||
) -> Result<Vc<ChunkGroupResult>> {
|
||||
let span = tracing::info_span!("chunking", name = display(ident.to_string().await?));
|
||||
async move {
|
||||
let this = self.await?;
|
||||
let entries = chunk_group.entries();
|
||||
let input_availability_info = availability_info;
|
||||
let MakeChunkGroupResult {
|
||||
chunks,
|
||||
@@ -777,7 +776,7 @@ impl ChunkingContext for BrowserChunkingContext {
|
||||
references,
|
||||
availability_info,
|
||||
} = make_chunk_group(
|
||||
entries,
|
||||
chunk_group,
|
||||
module_graph,
|
||||
ResolvedVc::upcast(self),
|
||||
input_availability_info,
|
||||
@@ -834,7 +833,7 @@ impl ChunkingContext for BrowserChunkingContext {
|
||||
self: ResolvedVc<Self>,
|
||||
ident: Vc<AssetIdent>,
|
||||
chunk_group: ChunkGroup,
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
input_availability_info: AvailabilityInfo,
|
||||
) -> Result<Vc<ChunkGroupResult>> {
|
||||
let span = tracing::info_span!(
|
||||
@@ -844,14 +843,13 @@ impl ChunkingContext for BrowserChunkingContext {
|
||||
);
|
||||
async move {
|
||||
let this = self.await?;
|
||||
let entries = chunk_group.entries();
|
||||
let MakeChunkGroupResult {
|
||||
chunks,
|
||||
referenced_output_assets,
|
||||
references,
|
||||
availability_info,
|
||||
} = make_chunk_group(
|
||||
entries,
|
||||
chunk_group.clone(),
|
||||
module_graph,
|
||||
ResolvedVc::upcast(self),
|
||||
input_availability_info,
|
||||
@@ -903,7 +901,7 @@ impl ChunkingContext for BrowserChunkingContext {
|
||||
}
|
||||
|
||||
assets.push(
|
||||
self.generate_evaluate_chunk(ident, other_assets, entries, module_graph)
|
||||
self.generate_evaluate_chunk(ident, other_assets, entries, *module_graph)
|
||||
.to_resolved()
|
||||
.await?,
|
||||
);
|
||||
|
||||
@@ -2,7 +2,7 @@ use anyhow::Result;
|
||||
use bincode::{Decode, Encode};
|
||||
use bitfield::bitfield;
|
||||
use turbo_rcstr::RcStr;
|
||||
use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, Vc, trace::TraceRawVcs};
|
||||
use turbo_tasks::{NonLocalValue, OperationVc, ResolvedVc, TaskInput, trace::TraceRawVcs};
|
||||
|
||||
use crate::chunk::available_modules::{AvailableModules, AvailableModulesSet};
|
||||
|
||||
@@ -34,7 +34,7 @@ impl AvailabilityInfo {
|
||||
self.available_modules
|
||||
}
|
||||
|
||||
pub async fn with_modules(self, modules: Vc<AvailableModulesSet>) -> Result<Self> {
|
||||
pub async fn with_modules(self, modules: OperationVc<AvailableModulesSet>) -> Result<Self> {
|
||||
Ok(if let Some(available_modules) = self.available_modules {
|
||||
Self {
|
||||
flags: self.flags,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use anyhow::Result;
|
||||
use bincode::{Decode, Encode};
|
||||
use turbo_tasks::{
|
||||
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, ValueToString, Vc,
|
||||
trace::TraceRawVcs, turbofmt,
|
||||
FxIndexSet, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt,
|
||||
ValueToString, Vc, trace::TraceRawVcs, turbofmt,
|
||||
};
|
||||
use turbo_tasks_hash::Xxh3Hash64Hasher;
|
||||
|
||||
@@ -61,13 +61,13 @@ pub struct AvailableModulesSet(
|
||||
#[turbo_tasks::value]
|
||||
pub struct AvailableModules {
|
||||
parent: Option<ResolvedVc<AvailableModules>>,
|
||||
modules: ResolvedVc<AvailableModulesSet>,
|
||||
modules: OperationVc<AvailableModulesSet>,
|
||||
}
|
||||
|
||||
#[turbo_tasks::value_impl]
|
||||
impl AvailableModules {
|
||||
#[turbo_tasks::function]
|
||||
pub fn new(modules: ResolvedVc<AvailableModulesSet>) -> Vc<Self> {
|
||||
pub fn new(modules: OperationVc<AvailableModulesSet>) -> Vc<Self> {
|
||||
AvailableModules {
|
||||
parent: None,
|
||||
modules,
|
||||
@@ -78,7 +78,7 @@ impl AvailableModules {
|
||||
#[turbo_tasks::function]
|
||||
pub fn with_modules(
|
||||
self: ResolvedVc<Self>,
|
||||
modules: ResolvedVc<AvailableModulesSet>,
|
||||
modules: OperationVc<AvailableModulesSet>,
|
||||
) -> Result<Vc<Self>> {
|
||||
Ok(AvailableModules {
|
||||
parent: Some(self),
|
||||
@@ -97,6 +97,7 @@ impl AvailableModules {
|
||||
}
|
||||
let item_idents = self
|
||||
.modules
|
||||
.connect()
|
||||
.await?
|
||||
.iter()
|
||||
.map(async |&module| module.ident_strings().await)
|
||||
@@ -118,7 +119,7 @@ impl AvailableModules {
|
||||
|
||||
#[turbo_tasks::function]
|
||||
pub async fn get(&self, item: AvailableModuleItem) -> Result<Vc<bool>> {
|
||||
if self.modules.await?.contains(&item) {
|
||||
if self.modules.connect().await?.contains(&item) {
|
||||
return Ok(Vc::cell(true));
|
||||
};
|
||||
if let Some(parent) = self.parent {
|
||||
@@ -129,7 +130,7 @@ impl AvailableModules {
|
||||
|
||||
#[turbo_tasks::function]
|
||||
pub async fn snapshot(&self) -> Result<Vc<AvailableModulesSnapshot>> {
|
||||
let modules = self.modules.await?;
|
||||
let modules = self.modules.connect().await?;
|
||||
let parent = if let Some(parent) = self.parent {
|
||||
Some(parent.snapshot().await?)
|
||||
} else {
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use bincode::{Decode, Encode};
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::Instrument;
|
||||
use turbo_rcstr::rcstr;
|
||||
use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
|
||||
use turbo_tasks::{
|
||||
FxIndexSet, NonLocalValue, OperationVc, ResolvedVc, TaskInput, TryFlatJoinIterExt,
|
||||
TryJoinIterExt, Vc, trace::TraceRawVcs,
|
||||
};
|
||||
|
||||
use super::{
|
||||
ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext,
|
||||
availability_info::AvailabilityInfo, chunking::make_chunks,
|
||||
ChunkItemWithAsyncModuleInfo, ChunkingContext, availability_info::AvailabilityInfo,
|
||||
chunking::make_chunks,
|
||||
};
|
||||
use crate::{
|
||||
chunk::{
|
||||
ChunkableModule, ChunkingType, Chunks,
|
||||
available_modules::AvailableModuleItem,
|
||||
ChunkGroupContent, ChunkGroupContentInner, ChunkableModule, ChunkingType, Chunks,
|
||||
available_modules::{AvailableModuleItem, AvailableModulesSet},
|
||||
chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
|
||||
},
|
||||
environment::ChunkLoading,
|
||||
module::Module,
|
||||
module_graph::{
|
||||
GraphTraversalAction, ModuleGraph,
|
||||
chunk_group_info::ChunkGroup,
|
||||
merged_modules::MergedModuleInfo,
|
||||
module_batch::{
|
||||
ChunkableModuleBatchGroup, ChunkableModuleOrBatch, ModuleBatch, ModuleBatchGroup,
|
||||
@@ -39,11 +45,8 @@ pub struct MakeChunkGroupResult {
|
||||
|
||||
/// Creates a chunk group from a set of entries.
|
||||
pub async fn make_chunk_group(
|
||||
chunk_group_entries: impl IntoIterator<
|
||||
IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
|
||||
> + Send
|
||||
+ Clone,
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
chunk_group: ChunkGroup,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
|
||||
availability_info: AvailabilityInfo,
|
||||
) -> Result<MakeChunkGroupResult> {
|
||||
@@ -56,17 +59,14 @@ pub async fn make_chunk_group(
|
||||
.await?;
|
||||
let should_trace = *chunking_context.is_tracing_enabled().await?;
|
||||
let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
|
||||
let batching_config = chunking_context.batching_config();
|
||||
let batching_config = chunking_context.batching_config().to_resolved().await?;
|
||||
|
||||
let ChunkGroupContent {
|
||||
chunkable_items,
|
||||
batch_groups,
|
||||
async_modules,
|
||||
traced_modules,
|
||||
inner,
|
||||
availability_info: new_availability_info,
|
||||
} = chunk_group_content(
|
||||
module_graph,
|
||||
chunk_group_entries.clone(),
|
||||
chunk_group,
|
||||
ChunkGroupContentOptions {
|
||||
availability_info,
|
||||
can_split_async,
|
||||
@@ -76,6 +76,13 @@ pub async fn make_chunk_group(
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let ChunkGroupContentInner {
|
||||
chunkable_items,
|
||||
batch_groups,
|
||||
async_modules,
|
||||
traced_modules,
|
||||
available_modules: _,
|
||||
} = &*inner;
|
||||
|
||||
let async_module_info = module_graph.async_module_info();
|
||||
|
||||
@@ -87,7 +94,7 @@ pub async fn make_chunk_group(
|
||||
ChunkItemOrBatchWithAsyncModuleInfo::from_chunkable_module_or_batch(
|
||||
m,
|
||||
async_module_info,
|
||||
module_graph,
|
||||
*module_graph,
|
||||
*chunking_context,
|
||||
)
|
||||
})
|
||||
@@ -102,7 +109,7 @@ pub async fn make_chunk_group(
|
||||
.map(|&batch_group| {
|
||||
ChunkItemBatchGroup::from_module_batch_group(
|
||||
ChunkableModuleBatchGroup::from_module_batch_group(*batch_group),
|
||||
module_graph,
|
||||
*module_graph,
|
||||
*chunking_context,
|
||||
)
|
||||
.to_resolved()
|
||||
@@ -118,10 +125,11 @@ pub async fn make_chunk_group(
|
||||
availability_info
|
||||
};
|
||||
let async_loaders = async_modules
|
||||
.into_iter()
|
||||
.iter()
|
||||
.copied()
|
||||
.map(async |module| {
|
||||
chunking_context
|
||||
.async_loader_chunk_item(*module, module_graph, async_availability_info)
|
||||
.async_loader_chunk_item(*module, *module_graph, async_availability_info)
|
||||
.to_resolved()
|
||||
.await
|
||||
})
|
||||
@@ -149,7 +157,8 @@ pub async fn make_chunk_group(
|
||||
.await?;
|
||||
|
||||
let referenced_output_assets = traced_modules
|
||||
.into_iter()
|
||||
.iter()
|
||||
.copied()
|
||||
.map(|module| async move {
|
||||
Ok(ResolvedVc::upcast(
|
||||
TracedAsset::new(*module).to_resolved().await?,
|
||||
@@ -162,7 +171,7 @@ pub async fn make_chunk_group(
|
||||
|
||||
// Pass chunk items to chunking algorithm
|
||||
let chunks = make_chunks(
|
||||
module_graph,
|
||||
*module_graph,
|
||||
*chunking_context,
|
||||
Vc::cell(chunk_items),
|
||||
Vc::cell(chunk_item_batch_groups),
|
||||
@@ -179,6 +188,9 @@ pub async fn make_chunk_group(
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, NonLocalValue, Encode, Decode,
|
||||
)]
|
||||
pub struct ChunkGroupContentOptions {
|
||||
/// The availability info of the chunk group
|
||||
pub availability_info: AvailabilityInfo,
|
||||
@@ -189,15 +201,37 @@ pub struct ChunkGroupContentOptions {
|
||||
/// Whether module merging is enabled
|
||||
pub should_merge_modules: bool,
|
||||
/// The batching config to use
|
||||
pub batching_config: Vc<BatchingConfig>,
|
||||
pub batching_config: ResolvedVc<BatchingConfig>,
|
||||
}
|
||||
|
||||
/// Computes the content of a chunk group.
|
||||
pub async fn chunk_group_content(
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
chunk_group_entries: impl IntoIterator<
|
||||
IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
|
||||
> + Send,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunk_group: ChunkGroup,
|
||||
options: ChunkGroupContentOptions,
|
||||
) -> Result<ChunkGroupContent> {
|
||||
let availability_info = options.availability_info;
|
||||
let chunk_group_content = chunk_group_content_operation(module_graph, chunk_group, options);
|
||||
let available_modules = available_modules_operation(chunk_group_content);
|
||||
let inner = chunk_group_content.connect().await?;
|
||||
|
||||
Ok(ChunkGroupContent {
|
||||
inner,
|
||||
availability_info: availability_info.with_modules(available_modules).await?,
|
||||
})
|
||||
}
|
||||
|
||||
#[turbo_tasks::function(operation)]
|
||||
async fn available_modules_operation(
|
||||
chunk_group_content: OperationVc<ChunkGroupContentInner>,
|
||||
) -> Result<Vc<AvailableModulesSet>> {
|
||||
Ok(*chunk_group_content.connect().await?.available_modules)
|
||||
}
|
||||
|
||||
#[turbo_tasks::function(operation)]
|
||||
async fn chunk_group_content_operation(
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunk_group: ChunkGroup,
|
||||
ChunkGroupContentOptions {
|
||||
availability_info,
|
||||
can_split_async,
|
||||
@@ -205,8 +239,8 @@ pub async fn chunk_group_content(
|
||||
should_merge_modules,
|
||||
batching_config,
|
||||
}: ChunkGroupContentOptions,
|
||||
) -> Result<ChunkGroupContent> {
|
||||
let module_batches_graph = module_graph.module_batches(batching_config).await?;
|
||||
) -> Result<Vc<ChunkGroupContentInner>> {
|
||||
let module_batches_graph = module_graph.module_batches(*batching_config).await?;
|
||||
|
||||
type ModuleToChunkableMap = FxHashMap<ModuleOrBatch, ChunkableModuleOrBatch>;
|
||||
|
||||
@@ -229,63 +263,50 @@ pub async fn chunk_group_content(
|
||||
None => None,
|
||||
};
|
||||
|
||||
let chunk_group_entries = chunk_group_entries.into_iter();
|
||||
let mut entries = Vec::with_capacity(chunk_group_entries.size_hint().0);
|
||||
for entry in chunk_group_entries {
|
||||
let mut entries = Vec::with_capacity(chunk_group.entries_count());
|
||||
for entry in chunk_group.entries() {
|
||||
entries.push(module_batches_graph.get_entry_index(entry).await?);
|
||||
}
|
||||
|
||||
module_batches_graph.traverse_edges_from_entries_dfs(
|
||||
entries,
|
||||
&mut state,
|
||||
|parent_info, &node, state| {
|
||||
if matches!(node, ModuleOrBatch::None(_)) {
|
||||
return Ok(GraphTraversalAction::Continue);
|
||||
}
|
||||
// Traced modules need to have a special handling
|
||||
if let Some((
|
||||
_,
|
||||
ModuleBatchesGraphEdge {
|
||||
ty: ChunkingType::Traced,
|
||||
..
|
||||
},
|
||||
)) = parent_info
|
||||
{
|
||||
if should_trace {
|
||||
let ModuleOrBatch::Module(module) = node else {
|
||||
unreachable!();
|
||||
};
|
||||
state.traced_modules.insert(module);
|
||||
{
|
||||
let _span = tracing::trace_span!("traversal").entered();
|
||||
module_batches_graph.traverse_edges_from_entries_dfs(
|
||||
entries,
|
||||
&mut state,
|
||||
|parent_info, &node, state| {
|
||||
if matches!(node, ModuleOrBatch::None(_)) {
|
||||
return Ok(GraphTraversalAction::Continue);
|
||||
}
|
||||
return Ok(GraphTraversalAction::Exclude);
|
||||
}
|
||||
|
||||
let Some(chunkable_node) = ChunkableModuleOrBatch::from_module_or_batch(node) else {
|
||||
return Ok(GraphTraversalAction::Exclude);
|
||||
};
|
||||
|
||||
let is_available = available_modules
|
||||
.as_ref()
|
||||
.is_some_and(|available_modules| available_modules.get(chunkable_node.into()));
|
||||
|
||||
let Some((_, edge)) = parent_info else {
|
||||
// An entry from the entries list
|
||||
return Ok(if is_available {
|
||||
GraphTraversalAction::Exclude
|
||||
} else if state
|
||||
.unsorted_items
|
||||
.try_insert(node, chunkable_node)
|
||||
.is_ok()
|
||||
// Traced modules need to have a special handling
|
||||
if let Some((
|
||||
_,
|
||||
ModuleBatchesGraphEdge {
|
||||
ty: ChunkingType::Traced,
|
||||
..
|
||||
},
|
||||
)) = parent_info
|
||||
{
|
||||
GraphTraversalAction::Continue
|
||||
} else {
|
||||
GraphTraversalAction::Exclude
|
||||
});
|
||||
};
|
||||
if should_trace {
|
||||
let ModuleOrBatch::Module(module) = node else {
|
||||
unreachable!();
|
||||
};
|
||||
state.traced_modules.insert(module);
|
||||
}
|
||||
return Ok(GraphTraversalAction::Exclude);
|
||||
}
|
||||
|
||||
Ok(match edge.ty {
|
||||
ChunkingType::Parallel { .. } | ChunkingType::Shared { .. } => {
|
||||
if is_available {
|
||||
let Some(chunkable_node) = ChunkableModuleOrBatch::from_module_or_batch(node)
|
||||
else {
|
||||
return Ok(GraphTraversalAction::Exclude);
|
||||
};
|
||||
|
||||
let is_available = available_modules
|
||||
.as_ref()
|
||||
.is_some_and(|available_modules| available_modules.get(chunkable_node.into()));
|
||||
|
||||
let Some((_, edge)) = parent_info else {
|
||||
// An entry from the entries list
|
||||
return Ok(if is_available {
|
||||
GraphTraversalAction::Exclude
|
||||
} else if state
|
||||
.unsorted_items
|
||||
@@ -295,53 +316,70 @@ pub async fn chunk_group_content(
|
||||
GraphTraversalAction::Continue
|
||||
} else {
|
||||
GraphTraversalAction::Exclude
|
||||
}
|
||||
}
|
||||
ChunkingType::Async => {
|
||||
if can_split_async {
|
||||
let chunkable_module = ResolvedVc::try_downcast(edge.module.unwrap())
|
||||
.context("Module in async chunking edge is not chunkable")?;
|
||||
let is_async_loader_available =
|
||||
available_modules.as_ref().is_some_and(|available_modules| {
|
||||
available_modules
|
||||
.get(AvailableModuleItem::AsyncLoader(chunkable_module))
|
||||
});
|
||||
if !is_async_loader_available {
|
||||
state.async_modules.insert(chunkable_module);
|
||||
});
|
||||
};
|
||||
|
||||
Ok(match edge.ty {
|
||||
ChunkingType::Parallel { .. } | ChunkingType::Shared { .. } => {
|
||||
if is_available {
|
||||
GraphTraversalAction::Exclude
|
||||
} else if state
|
||||
.unsorted_items
|
||||
.try_insert(node, chunkable_node)
|
||||
.is_ok()
|
||||
{
|
||||
GraphTraversalAction::Continue
|
||||
} else {
|
||||
GraphTraversalAction::Exclude
|
||||
}
|
||||
GraphTraversalAction::Exclude
|
||||
} else if is_available {
|
||||
GraphTraversalAction::Exclude
|
||||
} else if state
|
||||
.unsorted_items
|
||||
.try_insert(node, chunkable_node)
|
||||
.is_ok()
|
||||
{
|
||||
GraphTraversalAction::Continue
|
||||
} else {
|
||||
}
|
||||
ChunkingType::Async => {
|
||||
if can_split_async {
|
||||
let chunkable_module =
|
||||
ResolvedVc::try_downcast(edge.module.unwrap())
|
||||
.context("Module in async chunking edge is not chunkable")?;
|
||||
let is_async_loader_available =
|
||||
available_modules.as_ref().is_some_and(|available_modules| {
|
||||
available_modules
|
||||
.get(AvailableModuleItem::AsyncLoader(chunkable_module))
|
||||
});
|
||||
if !is_async_loader_available {
|
||||
state.async_modules.insert(chunkable_module);
|
||||
}
|
||||
GraphTraversalAction::Exclude
|
||||
} else if is_available {
|
||||
GraphTraversalAction::Exclude
|
||||
} else if state
|
||||
.unsorted_items
|
||||
.try_insert(node, chunkable_node)
|
||||
.is_ok()
|
||||
{
|
||||
GraphTraversalAction::Continue
|
||||
} else {
|
||||
GraphTraversalAction::Exclude
|
||||
}
|
||||
}
|
||||
ChunkingType::Traced => {
|
||||
// handled above before the sidecast
|
||||
unreachable!();
|
||||
}
|
||||
ChunkingType::Isolated { .. } => {
|
||||
// TODO currently not implemented
|
||||
GraphTraversalAction::Exclude
|
||||
}
|
||||
})
|
||||
},
|
||||
|_, node, state| {
|
||||
// Insert modules in topological order
|
||||
if let Some(chunkable_module) = state.unsorted_items.get(node).copied() {
|
||||
state.chunkable_items.insert(chunkable_module);
|
||||
}
|
||||
ChunkingType::Traced => {
|
||||
// handled above before the sidecast
|
||||
unreachable!();
|
||||
}
|
||||
ChunkingType::Isolated { .. } => {
|
||||
// TODO currently not implemented
|
||||
GraphTraversalAction::Exclude
|
||||
}
|
||||
})
|
||||
},
|
||||
|_, node, state| {
|
||||
// Insert modules in topological order
|
||||
if let Some(chunkable_module) = state.unsorted_items.get(node).copied() {
|
||||
state.chunkable_items.insert(chunkable_module);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
// This needs to use the unmerged items
|
||||
let available_modules = state
|
||||
let available_modules: FxIndexSet<AvailableModuleItem> = state
|
||||
.chunkable_items
|
||||
.iter()
|
||||
.copied()
|
||||
@@ -354,9 +392,10 @@ pub async fn chunk_group_content(
|
||||
.map(AvailableModuleItem::AsyncLoader),
|
||||
)
|
||||
.collect();
|
||||
let availability_info = availability_info
|
||||
.with_modules(Vc::cell(available_modules))
|
||||
.await?;
|
||||
let available_modules: ResolvedVc<AvailableModulesSet> =
|
||||
Vc::<AvailableModulesSet>::cell(available_modules)
|
||||
.to_resolved()
|
||||
.await?;
|
||||
|
||||
let should_merge_modules = if should_merge_modules {
|
||||
let merged_modules = module_graph.merged_modules();
|
||||
@@ -373,20 +412,13 @@ pub async fn chunk_group_content(
|
||||
.into_iter()
|
||||
.map(async |chunkable_module| match chunkable_module {
|
||||
ChunkableModuleOrBatch::Module(module) => {
|
||||
if !merged_modules_ref
|
||||
.should_create_chunk_item_for(ResolvedVc::upcast(module))
|
||||
.await?
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let module = if let Some(replacement) = merged_modules_ref
|
||||
let module = match merged_modules_ref
|
||||
.should_replace_module(ResolvedVc::upcast(module))
|
||||
.await?
|
||||
{
|
||||
replacement
|
||||
} else {
|
||||
module
|
||||
Some(None) => return Ok(None),
|
||||
Some(Some(replacement)) => replacement,
|
||||
None => module,
|
||||
};
|
||||
|
||||
Ok(Some(ChunkableModuleOrBatch::Module(module)))
|
||||
@@ -399,6 +431,7 @@ pub async fn chunk_group_content(
|
||||
ChunkableModuleOrBatch::None(i) => Ok(Some(ChunkableModuleOrBatch::None(i))),
|
||||
})
|
||||
.try_flat_join()
|
||||
.instrument(tracing::trace_span!("replace with merged modules"))
|
||||
.await?
|
||||
} else {
|
||||
state.chunkable_items.into_iter().collect()
|
||||
@@ -421,13 +454,14 @@ pub async fn chunk_group_content(
|
||||
batch_groups.into_iter().collect()
|
||||
};
|
||||
|
||||
Ok(ChunkGroupContent {
|
||||
Ok(ChunkGroupContentInner {
|
||||
chunkable_items,
|
||||
batch_groups,
|
||||
async_modules: state.async_modules,
|
||||
traced_modules: state.traced_modules,
|
||||
availability_info,
|
||||
})
|
||||
available_modules,
|
||||
}
|
||||
.cell())
|
||||
}
|
||||
|
||||
#[turbo_tasks::function]
|
||||
@@ -444,22 +478,19 @@ async fn map_module_batch(
|
||||
.iter()
|
||||
.copied()
|
||||
.map(async |module| {
|
||||
if !merged_modules
|
||||
.should_create_chunk_item_for(ResolvedVc::upcast(module))
|
||||
.await?
|
||||
{
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let module = if let Some(replacement) = merged_modules
|
||||
let module = match merged_modules
|
||||
.should_replace_module(ResolvedVc::upcast(module))
|
||||
.await?
|
||||
{
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
replacement
|
||||
} else {
|
||||
module
|
||||
Some(None) => {
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
return Ok(None);
|
||||
}
|
||||
Some(Some(replacement)) => {
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
replacement
|
||||
}
|
||||
None => module,
|
||||
};
|
||||
|
||||
Ok(Some(module))
|
||||
@@ -492,21 +523,16 @@ async fn map_module_batch_group(
|
||||
.copied()
|
||||
.map(async |chunkable_module| match chunkable_module {
|
||||
ModuleOrBatch::Module(module) => {
|
||||
if !merged_modules_ref
|
||||
.should_create_chunk_item_for(module)
|
||||
.await?
|
||||
{
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let module = if let Some(replacement) =
|
||||
merged_modules_ref.should_replace_module(module).await?
|
||||
{
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
ResolvedVc::upcast(replacement)
|
||||
} else {
|
||||
module
|
||||
let module = match merged_modules_ref.should_replace_module(module).await? {
|
||||
Some(None) => {
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
return Ok(None);
|
||||
}
|
||||
Some(Some(replacement)) => {
|
||||
modified.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
ResolvedVc::upcast(replacement)
|
||||
}
|
||||
None => module,
|
||||
};
|
||||
|
||||
Ok(Some(ModuleOrBatch::Module(module)))
|
||||
|
||||
@@ -16,7 +16,7 @@ use bincode::{Decode, Encode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use turbo_rcstr::RcStr;
|
||||
use turbo_tasks::{
|
||||
FxIndexSet, NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
|
||||
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
|
||||
debug::ValueDebugFormat, trace::TraceRawVcs,
|
||||
};
|
||||
use turbo_tasks_hash::DeterministicHash;
|
||||
@@ -36,7 +36,7 @@ pub use crate::chunk::{
|
||||
};
|
||||
use crate::{
|
||||
asset::Asset,
|
||||
chunk::availability_info::AvailabilityInfo,
|
||||
chunk::{availability_info::AvailabilityInfo, available_modules::AvailableModulesSet},
|
||||
ident::AssetIdent,
|
||||
module::Module,
|
||||
module_graph::{
|
||||
@@ -427,11 +427,19 @@ impl ChunkingType {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ChunkGroupContent {
|
||||
#[turbo_tasks::value(cell = "new")]
|
||||
pub struct ChunkGroupContentInner {
|
||||
pub chunkable_items: Vec<ChunkableModuleOrBatch>,
|
||||
pub batch_groups: Vec<ResolvedVc<ModuleBatchGroup>>,
|
||||
#[bincode(with = "turbo_bincode::indexset")]
|
||||
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
|
||||
#[bincode(with = "turbo_bincode::indexset")]
|
||||
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
|
||||
pub available_modules: ResolvedVc<AvailableModulesSet>,
|
||||
}
|
||||
|
||||
pub struct ChunkGroupContent {
|
||||
pub inner: ReadRef<ChunkGroupContentInner>,
|
||||
pub availability_info: AvailabilityInfo,
|
||||
}
|
||||
|
||||
|
||||
@@ -161,7 +161,9 @@ impl ChunkGroupEntry {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, Encode, Decode)]
|
||||
#[derive(
|
||||
Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, Encode, Decode, NonLocalValue,
|
||||
)]
|
||||
pub enum ChunkGroup {
|
||||
/// The entry chunk group of the compilation, e.g. src/index.js for a SPA, or app/foo/page.js
|
||||
/// for Next.js.
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::{
|
||||
#[turbo_tasks::value(transparent, cell = "keyed")]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub struct MergedModulesReplacements(
|
||||
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn ChunkableModule>>>,
|
||||
FxHashMap<ResolvedVc<Box<dyn Module>>, Option<ResolvedVc<Box<dyn ChunkableModule>>>>,
|
||||
);
|
||||
|
||||
#[turbo_tasks::value(transparent, cell = "keyed")]
|
||||
@@ -29,27 +29,30 @@ pub struct MergedModulesOriginalModules(
|
||||
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn Module>>>,
|
||||
);
|
||||
|
||||
#[turbo_tasks::value(transparent, cell = "keyed")]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub struct MergedModulesIncluded(FxHashSet<ResolvedVc<Box<dyn Module>>>);
|
||||
|
||||
#[turbo_tasks::value]
|
||||
pub struct MergedModuleInfo {
|
||||
/// A map of modules to the merged module containing the module plus additional modules.
|
||||
/// A map of modules describing how they participate in module merging:
|
||||
/// - Not present: the module is not affected by merging and a regular chunk item should be
|
||||
/// created for it.
|
||||
/// - Present with `Some(replacement)`: the module should be replaced with the given merged
|
||||
/// module when creating a chunk item.
|
||||
/// - Present with `None`: the module is already included in some other merged module returned
|
||||
/// by a `Some(replacement)` entry and no chunk item should be created for it.
|
||||
pub replacements: ResolvedVc<MergedModulesReplacements>,
|
||||
/// A map of replacement modules to their corresponding chunk group info (which is the same as
|
||||
/// the chunk group info of the original module it replaced).
|
||||
pub replacements_to_original: ResolvedVc<MergedModulesOriginalModules>,
|
||||
/// A map of modules that are already contained as values in replacements.
|
||||
pub included: ResolvedVc<MergedModulesIncluded>,
|
||||
}
|
||||
|
||||
impl MergedModuleInfo {
|
||||
/// Whether the given module should be replaced with a merged module.
|
||||
/// Returns the merging decision for the given module:
|
||||
/// - `None`: the module is not affected by merging, keep it as-is.
|
||||
/// - `Some(None)`: the module is already included in another merged module, skip it.
|
||||
/// - `Some(Some(replacement))`: the module should be replaced with `replacement`.
|
||||
pub async fn should_replace_module(
|
||||
&self,
|
||||
module: ResolvedVc<Box<dyn Module>>,
|
||||
) -> Result<Option<ResolvedVc<Box<dyn ChunkableModule>>>> {
|
||||
) -> Result<Option<Option<ResolvedVc<Box<dyn ChunkableModule>>>>> {
|
||||
Ok(self.replacements.get(&module).await?.as_deref().copied())
|
||||
}
|
||||
|
||||
@@ -66,15 +69,6 @@ impl MergedModuleInfo {
|
||||
.as_deref()
|
||||
.copied())
|
||||
}
|
||||
|
||||
// Whether the given module should be skipped during chunking, as it is already included in a
|
||||
// module returned by some `should_replace_module` call.
|
||||
pub async fn should_create_chunk_item_for(
|
||||
&self,
|
||||
module: ResolvedVc<Box<dyn Module>>,
|
||||
) -> Result<bool> {
|
||||
Ok(!self.included.contains_key(&module).await?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine which modules can be merged together:
|
||||
@@ -774,28 +768,32 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
|
||||
#[allow(clippy::type_complexity)]
|
||||
let mut replacements: FxHashMap<
|
||||
ResolvedVc<Box<dyn Module>>,
|
||||
ResolvedVc<Box<dyn ChunkableModule>>,
|
||||
Option<ResolvedVc<Box<dyn ChunkableModule>>>,
|
||||
> = Default::default();
|
||||
#[allow(clippy::type_complexity)]
|
||||
let mut replacements_to_original: FxHashMap<
|
||||
ResolvedVc<Box<dyn Module>>,
|
||||
ResolvedVc<Box<dyn Module>>,
|
||||
> = Default::default();
|
||||
let mut included: FxHashSet<ResolvedVc<Box<dyn Module>>> = FxHashSet::default();
|
||||
let mut merged_groups = 0;
|
||||
let mut included_modules = 0;
|
||||
|
||||
for (original, replacement, replacement_included) in result.into_iter().flatten() {
|
||||
replacements.insert(original, replacement);
|
||||
replacements.insert(original, Some(replacement));
|
||||
replacements_to_original.insert(ResolvedVc::upcast(replacement), original);
|
||||
included.extend(replacement_included);
|
||||
merged_groups += 1;
|
||||
included_modules += replacement_included.len();
|
||||
for included in replacement_included {
|
||||
replacements.insert(included, None);
|
||||
}
|
||||
}
|
||||
|
||||
span.record("merged_groups", replacements.len());
|
||||
span.record("included_modules", included.len());
|
||||
span.record("merged_groups", merged_groups);
|
||||
span.record("included_modules", included_modules);
|
||||
|
||||
Ok(MergedModuleInfo {
|
||||
replacements: ResolvedVc::cell(replacements),
|
||||
replacements_to_original: ResolvedVc::cell(replacements_to_original),
|
||||
included: ResolvedVc::cell(included),
|
||||
}
|
||||
.cell())
|
||||
}
|
||||
|
||||
@@ -528,19 +528,18 @@ impl ChunkingContext for NodeJsChunkingContext {
|
||||
self: ResolvedVc<Self>,
|
||||
ident: Vc<AssetIdent>,
|
||||
chunk_group: ChunkGroup,
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
availability_info: AvailabilityInfo,
|
||||
) -> Result<Vc<ChunkGroupResult>> {
|
||||
let span = tracing::info_span!("chunking", name = display(ident.to_string().await?));
|
||||
async move {
|
||||
let modules = chunk_group.entries();
|
||||
let MakeChunkGroupResult {
|
||||
chunks,
|
||||
referenced_output_assets,
|
||||
references,
|
||||
availability_info,
|
||||
} = make_chunk_group(
|
||||
modules,
|
||||
chunk_group,
|
||||
module_graph,
|
||||
ResolvedVc::upcast(self),
|
||||
availability_info,
|
||||
@@ -572,7 +571,7 @@ impl ChunkingContext for NodeJsChunkingContext {
|
||||
self: ResolvedVc<Self>,
|
||||
path: FileSystemPath,
|
||||
chunk_group: ChunkGroup,
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
extra_chunks: Vc<OutputAssets>,
|
||||
extra_referenced_assets: Vc<OutputAssets>,
|
||||
availability_info: AvailabilityInfo,
|
||||
@@ -583,14 +582,13 @@ impl ChunkingContext for NodeJsChunkingContext {
|
||||
chunking_type = "entry",
|
||||
);
|
||||
async move {
|
||||
let entries = chunk_group.entries();
|
||||
let MakeChunkGroupResult {
|
||||
chunks,
|
||||
mut referenced_output_assets,
|
||||
references,
|
||||
availability_info,
|
||||
} = make_chunk_group(
|
||||
entries,
|
||||
chunk_group.clone(),
|
||||
module_graph,
|
||||
ResolvedVc::upcast(self),
|
||||
availability_info,
|
||||
@@ -630,7 +628,7 @@ impl ChunkingContext for NodeJsChunkingContext {
|
||||
*module,
|
||||
Vc::cell(referenced_output_assets),
|
||||
Vc::cell(references),
|
||||
module_graph,
|
||||
*module_graph,
|
||||
*self,
|
||||
)
|
||||
.to_resolved()
|
||||
|
||||
Reference in New Issue
Block a user