Turbopack: make available_modules an OperationVc and simplify merged module bookkeeping (#94055)

### What?

Reworks how `chunk_group_content` exposes `available_modules` for
downstream chunking, and consolidates the bookkeeping for merged modules
in `ChunkGroup`.

- Splits `ChunkGroupContent` into a `turbo_tasks::value`
`ChunkGroupContentInner` (carrying `chunkable_items`, `batch_groups`,
`async_modules`, `traced_modules`, and `available_modules`) plus a thin
wrapper that pairs it with the resulting `AvailabilityInfo`.
`chunk_group_content_operation` now runs entirely inside `turbo_tasks`,
allowing `AvailableModules` to store its set as an
`OperationVc<AvailableModulesSet>` and `.connect()` it on read.
- Replaces the separate `included` set and
`should_create_chunk_item_for` method on `MergedModuleInfo` with an
`Option<ChunkableModule>` value type on the `replacements` map. Call
sites now decide all three cases (replace / skip / keep) in one match
via `should_replace_module`.

### Why?

- `AvailableModules::snapshot` did not correctly derive its invalidation
priority, which caused double invalidations: the snapshot was treated as
a low-priority dependency even though it gated chunking work, so when
its inputs changed turbo-tasks would invalidate it after dependent
computations had already started, triggering them a second time.
Exposing `available_modules` as an `OperationVc` makes
`AvailableModules` inherit the correct invalidation priority from the
operation that produces it, which removes the double-invalidation.
- The previous `MergedModuleInfo` API had two parallel data structures
(`replacements` and `included`) and a helper that combined them. Folding
`included` into the value of `replacements` removes the duplication,
makes the three states (replace, skip-because-merged, keep) explicit,
and shrinks the call-site logic in `chunk_group.rs`.

### How?

**`available_modules` as an `OperationVc`:**

- New `ChunkGroupContentInner` is a `turbo_tasks::value` holding the
chunkable items, batch groups, async/traced modules, and an
`OperationVc<AvailableModulesSet>` for `available_modules`.
- `ChunkGroupContent` becomes a plain struct wrapping
`ResolvedVc<ChunkGroupContentInner>` together with the
`AvailabilityInfo` computed for the group.
- `chunk_group_content_operation` is an `OperationVc`-producing function
so its inner `available_modules` can be returned as an
`OperationVc<AvailableModulesSet>`. `AvailableModules` now stores that
operation Vc and `.connect()`s it whenever the set is read, so consumers
reach the set through the operation and inherit its priority.
- Browser and Node.js chunking contexts are updated to thread the new
shape through `make_chunk_group` and friends.

**`MergedModuleInfo` simplification:**

- `replacements` changes from `HashMap<ChunkableModule,
ChunkableModule>` to `HashMap<ChunkableModule,
Option<ChunkableModule>>`: `Some(m)` means "replace with `m`", `None`
means "skip — already included in another merged module", absent means
"keep as-is".
- The old `included` set and `should_create_chunk_item_for` helper are
removed; `should_replace_module` returns the three-way outcome and
consumers match on it once.
- Call sites in `chunk_group.rs` are restructured around the single
match so the logic is no longer split across two lookups.

### Notes

- No user-facing behavior change is intended — this is an internal
refactor in `turbopack-core` (`chunk_group_content`,
`available_modules`, `chunk_group`) and the browser/nodejs chunking
contexts.
- Existing chunking tests in `turbopack-core` exercise the new shape.

<!-- NEXT_JS_LLM_PR -->
This commit is contained in:
Tobias Koppers
2026-05-26 13:10:04 +02:00
committed by GitHub
parent 01c9cbfe82
commit 0070514701
8 changed files with 250 additions and 219 deletions
@@ -763,13 +763,12 @@ impl ChunkingContext for BrowserChunkingContext {
self: ResolvedVc<Self>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> {
let span = tracing::info_span!("chunking", name = display(ident.to_string().await?));
async move {
let this = self.await?;
let entries = chunk_group.entries();
let input_availability_info = availability_info;
let MakeChunkGroupResult {
chunks,
@@ -777,7 +776,7 @@ impl ChunkingContext for BrowserChunkingContext {
references,
availability_info,
} = make_chunk_group(
entries,
chunk_group,
module_graph,
ResolvedVc::upcast(self),
input_availability_info,
@@ -834,7 +833,7 @@ impl ChunkingContext for BrowserChunkingContext {
self: ResolvedVc<Self>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
input_availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> {
let span = tracing::info_span!(
@@ -844,14 +843,13 @@ impl ChunkingContext for BrowserChunkingContext {
);
async move {
let this = self.await?;
let entries = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
chunk_group.clone(),
module_graph,
ResolvedVc::upcast(self),
input_availability_info,
@@ -903,7 +901,7 @@ impl ChunkingContext for BrowserChunkingContext {
}
assets.push(
self.generate_evaluate_chunk(ident, other_assets, entries, module_graph)
self.generate_evaluate_chunk(ident, other_assets, entries, *module_graph)
.to_resolved()
.await?,
);
@@ -2,7 +2,7 @@ use anyhow::Result;
use bincode::{Decode, Encode};
use bitfield::bitfield;
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, Vc, trace::TraceRawVcs};
use turbo_tasks::{NonLocalValue, OperationVc, ResolvedVc, TaskInput, trace::TraceRawVcs};
use crate::chunk::available_modules::{AvailableModules, AvailableModulesSet};
@@ -34,7 +34,7 @@ impl AvailabilityInfo {
self.available_modules
}
pub async fn with_modules(self, modules: Vc<AvailableModulesSet>) -> Result<Self> {
pub async fn with_modules(self, modules: OperationVc<AvailableModulesSet>) -> Result<Self> {
Ok(if let Some(available_modules) = self.available_modules {
Self {
flags: self.flags,
@@ -1,8 +1,8 @@
use anyhow::Result;
use bincode::{Decode, Encode};
use turbo_tasks::{
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, ValueToString, Vc,
trace::TraceRawVcs, turbofmt,
FxIndexSet, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt,
ValueToString, Vc, trace::TraceRawVcs, turbofmt,
};
use turbo_tasks_hash::Xxh3Hash64Hasher;
@@ -61,13 +61,13 @@ pub struct AvailableModulesSet(
#[turbo_tasks::value]
pub struct AvailableModules {
parent: Option<ResolvedVc<AvailableModules>>,
modules: ResolvedVc<AvailableModulesSet>,
modules: OperationVc<AvailableModulesSet>,
}
#[turbo_tasks::value_impl]
impl AvailableModules {
#[turbo_tasks::function]
pub fn new(modules: ResolvedVc<AvailableModulesSet>) -> Vc<Self> {
pub fn new(modules: OperationVc<AvailableModulesSet>) -> Vc<Self> {
AvailableModules {
parent: None,
modules,
@@ -78,7 +78,7 @@ impl AvailableModules {
#[turbo_tasks::function]
pub fn with_modules(
self: ResolvedVc<Self>,
modules: ResolvedVc<AvailableModulesSet>,
modules: OperationVc<AvailableModulesSet>,
) -> Result<Vc<Self>> {
Ok(AvailableModules {
parent: Some(self),
@@ -97,6 +97,7 @@ impl AvailableModules {
}
let item_idents = self
.modules
.connect()
.await?
.iter()
.map(async |&module| module.ident_strings().await)
@@ -118,7 +119,7 @@ impl AvailableModules {
#[turbo_tasks::function]
pub async fn get(&self, item: AvailableModuleItem) -> Result<Vc<bool>> {
if self.modules.await?.contains(&item) {
if self.modules.connect().await?.contains(&item) {
return Ok(Vc::cell(true));
};
if let Some(parent) = self.parent {
@@ -129,7 +130,7 @@ impl AvailableModules {
#[turbo_tasks::function]
pub async fn snapshot(&self) -> Result<Vc<AvailableModulesSnapshot>> {
let modules = self.modules.await?;
let modules = self.modules.connect().await?;
let parent = if let Some(parent) = self.parent {
Some(parent.snapshot().await?)
} else {
@@ -1,24 +1,30 @@
use std::sync::atomic::AtomicBool;
use anyhow::{Context, Result};
use bincode::{Decode, Encode};
use rustc_hash::FxHashMap;
use tracing::Instrument;
use turbo_rcstr::rcstr;
use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
use turbo_tasks::{
FxIndexSet, NonLocalValue, OperationVc, ResolvedVc, TaskInput, TryFlatJoinIterExt,
TryJoinIterExt, Vc, trace::TraceRawVcs,
};
use super::{
ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext,
availability_info::AvailabilityInfo, chunking::make_chunks,
ChunkItemWithAsyncModuleInfo, ChunkingContext, availability_info::AvailabilityInfo,
chunking::make_chunks,
};
use crate::{
chunk::{
ChunkableModule, ChunkingType, Chunks,
available_modules::AvailableModuleItem,
ChunkGroupContent, ChunkGroupContentInner, ChunkableModule, ChunkingType, Chunks,
available_modules::{AvailableModuleItem, AvailableModulesSet},
chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
},
environment::ChunkLoading,
module::Module,
module_graph::{
GraphTraversalAction, ModuleGraph,
chunk_group_info::ChunkGroup,
merged_modules::MergedModuleInfo,
module_batch::{
ChunkableModuleBatchGroup, ChunkableModuleOrBatch, ModuleBatch, ModuleBatchGroup,
@@ -39,11 +45,8 @@ pub struct MakeChunkGroupResult {
/// Creates a chunk group from a set of entries.
pub async fn make_chunk_group(
chunk_group_entries: impl IntoIterator<
IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
> + Send
+ Clone,
module_graph: Vc<ModuleGraph>,
chunk_group: ChunkGroup,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
availability_info: AvailabilityInfo,
) -> Result<MakeChunkGroupResult> {
@@ -56,17 +59,14 @@ pub async fn make_chunk_group(
.await?;
let should_trace = *chunking_context.is_tracing_enabled().await?;
let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
let batching_config = chunking_context.batching_config();
let batching_config = chunking_context.batching_config().to_resolved().await?;
let ChunkGroupContent {
chunkable_items,
batch_groups,
async_modules,
traced_modules,
inner,
availability_info: new_availability_info,
} = chunk_group_content(
module_graph,
chunk_group_entries.clone(),
chunk_group,
ChunkGroupContentOptions {
availability_info,
can_split_async,
@@ -76,6 +76,13 @@ pub async fn make_chunk_group(
},
)
.await?;
let ChunkGroupContentInner {
chunkable_items,
batch_groups,
async_modules,
traced_modules,
available_modules: _,
} = &*inner;
let async_module_info = module_graph.async_module_info();
@@ -87,7 +94,7 @@ pub async fn make_chunk_group(
ChunkItemOrBatchWithAsyncModuleInfo::from_chunkable_module_or_batch(
m,
async_module_info,
module_graph,
*module_graph,
*chunking_context,
)
})
@@ -102,7 +109,7 @@ pub async fn make_chunk_group(
.map(|&batch_group| {
ChunkItemBatchGroup::from_module_batch_group(
ChunkableModuleBatchGroup::from_module_batch_group(*batch_group),
module_graph,
*module_graph,
*chunking_context,
)
.to_resolved()
@@ -118,10 +125,11 @@ pub async fn make_chunk_group(
availability_info
};
let async_loaders = async_modules
.into_iter()
.iter()
.copied()
.map(async |module| {
chunking_context
.async_loader_chunk_item(*module, module_graph, async_availability_info)
.async_loader_chunk_item(*module, *module_graph, async_availability_info)
.to_resolved()
.await
})
@@ -149,7 +157,8 @@ pub async fn make_chunk_group(
.await?;
let referenced_output_assets = traced_modules
.into_iter()
.iter()
.copied()
.map(|module| async move {
Ok(ResolvedVc::upcast(
TracedAsset::new(*module).to_resolved().await?,
@@ -162,7 +171,7 @@ pub async fn make_chunk_group(
// Pass chunk items to chunking algorithm
let chunks = make_chunks(
module_graph,
*module_graph,
*chunking_context,
Vc::cell(chunk_items),
Vc::cell(chunk_item_batch_groups),
@@ -179,6 +188,9 @@ pub async fn make_chunk_group(
})
}
#[derive(
Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, NonLocalValue, Encode, Decode,
)]
pub struct ChunkGroupContentOptions {
/// The availability info of the chunk group
pub availability_info: AvailabilityInfo,
@@ -189,15 +201,37 @@ pub struct ChunkGroupContentOptions {
/// Whether module merging is enabled
pub should_merge_modules: bool,
/// The batching config to use
pub batching_config: Vc<BatchingConfig>,
pub batching_config: ResolvedVc<BatchingConfig>,
}
/// Computes the content of a chunk group.
pub async fn chunk_group_content(
module_graph: Vc<ModuleGraph>,
chunk_group_entries: impl IntoIterator<
IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
> + Send,
module_graph: ResolvedVc<ModuleGraph>,
chunk_group: ChunkGroup,
options: ChunkGroupContentOptions,
) -> Result<ChunkGroupContent> {
let availability_info = options.availability_info;
let chunk_group_content = chunk_group_content_operation(module_graph, chunk_group, options);
let available_modules = available_modules_operation(chunk_group_content);
let inner = chunk_group_content.connect().await?;
Ok(ChunkGroupContent {
inner,
availability_info: availability_info.with_modules(available_modules).await?,
})
}
#[turbo_tasks::function(operation)]
async fn available_modules_operation(
chunk_group_content: OperationVc<ChunkGroupContentInner>,
) -> Result<Vc<AvailableModulesSet>> {
Ok(*chunk_group_content.connect().await?.available_modules)
}
#[turbo_tasks::function(operation)]
async fn chunk_group_content_operation(
module_graph: ResolvedVc<ModuleGraph>,
chunk_group: ChunkGroup,
ChunkGroupContentOptions {
availability_info,
can_split_async,
@@ -205,8 +239,8 @@ pub async fn chunk_group_content(
should_merge_modules,
batching_config,
}: ChunkGroupContentOptions,
) -> Result<ChunkGroupContent> {
let module_batches_graph = module_graph.module_batches(batching_config).await?;
) -> Result<Vc<ChunkGroupContentInner>> {
let module_batches_graph = module_graph.module_batches(*batching_config).await?;
type ModuleToChunkableMap = FxHashMap<ModuleOrBatch, ChunkableModuleOrBatch>;
@@ -229,63 +263,50 @@ pub async fn chunk_group_content(
None => None,
};
let chunk_group_entries = chunk_group_entries.into_iter();
let mut entries = Vec::with_capacity(chunk_group_entries.size_hint().0);
for entry in chunk_group_entries {
let mut entries = Vec::with_capacity(chunk_group.entries_count());
for entry in chunk_group.entries() {
entries.push(module_batches_graph.get_entry_index(entry).await?);
}
module_batches_graph.traverse_edges_from_entries_dfs(
entries,
&mut state,
|parent_info, &node, state| {
if matches!(node, ModuleOrBatch::None(_)) {
return Ok(GraphTraversalAction::Continue);
}
// Traced modules need to have a special handling
if let Some((
_,
ModuleBatchesGraphEdge {
ty: ChunkingType::Traced,
..
},
)) = parent_info
{
if should_trace {
let ModuleOrBatch::Module(module) = node else {
unreachable!();
};
state.traced_modules.insert(module);
{
let _span = tracing::trace_span!("traversal").entered();
module_batches_graph.traverse_edges_from_entries_dfs(
entries,
&mut state,
|parent_info, &node, state| {
if matches!(node, ModuleOrBatch::None(_)) {
return Ok(GraphTraversalAction::Continue);
}
return Ok(GraphTraversalAction::Exclude);
}
let Some(chunkable_node) = ChunkableModuleOrBatch::from_module_or_batch(node) else {
return Ok(GraphTraversalAction::Exclude);
};
let is_available = available_modules
.as_ref()
.is_some_and(|available_modules| available_modules.get(chunkable_node.into()));
let Some((_, edge)) = parent_info else {
// An entry from the entries list
return Ok(if is_available {
GraphTraversalAction::Exclude
} else if state
.unsorted_items
.try_insert(node, chunkable_node)
.is_ok()
// Traced modules need to have a special handling
if let Some((
_,
ModuleBatchesGraphEdge {
ty: ChunkingType::Traced,
..
},
)) = parent_info
{
GraphTraversalAction::Continue
} else {
GraphTraversalAction::Exclude
});
};
if should_trace {
let ModuleOrBatch::Module(module) = node else {
unreachable!();
};
state.traced_modules.insert(module);
}
return Ok(GraphTraversalAction::Exclude);
}
Ok(match edge.ty {
ChunkingType::Parallel { .. } | ChunkingType::Shared { .. } => {
if is_available {
let Some(chunkable_node) = ChunkableModuleOrBatch::from_module_or_batch(node)
else {
return Ok(GraphTraversalAction::Exclude);
};
let is_available = available_modules
.as_ref()
.is_some_and(|available_modules| available_modules.get(chunkable_node.into()));
let Some((_, edge)) = parent_info else {
// An entry from the entries list
return Ok(if is_available {
GraphTraversalAction::Exclude
} else if state
.unsorted_items
@@ -295,53 +316,70 @@ pub async fn chunk_group_content(
GraphTraversalAction::Continue
} else {
GraphTraversalAction::Exclude
}
}
ChunkingType::Async => {
if can_split_async {
let chunkable_module = ResolvedVc::try_downcast(edge.module.unwrap())
.context("Module in async chunking edge is not chunkable")?;
let is_async_loader_available =
available_modules.as_ref().is_some_and(|available_modules| {
available_modules
.get(AvailableModuleItem::AsyncLoader(chunkable_module))
});
if !is_async_loader_available {
state.async_modules.insert(chunkable_module);
});
};
Ok(match edge.ty {
ChunkingType::Parallel { .. } | ChunkingType::Shared { .. } => {
if is_available {
GraphTraversalAction::Exclude
} else if state
.unsorted_items
.try_insert(node, chunkable_node)
.is_ok()
{
GraphTraversalAction::Continue
} else {
GraphTraversalAction::Exclude
}
GraphTraversalAction::Exclude
} else if is_available {
GraphTraversalAction::Exclude
} else if state
.unsorted_items
.try_insert(node, chunkable_node)
.is_ok()
{
GraphTraversalAction::Continue
} else {
}
ChunkingType::Async => {
if can_split_async {
let chunkable_module =
ResolvedVc::try_downcast(edge.module.unwrap())
.context("Module in async chunking edge is not chunkable")?;
let is_async_loader_available =
available_modules.as_ref().is_some_and(|available_modules| {
available_modules
.get(AvailableModuleItem::AsyncLoader(chunkable_module))
});
if !is_async_loader_available {
state.async_modules.insert(chunkable_module);
}
GraphTraversalAction::Exclude
} else if is_available {
GraphTraversalAction::Exclude
} else if state
.unsorted_items
.try_insert(node, chunkable_node)
.is_ok()
{
GraphTraversalAction::Continue
} else {
GraphTraversalAction::Exclude
}
}
ChunkingType::Traced => {
// handled above before the sidecast
unreachable!();
}
ChunkingType::Isolated { .. } => {
// TODO currently not implemented
GraphTraversalAction::Exclude
}
})
},
|_, node, state| {
// Insert modules in topological order
if let Some(chunkable_module) = state.unsorted_items.get(node).copied() {
state.chunkable_items.insert(chunkable_module);
}
ChunkingType::Traced => {
// handled above before the sidecast
unreachable!();
}
ChunkingType::Isolated { .. } => {
// TODO currently not implemented
GraphTraversalAction::Exclude
}
})
},
|_, node, state| {
// Insert modules in topological order
if let Some(chunkable_module) = state.unsorted_items.get(node).copied() {
state.chunkable_items.insert(chunkable_module);
}
},
)?;
},
)?;
}
// This needs to use the unmerged items
let available_modules = state
let available_modules: FxIndexSet<AvailableModuleItem> = state
.chunkable_items
.iter()
.copied()
@@ -354,9 +392,10 @@ pub async fn chunk_group_content(
.map(AvailableModuleItem::AsyncLoader),
)
.collect();
let availability_info = availability_info
.with_modules(Vc::cell(available_modules))
.await?;
let available_modules: ResolvedVc<AvailableModulesSet> =
Vc::<AvailableModulesSet>::cell(available_modules)
.to_resolved()
.await?;
let should_merge_modules = if should_merge_modules {
let merged_modules = module_graph.merged_modules();
@@ -373,20 +412,13 @@ pub async fn chunk_group_content(
.into_iter()
.map(async |chunkable_module| match chunkable_module {
ChunkableModuleOrBatch::Module(module) => {
if !merged_modules_ref
.should_create_chunk_item_for(ResolvedVc::upcast(module))
.await?
{
return Ok(None);
}
let module = if let Some(replacement) = merged_modules_ref
let module = match merged_modules_ref
.should_replace_module(ResolvedVc::upcast(module))
.await?
{
replacement
} else {
module
Some(None) => return Ok(None),
Some(Some(replacement)) => replacement,
None => module,
};
Ok(Some(ChunkableModuleOrBatch::Module(module)))
@@ -399,6 +431,7 @@ pub async fn chunk_group_content(
ChunkableModuleOrBatch::None(i) => Ok(Some(ChunkableModuleOrBatch::None(i))),
})
.try_flat_join()
.instrument(tracing::trace_span!("replace with merged modules"))
.await?
} else {
state.chunkable_items.into_iter().collect()
@@ -421,13 +454,14 @@ pub async fn chunk_group_content(
batch_groups.into_iter().collect()
};
Ok(ChunkGroupContent {
Ok(ChunkGroupContentInner {
chunkable_items,
batch_groups,
async_modules: state.async_modules,
traced_modules: state.traced_modules,
availability_info,
})
available_modules,
}
.cell())
}
#[turbo_tasks::function]
@@ -444,22 +478,19 @@ async fn map_module_batch(
.iter()
.copied()
.map(async |module| {
if !merged_modules
.should_create_chunk_item_for(ResolvedVc::upcast(module))
.await?
{
modified.store(true, std::sync::atomic::Ordering::Relaxed);
return Ok(None);
}
let module = if let Some(replacement) = merged_modules
let module = match merged_modules
.should_replace_module(ResolvedVc::upcast(module))
.await?
{
modified.store(true, std::sync::atomic::Ordering::Relaxed);
replacement
} else {
module
Some(None) => {
modified.store(true, std::sync::atomic::Ordering::Relaxed);
return Ok(None);
}
Some(Some(replacement)) => {
modified.store(true, std::sync::atomic::Ordering::Relaxed);
replacement
}
None => module,
};
Ok(Some(module))
@@ -492,21 +523,16 @@ async fn map_module_batch_group(
.copied()
.map(async |chunkable_module| match chunkable_module {
ModuleOrBatch::Module(module) => {
if !merged_modules_ref
.should_create_chunk_item_for(module)
.await?
{
modified.store(true, std::sync::atomic::Ordering::Relaxed);
return Ok(None);
}
let module = if let Some(replacement) =
merged_modules_ref.should_replace_module(module).await?
{
modified.store(true, std::sync::atomic::Ordering::Relaxed);
ResolvedVc::upcast(replacement)
} else {
module
let module = match merged_modules_ref.should_replace_module(module).await? {
Some(None) => {
modified.store(true, std::sync::atomic::Ordering::Relaxed);
return Ok(None);
}
Some(Some(replacement)) => {
modified.store(true, std::sync::atomic::Ordering::Relaxed);
ResolvedVc::upcast(replacement)
}
None => module,
};
Ok(Some(ModuleOrBatch::Module(module)))
@@ -16,7 +16,7 @@ use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexSet, NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs,
};
use turbo_tasks_hash::DeterministicHash;
@@ -36,7 +36,7 @@ pub use crate::chunk::{
};
use crate::{
asset::Asset,
chunk::availability_info::AvailabilityInfo,
chunk::{availability_info::AvailabilityInfo, available_modules::AvailableModulesSet},
ident::AssetIdent,
module::Module,
module_graph::{
@@ -427,11 +427,19 @@ impl ChunkingType {
}
}
pub struct ChunkGroupContent {
#[turbo_tasks::value(cell = "new")]
pub struct ChunkGroupContentInner {
pub chunkable_items: Vec<ChunkableModuleOrBatch>,
pub batch_groups: Vec<ResolvedVc<ModuleBatchGroup>>,
#[bincode(with = "turbo_bincode::indexset")]
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
#[bincode(with = "turbo_bincode::indexset")]
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
pub available_modules: ResolvedVc<AvailableModulesSet>,
}
pub struct ChunkGroupContent {
pub inner: ReadRef<ChunkGroupContentInner>,
pub availability_info: AvailabilityInfo,
}
@@ -161,7 +161,9 @@ impl ChunkGroupEntry {
}
}
#[derive(Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, Encode, Decode)]
#[derive(
Debug, Clone, Hash, TaskInput, PartialEq, Eq, TraceRawVcs, Encode, Decode, NonLocalValue,
)]
pub enum ChunkGroup {
/// The entry chunk group of the compilation, e.g. src/index.js for a SPA, or app/foo/page.js
/// for Next.js.
@@ -20,7 +20,7 @@ use crate::{
#[turbo_tasks::value(transparent, cell = "keyed")]
#[allow(clippy::type_complexity)]
pub struct MergedModulesReplacements(
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn ChunkableModule>>>,
FxHashMap<ResolvedVc<Box<dyn Module>>, Option<ResolvedVc<Box<dyn ChunkableModule>>>>,
);
#[turbo_tasks::value(transparent, cell = "keyed")]
@@ -29,27 +29,30 @@ pub struct MergedModulesOriginalModules(
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn Module>>>,
);
#[turbo_tasks::value(transparent, cell = "keyed")]
#[allow(clippy::type_complexity)]
pub struct MergedModulesIncluded(FxHashSet<ResolvedVc<Box<dyn Module>>>);
#[turbo_tasks::value]
pub struct MergedModuleInfo {
/// A map of modules to the merged module containing the module plus additional modules.
/// A map of modules describing how they participate in module merging:
/// - Not present: the module is not affected by merging and a regular chunk item should be
/// created for it.
/// - Present with `Some(replacement)`: the module should be replaced with the given merged
/// module when creating a chunk item.
/// - Present with `None`: the module is already included in some other merged module returned
/// by a `Some(replacement)` entry and no chunk item should be created for it.
pub replacements: ResolvedVc<MergedModulesReplacements>,
/// A map of replacement modules to their corresponding chunk group info (which is the same as
/// the chunk group info of the original module it replaced).
pub replacements_to_original: ResolvedVc<MergedModulesOriginalModules>,
/// A map of modules that are already contained as values in replacements.
pub included: ResolvedVc<MergedModulesIncluded>,
}
impl MergedModuleInfo {
/// Whether the given module should be replaced with a merged module.
/// Returns the merging decision for the given module:
/// - `None`: the module is not affected by merging, keep it as-is.
/// - `Some(None)`: the module is already included in another merged module, skip it.
/// - `Some(Some(replacement))`: the module should be replaced with `replacement`.
pub async fn should_replace_module(
&self,
module: ResolvedVc<Box<dyn Module>>,
) -> Result<Option<ResolvedVc<Box<dyn ChunkableModule>>>> {
) -> Result<Option<Option<ResolvedVc<Box<dyn ChunkableModule>>>>> {
Ok(self.replacements.get(&module).await?.as_deref().copied())
}
@@ -66,15 +69,6 @@ impl MergedModuleInfo {
.as_deref()
.copied())
}
// Whether the given module should be skipped during chunking, as it is already included in a
// module returned by some `should_replace_module` call.
pub async fn should_create_chunk_item_for(
&self,
module: ResolvedVc<Box<dyn Module>>,
) -> Result<bool> {
Ok(!self.included.contains_key(&module).await?)
}
}
/// Determine which modules can be merged together:
@@ -774,28 +768,32 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
#[allow(clippy::type_complexity)]
let mut replacements: FxHashMap<
ResolvedVc<Box<dyn Module>>,
ResolvedVc<Box<dyn ChunkableModule>>,
Option<ResolvedVc<Box<dyn ChunkableModule>>>,
> = Default::default();
#[allow(clippy::type_complexity)]
let mut replacements_to_original: FxHashMap<
ResolvedVc<Box<dyn Module>>,
ResolvedVc<Box<dyn Module>>,
> = Default::default();
let mut included: FxHashSet<ResolvedVc<Box<dyn Module>>> = FxHashSet::default();
let mut merged_groups = 0;
let mut included_modules = 0;
for (original, replacement, replacement_included) in result.into_iter().flatten() {
replacements.insert(original, replacement);
replacements.insert(original, Some(replacement));
replacements_to_original.insert(ResolvedVc::upcast(replacement), original);
included.extend(replacement_included);
merged_groups += 1;
included_modules += replacement_included.len();
for included in replacement_included {
replacements.insert(included, None);
}
}
span.record("merged_groups", replacements.len());
span.record("included_modules", included.len());
span.record("merged_groups", merged_groups);
span.record("included_modules", included_modules);
Ok(MergedModuleInfo {
replacements: ResolvedVc::cell(replacements),
replacements_to_original: ResolvedVc::cell(replacements_to_original),
included: ResolvedVc::cell(included),
}
.cell())
}
@@ -528,19 +528,18 @@ impl ChunkingContext for NodeJsChunkingContext {
self: ResolvedVc<Self>,
ident: Vc<AssetIdent>,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
availability_info: AvailabilityInfo,
) -> Result<Vc<ChunkGroupResult>> {
let span = tracing::info_span!("chunking", name = display(ident.to_string().await?));
async move {
let modules = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
modules,
chunk_group,
module_graph,
ResolvedVc::upcast(self),
availability_info,
@@ -572,7 +571,7 @@ impl ChunkingContext for NodeJsChunkingContext {
self: ResolvedVc<Self>,
path: FileSystemPath,
chunk_group: ChunkGroup,
module_graph: Vc<ModuleGraph>,
module_graph: ResolvedVc<ModuleGraph>,
extra_chunks: Vc<OutputAssets>,
extra_referenced_assets: Vc<OutputAssets>,
availability_info: AvailabilityInfo,
@@ -583,14 +582,13 @@ impl ChunkingContext for NodeJsChunkingContext {
chunking_type = "entry",
);
async move {
let entries = chunk_group.entries();
let MakeChunkGroupResult {
chunks,
mut referenced_output_assets,
references,
availability_info,
} = make_chunk_group(
entries,
chunk_group.clone(),
module_graph,
ResolvedVc::upcast(self),
availability_info,
@@ -630,7 +628,7 @@ impl ChunkingContext for NodeJsChunkingContext {
*module,
Vc::cell(referenced_output_assets),
Vc::cell(references),
module_graph,
*module_graph,
*self,
)
.to_resolved()