mirror of
https://github.com/hanzoai/next.js.git
synced 2026-09-24 17:00:51 +00:00
[Turbopack] implement chunking based on the module graph (#74979)
### What? Visit the module graph by using the computed module graph during chunking --------- Co-authored-by: Will Binns-Smith <[email protected]> Co-authored-by: Niklas Mischkulnig <[email protected]>
This commit is contained in:
co-authored by
Will Binns-Smith
Niklas Mischkulnig
parent
a0de92e668
commit
645ef165dd
+36
-15
@@ -728,28 +728,35 @@ impl AppProject {
|
||||
&self,
|
||||
endpoint: Vc<AppEndpoint>,
|
||||
rsc_entry: ResolvedVc<Box<dyn Module>>,
|
||||
extra_entries: Vc<EvaluatableAssets>,
|
||||
has_layout_segments: bool,
|
||||
) -> Result<Vc<ModuleGraphs>> {
|
||||
let extra_entries = extra_entries
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|m| *ResolvedVc::upcast(*m));
|
||||
|
||||
if *self.project.per_page_module_graph().await? {
|
||||
// Implements layout segment optimization to compute a graph "chain" for each layout
|
||||
// segment
|
||||
async move {
|
||||
let mut graphs = vec![];
|
||||
let mut visited_modules = VisitedModules::empty();
|
||||
|
||||
if has_layout_segments {
|
||||
let mut visited_modules = if has_layout_segments {
|
||||
let ServerEntries {
|
||||
server_utils,
|
||||
server_component_entries,
|
||||
} = &*find_server_entries(*rsc_entry).await?;
|
||||
if !server_utils.is_empty() {
|
||||
let graph = SingleModuleGraph::new_with_entries_visited(
|
||||
server_utils.iter().map(|m| **m).collect(),
|
||||
visited_modules,
|
||||
);
|
||||
graphs.push(graph);
|
||||
visited_modules = VisitedModules::from_graph(graph)
|
||||
}
|
||||
|
||||
let graph = SingleModuleGraph::new_with_entries_visited(
|
||||
server_utils
|
||||
.iter()
|
||||
.map(|m| **m)
|
||||
.chain(extra_entries)
|
||||
.collect(),
|
||||
VisitedModules::empty(),
|
||||
);
|
||||
graphs.push(graph);
|
||||
let mut visited_modules = VisitedModules::from_graph(graph);
|
||||
|
||||
for module in server_component_entries.iter() {
|
||||
let graph = SingleModuleGraph::new_with_entries_visited(
|
||||
@@ -759,14 +766,27 @@ impl AppProject {
|
||||
graphs.push(graph);
|
||||
let is_layout =
|
||||
module.server_path().file_stem().await?.as_deref() == Some("layout");
|
||||
if is_layout {
|
||||
visited_modules = if is_layout {
|
||||
// Only propagate the visited_modules of the parent layout(s), not
|
||||
// across siblings such as loading.js and
|
||||
// page.js.
|
||||
visited_modules = visited_modules.concatenate(graph);
|
||||
}
|
||||
visited_modules.concatenate(graph)
|
||||
} else {
|
||||
// Prevents graph index from getting out of sync.
|
||||
// TODO We should remove VisitedModule entirely in favor of lookups
|
||||
// in SingleModuleGraph
|
||||
visited_modules.with_incremented_index()
|
||||
};
|
||||
}
|
||||
}
|
||||
visited_modules
|
||||
} else {
|
||||
let graph = SingleModuleGraph::new_with_entries_visited(
|
||||
extra_entries.collect::<_>(),
|
||||
VisitedModules::empty(),
|
||||
);
|
||||
graphs.push(graph);
|
||||
VisitedModules::from_graph(graph)
|
||||
};
|
||||
|
||||
let graph =
|
||||
SingleModuleGraph::new_with_entries_visited(vec![*rsc_entry], visited_modules);
|
||||
@@ -1021,6 +1041,7 @@ impl AppEndpoint {
|
||||
.app_module_graphs(
|
||||
self,
|
||||
*rsc_entry,
|
||||
this.app_project.client_runtime_entries(),
|
||||
matches!(this.ty, AppEndpointType::Page { .. }),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -102,8 +102,6 @@ impl MiddlewareEndpoint {
|
||||
)
|
||||
.resolve_entries(*this.asset_context);
|
||||
|
||||
let module_graph = this.project.module_graph_for_entries(evaluatable_assets);
|
||||
|
||||
let mut evaluatable_assets = evaluatable_assets.await?.clone_value();
|
||||
|
||||
let Some(module) =
|
||||
@@ -116,11 +114,14 @@ impl MiddlewareEndpoint {
|
||||
.context("Entry module must be evaluatable")?;
|
||||
evaluatable_assets.push(evaluatable.to_resolved().await?);
|
||||
|
||||
let evaluatable_assets = Vc::cell(evaluatable_assets);
|
||||
let module_graph = this.project.module_graph_for_entries(evaluatable_assets);
|
||||
|
||||
let edge_chunking_context = this.project.edge_chunking_context(false);
|
||||
|
||||
let edge_files = edge_chunking_context.evaluated_chunk_group_assets(
|
||||
module.ident(),
|
||||
Vc::cell(evaluatable_assets),
|
||||
evaluatable_assets,
|
||||
module_graph,
|
||||
Value::new(AvailabilityInfo::Root),
|
||||
);
|
||||
|
||||
@@ -328,13 +328,13 @@ impl ClientReferencesGraph {
|
||||
|
||||
state_map.insert(module, current_server_component);
|
||||
|
||||
match module_type {
|
||||
Ok(match module_type {
|
||||
Some(
|
||||
ClientReferenceMapType::EcmascriptClientReference { .. }
|
||||
| ClientReferenceMapType::CssClientReference { .. },
|
||||
) => GraphTraversalAction::Skip,
|
||||
_ => GraphTraversalAction::Continue,
|
||||
}
|
||||
})
|
||||
},
|
||||
|parent_info, node, state_map| {
|
||||
let Some((parent_node, _)) = parent_info else {
|
||||
|
||||
@@ -941,7 +941,7 @@ impl PageEndpoint {
|
||||
)) = next_dynamic_imports
|
||||
{
|
||||
collect_next_dynamic_chunks(
|
||||
module_graph,
|
||||
self.client_module_graph(),
|
||||
Vc::upcast(project.client_chunking_context()),
|
||||
next_dynamic_imports,
|
||||
NextDynamicChunkAvailability::AvailabilityInfo(client_availability_info),
|
||||
|
||||
@@ -19,7 +19,6 @@ use turbopack_core::{
|
||||
context::AssetContext,
|
||||
ident::AssetIdent,
|
||||
issue::{IssueExt, IssueSeverity},
|
||||
module_graph::ModuleGraph,
|
||||
reference_type::{InnerAssets, ReferenceType},
|
||||
resolve::{
|
||||
options::{ImportMapResult, ImportMappingReplacement, ReplacedImportMapping},
|
||||
@@ -695,14 +694,12 @@ async fn get_mock_stylesheet(
|
||||
.module();
|
||||
|
||||
let root = mock_fs.root();
|
||||
let module_graph = ModuleGraph::from_module(mocked_response_asset);
|
||||
let val = evaluate(
|
||||
mocked_response_asset,
|
||||
root,
|
||||
*env,
|
||||
AssetIdent::from_path(loader_path),
|
||||
asset_context,
|
||||
module_graph,
|
||||
*chunking_context,
|
||||
None,
|
||||
vec![],
|
||||
|
||||
@@ -1,18 +1,26 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use auto_hash_map::AutoSet;
|
||||
use futures::future::Either;
|
||||
use turbo_tasks::{FxIndexMap, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc};
|
||||
use swc_core::alloc::collections::FxHashMap;
|
||||
use turbo_tasks::{
|
||||
FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc,
|
||||
};
|
||||
|
||||
use super::{
|
||||
availability_info::AvailabilityInfo, available_modules::AvailableModulesInfo, chunk_content,
|
||||
chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem, ChunkItemTy,
|
||||
availability_info::AvailabilityInfo, available_modules::AvailableModulesInfo,
|
||||
chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkGroupContent, ChunkItem, ChunkItemTy,
|
||||
ChunkItemWithAsyncModuleInfo, ChunkableModule, ChunkingContext,
|
||||
};
|
||||
use crate::{
|
||||
environment::ChunkLoading, module::Module, module_graph::ModuleGraph, output::OutputAssets,
|
||||
rebase::RebasedAsset, reference::ModuleReference,
|
||||
chunk::ChunkingType,
|
||||
environment::ChunkLoading,
|
||||
module::Module,
|
||||
module_graph::{GraphTraversalAction, ModuleGraph},
|
||||
output::OutputAssets,
|
||||
rebase::RebasedAsset,
|
||||
reference::ModuleReference,
|
||||
};
|
||||
|
||||
pub struct MakeChunkGroupResult {
|
||||
@@ -33,7 +41,7 @@ pub async fn make_chunk_group(
|
||||
);
|
||||
let should_trace = *chunking_context.is_tracing_enabled().await?;
|
||||
|
||||
let ChunkContentResult {
|
||||
let ChunkGroupContent {
|
||||
chunkable_modules,
|
||||
async_modules,
|
||||
traced_modules,
|
||||
@@ -41,7 +49,8 @@ pub async fn make_chunk_group(
|
||||
forward_edges_inherit_async,
|
||||
local_back_edges_inherit_async,
|
||||
available_async_modules_back_edges_inherit_async,
|
||||
} = chunk_content(
|
||||
} = chunk_group_content(
|
||||
&*module_graph.await?,
|
||||
chunk_group_entries,
|
||||
availability_info,
|
||||
can_split_async,
|
||||
@@ -272,3 +281,154 @@ pub async fn references_to_output_assets(
|
||||
.collect::<Vec<_>>();
|
||||
Ok(OutputAssets::new(output_assets))
|
||||
}
|
||||
|
||||
pub async fn chunk_group_content(
|
||||
module_graph: &ModuleGraph,
|
||||
chunk_group_entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
|
||||
availability_info: AvailabilityInfo,
|
||||
can_split_async: bool,
|
||||
should_trace: bool,
|
||||
) -> Result<ChunkGroupContent> {
|
||||
type ModuleToChunkableMap =
|
||||
FxHashMap<ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn ChunkableModule>>>;
|
||||
|
||||
struct TraverseState {
|
||||
unsorted_chunkable_modules: ModuleToChunkableMap,
|
||||
result: ChunkGroupContent,
|
||||
}
|
||||
|
||||
let mut state = TraverseState {
|
||||
unsorted_chunkable_modules: FxHashMap::default(),
|
||||
result: ChunkGroupContent {
|
||||
chunkable_modules: FxIndexSet::default(),
|
||||
async_modules: FxIndexSet::default(),
|
||||
traced_modules: FxIndexSet::default(),
|
||||
passthrough_modules: FxIndexSet::default(),
|
||||
forward_edges_inherit_async: FxIndexMap::default(),
|
||||
local_back_edges_inherit_async: FxIndexMap::default(),
|
||||
available_async_modules_back_edges_inherit_async: FxIndexMap::default(),
|
||||
},
|
||||
};
|
||||
|
||||
let available_modules = match availability_info.available_modules() {
|
||||
Some(available_modules) => Some(available_modules.snapshot().await?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
module_graph
|
||||
.traverse_edges_from_entries_topological(
|
||||
chunk_group_entries,
|
||||
&mut state,
|
||||
|parent_info,
|
||||
node,
|
||||
TraverseState {
|
||||
unsorted_chunkable_modules,
|
||||
result,
|
||||
}| {
|
||||
if let Some((_, ChunkingType::Traced)) = parent_info {
|
||||
if should_trace {
|
||||
result.traced_modules.insert(node.module);
|
||||
}
|
||||
return Ok(GraphTraversalAction::Skip);
|
||||
}
|
||||
|
||||
let Some(chunkable_module) =
|
||||
ResolvedVc::try_sidecast_sync::<Box<dyn ChunkableModule>>(node.module)
|
||||
else {
|
||||
return Ok(GraphTraversalAction::Skip);
|
||||
};
|
||||
|
||||
let available_info = available_modules
|
||||
.as_ref()
|
||||
.and_then(|available_modules| available_modules.get(chunkable_module));
|
||||
|
||||
let Some((parent_node, edge)) = parent_info else {
|
||||
return Ok(if available_info.is_some() {
|
||||
GraphTraversalAction::Skip
|
||||
} else {
|
||||
unsorted_chunkable_modules.insert(node.module, chunkable_module);
|
||||
GraphTraversalAction::Continue
|
||||
});
|
||||
};
|
||||
|
||||
let parent_module =
|
||||
ResolvedVc::try_sidecast_sync::<Box<dyn ChunkableModule>>(parent_node.module)
|
||||
.context("Expected parent module to be chunkable")?;
|
||||
|
||||
Ok(match edge {
|
||||
ChunkingType::Passthrough => {
|
||||
result.passthrough_modules.insert(chunkable_module);
|
||||
GraphTraversalAction::Continue
|
||||
}
|
||||
ChunkingType::Parallel => {
|
||||
if available_info.is_some() {
|
||||
GraphTraversalAction::Skip
|
||||
} else {
|
||||
unsorted_chunkable_modules.insert(node.module, chunkable_module);
|
||||
GraphTraversalAction::Continue
|
||||
}
|
||||
}
|
||||
ChunkingType::ParallelInheritAsync => {
|
||||
result
|
||||
.forward_edges_inherit_async
|
||||
.entry(parent_module)
|
||||
.or_default()
|
||||
.push(chunkable_module);
|
||||
if let Some(info) = available_info {
|
||||
if info.is_async {
|
||||
result
|
||||
.available_async_modules_back_edges_inherit_async
|
||||
.entry(chunkable_module)
|
||||
.or_default()
|
||||
.push(parent_module);
|
||||
}
|
||||
GraphTraversalAction::Skip
|
||||
} else {
|
||||
result
|
||||
.local_back_edges_inherit_async
|
||||
.entry(chunkable_module)
|
||||
.or_default()
|
||||
.push(parent_module);
|
||||
unsorted_chunkable_modules.insert(node.module, chunkable_module);
|
||||
GraphTraversalAction::Continue
|
||||
}
|
||||
}
|
||||
ChunkingType::Async => {
|
||||
if can_split_async {
|
||||
result.async_modules.insert(chunkable_module);
|
||||
GraphTraversalAction::Skip
|
||||
} else if available_info.is_some() {
|
||||
GraphTraversalAction::Skip
|
||||
} else {
|
||||
unsorted_chunkable_modules.insert(node.module, chunkable_module);
|
||||
GraphTraversalAction::Continue
|
||||
}
|
||||
}
|
||||
ChunkingType::Traced => {
|
||||
// handled above before the sidecast
|
||||
unreachable!();
|
||||
}
|
||||
ChunkingType::Isolated { .. } => {
|
||||
// TODO currently not implemented
|
||||
GraphTraversalAction::Skip
|
||||
}
|
||||
})
|
||||
},
|
||||
|_,
|
||||
node,
|
||||
TraverseState {
|
||||
unsorted_chunkable_modules,
|
||||
result,
|
||||
}| {
|
||||
// Insert modules in topological order
|
||||
if let Some(chunkable_module) =
|
||||
unsorted_chunkable_modules.get(&node.module).copied()
|
||||
{
|
||||
result.chunkable_modules.insert(chunkable_module);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(state.result)
|
||||
}
|
||||
|
||||
@@ -9,29 +9,19 @@ pub(crate) mod evaluate;
|
||||
pub mod module_id_strategies;
|
||||
pub mod optimize;
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
fmt::{Debug, Display},
|
||||
future::Future,
|
||||
hash::Hash,
|
||||
};
|
||||
use std::fmt::Display;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use anyhow::Result;
|
||||
use auto_hash_map::AutoSet;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info_span, Span};
|
||||
use turbo_rcstr::RcStr;
|
||||
use turbo_tasks::{
|
||||
debug::ValueDebugFormat,
|
||||
graph::{AdjacencyMap, GraphTraversal, GraphTraversalResult, Visit, VisitControlFlow},
|
||||
trace::TraceRawVcs,
|
||||
FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryFlatJoinIterExt,
|
||||
TryJoinIterExt, Upcast, ValueToString, Vc,
|
||||
debug::ValueDebugFormat, trace::TraceRawVcs, FxIndexMap, FxIndexSet, NonLocalValue, ResolvedVc,
|
||||
TaskInput, Upcast, ValueToString, Vc,
|
||||
};
|
||||
use turbo_tasks_fs::FileSystemPath;
|
||||
use turbo_tasks_hash::DeterministicHash;
|
||||
|
||||
use self::{availability_info::AvailabilityInfo, available_modules::AvailableModules};
|
||||
pub use self::{
|
||||
chunking_context::{
|
||||
ChunkGroupResult, ChunkGroupType, ChunkingContext, ChunkingContextExt,
|
||||
@@ -219,7 +209,8 @@ pub trait ChunkableModuleReference: ModuleReference + ValueToString {
|
||||
type AsyncInfo =
|
||||
FxIndexMap<ResolvedVc<Box<dyn ChunkableModule>>, Vec<ResolvedVc<Box<dyn ChunkableModule>>>>;
|
||||
|
||||
pub struct ChunkContentResult {
|
||||
#[derive(Default)]
|
||||
pub struct ChunkGroupContent {
|
||||
pub chunkable_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
|
||||
pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
|
||||
pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
|
||||
@@ -235,469 +226,6 @@ pub struct ChunkContentResult {
|
||||
pub available_async_modules_back_edges_inherit_async: AsyncInfo,
|
||||
}
|
||||
|
||||
pub async fn chunk_content(
|
||||
chunk_entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
|
||||
availability_info: AvailabilityInfo,
|
||||
can_split_async: bool,
|
||||
should_trace: bool,
|
||||
) -> Result<ChunkContentResult> {
|
||||
chunk_content_internal_parallel(
|
||||
chunk_entries,
|
||||
availability_info,
|
||||
can_split_async,
|
||||
should_trace,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs, Debug, NonLocalValue,
|
||||
)]
|
||||
enum InheritAsyncEdge {
|
||||
/// The chunk item is in the current chunk group and async module info need
|
||||
/// to be computed for it
|
||||
LocalModule,
|
||||
/// The chunk item is already available in the parent chunk group and is an
|
||||
/// async module. Chunk items that are available but not async modules are
|
||||
/// not included in back edges at all since they don't influence the parent
|
||||
/// module in terms of being an async module.
|
||||
AvailableAsyncModule,
|
||||
}
|
||||
|
||||
#[derive(Eq, PartialEq, Clone, Hash, Serialize, Deserialize, TraceRawVcs, Debug, NonLocalValue)]
|
||||
enum ChunkContentGraphNode {
|
||||
// A module not placed in the current chunk, but whose references we will
|
||||
// follow to find more graph nodes.
|
||||
PassthroughModule {
|
||||
module: ResolvedVc<Box<dyn ChunkableModule>>,
|
||||
},
|
||||
// Modules that are placed into the current chunk group
|
||||
Module {
|
||||
module: ResolvedVc<Box<dyn ChunkableModule>>,
|
||||
ident: ReadRef<RcStr>,
|
||||
},
|
||||
// Async module that is referenced from the chunk group
|
||||
AsyncModule {
|
||||
module: ResolvedVc<Box<dyn ChunkableModule>>,
|
||||
},
|
||||
// Module that is referenced as traced and will be turned into a separate RebasedAsset
|
||||
TracedModule {
|
||||
module: ResolvedVc<Box<dyn Module>>,
|
||||
},
|
||||
/// A list of directly referenced modules from which `is_async_module`
|
||||
/// will be inherited.
|
||||
InheritAsyncInfo {
|
||||
module: ResolvedVc<Box<dyn ChunkableModule>>,
|
||||
references: Vec<(ResolvedVc<Box<dyn ChunkableModule>>, InheritAsyncEdge)>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, TaskInput, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
enum ChunkGraphNodeToReferences {
|
||||
PassthroughModule(ResolvedVc<Box<dyn ChunkableModule>>),
|
||||
Module(ResolvedVc<Box<dyn ChunkableModule>>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, TraceRawVcs, NonLocalValue)]
|
||||
struct ChunkGraphEdge {
|
||||
key: Option<ResolvedVc<Box<dyn Module>>>,
|
||||
node: ChunkContentGraphNode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[turbo_tasks::value(transparent)]
|
||||
struct ChunkGraphEdges(Vec<ChunkGraphEdge>);
|
||||
|
||||
#[turbo_tasks::function]
|
||||
async fn graph_node_to_referenced_nodes_with_available_modules(
|
||||
node: ChunkGraphNodeToReferences,
|
||||
available_modules: Vc<AvailableModules>,
|
||||
can_split_async: bool,
|
||||
should_trace: bool,
|
||||
) -> Result<Vc<ChunkGraphEdges>> {
|
||||
let edges = graph_node_to_referenced_nodes(node, can_split_async, should_trace);
|
||||
let edges_ref = edges.await?;
|
||||
for (unchanged, edge) in edges_ref.iter().enumerate() {
|
||||
if let ChunkContentGraphNode::Module { module, .. } = edge.node {
|
||||
if let Some(info) = *available_modules.get(*module).await? {
|
||||
let mut new_edges = Vec::with_capacity(edges_ref.len());
|
||||
new_edges.extend(edges_ref[0..unchanged].iter().cloned());
|
||||
let mut available_module_info = HashMap::new();
|
||||
available_module_info.insert(module, info);
|
||||
for edge in edges_ref[unchanged + 1..].iter() {
|
||||
match edge.node {
|
||||
ChunkContentGraphNode::Module { module, .. } => {
|
||||
if let Some(info) = *available_modules.get(*module).await? {
|
||||
available_module_info.insert(module, info);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ChunkContentGraphNode::InheritAsyncInfo {
|
||||
module,
|
||||
ref references,
|
||||
} => {
|
||||
let new_references = references
|
||||
.iter()
|
||||
.filter_map(|&(r, _)| {
|
||||
if let Some(info) = available_module_info.get(&r) {
|
||||
if info.is_async {
|
||||
Some((r, InheritAsyncEdge::AvailableAsyncModule))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
Some((r, InheritAsyncEdge::LocalModule))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
new_edges.push(ChunkGraphEdge {
|
||||
key: edge.key,
|
||||
node: ChunkContentGraphNode::InheritAsyncInfo {
|
||||
module,
|
||||
references: new_references,
|
||||
},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
new_edges.push(edge.clone())
|
||||
}
|
||||
return Ok(Vc::cell(new_edges));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(edges)
|
||||
}
|
||||
|
||||
#[turbo_tasks::function]
|
||||
async fn graph_node_to_referenced_nodes(
|
||||
node: ChunkGraphNodeToReferences,
|
||||
can_split_async: bool,
|
||||
should_trace: bool,
|
||||
) -> Result<Vc<ChunkGraphEdges>> {
|
||||
let (parent, module_references) = match &node {
|
||||
ChunkGraphNodeToReferences::PassthroughModule(item) => (None, item.references()),
|
||||
ChunkGraphNodeToReferences::Module(item) => (Some(*item), item.references()),
|
||||
};
|
||||
|
||||
let module_references = module_references.await?;
|
||||
let graph_nodes = module_references
|
||||
.iter()
|
||||
.map(|reference| async {
|
||||
let reference = *reference;
|
||||
let Some(chunkable_module_reference) =
|
||||
ResolvedVc::try_downcast::<Box<dyn ChunkableModuleReference>>(reference).await?
|
||||
else {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
|
||||
let Some(chunking_type) = &*chunkable_module_reference.chunking_type().await? else {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
|
||||
let module_data = reference
|
||||
.resolve_reference()
|
||||
.resolve()
|
||||
.await?
|
||||
.primary_modules()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|&module| async move {
|
||||
if matches!(chunking_type, ChunkingType::Traced) {
|
||||
if should_trace {
|
||||
return Ok((
|
||||
Some(ChunkGraphEdge {
|
||||
key: None,
|
||||
node: ChunkContentGraphNode::TracedModule { module },
|
||||
}),
|
||||
None,
|
||||
));
|
||||
} else {
|
||||
return Ok((None, None));
|
||||
}
|
||||
}
|
||||
|
||||
let Some(chunkable_module) =
|
||||
ResolvedVc::try_sidecast::<Box<dyn ChunkableModule>>(module).await?
|
||||
else {
|
||||
return Ok((None, None));
|
||||
};
|
||||
|
||||
match chunking_type {
|
||||
ChunkingType::Parallel => Ok((
|
||||
Some(ChunkGraphEdge {
|
||||
key: Some(module),
|
||||
node: ChunkContentGraphNode::Module {
|
||||
module: chunkable_module,
|
||||
ident: module.ident().to_string().await?,
|
||||
},
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
ChunkingType::ParallelInheritAsync => Ok((
|
||||
Some(ChunkGraphEdge {
|
||||
key: Some(module),
|
||||
node: ChunkContentGraphNode::Module {
|
||||
module: chunkable_module,
|
||||
ident: module.ident().to_string().await?,
|
||||
},
|
||||
}),
|
||||
Some((chunkable_module, InheritAsyncEdge::LocalModule)),
|
||||
)),
|
||||
ChunkingType::Passthrough => Ok((
|
||||
Some(ChunkGraphEdge {
|
||||
key: None,
|
||||
node: ChunkContentGraphNode::PassthroughModule {
|
||||
module: chunkable_module,
|
||||
},
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
ChunkingType::Async => {
|
||||
if can_split_async {
|
||||
Ok((
|
||||
Some(ChunkGraphEdge {
|
||||
key: None,
|
||||
node: ChunkContentGraphNode::AsyncModule {
|
||||
module: chunkable_module,
|
||||
},
|
||||
}),
|
||||
None,
|
||||
))
|
||||
} else {
|
||||
Ok((
|
||||
Some(ChunkGraphEdge {
|
||||
key: Some(module),
|
||||
node: ChunkContentGraphNode::Module {
|
||||
module: chunkable_module,
|
||||
ident: module.ident().to_string().await?,
|
||||
},
|
||||
}),
|
||||
None,
|
||||
))
|
||||
}
|
||||
}
|
||||
ChunkingType::Isolated { .. } => {
|
||||
// TODO implement
|
||||
Ok((None, None))
|
||||
}
|
||||
ChunkingType::Traced => {
|
||||
bail!("unreachable ChunkingType::Traced");
|
||||
}
|
||||
}
|
||||
})
|
||||
.try_join()
|
||||
.await?;
|
||||
|
||||
let mut graph_nodes = vec![];
|
||||
let mut inherit_async_references = vec![];
|
||||
for (n, iar) in module_data {
|
||||
if let Some(n) = n {
|
||||
graph_nodes.push(n);
|
||||
}
|
||||
if let Some(iar) = iar {
|
||||
inherit_async_references.push(iar);
|
||||
}
|
||||
}
|
||||
|
||||
if !inherit_async_references.is_empty() {
|
||||
if let Some(parent) = parent {
|
||||
graph_nodes.push(ChunkGraphEdge {
|
||||
key: None,
|
||||
node: ChunkContentGraphNode::InheritAsyncInfo {
|
||||
module: parent,
|
||||
references: inherit_async_references,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Ok(graph_nodes)
|
||||
})
|
||||
.try_flat_join()
|
||||
.await?;
|
||||
|
||||
Ok(Vc::cell(graph_nodes))
|
||||
}
|
||||
|
||||
struct ChunkContentVisit {
|
||||
available_chunk_items: Option<ResolvedVc<AvailableModules>>,
|
||||
processed_modules: HashSet<ResolvedVc<Box<dyn Module>>>,
|
||||
should_trace: bool,
|
||||
can_split_async: bool,
|
||||
}
|
||||
|
||||
type ChunkItemToGraphNodesEdges = impl Iterator<Item = ChunkGraphEdge>;
|
||||
|
||||
type ChunkItemToGraphNodesFuture = impl Future<Output = Result<ChunkItemToGraphNodesEdges>>;
|
||||
|
||||
impl Visit<ChunkContentGraphNode, ()> for ChunkContentVisit {
|
||||
type Edge = ChunkGraphEdge;
|
||||
type EdgesIntoIter = ChunkItemToGraphNodesEdges;
|
||||
type EdgesFuture = ChunkItemToGraphNodesFuture;
|
||||
|
||||
fn visit(&mut self, edge: ChunkGraphEdge) -> VisitControlFlow<ChunkContentGraphNode, ()> {
|
||||
let ChunkGraphEdge { key, node } = edge;
|
||||
let Some(module) = key else {
|
||||
if matches!(node, ChunkContentGraphNode::PassthroughModule { .. }) {
|
||||
return VisitControlFlow::Continue(node);
|
||||
} else {
|
||||
// All other types don't have edges
|
||||
return VisitControlFlow::Skip(node);
|
||||
}
|
||||
};
|
||||
|
||||
if !self.processed_modules.insert(module) {
|
||||
return VisitControlFlow::Skip(node);
|
||||
}
|
||||
|
||||
VisitControlFlow::Continue(node)
|
||||
}
|
||||
|
||||
fn edges(&mut self, node: &ChunkContentGraphNode) -> Self::EdgesFuture {
|
||||
let node = node.clone();
|
||||
|
||||
let available_chunk_items = self.available_chunk_items;
|
||||
let can_split_async = self.can_split_async;
|
||||
let should_trace = self.should_trace;
|
||||
|
||||
async move {
|
||||
let node = match node {
|
||||
ChunkContentGraphNode::PassthroughModule { module: item } => {
|
||||
ChunkGraphNodeToReferences::PassthroughModule(item)
|
||||
}
|
||||
ChunkContentGraphNode::Module { module: item, .. } => {
|
||||
ChunkGraphNodeToReferences::Module(item)
|
||||
}
|
||||
_ => {
|
||||
return Ok(None.into_iter().flatten());
|
||||
}
|
||||
};
|
||||
|
||||
let nodes = if let Some(available_chunk_items) = available_chunk_items {
|
||||
graph_node_to_referenced_nodes_with_available_modules(
|
||||
node,
|
||||
*available_chunk_items,
|
||||
can_split_async,
|
||||
should_trace,
|
||||
)
|
||||
} else {
|
||||
graph_node_to_referenced_nodes(node, can_split_async, should_trace)
|
||||
}
|
||||
.await?;
|
||||
Ok(Some(nodes.into_iter().cloned()).into_iter().flatten())
|
||||
}
|
||||
}
|
||||
|
||||
fn span(&mut self, node: &ChunkContentGraphNode) -> Span {
|
||||
if let ChunkContentGraphNode::Module { ident, .. } = node {
|
||||
info_span!("chunking module", name = display(ident))
|
||||
} else {
|
||||
Span::current()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn chunk_content_internal_parallel(
|
||||
chunk_entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
|
||||
availability_info: AvailabilityInfo,
|
||||
can_split_async: bool,
|
||||
should_trace: bool,
|
||||
) -> Result<ChunkContentResult> {
|
||||
let root_edges = chunk_entries
|
||||
.into_iter()
|
||||
.map(|entry| async move {
|
||||
let Some(chunkable_module) =
|
||||
ResolvedVc::try_downcast::<Box<dyn ChunkableModule>>(entry).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(ChunkGraphEdge {
|
||||
key: Some(entry),
|
||||
node: ChunkContentGraphNode::Module {
|
||||
module: chunkable_module,
|
||||
ident: chunkable_module.ident().to_string().await?,
|
||||
},
|
||||
}))
|
||||
})
|
||||
.try_flat_join()
|
||||
.await?;
|
||||
|
||||
let visit = ChunkContentVisit {
|
||||
available_chunk_items: availability_info.available_modules(),
|
||||
processed_modules: Default::default(),
|
||||
can_split_async,
|
||||
should_trace,
|
||||
};
|
||||
|
||||
let GraphTraversalResult::Completed(traversal_result) =
|
||||
AdjacencyMap::new().visit(root_edges, visit).await
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
|
||||
let graph_nodes: Vec<_> = traversal_result?.into_reverse_topological().collect();
|
||||
|
||||
let mut chunkable_modules = FxIndexSet::default();
|
||||
let mut async_modules = FxIndexSet::default();
|
||||
let mut passthrough_modules = FxIndexSet::default();
|
||||
let mut forward_edges_inherit_async = FxIndexMap::default();
|
||||
let mut local_back_edges_inherit_async = FxIndexMap::default();
|
||||
let mut available_async_modules_back_edges_inherit_async = FxIndexMap::default();
|
||||
let mut traced_modules = FxIndexSet::default();
|
||||
|
||||
for graph_node in graph_nodes {
|
||||
match graph_node {
|
||||
ChunkContentGraphNode::PassthroughModule { module } => {
|
||||
passthrough_modules.insert(module);
|
||||
}
|
||||
ChunkContentGraphNode::TracedModule { module } => {
|
||||
traced_modules.insert(module);
|
||||
}
|
||||
ChunkContentGraphNode::Module { module: item, .. } => {
|
||||
chunkable_modules.insert(item);
|
||||
}
|
||||
ChunkContentGraphNode::AsyncModule { module } => {
|
||||
async_modules.insert(module);
|
||||
}
|
||||
ChunkContentGraphNode::InheritAsyncInfo {
|
||||
module: item,
|
||||
references,
|
||||
} => {
|
||||
for &(reference, ty) in &references {
|
||||
match ty {
|
||||
InheritAsyncEdge::LocalModule => local_back_edges_inherit_async
|
||||
.entry(reference)
|
||||
.or_insert_with(Vec::new)
|
||||
.push(item),
|
||||
InheritAsyncEdge::AvailableAsyncModule => {
|
||||
available_async_modules_back_edges_inherit_async
|
||||
.entry(reference)
|
||||
.or_insert_with(Vec::new)
|
||||
.push(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
forward_edges_inherit_async
|
||||
.entry(item)
|
||||
.or_insert_with(Vec::new)
|
||||
.extend(references.into_iter().map(|(r, _)| r));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ChunkContentResult {
|
||||
chunkable_modules,
|
||||
async_modules,
|
||||
traced_modules,
|
||||
passthrough_modules,
|
||||
forward_edges_inherit_async,
|
||||
local_back_edges_inherit_async,
|
||||
available_async_modules_back_edges_inherit_async,
|
||||
})
|
||||
}
|
||||
|
||||
#[turbo_tasks::value_trait]
|
||||
pub trait ChunkItem {
|
||||
/// The [AssetIdent] of the [Module] that this [ChunkItem] was created from.
|
||||
|
||||
@@ -180,13 +180,13 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
|
||||
// ----
|
||||
|
||||
let mut visitor =
|
||||
async |parent_info: Option<(&'_ SingleModuleGraphModuleNode, &'_ ChunkingType)>,
|
||||
node: &'_ SingleModuleGraphModuleNode,
|
||||
module_chunk_groups: &mut HashMap<
|
||||
|parent_info: Option<(&'_ SingleModuleGraphModuleNode, &'_ ChunkingType)>,
|
||||
node: &'_ SingleModuleGraphModuleNode,
|
||||
module_chunk_groups: &mut HashMap<
|
||||
ResolvedVc<Box<dyn Module>>,
|
||||
RoaringBitmapWrapper,
|
||||
>|
|
||||
-> Result<GraphTraversalAction> {
|
||||
-> GraphTraversalAction {
|
||||
enum ChunkGroupInheritance<It: Iterator<Item = ChunkGroup>> {
|
||||
Inherit(ResolvedVc<Box<dyn Module>>),
|
||||
ChunkGroup(It),
|
||||
@@ -218,7 +218,10 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
|
||||
},
|
||||
),
|
||||
)),
|
||||
ChunkingType::Traced => unreachable!(),
|
||||
ChunkingType::Traced => {
|
||||
// Traced modules are not placed in chunk groups
|
||||
return GraphTraversalAction::Skip;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ChunkGroupInheritance::ChunkGroup(Either::Left(std::iter::once(
|
||||
@@ -226,7 +229,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
|
||||
)))
|
||||
};
|
||||
|
||||
Ok(match chunk_groups {
|
||||
match chunk_groups {
|
||||
ChunkGroupInheritance::ChunkGroup(chunk_groups) => {
|
||||
// Start of a new chunk group, don't inherit anything from parent
|
||||
let chunk_group_ids = chunk_groups.map(|chunk_group| {
|
||||
@@ -287,28 +290,27 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
let mut visit_count = 0usize;
|
||||
|
||||
{
|
||||
let mut queue_set = HashSet::new();
|
||||
let mut queue = entries
|
||||
.iter()
|
||||
.map(|e| NodeWithPriority {
|
||||
let mut queue = BinaryHeap::with_capacity(entries.len());
|
||||
for e in entries {
|
||||
queue.push(NodeWithPriority {
|
||||
depth: *module_depth.get(e).unwrap(),
|
||||
chunk_group_len: 0,
|
||||
node: ModuleGraph::get_entry(&graphs, e).unwrap(),
|
||||
})
|
||||
.collect::<BinaryHeap<_>>();
|
||||
node: ModuleGraph::get_entry(&graphs, *e).await?,
|
||||
});
|
||||
}
|
||||
for entry_node in &queue {
|
||||
visitor(
|
||||
None,
|
||||
get_node!(graphs, entry_node.node),
|
||||
&mut module_chunk_groups,
|
||||
)
|
||||
.await?;
|
||||
);
|
||||
}
|
||||
while let Some(NodeWithPriority { node, .. }) = queue.pop() {
|
||||
queue_set.remove(&node);
|
||||
@@ -329,8 +331,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
|
||||
Some((node_weight, edge_weight)),
|
||||
succ_weight,
|
||||
&mut module_chunk_groups,
|
||||
)
|
||||
.await?;
|
||||
);
|
||||
|
||||
if action == GraphTraversalAction::Continue && queue_set.insert(succ) {
|
||||
queue.push(NodeWithPriority {
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::{
|
||||
use anyhow::{bail, Context, Result};
|
||||
use petgraph::{
|
||||
graph::{DiGraph, EdgeIndex, NodeIndex},
|
||||
visit::{Dfs, EdgeRef, VisitMap, Visitable},
|
||||
visit::{Dfs, EdgeRef, IntoNodeReferences, VisitMap, Visitable},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::Instrument;
|
||||
@@ -77,7 +77,16 @@ impl VisitedModules {
|
||||
SingleModuleGraphNode::VisitedModule { .. } => None,
|
||||
})
|
||||
.collect(),
|
||||
next_graph_idx: 0,
|
||||
next_graph_idx: 1,
|
||||
}
|
||||
.cell())
|
||||
}
|
||||
|
||||
#[turbo_tasks::function]
|
||||
pub async fn with_incremented_index(&self) -> Result<Vc<Self>> {
|
||||
Ok(Self {
|
||||
modules: self.modules.clone(),
|
||||
next_graph_idx: self.next_graph_idx + 1,
|
||||
}
|
||||
.cell())
|
||||
}
|
||||
@@ -131,8 +140,8 @@ pub struct SingleModuleGraph {
|
||||
|
||||
/// The number of modules in the graph (excluding VisitedModule nodes)
|
||||
pub number_of_modules: usize,
|
||||
// NodeIndex isn't necessarily stable, but these are first nodes in the graph, so shouldn't
|
||||
// ever be involved in a swap_remove operation
|
||||
|
||||
// NodeIndex isn't necessarily stable (because of swap_remove), but we never remove nodes.
|
||||
//
|
||||
// HashMaps have nondeterministic order, but this map is only used for lookups (in
|
||||
// `get_module`) and not iteration.
|
||||
@@ -140,6 +149,7 @@ pub struct SingleModuleGraph {
|
||||
// This contains Vcs, but they are already contained in the graph, so no need to trace this.
|
||||
#[turbo_tasks(trace_ignore)]
|
||||
modules: HashMap<ResolvedVc<Box<dyn Module>>, NodeIndex>,
|
||||
|
||||
#[turbo_tasks(trace_ignore)]
|
||||
pub entries: Vec<ResolvedVc<Box<dyn Module>>>,
|
||||
}
|
||||
@@ -262,7 +272,7 @@ impl SingleModuleGraph {
|
||||
self.modules
|
||||
.get(&module)
|
||||
.copied()
|
||||
.context("Couldn't find entry module in graph")
|
||||
.context("Couldn't find module in graph")
|
||||
}
|
||||
|
||||
/// Iterate over all nodes in the graph
|
||||
@@ -277,9 +287,7 @@ impl SingleModuleGraph {
|
||||
pub fn enumerate_nodes(
|
||||
&self,
|
||||
) -> impl Iterator<Item = (NodeIndex, &'_ SingleModuleGraphNode)> + '_ {
|
||||
self.graph
|
||||
.node_indices()
|
||||
.map(move |idx| (idx, self.graph.node_weight(idx).unwrap()))
|
||||
self.graph.node_references()
|
||||
}
|
||||
|
||||
/// Traverses all reachable nodes (once)
|
||||
@@ -446,7 +454,7 @@ impl SingleModuleGraph {
|
||||
Option<(&'a SingleModuleGraphModuleNode, &'a ChunkingType)>,
|
||||
&'a SingleModuleGraphModuleNode,
|
||||
&mut S,
|
||||
) -> GraphTraversalAction,
|
||||
) -> Result<GraphTraversalAction>,
|
||||
mut visit_postorder: impl FnMut(
|
||||
Option<(&'a SingleModuleGraphModuleNode, &'a ChunkingType)>,
|
||||
&'a SingleModuleGraphModuleNode,
|
||||
@@ -496,7 +504,7 @@ impl SingleModuleGraph {
|
||||
{
|
||||
let action = visit_preorder(parent_arg, current_node, state);
|
||||
stack.push((ReverseTopologicalPass::Visit, parent, current));
|
||||
if expanded.insert(current) && action == GraphTraversalAction::Continue {
|
||||
if action? == GraphTraversalAction::Continue && expanded.insert(current) {
|
||||
stack.extend(iter_neighbors(graph, current).map(|(edge, child)| {
|
||||
(
|
||||
ReverseTopologicalPass::ExpandAndVisit,
|
||||
@@ -582,20 +590,28 @@ impl ModuleGraph {
|
||||
self.graphs.iter().try_join().await
|
||||
}
|
||||
|
||||
fn get_entry(
|
||||
async fn get_entry(
|
||||
graphs: &[ReadRef<SingleModuleGraph>],
|
||||
entry: &ResolvedVc<Box<dyn Module>>,
|
||||
entry: ResolvedVc<Box<dyn Module>>,
|
||||
) -> Result<GraphNodeIndex> {
|
||||
graphs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(graph_idx, graph)| {
|
||||
graph.modules.get(entry).map(|node_idx| GraphNodeIndex {
|
||||
graph_idx,
|
||||
node_idx: *node_idx,
|
||||
})
|
||||
let Some(idx) = graphs.iter().enumerate().find_map(|(graph_idx, graph)| {
|
||||
graph.modules.get(&entry).map(|node_idx| GraphNodeIndex {
|
||||
graph_idx,
|
||||
node_idx: *node_idx,
|
||||
})
|
||||
.context("Couldn't find entry module in graph")
|
||||
}) else {
|
||||
bail!(
|
||||
"Couldn't find entry module {} in module graph (potential entries: {:?})",
|
||||
entry.ident().to_string().await?,
|
||||
graphs
|
||||
.iter()
|
||||
.flat_map(|g| g.entries.iter())
|
||||
.map(|e| e.ident().to_string())
|
||||
.try_join()
|
||||
.await?
|
||||
);
|
||||
};
|
||||
Ok(idx)
|
||||
}
|
||||
|
||||
/// Traverses all reachable edges exactly once and calls the visitor with the edge source and
|
||||
@@ -618,10 +634,11 @@ impl ModuleGraph {
|
||||
) -> Result<()> {
|
||||
let graphs = self.get_graphs().await?;
|
||||
|
||||
let mut queue = entries
|
||||
.into_iter()
|
||||
.map(|e| ModuleGraph::get_entry(&graphs, e).unwrap())
|
||||
.collect::<VecDeque<_>>();
|
||||
let entries = entries.into_iter();
|
||||
let mut queue = VecDeque::with_capacity(entries.size_hint().0);
|
||||
for e in entries {
|
||||
queue.push_back(ModuleGraph::get_entry(&graphs, *e).await?);
|
||||
}
|
||||
let mut visited = HashSet::new();
|
||||
for entry_node in &queue {
|
||||
visitor(None, get_node!(graphs, entry_node));
|
||||
@@ -660,9 +677,9 @@ impl ModuleGraph {
|
||||
/// - Receives (originating &SingleModuleGraphNode, edge &ChunkingType), target
|
||||
/// &SingleModuleGraphNode, state &S
|
||||
/// - Can return [GraphTraversalAction]s to control the traversal
|
||||
pub async fn traverse_edges_from_entry<'a>(
|
||||
pub async fn traverse_edges_from_entry(
|
||||
&self,
|
||||
entries: impl IntoIterator<Item = &'a ResolvedVc<Box<dyn Module>>>,
|
||||
entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
|
||||
mut visitor: impl FnMut(
|
||||
Option<(&'_ SingleModuleGraphModuleNode, &'_ ChunkingType)>,
|
||||
&'_ SingleModuleGraphModuleNode,
|
||||
@@ -670,10 +687,11 @@ impl ModuleGraph {
|
||||
) -> Result<()> {
|
||||
let graphs = self.get_graphs().await?;
|
||||
|
||||
let mut stack = entries
|
||||
.into_iter()
|
||||
.map(|e| ModuleGraph::get_entry(&graphs, e).unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
let entries = entries.into_iter();
|
||||
let mut stack = Vec::with_capacity(entries.size_hint().0);
|
||||
for entry in entries {
|
||||
stack.push(ModuleGraph::get_entry(&graphs, entry).await?);
|
||||
}
|
||||
let mut visited = HashSet::new();
|
||||
for entry_node in &stack {
|
||||
visitor(None, get_node!(graphs, entry_node));
|
||||
@@ -720,15 +738,15 @@ impl ModuleGraph {
|
||||
/// - Receives: (originating &SingleModuleGraphNode, edge &ChunkingType), target
|
||||
/// &SingleModuleGraphNode, state &S
|
||||
/// - Can return [GraphTraversalAction]s to control the traversal
|
||||
pub async fn traverse_edges_from_entries_topological<'a, S>(
|
||||
pub async fn traverse_edges_from_entries_topological<S>(
|
||||
&self,
|
||||
entries: impl IntoIterator<Item = &'a ResolvedVc<Box<dyn Module>>>,
|
||||
entries: impl IntoIterator<Item = ResolvedVc<Box<dyn Module>>>,
|
||||
state: &mut S,
|
||||
mut visit_preorder: impl FnMut(
|
||||
Option<(&'_ SingleModuleGraphModuleNode, &'_ ChunkingType)>,
|
||||
&'_ SingleModuleGraphModuleNode,
|
||||
&mut S,
|
||||
) -> GraphTraversalAction,
|
||||
) -> Result<GraphTraversalAction>,
|
||||
mut visit_postorder: impl FnMut(
|
||||
Option<(&'_ SingleModuleGraphModuleNode, &'_ ChunkingType)>,
|
||||
&'_ SingleModuleGraphModuleNode,
|
||||
@@ -737,6 +755,8 @@ impl ModuleGraph {
|
||||
) -> Result<()> {
|
||||
let graphs = self.get_graphs().await?;
|
||||
|
||||
let entries = entries.into_iter().collect::<Vec<_>>();
|
||||
|
||||
enum ReverseTopologicalPass {
|
||||
Visit,
|
||||
ExpandAndVisit,
|
||||
@@ -746,26 +766,20 @@ impl ModuleGraph {
|
||||
ReverseTopologicalPass,
|
||||
Option<(GraphNodeIndex, EdgeIndex)>,
|
||||
GraphNodeIndex,
|
||||
)> = entries
|
||||
.into_iter()
|
||||
.map(|e| {
|
||||
(
|
||||
ReverseTopologicalPass::ExpandAndVisit,
|
||||
None,
|
||||
ModuleGraph::get_entry(&graphs, e).unwrap(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
)> = Vec::with_capacity(entries.len());
|
||||
for entry in entries.into_iter().rev() {
|
||||
stack.push((
|
||||
ReverseTopologicalPass::ExpandAndVisit,
|
||||
None,
|
||||
ModuleGraph::get_entry(&graphs, entry).await?,
|
||||
));
|
||||
}
|
||||
let mut expanded = HashSet::new();
|
||||
while let Some((pass, parent, current)) = stack.pop() {
|
||||
let parent_arg = parent.map(|(parent_node, parent_edge)| {
|
||||
(
|
||||
get_node!(graphs, parent_node),
|
||||
graphs[parent_node.graph_idx]
|
||||
.graph
|
||||
.edge_weight(parent_edge)
|
||||
.unwrap(),
|
||||
)
|
||||
let edge_weight = graphs[parent_node.graph_idx].graph.edge_weight(parent_edge);
|
||||
|
||||
(get_node!(graphs, parent_node), edge_weight.unwrap())
|
||||
});
|
||||
let current_node = get_node!(graphs, current);
|
||||
match pass {
|
||||
@@ -775,17 +789,17 @@ impl ModuleGraph {
|
||||
ReverseTopologicalPass::ExpandAndVisit => {
|
||||
let action = visit_preorder(parent_arg, current_node, state);
|
||||
stack.push((ReverseTopologicalPass::Visit, parent, current));
|
||||
if expanded.insert(current) && action == GraphTraversalAction::Continue {
|
||||
if action? == GraphTraversalAction::Continue && expanded.insert(current) {
|
||||
let graph = &graphs[current.graph_idx].graph;
|
||||
let (neighbors, child_graph_idx) =
|
||||
let (neighbors, current) =
|
||||
match graph.node_weight(current.node_idx).unwrap() {
|
||||
SingleModuleGraphNode::Module(_) => {
|
||||
(iter_neighbors(graph, current.node_idx), current.graph_idx)
|
||||
(iter_neighbors(graph, current.node_idx), current)
|
||||
}
|
||||
SingleModuleGraphNode::VisitedModule { idx } => (
|
||||
// We switch graphs
|
||||
iter_neighbors(&graphs[idx.graph_idx].graph, idx.node_idx),
|
||||
idx.graph_idx,
|
||||
*idx,
|
||||
),
|
||||
};
|
||||
stack.extend(neighbors.map(|(edge, child)| {
|
||||
@@ -793,7 +807,7 @@ impl ModuleGraph {
|
||||
ReverseTopologicalPass::ExpandAndVisit,
|
||||
Some((current, edge)),
|
||||
GraphNodeIndex {
|
||||
graph_idx: child_graph_idx,
|
||||
graph_idx: current.graph_idx,
|
||||
node_idx: child,
|
||||
},
|
||||
)
|
||||
@@ -1016,19 +1030,14 @@ impl Visit<SingleModuleGraphBuilderNode> for SingleModuleGraphBuilder<'_> {
|
||||
// .await?;
|
||||
|
||||
refs.iter()
|
||||
.flat_map(|(ty, modules)| {
|
||||
if matches!(ty, ChunkingType::Traced) {
|
||||
None
|
||||
} else {
|
||||
Some(modules.iter().map(|m| (ty.clone(), *m)))
|
||||
}
|
||||
})
|
||||
.flatten()
|
||||
.flat_map(|(ty, modules)| modules.iter().map(|m| (ty.clone(), *m)))
|
||||
.map(async |(ty, target)| {
|
||||
let to = if let Some(idx) = visited_modules.get(&target) {
|
||||
SingleModuleGraphBuilderNode::new_visited_module(target, *idx)
|
||||
} else if ty == COMMON_CHUNKING_TYPE {
|
||||
SingleModuleGraphBuilderNode::new_module(target).await?
|
||||
let to = if ty == COMMON_CHUNKING_TYPE {
|
||||
if let Some(idx) = visited_modules.get(&target) {
|
||||
SingleModuleGraphBuilderNode::new_visited_module(target, *idx)
|
||||
} else {
|
||||
SingleModuleGraphBuilderNode::new_module(target).await?
|
||||
}
|
||||
} else {
|
||||
SingleModuleGraphBuilderNode::new_chunkable_ref(module, target, ty)
|
||||
.await?
|
||||
@@ -1040,7 +1049,14 @@ impl Visit<SingleModuleGraphBuilderNode> for SingleModuleGraphBuilder<'_> {
|
||||
}
|
||||
(None, Some(chunkable_ref_target)) => {
|
||||
vec![SingleModuleGraphBuilderEdge {
|
||||
to: SingleModuleGraphBuilderNode::new_module(chunkable_ref_target).await?,
|
||||
to: if let Some(idx) = visited_modules.get(&chunkable_ref_target) {
|
||||
SingleModuleGraphBuilderNode::new_visited_module(
|
||||
chunkable_ref_target,
|
||||
*idx,
|
||||
)
|
||||
} else {
|
||||
SingleModuleGraphBuilderNode::new_module(chunkable_ref_target).await?
|
||||
},
|
||||
}]
|
||||
}
|
||||
_ => unreachable!(),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::{borrow::Cow, ops::ControlFlow, thread::available_parallelism, time::Duration};
|
||||
use std::{borrow::Cow, iter, ops::ControlFlow, thread::available_parallelism, time::Duration};
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use async_stream::try_stream as generator;
|
||||
@@ -89,7 +89,6 @@ struct EmittedEvaluatePoolAssets {
|
||||
async fn emit_evaluate_pool_assets_operation(
|
||||
module_asset: ResolvedVc<Box<dyn Module>>,
|
||||
asset_context: ResolvedVc<Box<dyn AssetContext>>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
|
||||
runtime_entries: Option<ResolvedVc<EvaluatableAssets>>,
|
||||
) -> Result<Vc<EmittedEvaluatePoolAssets>> {
|
||||
@@ -151,16 +150,21 @@ async fn emit_evaluate_pool_assets_operation(
|
||||
entries.push(entry)
|
||||
}
|
||||
}
|
||||
|
||||
Vc::<EvaluatableAssets>::cell(entries)
|
||||
entries
|
||||
};
|
||||
|
||||
let module_graph = ModuleGraph::from_modules(Vc::cell(
|
||||
iter::once(entry_module.to_resolved().await?)
|
||||
.chain(runtime_entries.iter().copied().map(ResolvedVc::upcast))
|
||||
.collect(),
|
||||
));
|
||||
|
||||
let bootstrap = chunking_context.root_entry_chunk_group_asset(
|
||||
entrypoint,
|
||||
entry_module,
|
||||
*module_graph,
|
||||
module_graph,
|
||||
OutputAssets::empty(),
|
||||
runtime_entries,
|
||||
Vc::<EvaluatableAssets>::cell(runtime_entries),
|
||||
);
|
||||
|
||||
let output_root = chunking_context.output_root().to_resolved().await?;
|
||||
@@ -179,14 +183,12 @@ async fn emit_evaluate_pool_assets_operation(
|
||||
async fn emit_evaluate_pool_assets_with_effects(
|
||||
module_asset: ResolvedVc<Box<dyn Module>>,
|
||||
asset_context: ResolvedVc<Box<dyn AssetContext>>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
|
||||
runtime_entries: Option<ResolvedVc<EvaluatableAssets>>,
|
||||
) -> Result<Vc<EmittedEvaluatePoolAssets>> {
|
||||
let operation = emit_evaluate_pool_assets_operation(
|
||||
module_asset,
|
||||
asset_context,
|
||||
module_graph,
|
||||
chunking_context,
|
||||
runtime_entries,
|
||||
);
|
||||
@@ -209,7 +211,6 @@ pub async fn get_evaluate_pool(
|
||||
cwd: Vc<FileSystemPath>,
|
||||
env: Vc<Box<dyn ProcessEnv>>,
|
||||
asset_context: Vc<Box<dyn AssetContext>>,
|
||||
module_graph: Vc<ModuleGraph>,
|
||||
chunking_context: Vc<Box<dyn ChunkingContext>>,
|
||||
runtime_entries: Option<Vc<EvaluatableAssets>>,
|
||||
additional_invalidation: Vc<Completion>,
|
||||
@@ -223,7 +224,6 @@ pub async fn get_evaluate_pool(
|
||||
} = *emit_evaluate_pool_assets_with_effects(
|
||||
module_asset,
|
||||
asset_context,
|
||||
module_graph,
|
||||
chunking_context,
|
||||
runtime_entries,
|
||||
)
|
||||
@@ -388,7 +388,6 @@ pub fn evaluate(
|
||||
env: ResolvedVc<Box<dyn ProcessEnv>>,
|
||||
context_ident_for_issue: ResolvedVc<AssetIdent>,
|
||||
asset_context: ResolvedVc<Box<dyn AssetContext>>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
|
||||
runtime_entries: Option<ResolvedVc<EvaluatableAssets>>,
|
||||
args: Vec<ResolvedVc<JsonValue>>,
|
||||
@@ -401,7 +400,6 @@ pub fn evaluate(
|
||||
env,
|
||||
context_ident_for_issue,
|
||||
asset_context,
|
||||
module_graph,
|
||||
chunking_context,
|
||||
runtime_entries,
|
||||
args,
|
||||
@@ -571,7 +569,6 @@ struct BasicEvaluateContext {
|
||||
env: ResolvedVc<Box<dyn ProcessEnv>>,
|
||||
context_ident_for_issue: ResolvedVc<AssetIdent>,
|
||||
asset_context: ResolvedVc<Box<dyn AssetContext>>,
|
||||
module_graph: ResolvedVc<ModuleGraph>,
|
||||
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
|
||||
runtime_entries: Option<ResolvedVc<EvaluatableAssets>>,
|
||||
args: Vec<ResolvedVc<JsonValue>>,
|
||||
@@ -596,7 +593,6 @@ impl EvaluateContext for BasicEvaluateContext {
|
||||
*self.cwd,
|
||||
*self.env,
|
||||
*self.asset_context,
|
||||
*self.module_graph,
|
||||
*self.chunking_context,
|
||||
self.runtime_entries.map(|r| *r),
|
||||
*self.additional_invalidation,
|
||||
|
||||
@@ -19,7 +19,6 @@ use turbopack_core::{
|
||||
issue::{
|
||||
Issue, IssueDescriptionExt, IssueSeverity, IssueStage, OptionStyledString, StyledString,
|
||||
},
|
||||
module_graph::ModuleGraph,
|
||||
reference_type::{EntryReferenceSubType, InnerAssets, ReferenceType},
|
||||
resolve::{find_context_file_or_package_key, options::ImportMapping, FindContextFileResult},
|
||||
source::Source,
|
||||
@@ -531,17 +530,12 @@ impl PostCssTransformedAsset {
|
||||
"".into()
|
||||
};
|
||||
|
||||
let module_graph = ModuleGraph::from_module(*postcss_executor)
|
||||
.to_resolved()
|
||||
.await?;
|
||||
|
||||
let config_value = evaluate_webpack_loader(WebpackLoaderContext {
|
||||
module_asset: postcss_executor,
|
||||
cwd: *project_path,
|
||||
env: *env,
|
||||
context_ident_for_issue: self.source.ident().to_resolved().await?,
|
||||
asset_context: evaluate_context,
|
||||
module_graph,
|
||||
chunking_context: *chunking_context,
|
||||
resolve_options_context: None,
|
||||
args: vec![
|
||||
|
||||
@@ -25,7 +25,6 @@ use turbopack_core::{
|
||||
ident::AssetIdent,
|
||||
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
|
||||
module::Module,
|
||||
module_graph::ModuleGraph,
|
||||
reference_type::{InnerAssets, ReferenceType},
|
||||
resolve::{
|
||||
options::{ConditionValue, ResolveInPackage, ResolveIntoPackage, ResolveOptions},
|
||||
@@ -224,10 +223,6 @@ impl WebpackLoadersProcessedAsset {
|
||||
.to_resolved()
|
||||
.await?;
|
||||
|
||||
let module_graph = ModuleGraph::from_module(*webpack_loaders_executor)
|
||||
.to_resolved()
|
||||
.await?;
|
||||
|
||||
let resource_fs_path = this.source.ident().path();
|
||||
let resource_fs_path_ref = resource_fs_path.await?;
|
||||
let Some(resource_path) = project_path
|
||||
@@ -247,7 +242,6 @@ impl WebpackLoadersProcessedAsset {
|
||||
env,
|
||||
context_ident_for_issue: this.source.ident().to_resolved().await?,
|
||||
asset_context: evaluate_context,
|
||||
module_graph,
|
||||
chunking_context,
|
||||
resolve_options_context: Some(transform.resolve_options_context),
|
||||
args: vec![
|
||||
@@ -404,7 +398,6 @@ pub struct WebpackLoaderContext {
|
||||
pub env: ResolvedVc<Box<dyn ProcessEnv>>,
|
||||
pub context_ident_for_issue: ResolvedVc<AssetIdent>,
|
||||
pub asset_context: ResolvedVc<Box<dyn AssetContext>>,
|
||||
pub module_graph: ResolvedVc<ModuleGraph>,
|
||||
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
|
||||
pub resolve_options_context: Option<ResolvedVc<ResolveOptionsContext>>,
|
||||
pub args: Vec<ResolvedVc<JsonValue>>,
|
||||
@@ -428,7 +421,6 @@ impl EvaluateContext for WebpackLoaderContext {
|
||||
*self.cwd,
|
||||
*self.env,
|
||||
*self.asset_context,
|
||||
*self.module_graph,
|
||||
*self.chunking_context,
|
||||
None,
|
||||
*self.additional_invalidation,
|
||||
|
||||
@@ -35,7 +35,6 @@ use turbopack_core::{
|
||||
environment::{Environment, ExecutionEnvironment, NodeJsEnvironment},
|
||||
file_source::FileSource,
|
||||
issue::{Issue, IssueDescriptionExt},
|
||||
module_graph::ModuleGraph,
|
||||
reference_type::{InnerAssets, ReferenceType},
|
||||
resolve::{
|
||||
options::{ImportMap, ImportMapping},
|
||||
@@ -409,15 +408,12 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
|
||||
)
|
||||
.module();
|
||||
|
||||
let module_graph = ModuleGraph::from_module(jest_entry_asset);
|
||||
|
||||
let res = evaluate(
|
||||
jest_entry_asset,
|
||||
*path,
|
||||
Vc::upcast(CommandLineProcessEnv::new()),
|
||||
test_source.ident(),
|
||||
asset_context,
|
||||
module_graph,
|
||||
Vc::upcast(chunking_context),
|
||||
None,
|
||||
vec![],
|
||||
|
||||
Reference in New Issue
Block a user