hanzoai/next.js
2
0
Fork 0
mirror of https://github.com/hanzoai/next.js.git synced 2026-09-24 17:00:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

next-core: deduplicate output assets and detect content conflicts on emit (#92292)

Browse Source 
### What?

Adds deduplication and conflict detection to the asset emission stage in
`crates/next-core/src/emit.rs`, and a new `IssueStage::Emit` variant in
`turbopack-core`.

Before emitting, assets are grouped by their output path. If multiple
assets map to the same path:

- If their content is identical, one is silently chosen (deduplication).
- If their content differs, both versions are written to
`<node_root>/<content_hash>.<ext>` and an `EmitConflictIssue` is raised
for each conflict. All assets are still emitted — conflicts do not abort
the build.

### Why?

Previously, duplicate output assets for the same path were emitted
unconditionally — whichever write happened last silently won. This
masked build graph bugs where two different modules produced conflicting
output files. Reporting conflicts as issues (rather than silently
overwriting) makes them visible and easy to diagnose without breaking
the build.

### How?

- Collect all assets with their resolved paths via `try_flat_join`.
- Bucket them into two `FxIndexMap<FileSystemPath,
Vec<ResolvedVc<Box<dyn OutputAsset>>>>` — one for node-root assets and
one for client assets.
- For each bucket entry, call `check_duplicates`: compare every asset
against the first using `assets_diff`. If content differs, emit an
`EmitConflictIssue` as a turbo-tasks collectible — but still return the
first asset so emission continues.
- `assets_diff` is a `#[turbo_tasks::function]` that takes only
`(asset1, asset2, extension, node_root)` — the `asset_path` stays out of
the task key to avoid unnecessary task cardinality. When file content
differs, it hashes each version with xxh3, writes them to
`<node_root>/<hash>.<ext>`, and returns the paths in the detail message
so the user can diff them.
- `EmitConflictIssue` implements the `Issue` trait with
`IssueStage::Emit` (new variant added to `turbopack-core`),
`IssueSeverity::Error`, a descriptive title, and a detail message
explaining the type of conflict.
- Node-root and client assets are emitted in parallel via
`futures::join!` (not `try_join!`) to ensure deterministic error
reporting — both branches always run to completion so errors are
reported in a consistent order.

---------

Co-authored-by: Tobias Koppers <[email protected]>
Co-authored-by: Claude <[email protected]>
This commit is contained in:
Tobias Koppers
2026-04-05 20:37:58 +02:00
committed by GitHub
co-authored by Tobias Koppers Claude
parent 5a922bc9bb
commit 81d5e070f4
2 changed files with 255 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/next-core/src/emit.rs
+253 -32
View File
@@ -1,9 +1,16 @@
use anyhow::Result;
use anyhow::{Ok, Result};
use futures::join;
use smallvec::{SmallVec, smallvec};
use tracing::Instrument;
use turbo_tasks::{TryFlatJoinIterExt, ValueToStringRef, Vc};
use turbo_tasks_fs::{FileSystemPath, rebase};
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexMap, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToStringRef, Vc,
};
use turbo_tasks_fs::{FileContent, FileSystemPath, rebase};
use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64};
use turbopack_core::{
asset::Asset,
asset::{Asset, AssetContent},
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
output::{ExpandedOutputAssets, OutputAsset, OutputAssets},
reference::all_assets_from_entries,
};
@@ -43,40 +50,124 @@ pub async fn emit_assets(
client_relative_path: FileSystemPath,
client_output_path: FileSystemPath,
) -> Result<()> {
let _: Vec<()> = assets
enum Location {
Node,
Client,
}
let assets = assets
.await?
.iter()
.copied()
.map(|asset| {
let node_root = node_root.clone();
let client_relative_path = client_relative_path.clone();
let client_output_path = client_output_path.clone();
async move {
let path = asset.path().owned().await?;
let span = tracing::info_span!("emit asset", name = %path.to_string_ref().await?);
async move {
Ok(if path.is_inside_ref(&node_root) {
Some(emit(*asset).as_side_effect().await?)
} else if path.is_inside_ref(&client_relative_path) {
// Client assets are emitted to the client output path, which is prefixed
// with _next. We need to rebase them to remove that
// prefix.
Some(
emit_rebase(*asset, client_relative_path, client_output_path)
.as_side_effect()
.await?,
)
} else {
None
})
}
.instrument(span)
.await
}
.map(async |asset| {
let path = asset.path().owned().await?;
let location = if path.is_inside_ref(&node_root) {
Location::Node
} else if path.is_inside_ref(&client_relative_path) {
Location::Client
} else {
return Ok(None);
};
Ok(Some((location, path, asset)))
})
.try_flat_join()
.await?;
type AssetVec = SmallVec<[ResolvedVc<Box<dyn OutputAsset>>; 1]>;
let mut node_assets_by_path: FxIndexMap<FileSystemPath, AssetVec> = FxIndexMap::default();
let mut client_assets_by_path: FxIndexMap<FileSystemPath, AssetVec> = FxIndexMap::default();
for (location, path, asset) in assets {
match location {
Location::Node => {
node_assets_by_path
.entry(path)
.or_insert_with(|| smallvec![])
.push(asset);
}
Location::Client => {
client_assets_by_path
.entry(path)
.or_insert_with(|| smallvec![])
.push(asset);
}
}
}
/// Checks for duplicate assets at the same path. If duplicates with
/// different content are found, emits an `EmitConflictIssue` for each
/// conflict but still returns the first asset so emission can continue.
async fn check_duplicates(
path: &FileSystemPath,
assets: AssetVec,
node_root: &FileSystemPath,
) -> Result<ResolvedVc<Box<dyn OutputAsset>>> {
let mut iter = assets.into_iter();
let first = iter.next().unwrap();
for next in iter {
let ext: RcStr = path.extension().unwrap_or_default().into();
if let Some(detail) = assets_diff(*next, *first, ext, node_root.clone())
.owned()
.await?
{
EmitConflictIssue {
asset_path: path.clone(),
detail,
}
.resolved_cell()
.emit();
}
}
Ok(first)
}
// Use join! instead of try_join! to collect all errors deterministically
// rather than returning whichever branch fails first non-deterministically.
let (node_result, client_result) = join!(
node_assets_by_path
.into_iter()
.map(|(path, assets)| {
let node_root = node_root.clone();
async move {
let asset = check_duplicates(&path, assets, &node_root).await?;
let span = tracing::info_span!(
"emit asset",
name = %path.to_string_ref().await?
);
async move { emit(*asset).as_side_effect().await }
.instrument(span)
.await
}
})
.try_join(),
client_assets_by_path
.into_iter()
.map(|(path, assets)| {
let node_root = node_root.clone();
let client_relative_path = client_relative_path.clone();
let client_output_path = client_output_path.clone();
async move {
let asset = check_duplicates(&path, assets, &node_root).await?;
let span = tracing::info_span!(
"emit asset",
name = %path.to_string_ref().await?
);
async move {
// Client assets are emitted to the client output path, which is
// prefixed with _next. We need to rebase them to
// remove that prefix.
emit_rebase(*asset, client_relative_path, client_output_path)
.as_side_effect()
.await
}
.instrument(span)
.await
}
})
.try_join(),
);
node_result?;
client_result?;
Ok(())
}
@@ -110,3 +201,133 @@ async fn emit_rebase(
.await?;
Ok(())
}
/// Compares two assets that target the same output path. If their content
/// differs, writes both versions under `node_root` as `<hash>.<ext>` and
/// returns a description of the difference.
#[turbo_tasks::function]
async fn assets_diff(
asset1: Vc<Box<dyn OutputAsset>>,
asset2: Vc<Box<dyn OutputAsset>>,
extension: RcStr,
node_root: FileSystemPath,
) -> Result<Vc<Option<RcStr>>> {
let content1 = asset1.content().await?;
let content2 = asset2.content().await?;
let detail = match (&*content1, &*content2) {
(AssetContent::File(content1), AssetContent::File(content2)) => {
let content1 = content1.await?;
let content2 = content2.await?;
match (&*content1, &*content2) {
(FileContent::NotFound, FileContent::NotFound) => None,
(FileContent::Content(file1), FileContent::Content(file2)) => {
if file1 == file2 {
None
} else {
// Write both versions under node_root as <hash>.<ext> so the
// user can diff them.
let ext = &*extension;
let hash1 = encode_hex(hash_xxh3_hash64(file1.content().content_hash()));
let hash2 = encode_hex(hash_xxh3_hash64(file2.content().content_hash()));
let name1 = if ext.is_empty() {
hash1
} else {
format!("{hash1}.{ext}")
};
let name2 = if ext.is_empty() {
hash2
} else {
format!("{hash2}.{ext}")
};
let path1 = node_root.join(&name1)?;
let path2 = node_root.join(&name2)?;
path1
.write(FileContent::Content(file1.clone()).cell())
.as_side_effect()
.await?;
path2
.write(FileContent::Content(file2.clone()).cell())
.as_side_effect()
.await?;
Some(format!(
"file content differs, written to:\n {}\n {}",
path1.to_string_ref().await?,
path2.to_string_ref().await?,
))
}
}
_ => Some(
"assets at the same path have mismatched file content types (one task wants \
to write the file, another wants to delete it)"
.into(),
),
}
}
(
AssetContent::Redirect {
target: target1,
link_type: link_type1,
},
AssetContent::Redirect {
target: target2,
link_type: link_type2,
},
) => {
if target1 == target2 && link_type1 == link_type2 {
None
} else {
Some(format!(
"assets at the same path are both redirects but point to different targets: \
{target1} vs {target2}"
))
}
}
_ => Some(
"assets at the same path have different content types (one is a file, the other is a \
redirect)"
.into(),
),
};
Ok(Vc::cell(detail.map(|d| d.into())))
}
#[turbo_tasks::value]
struct EmitConflictIssue {
asset_path: FileSystemPath,
detail: RcStr,
}
#[turbo_tasks::value_impl]
impl Issue for EmitConflictIssue {
#[turbo_tasks::function]
fn file_path(&self) -> Vc<FileSystemPath> {
self.asset_path.clone().cell()
}
#[turbo_tasks::function]
fn stage(&self) -> Vc<IssueStage> {
IssueStage::Emit.cell()
}
fn severity(&self) -> IssueSeverity {
IssueSeverity::Error
}
#[turbo_tasks::function]
fn title(&self) -> Vc<StyledString> {
StyledString::Text(
"Two or more assets with different content were emitted to the same output path".into(),
)
.cell()
}
#[turbo_tasks::function]
fn description(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(
StyledString::Text(self.detail.clone()).resolved_cell(),
))
}
}
turbopack/crates/turbopack-core/src/issue/mod.rs
+2
View File
@@ -875,6 +875,7 @@ pub enum IssueStage {
Resolve,
Bindings,
CodeGen,
Emit,
Unsupported,
Misc,
Other(RcStr),
@@ -893,6 +894,7 @@ impl Display for IssueStage {
IssueStage::Analysis => write!(f, "analysis"),
IssueStage::Bindings => write!(f, "bindings"),
IssueStage::CodeGen => write!(f, "code gen"),
IssueStage::Emit => write!(f, "emit"),
IssueStage::Unsupported => write!(f, "unsupported"),
IssueStage::AppStructure => write!(f, "app structure"),
IssueStage::Misc => write!(f, "misc"),