hanzoai/next.js
2
0
Fork 0
mirror of https://github.com/hanzoai/next.js.git synced 2026-09-24 17:00:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Share Ecmascript HMR chunk versioning, diffing, and merging between browser and node (#96325)

Browse Source 
## Summary

`turbopack-nodejs` and `turbopack-browser` each carried their own copy
of the Ecmascript chunk versioning and HMR diffing stack.
`chunk_list/merged_update.rs` already documented the intent to share
this, noting the turbo-tasks value types "cannot be generic and
therefore remain per-runtime". A `value_trait` resolves that: the shared
machinery is written against `EcmascriptHmrChunkContent`, which each
runtime implements in a few lines to expose `entries()` and
`own_version()`.

The shared code now lives in `turbopack-ecmascript/src/hmr/`
(`version.rs`, `update.rs`, `content.rs`, `merger.rs`). Both runtimes'
`ecmascript/version.rs`, `ecmascript/update.rs`, and
`ecmascript/**/merged/` directories are deleted.

No struct fields were widened and no `ChunkingContext` methods changed.

### Behavior changes

- Browser chunk version IDs now hash `minify_type`, matching node. IDs
are opaque and recomputed in-memory on both ends of an HMR connection,
so this cannot desync a client.
- The shared diff uses node's lazy `entries()` materialization, so
chunks with only deletions no longer materialize any `Vc<Code>`. This
now applies to the browser path too.
- Node error text said "chunk path ... is not in client root", a
copy-paste artifact. Now says "output root".
- `#[turbo_tasks(trace_ignore)]` is applied consistently to the merged
chunk version; previously only the browser copy had it.

Browser chunk content keeps the default `VersionedContent::update`.
Overriding it bypasses the `ChunkListUpdate` envelope the client runtime
expects.

### Not included

Unifying the chunk-content structs. That requires widening to `Box<dyn
ChunkingContext>`, and both chunking contexts define `minify_type` twice
— an inherent method and a `ChunkingContext` method whose trait default
is `NoMinify` — so it needs separate verification. The trait added here
lets the two structs coexist at no cost.

The runtime, evaluate, and entry chunk types are also out of scope:
`get_browser_runtime_code` takes eight more parameters than
`get_node_runtime_code`, so those differ behaviorally rather than by
duplication.

<!-- NEXT_JS_LLM -->
This commit is contained in:
Will Binns-Smith
2026-07-29 14:51:33 -07:00
committed by GitHub
parent be1424f3dc
commit 6fc7bf8af1
25 changed files with 606 additions and 1116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
turbopack/crates/turbopack-browser/src/ecmascript/content.rs
+25 -23
View File
@@ -15,14 +15,15 @@ use turbopack_core::{
};
use turbopack_ecmascript::{
chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries},
hmr::{
EcmascriptHmrChunkContent, merger::EcmascriptChunkContentMerger,
version::EcmascriptChunkVersion,
},
minify::minify,
utils::StringifyJs,
};
use super::{
chunk::EcmascriptBrowserChunk, merged::merger::EcmascriptBrowserChunkContentMerger,
version::EcmascriptBrowserChunkVersion,
};
use super::chunk::EcmascriptBrowserChunk;
use crate::{
BrowserChunkingContext,
chunking_context::{CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod},
@@ -54,23 +55,6 @@ impl EcmascriptBrowserChunkContent {
.cell())
}
#[turbo_tasks::function]
pub fn entries(&self) -> Vc<EcmascriptChunkContentEntries> {
EcmascriptChunkContentEntries::new(*self.content)
}
}
#[turbo_tasks::value_impl]
impl EcmascriptBrowserChunkContent {
#[turbo_tasks::function]
pub(crate) async fn own_version(&self) -> Result<Vc<EcmascriptBrowserChunkVersion>> {
Ok(EcmascriptBrowserChunkVersion::new(
self.chunking_context.output_root().owned().await?,
self.chunk.path().owned().await?,
*self.content,
))
}
#[turbo_tasks::function]
pub(crate) async fn code(self: Vc<Self>) -> Result<Vc<Code>> {
let this = self.await?;
@@ -164,7 +148,25 @@ impl VersionedContent for EcmascriptBrowserChunkContent {
#[turbo_tasks::function]
fn version(self: Vc<Self>) -> Vc<Box<dyn Version>> {
Vc::upcast(self.own_version())
Vc::upcast(self.ecmascript_chunk_version())
}
}
#[turbo_tasks::value_impl]
impl EcmascriptHmrChunkContent for EcmascriptBrowserChunkContent {
#[turbo_tasks::function]
fn entries(&self) -> Vc<EcmascriptChunkContentEntries> {
EcmascriptChunkContentEntries::new(*self.content)
}
#[turbo_tasks::function]
async fn ecmascript_chunk_version(&self) -> Result<Vc<EcmascriptChunkVersion>> {
Ok(EcmascriptChunkVersion::new(
self.chunking_context.output_root().owned().await?,
self.chunk.path().owned().await?,
*self.content,
*self.chunking_context.minify_type().await?,
))
}
}
@@ -172,7 +174,7 @@ impl VersionedContent for EcmascriptBrowserChunkContent {
impl MergeableVersionedContent for EcmascriptBrowserChunkContent {
#[turbo_tasks::function]
fn get_merger(&self) -> Vc<Box<dyn VersionedContentMerger>> {
Vc::upcast(EcmascriptBrowserChunkContentMerger::new())
Vc::upcast(EcmascriptChunkContentMerger::new())
}
}
turbopack/crates/turbopack-browser/src/ecmascript/merged/content.rs
-60
View File
@@ -1,60 +0,0 @@
use anyhow::{Result, bail};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
asset::AssetContent,
version::{Update, Version, VersionedContent},
};
use super::{
super::content::EcmascriptBrowserChunkContent, update::update_ecmascript_merged_chunk,
version::EcmascriptBrowserMergedChunkVersion,
};
/// Composite [`EcmascriptChunkContent`] that is the result of merging multiple
/// EcmaScript chunk's contents together through the
/// [`EcmascriptChunkContentMerger`].
///
/// [`EcmascriptChunkContentMerger`]: super::merger::EcmascriptChunkContentMerger
#[turbo_tasks::value(serialization = "skip", shared)]
pub(super) struct EcmascriptBrowserMergedChunkContent {
pub contents: Vec<ResolvedVc<EcmascriptBrowserChunkContent>>,
}
#[turbo_tasks::value_impl]
impl EcmascriptBrowserMergedChunkContent {
#[turbo_tasks::function]
pub async fn version(&self) -> Result<Vc<EcmascriptBrowserMergedChunkVersion>> {
Ok(EcmascriptBrowserMergedChunkVersion {
versions: self
.contents
.iter()
.map(|content| async move { content.own_version().await })
.try_join()
.await?,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl VersionedContent for EcmascriptBrowserMergedChunkContent {
#[turbo_tasks::function]
fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
bail!("EcmascriptDevMergedChunkContent does not have content")
}
#[turbo_tasks::function]
fn version(self: Vc<Self>) -> Vc<Box<dyn Version>> {
Vc::upcast(self.version())
}
#[turbo_tasks::function]
async fn update(
self: Vc<Self>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> {
Ok(update_ecmascript_merged_chunk(self, from_version)
.await?
.cell())
}
}
turbopack/crates/turbopack-browser/src/ecmascript/merged/merger.rs
-50
View File
@@ -1,50 +0,0 @@
use anyhow::{Result, bail};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::version::{VersionedContent, VersionedContentMerger, VersionedContents};
use super::{
super::content::EcmascriptBrowserChunkContent, content::EcmascriptBrowserMergedChunkContent,
};
/// Merges multiple [`EcmascriptChunkContent`] into a single
/// [`EcmascriptDevMergedChunkContent`]. This is useful for generating a single
/// update for multiple ES chunks updating all at the same time.
#[turbo_tasks::value]
pub(crate) struct EcmascriptBrowserChunkContentMerger;
#[turbo_tasks::value_impl]
impl EcmascriptBrowserChunkContentMerger {
/// Creates a new [`EcmascriptDevChunkContentMerger`].
#[turbo_tasks::function]
pub fn new() -> Vc<Self> {
Self::cell(EcmascriptBrowserChunkContentMerger)
}
}
#[turbo_tasks::value_impl]
impl VersionedContentMerger for EcmascriptBrowserChunkContentMerger {
#[turbo_tasks::function]
async fn merge(
&self,
contents: Vc<VersionedContents>,
) -> Result<Vc<Box<dyn VersionedContent>>> {
let contents = contents
.await?
.iter()
.map(|content| async move {
if let Some(content) =
ResolvedVc::try_downcast_type::<EcmascriptBrowserChunkContent>(*content)
{
Ok(content)
} else {
bail!("expected Vc<EcmascriptBrowserChunkContent>")
}
})
.try_join()
.await?;
Ok(Vc::upcast(
EcmascriptBrowserMergedChunkContent { contents }.cell(),
))
}
}
turbopack/crates/turbopack-browser/src/ecmascript/merged/mod.rs
-4
View File
@@ -1,4 +0,0 @@
pub(crate) mod content;
pub(crate) mod merger;
pub(crate) mod update;
pub(crate) mod version;
turbopack/crates/turbopack-browser/src/ecmascript/merged/update.rs
-191
View File
@@ -1,191 +0,0 @@
use std::sync::Arc;
use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
chunk::{ChunkingContext, ModuleId},
output::OutputAsset,
version::{PartialUpdate, TotalUpdate, Update, Version},
};
use turbopack_ecmascript::chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
};
use super::{
super::{
update::{EcmascriptChunkUpdate, update_ecmascript_chunk},
version::EcmascriptBrowserChunkVersion,
},
content::EcmascriptBrowserMergedChunkContent,
version::EcmascriptBrowserMergedChunkVersion,
};
/// Helper structure to get a module's hash from multiple different chunk
/// versions, without having to actually merge the versions into a single
/// hashmap, which would be expensive.
struct MergedModuleMap {
versions: Vec<ReadRef<EcmascriptBrowserChunkVersion>>,
}
impl MergedModuleMap {
/// Creates a new `MergedModuleMap` from the given versions.
fn new(versions: Vec<ReadRef<EcmascriptBrowserChunkVersion>>) -> Self {
Self { versions }
}
/// Returns the hash of the module with the given id, or `None` if the
/// module is not present in any of the versions.
fn get(&self, id: &ModuleId) -> Option<u128> {
for version in &self.versions {
if let Some(hash) = version.entries_hashes.get(id) {
return Some(*hash);
}
}
None
}
}
pub(super) async fn update_ecmascript_merged_chunk(
content: Vc<EcmascriptBrowserMergedChunkContent>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Update> {
let to_merged_version = content.version();
let from_merged_version = if let Some(from) =
ResolvedVc::try_downcast_type::<EcmascriptBrowserMergedChunkVersion>(from_version)
{
from
} else {
// It's likely `from_version` is `NotFoundVersion`.
return Ok(Update::Total(TotalUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
}));
};
let to = to_merged_version.await?;
let from = from_merged_version.await?;
// When to and from point to the same value we can skip comparing them. This will happen since
// `TraitRef::<Box<dyn Version>>::cell` will not clone the value, but only make the cell point
// to the same immutable value (`Arc`).
if from.ptr_eq(&to) {
return Ok(Update::None);
}
let mut from_versions_by_chunk_path: FxIndexMap<_, _> = from
.versions
.iter()
.map(|version| (&*version.chunk_path, version))
.collect();
let merged_module_map = MergedModuleMap::new(from.versions.to_vec());
let content = content.await?;
let to_contents = content
.contents
.iter()
.map(|content| async move {
let entries = content.entries().await?;
let content_ref = content.await?;
let output_root = content_ref.chunking_context.output_root().await?;
let path = content_ref.chunk.path().await?;
Ok((*content, entries, output_root, path))
})
.try_join()
.await?;
let mut merged_update = EcmascriptMergedUpdate::default();
for (content, entries, output_root, path) in &to_contents {
let Some(chunk_path) = output_root.get_path_to(path) else {
continue;
};
let chunk_update = if let Some(from_version) =
from_versions_by_chunk_path.swap_remove(chunk_path)
{
// The chunk was present in the previous version, so we must update it.
let update = update_ecmascript_chunk(**content, from_version).await?;
match update {
EcmascriptChunkUpdate::None => {
// Nothing changed, so we can skip this chunk.
continue;
}
EcmascriptChunkUpdate::Partial(chunk_partial) => {
// The chunk was updated.
let mut partial = EcmascriptMergedChunkPartial::default();
for (module_id, (module_hash, module_code)) in chunk_partial.added {
partial.added.insert(module_id.clone());
if merged_module_map.get(&module_id) != Some(module_hash) {
let entry = EcmascriptModuleEntry::from_code(
&module_id,
*module_code,
chunk_path,
)
.await?;
merged_update.entries.insert(module_id, entry);
}
}
partial.deleted.extend(chunk_partial.deleted.into_keys());
for (module_id, module_code) in chunk_partial.modified {
let entry =
EcmascriptModuleEntry::from_code(&module_id, *module_code, chunk_path)
.await?;
merged_update.entries.insert(module_id, entry);
}
EcmascriptMergedChunkUpdate::Partial(partial)
}
}
} else {
// The chunk was added in this version.
let mut added = EcmascriptMergedChunkAdded::default();
for (id, entry) in entries {
let hash = *entry.hash.await?;
added.modules.insert(id.clone());
if merged_module_map.get(id) != Some(hash) {
let entry =
EcmascriptModuleEntry::from_code(id, *entry.code, chunk_path).await?;
merged_update.entries.insert(id.clone(), entry);
}
}
EcmascriptMergedChunkUpdate::Added(added)
};
merged_update.chunks.insert(chunk_path, chunk_update);
}
// Deleted chunks.
for (chunk_path, chunk_version) in from_versions_by_chunk_path {
let hashes = &chunk_version.entries_hashes;
merged_update.chunks.insert(
chunk_path,
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: hashes.keys().cloned().collect(),
}),
);
}
let update = if merged_update.is_empty() {
Update::None
} else {
Update::Partial(PartialUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
instruction: Arc::new(serde_json::to_value(&merged_update)?),
})
};
Ok(update)
}
turbopack/crates/turbopack-browser/src/ecmascript/merged/version.rs
-41
View File
@@ -1,41 +0,0 @@
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{ReadRef, TryJoinIterExt, Vc};
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::version::Version;
use super::super::version::EcmascriptBrowserChunkVersion;
/// The version of a [`super::content::EcmascriptMergedChunkContent`]. This is
/// essentially a composite [`EcmascriptChunkVersion`].
#[turbo_tasks::value(serialization = "skip", shared)]
pub(super) struct EcmascriptBrowserMergedChunkVersion {
#[turbo_tasks(trace_ignore)]
pub(super) versions: Vec<ReadRef<EcmascriptBrowserChunkVersion>>,
}
#[turbo_tasks::value_impl]
impl Version for EcmascriptBrowserMergedChunkVersion {
#[turbo_tasks::function]
async fn id(&self) -> Result<Vc<RcStr>> {
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_value(self.versions.len());
let sorted_ids = {
let mut sorted_ids = self
.versions
.iter()
// This ReadRef::cell call is important, as it means that `.id()` is cached.
.map(|version| ReadRef::cell(version.clone()).id())
.try_join()
.await?;
sorted_ids.sort();
sorted_ids
};
for id in sorted_ids {
hasher.write_value(id);
}
let hash = hasher.finish();
let hash = encode_base64(hash);
Ok(Vc::cell(hash.into()))
}
}
turbopack/crates/turbopack-browser/src/ecmascript/mod.rs
-3
View File
@@ -2,9 +2,6 @@ pub(crate) mod chunk;
pub(crate) mod content;
pub(crate) mod evaluate;
pub mod list;
pub(crate) mod merged;
pub(crate) mod update;
pub(crate) mod version;
pub(crate) mod worker;
pub use chunk::EcmascriptBrowserChunk;
turbopack/crates/turbopack-browser/src/ecmascript/update.rs
-65
View File
@@ -1,65 +0,0 @@
use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, Vc};
use turbopack_core::{chunk::ModuleId, code_builder::Code};
use super::{content::EcmascriptBrowserChunkContent, version::EcmascriptBrowserChunkVersion};
#[allow(clippy::large_enum_variant)]
pub(super) enum EcmascriptChunkUpdate {
None,
Partial(EcmascriptChunkPartialUpdate),
}
pub(super) struct EcmascriptChunkPartialUpdate {
pub added: FxIndexMap<ModuleId, (u128, ResolvedVc<Code>)>,
pub deleted: FxIndexMap<ModuleId, u128>,
pub modified: FxIndexMap<ModuleId, ResolvedVc<Code>>,
}
pub(super) async fn update_ecmascript_chunk(
content: Vc<EcmascriptBrowserChunkContent>,
from: &ReadRef<EcmascriptBrowserChunkVersion>,
) -> Result<EcmascriptChunkUpdate> {
let to = content.own_version().await?;
// When to and from point to the same value we can skip comparing them. This will happen since
// `TraitRef::<Box<dyn Version>>::cell` will not clone the value, but only make the cell point
// to the same immutable value (`Arc`).
if from.ptr_eq(&to) {
return Ok(EcmascriptChunkUpdate::None);
}
let entries = content.entries().await?;
let mut added = FxIndexMap::default();
let mut modified = FxIndexMap::default();
let mut deleted = FxIndexMap::default();
for (id, from_hash) in &from.entries_hashes {
if let Some(entry) = entries.get(id) {
if *entry.hash.await? != *from_hash {
modified.insert(id.clone(), entry.code);
}
} else {
deleted.insert(id.clone(), *from_hash);
}
}
// Remaining entries are added
for (id, entry) in entries.iter() {
if !from.entries_hashes.contains_key(id) {
added.insert(id.clone(), (*entry.hash.await?, entry.code));
}
}
let update = if added.is_empty() && modified.is_empty() && deleted.is_empty() {
EcmascriptChunkUpdate::None
} else {
EcmascriptChunkUpdate::Partial(EcmascriptChunkPartialUpdate {
added,
modified,
deleted,
})
};
Ok(update)
}
turbopack/crates/turbopack-browser/src/ecmascript/version.rs
-64
View File
@@ -1,64 +0,0 @@
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::{chunk::ModuleId, version::Version};
use turbopack_ecmascript::chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries};
#[turbo_tasks::value(serialization = "skip")]
pub(super) struct EcmascriptBrowserChunkVersion {
pub(super) chunk_path: String,
pub(super) entries_hashes: FxIndexMap<ModuleId, u128>,
}
#[turbo_tasks::value_impl]
impl EcmascriptBrowserChunkVersion {
#[turbo_tasks::function]
pub async fn new(
output_root: FileSystemPath,
chunk_path: FileSystemPath,
content: Vc<EcmascriptChunkContent>,
) -> Result<Vc<Self>> {
let output_root = output_root.clone();
let chunk_path = chunk_path.clone();
let chunk_path = if let Some(path) = output_root.get_path_to(&chunk_path) {
path
} else {
turbobail!("chunk path {chunk_path} is not in client root {output_root}");
};
let entries_hashes = EcmascriptChunkContentEntries::new(content)
.await?
.iter()
.map(async |(id, entry)| Ok((id.clone(), *entry.hash.await?)))
.try_join()
.await?
.into_iter()
.collect();
Ok(EcmascriptBrowserChunkVersion {
chunk_path: chunk_path.to_string(),
entries_hashes,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl Version for EcmascriptBrowserChunkVersion {
#[turbo_tasks::function]
fn id(&self) -> Vc<RcStr> {
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_ref(&self.chunk_path);
let sorted_hashes = {
let mut hashes: Vec<_> = self.entries_hashes.values().copied().collect();
hashes.sort();
hashes
};
for hash in sorted_hashes {
hasher.write_value(hash);
}
let hash = hasher.finish();
let hash = encode_base64(hash);
Vc::cell(hash.into())
}
}
turbopack/crates/turbopack-ecmascript/src/hmr/content.rs
+63
View File
@@ -0,0 +1,63 @@
use anyhow::{Result, bail};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
asset::AssetContent,
version::{Update, Version, VersionedContent},
};
use crate::hmr::{
EcmascriptHmrChunkContent, update::update_ecmascript_merged_chunk,
version::EcmascriptMergedChunkVersion,
};
/// Composite [`VersionedContent`] that is the result of merging multiple
/// Ecmascript chunk contents together through the
/// [`super::merger::EcmascriptChunkContentMerger`]. This allows a chunk
/// list to produce a single `EcmascriptMergedUpdate` for multiple chunks updating
/// at the same time.
///
/// "Merged" here refers to [`turbopack_core::version::VersionedContentMerger`]
/// combining several chunks into one update payload.
#[turbo_tasks::value(serialization = "skip", shared)]
pub struct EcmascriptMergedChunkContent {
pub contents: Vec<ResolvedVc<Box<dyn EcmascriptHmrChunkContent>>>,
}
#[turbo_tasks::value_impl]
impl EcmascriptMergedChunkContent {
#[turbo_tasks::function]
pub async fn version(&self) -> Result<Vc<EcmascriptMergedChunkVersion>> {
Ok(EcmascriptMergedChunkVersion {
versions: self
.contents
.iter()
.map(|content| async move { content.ecmascript_chunk_version().await })
.try_join()
.await?,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl VersionedContent for EcmascriptMergedChunkContent {
#[turbo_tasks::function]
fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
bail!("EcmascriptMergedChunkContent does not have content")
}
#[turbo_tasks::function]
fn version(self: Vc<Self>) -> Vc<Box<dyn Version>> {
Vc::upcast(self.version())
}
#[turbo_tasks::function]
async fn update(
self: Vc<Self>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> {
Ok(update_ecmascript_merged_chunk(self, from_version)
.await?
.cell())
}
}
turbopack/crates/turbopack-ecmascript/src/hmr/merger.rs
+45
View File
@@ -0,0 +1,45 @@
use anyhow::{Result, bail};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::version::{VersionedContent, VersionedContentMerger, VersionedContents};
use crate::hmr::{EcmascriptHmrChunkContent, content::EcmascriptMergedChunkContent};
/// Merges multiple [`EcmascriptHmrChunkContent`] into a single
/// [`EcmascriptMergedChunkContent`]. This allows the chunk list to produce a
/// single `EcmascriptMergedUpdate` for multiple chunks updating at the same time.
#[turbo_tasks::value]
pub struct EcmascriptChunkContentMerger;
#[turbo_tasks::value_impl]
impl EcmascriptChunkContentMerger {
#[turbo_tasks::function]
pub fn new() -> Vc<Self> {
Self::cell(EcmascriptChunkContentMerger)
}
}
#[turbo_tasks::value_impl]
impl VersionedContentMerger for EcmascriptChunkContentMerger {
#[turbo_tasks::function]
async fn merge(
&self,
contents: Vc<VersionedContents>,
) -> Result<Vc<Box<dyn VersionedContent>>> {
let contents = contents
.await?
.iter()
.map(|content| async move {
if let Some(content) =
ResolvedVc::try_sidecast::<Box<dyn EcmascriptHmrChunkContent>>(*content)
{
Ok(content)
} else {
bail!("expected Vc<Box<dyn EcmascriptHmrChunkContent>>")
}
})
.try_join()
.await?;
Ok(Vc::upcast(EcmascriptMergedChunkContent { contents }.cell()))
}
}
turbopack/crates/turbopack-ecmascript/src/hmr/mod.rs
+31
View File
@@ -0,0 +1,31 @@
use turbo_tasks::Vc;
use turbopack_core::version::VersionedContent;
use crate::{chunk::EcmascriptChunkContentEntries, hmr::version::EcmascriptChunkVersion};
/// Diffing and merging of Ecmascript chunk contents for hot module replacement.
///
/// Each module covers both the single-chunk and the merged (multi-chunk) case,
/// the latter being what a chunk list produces a single update from.
pub mod content;
pub mod merger;
pub mod update;
pub mod version;
/// An Ecmascript chunk content that participates in HMR.
///
/// Exists so the version/diff/merge machinery can be written once against the
/// trait rather than duplicated per runtime. The turbo-tasks value types cannot
/// be generic, so the runtimes keep their own content structs and implement this
/// to expose the two things the shared machinery needs.
#[turbo_tasks::value_trait]
pub trait EcmascriptHmrChunkContent: VersionedContent {
/// The per-module code and hashes making up this chunk.
#[turbo_tasks::function]
fn entries(self: Vc<Self>) -> Vc<EcmascriptChunkContentEntries>;
/// The same value as [`VersionedContent::version`], but with the concrete
/// type the diffing machinery needs instead of `Box<dyn Version>`.
#[turbo_tasks::function]
fn ecmascript_chunk_version(self: Vc<Self>) -> Vc<EcmascriptChunkVersion>;
}
turbopack/crates/turbopack-ecmascript/src/hmr/update.rs
+294
View File
@@ -0,0 +1,294 @@
use std::sync::Arc;
use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
chunk::ModuleId,
code_builder::Code,
version::{PartialUpdate, TotalUpdate, Update, Version},
};
use crate::{
chunk::EcmascriptChunkContentEntries,
chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
},
hmr::{
EcmascriptHmrChunkContent,
content::EcmascriptMergedChunkContent,
version::{EcmascriptChunkVersion, EcmascriptMergedChunkVersion},
},
};
/// The module-level difference between two versions of a single chunk.
enum EcmascriptChunkUpdate {
None,
Partial {
added: FxIndexMap<ModuleId, AddedModule>,
modified: FxIndexMap<ModuleId, ResolvedVc<Code>>,
deleted: FxIndexMap<ModuleId, u128>,
},
}
struct AddedModule {
hash: u128,
code: ResolvedVc<Code>,
}
/// Diffs two versions of a single chunk's content, as one step of building a
/// merged update.
async fn update_ecmascript_hmr_chunk_content(
content: Vc<Box<dyn EcmascriptHmrChunkContent>>,
to: &ReadRef<EcmascriptChunkVersion>,
from: &ReadRef<EcmascriptChunkVersion>,
) -> Result<EcmascriptChunkUpdate> {
let mut added = FxIndexMap::default();
let mut modified = FxIndexMap::default();
let mut deleted = FxIndexMap::default();
// Lazily resolve the entries map only when we actually need to ship code
// bytes for an added or modified module. For chunks that only have deletions
// (or no changes that need code beyond hashes), this avoids materializing
// any `Vc<Code>`.
let mut entries_ref = None;
// Check for deleted and modified modules
for (id, from_hash) in &from.entries_hashes {
if let Some(to_hash) = to.entries_hashes.get(id) {
if *to_hash != *from_hash {
// Module was modified
let entries = match &entries_ref {
Some(entries) => entries,
None => entries_ref.insert(content.entries().await?),
};
if let Some(entry) = entries.get(id) {
modified.insert(id.clone(), entry.code);
}
}
} else {
// Module was deleted
deleted.insert(id.clone(), *from_hash);
}
}
// Check for added modules
for (id, hash) in &to.entries_hashes {
if !from.entries_hashes.contains_key(id) {
let entries = match &entries_ref {
Some(entries) => entries,
None => entries_ref.insert(content.entries().await?),
};
if let Some(entry) = entries.get(id) {
added.insert(
id.clone(),
AddedModule {
hash: *hash,
code: entry.code,
},
);
}
}
}
Ok(
if added.is_empty() && modified.is_empty() && deleted.is_empty() {
EcmascriptChunkUpdate::None
} else {
EcmascriptChunkUpdate::Partial {
added,
modified,
deleted,
}
},
)
}
/// Looks up a module's hash across several chunk versions, avoiding the cost of
/// merging them into a single map.
fn module_hash(versions: &[ReadRef<EcmascriptChunkVersion>], id: &ModuleId) -> Option<u128> {
versions
.iter()
.find_map(|version| version.entries_hashes.get(id).copied())
}
/// Code only has to be shipped once per update: skip any module another chunk in
/// the group already provides at the same hash.
async fn insert_entry_unless_shipped(
entries: &mut FxIndexMap<ModuleId, EcmascriptModuleEntry>,
from_versions: &[ReadRef<EcmascriptChunkVersion>],
id: ModuleId,
hash: u128,
code: Vc<Code>,
chunk_path: &str,
) -> Result<()> {
if module_hash(from_versions, &id) != Some(hash) {
let entry = EcmascriptModuleEntry::from_code(&id, code, chunk_path).await?;
entries.insert(id, entry);
}
Ok(())
}
/// Translates a single chunk's module diff into its merged-update payload.
async fn partial_chunk_update(
update: EcmascriptChunkUpdate,
chunk_path: &str,
from_versions: &[ReadRef<EcmascriptChunkVersion>],
entries: &mut FxIndexMap<ModuleId, EcmascriptModuleEntry>,
) -> Result<EcmascriptMergedChunkUpdate> {
let EcmascriptChunkUpdate::Partial {
added,
modified,
deleted,
} = update
else {
unreachable!("caller filters out EcmascriptChunkUpdate::None");
};
let mut partial = EcmascriptMergedChunkPartial::default();
for (id, AddedModule { hash, code }) in added {
partial.added.insert(id.clone());
insert_entry_unless_shipped(entries, from_versions, id, hash, *code, chunk_path).await?;
}
partial.deleted.extend(deleted.into_keys());
for (id, code) in modified {
let entry = EcmascriptModuleEntry::from_code(&id, *code, chunk_path).await?;
entries.insert(id, entry);
}
Ok(EcmascriptMergedChunkUpdate::Partial(partial))
}
/// Builds the payload for a chunk that wasn't present in the previous version.
async fn added_chunk_update(
chunk_entries: &ReadRef<EcmascriptChunkContentEntries>,
chunk_path: &str,
from_versions: &[ReadRef<EcmascriptChunkVersion>],
entries: &mut FxIndexMap<ModuleId, EcmascriptModuleEntry>,
) -> Result<EcmascriptMergedChunkUpdate> {
let mut added = EcmascriptMergedChunkAdded::default();
for (id, entry) in chunk_entries.iter() {
added.modules.insert(id.clone());
insert_entry_unless_shipped(
entries,
from_versions,
id.clone(),
*entry.hash.await?,
*entry.code,
chunk_path,
)
.await?;
}
Ok(EcmascriptMergedChunkUpdate::Added(added))
}
/// Computes a single [`Update`] covering every chunk in a merged chunk content.
///
/// Runtime-agnostic: both the browser and node chunk lists share this one
/// implementation.
pub async fn update_ecmascript_merged_chunk(
content: Vc<EcmascriptMergedChunkContent>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Update> {
let to_merged_version = content.version();
let Some(from_merged_version) =
ResolvedVc::try_downcast_type::<EcmascriptMergedChunkVersion>(from_version)
else {
// It's likely `from_version` is `NotFoundVersion`.
return Ok(Update::Total(TotalUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
}));
};
let to = to_merged_version.await?;
let from = from_merged_version.await?;
// When to and from point to the same value we can skip comparing them
if from.ptr_eq(&to) {
return Ok(Update::None);
}
let mut from_versions_by_chunk_path: FxIndexMap<_, _> = from
.versions
.iter()
.map(|version| (&*version.chunk_path, version))
.collect();
let from_versions = &from.versions;
let content = content.await?;
let to_contents = content
.contents
.iter()
.map(|content| async move {
let entries = content.entries().await?;
let version = content.ecmascript_chunk_version().await?;
Ok((*content, entries, version))
})
.try_join()
.await?;
let mut merged_update = EcmascriptMergedUpdate::default();
for (content, entries, to_version) in &to_contents {
let chunk_path = to_version.chunk_path.as_str();
let chunk_update = match from_versions_by_chunk_path.swap_remove(chunk_path) {
Some(from_version) => {
match update_ecmascript_hmr_chunk_content(**content, to_version, from_version)
.await?
{
EcmascriptChunkUpdate::None => continue,
update => {
partial_chunk_update(
update,
chunk_path,
from_versions,
&mut merged_update.entries,
)
.await?
}
}
}
None => {
added_chunk_update(
entries,
chunk_path,
from_versions,
&mut merged_update.entries,
)
.await?
}
};
merged_update.chunks.insert(chunk_path, chunk_update);
}
for (chunk_path, chunk_version) in from_versions_by_chunk_path {
let hashes = &chunk_version.entries_hashes;
merged_update.chunks.insert(
chunk_path,
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: hashes.keys().cloned().collect(),
}),
);
}
Ok(if merged_update.is_empty() {
Update::None
} else {
Update::Partial(PartialUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
instruction: Arc::new(serde_json::to_value(&merged_update)?),
})
})
}
turbopack/crates/turbopack-ecmascript/src/hmr/version.rs
+111
View File
@@ -0,0 +1,111 @@
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, ReadRef, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::{
chunk::{MinifyType, ModuleId},
version::Version,
};
use crate::chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries};
/// The version of a single Ecmascript chunk's content, tracked as the set of
/// per-module content hashes.
///
/// Runtime-agnostic: the browser and node chunking contexts share this one
/// implementation rather than each carrying a copy. `minify_type` participates
/// in the hash because minification changes the emitted bytes without changing
/// any module's own hash.
#[turbo_tasks::value(serialization = "skip")]
pub struct EcmascriptChunkVersion {
pub chunk_path: RcStr,
pub minify_type: MinifyType,
pub entries_hashes: FxIndexMap<ModuleId, u128>,
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkVersion {
#[turbo_tasks::function]
pub async fn new(
output_root: FileSystemPath,
chunk_path: FileSystemPath,
content: Vc<EcmascriptChunkContent>,
minify_type: MinifyType,
) -> Result<Vc<Self>> {
let Some(chunk_path) = output_root.get_path_to(&chunk_path) else {
turbobail!("chunk path {chunk_path} is not in output root {output_root}");
};
let entries_hashes = EcmascriptChunkContentEntries::new(content)
.await?
.iter()
.map(async |(id, entry)| Ok((id.clone(), *entry.hash.await?)))
.try_join()
.await?
.into_iter()
.collect();
Ok(EcmascriptChunkVersion {
chunk_path: chunk_path.into(),
minify_type,
entries_hashes,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl Version for EcmascriptChunkVersion {
#[turbo_tasks::function]
fn id(&self) -> Vc<RcStr> {
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_ref(&self.chunk_path);
hasher.write_ref(&self.minify_type);
let sorted_hashes = {
let mut hashes: Vec<_> = self.entries_hashes.values().copied().collect();
hashes.sort();
hashes
};
for hash in sorted_hashes {
hasher.write_value(hash);
}
let hash = hasher.finish();
let hash = encode_base64(hash);
Vc::cell(hash.into())
}
}
/// The version of a [`super::content::EcmascriptMergedChunkContent`]. This is
/// essentially a composite [`EcmascriptChunkVersion`].
#[turbo_tasks::value(serialization = "skip", shared)]
pub struct EcmascriptMergedChunkVersion {
#[turbo_tasks(trace_ignore)]
pub versions: Vec<ReadRef<EcmascriptChunkVersion>>,
}
#[turbo_tasks::value_impl]
impl Version for EcmascriptMergedChunkVersion {
#[turbo_tasks::function]
async fn id(&self) -> Result<Vc<RcStr>> {
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_value(self.versions.len());
let sorted_ids = {
let mut sorted_ids = self
.versions
.iter()
// This `ReadRef::cell` call is important: it ensures the id is
// computed from a cell, so it is cached.
.map(|version| ReadRef::cell(version.clone()).id())
.try_join()
.await?;
sorted_ids.sort();
sorted_ids
};
for id in sorted_ids {
hasher.write_value(id);
}
let hash = hasher.finish();
let hash = encode_base64(hash);
Ok(Vc::cell(hash.into()))
}
}
turbopack/crates/turbopack-ecmascript/src/lib.rs
+1
View File
@@ -16,6 +16,7 @@ pub mod chunk_list;
pub mod code_gen;
pub mod embed_js;
mod errors;
pub mod hmr;
pub mod json_source_transform;
pub mod magic_identifier;
pub mod manifest;
turbopack/crates/turbopack-nodejs/src/ecmascript/node/chunk.rs
+3 -3
View File
@@ -12,7 +12,7 @@ use turbopack_core::{
};
use turbopack_ecmascript::chunk::EcmascriptChunk;
use super::content::EcmascriptBuildNodeChunkContent;
use super::content::EcmascriptNodeChunkContent;
use crate::NodeJsChunkingContext;
/// Production Ecmascript chunk targeting Node.js.
@@ -62,9 +62,9 @@ fn modifier() -> RcStr {
#[turbo_tasks::value_impl]
impl EcmascriptBuildNodeChunk {
#[turbo_tasks::function]
async fn own_content(self: Vc<Self>) -> Result<Vc<EcmascriptBuildNodeChunkContent>> {
async fn own_content(self: Vc<Self>) -> Result<Vc<EcmascriptNodeChunkContent>> {
let this = self.await?;
Ok(EcmascriptBuildNodeChunkContent::new(
Ok(EcmascriptNodeChunkContent::new(
*this.chunking_context,
self,
this.chunk.chunk_content(),
turbopack/crates/turbopack-nodejs/src/ecmascript/node/content.rs
+33 -39
View File
@@ -7,24 +7,23 @@ use turbopack_core::{
code_builder::{Code, CodeBuilder},
output::OutputAsset,
source_map::{GenerateSourceMap, SourceMapAsset},
version::{
MergeableVersionedContent, Update, Version, VersionedContent, VersionedContentMerger,
},
version::{MergeableVersionedContent, Version, VersionedContent, VersionedContentMerger},
};
use turbopack_ecmascript::{
chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries},
hmr::{
EcmascriptHmrChunkContent, merger::EcmascriptChunkContentMerger,
version::EcmascriptChunkVersion,
},
minify::minify,
utils::StringifyJs,
};
use super::{
chunk::EcmascriptBuildNodeChunk, merged::merger::EcmascriptBuildNodeChunkContentMerger,
update::update_node_chunk, version::EcmascriptBuildNodeChunkVersion,
};
use super::chunk::EcmascriptBuildNodeChunk;
use crate::NodeJsChunkingContext;
#[turbo_tasks::value]
pub(super) struct EcmascriptBuildNodeChunkContent {
pub(super) struct EcmascriptNodeChunkContent {
pub(super) content: ResolvedVc<EcmascriptChunkContent>,
pub(super) chunking_context: ResolvedVc<NodeJsChunkingContext>,
pub(super) chunk: ResolvedVc<EcmascriptBuildNodeChunk>,
@@ -32,7 +31,7 @@ pub(super) struct EcmascriptBuildNodeChunkContent {
}
#[turbo_tasks::value_impl]
impl EcmascriptBuildNodeChunkContent {
impl EcmascriptNodeChunkContent {
#[turbo_tasks::function]
pub(crate) fn new(
chunking_context: ResolvedVc<NodeJsChunkingContext>,
@@ -40,7 +39,7 @@ impl EcmascriptBuildNodeChunkContent {
content: ResolvedVc<EcmascriptChunkContent>,
source_map: ResolvedVc<SourceMapAsset>,
) -> Vc<Self> {
EcmascriptBuildNodeChunkContent {
EcmascriptNodeChunkContent {
content,
chunking_context,
chunk,
@@ -48,15 +47,10 @@ impl EcmascriptBuildNodeChunkContent {
}
.cell()
}
#[turbo_tasks::function]
pub(crate) fn entries(&self) -> Vc<EcmascriptChunkContentEntries> {
EcmascriptChunkContentEntries::new(*self.content)
}
}
#[turbo_tasks::value_impl]
impl EcmascriptBuildNodeChunkContent {
impl EcmascriptNodeChunkContent {
#[turbo_tasks::function]
async fn code(&self) -> Result<Vc<Code>> {
use std::io::Write;
@@ -94,20 +88,10 @@ impl EcmascriptBuildNodeChunkContent {
Ok(code.cell())
}
#[turbo_tasks::function]
pub(crate) async fn own_version(&self) -> Result<Vc<EcmascriptBuildNodeChunkVersion>> {
Ok(EcmascriptBuildNodeChunkVersion::new(
self.chunking_context.output_root().owned().await?,
self.chunk.path().owned().await?,
*self.content,
*self.chunking_context.minify_type().await?,
))
}
}
#[turbo_tasks::value_impl]
impl GenerateSourceMap for EcmascriptBuildNodeChunkContent {
impl GenerateSourceMap for EcmascriptNodeChunkContent {
#[turbo_tasks::function]
fn generate_source_map(self: Vc<Self>) -> Vc<FileContent> {
self.code().generate_source_map()
@@ -115,7 +99,7 @@ impl GenerateSourceMap for EcmascriptBuildNodeChunkContent {
}
#[turbo_tasks::value_impl]
impl VersionedContent for EcmascriptBuildNodeChunkContent {
impl VersionedContent for EcmascriptNodeChunkContent {
#[turbo_tasks::function]
async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
let this = self.await?;
@@ -131,22 +115,32 @@ impl VersionedContent for EcmascriptBuildNodeChunkContent {
#[turbo_tasks::function]
fn version(self: Vc<Self>) -> Vc<Box<dyn Version>> {
Vc::upcast(self.own_version())
}
#[turbo_tasks::function]
async fn update(
self: Vc<Self>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> {
Ok(update_node_chunk(self, from_version).await?.cell())
Vc::upcast(self.ecmascript_chunk_version())
}
}
#[turbo_tasks::value_impl]
impl MergeableVersionedContent for EcmascriptBuildNodeChunkContent {
impl EcmascriptHmrChunkContent for EcmascriptNodeChunkContent {
#[turbo_tasks::function]
fn entries(&self) -> Vc<EcmascriptChunkContentEntries> {
EcmascriptChunkContentEntries::new(*self.content)
}
#[turbo_tasks::function]
async fn ecmascript_chunk_version(&self) -> Result<Vc<EcmascriptChunkVersion>> {
Ok(EcmascriptChunkVersion::new(
self.chunking_context.output_root().owned().await?,
self.chunk.path().owned().await?,
*self.content,
*self.chunking_context.minify_type().await?,
))
}
}
#[turbo_tasks::value_impl]
impl MergeableVersionedContent for EcmascriptNodeChunkContent {
#[turbo_tasks::function]
fn get_merger(&self) -> Vc<Box<dyn VersionedContentMerger>> {
Vc::upcast(EcmascriptBuildNodeChunkContentMerger::new())
Vc::upcast(EcmascriptChunkContentMerger::new())
}
}
turbopack/crates/turbopack-nodejs/src/ecmascript/node/merged/content.rs
-55
View File
@@ -1,55 +0,0 @@
use anyhow::{Result, bail};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::version::{Update, Version, VersionedContent};
use super::{
update::update_ecmascript_merged_chunk, version::EcmascriptBuildNodeMergedChunkVersion,
};
use crate::ecmascript::node::content::EcmascriptBuildNodeChunkContent;
/// Composite [`VersionedContent`] that is the result of merging multiple
/// node ecmascript chunk contents together through the
/// [`EcmascriptBuildNodeChunkContentMerger`].
#[turbo_tasks::value(serialization = "skip", shared)]
pub(crate) struct EcmascriptBuildNodeMergedChunkContent {
pub(super) contents: Vec<ResolvedVc<EcmascriptBuildNodeChunkContent>>,
}
#[turbo_tasks::value_impl]
impl EcmascriptBuildNodeMergedChunkContent {
#[turbo_tasks::function]
pub async fn version(&self) -> Result<Vc<EcmascriptBuildNodeMergedChunkVersion>> {
Ok(EcmascriptBuildNodeMergedChunkVersion {
versions: self
.contents
.iter()
.map(|content| async move { content.own_version().await })
.try_join()
.await?,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl VersionedContent for EcmascriptBuildNodeMergedChunkContent {
#[turbo_tasks::function]
fn content(self: Vc<Self>) -> Result<Vc<turbopack_core::asset::AssetContent>> {
bail!("EcmascriptBuildNodeMergedChunkContent does not have content")
}
#[turbo_tasks::function]
fn version(self: Vc<Self>) -> Vc<Box<dyn Version>> {
Vc::upcast(self.version())
}
#[turbo_tasks::function]
async fn update(
self: Vc<Self>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> {
Ok(update_ecmascript_merged_chunk(self, from_version)
.await?
.cell())
}
}
turbopack/crates/turbopack-nodejs/src/ecmascript/node/merged/merger.rs
-51
View File
@@ -1,51 +0,0 @@
use anyhow::{Result, bail};
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::version::{VersionedContent, VersionedContentMerger, VersionedContents};
use crate::ecmascript::node::{
content::EcmascriptBuildNodeChunkContent,
merged::content::EcmascriptBuildNodeMergedChunkContent,
};
/// Merges multiple [`EcmascriptBuildNodeChunkContent`] into a single
/// [`EcmascriptBuildNodeMergedChunkContent`]. This allows the chunk list to
/// produce a single `EcmascriptMergedUpdate` for multiple chunks updating at
/// the same time.
#[turbo_tasks::value]
pub(crate) struct EcmascriptBuildNodeChunkContentMerger;
#[turbo_tasks::value_impl]
impl EcmascriptBuildNodeChunkContentMerger {
#[turbo_tasks::function]
pub fn new() -> Vc<Self> {
Self::cell(EcmascriptBuildNodeChunkContentMerger)
}
}
#[turbo_tasks::value_impl]
impl VersionedContentMerger for EcmascriptBuildNodeChunkContentMerger {
#[turbo_tasks::function]
async fn merge(
&self,
contents: Vc<VersionedContents>,
) -> Result<Vc<Box<dyn VersionedContent>>> {
let contents = contents
.await?
.iter()
.map(|content| async move {
if let Some(content) =
ResolvedVc::try_downcast_type::<EcmascriptBuildNodeChunkContent>(*content)
{
Ok(content)
} else {
bail!("expected Vc<EcmascriptBuildNodeChunkContent>")
}
})
.try_join()
.await?;
Ok(Vc::upcast(
EcmascriptBuildNodeMergedChunkContent { contents }.cell(),
))
}
}
turbopack/crates/turbopack-nodejs/src/ecmascript/node/merged/mod.rs
-4
View File
@@ -1,4 +0,0 @@
pub(crate) mod content;
pub(crate) mod merger;
pub(crate) mod update;
pub(crate) mod version;
turbopack/crates/turbopack-nodejs/src/ecmascript/node/merged/update.rs
-182
View File
@@ -1,182 +0,0 @@
use std::sync::Arc;
use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
chunk::{ChunkingContext, ModuleId},
output::OutputAsset,
version::{PartialUpdate, TotalUpdate, Update, Version},
};
use turbopack_ecmascript::chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
};
use crate::ecmascript::node::{
merged::{
content::EcmascriptBuildNodeMergedChunkContent,
version::EcmascriptBuildNodeMergedChunkVersion,
},
update::{NodeChunkUpdate, update_ecmascript_node_chunk_content},
version::EcmascriptBuildNodeChunkVersion,
};
struct MergedModuleMap {
versions: Vec<ReadRef<EcmascriptBuildNodeChunkVersion>>,
}
impl MergedModuleMap {
fn new(versions: Vec<ReadRef<EcmascriptBuildNodeChunkVersion>>) -> Self {
Self { versions }
}
fn get(&self, id: &ModuleId) -> Option<u128> {
for version in &self.versions {
if let Some(hash) = version.entries_hashes.get(id) {
return Some(*hash);
}
}
None
}
}
pub(crate) async fn update_ecmascript_merged_chunk(
content: Vc<EcmascriptBuildNodeMergedChunkContent>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Update> {
let to_merged_version = content.version();
let from_merged_version = if let Some(from) =
ResolvedVc::try_downcast_type::<EcmascriptBuildNodeMergedChunkVersion>(from_version)
{
from
} else {
return Ok(Update::Total(TotalUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
}));
};
let to = to_merged_version.await?;
let from = from_merged_version.await?;
if from.ptr_eq(&to) {
return Ok(Update::None);
}
let mut from_versions_by_chunk_path: FxIndexMap<_, _> = from
.versions
.iter()
.map(|version| (&*version.chunk_path, version))
.collect();
let merged_module_map = MergedModuleMap::new(from.versions.to_vec());
let content = content.await?;
let to_contents = content
.contents
.iter()
.map(|content| async move {
let entries = content.entries().await?;
let content_ref = content.await?;
let output_root = content_ref.chunking_context.output_root().await?;
let path = content_ref.chunk.path().await?;
Ok((*content, entries, output_root, path))
})
.try_join()
.await?;
let mut merged_update = EcmascriptMergedUpdate::default();
for (content, entries, output_root, path) in &to_contents {
let Some(chunk_path) = output_root.get_path_to(path) else {
continue;
};
let chunk_update = if let Some(from_version) =
from_versions_by_chunk_path.swap_remove(chunk_path)
{
// Reuse the single-chunk diff so the merged path stays in sync with
// the standalone chunk update path.
let to_version = content.own_version().await?;
match update_ecmascript_node_chunk_content(**content, &to_version, from_version).await?
{
NodeChunkUpdate::None => continue,
NodeChunkUpdate::Partial {
added,
modified,
deleted,
} => {
let mut partial = EcmascriptMergedChunkPartial::default();
for (module_id, (module_hash, module_code)) in added {
partial.added.insert(module_id.clone());
// Only ship the code if no other chunk in the group
// already provides this module at the same hash.
if merged_module_map.get(&module_id) != Some(module_hash) {
let entry = EcmascriptModuleEntry::from_code(
&module_id,
module_code,
chunk_path,
)
.await?;
merged_update.entries.insert(module_id, entry);
}
}
partial.deleted.extend(deleted.into_keys());
for (module_id, module_code) in modified {
let entry =
EcmascriptModuleEntry::from_code(&module_id, module_code, chunk_path)
.await?;
merged_update.entries.insert(module_id, entry);
}
EcmascriptMergedChunkUpdate::Partial(partial)
}
}
} else {
let mut added = EcmascriptMergedChunkAdded::default();
for (id, entry) in entries {
let hash = *entry.hash.await?;
added.modules.insert(id.clone());
if merged_module_map.get(id) != Some(hash) {
let entry =
EcmascriptModuleEntry::from_code(id, *entry.code, chunk_path).await?;
merged_update.entries.insert(id.clone(), entry);
}
}
EcmascriptMergedChunkUpdate::Added(added)
};
merged_update.chunks.insert(chunk_path, chunk_update);
}
for (chunk_path, chunk_version) in from_versions_by_chunk_path {
let hashes = &chunk_version.entries_hashes;
merged_update.chunks.insert(
chunk_path,
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: hashes.keys().cloned().collect(),
}),
);
}
let update = if merged_update.is_empty() {
Update::None
} else {
Update::Partial(PartialUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
instruction: Arc::new(serde_json::to_value(&merged_update)?),
})
};
Ok(update)
}
turbopack/crates/turbopack-nodejs/src/ecmascript/node/merged/version.rs
-38
View File
@@ -1,38 +0,0 @@
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{ReadRef, TryJoinIterExt, Vc};
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::version::Version;
use crate::ecmascript::node::version::EcmascriptBuildNodeChunkVersion;
/// The version of a [`super::content::EcmascriptBuildNodeMergedChunkContent`].
#[turbo_tasks::value(serialization = "skip", shared)]
pub(crate) struct EcmascriptBuildNodeMergedChunkVersion {
pub(super) versions: Vec<ReadRef<EcmascriptBuildNodeChunkVersion>>,
}
#[turbo_tasks::value_impl]
impl Version for EcmascriptBuildNodeMergedChunkVersion {
#[turbo_tasks::function]
async fn id(&self) -> Result<Vc<RcStr>> {
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_value(self.versions.len());
let sorted_ids = {
let mut sorted_ids = self
.versions
.iter()
.map(|version| ReadRef::cell(version.clone()).id())
.try_join()
.await?;
sorted_ids.sort();
sorted_ids
};
for id in sorted_ids {
hasher.write_value(id);
}
let hash = hasher.finish();
let hash = encode_base64(hash);
Ok(Vc::cell(hash.into()))
}
}
turbopack/crates/turbopack-nodejs/src/ecmascript/node/mod.rs
-3
View File
@@ -1,6 +1,3 @@
pub(crate) mod chunk;
pub(crate) mod content;
pub mod entry;
pub(crate) mod merged;
pub(crate) mod update;
pub(crate) mod version;
turbopack/crates/turbopack-nodejs/src/ecmascript/node/update.rs
-168
View File
@@ -1,168 +0,0 @@
use std::sync::Arc;
use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, Vc};
use turbopack_core::{
chunk::ModuleId,
code_builder::Code,
version::{PartialUpdate, TotalUpdate, Update, Version},
};
use turbopack_ecmascript::chunk_list::merged_update::{
EcmascriptMergedChunkPartial, EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate,
EcmascriptModuleEntry,
};
use super::{content::EcmascriptBuildNodeChunkContent, version::EcmascriptBuildNodeChunkVersion};
pub(super) async fn update_node_chunk(
content: Vc<EcmascriptBuildNodeChunkContent>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Update> {
let to_version = content.own_version();
let from_version = if let Some(from) =
ResolvedVc::try_downcast_type::<EcmascriptBuildNodeChunkVersion>(from_version)
{
from
} else {
// It's likely `from_version` is `NotFoundVersion`.
return Ok(Update::Total(TotalUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_version)
.into_trait_ref()
.await?,
}));
};
let to = to_version.await?;
let from = from_version.await?;
// When to and from point to the same value we can skip comparing them
if from.ptr_eq(&to) {
return Ok(Update::None);
}
let chunk_path = to.chunk_path.as_str();
let chunk_update = update_ecmascript_node_chunk_content(content, &to, &from).await?;
let mut merged_update = EcmascriptMergedUpdate::default();
match chunk_update {
NodeChunkUpdate::None => {
return Ok(Update::None);
}
NodeChunkUpdate::Partial {
added,
modified,
deleted,
} => {
let mut partial = EcmascriptMergedChunkPartial::default();
for (module_id, (_hash, module_code)) in added {
partial.added.insert(module_id.clone());
let entry =
EcmascriptModuleEntry::from_code(&module_id, module_code, chunk_path).await?;
merged_update.entries.insert(module_id, entry);
}
partial.deleted.extend(deleted.into_keys());
for (module_id, module_code) in modified {
let entry =
EcmascriptModuleEntry::from_code(&module_id, module_code, chunk_path).await?;
merged_update.entries.insert(module_id, entry);
}
merged_update
.chunks
.insert(chunk_path, EcmascriptMergedChunkUpdate::Partial(partial));
}
}
let update = if merged_update.is_empty() {
Update::None
} else {
// Serialize EcmascriptMergedUpdate directly
// The hot-reloader will wrap it in ChunkListUpdate format for the runtime
let instruction_value = serde_json::to_value(&merged_update)?;
Update::Partial(PartialUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_version)
.into_trait_ref()
.await?,
instruction: Arc::new(instruction_value),
})
};
Ok(update)
}
pub(crate) enum NodeChunkUpdate {
None,
Partial {
/// Added modules, keyed by id, with their content hash (used by the
/// merged-chunk path to dedup code emission across chunks) and code.
added: FxIndexMap<ModuleId, (u128, Vc<Code>)>,
modified: FxIndexMap<ModuleId, Vc<Code>>,
deleted: FxIndexMap<ModuleId, u128>,
},
}
pub(super) async fn update_ecmascript_node_chunk_content(
content: Vc<EcmascriptBuildNodeChunkContent>,
to: &ReadRef<EcmascriptBuildNodeChunkVersion>,
from: &ReadRef<EcmascriptBuildNodeChunkVersion>,
) -> Result<NodeChunkUpdate> {
let mut added = FxIndexMap::default();
let mut modified = FxIndexMap::default();
let mut deleted = FxIndexMap::default();
// Lazily resolve the entries map only when we actually need to ship code
// bytes for an added or modified module. For chunks that only have deletions
// (or no changes that need code beyond hashes), this avoids materializing
// any `Vc<Code>`.
let mut entries_ref = None;
// Check for deleted and modified modules
for (id, from_hash) in &from.entries_hashes {
if let Some(to_hash) = to.entries_hashes.get(id) {
if *to_hash != *from_hash {
// Module was modified
let entries = match &entries_ref {
Some(entries) => entries,
None => entries_ref.insert(content.entries().await?),
};
if let Some(entry) = entries.get(id) {
modified.insert(id.clone(), *entry.code);
}
}
} else {
// Module was deleted
deleted.insert(id.clone(), *from_hash);
}
}
// Check for added modules
for (id, hash) in &to.entries_hashes {
if !from.entries_hashes.contains_key(id) {
let entries = match &entries_ref {
Some(entries) => entries,
None => entries_ref.insert(content.entries().await?),
};
if let Some(entry) = entries.get(id) {
added.insert(id.clone(), (*hash, *entry.code));
}
}
}
let update = if added.is_empty() && modified.is_empty() && deleted.is_empty() {
NodeChunkUpdate::None
} else {
NodeChunkUpdate::Partial {
added,
modified,
deleted,
}
};
Ok(update)
}
turbopack/crates/turbopack-nodejs/src/ecmascript/node/version.rs
-72
View File
@@ -1,72 +0,0 @@
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::{
chunk::{MinifyType, ModuleId},
version::Version,
};
use turbopack_ecmascript::chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries};
#[turbo_tasks::value(serialization = "skip")]
pub(super) struct EcmascriptBuildNodeChunkVersion {
pub(super) chunk_path: RcStr,
pub(super) minify_type: MinifyType,
pub(super) entries_hashes: FxIndexMap<ModuleId, u128>,
}
#[turbo_tasks::value_impl]
impl EcmascriptBuildNodeChunkVersion {
#[turbo_tasks::function]
pub async fn new(
output_root: FileSystemPath,
chunk_path: FileSystemPath,
content: Vc<EcmascriptChunkContent>,
minify_type: MinifyType,
) -> Result<Vc<Self>> {
let output_root = output_root.clone();
let chunk_path = chunk_path.clone();
let chunk_path = if let Some(path) = output_root.get_path_to(&chunk_path) {
path
} else {
turbobail!("chunk path {chunk_path} is not in client root {output_root}");
};
let entries_hashes = EcmascriptChunkContentEntries::new(content)
.await?
.iter()
.map(async |(id, entry)| Ok((id.clone(), *entry.hash.await?)))
.try_join()
.await?
.into_iter()
.collect();
Ok(EcmascriptBuildNodeChunkVersion {
chunk_path: chunk_path.into(),
minify_type,
entries_hashes,
}
.cell())
}
}
#[turbo_tasks::value_impl]
impl Version for EcmascriptBuildNodeChunkVersion {
#[turbo_tasks::function]
fn id(&self) -> Vc<RcStr> {
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_ref(&self.chunk_path);
hasher.write_ref(&self.minify_type);
let sorted_hashes = {
let mut hashes: Vec<_> = self.entries_hashes.values().copied().collect();
hashes.sort();
hashes
};
for hash in sorted_hashes {
hasher.write_value(hash);
}
let hash = hasher.finish();
let hash = encode_base64(hash);
Vc::cell(hash.into())
}
}