hanzoai/next.js
2
0
Fork 0
mirror of https://github.com/hanzoai/next.js.git synced 2026-09-24 17:00:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Turbopack: Share entrypoint template logic between Turbopack/webpack (#82385)

Browse Source 
## Goal

With three different bundlers, we're often forced to duplicate next.js-specific logic and configuration across two or three different bundlers.

E.g. https://github.com/SyMind/next-rspack-binding/ would cause us to duplicate our current externals logic across all three.

To help, we'd like to have a shared crate that can be used in a few different ways:

- Via turbopack (which is itself called through native napi bindings)
- Via direct native napi bindings to JS for webpack
- Via wasm bindings to JS for webpack [*(note: ideally these would be replaced with napi's recent wasm support, but it doesn't make sense to do that until after the napi v3 migration is done)*](https://napi.rs/blog/announce-v3#webassembly)
- Via Rspack (we'd build a custom binary)

The wasm bindings (in their current state) and Rspack cannot (or should not) depend on `turbo-tasks`. However, all of the current `next-*` crates depend on `turbo-tasks`.

This introduces a `next-taskless` crate for code that can be isolated from performing IO and that does not need to depend on `turbo-tasks`. Code in this crate can be used from wasm or Rspack.

## This PR

As a proof-of-concept, this deduplicates our next.js entrypoint templating logic across webpack and Turbopack. It's a large function that doesn't need to directly depend on file IO.

Some path manipulation had to be moved out of `turbo-tasks-fs` to support this, so that code is in `turbo-unix-path`.

I'm happy to accept different bikeshedded crate names.

## Testing

Tested wasm with:

```
pnpm i
pnpm swc-build-wasm --target nodejs
pnpm build
NODE_TEST_MODE=start NEXT_TEST_WASM=true NEXT_SKIP_NATIVE_POSTINSTALL=1 NEXT_TEST_PREFER_OFFLINE=1 node run-tests.js test/production/pages-dir/production/test/index.test.ts
```

Tested turbopack with:

```
pnpm i
pnpm swc-build-native
pnpm build
pnpm test-start-turbo test/production/pages-dir/production/test/index.test.ts
```

Tested webpack with:

```
pnpm i
pnpm swc-build-native
pnpm build
pnpm test-start test/production/pages-dir/production/test/index.test.ts
```
This commit is contained in:
Benjamin Woodruff
2025-08-07 19:26:48 -07:00
committed by GitHub
parent c26cb69e03
commit 4eb46d8ea5
35 changed files with 820 additions and 746 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+26
View File
@@ -4280,6 +4280,7 @@ dependencies = [
"turbo-tasks-env",
"turbo-tasks-fs",
"turbo-tasks-malloc",
"turbo-unix-path",
"turbopack",
"turbopack-browser",
"turbopack-core",
@@ -4340,6 +4341,7 @@ dependencies = [
"mime_guess",
"modularize_imports",
"next-custom-transforms",
"next-taskless",
"once_cell",
"percent-encoding",
"qstring",
@@ -4431,6 +4433,7 @@ dependencies = [
"next-build",
"next-core",
"next-custom-transforms",
"next-taskless",
"once_cell",
"owo-colors 3.5.0",
"rand 0.9.0",
@@ -4450,6 +4453,7 @@ dependencies = [
"turbo-tasks-build",
"turbo-tasks-fs",
"turbo-tasks-malloc",
"turbo-unix-path",
"turbopack-core",
"turbopack-ecmascript-hmr-protocol",
"turbopack-ecmascript-plugins",
@@ -4460,6 +4464,16 @@ dependencies = [
"vergen-gitcl",
]
[[package]]
name = "next-taskless"
version = "0.0.1"
dependencies = [
"anyhow",
"regex",
"serde_json",
"turbo-unix-path",
]
[[package]]
name = "nix"
version = "0.26.4"
@@ -9372,6 +9386,7 @@ dependencies = [
"turbo-tasks-build",
"turbo-tasks-hash",
"turbo-tasks-testing",
"turbo-unix-path",
"urlencoding",
]
@@ -9455,6 +9470,13 @@ dependencies = [
"turbo-tasks",
]
[[package]]
name = "turbo-unix-path"
version = "0.0.1"
dependencies = [
"rstest",
]
[[package]]
name = "turbopack"
version = "0.1.0"
@@ -9638,6 +9660,7 @@ dependencies = [
"turbo-tasks-fs",
"turbo-tasks-hash",
"turbo-tasks-testing",
"turbo-unix-path",
"urlencoding",
]
@@ -10011,6 +10034,7 @@ dependencies = [
"turbo-tasks-bytes",
"turbo-tasks-env",
"turbo-tasks-fs",
"turbo-unix-path",
"turbopack",
"turbopack-browser",
"turbopack-core",
@@ -10567,6 +10591,8 @@ dependencies = [
"js-sys",
"mdxjs",
"next-custom-transforms",
"next-taskless",
"rustc-hash 2.1.1",
"serde-wasm-bindgen 0.4.5",
"serde_json",
"swc_core",
Cargo.toml
+3
View File
@@ -10,6 +10,7 @@ members = [
"crates/next-build",
"crates/next-core",
"crates/next-custom-transforms",
"crates/next-taskless",
"turbopack/crates/*",
"turbopack/crates/*/fuzz",
"turbopack/xtask",
@@ -252,6 +253,7 @@ next-api = { path = "crates/next-api" }
next-build = { path = "crates/next-build" }
next-core = { path = "crates/next-core" }
next-custom-transforms = { path = "crates/next-custom-transforms" }
next-taskless = { path = "crates/next-taskless" }
# Turbopack
auto-hash-map = { path = "turbopack/crates/turbo-tasks-auto-hash-map" }
@@ -260,6 +262,7 @@ turbo-rcstr = { path = "turbopack/crates/turbo-rcstr" }
turbo-dyn-eq-hash = { path = "turbopack/crates/turbo-dyn-eq-hash" }
turbo-esregex = { path = "turbopack/crates/turbo-esregex" }
turbo-persistence = { path = "turbopack/crates/turbo-persistence" }
turbo-unix-path = { path = "turbopack/crates/turbo-unix-path" }
turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }
crates/napi/Cargo.toml
+2
View File
@@ -64,6 +64,7 @@ owo-colors = { workspace = true }
napi = { workspace = true }
napi-derive = "2"
next-custom-transforms = { workspace = true }
next-taskless = { workspace = true }
rand = { workspace = true }
rustc-hash = { workspace = true }
serde = "1"
@@ -105,6 +106,7 @@ turbo-rcstr = { workspace = true, features = ["napi"] }
turbo-tasks = { workspace = true }
turbo-tasks-backend = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-unix-path = { workspace = true }
next-api = { workspace = true }
next-build = { workspace = true }
next-core = { workspace = true }
crates/napi/src/next_api/project.rs
+2 -2
View File
@@ -39,9 +39,9 @@ use turbo_tasks::{
};
use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystem, FileSystemPath, get_relative_path_to,
util::uri_from_file,
DiskFileSystem, FileContent, FileSystem, FileSystemPath, util::uri_from_file,
};
use turbo_unix_path::get_relative_path_to;
use turbopack_core::{
PROJECT_FILESYSTEM_NAME, SOURCE_URL_PROTOCOL,
diagnostics::PlainDiagnostic,
crates/napi/src/next_api/utils.rs
+23 -1
View File
@@ -4,7 +4,7 @@ use anyhow::{Context, Result, anyhow};
use futures_util::TryFutureExt;
use napi::{
JsFunction, JsObject, JsUnknown, NapiRaw, NapiValue, Status,
bindgen_prelude::{External, ToNapiValue},
bindgen_prelude::{Buffer, External, ToNapiValue},
threadsafe_function::{ThreadSafeCallContext, ThreadsafeFunction, ThreadsafeFunctionCallMode},
};
use rustc_hash::FxHashMap;
@@ -380,3 +380,25 @@ pub async fn strongly_consistent_catch_collectables<R: VcValueType + Send>(
Ok((result, issues, diagnostics, effects))
}
#[napi]
pub fn expand_next_js_template(
content: Buffer,
template_path: String,
next_package_dir_path: String,
#[napi(ts_arg_type = "Record<string, string>")] replacements: FxHashMap<String, String>,
#[napi(ts_arg_type = "Record<string, string>")] injections: FxHashMap<String, String>,
#[napi(ts_arg_type = "Record<string, string | null>")] imports: FxHashMap<
String,
Option<String>,
>,
) -> napi::Result<String> {
Ok(next_taskless::expand_next_js_template(
str::from_utf8(&content).context("template content must be valid utf-8")?,
&template_path,
&next_package_dir_path,
replacements.iter().map(|(k, v)| (&**k, &**v)),
injections.iter().map(|(k, v)| (&**k, &**v)),
imports.iter().map(|(k, v)| (&**k, v.as_deref())),
)?)
}
crates/next-api/Cargo.toml
+1
View File
@@ -33,6 +33,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true, features = ["non_operation_vc_strongly_consistent"] }
turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-unix-path = { workspace = true }
turbopack = { workspace = true }
turbopack-browser = { workspace = true }
turbopack-core = { workspace = true }
crates/next-api/src/project.rs
+2 -4
View File
@@ -36,10 +36,8 @@ use turbo_tasks::{
trace::TraceRawVcs,
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{
DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, invalidation,
util::{join_path, unix_to_sys},
};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, invalidation};
use turbo_unix_path::{join_path, unix_to_sys};
use turbopack::{
ModuleAssetContext, evaluate_context::node_build_environment,
global_module_ids::get_global_module_id_strategy, transition::TransitionOptions,
crates/next-core/Cargo.toml
+3 -1
View File
@@ -16,7 +16,6 @@ anyhow = { workspace = true }
async-trait = { workspace = true }
base64 = "0.21.0"
either = { workspace = true }
next-custom-transforms = { workspace = true }
once_cell = { workspace = true }
qstring = { workspace = true }
regex = { workspace = true }
@@ -55,6 +54,9 @@ swc_core = { workspace = true, features = [
] }
modularize_imports = { workspace = true }
next-custom-transforms = { workspace = true }
next-taskless = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-esregex = { workspace = true }
turbo-tasks = { workspace = true }
crates/next-core/src/middleware.rs
+9 -9
View File
@@ -1,6 +1,6 @@
use anyhow::Result;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexMap, ResolvedVc, Vc, fxindexmap};
use turbo_tasks::{ResolvedVc, Vc, fxindexmap};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{context::AssetContext, module::Module, reference_type::ReferenceType};
@@ -27,23 +27,23 @@ pub async fn get_middleware_module(
project_root: FileSystemPath,
userland_module: ResolvedVc<Box<dyn Module>>,
) -> Result<Vc<Box<dyn Module>>> {
let inner = rcstr!("INNER_MIDDLEWARE_MODULE");
const INNER: &str = "INNER_MIDDLEWARE_MODULE";
// Load the file from the next.js codebase.
let source = load_next_js_template(
"middleware.js",
project_root,
fxindexmap! {
"VAR_USERLAND" => inner.clone(),
"VAR_DEFINITION_PAGE" => rcstr!("/middleware"),
},
FxIndexMap::default(),
FxIndexMap::default(),
&[
("VAR_USERLAND", INNER),
("VAR_DEFINITION_PAGE", "/middleware"),
],
&[],
&[],
)
.await?;
let inner_assets = fxindexmap! {
inner => userland_module
rcstr!(INNER) => userland_module
};
let module = asset_context
crates/next-core/src/next_app/app_page_entry.rs
+25 -27
View File
@@ -1,7 +1,7 @@
use std::io::Write;
use anyhow::Result;
use turbo_rcstr::{RcStr, rcstr};
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, fxindexmap};
use turbo_tasks_fs::{self, File, FileSystemPath, rope::RopeBuilder};
use turbopack::ModuleAssetContext;
@@ -86,22 +86,25 @@ pub async fn get_app_page_entry(
let source = load_next_js_template(
"app-page.js",
project_root.clone(),
fxindexmap! {
"VAR_DEFINITION_PAGE" => page.to_string().into(),
"VAR_DEFINITION_PATHNAME" => pathname.clone(),
"VAR_MODULE_GLOBAL_ERROR" => if inner_assets.contains_key(GLOBAL_ERROR) {
GLOBAL_ERROR.into()
} else {
rcstr!("next/dist/client/components/builtin/global-error")
},
},
fxindexmap! {
"tree" => loader_tree_code,
"pages" => StringifyJs(&pages).to_string().into(),
"__next_app_require__" => TURBOPACK_REQUIRE.bound().into(),
"__next_app_load_chunk__" => TURBOPACK_LOAD.bound().into(),
},
fxindexmap! {},
&[
("VAR_DEFINITION_PAGE", &*page.to_string()),
("VAR_DEFINITION_PATHNAME", &pathname),
(
"VAR_MODULE_GLOBAL_ERROR",
if inner_assets.contains_key(GLOBAL_ERROR) {
GLOBAL_ERROR
} else {
"next/dist/client/components/builtin/global-error"
},
),
],
&[
("tree", &*loader_tree_code),
("pages", &StringifyJs(&pages).to_string()),
("__next_app_require__", &TURBOPACK_REQUIRE.bound()),
("__next_app_load_chunk__", &TURBOPACK_LOAD.bound()),
],
&[],
)
.await?;
@@ -158,18 +161,13 @@ async fn wrap_edge_page(
let source = load_next_js_template(
"edge-ssr-app.js",
project_root.clone(),
fxindexmap! {
"VAR_USERLAND" => INNER.into(),
"VAR_PAGE" => page.to_string().into(),
},
fxindexmap! {
&[("VAR_USERLAND", INNER), ("VAR_PAGE", &page.to_string())],
&[
// TODO do we really need to pass the entire next config here?
// This is bad for invalidation as any config change will invalidate this
"nextConfig" => serde_json::to_string(next_config_val)?.into(),
},
fxindexmap! {
"incrementalCacheHandler" => None,
},
("nextConfig", &*serde_json::to_string(next_config_val)?),
],
&[("incrementalCacheHandler", None)],
)
.await?;
crates/next-core/src/next_app/app_route_entry.rs
+17 -24
View File
@@ -59,33 +59,31 @@ pub async fn get_app_route_entry(
let inner = rcstr!("INNER_APP_ROUTE");
let output_type: RcStr = next_config
let output_type: &str = next_config
.await?
.output
.as_ref()
.map(|o| match o {
OutputType::Standalone => rcstr!("\"standalone\""),
OutputType::Export => rcstr!("\"export\""),
OutputType::Standalone => "\"standalone\"",
OutputType::Export => "\"export\"",
})
.unwrap_or(rcstr!("\"\""));
.unwrap_or("\"\"");
// Load the file from the next.js codebase.
let virtual_source = load_next_js_template(
"app-route.js",
project_root.clone(),
fxindexmap! {
"VAR_DEFINITION_PAGE" => page.to_string().into(),
"VAR_DEFINITION_PATHNAME" => pathname.clone(),
"VAR_DEFINITION_FILENAME" => path.file_stem().unwrap().into(),
&[
("VAR_DEFINITION_PAGE", &*page.to_string()),
("VAR_DEFINITION_PATHNAME", &pathname),
("VAR_DEFINITION_FILENAME", path.file_stem().unwrap()),
// TODO(alexkirsz) Is this necessary?
"VAR_DEFINITION_BUNDLE_PATH" => "".to_string().into(),
"VAR_RESOLVED_PAGE_PATH" => path.value_to_string().owned().await?,
"VAR_USERLAND" => inner.clone(),
},
fxindexmap! {
"nextConfigOutput" => output_type
},
fxindexmap! {},
("VAR_DEFINITION_BUNDLE_PATH", ""),
("VAR_RESOLVED_PAGE_PATH", &path.value_to_string().await?),
("VAR_USERLAND", &inner),
],
&[("nextConfigOutput", output_type)],
&[],
)
.await?;
@@ -143,14 +141,9 @@ async fn wrap_edge_route(
let source = load_next_js_template(
"edge-app-route.js",
project_root.clone(),
fxindexmap! {
"VAR_USERLAND" => inner.clone(),
"VAR_PAGE" => page.to_string().into(),
},
fxindexmap! {
"nextConfig" => serde_json::to_string(next_config)?.into(),
},
fxindexmap! {},
&[("VAR_USERLAND", &*inner), ("VAR_PAGE", &page.to_string())],
&[("nextConfig", &*serde_json::to_string(next_config)?)],
&[],
)
.await?;
crates/next-core/src/next_pages/page_entry.rs
+42 -33
View File
@@ -3,7 +3,7 @@ use std::io::Write;
use anyhow::{Result, bail};
use serde::Serialize;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexMap, ResolvedVc, Vc, fxindexmap};
use turbo_tasks::{ResolvedVc, Vc, fxindexmap};
use turbo_tasks_fs::{File, FileSystemPath, rope::RopeBuilder};
use turbopack_core::{
asset::{Asset, AssetContent},
@@ -71,26 +71,20 @@ pub async fn create_page_ssr_entry_module(
let inner_document = rcstr!("INNER_DOCUMENT");
let inner_app = rcstr!("INNER_APP");
let mut replacements = fxindexmap! {
"VAR_DEFINITION_PAGE" => definition_page.clone(),
"VAR_DEFINITION_PATHNAME" => definition_pathname.clone(),
"VAR_USERLAND" => inner.clone(),
};
let mut replacements = vec![
("VAR_DEFINITION_PAGE", &*definition_page),
("VAR_DEFINITION_PATHNAME", &definition_pathname),
("VAR_USERLAND", &inner),
];
if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page) {
replacements.insert("VAR_MODULE_DOCUMENT", inner_document.clone());
replacements.insert("VAR_MODULE_APP", inner_app.clone());
replacements.push(("VAR_MODULE_DOCUMENT", &inner_document));
replacements.push(("VAR_MODULE_APP", &inner_app));
}
// Load the file from the next.js codebase.
let mut source = load_next_js_template(
template_file,
project_root.clone(),
replacements,
FxIndexMap::default(),
FxIndexMap::default(),
)
.await?;
let mut source =
load_next_js_template(template_file, project_root.clone(), &replacements, &[], &[]).await?;
// When we're building the instrumentation page (only when the
// instrumentation file conflicts with a page also labeled
@@ -215,26 +209,41 @@ async fn wrap_edge_page(
let source = load_next_js_template(
"edge-ssr.js",
project_root.clone(),
fxindexmap! {
"VAR_USERLAND" => INNER.into(),
"VAR_PAGE" => pathname.clone(),
"VAR_MODULE_DOCUMENT" => INNER_DOCUMENT.into(),
"VAR_MODULE_APP" => INNER_APP.into(),
"VAR_MODULE_GLOBAL_ERROR" => INNER_ERROR.into(),
},
fxindexmap! {
&[
("VAR_USERLAND", INNER),
("VAR_PAGE", &pathname),
("VAR_MODULE_DOCUMENT", INNER_DOCUMENT),
("VAR_MODULE_APP", INNER_APP),
("VAR_MODULE_GLOBAL_ERROR", INNER_ERROR),
],
&[
// TODO do we really need to pass the entire next config here?
// This is bad for invalidation as any config change will invalidate this
"nextConfig" => serde_json::to_string(next_config_val)?.into(),
"pageRouteModuleOptions" => serde_json::to_string(&get_route_module_options(page.clone(), pathname.clone()))?.into(),
"errorRouteModuleOptions" => serde_json::to_string(&get_route_module_options(rcstr!("/_error"), rcstr!("/_error")))?.into(),
"user500RouteModuleOptions" => serde_json::to_string(&get_route_module_options(rcstr!("/500"), rcstr!("/500")))?.into(),
},
fxindexmap! {
("nextConfig", &*serde_json::to_string(next_config_val)?),
(
"pageRouteModuleOptions",
&serde_json::to_string(&get_route_module_options(page.clone(), pathname.clone()))?,
),
(
"errorRouteModuleOptions",
&serde_json::to_string(&get_route_module_options(
rcstr!("/_error"),
rcstr!("/_error"),
))?,
),
(
"user500RouteModuleOptions",
&serde_json::to_string(&get_route_module_options(rcstr!("/500"), rcstr!("/500")))?,
),
],
&[
// TODO
"incrementalCacheHandler" => None,
"userland500Page" => pages_structure.await?.error_500.map(|_| INNER_ERROR_500.into()),
},
("incrementalCacheHandler", None),
(
"userland500Page",
pages_structure.await?.error_500.map(|_| INNER_ERROR_500),
),
],
)
.await?;
crates/next-core/src/util.rs
+25 -245
View File
@@ -1,7 +1,7 @@
use std::{str::FromStr, sync::LazyLock};
use std::{future::Future, str::FromStr};
use anyhow::{Context, Result, bail};
use regex::Regex;
use anyhow::{Result, bail};
use next_taskless::expand_next_js_template;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use swc_core::{
common::{GLOBALS, Spanned, source_map::SmallPos},
@@ -9,12 +9,12 @@ use swc_core::{
};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
FxIndexMap, FxIndexSet, NonLocalValue, ResolvedVc, TaskInput, ValueDefault, Vc,
trace::TraceRawVcs, util::WrapFuture,
FxIndexMap, NonLocalValue, ResolvedVc, TaskInput, ValueDefault, Vc, trace::TraceRawVcs,
util::WrapFuture,
};
use turbo_tasks_fs::{
self, File, FileContent, FileSystem, FileSystemPath, json::parse_json_rope_with_source_context,
rope::Rope, util::join_path,
rope::Rope,
};
use turbopack_core::{
asset::AssetContent,
@@ -32,7 +32,6 @@ use turbopack_ecmascript::{
EcmascriptParsable,
analyzer::{ConstantValue, JsValue, ObjectPart},
parse::ParseResult,
utils::StringifyJs,
};
use crate::{
@@ -725,250 +724,31 @@ async fn parse_config_from_js_value(
/// Loads a next.js template, replaces `replacements` and `injections` and makes
/// sure there are none left over.
pub async fn load_next_js_template(
path: &str,
template_path: &str,
project_path: FileSystemPath,
replacements: FxIndexMap<&'static str, RcStr>,
injections: FxIndexMap<&'static str, RcStr>,
imports: FxIndexMap<&'static str, Option<RcStr>>,
replacements: &[(&str, &str)],
injections: &[(&str, &str)],
imports: &[(&str, Option<&str>)],
) -> Result<Vc<Box<dyn Source>>> {
let path = virtual_next_js_template_path(project_path.clone(), path.to_string()).await?;
let template_path = virtual_next_js_template_path(project_path.clone(), template_path).await?;
let content = &*file_content_rope(path.read()).await?;
let content = content.to_str()?.into_owned();
let content = file_content_rope(template_path.read()).await?;
let content = content.to_str()?;
let parent_path = path.parent();
let parent_path_value = parent_path.clone();
let package_root = &*get_next_package(project_path).await?;
let package_root = get_next_package(project_path).await?.parent();
let package_root_value = package_root.clone();
/// See [regex::Regex::replace_all].
fn replace_all<E>(
re: &regex::Regex,
haystack: &str,
mut replacement: impl FnMut(&regex::Captures) -> Result<String, E>,
) -> Result<String, E> {
let mut new = String::with_capacity(haystack.len());
let mut last_match = 0;
for caps in re.captures_iter(haystack) {
let m = caps.get(0).unwrap();
new.push_str(&haystack[last_match..m.start()]);
new.push_str(&replacement(&caps)?);
last_match = m.end();
}
new.push_str(&haystack[last_match..]);
Ok(new)
}
// Update the relative imports to be absolute. This will update any relative
// imports to be relative to the root of the `next` package.
static IMPORT_PATH_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new("(?:from '(\\..*)'|import '(\\..*)')").unwrap());
let mut count = 0;
let mut content = replace_all(&IMPORT_PATH_RE, &content, |caps| {
let from_request = caps.get(1).map_or("", |c| c.as_str());
let import_request = caps.get(2).map_or("", |c| c.as_str());
count += 1;
let is_from_request = !from_request.is_empty();
let imported = FileSystemPath {
fs: package_root_value.fs,
path: join_path(
&parent_path_value.path,
if is_from_request {
from_request
} else {
import_request
},
)
.context("path should not leave the fs")?
.into(),
};
let relative = package_root_value
.get_relative_path_to(&imported)
.context("path has to be relative to package root")?;
if !relative.starts_with("./next/") {
bail!(
"Invariant: Expected relative import to start with \"./next/\", found \"{}\"",
relative
)
}
let relative = relative
.strip_prefix("./")
.context("should be able to strip the prefix")?;
Ok(if is_from_request {
format!("from {}", StringifyJs(relative))
} else {
format!("import {}", StringifyJs(relative))
})
})
.context("replacing imports failed")?;
// Verify that at least one import was replaced. It's the case today where
// every template file has at least one import to update, so this ensures that
// we don't accidentally remove the import replacement code or use the wrong
// template file.
if count == 0 {
bail!("Invariant: Expected to replace at least one import")
}
// Replace all the template variables with the actual values. If a template
// variable is missing, throw an error.
let mut replaced = FxIndexSet::default();
for (key, replacement) in &replacements {
let full = format!("'{key}'");
if content.contains(&full) {
replaced.insert(*key);
content = content.replace(&full, &StringifyJs(&replacement).to_string());
}
}
// Check to see if there's any remaining template variables.
static TEMPLATE_VAR_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new("VAR_[A-Z_]+").unwrap());
let mut matches = TEMPLATE_VAR_RE.find_iter(&content).peekable();
if matches.peek().is_some() {
bail!(
"Invariant: Expected to replace all template variables, found {}",
matches.map(|m| m.as_str()).collect::<Vec<_>>().join(", "),
)
}
// Check to see if any template variable was provided but not used.
if replaced.len() != replacements.len() {
// Find the difference between the provided replacements and the replaced
// template variables. This will let us notify the user of any template
// variables that were not used but were provided.
let difference = replacements
.keys()
.filter(|k| !replaced.contains(*k))
.copied()
.collect::<Vec<_>>();
bail!(
"Invariant: Expected to replace all template variables, missing {} in template",
difference.join(", "),
)
}
// Replace the injections.
let mut injected = FxIndexSet::default();
for (key, injection) in &injections {
let full = format!("// INJECT:{key}");
if content.contains(&full) {
// Track all the injections to ensure that we're not missing any.
injected.insert(*key);
content = content.replace(&full, &format!("const {key} = {injection}"));
}
}
// Check to see if there's any remaining injections.
static INJECT_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new("// INJECT:[A-Za-z0-9_]+").unwrap());
let mut matches = INJECT_RE.find_iter(&content).peekable();
if matches.peek().is_some() {
bail!(
"Invariant: Expected to inject all injections, found {}",
matches.map(|m| m.as_str()).collect::<Vec<_>>().join(", "),
)
}
// Check to see if any injection was provided but not used.
if injected.len() != injections.len() {
// Find the difference between the provided replacements and the replaced
// template variables. This will let us notify the user of any template
// variables that were not used but were provided.
let difference = injections
.keys()
.filter(|k| !injected.contains(*k))
.copied()
.collect::<Vec<_>>();
bail!(
"Invariant: Expected to inject all injections, missing {} in template",
difference.join(", "),
)
}
// Replace the optional imports.
let mut imports_added = FxIndexSet::default();
for (key, import_path) in &imports {
let mut full = format!("// OPTIONAL_IMPORT:{key}");
let namespace = if !content.contains(&full) {
full = format!("// OPTIONAL_IMPORT:* as {key}");
if content.contains(&full) {
true
} else {
continue;
}
} else {
false
};
// Track all the imports to ensure that we're not missing any.
imports_added.insert(*key);
if let Some(path) = import_path {
content = content.replace(
&full,
&format!(
"import {}{} from {}",
if namespace { "* as " } else { "" },
key,
&StringifyJs(&path).to_string()
),
);
} else {
content = content.replace(&full, &format!("const {key} = null"));
}
}
// Check to see if there's any remaining imports.
static OPTIONAL_IMPORT_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new("// OPTIONAL_IMPORT:(\\* as )?[A-Za-z0-9_]+").unwrap());
let mut matches = OPTIONAL_IMPORT_RE.find_iter(&content).peekable();
if matches.peek().is_some() {
bail!(
"Invariant: Expected to inject all imports, found {}",
matches.map(|m| m.as_str()).collect::<Vec<_>>().join(", "),
)
}
// Check to see if any import was provided but not used.
if imports_added.len() != imports.len() {
// Find the difference between the provided imports and the injected
// imports. This will let us notify the user of any imports that were
// not used but were provided.
let difference = imports
.keys()
.filter(|k| !imports_added.contains(*k))
.cloned()
.collect::<Vec<_>>();
bail!(
"Invariant: Expected to inject all imports, missing {} in template",
difference.join(", "),
)
}
// Ensure that the last line is a newline.
if !content.ends_with('\n') {
content.push('\n');
}
let content = expand_next_js_template(
&content,
&template_path.path,
&package_root.path,
replacements.iter().copied(),
injections.iter().copied(),
imports.iter().copied(),
)?;
let file = File::from(content);
let source = VirtualSource::new(path, AssetContent::file(file.into()));
let source = VirtualSource::new(template_path, AssetContent::file(file.into()));
Ok(Vc::upcast(source))
}
@@ -984,9 +764,9 @@ pub async fn file_content_rope(content: Vc<FileContent>) -> Result<Vc<Rope>> {
Ok(file.content().to_owned().cell())
}
pub async fn virtual_next_js_template_path(
async fn virtual_next_js_template_path(
project_path: FileSystemPath,
file: String,
file: &str,
) -> Result<FileSystemPath> {
debug_assert!(!file.contains('/'));
get_next_package(project_path)
crates/next-taskless/Cargo.toml
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "next-taskless"
version = "0.0.1"
description = "TBD"
license = "MIT"
edition = "2024"
[lib]
bench = false
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
regex = { workspace = true }
serde_json = { workspace = true }
turbo-unix-path = { workspace = true }
crates/next-taskless/README.md
+7
View File
@@ -0,0 +1,7 @@
Utilities for Next.js written without a dependency on turbo-tasks, allowing them to be shared with
webpack and/or rspack codepaths. This crate must be compilable to WASM, so that it works in
environments where no native bindings are available.
These utilities should not perform file IO directly, but instead accept files they depend on as
arguments (preferred) or via async trait methods or callbacks (acceptable). Follow "sans-io"
patterns where possible.
crates/next-taskless/src/lib.rs
+265
View File
@@ -0,0 +1,265 @@
#![doc = include_str!("../README.md")]
use std::sync::LazyLock;
use anyhow::{Context, Result, bail};
use regex::Regex;
use turbo_unix_path::{get_parent_path, get_relative_path_to, join_path, normalize_path};
/// Given a next.js template file's contents, replaces `replacements` and `injections` and makes
/// sure there are none left over.
///
/// See `packages/next/src/build/templates/` for examples.
///
/// Paths should be unix or node.js-style paths where `/` is used as the path separator. They should
/// not be windows-style paths.
pub fn expand_next_js_template<'a>(
content: &str,
template_path: &str,
next_package_dir_path: &str,
replacements: impl IntoIterator<Item = (&'a str, &'a str)>,
injections: impl IntoIterator<Item = (&'a str, &'a str)>,
imports: impl IntoIterator<Item = (&'a str, Option<&'a str>)>,
) -> Result<String> {
let template_parent_path = normalize_path(get_parent_path(template_path))
.context("failed to normalize template path")?;
let next_package_dir_parent_path = normalize_path(get_parent_path(next_package_dir_path))
.context("failed to normalize package dir path")?;
/// See [regex::Regex::replace_all].
fn replace_all<E>(
re: &regex::Regex,
haystack: &str,
mut replacement: impl FnMut(&regex::Captures) -> Result<String, E>,
) -> Result<String, E> {
let mut new = String::with_capacity(haystack.len());
let mut last_match = 0;
for caps in re.captures_iter(haystack) {
let m = caps.get(0).unwrap();
new.push_str(&haystack[last_match..m.start()]);
new.push_str(&replacement(&caps)?);
last_match = m.end();
}
new.push_str(&haystack[last_match..]);
Ok(new)
}
// Update the relative imports to be absolute. This will update any relative imports to be
// relative to the root of the `next` package.
static IMPORT_PATH_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new("(?:from '(\\..*)'|import '(\\..*)')").unwrap());
let mut count = 0;
let mut content = replace_all(&IMPORT_PATH_RE, content, |caps| {
let from_request = caps.get(1).map_or("", |c| c.as_str());
count += 1;
let is_from_request = !from_request.is_empty();
let imported_path = join_path(
&template_parent_path,
if is_from_request {
from_request
} else {
caps.get(2).context("import path must exist")?.as_str()
},
)
.context("path should not leave the fs")?;
let relative = get_relative_path_to(&next_package_dir_parent_path, &imported_path);
if !relative.starts_with("./next/") {
bail!(
"Invariant: Expected relative import to start with \"./next/\", found \
{relative:?}. Path computed from {next_package_dir_parent_path:?} to \
{imported_path:?}.",
)
}
let relative = relative
.strip_prefix("./")
.context("should be able to strip the prefix")?;
Ok(if is_from_request {
format!("from {}", serde_json::to_string(relative).unwrap())
} else {
format!("import {}", serde_json::to_string(relative).unwrap())
})
})
.context("replacing imports failed")?;
// Verify that at least one import was replaced. It's the case today where every template file
// has at least one import to update, so this ensures that we don't accidentally remove the
// import replacement code or use the wrong template file.
if count == 0 {
bail!("Invariant: Expected to replace at least one import")
}
// Replace all the template variables with the actual values. If a template variable is missing,
// throw an error.
let mut missing_replacements = Vec::new();
for (key, replacement) in replacements {
let full = format!("'{key}'");
if content.contains(&full) {
content = content.replace(&full, &serde_json::to_string(&replacement).unwrap());
} else {
missing_replacements.push(key)
}
}
// Check to see if there's any remaining template variables.
static TEMPLATE_VAR_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new("VAR_[A-Z_]+").unwrap());
let mut matches = TEMPLATE_VAR_RE.find_iter(&content).peekable();
if matches.peek().is_some() {
bail!(
"Invariant: Expected to replace all template variables, found {}",
matches.map(|m| m.as_str()).collect::<Vec<_>>().join(", "),
)
}
// Check to see if any template variable was provided but not used.
if !missing_replacements.is_empty() {
bail!(
"Invariant: Expected to replace all template variables, missing {} in template",
missing_replacements.join(", "),
)
}
// Replace the injections.
let mut missing_injections = Vec::new();
for (key, injection) in injections {
let full = format!("// INJECT:{key}");
if content.contains(&full) {
content = content.replace(&full, &format!("const {key} = {injection}"));
} else {
missing_injections.push(key);
}
}
// Check to see if there's any remaining injections.
static INJECT_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new("// INJECT:[A-Za-z0-9_]+").unwrap());
let mut matches = INJECT_RE.find_iter(&content).peekable();
if matches.peek().is_some() {
bail!(
"Invariant: Expected to inject all injections, found {}",
matches.map(|m| m.as_str()).collect::<Vec<_>>().join(", "),
)
}
// Check to see if any injection was provided but not used.
if !missing_injections.is_empty() {
bail!(
"Invariant: Expected to inject all injections, missing {} in template",
missing_injections.join(", "),
)
}
// Replace the optional imports.
let mut missing_imports = Vec::new();
for (key, import_path) in imports {
let mut full = format!("// OPTIONAL_IMPORT:{key}");
let namespace = if !content.contains(&full) {
full = format!("// OPTIONAL_IMPORT:* as {key}");
if content.contains(&full) {
true
} else {
missing_imports.push(key);
continue;
}
} else {
false
};
if let Some(path) = import_path {
content = content.replace(
&full,
&format!(
"import {}{} from {}",
if namespace { "* as " } else { "" },
key,
serde_json::to_string(&path).unwrap(),
),
);
} else {
content = content.replace(&full, &format!("const {key} = null"));
}
}
// Check to see if there's any remaining imports.
static OPTIONAL_IMPORT_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new("// OPTIONAL_IMPORT:(\\* as )?[A-Za-z0-9_]+").unwrap());
let mut matches = OPTIONAL_IMPORT_RE.find_iter(&content).peekable();
if matches.peek().is_some() {
bail!(
"Invariant: Expected to inject all imports, found {}",
matches.map(|m| m.as_str()).collect::<Vec<_>>().join(", "),
)
}
// Check to see if any import was provided but not used.
if !missing_imports.is_empty() {
bail!(
"Invariant: Expected to inject all imports, missing {} in template",
missing_imports.join(", "),
)
}
// Ensure that the last line is a newline.
if !content.ends_with('\n') {
content.push('\n');
}
Ok(content)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_expand_next_js_template() {
let input = r#"
import '../../foo/bar';
import * as userlandPage from 'VAR_USERLAND'
// OPTIONAL_IMPORT:* as userland500Page
// OPTIONAL_IMPORT:incrementalCacheHandler
// INJECT:nextConfig
const srcPage = 'VAR_PAGE'
"#;
let expected = r#"
import "next/src/foo/bar";
import * as userlandPage from "INNER_PAGE_ENTRY"
import * as userland500Page from "INNER_ERROR_500"
const incrementalCacheHandler = null
const nextConfig = {}
const srcPage = "./some/path.js"
"#;
let output = expand_next_js_template(
input,
"project/node_modules/next/src/build/templates/test-case.js",
"project/node_modules/next",
[
("VAR_USERLAND", "INNER_PAGE_ENTRY"),
("VAR_PAGE", "./some/path.js"),
],
[("nextConfig", "{}")],
[
("incrementalCacheHandler", None),
("userland500Page", Some("INNER_ERROR_500")),
],
)
.unwrap();
println!("{output}");
assert_eq!(output.trim_end(), expected.trim_end());
}
}
crates/wasm/Cargo.toml
+6 -4
View File
@@ -17,15 +17,17 @@ plugin = ["swc_core/plugin_transform_host_js"]
workspace = true
[dependencies]
anyhow = "1.0.66"
anyhow = { workspace = true }
console_error_panic_hook = "0.1.6"
getrandom = { version = "0.3", default-features = false, features = ["wasm_js"] }
js-sys = "0.3.59"
next-custom-transforms = { workspace = true }
next-taskless = { workspace = true }
rustc-hash = { workspace = true }
serde-wasm-bindgen = "0.4.3"
serde_json = "1"
wasm-bindgen = { version = "0.2", features = ["enable-interning"] }
wasm-bindgen-futures = "0.4.8"
getrandom = { version = "0.3", default-features = false, features = ["wasm_js"] }
js-sys = "0.3.59"
serde-wasm-bindgen = "0.4.3"
swc_core = { workspace = true, features = [
"common",
crates/wasm/src/lib.rs
+35 -8
View File
@@ -1,8 +1,9 @@
use std::sync::Arc;
use std::{fmt::Debug, sync::Arc};
use anyhow::{Context, Error};
use anyhow::Context;
use js_sys::JsString;
use next_custom_transforms::chain_transforms::{custom_before_pass, TransformOptions};
use rustc_hash::FxHashMap;
use swc_core::{
base::{
config::{JsMinifyOptions, ParseOptions},
@@ -20,8 +21,8 @@ use wasm_bindgen_futures::future_to_promise;
pub mod mdx;
fn convert_err(err: Error) -> JsValue {
format!("{err:?}").into()
fn convert_err(err: impl Debug) -> JsError {
JsError::new(&format!("{err:?}"))
}
#[wasm_bindgen(js_name = "minifySync")]
@@ -63,7 +64,7 @@ pub fn minify(s: JsString, opts: JsValue) -> js_sys::Promise {
}
#[wasm_bindgen(js_name = "transformSync")]
pub fn transform_sync(s: JsValue, opts: JsValue) -> Result<JsValue, JsValue> {
pub fn transform_sync(s: JsValue, opts: JsValue) -> Result<JsValue, JsError> {
console_error_panic_hook::set_once();
let c = compiler();
@@ -136,11 +137,11 @@ pub fn transform_sync(s: JsValue, opts: JsValue) -> Result<JsValue, JsValue> {
pub fn transform(s: JsValue, opts: JsValue) -> js_sys::Promise {
// TODO: This'll be properly scheduled once wasm have standard backed thread
// support.
future_to_promise(async { transform_sync(s, opts) })
future_to_promise(async { Ok(transform_sync(s, opts)?) })
}
#[wasm_bindgen(js_name = "parseSync")]
pub fn parse_sync(s: JsString, opts: JsValue) -> Result<JsValue, JsValue> {
pub fn parse_sync(s: JsString, opts: JsValue) -> Result<JsValue, JsError> {
console_error_panic_hook::set_once();
let c = swc_core::base::Compiler::new(Arc::new(SourceMap::new(FilePathMapping::empty())));
@@ -188,7 +189,7 @@ pub fn parse_sync(s: JsString, opts: JsValue) -> Result<JsValue, JsValue> {
pub fn parse(s: JsString, opts: JsValue) -> js_sys::Promise {
// TODO: This'll be properly scheduled once wasm have standard backed thread
// support.
future_to_promise(async { parse_sync(s, opts) })
future_to_promise(async { Ok(parse_sync(s, opts)?) })
}
/// Get global sourcemap
@@ -197,3 +198,29 @@ fn compiler() -> Arc<Compiler> {
Arc::new(Compiler::new(cm))
}
#[wasm_bindgen(js_name = "expandNextJsTemplate")]
pub fn expand_next_js_template(
content: Box<[u8]>,
template_path: &str,
next_package_dir_path: &str,
replacements: JsValue,
injections: JsValue,
imports: JsValue,
) -> Result<String, JsError> {
next_taskless::expand_next_js_template(
str::from_utf8(&content).map_err(convert_err)?,
template_path,
next_package_dir_path,
serde_wasm_bindgen::from_value::<FxHashMap<String, String>>(replacements)?
.iter()
.map(|(k, v)| (&**k, &**v)),
serde_wasm_bindgen::from_value::<FxHashMap<String, String>>(injections)?
.iter()
.map(|(k, v)| (&**k, &**v)),
serde_wasm_bindgen::from_value::<FxHashMap<String, Option<String>>>(imports)?
.iter()
.map(|(k, v)| (&**k, v.as_deref())),
)
.map_err(convert_err)
}
packages/next/src/build/load-entrypoint.ts
+15 -189
View File
@@ -1,9 +1,10 @@
import fs from 'fs/promises'
import path from 'path'
import { loadBindings } from './swc'
// NOTE: this should be updated if this loader file is moved.
const PACKAGE_ROOT = path.normalize(path.join(__dirname, '../../..'))
const TEMPLATE_FOLDER = path.join(__dirname, 'templates')
const PACKAGE_ROOT = path.normalize(path.join(__dirname, '../..'))
const TEMPLATE_SRC_FOLDER = path.normalize(path.join(__dirname, './templates'))
const TEMPLATES_ESM_FOLDER = path.normalize(
path.join(__dirname, '../../dist/esm/build/templates')
)
@@ -38,195 +39,20 @@ export async function loadEntrypoint(
injections?: Record<string, string>,
imports?: Record<string, string | null>
): Promise<string> {
const filepath = path.resolve(
let bindings = await loadBindings()
const templatePath = path.resolve(
path.join(TEMPLATES_ESM_FOLDER, `${entrypoint}.js`)
)
let content = await fs.readFile(templatePath)
let file = await fs.readFile(filepath, 'utf8')
// Update the relative imports to be absolute. This will update any relative
// imports to be relative to the root of the `next` package.
let count = 0
file = file.replaceAll(
/from '(\..*)'|import '(\..*)'/g,
function (_, fromRequest, importRequest) {
count++
const relative = path
.relative(
PACKAGE_ROOT,
path.resolve(TEMPLATE_FOLDER, fromRequest ?? importRequest)
)
// Ensure that we use linux style path separators for node.
.replace(/\\/g, '/')
// Verify that the relative import is relative to the `next` package. This
// will catch cases where the constants at the top of the file were not
// updated after the file was moved.
if (!relative.startsWith('next/')) {
throw new Error(
`Invariant: Expected relative import to start with "next/", found "${relative}"`
)
}
return fromRequest
? `from ${JSON.stringify(relative)}`
: `import ${JSON.stringify(relative)}`
}
return bindings.expandNextJsTemplate(
content,
// Ensure that we use unix-style path separators for the import paths
path.join(TEMPLATE_SRC_FOLDER, `${entrypoint}.js`).replace(/\\/g, '/'),
PACKAGE_ROOT.replace(/\\/g, '/'),
replacements,
injections ?? {},
imports ?? {}
)
// Verify that at least one import was replaced. It's the case today where
// every template file has at least one import to update, so this ensures that
// we don't accidentally remove the import replacement code or use the wrong
// template file.
if (count === 0) {
throw new Error('Invariant: Expected to replace at least one import')
}
const replaced = new Set<string>()
// Replace all the template variables with the actual values. If a template
// variable is missing, throw an error.
file = file.replaceAll(
new RegExp(
`${Object.keys(replacements)
.map((k) => `'${k}'`)
.join('|')}`,
'g'
),
(match) => {
const key = JSON.parse(match.replace(/'/g, `"`))
if (!(key in replacements)) {
throw new Error(`Invariant: Unexpected template variable ${key}`)
}
replaced.add(key)
return JSON.stringify(replacements[key])
}
)
// Check to see if there's any remaining template variables.
let matches = file.match(/VAR_[A-Z_]+/g)
if (matches) {
throw new Error(
`Invariant: Expected to replace all template variables, found ${matches.join(
', '
)}`
)
}
// Check to see if any template variable was provided but not used.
if (replaced.size !== Object.keys(replacements).length) {
// Find the difference between the provided replacements and the replaced
// template variables. This will let us notify the user of any template
// variables that were not used but were provided.
const difference = Object.keys(replacements).filter(
(key) => !replaced.has(key)
)
throw new Error(
`Invariant: Expected to replace all template variables, missing ${difference.join(
', '
)} in template`
)
}
// Replace the injections.
const injected = new Set<string>()
if (injections) {
// Track all the injections to ensure that we're not missing any.
file = file.replaceAll(
new RegExp(`// INJECT:(${Object.keys(injections).join('|')})`, 'g'),
(_, key) => {
if (!(key in injections)) {
throw new Error(`Invariant: Unexpected injection ${key}`)
}
injected.add(key)
return `const ${key} = ${injections[key]}`
}
)
}
// Check to see if there's any remaining injections.
matches = file.match(/\/\/ INJECT:[A-Za-z0-9_]+/g)
if (matches) {
throw new Error(
`Invariant: Expected to inject all injections, found ${matches.join(
', '
)}`
)
}
// Check to see if any injection was provided but not used.
if (injected.size !== Object.keys(injections ?? {}).length) {
// Find the difference between the provided injections and the injected
// injections. This will let us notify the user of any injections that were
// not used but were provided.
const difference = Object.keys(injections ?? {}).filter(
(key) => !injected.has(key)
)
throw new Error(
`Invariant: Expected to inject all injections, missing ${difference.join(
', '
)} in template`
)
}
// Replace the optional imports.
const importsAdded = new Set<string>()
if (imports) {
// Track all the imports to ensure that we're not missing any.
file = file.replaceAll(
new RegExp(
`// OPTIONAL_IMPORT:(\\* as )?(${Object.keys(imports).join('|')})`,
'g'
),
(_, asNamespace = '', key) => {
if (!(key in imports)) {
throw new Error(`Invariant: Unexpected optional import ${key}`)
}
importsAdded.add(key)
if (imports[key]) {
return `import ${asNamespace}${key} from ${JSON.stringify(
imports[key]
)}`
} else {
return `const ${key} = null`
}
}
)
}
// Check to see if there's any remaining imports.
matches = file.match(/\/\/ OPTIONAL_IMPORT:(\* as )?[A-Za-z0-9_]+/g)
if (matches) {
throw new Error(
`Invariant: Expected to inject all imports, found ${matches.join(', ')}`
)
}
// Check to see if any import was provided but not used.
if (importsAdded.size !== Object.keys(imports ?? {}).length) {
// Find the difference between the provided imports and the injected
// imports. This will let us notify the user of any imports that were
// not used but were provided.
const difference = Object.keys(imports ?? {}).filter(
(key) => !importsAdded.has(key)
)
throw new Error(
`Invariant: Expected to inject all imports, missing ${difference.join(
', '
)} in template`
)
}
return file
}
packages/next/src/build/swc/generated-native.d.ts
Vendored
+8
View File
@@ -424,6 +424,14 @@ export interface NapiDiagnostic {
name: string
payload: Record<string, string>
}
export declare function expandNextJsTemplate(
content: Buffer,
templatePath: string,
nextPackageDirPath: string,
replacements: Record<string, string>,
injections: Record<string, string>,
imports: Record<string, string | null>
): string
export declare function parse(
src: string,
options: Buffer,
packages/next/src/build/swc/generated-wasm.d.ts
Vendored
+8
View File
@@ -11,3 +11,11 @@ export function transformSync(s: any, opts: any): any
export function transform(s: any, opts: any): Promise<any>
export function parseSync(s: string, opts: any): any
export function parse(s: string, opts: any): Promise<any>
export function expandNextJsTemplate(
content: Uint8Array,
template_path: string,
next_package_dir_path: string,
replacements: any,
injections: any,
imports: any
): string
packages/next/src/build/swc/index.ts
+34
View File
@@ -1238,6 +1238,23 @@ async function loadWasm(importPath = '') {
)
},
},
expandNextJsTemplate(
content: Buffer,
templatePath: string,
nextPackageDirPath: string,
replacements: Record<`VAR_${string}`, string>,
injections: Record<string, string>,
imports: Record<string, string | null>
): string {
return rawBindings.expandNextJsTemplate(
content,
templatePath,
nextPackageDirPath,
replacements,
injections,
imports
)
},
}
return wasmBindings
}
@@ -1420,6 +1437,23 @@ function loadNative(importPath?: string) {
return bindings.warnForEdgeRuntime(source, isProduction)
},
},
expandNextJsTemplate(
content: Buffer,
templatePath: string,
nextPackageDirPath: string,
replacements: Record<`VAR_${string}`, string>,
injections: Record<string, string>,
imports: Record<string, string | null>
): string {
return bindings.expandNextJsTemplate(
content,
templatePath,
nextPackageDirPath,
replacements,
injections,
imports
)
},
}
return nativeBindings
}
packages/next/src/build/swc/types.ts
+8
View File
@@ -52,6 +52,14 @@ export interface Binding {
isProduction: boolean
): Promise<NapiSourceDiagnostic[]>
}
expandNextJsTemplate(
content: Buffer,
templatePath: string,
nextPackageDirPath: string,
replacements: Record<`VAR_${string}`, string>,
injections: Record<string, string>,
imports: Record<string, string | null>
): string
}
export type StyledString =
turbopack/crates/turbo-tasks-fs/Cargo.toml
+1
View File
@@ -50,6 +50,7 @@ triomphe = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true }
urlencoding = { workspace = true }
[dev-dependencies]
turbopack/crates/turbo-tasks-fs/src/lib.rs
+8 -40
View File
@@ -59,16 +59,21 @@ use turbo_tasks::{
mark_session_dependent, mark_stateful, trace::TraceRawVcs,
};
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, hash_xxh3_hash64};
use turbo_unix_path::{
get_parent_path, get_relative_path_to, join_path, normalize_path, sys_to_unix, unix_to_sys,
};
use self::{invalidation::Write, json::UnparsableJson, mutex_map::MutexMap};
use crate::{
attach::AttachedFileSystem,
glob::Glob,
invalidation::Write,
invalidator_map::{InvalidatorMap, WriteContent},
json::UnparsableJson,
mutex_map::MutexMap,
read_glob::{read_glob, track_glob},
retry::retry_blocking,
rope::{Rope, RopeReader},
util::{extract_disk_access, join_path, normalize_path, sys_to_unix, unix_to_sys},
util::extract_disk_access,
watcher::DiskWatcher,
};
pub use crate::{read_glob::ReadGlobResult, virtual_fs::VirtualFileSystem};
@@ -1000,39 +1005,6 @@ impl ValueToString for DiskFileSystem {
}
}
/// Note: this only works for Unix-style paths (with `/` as a separator).
pub fn get_relative_path_to(path: &str, other_path: &str) -> String {
fn split(s: &str) -> impl Iterator<Item = &str> {
let empty = s.is_empty();
let mut iterator = s.split('/');
if empty {
iterator.next();
}
iterator
}
let mut self_segments = split(path).peekable();
let mut other_segments = split(other_path).peekable();
while self_segments.peek() == other_segments.peek() {
self_segments.next();
if other_segments.next().is_none() {
return ".".to_string();
}
}
let mut result = Vec::new();
if self_segments.peek().is_none() {
result.push(".");
} else {
while self_segments.next().is_some() {
result.push("..");
}
}
for segment in other_segments {
result.push(segment);
}
result.join("/")
}
#[turbo_tasks::value(shared)]
#[derive(Debug, Clone, Hash, TaskInput)]
pub struct FileSystemPath {
@@ -1456,11 +1428,7 @@ impl FileSystemPath {
if path.is_empty() {
return self.clone();
}
let p = match str::rfind(path, '/') {
Some(index) => path[..index].to_string(),
None => "".to_string(),
};
FileSystemPath::new_normalized(self.fs, p.into())
FileSystemPath::new_normalized(self.fs, RcStr::from(get_parent_path(path)))
}
// It is important that get_type uses read_dir and not stat/metadata.
turbopack/crates/turbo-tasks-fs/src/util.rs
+2 -149
View File
@@ -1,5 +1,4 @@
use std::{
borrow::Cow,
io::{self, ErrorKind},
path::Path,
};
@@ -9,123 +8,6 @@ use turbo_tasks::Vc;
use crate::{DiskFileSystem, FileSystemPath};
/// Joins two /-separated paths into a normalized path.
/// Paths are concatenated with /.
///
/// see also [normalize_path] for normalization.
pub fn join_path(fs_path: &str, join: &str) -> Option<String> {
// Paths that we join are written as source code (eg, `join_path(fs_path,
// "foo/bar.js")`) and it's expected that they will never contain a
// backslash.
debug_assert!(
!join.contains('\\'),
"joined path {join} must not contain a Windows directory '\\', it must be normalized to \
Unix '/'"
);
// TODO: figure out why this freezes the benchmarks.
// // an absolute path would leave the file system root
// if Path::new(join).is_absolute() {
// return None;
// }
if fs_path.is_empty() {
normalize_path(join)
} else if join.is_empty() {
normalize_path(fs_path)
} else {
normalize_path(&[fs_path, "/", join].concat())
}
}
/// Converts System paths into Unix paths. This is a noop on Unix systems, and
/// replaces backslash directory separators with forward slashes on Windows.
#[inline]
pub fn sys_to_unix(path: &str) -> Cow<'_, str> {
#[cfg(not(target_family = "windows"))]
{
Cow::from(path)
}
#[cfg(target_family = "windows")]
{
Cow::Owned(path.replace(std::path::MAIN_SEPARATOR_STR, "/"))
}
}
/// Converts Unix paths into System paths. This is a noop on Unix systems, and
/// replaces forward slash directory separators with backslashes on Windows.
#[inline]
pub fn unix_to_sys(path: &str) -> Cow<'_, str> {
#[cfg(not(target_family = "windows"))]
{
Cow::from(path)
}
#[cfg(target_family = "windows")]
{
Cow::Owned(path.replace('/', std::path::MAIN_SEPARATOR_STR))
}
}
/// Normalizes a /-separated path into a form that contains no leading /, no
/// double /, no "." segment, no ".." segment.
///
/// Returns None if the path would need to start with ".." to be equal.
pub fn normalize_path(str: &str) -> Option<String> {
let mut segments = Vec::new();
for segment in str.split('/') {
match segment {
"." | "" => {}
".." => {
segments.pop()?;
}
segment => {
segments.push(segment);
}
}
}
Some(segments.join("/"))
}
/// Normalizes a /-separated request into a form that contains no leading /, no
/// double /, and no "." or ".." segments in the middle of the request.
///
/// A request might only start with a single "." segment and no ".." segments, or
/// any positive number of ".." segments but no "." segment.
pub fn normalize_request(str: &str) -> String {
let mut segments = vec!["."];
// Keeps track of our directory depth so that we can pop directories when
// encountering a "..". If this is positive, then we're inside a directory
// and we can pop that. If it's 0, then we can't pop the directory and we must
// keep the ".." in our segments. This is not the same as the segments.len(),
// because we cannot pop a kept ".." when encountering another "..".
let mut depth = 0;
let mut popped_dot = false;
for segment in str.split('/') {
match segment {
"." => {}
".." => {
if depth > 0 {
depth -= 1;
segments.pop();
} else {
// The first time we push a "..", we need to remove the "." we include by
// default.
if !popped_dot {
popped_dot = true;
segments.pop();
}
segments.push(segment);
}
}
segment => {
segments.push(segment);
depth += 1;
}
}
}
segments.join("/")
}
/// Converts a disk access Result<T> into a Result<Some<T>>, where a NotFound
/// error results in a None value. This is purely to reduce boilerplate code
/// comparing NotFound errors against all other errors.
@@ -139,6 +21,8 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio
#[cfg(not(target_os = "windows"))]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
use turbo_unix_path::sys_to_unix;
let root_fs = root.fs();
let root_fs = &*Vc::try_resolve_downcast_type::<DiskFileSystem>(root_fs)
.await?
@@ -190,34 +74,3 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
Ok(uri)
}
#[cfg(test)]
mod tests {
use rstest::*;
use crate::util::normalize_path;
#[rstest]
#[case("file.js")]
#[case("a/b/c/d/e/file.js")]
fn test_normalize_path_no_op(#[case] path: &str) {
assert_eq!(path, normalize_path(path).unwrap());
}
#[rstest]
#[case("/file.js", "file.js")]
#[case("./file.js", "file.js")]
#[case("././file.js", "file.js")]
#[case("a/../c/../file.js", "file.js")]
fn test_normalize_path(#[case] path: &str, #[case] normalized: &str) {
assert_eq!(normalized, normalize_path(path).unwrap());
}
#[rstest]
#[case("../file.js")]
#[case("a/../../file.js")]
fn test_normalize_path_invalid(#[case] path: &str) {
assert_eq!(None, normalize_path(path));
}
}
turbopack/crates/turbo-unix-path/Cargo.toml
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "turbo-unix-path"
version = "0.0.1"
description = "TBD"
license = "MIT"
edition = "2024"
[lib]
bench = false
[lints]
workspace = true
[dependencies]
[dev-dependencies]
rstest = { workspace = true }
turbopack/crates/turbo-unix-path/README.md
+8
View File
@@ -0,0 +1,8 @@
Utilities for lexical manipulation of unix-style (`/`-separated) paths represented as unicode
strings.
These paths types are frequently used for JS imports, and are used within Turbopack to represent
platform-agnostic relative paths.
This crate does not perform any IO, and does not depend on turbo-tasks. Most users should prefer the
`turbo-tasks-fs` crate.
turbopack/crates/turbo-unix-path/src/lib.rs
+188
View File
@@ -0,0 +1,188 @@
#![doc = include_str!("../README.md")]
use std::borrow::Cow;
/// Converts system paths into Unix paths. This is a noop on Unix systems, and replaces backslash
/// directory separators with forward slashes on Windows.
#[inline]
pub fn sys_to_unix(path: &str) -> Cow<'_, str> {
#[cfg(not(target_family = "windows"))]
{
Cow::from(path)
}
#[cfg(target_family = "windows")]
{
Cow::Owned(path.replace(std::path::MAIN_SEPARATOR_STR, "/"))
}
}
/// Converts Unix paths into system paths. This is a noop on Unix systems, and replaces forward
/// slash directory separators with backslashes on Windows.
#[inline]
pub fn unix_to_sys(path: &str) -> Cow<'_, str> {
#[cfg(not(target_family = "windows"))]
{
Cow::from(path)
}
#[cfg(target_family = "windows")]
{
Cow::Owned(path.replace('/', std::path::MAIN_SEPARATOR_STR))
}
}
/// Joins two /-separated paths into a normalized path.
/// Paths are concatenated with /.
///
/// see also [normalize_path] for normalization.
pub fn join_path(fs_path: &str, join: &str) -> Option<String> {
// Paths that we join are written as source code (eg, `join_path(fs_path, "foo/bar.js")`) and
// it's expected that they will never contain a backslash.
debug_assert!(
!join.contains('\\'),
"joined path {join} must not contain a Windows directory '\\', it must be normalized to \
Unix '/'"
);
// TODO: figure out why this freezes the benchmarks.
// // an absolute path would leave the file system root
// if Path::new(join).is_absolute() {
// return None;
// }
if fs_path.is_empty() {
normalize_path(join)
} else if join.is_empty() {
normalize_path(fs_path)
} else {
normalize_path(&[fs_path, "/", join].concat())
}
}
/// Normalizes a /-separated path into a form that contains no leading /, no double /, no "."
/// segment, no ".." segment.
///
/// Returns None if the path would need to start with ".." to be equal.
pub fn normalize_path(str: &str) -> Option<String> {
let mut segments = Vec::new();
for segment in str.split('/') {
match segment {
"." | "" => {}
".." => {
segments.pop()?;
}
segment => {
segments.push(segment);
}
}
}
Some(segments.join("/"))
}
/// Normalizes a /-separated request into a form that contains no leading /, no double /, and no "."
/// or ".." segments in the middle of the request.
///
/// A request might only start with a single "." segment and no ".." segments, or any positive
/// number of ".." segments but no "." segment.
pub fn normalize_request(str: &str) -> String {
let mut segments = vec!["."];
// Keeps track of our directory depth so that we can pop directories when encountering a "..".
// If this is positive, then we're inside a directory and we can pop that. If it's 0, then we
// can't pop the directory and we must keep the ".." in our segments. This is not the same as
// the segments.len(), because we cannot pop a kept ".." when encountering another "..".
let mut depth = 0;
let mut popped_dot = false;
for segment in str.split('/') {
match segment {
"." => {}
".." => {
if depth > 0 {
depth -= 1;
segments.pop();
} else {
// The first time we push a "..", we need to remove the "." we include by
// default.
if !popped_dot {
popped_dot = true;
segments.pop();
}
segments.push(segment);
}
}
segment => {
segments.push(segment);
depth += 1;
}
}
}
segments.join("/")
}
pub fn get_relative_path_to(from: &str, target: &str) -> String {
fn split(s: &str) -> impl Iterator<Item = &str> {
let empty = s.is_empty();
let mut iterator = s.split('/');
if empty {
iterator.next();
}
iterator
}
let mut from_segments = split(from).peekable();
let mut target_segments = split(target).peekable();
while from_segments.peek() == target_segments.peek() {
from_segments.next();
if target_segments.next().is_none() {
return ".".to_string();
}
}
let mut result = Vec::new();
if from_segments.peek().is_none() {
result.push(".");
} else {
while from_segments.next().is_some() {
result.push("..");
}
}
for segment in target_segments {
result.push(segment);
}
result.join("/")
}
pub fn get_parent_path(path: &str) -> &str {
match str::rfind(path, '/') {
Some(index) => &path[..index],
None => "",
}
}
#[cfg(test)]
mod tests {
use rstest::*;
use super::*;
#[rstest]
#[case("file.js")]
#[case("a/b/c/d/e/file.js")]
fn test_normalize_path_no_op(#[case] path: &str) {
assert_eq!(path, normalize_path(path).unwrap());
}
#[rstest]
#[case("/file.js", "file.js")]
#[case("./file.js", "file.js")]
#[case("././file.js", "file.js")]
#[case("a/../c/../file.js", "file.js")]
fn test_normalize_path(#[case] path: &str, #[case] normalized: &str) {
assert_eq!(normalized, normalize_path(path).unwrap());
}
#[rstest]
#[case("../file.js")]
#[case("a/../../file.js")]
fn test_normalize_path_invalid(#[case] path: &str) {
assert_eq!(None, normalize_path(path));
}
}
turbopack/crates/turbopack-core/Cargo.toml
+1
View File
@@ -41,6 +41,7 @@ turbo-tasks = { workspace = true }
turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true }
urlencoding = { workspace = true }
bytes-str = { workspace = true }
turbopack/crates/turbopack-core/src/resolve/mod.rs
+3 -4
View File
@@ -15,9 +15,8 @@ use turbo_tasks::{
FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, SliceMap, TaskInput,
TryJoinIterExt, ValueToString, Vc, trace::TraceRawVcs,
};
use turbo_tasks_fs::{
FileSystemEntryType, FileSystemPath, RealPathResult, util::normalize_request,
};
use turbo_tasks_fs::{FileSystemEntryType, FileSystemPath, RealPathResult};
use turbo_unix_path::normalize_request;
use self::{
options::{
@@ -2108,7 +2107,7 @@ async fn resolve_into_folder(
.await?
&& let Some(field_value) = package_json[name.as_str()].as_str()
{
let normalized_request: RcStr = normalize_request(field_value).into();
let normalized_request = RcStr::from(normalize_request(field_value));
if normalized_request.is_empty()
|| &*normalized_request == "."
|| &*normalized_request == "./"
turbopack/crates/turbopack-core/src/resolve/pattern.rs
+2 -2
View File
@@ -16,8 +16,8 @@ use turbo_tasks::{
};
use turbo_tasks_fs::{
FileSystemPath, LinkContent, LinkType, RawDirectoryContent, RawDirectoryEntry,
util::normalize_path,
};
use turbo_unix_path::normalize_path;
#[turbo_tasks::value]
#[derive(Hash, Clone, Debug, Default)]
@@ -460,7 +460,7 @@ impl Pattern {
match pattern {
Pattern::Constant(c) => {
let normalized = c.replace('\\', "/");
*c = (*(normalize_path(normalized.as_str())?)).into();
*c = RcStr::from(normalize_path(normalized.as_str())?);
Some(())
}
Pattern::Dynamic => Some(()),
turbopack/crates/turbopack-tests/Cargo.toml
+1
View File
@@ -31,6 +31,7 @@ turbo-tasks-bytes = { workspace = true }
turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-tasks-backend = { workspace = true }
turbo-unix-path = { workspace = true }
turbopack-browser = { workspace = true, features = ["test"] }
turbopack-core = { workspace = true, features = ["issue_path"] }
turbopack-ecmascript-plugins = { workspace = true, features = [
turbopack/crates/turbopack-tests/tests/execution.rs
+3 -2
View File
@@ -21,8 +21,9 @@ use turbo_tasks_bytes::stream::SingleValue;
use turbo_tasks_env::CommandLineProcessEnv;
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystem, FileSystemEntryType, FileSystemPath,
json::parse_json_with_source_context, util::sys_to_unix,
json::parse_json_with_source_context,
};
use turbo_unix_path::sys_to_unix;
use turbopack::{
ModuleAssetContext,
css::chunk::CssChunkType,
@@ -290,7 +291,7 @@ async fn prepare_test(resource: RcStr) -> Result<Vc<PreparedTest>> {
&*REPO_ROOT,
resource_path.display()
))?;
let relative_path: RcStr = sys_to_unix(relative_path.to_str().unwrap()).into();
let relative_path = RcStr::from(sys_to_unix(relative_path.to_str().unwrap()));
let path = root_fs.root().await?.join(&relative_path)?;
let project_path = project_root.join(&relative_path)?;
let tests_path = project_fs
turbopack/crates/turbopack-tests/tests/snapshot.rs
+2 -2
View File
@@ -16,8 +16,8 @@ use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storag
use turbo_tasks_env::DotenvProcessEnv;
use turbo_tasks_fs::{
DiskFileSystem, FileSystem, FileSystemPath, json::parse_json_with_source_context,
util::sys_to_unix,
};
use turbo_unix_path::sys_to_unix;
use turbopack::{
ModuleAssetContext,
ecmascript::{EcmascriptInputTransform, TreeShakingMode, chunk::EcmascriptChunkType},
@@ -264,7 +264,7 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
let project_root = project_fs.root().owned().await?;
let relative_path = test_path.strip_prefix(&*REPO_ROOT)?;
let relative_path: RcStr = sys_to_unix(relative_path.to_str().unwrap()).into();
let relative_path = RcStr::from(sys_to_unix(relative_path.to_str().unwrap()));
let project_path = project_root.join(&relative_path)?;
let project_path_to_project_root = project_path