hanzoai/next.js
2
0
Fork 0
mirror of https://github.com/hanzoai/next.js.git synced 2026-09-24 17:00:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Turbopack: symlinks and parent globs in outputFileTracingIncludes (#82528)

Browse Source 
1. Don't resolve symlinks in `outputFileTracingIncludes`. Otherwise if you specify `node_modules/foo/*` in there, it will resolve to `node_modules/.pnpm/foo/*` and `require("foo")` won't work anymore because there no symlink in `node_modules/foo` anymore. This is also what happens with Webpack
2. Align `read_glob` with `read_dir` in that the hashmap keys only contain the path segment, and not the full path.
3. Support globing parents of the start folder: if the glob can match `..`, then also try to match `directory.parent_folder().read_glob(glob)`

Closes PACK-5219
This commit is contained in:
Niklas Mischkulnig
2025-08-12 15:03:48 +02:00
committed by GitHub
parent bd802d1327
commit 7f20cbd68d
13 changed files with 224 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/next-api/src/nft_json.rs
+4 -3
View File
@@ -104,6 +104,7 @@ async fn apply_includes(
) -> Result<BTreeSet<RcStr>> {
debug_assert_eq!(project_root_path.fs, ident_folder.fs);
// Read files matching the glob pattern from the project root
// This result itself has random order, but the BTreeSet will ensure a deterministic ordering.
let glob_result = project_root_path.read_glob(glob).await?;
// Walk the full glob_result using an explicit stack to avoid async recursion overheads.
@@ -113,15 +114,15 @@ async fn apply_includes(
while let Some(glob_result) = stack.pop_back() {
// Process direct results (files and directories at this level)
for entry in glob_result.results.values() {
let DirectoryEntry::File(file_path) = entry else {
let (DirectoryEntry::File(file_path) | DirectoryEntry::Symlink(file_path)) = entry
else {
continue;
};
let file_path_ref = file_path;
// Convert to relative path from ident_folder to the file
// unwrap is safe because project_root_path and ident_folder have the same filesystem
// and paths produced by read_glob stay in the filesystem
let relative_path = ident_folder.get_relative_path_to(file_path_ref).unwrap();
let relative_path = ident_folder.get_relative_path_to(file_path).unwrap();
result.insert(relative_path);
}
test/integration/build-trace-extra-entries-monorepo/app/app/route1/route.js
+5
View File
@@ -0,0 +1,5 @@
export const dynamic = 'force-dynamic'
export async function GET() {
return new Response('foo')
}
test/integration/build-trace-extra-entries-monorepo/app/next.config.js
+5
View File
@@ -0,0 +1,5 @@
module.exports = {
outputFileTracingIncludes: {
'/route1': ['../other/included.txt'],
},
}
test/integration/build-trace-extra-entries-monorepo/other/included.txt
+1
View File
@@ -0,0 +1 @@
data
test/integration/build-trace-extra-entries-monorepo/test/index.test.ts
+33
View File
@@ -0,0 +1,33 @@
/* eslint-env jest */
import fs from 'fs-extra'
import { join } from 'path'
import { nextBuild } from 'next-test-utils'
const appDir = join(__dirname, '../app')
describe('build trace with extra entries in monorepo', () => {
;(process.env.TURBOPACK_DEV ? describe.skip : describe)(
'production mode',
() => {
it('should build and trace correctly', async () => {
const result = await nextBuild(appDir, undefined, {
cwd: appDir,
stderr: true,
stdout: true,
})
expect(result.code).toBe(0)
console.log(result.stderr)
console.log(result.stdout)
const appDirRoute1Trace = await fs.readJSON(
join(appDir, '.next/server/app/route1/route.js.nft.json')
)
expect(appDirRoute1Trace.files).toContain(
'../../../../../other/included.txt'
)
})
}
)
})
test/integration/build-trace-extra-entries/app/include-me-global.txt
View File
test/integration/build-trace-extra-entries/app/next.config.js
+2 -3
View File
@@ -2,7 +2,6 @@ const path = require('path')
module.exports = {
webpack(cfg, { isServer, nextRuntime }) {
console.log(cfg.entry)
const origEntry = cfg.entry
cfg.entry = async () => {
const origEntries = await origEntry()
@@ -13,7 +12,6 @@ module.exports = {
path.join(__dirname, 'lib/get-data.js'),
...curEntry,
]
console.log(origEntries)
}
return origEntries
}
@@ -21,7 +19,8 @@ module.exports = {
},
outputFileTracingIncludes: {
'/index': ['include-me/**/*'],
'/route1': ['./include-me/**/*'],
'/route1': ['./include-me/**/*', 'node_modules/pkg-behind-symlink/*'],
'/*': ['include-me-global.txt'],
},
outputFileTracingExcludes: {
'/index': ['public/exclude-me/**/*'],
test/integration/build-trace-extra-entries/app/node_modules/pkg-behind-symlink
Generated Vendored Symlink
+1
View File
@@ -0,0 +1 @@
pkg
test/integration/build-trace-extra-entries/app/node_modules/pkg/index.js
Generated Vendored
+1
View File
@@ -0,0 +1 @@
module.exports = 'pkg'
test/integration/build-trace-extra-entries/app/node_modules/pkg/package.json
Generated Vendored
+4
View File
@@ -0,0 +1,4 @@
{
"name": "pkg",
"main": "./index.js"
}
test/integration/build-trace-extra-entries/test/index.test.js
+45 -8
View File
@@ -16,7 +16,6 @@ describe('build trace with extra entries', () => {
stderr: true,
stdout: true,
})
console.log(result)
expect(result.code).toBe(0)
const appTrace = await fs.readJSON(
@@ -41,16 +40,26 @@ describe('build trace with extra entries', () => {
expect(appDirRoute1Trace.files).toContain(
'../../../../include-me/second.txt'
)
expect(
appDirRoute1Trace.files.some(
(file) => file === '../../../../include-me-global.txt'
)
).toBe(true)
expect(
appDirRoute1Trace.files.some((file) => file.includes('exclude-me'))
).toBe(false)
// Skip hello.json check for Turbopack as it doesn't support webpack entry modifications
if (!process.env.TURBOPACK_BUILD) {
expect(
appTrace.files.some((file) => file.endsWith('hello.json'))
).toBe(true)
}
expect(appDirRoute1Trace.files).toEqual(
expect.arrayContaining([
'../../../../node_modules/pkg-behind-symlink/index.js',
'../../../../node_modules/pkg-behind-symlink/package.json',
])
)
// This assertion not really needed, but ensures consistency between Turbopack and Webpack
expect(
appDirRoute1Trace.files.some((file) =>
file.startsWith('../../../../node_modules/pkg/')
)
).toBe(false)
expect(
indexTrace.files.filter(
@@ -62,12 +71,24 @@ describe('build trace with extra entries', () => {
).length
)
// Skip hello.json check for Turbopack as it doesn't support webpack entry modifications
if (!process.env.TURBOPACK_BUILD) {
expect(
appTrace.files.some((file) => file.endsWith('hello.json'))
).toBe(true)
}
// Skip lib/get-data.js check for Turbopack as it doesn't support webpack entry modifications
if (!process.env.TURBOPACK_BUILD) {
expect(
appTrace.files.some((file) => file.endsWith('lib/get-data.js'))
).toBe(true)
}
expect(
appTrace.files.some(
(file) => file === '../../../include-me-global.txt'
)
).toBe(true)
expect(
indexTrace.files.some((file) => file.endsWith('hello.json'))
).toBeFalsy()
@@ -90,6 +111,11 @@ describe('build trace with extra entries', () => {
expect(
indexTrace.files.some((file) => file.includes('exclude-me'))
).toBe(false)
expect(
indexTrace.files.some(
(file) => file === '../../../include-me-global.txt'
)
).toBe(true)
expect(
anotherTrace.files.some((file) =>
@@ -106,12 +132,23 @@ describe('build trace with extra entries', () => {
file.includes('nested-structure/dist/constants.js')
)
).toBe(true)
expect(
anotherTrace.files.some(
(file) => file === '../../../include-me-global.txt'
)
).toBe(true)
expect(
imageTrace.files.some((file) => file.includes('public/another.jpg'))
).toBe(true)
expect(
imageTrace.files.some((file) => file.includes('public/test.jpg'))
).toBe(false)
expect(
imageTrace.files.some(
(file) => file === '../../../include-me-global.txt'
)
).toBe(true)
})
}
)
turbopack/crates/turbo-tasks-fs/src/lib.rs
+2
View File
@@ -1249,6 +1249,8 @@ impl FileSystemPath {
Ok(None)
}
/// DETERMINISM: Result is in random order. Either sort result or do not depend
/// on the order.
pub fn read_glob(&self, glob: Vc<Glob>) -> Vc<ReadGlobResult> {
read_glob(self.clone(), glob)
}
turbopack/crates/turbo-tasks-fs/src/read_glob.rs
+121 -49
View File
@@ -4,16 +4,18 @@ use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
use turbo_tasks::{Completion, ResolvedVc, TryJoinIterExt, Vc};
use crate::{DirectoryContent, DirectoryEntry, FileSystem, FileSystemPath, glob::Glob};
use crate::{
DirectoryContent, DirectoryEntry, FileSystem, FileSystemPath, LinkContent, LinkType, glob::Glob,
};
#[turbo_tasks::value]
#[derive(Default, Debug)]
pub struct ReadGlobResult {
pub results: FxHashMap<String, DirectoryEntry>,
pub inner: FxHashMap<String, ResolvedVc<ReadGlobResult>>,
pub results: FxHashMap<RcStr, DirectoryEntry>,
pub inner: FxHashMap<RcStr, ResolvedVc<ReadGlobResult>>,
}
/// Reads matches of a glob pattern.
/// Reads matches of a glob pattern. Symlinks are not resolved (and returned as-is)
///
/// DETERMINISM: Result is in random order. Either sort result or do not depend
/// on the order.
@@ -40,29 +42,64 @@ async fn read_glob_internal(
let dir = directory.read_dir().await?;
let mut result = ReadGlobResult::default();
let glob_value = glob.await?;
let handle_file = |result: &mut ReadGlobResult,
entry_path: &RcStr,
segment: &RcStr,
entry: &DirectoryEntry| {
if glob_value.matches(entry_path) {
result.results.insert(segment.clone(), entry.clone());
}
};
let handle_dir = async |result: &mut ReadGlobResult,
entry_path: RcStr,
segment: &RcStr,
path: &FileSystemPath| {
if glob_value.can_match_in_directory(&entry_path) {
result.inner.insert(
segment.clone(),
read_glob_inner(entry_path, path.clone(), glob)
.to_resolved()
.await?,
);
}
anyhow::Ok(())
};
match &*dir {
DirectoryContent::Entries(entries) => {
for (segment, entry) in entries.iter() {
// This is redundant with logic inside of `read_dir` but here we track it separately
// so we don't follow symlinks.
let entry_path: RcStr = if prefix.is_empty() {
segment.clone()
} else {
format!("{prefix}/{segment}").into()
};
let entry = resolve_symlink_safely(entry.clone()).await?;
if glob_value.matches(&entry_path) {
result.results.insert(entry_path.to_string(), entry.clone());
}
if let DirectoryEntry::Directory(path) = entry
&& glob_value.can_match_in_directory(&entry_path)
{
result.inner.insert(
entry_path.to_string(),
read_glob_inner(entry_path, path.clone(), glob)
.to_resolved()
.await?,
);
match entry {
DirectoryEntry::File(_) => {
handle_file(&mut result, &entry_path, segment, entry);
}
DirectoryEntry::Directory(path) => {
// Add the directory to `results` if it is a whole match of the glob
handle_file(&mut result, &entry_path, segment, entry);
// Recursively handle the directory
handle_dir(&mut result, entry_path, segment, path).await?;
}
DirectoryEntry::Symlink(path) => {
if let LinkContent::Link { link_type, .. } = &*path.read_link().await? {
if link_type.contains(LinkType::DIRECTORY) {
// Ensure that there are no infinite link loops, but don't resolve
resolve_symlink_safely(entry.clone()).await?;
// Add the directory to `results` if it is a whole match of the glob
handle_file(&mut result, &entry_path, segment, entry);
// Recursively handle the directory
handle_dir(&mut result, entry_path, segment, path).await?;
} else {
handle_file(&mut result, &entry_path, segment, entry);
}
}
}
DirectoryEntry::Other(_) | DirectoryEntry::Error => continue,
}
}
}
@@ -71,7 +108,7 @@ async fn read_glob_internal(
Ok(ReadGlobResult::cell(result))
}
// Resolve a symlink checking for recursion.
/// Resolve a symlink checking for recursion.
async fn resolve_symlink_safely(entry: DirectoryEntry) -> Result<DirectoryEntry> {
let resolved_entry = entry.clone().resolve_symlink().await?;
if resolved_entry != entry && matches!(&resolved_entry, DirectoryEntry::Directory(_)) {
@@ -189,8 +226,10 @@ async fn track_glob_internal(
pub mod tests {
use std::{
collections::HashMap,
fs::{File, create_dir},
io::prelude::*,
os::unix::fs::symlink,
};
use turbo_rcstr::{RcStr, rcstr};
@@ -224,13 +263,9 @@ pub mod tests {
));
let path: RcStr = scratch.path().to_str().unwrap().into();
tt.run_once(async {
let fs = Vc::upcast::<Box<dyn FileSystem>>(DiskFileSystem::new(rcstr!("temp"), path));
let read_dir = fs
.root()
.await?
.read_glob(Glob::new(rcstr!("**")))
.await
.unwrap();
let fs = DiskFileSystem::new(rcstr!("temp"), path);
let root = fs.root().await?;
let read_dir = root.read_glob(Glob::new(rcstr!("**"))).await.unwrap();
assert_eq!(read_dir.results.len(), 2);
assert_eq!(
read_dir.results.get("foo"),
@@ -244,26 +279,22 @@ pub mod tests {
let inner = &*read_dir.inner.get("sub").unwrap().await?;
assert_eq!(inner.results.len(), 1);
assert_eq!(
inner.results.get("sub/bar"),
inner.results.get("bar"),
Some(&DirectoryEntry::File(fs.root().await?.join("sub/bar")?))
);
assert_eq!(inner.inner.len(), 0);
// Now with a more specific pattern
let read_dir = fs
.root()
.await?
.read_glob(Glob::new(rcstr!("**/bar")))
.await
.unwrap();
let read_dir = root.read_glob(Glob::new(rcstr!("**/bar"))).await.unwrap();
assert_eq!(read_dir.results.len(), 0);
assert_eq!(read_dir.inner.len(), 1);
let inner = &*read_dir.inner.get("sub").unwrap().await?;
assert_eq!(inner.results.len(), 1);
assert_eq!(
inner.results.get("sub/bar"),
inner.results.get("bar"),
Some(&DirectoryEntry::File(fs.root().await?.join("sub/bar")?))
);
assert_eq!(inner.inner.len(), 0);
anyhow::Ok(())
@@ -278,15 +309,27 @@ pub mod tests {
crate::register();
let scratch = tempfile::tempdir().unwrap();
{
use std::os::unix::fs::symlink;
// Create a simple directory with 1 file and a symlink pointing at at a file in a
// subdirectory
// root.js
// sub/foo.js
// sub/link-foo.js -> ./foo.js
// sub/link-root.js -> ../root.js
let path = scratch.path();
create_dir(path.join("sub")).unwrap();
let foo = path.join("sub/foo.js");
File::create_new(&foo).unwrap().write_all(b"foo").unwrap();
symlink(&foo, path.join("link.js")).unwrap();
symlink(&foo, path.join("sub/link-foo.js")).unwrap();
let root = path.join("root.js");
File::create_new(&root).unwrap().write_all(b"root").unwrap();
symlink(&root, path.join("sub/link-root.js")).unwrap();
let dir = path.join("dir");
create_dir(&dir).unwrap();
File::create_new(dir.join("index.js"))
.unwrap()
.write_all(b"dir index")
.unwrap();
symlink(&dir, path.join("sub/dir")).unwrap();
}
let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
BackendOptions::default(),
@@ -294,19 +337,48 @@ pub mod tests {
));
let path: RcStr = scratch.path().to_str().unwrap().into();
tt.run_once(async {
let fs = Vc::upcast::<Box<dyn FileSystem>>(DiskFileSystem::new(rcstr!("temp"), path));
let read_dir = fs
.root()
.await?
.read_glob(Glob::new(rcstr!("*.js")))
let fs = DiskFileSystem::new(rcstr!("temp"), path);
let root = fs.root().await?;
// Symlinked files
let read_dir = root.read_glob(Glob::new(rcstr!("sub/*.js"))).await.unwrap();
assert_eq!(read_dir.results.len(), 0);
let inner = &*read_dir.inner.get("sub").unwrap().await?;
assert_eq!(
inner.results,
HashMap::from_iter([
(
"link-foo.js".into(),
DirectoryEntry::Symlink(root.join("sub/link-foo.js")?),
),
(
"link-root.js".into(),
DirectoryEntry::Symlink(root.join("sub/link-root.js")?),
),
(
"foo.js".into(),
DirectoryEntry::File(root.join("sub/foo.js")?),
),
])
);
assert_eq!(inner.inner.len(), 0);
// A symlinked folder
let read_dir = root
.read_glob(Glob::new(rcstr!("sub/dir/*")))
.await
.unwrap();
assert_eq!(read_dir.results.len(), 1);
assert_eq!(read_dir.results.len(), 0);
let inner_sub = &*read_dir.inner.get("sub").unwrap().await?;
assert_eq!(inner_sub.results.len(), 0);
let inner_sub_dir = &*inner_sub.inner.get("dir").unwrap().await?;
assert_eq!(
read_dir.results.get("link.js"),
Some(&DirectoryEntry::File(fs.root().await?.join("sub/foo.js")?))
inner_sub_dir.results,
HashMap::from_iter([(
"index.js".into(),
DirectoryEntry::File(root.join("sub/dir/index.js")?),
)])
);
assert_eq!(read_dir.inner.len(), 0);
assert_eq!(inner_sub_dir.inner.len(), 0);
anyhow::Ok(())
})