Unverified Commit 08aab14b authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: avoid project-wide traces from package directories (#99238)

### What?

Limit Turbopack's package-origin project-root tracing to statically
named **files**, while resolving the complete original request relative
to the package as before. This applies to filesystem, directory,
child-process and Node worker references.

### Why?

Static does not necessarily mean narrow: `node-gyp-build`'s
`path.resolve(dir || '.')` contains a static `'.'` branch. Tracing that
from the application root as a directory recursively includes unrelated
application assets. We still need packages to find explicitly named
application config files.

### How?

Analysis records two candidate contexts without inspecting the
filesystem: an optional app-root context for static file paths, and a
package-local fallback for the original request. At reference
resolution, the app-side lookup keeps only fully static file matches;
directories and symlinked directories cannot recursively expand there.
The package-side lookup independently traces the full pattern, including
dynamic alternatives and directory contents. Both results are combined
so even a mixed file/directory expression can retain its app config
**and** package-local assets; the combination preserves distinct files
sharing a request key and deduplicates identical modules. Protobuf
`includeDirs` uses the same reference path.

Regression fixtures cover a `node-gyp-build`-style `'.'` fallback, mixed
file/directory alternatives with same-named files in both contexts,
symlinked directories and files, a missing optional config, and Node
worker references. The webpack/@vercel/nft path is unchanged.

**Verification:** 93 tracing tests and 65 core resolve tests pass;
focused Clippy with warnings denied, Rust formatting and diff checks
pass. Production next-site size impact has not been independently
measured in this fixture.

Related: #98628, #98712, #99080.

<!-- NEXT_JS_LLM -->


<!-- fleet fe000a60-fbb8-4835-b6ef-38d5684822a9 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 725705bd
......@@ -486,6 +486,35 @@ impl ModuleResolveResult {
Ok(*ModuleResolveResult::unresolvable())
}
}
/// Combines results from distinct lookup directories. Unlike `alternatives`, entries with
/// identical request keys can point at different modules and must both be kept.
#[turbo_tasks::function]
pub async fn concat(results: Vec<Vc<ModuleResolveResult>>) -> Result<Vc<Self>> {
if results.len() == 1 {
return Ok(results.into_iter().next().unwrap());
}
let mut primary = Vec::new();
let mut affecting_sources = Vec::new();
let mut seen_sources = FxHashSet::default();
for result in results.into_iter().try_join().await? {
primary.extend(
result.primary.iter().map(|(key, item)| {
(key.clone(), expand_duplicate(&result.primary, item).clone())
}),
);
for source in result.affecting_sources.iter().copied() {
if seen_sources.insert(source) {
affecting_sources.push(source);
}
}
}
Self::mark_duplicates(&mut primary);
Ok(Self::cell(Self {
primary: primary.into_boxed_slice(),
affecting_sources: affecting_sources.into_boxed_slice(),
}))
}
}
#[turbo_tasks::task_input]
......@@ -4243,4 +4272,36 @@ mod tests {
.await
.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concat_preserves_distinct_modules_with_same_request_key() {
let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
BackendOptions::default(),
noop_backing_storage(),
));
#[turbo_tasks::function(operation, root)]
async fn run_test() -> Result<Vc<DupCheckResult>> {
let app = make_module(rcstr!("app.js")).to_resolved().await?;
let package = make_module(rcstr!("package.js")).to_resolved().await?;
let key = RequestKey::new(rcstr!("./config.js"));
let app_result = *ModuleResolveResult::module_with_key(key.clone(), app);
let package_result = *ModuleResolveResult::modules([
(key, package),
(RequestKey::new(rcstr!("also-app")), app),
]);
let combined = ModuleResolveResult::concat(vec![app_result, package_result]).await?;
assert_eq!(combined.primary_modules().await?.as_slice(), [app, package]);
Ok(Vc::cell(snapshot_primary(&combined).await?))
}
tt.run_once(async move {
let snap = run_test().read_strongly_consistent().await?;
assert_eq!(
snap.iter().map(String::as_str).collect::<Vec<_>>(),
vec!["module:app.js", "module:package.js", "dup:0"]
);
Ok(())
})
.await
.unwrap();
}
}
......@@ -2059,21 +2059,17 @@ where
ResolveErrorMode::Error
};
let get_traced_project_dir = async |pattern: Pattern| -> Result<(FileSystemPath, Pattern)> {
// Relative filesystem references are resolved from the project root at runtime. Inside
// node_modules, only trace fully static alternatives from there: a dynamic pattern could
// otherwise include much or all of the project. Dynamic-only patterns continue to fall
// back to the package directory, preserving package-local tracing without exposing the
// project root.
let get_traced_project_dirs = async || -> Result<(Option<FileSystemPath>, FileSystemPath)> {
let fallback = source.ident().await?.path.parent();
if let Some(cwd) = compile_time_info.environment().cwd().owned().await? {
if state.allow_project_root_tracing {
return Ok((cwd, pattern));
}
if let Some(static_pattern) = pattern.filter_static() {
return Ok((cwd, static_pattern));
return Ok((None, cwd));
}
// Resolve only static files from the project root. All paths, including dynamic
// alternatives and directories, are still traced from the package directory.
return Ok((Some(cwd), fallback));
}
Ok((source.ident().await?.path.parent(), pattern))
Ok((None, fallback))
};
let get_issue_source =
......@@ -2249,18 +2245,19 @@ where
};
// WorkerThreads resolve URLs relative to import.meta.url
// and string paths relative to the process root
let (context_dir, pat) = if matches!(
let (static_files_only_context_dir, fallback_context_dir) = if matches!(
args.first(),
Some(JsValue::Url(_, JsValueUrlKind::Relative))
) {
(origin.into_trait_ref().await?.origin_path().parent(), pat)
(None, origin.into_trait_ref().await?.origin_path().parent())
} else {
get_traced_project_dir(pat).await?
get_traced_project_dirs().await?
};
analysis.add_reference_code_gen(
WorkerAssetReference::new_node_worker_thread(
origin,
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(pat).to_resolved().await?,
collect_affecting_sources,
get_issue_source(),
......@@ -2588,10 +2585,12 @@ where
return Ok(());
}
}
let (context_dir, pat) = get_traced_project_dir(pat).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
FileSourceReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(pat),
collect_affecting_sources,
get_issue_source(),
......@@ -2626,10 +2625,12 @@ where
return Ok(());
}
}
let (context_dir, pat) = get_traced_project_dir(pat).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
DirAssetReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(pat),
get_issue_source(),
rcstr!("fs.readdir"),
......@@ -2684,10 +2685,12 @@ where
return Ok(());
}
}
let (context_dir, pat) = get_traced_project_dir(pat).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
DirAssetReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(pat),
get_issue_source(),
rcstr!("path.resolve"),
......@@ -2737,10 +2740,12 @@ where
return Ok(());
}
}
let (context_dir, pat) = get_traced_project_dir(pat).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
DirAssetReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(pat),
get_issue_source(),
rcstr!("path.join"),
......@@ -2800,10 +2805,12 @@ where
show_dynamic_warning = true;
}
if !dynamic || !ignore_dynamic_requests {
let (context_dir, pat) = get_traced_project_dir(pat).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
FileSourceReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(pat),
collect_affecting_sources,
IssueSource::from_swc_offsets(
......@@ -3043,11 +3050,12 @@ where
.await?;
js_value_to_pattern(&linked_func_call)
};
let (context_dir, abs_pattern) =
get_traced_project_dir(abs_pattern).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
DirAssetReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(abs_pattern),
get_issue_source(),
rcstr!("express().set"),
......@@ -3117,10 +3125,12 @@ where
.await?;
js_value_to_pattern(&linked_func_call)
};
let (context_dir, abs_pattern) = get_traced_project_dir(abs_pattern).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
analysis.add_reference(
DirAssetReference::new(
context_dir,
static_files_only_context_dir,
fallback_context_dir,
Pattern::new(abs_pattern),
get_issue_source(),
rcstr!("strong-globalize.SetRootDir"),
......@@ -3176,10 +3186,8 @@ where
if args.len() == 2
&& let Some(JsValue::Object { parts, .. }) = args.get(1)
{
// Every `includeDirs` entry below is a constant, so the traced directory is the
// same for all of them and can be computed once from a static pattern.
let (context_dir, _) =
get_traced_project_dir(Pattern::Constant(rcstr!(""))).await?;
let (static_files_only_context_dir, fallback_context_dir) =
get_traced_project_dirs().await?;
let resolved_dirs = parts
.iter()
.filter_map(|object_part| match object_part {
......@@ -3194,7 +3202,8 @@ where
.flatten()
.map(|dir| {
DirAssetReference::new(
context_dir.clone(),
static_files_only_context_dir.clone(),
fallback_context_dir.clone(),
Pattern::new(Pattern::Constant(dir.into())),
get_issue_source(),
rcstr!("protobufjs.load"),
......
......@@ -10,7 +10,7 @@ use turbopack_core::{
raw_module::RawModule,
reference::{DynamicTraceReference, ModuleReference},
resolve::{
ModuleResolveResult, RequestKey,
ModuleResolveResult, RequestKey, ResolveResult,
pattern::{Pattern, PatternMatch, read_matches},
resolve_raw,
},
......@@ -18,11 +18,33 @@ use turbopack_core::{
use crate::references::util::check_and_emit_too_many_matches_warning;
/// Look for statically named files in the project root without expanding directories. The
/// package-local reference is resolved separately with its entire original pattern.
pub async fn resolve_static_files(
context_dir: &Option<FileSystemPath>,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
) -> Result<Option<Vc<ResolveResult>>> {
let Some(context_dir) = context_dir else {
return Ok(None);
};
let Some(static_pattern) = path.await?.filter_static() else {
return Ok(None);
};
Ok(Some(resolve_raw(
context_dir.clone(),
Pattern::new(static_pattern),
collect_affecting_sources,
/* force_in_lookup_dir */ true,
)))
}
#[turbo_tasks::value]
#[derive(Hash, Debug, ValueToString)]
#[value_to_string("raw asset {path}")]
pub struct FileSourceReference {
context_dir: FileSystemPath,
static_files_only_context_dir: Option<FileSystemPath>,
fallback_context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
......@@ -35,14 +57,16 @@ pub struct FileSourceReference {
impl FileSourceReference {
#[turbo_tasks::function]
pub fn new(
context_dir: FileSystemPath,
static_files_only_context_dir: Option<FileSystemPath>,
fallback_context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
origin_fn_name: RcStr,
) -> Vc<Self> {
Self::cell(FileSourceReference {
context_dir,
static_files_only_context_dir,
fallback_context_dir,
path,
collect_affecting_sources,
issue_source,
......@@ -60,24 +84,33 @@ impl ModuleReference for FileSourceReference {
pattern = display(self.path.to_string().await?)
);
async {
let result = resolve_raw(
self.context_dir.clone(),
let fallback = resolve_raw(
self.fallback_context_dir.clone(),
*self.path,
self.collect_affecting_sources,
/* force_in_lookup_dir */ false,
)
.as_raw_module_result()
.to_resolved()
.await?;
.as_raw_module_result();
let result = if let Some(static_files) = resolve_static_files(
&self.static_files_only_context_dir,
self.path,
self.collect_affecting_sources,
)
.await?
{
ModuleResolveResult::concat(vec![static_files.as_raw_module_result(), fallback])
} else {
fallback
};
check_and_emit_too_many_matches_warning(
*result,
result,
self.issue_source,
self.context_dir.clone(),
self.fallback_context_dir.clone(),
self.path,
)
.await?;
Ok(*result)
Ok(result)
}
.instrument(span)
.await
......@@ -105,7 +138,8 @@ impl DynamicTraceReference for FileSourceReference {
#[derive(Hash, Debug, ValueToString)]
#[value_to_string("directory assets {path}")]
pub struct DirAssetReference {
context_dir: FileSystemPath,
static_files_only_context_dir: Option<FileSystemPath>,
fallback_context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
issue_source: IssueSource,
/// The dynamic function whose access triggered this reference (e.g.
......@@ -117,13 +151,15 @@ pub struct DirAssetReference {
impl DirAssetReference {
#[turbo_tasks::function]
pub fn new(
context_dir: FileSystemPath,
static_files_only_context_dir: Option<FileSystemPath>,
fallback_context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
issue_source: IssueSource,
origin_fn_name: RcStr,
) -> Vc<Self> {
Self::cell(DirAssetReference {
context_dir,
static_files_only_context_dir,
fallback_context_dir,
path,
issue_source,
origin_fn_name,
......@@ -215,11 +251,19 @@ impl ModuleReference for DirAssetReference {
pattern = display(self.path.to_string().await?)
);
async {
let result = resolve_reference_from_dir(self.context_dir.clone(), *self.path).await?;
let fallback =
resolve_reference_from_dir(self.fallback_context_dir.clone(), *self.path).await?;
let result = if let Some(static_files) =
resolve_static_files(&self.static_files_only_context_dir, self.path, true).await?
{
ModuleResolveResult::concat(vec![static_files.as_raw_module_result(), fallback])
} else {
fallback
};
check_and_emit_too_many_matches_warning(
result,
self.issue_source,
self.context_dir.clone(),
self.fallback_context_dir.clone(),
self.path,
)
.await?;
......
......@@ -28,7 +28,10 @@ use crate::{
ast_path_trie::{AstPathId, AstPathTrie, AstPathTrieBuilder},
code_gen::{CodeGen, CodeGeneration, IntoCodeGenReference},
create_visitor,
references::pattern_mapping::{PatternMapping, ResolveType},
references::{
pattern_mapping::{PatternMapping, ResolveType},
raw::resolve_static_files,
},
worker_chunk::{WorkerType, module::WorkerLoaderModule},
};
......@@ -53,10 +56,10 @@ pub struct WorkerAssetReference {
pub enum WorkerRequest {
/// Web workers use Request (URLs)
Url(ResolvedVc<Request>),
/// Node.js workers use Pattern (file paths) with a context directory that should be the server
/// working directory
/// Node.js workers resolve static project files and the complete request in the package.
Pattern {
context_dir: FileSystemPath,
static_files_only_context_dir: Option<FileSystemPath>,
fallback_context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
},
......@@ -87,7 +90,8 @@ impl WorkerAssetReference {
pub fn new_node_worker_thread(
origin: ResolvedVc<Box<dyn ResolveOrigin>>,
context_dir: FileSystemPath,
static_files_only_context_dir: Option<FileSystemPath>,
fallback_context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
......@@ -98,7 +102,8 @@ impl WorkerAssetReference {
worker_type: WorkerType::NodeWorkerThread,
origin,
request: WorkerRequest::Pattern {
context_dir,
static_files_only_context_dir,
fallback_context_dir,
path,
collect_affecting_sources,
},
......@@ -130,22 +135,36 @@ impl ModuleReference for WorkerAssetReference {
(
WorkerType::NodeWorkerThread,
WorkerRequest::Pattern {
context_dir,
static_files_only_context_dir,
fallback_context_dir,
path,
collect_affecting_sources,
},
) => {
// Node.js worker resolution uses resolve_raw
let result = resolve_raw(
context_dir.clone(),
let reference_type = ReferenceType::Worker(WorkerReferenceSubType::NodeWorker);
let mut results = Vec::new();
if let Some(static_files) = resolve_static_files(
static_files_only_context_dir,
*path,
*collect_affecting_sources,
)
.await?
{
results.push(
asset_context.process_resolve_result(static_files, reference_type.clone()),
);
}
let fallback = resolve_raw(
fallback_context_dir.clone(),
**path,
*collect_affecting_sources,
/* force_in_lookup_dir */ false,
);
let reference_type = ReferenceType::Worker(WorkerReferenceSubType::NodeWorker);
let result = asset_context.process_resolve_result(result, reference_type.clone());
results
.push(asset_context.process_resolve_result(fallback, reference_type.clone()));
let result = ModuleResolveResult::concat(results);
// Report an error if we cannot resolve
// Report an error if neither location resolves
handle_resolve_error(
result,
reference_type.clone(),
......
const path = require('path')
// Like node-gyp-build: the missing argument has a static cwd fallback.
function resolve(dir) {
return path.resolve(dir || '.')
}
resolve(process.env.ADDON_DIR)
;[
'package.json',
'test/unit/pkg-project-root-directory/input.js',
'test/unit/pkg-project-root-directory/node_modules/some-pkg/index.js',
'test/unit/pkg-project-root-directory/node_modules/some-pkg/package.json',
'test/unit/pkg-project-root-directory/node_modules/some-pkg/local-asset.txt',
]
// This app-owned file is traced as a static alternative, without expanding the `.` directory from the project root.
const path = require('path')
let target = './app-config.js'
if (process.env.USE_ADDON_DIR) target = '.'
path.resolve(target)
;[
'package.json',
'test/unit/pkg-project-root-mixed-directory/app-config.js',
'test/unit/pkg-project-root-mixed-directory/input.js',
'test/unit/pkg-project-root-mixed-directory/node_modules/some-pkg/app-config.js',
'test/unit/pkg-project-root-mixed-directory/node_modules/some-pkg/index.js',
'test/unit/pkg-project-root-mixed-directory/node_modules/some-pkg/package.json',
'test/unit/pkg-project-root-mixed-directory/node_modules/some-pkg/local-asset.txt',
]
......@@ -5,5 +5,12 @@ let configPath = './app-config.js'
if (process.env.CONFIG_PATH) {
configPath = process.env.CONFIG_PATH
}
if (process.env.OPTIONAL_CONFIG) {
configPath = './missing-config.js'
}
fs.existsSync(path.resolve(configPath))
fs.existsSync('./linked-file.js')
const { Worker } = require('node:worker_threads')
new Worker('./worker.js')
......@@ -2,6 +2,11 @@
'package.json',
'test/unit/pkg-project-root-static/app-config.js',
'test/unit/pkg-project-root-static/input.js',
'test/unit/pkg-project-root-static/linked-file-target.js',
'test/unit/pkg-project-root-static/linked-file.js',
'test/unit/pkg-project-root-static/node_modules/some-pkg/index.js',
'test/unit/pkg-project-root-static/node_modules/some-pkg/package.json',
'test/unit/pkg-project-root-static/node_modules/some-pkg/unrelated.js',
'test/unit/pkg-project-root-static/node_modules/some-pkg/worker.js',
'test/unit/pkg-project-root-static/worker.js',
]
;[
'package.json',
'test/unit/pkg-project-root-symlink-directory/input.js',
'test/unit/pkg-project-root-symlink-directory/node_modules/some-pkg/index.js',
'test/unit/pkg-project-root-symlink-directory/node_modules/some-pkg/package.json',
]
......@@ -174,6 +174,9 @@ static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
#[case::pkg_dir_outside_base("pkg-dir-outside-base")]
#[case::pkg_file_outside_base("pkg-file-outside-base")]
#[case::pkg_project_root_static("pkg-project-root-static")]
#[case::pkg_project_root_directory("pkg-project-root-directory")]
#[case::pkg_project_root_mixed_directory("pkg-project-root-mixed-directory")]
#[case::pkg_project_root_symlink_directory("pkg-project-root-symlink-directory")]
// #[case::pkginfo("pkginfo")]
// #[case::pnpm_symlinks("pnpm-symlinks")]
// #[case::prisma_photon("prisma-photon")]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment