Unverified Commit 49dc4c84 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: fix output tracing include glob traversal (#98382)

### What?

Make Turbopack's `outputFileTracingIncludes` file patterns root-relative
and targeted during output tracing. Reject unanchored partial globs when
code attempts directory-prefix matching. Add production coverage using a
real pnpm-installed WASM package, plus unit coverage for anchored
directory pruning and include-pattern expansion.

### Why?

Turbopack previously treated include file patterns as unanchored partial
matches. The directory-prefix matcher could therefore accept every
descendant after matching the first literal segment: an exact WASM
include under `node_modules/.pnpm` caused 2,093 `read_glob_inner`
executions instead of traversing only its 5 directory prefixes.

Unanchored patterns also cannot soundly answer whether a directory may
contain a future match, so allowing `contains: true` with
directory-prefix matching risks both over-traversal and missed files.
Include behavior also differed from webpack and the documented
project-root-relative semantics.

### How?

Build anchored alternatives for each include pattern: one for the direct
match and one for descendants of a matched directory. Anchoring lets
`read_glob` prune unrelated directories, while the recursive alternative
preserves existing behavior for matched directories and directory
symlinks, including their resolved target contents.

`can_match_in_directory` now fails immediately for `contains: true`,
while direct partial matching remains supported. Route-key and exclusion
matching continue to use direct partial matches and are unchanged.

The production fixture resolves
`lightningcss-wasm/lightningcss_node.wasm` from a real
`lightningcss-wasm@1.28.2` dependency, avoiding assumptions about pnpm's
store naming or patch hashes.

### Verification

- `CARGO_INCREMENTAL=0 pnpm build-all`
- `cargo fmt -p turbo-tasks-fs -p next-api -- --check`
- `cargo test -p turbo-tasks-fs glob` (84 passed)
- `cargo test -p next-api` (17 passed)
- `pnpm test-start-turbo
test/production/app-dir/output-file-tracing-includes-read-glob/output-file-tracing-includes-read-glob.test.ts
test/production/build-trace-extra-entries-turbo/build-trace-extra-entries-turbo.test.ts
test/production/build-trace-extra-entries-monorepo/build-trace-extra-entries-monorepo.test.ts`
(3 passed)
- `pnpm test-start-webpack
test/production/app-dir/output-file-tracing-includes-read-glob/output-file-tracing-includes-read-glob.test.ts`
- The shared fixture asserts that both bundlers include a root-relative
exact match and exclude a nested file with the same suffix
- Measured the exact reproduction at 5 include-specific
`read_glob_inner` executions, down from 2,093

<!-- NEXT_JS_LLM -->


<!-- fleet b3cefb44-e1ce-4bd6-83a9-da2f22848cad -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 4cb54721
......@@ -155,13 +155,7 @@ pub async fn trace_endpoint(
let includes = combined_includes_by_root
.into_iter()
.map(|(root, globs)| {
let glob = Glob::new(
format!("{{{}}}", globs.join(",")).into(),
GlobOptions {
contains: true,
..Default::default()
},
);
let glob = Glob::new(include_glob_pattern(&globs), GlobOptions::default());
get_glob_includes(root, glob)
})
.try_join()
......@@ -183,6 +177,32 @@ pub async fn trace_endpoint(
.await
}
/// Build root-relative include alternatives. The recursive alternative preserves the contents of
/// a directory (including a directory symlink) matched by the user pattern without making the
/// original pattern an unanchored partial match.
fn include_glob_pattern(globs: &[&str]) -> RcStr {
if globs.contains(&"**") {
return rcstr!("**");
}
let mut pattern = String::new();
pattern.push('{');
for (index, glob) in globs.iter().enumerate() {
if index > 0 {
pattern.push(',');
}
pattern.push_str(glob);
// A recursive suffix must remain at the end of its alternative. Appending another suffix
// would produce e.g. `assets/**/**`, which is invalid in the glob parser.
if !glob.ends_with("/**") {
pattern.push(',');
pattern.push_str(glob);
pattern.push_str("/**");
}
}
pattern.push('}');
pattern.into()
}
/// Apply outputFileTracingIncludes patterns to find additional files
async fn get_glob_includes(
project_root_path: FileSystemPath,
......@@ -574,6 +594,81 @@ impl Issue for ForbiddenTracedFileIssue {
}
}
#[cfg(test)]
mod include_glob_tests {
use turbo_rcstr::RcStr;
use turbo_tasks_fs::glob::{Glob, GlobOptions};
use super::include_glob_pattern;
fn glob(patterns: &[&str]) -> Glob {
Glob::parse(include_glob_pattern(patterns), GlobOptions::default()).unwrap()
}
fn plain_glob(pattern: &str) -> Glob {
Glob::parse(RcStr::from(pattern), GlobOptions::default()).unwrap()
}
#[test]
fn include_glob_is_root_relative() {
let glob = glob(&["node_modules/pkg/file.wasm"]);
assert!(glob.matches("node_modules/pkg/file.wasm"));
assert!(!glob.matches("nested/node_modules/pkg/file.wasm"));
assert!(glob.can_match_in_directory("node_modules/pkg"));
assert!(!glob.can_match_in_directory("node_modules/other"));
}
#[test]
fn include_glob_recurses_below_matched_directories() {
let glob = glob(&["assets/*"]);
assert!(glob.matches("assets/file.txt"));
assert!(glob.matches("assets/directory/nested.txt"));
assert!(!glob.matches("other/assets/file.txt"));
}
#[test]
fn include_glob_preserves_recursive_patterns() {
for pattern in ["assets/**/*", "assets/**", "**"] {
let original = plain_glob(pattern);
let expanded = glob(&[pattern]);
for path in [
"assets",
"assets/file.txt",
"assets/directory/nested.txt",
"assets/directory/deeply/nested.txt",
"other/assets/file.txt",
] {
assert_eq!(
original.matches(path),
expanded.matches(path),
"pattern {pattern:?}, path {path:?}"
);
}
}
}
#[test]
fn include_glob_supports_braces_and_multiple_patterns() {
let combined = glob(&["assets/{one,two}", "config/*.json"]);
assert!(combined.matches("assets/one"));
assert!(combined.matches("assets/two/nested.txt"));
assert!(combined.matches("config/runtime.json"));
assert!(!combined.matches("config/nested/runtime.json"));
assert!(!combined.matches("assets/three"));
let recursive = glob(&["assets/**", "config/*.json"]);
assert!(recursive.matches("assets/any/deeply/nested/file"));
assert!(recursive.matches("config/runtime.json"));
let match_all = glob(&["config/*.json", "**"]);
assert!(match_all.matches("any/deeply/nested/file"));
}
}
#[cfg(all(test, unix))]
mod tests {
use std::{
......@@ -597,17 +692,8 @@ mod tests {
.root()
.owned()
.await?;
let includes = get_glob_includes(
root,
Glob::new(
rcstr!("**"),
GlobOptions {
contains: true,
..Default::default()
},
),
)
.await?;
let includes =
get_glob_includes(root, Glob::new(rcstr!("**"), GlobOptions::default())).await?;
assert_eq!(
includes
......
import type { ReactNode } from 'react'
export default function Root({ children }: { children: ReactNode }) {
return (
<html>
<body>{children}</body>
</html>
)
}
export default function Page() {
return <p>hello world</p>
}
import path from 'path'
import type { NextConfig } from 'next'
const wasmPath = path
.relative(
__dirname,
require.resolve('lightningcss-wasm/lightningcss_node.wasm')
)
.replaceAll(path.sep, '/')
const nextConfig: NextConfig = {
outputFileTracingIncludes: {
'/*': [wasmPath, 'include-me/file.txt'],
},
}
export default nextConfig
import path from 'path'
import { nextTestSetup } from 'e2e-utils'
describe('outputFileTracingIncludes read glob', () => {
const { next, skipped } = nextTestSetup({
files: __dirname,
dependencies: {
'lightningcss-wasm': '1.28.2',
},
skipStart: true,
skipDeployment: true,
})
if (skipped) return
it('traces a file from the monorepo node_modules directory', async () => {
const wasmPath = require.resolve(
'lightningcss-wasm/lightningcss_node.wasm',
{ paths: [next.testDir] }
)
const { exitCode } = await next.runCommand(['build'], {
cwd: path.join(next.testDir, 'app'),
})
expect(exitCode).toBe(0)
const traceDirectory = path.join(next.testDir, 'app/.next/server/app')
const toTracePath = (filePath: string) =>
path.relative(traceDirectory, filePath).replaceAll(path.sep, '/')
const trace = JSON.parse(
await next.readFile('app/.next/server/app/page.js.nft.json')
)
expect(trace.files).toContain(toTracePath(wasmPath))
expect(trace.files).toContain(
toTracePath(path.join(next.testDir, 'app/include-me/file.txt'))
)
expect(trace.files).not.toContain(
toTracePath(path.join(next.testDir, 'app/nested/include-me/file.txt'))
)
})
})
......@@ -77,7 +77,10 @@ pub struct GlobOptions {
/// Allows glob to match any part of the given string(s).
/// NOTE: this means that a pattern like `node_modules/package_name` with `contains:true` will
/// match `foo_node_modules/package_name_bar` If you want to match a _directory_ named
/// `node_modules/package_name` you should use `**/node_modules/package_name/**`
/// `node_modules/package_name` you should use `**/node_modules/package_name/**`.
///
/// A partial match cannot safely determine whether a directory might contain a match, so this
/// option cannot be used with [`Glob::can_match_in_directory`].
pub contains: bool,
/// Whether matching should ignore ASCII case differences.
pub case_insensitive: bool,
......@@ -92,6 +95,10 @@ impl Glob {
// Returns true if the glob might match a filename underneath this `path` where the
// path represents a directory.
pub fn can_match_in_directory(&self, path: &str) -> bool {
assert!(
!self.opts.contains,
"Glob::can_match_in_directory cannot be used when GlobOptions::contains is true"
);
debug_assert!(
!path.ends_with('/'),
"Path should be a directory name and not end with /"
......@@ -373,4 +380,34 @@ mod tests {
assert!(!glob.matches(path));
}
#[test]
fn literal_glob_directory_pruning() {
let pattern = rcstr!(
"node_modules/.pnpm/lightningcss-wasm@1.28.2/node_modules/lightningcss-wasm/\
lightningcss_node.wasm"
);
let anchored = Glob::parse(pattern, GlobOptions::default()).unwrap();
assert!(anchored.can_match_in_directory("node_modules"));
assert!(anchored.can_match_in_directory("node_modules/.pnpm"));
assert!(!anchored.can_match_in_directory("node_modules/next"));
}
#[test]
#[should_panic(
expected = "Glob::can_match_in_directory cannot be used when GlobOptions::contains is true"
)]
fn contains_glob_cannot_match_in_directory() {
let glob = Glob::parse(
rcstr!("node_modules/package_name"),
GlobOptions {
contains: true,
..Default::default()
},
)
.unwrap();
glob.can_match_in_directory("node_modules");
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment