Unverified Commit abac2089 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Allow webpack loaders to cross filesystem roots when reading dependencies (#99202)

Fixes https://github.com/vercel/next.js/issues/99157

Normally, we only allow crossing roots at symlink boundaries, but the
webpack loader could cross roots by following a symlink, and we should
allow that.
parent ff6ac752
const fs = require('node:fs')
const path = require('node:path')
module.exports = async function (source) {
const linkedPackage = fs.realpathSync(path.join(this.rootContext, 'linked'))
const siblingRelative = '../../node_modules/sibling/index.js'
const siblingAbsolute = path.resolve(linkedPackage, siblingRelative)
this.addDependency(path.resolve(this.rootContext, 'next.config.js'))
this.addDependency(siblingAbsolute)
const resolve = this.getResolve()
const resolved = await resolve(linkedPackage, siblingRelative)
if (resolved !== siblingAbsolute) {
throw new Error(`Expected ${siblingAbsolute}, got ${resolved}`)
}
return source.replace('unprocessed', 'processed')
}
import value from './input.loader'
export default function Page() {
return <p id="loader-value">{value}</p>
}
...@@ -8,6 +8,13 @@ const nextConfig = { ...@@ -8,6 +8,13 @@ const nextConfig = {
output: 'standalone', output: 'standalone',
outputHashSalt: 'adapter-symlink-test', outputHashSalt: 'adapter-symlink-test',
serverExternalPackages: ['sibling'], serverExternalPackages: ['sibling'],
turbopack: {
rules: {
'*.loader.js': {
loaders: [path.join(__dirname, 'additional-root-dependency-loader.js')],
},
},
},
experimental: { experimental: {
turbopackAdditionalRoots: { turbopackAdditionalRoots: {
linkedPackages: { linkedPackages: {
......
...@@ -65,6 +65,11 @@ describe('turbopack additional roots', () => { ...@@ -65,6 +65,11 @@ describe('turbopack additional roots', () => {
) )
}) })
it('handles absolute paths and an additional root in dependencies from a webpack loader', async () => {
const browser = await next.browser('/loader')
expect(await browser.elementByCss('#loader-value').text()).toBe('processed')
})
it('reports initialization warnings when startup succeeds', () => { it('reports initialization warnings when startup succeeds', () => {
expect(next.cliOutput).toContain('Invalid Turbopack additional root') expect(next.cliOutput).toContain('Invalid Turbopack additional root')
expect(next.cliOutput).not.toContain('overlaps the project root') expect(next.cliOutput).not.toContain('overlaps the project root')
......
...@@ -454,7 +454,7 @@ struct OptionRcStr(Option<RcStr>); ...@@ -454,7 +454,7 @@ struct OptionRcStr(Option<RcStr>);
/// Canonicalizes successive prefixes of `target_sys_path`, from the system root toward the full /// Canonicalizes successive prefixes of `target_sys_path`, from the system root toward the full
/// path, and passes each canonical prefix together with the untouched suffix to `visit`. /// path, and passes each canonical prefix together with the untouched suffix to `visit`.
/// ///
/// Helper for [`DiskFileSystem::resolve_link_target_ancestry_slow_path`] and /// Helper for [`DiskFileSystem::resolve_path_ancestry_slow_path`] and
/// [`DiskFileSystem::lookup_in_file_system_map`] /// [`DiskFileSystem::lookup_in_file_system_map`]
async fn visit_canonicalized_ancestry( async fn visit_canonicalized_ancestry(
target_sys_path: &Path, target_sys_path: &Path,
...@@ -631,6 +631,47 @@ impl DiskFileSystem { ...@@ -631,6 +631,47 @@ impl DiskFileSystem {
}) })
} }
/// Similar to [`try_from_sys_path`], but allows resolving paths that cross roots and may be in
/// a different [`DiskFileSystem`] linked via the [`DiskFileSystemMap`].
///
/// Unlike [`try_from_sys_path`], this conversion is not entirely lexical, and we may attempt to
/// canonicalize some segments of an absolute path.
///
/// Crossing roots should generally only happen at symlink boundaries. If you are not
/// handling a path that was potentially formed by crossing a symlink boundary, you should
/// prefer [`try_from_sys_path`].
///
/// [`try_from_sys_path`]: DiskFileSystem::try_from_sys_path
pub async fn try_from_sys_path_across_roots(
&self,
vc_self: ResolvedVc<DiskFileSystem>,
sys_path: &Path,
relative_to: &FileSystemPath,
) -> Result<Option<FileSystemPath>> {
debug_assert_eq!(
relative_to.fs,
ResolvedVc::upcast(vc_self),
"`relative_to` must be in the current disk filesystem"
);
if let Some(path) = self.try_from_sys_path(vc_self, sys_path, Some(relative_to)) {
return Ok(Some(path));
}
let absolute_path = self
.to_sys_path_raw(relative_to)
.join(sys_path)
.normalize_lexically()?;
if let Some(path) = self
.resolve_path_ancestry_slow_path(vc_self, &absolute_path)
.await?
{
return Ok(Some(path));
}
self.lookup_in_file_system_map(vc_self, &absolute_path)
.await
}
/// Returns the path as a system [`PathBuf`]. Similar to [`DiskFileSystem::to_sys_path`], but /// Returns the path as a system [`PathBuf`]. Similar to [`DiskFileSystem::to_sys_path`], but
/// keeps the internal representation as-is. /// keeps the internal representation as-is.
/// ///
...@@ -667,12 +708,11 @@ impl DiskFileSystem { ...@@ -667,12 +708,11 @@ impl DiskFileSystem {
return sys_path; return sys_path;
} }
/// Used by the slow path of [`DiskFileSystem::read_link`] for absolute link targets. Attempts /// Resolves an absolute path whose spelling differs from the [`DiskFileSystem`] root,
/// to strip the prefix of an absolute symlink target, creating a [`FileSystemPath`] relative to /// creating a [`FileSystemPath`] relative to that root.
/// the [`DiskFileSystem`] root.
/// ///
/// Returns [`None`] if the target never reaches the filesystem root or an ancestor can't be /// Returns [`None`] if the path never reaches the filesystem root or an ancestor can't be
/// canonicalized. `read_link` treats this as `LinkContent::Invalid`. /// canonicalized.
/// ///
/// In some cases that absolute path may contain symlinks, different capitalization, or Windows /// In some cases that absolute path may contain symlinks, different capitalization, or Windows
/// 8.3 short paths. Resolving this requires performing untracked IO outside of the filesystem /// 8.3 short paths. Resolving this requires performing untracked IO outside of the filesystem
...@@ -681,7 +721,7 @@ impl DiskFileSystem { ...@@ -681,7 +721,7 @@ impl DiskFileSystem {
/// ///
/// To avoid performing untracked reads of files outside of the filesystem root, we iteratively /// To avoid performing untracked reads of files outside of the filesystem root, we iteratively
/// canonicalize each prefix of the given `target_sys_path` using a session-dependent task. /// canonicalize each prefix of the given `target_sys_path` using a session-dependent task.
async fn resolve_link_target_ancestry_slow_path( async fn resolve_path_ancestry_slow_path(
&self, &self,
vc_self: ResolvedVc<Self>, vc_self: ResolvedVc<Self>,
target_sys_path: &Path, target_sys_path: &Path,
...@@ -701,7 +741,7 @@ impl DiskFileSystem { ...@@ -701,7 +741,7 @@ impl DiskFileSystem {
/// Looks up a system path in any configured filesystem other than the current filesystem. /// Looks up a system path in any configured filesystem other than the current filesystem.
/// ///
/// Like [`Self::resolve_link_target_ancestry_slow_path`], the fallback handles paths whose /// Like [`Self::resolve_path_ancestry_slow_path`], the fallback handles paths whose
/// spelling differs from a configured root. /// spelling differs from a configured root.
async fn lookup_in_file_system_map( async fn lookup_in_file_system_map(
&self, &self,
...@@ -1013,7 +1053,7 @@ impl FileSystem for DiskFileSystem { ...@@ -1013,7 +1053,7 @@ impl FileSystem for DiskFileSystem {
// path. // path.
if target_fs_path.is_none() { if target_fs_path.is_none() {
target_fs_path = this target_fs_path = this
.resolve_link_target_ancestry_slow_path(self, &target_sys_path) .resolve_path_ancestry_slow_path(self, &target_sys_path)
.await?; .await?;
} }
if target_fs_path.is_none() { if target_fs_path.is_none() {
......
...@@ -10,7 +10,7 @@ import type { Processor } from 'postcss' ...@@ -10,7 +10,7 @@ import type { Processor } from 'postcss'
import postcss from '@vercel/turbopack/postcss' import postcss from '@vercel/turbopack/postcss'
// @ts-ignore // @ts-ignore
import importedConfig from 'CONFIG' import importedConfig from 'CONFIG'
import { getReadEnvVariables, toPath, type TransformIpc } from './transforms' import { getReadEnvVariables, type TransformIpc } from './transforms'
let processor: Processor | undefined let processor: Processor | undefined
...@@ -99,16 +99,16 @@ export default async function transform( ...@@ -99,16 +99,16 @@ export default async function transform(
break break
case 'dependency': case 'dependency':
case 'missing-dependency': case 'missing-dependency':
filePaths.push(toPath(msg.file)) filePaths.push(msg.file)
break break
case 'build-dependency': case 'build-dependency':
buildFilePaths.push(toPath(msg.file)) buildFilePaths.push(msg.file)
break break
case 'dir-dependency': case 'dir-dependency':
directories.push([toPath(msg.dir), msg.glob]) directories.push([msg.dir, msg.glob])
break break
case 'context-dependency': case 'context-dependency':
directories.push([toPath(msg.dir), '**']) directories.push([msg.dir, '**'])
break break
default: default:
// TODO: do we need to do anything here? // TODO: do we need to do anything here?
......
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
*/ */
import type { Channel as Ipc } from '../types' import type { Channel as Ipc } from '../types'
import { relative, isAbsolute, join, sep } from 'path'
import { type StructuredError } from '../error' import { type StructuredError } from '../error'
import { type StackFrame } from '../compiled/stacktrace-parser' import { type StackFrame } from '../compiled/stacktrace-parser'
...@@ -49,23 +48,6 @@ export type IpcRequestMessage = ...@@ -49,23 +48,6 @@ export type IpcRequestMessage =
export type TransformIpc = Ipc<IpcInfoMessage, IpcRequestMessage> export type TransformIpc = Ipc<IpcInfoMessage, IpcRequestMessage>
const contextDir = process.cwd()
export const toPath = (file: string) => {
const relPath = relative(contextDir, file)
if (isAbsolute(relPath)) {
throw new Error(
`Cannot depend on path (${file}) outside of root directory (${contextDir})`
)
}
return sep !== '/' ? relPath.replaceAll(sep, '/') : relPath
}
export const fromPath = (path: string) => {
return join(
/* turbopackIgnore: true */ contextDir,
sep !== '/' ? path.replaceAll('/', sep) : path
)
}
// Patch process.env to track which env vars are read // Patch process.env to track which env vars are read
const originalEnv = process.env const originalEnv = process.env
const readEnvVars = new Set<string>() const readEnvVars = new Set<string>()
......
...@@ -3,18 +3,13 @@ declare const __turbopack_external_require__: { ...@@ -3,18 +3,13 @@ declare const __turbopack_external_require__: {
} & ((id: string, thunk: () => any, esm?: boolean) => any) } & ((id: string, thunk: () => any, esm?: boolean) => any)
import type { Channel as Ipc } from '../types' import type { Channel as Ipc } from '../types'
import { dirname, resolve as pathResolve, relative } from 'path' import { dirname, resolve as pathResolve } from 'path'
import { import {
StackFrame, StackFrame,
parse as parseStackTrace, parse as parseStackTrace,
} from '../compiled/stacktrace-parser' } from '../compiled/stacktrace-parser'
import { structuredError, type StructuredError } from '../error' import { structuredError, type StructuredError } from '../error'
import { import { getReadEnvVariables, type TransformIpc } from './transforms'
fromPath,
getReadEnvVariables,
toPath,
type TransformIpc,
} from './transforms'
import { import {
evaluateBundle, evaluateBundle,
type ImportModuleResult, type ImportModuleResult,
...@@ -211,7 +206,7 @@ const transform = ( ...@@ -211,7 +206,7 @@ const transform = (
ipc ipc
.sendRequest({ .sendRequest({
type: 'trackFileRead', type: 'trackFileRead',
file: relative(contextDir, pathResolve(p)), file: pathResolve(p),
}) })
.then( .then(
() => { () => {
...@@ -348,13 +343,13 @@ const transform = ( ...@@ -348,13 +343,13 @@ const transform = (
.sendRequest({ .sendRequest({
type: 'resolve', type: 'resolve',
options: rustOptions, options: rustOptions,
lookupPath: toPath(lookupPath), lookupPath,
request, request,
}) })
.then((unknownResult) => { .then((unknownResult) => {
let result = unknownResult as { path: string } let result = unknownResult as { path: string }
if (result && typeof result.path === 'string') { if (result && typeof result.path === 'string') {
return fromPath(result.path) return result.path
} else { } else {
throw Error( throw Error(
'Expected { path: string } from resolve request' 'Expected { path: string } from resolve request'
...@@ -409,7 +404,7 @@ const transform = ( ...@@ -409,7 +404,7 @@ const transform = (
const result = (await ipc.sendRequest({ const result = (await ipc.sendRequest({
type: 'importModule', type: 'importModule',
lookupPath: toPath(resourceDir), lookupPath: resourceDir,
request: actualRequest, request: actualRequest,
})) as ImportModuleResult })) as ImportModuleResult
...@@ -561,12 +556,9 @@ const transform = ( ...@@ -561,12 +556,9 @@ const transform = (
filePaths: [ filePaths: [
...result.fileDependencies, ...result.fileDependencies,
...result.missingDependencies, ...result.missingDependencies,
].map(toPath), ],
directories: result.contextDependencies.map((dep) => [ directories: result.contextDependencies.map((dep) => [dep, '**']),
toPath(dep), buildFilePaths: [...buildDependencies].sort(),
'**',
]),
buildFilePaths: [...buildDependencies].map(toPath).sort(),
}) })
if (err) { if (err) {
// Resolve loader paths to include in the error message using // Resolve loader paths to include in the error message using
......
use std::mem::take; use std::{mem::take, path::Path};
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use async_trait::async_trait; use async_trait::async_trait;
...@@ -17,10 +17,11 @@ use turbo_tasks::{ ...@@ -17,10 +17,11 @@ use turbo_tasks::{
}; };
use turbo_tasks_env::ProcessEnv; use turbo_tasks_env::ProcessEnv;
use turbo_tasks_fs::{ use turbo_tasks_fs::{
File, FileContent, FileSystemEntryType, FileSystemPath, DiskFileSystem, File, FileContent, FileSystemEntryType, FileSystemPath,
glob::{Glob, GlobOptions}, glob::{Glob, GlobOptions},
json::parse_json_with_source_context, json::parse_json_with_source_context,
rope::Rope, rope::Rope,
to_sys_path,
}; };
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -231,14 +232,23 @@ async fn webpack_loaders_executor( ...@@ -231,14 +232,23 @@ async fn webpack_loaders_executor(
)) ))
} }
async fn loader_path(cwd: &FileSystemPath, path: &str) -> Result<FileSystemPath> {
let fs = ResolvedVc::try_downcast_type::<DiskFileSystem>(cwd.fs)
.context("webpack loader working directory must be on a disk filesystem")?;
fs.await?
.try_from_sys_path_across_roots(fs, Path::new(path), cwd)
.await?
.with_context(|| format!("webpack loader path {path:?} is outside configured roots"))
}
// TODO: This creates a task that's keyed on the list of file paths, which might churn through tasks
// (creating more pressure for task GC).
#[turbo_tasks::function] #[turbo_tasks::function]
async fn build_files_changed( async fn build_files_changed(
cwd: FileSystemPath, paths: Vec<FileSystemPath>,
paths: Vec<RcStr>,
source: ResolvedVc<Box<dyn Source>>, source: ResolvedVc<Box<dyn Source>>,
) -> Result<Vc<Completion>> { ) -> Result<Vc<Completion>> {
for path in paths { for path in paths {
let path = cwd.join(&path)?;
let entry_type = path.get_type().await?; let entry_type = path.get_type().await?;
match &*entry_type { match &*entry_type {
FileSystemEntryType::File | FileSystemEntryType::NotFound => { FileSystemEntryType::File | FileSystemEntryType::NotFound => {
...@@ -717,22 +727,25 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -717,22 +727,25 @@ impl EvaluateContext for WebpackLoaderContext {
.try_join(); .try_join();
let file_subscriptions = file_paths let file_subscriptions = file_paths
.iter() .iter()
.map(async |p| self.cwd.join(p)?.read().await) .map(async |p| loader_path(&self.cwd, p).await?.read().await)
.try_join(); .try_join();
let build_file_subscriptions = async { let build_file_subscriptions = async {
build_files_changed( // Resolve to `FileSystemPath`s before calling the task: system paths
self.cwd.clone(), // aren't valid task inputs, as they aren't portable across machines.
build_file_paths, let build_file_paths = build_file_paths
*self.context_source_for_issue, .iter()
) .map(|p| loader_path(&self.cwd, p))
.await?; .try_join()
.await?;
build_files_changed(build_file_paths, *self.context_source_for_issue)
.await?;
Ok::<_, anyhow::Error>(()) Ok::<_, anyhow::Error>(())
}; };
let directory_subscriptions = directories let directory_subscriptions = directories
.iter() .iter()
.map(async |(dir, glob)| { .map(async |(dir, glob)| {
self.cwd loader_path(&self.cwd, dir)
.join(dir)? .await?
.track_glob(Glob::new(glob.clone(), GlobOptions::default()), false) .track_glob(Glob::new(glob.clone(), GlobOptions::default()), false)
.await .await
}) })
...@@ -779,7 +792,7 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -779,7 +792,7 @@ impl EvaluateContext for WebpackLoaderContext {
let Some(resolve_options_context) = self.resolve_options_context else { let Some(resolve_options_context) = self.resolve_options_context else {
bail!("Resolve options are not available in this context"); bail!("Resolve options are not available in this context");
}; };
let lookup_path = self.cwd.join(&lookup_path)?; let lookup_path = loader_path(&self.cwd, &lookup_path).await?;
let request = Request::parse(Pattern::Constant(request)); let request = Request::parse(Pattern::Constant(request));
let options = resolve_options(lookup_path.clone(), *resolve_options_context); let options = resolve_options(lookup_path.clone(), *resolve_options_context);
...@@ -793,15 +806,12 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -793,15 +806,12 @@ impl EvaluateContext for WebpackLoaderContext {
); );
if let Some(source) = resolved.await?.first_source() { if let Some(source) = resolved.await?.first_source() {
if let Some(path) = self.cwd.get_relative_path_to(&source.ident().await?.path) { // Always return absolute paths from the resolve function.
Ok(ResponseMessage::Resolve { path }) let path = to_sys_path(source.ident().await?.path.clone())
} else { .await?
bail!( .context("resolved path is not on a disk filesystem")?;
"Resolving {} in {} ends up on a different filesystem", let path = path.to_str().context("resolved path is not valid UTF-8")?;
request.to_string().await?, Ok(ResponseMessage::Resolve { path: path.into() })
lookup_path.to_string_ref().await?
);
}
} else { } else {
bail!( bail!(
"Unable to resolve {} in {}", "Unable to resolve {} in {}",
...@@ -813,14 +823,14 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -813,14 +823,14 @@ impl EvaluateContext for WebpackLoaderContext {
RequestMessage::TrackFileRead { file } => { RequestMessage::TrackFileRead { file } => {
// Ignore result, we read on the JS side again to prevent some IPC overhead. Still // Ignore result, we read on the JS side again to prevent some IPC overhead. Still
// await the read though to cover at least one class of race conditions. // await the read though to cover at least one class of race conditions.
let _ = &*self.cwd.join(&file)?.read().await?; let _ = &*loader_path(&self.cwd, &file).await?.read().await?;
Ok(ResponseMessage::TrackFileRead {}) Ok(ResponseMessage::TrackFileRead {})
} }
RequestMessage::ImportModule { RequestMessage::ImportModule {
lookup_path, lookup_path,
request, request,
} => { } => {
let lookup_path = self.cwd.join(&lookup_path)?; let lookup_path = loader_path(&self.cwd, &lookup_path).await?;
let request_vc = Request::parse(Pattern::Constant(request.clone())); let request_vc = Request::parse(Pattern::Constant(request.clone()));
let origin = PlainResolveOrigin::new(*self.asset_context, lookup_path.join("_")?); let origin = PlainResolveOrigin::new(*self.asset_context, lookup_path.join("_")?);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment