Unverified Commit d917d2e4 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Prevent accidental access to `.next` (#84714)

Certain patterns in source code cause turbopack to scan the project directories for resources.  This can go wrong when one of those patterns can traverse into .next (oroborous!).

There is no usecase for turbopack to read files from the distDir during analysis and we can prevent that with a simple filter at the FileSystem layer which is what this PR does.  Given the recent `build/dev` split this is a little subtle, so i added a new property to `NextConfigComplete` so we always pass the original `distDir` down even when we are performing the `dev` split.

The semantics of this are implemented at the filesystem layer which means:
* writing into a 'denied_path' is an error
* reading a denied path is 'not found'
* reading the parent directory of a denied path filters it.

Closes PACK-5593
parent b4080311
......@@ -9525,6 +9525,7 @@ dependencies = [
"turbo-tasks-env",
"turbo-tasks-fs",
"turbo-tasks-malloc",
"turbo-unix-path",
"turbopack",
"turbopack-bench",
"turbopack-browser",
......
......@@ -574,7 +574,7 @@ async fn benchmark_file_io(turbo_tasks: NextTurboTasks, directory: FileSystemPat
))?
.await?;
let directory = fs.to_sys_path(directory)?;
let directory = fs.to_sys_path(&directory);
let temp_path = directory.join(format!(
"tmp_file_io_benchmark_{:x}",
rand::random::<u128>()
......
......@@ -82,19 +82,17 @@ fn get_output_specifier(
ident_folder_in_project_fs: &FileSystemPath,
output_root: &FileSystemPath,
project_root: &FileSystemPath,
) -> Result<Option<RcStr>> {
) -> Result<RcStr> {
// include assets in the outputs such as referenced chunks
if path_ref.is_inside_ref(output_root) {
return Ok(Some(ident_folder.get_relative_path_to(path_ref).unwrap()));
return Ok(ident_folder.get_relative_path_to(path_ref).unwrap());
}
// include assets in the project root such as images and traced references (externals)
if path_ref.is_inside_ref(project_root) {
return Ok(Some(
ident_folder_in_project_fs
.get_relative_path_to(path_ref)
.unwrap(),
));
return Ok(ident_folder_in_project_fs
.get_relative_path_to(path_ref)
.unwrap());
}
// This should effectively be unreachable
......@@ -234,15 +232,20 @@ impl Asset for NftJsonAsset {
};
// Collect base assets first
for referenced_chunk in
all_assets_from_entries_filtered(Vc::cell(entries), Some(client_root), exclude_glob)
.await?
{
if chunk.eq(referenced_chunk) {
let all_assets = all_assets_from_entries_filtered(
Vc::cell(entries),
Some(client_root),
exclude_glob,
)
.await?;
for referenced_chunk in all_assets.iter().copied() {
if chunk.eq(&referenced_chunk) {
continue;
}
let referenced_chunk_path = referenced_chunk.path().await?;
if referenced_chunk_path.has_extension(".map") {
continue;
}
......@@ -279,16 +282,13 @@ impl Asset for NftJsonAsset {
}
}
let Some(specifier) = get_output_specifier(
let specifier = get_output_specifier(
&referenced_chunk_path,
&ident_folder,
&ident_folder_in_project_fs,
&output_root_ref,
&project_root_ref,
)?
else {
continue;
};
)?;
result.insert(specifier);
}
......@@ -357,7 +357,7 @@ impl Asset for NftJsonAsset {
/// The glob walker in turbopack is somewhat naive so we handle relative path directives first so
/// traversal doesn't need to consider them and can just traverse 'down' the tree.
/// The main alternative is to merge glob evaluation with directory traversal which is what the npm
/// `glob` package does, but this would be a substantial rewrite.`
/// `glob` package does, but this would be a substantial rewrite.
pub(crate) fn relativize_glob(
glob: &str,
relative_to: FileSystemPath,
......@@ -478,12 +478,12 @@ async fn get_referenced_server_assets(
client_root: Option<FileSystemPath>,
exclude_glob: Option<ReadRef<Glob>>,
) -> Result<Vec<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>> {
asset
.references()
.await?
.iter()
let refs = asset.references().await?;
refs.iter()
.map(async |asset| {
let asset_path = asset.path().await?;
if let Some(client_root) = &client_root
&& asset_path.is_inside_ref(client_root)
{
......
......@@ -479,18 +479,15 @@ impl ProjectContainer {
current_node_js_version = options.current_node_js_version.clone();
}
let dist_dir = next_config
.dist_dir()
.await?
.as_ref()
.map_or_else(|| rcstr!(".next"), |d| d.clone());
let dist_dir = next_config.dist_dir().owned().await?;
let dist_dir_root = next_config.dist_dir_root().owned().await?;
Ok(Project {
root_path,
project_path,
watch,
next_config: next_config.to_resolved().await?,
dist_dir,
dist_dir_root,
env: ResolvedVc::upcast(env_map.to_resolved().await?),
define_env: define_env.to_resolved().await?,
browserslist_query,
......@@ -554,6 +551,11 @@ pub struct Project {
/// E.g. `.next`
dist_dir: RcStr,
/// The root directory of the distDir. Generally the same as `distDir` but when
/// `isolatedDevBuild` is true it is the parent directory of `distDir`. This is used to
/// ensure that the bundler doesn't traverse into the output directory.
dist_dir_root: RcStr,
/// Filesystem watcher options.
watch: WatchOptions,
......@@ -668,8 +670,23 @@ impl Project {
}
#[turbo_tasks::function]
pub fn project_fs(&self) -> Vc<DiskFileSystem> {
DiskFileSystem::new(PROJECT_FILESYSTEM_NAME.into(), self.root_path.clone())
pub fn project_fs(&self) -> Result<Vc<DiskFileSystem>> {
let denied_path = match join_path(&self.project_path, &self.dist_dir_root) {
Some(dist_dir_root) => dist_dir_root.into(),
None => {
bail!(
"Invalid distDirRoot: {:?}. distDirRoot should not navigate out of the \
projectPath.",
self.dist_dir_root
);
}
};
Ok(DiskFileSystem::new_with_denied_path(
rcstr!(PROJECT_FILESYSTEM_NAME),
self.root_path.clone(),
denied_path,
))
}
#[turbo_tasks::function]
......
......@@ -96,7 +96,8 @@ pub struct NextConfig {
react_strict_mode: Option<bool>,
transpile_packages: Option<Vec<RcStr>>,
modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>,
dist_dir: Option<RcStr>,
dist_dir: RcStr,
dist_dir_root: RcStr,
deployment_id: Option<RcStr>,
sass_options: Option<serde_json::Value>,
trailing_slash: Option<bool>,
......@@ -1590,9 +1591,13 @@ impl NextConfig {
}
#[turbo_tasks::function]
pub fn dist_dir(&self) -> Vc<Option<RcStr>> {
pub fn dist_dir(&self) -> Vc<RcStr> {
Vc::cell(self.dist_dir.clone())
}
#[turbo_tasks::function]
pub fn dist_dir_root(&self) -> Vc<RcStr> {
Vc::cell(self.dist_dir_root.clone())
}
#[turbo_tasks::function]
pub fn experimental_cache_handlers(
......
......@@ -13,8 +13,13 @@
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "react-jsx",
"incremental": true
"incremental": true,
"plugins": [
{
"name": "next"
}
]
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
}
......@@ -25,6 +25,10 @@ export type NextConfigComplete = Required<Omit<NextConfig, 'configFile'>> & {
// because it's not defined in NextConfigComplete.experimental
htmlLimitedBots: string | undefined
experimental: ExperimentalConfig
// The root directory of the distDir. Generally the same as `distDir` but when `isolatedDevBuild`
// is true it is the parent directory of `distDir`. This is used to ensure that the bundler doesn't
// traverse into the output directory.
distDirRoot: string
}
export type I18NDomains = readonly DomainLocale[]
......
......@@ -1293,6 +1293,8 @@ function assignDefaultsAndValidate(
result.experimental.useCache = result.experimental.cacheComponents
}
// Store the distDirRoot in the config before it is modified by the isolatedDevBuild flag
;(result as NextConfigComplete).distDirRoot = result.distDir
if (
phase === PHASE_DEVELOPMENT_SERVER &&
result.experimental?.isolatedDevBuild
......
import fs from 'fs'
import path from 'path'
export default async function Page() {
let config = 'file from .next not found'
try {
const rootConfig = fs.readFileSync(
path.join(__dirname, '../../', '.next', globalThis.afile),
'utf-8'
)
config = `Config loaded: ${rootConfig.substring(0, 50)}...`
} catch (error) {
config = `File from next not found (this is expected in test)`
}
return (
<div>
<h1>Dynamic File Read Test</h1>
<p>This page uses dynamic fs.readFileSync that trace into odd places</p>
<p>{config}</p>
</div>
)
}
......@@ -26,8 +26,8 @@ const isReact18 = parseInt(process.env.NEXT_TEST_REACT_VERSION) === 18
}
async function readNormalizedNFT(name) {
let data = await next.readJSON(name)
let result = [
const data = await next.readJSON(name)
const result = [
...new Set(
data.files
.filter((file: string) => {
......@@ -39,9 +39,14 @@ const isReact18 = parseInt(process.env.NEXT_TEST_REACT_VERSION) === 18
}
// Filter out the many symlinks that power node_modules
let fileAbsolute = path.join(next.testDir, name, '..', file)
if (fs.lstatSync(fileAbsolute).isSymbolicLink()) {
return false
const fileAbsolute = path.join(next.testDir, name, '..', file)
try {
if (fs.lstatSync(fileAbsolute).isSymbolicLink()) {
return false
}
} catch (e) {
// File doesn't exist - this is a bug in the NFT generation!
// Keep it in the list so the test can catch it
}
return true
})
......@@ -59,8 +64,8 @@ const isReact18 = parseInt(process.env.NEXT_TEST_REACT_VERSION) === 18
}
// Strip double node_modules to simplify output
let firstNodeModules = file.indexOf('/node_modules/')
let lastNodeModules = file.lastIndexOf('/node_modules/')
const firstNodeModules = file.indexOf('/node_modules/')
const lastNodeModules = file.lastIndexOf('/node_modules/')
if (firstNodeModules !== lastNodeModules) {
return file.slice(lastNodeModules)
}
......@@ -74,12 +79,12 @@ const isReact18 = parseInt(process.env.NEXT_TEST_REACT_VERSION) === 18
}
it('should not trace too many files in next-server.js.nft.json', async () => {
let trace = await readNormalizedNFT('.next/next-server.js.nft.json')
const trace = await readNormalizedNFT('.next/next-server.js.nft.json')
// Group the entries together so that the snapshot doesn't change too often.
// This trace contains quite a lot of files that aren't actually needed. But there isn't much
// that Turbopack itself can do about that.
let traceGrouped = [
const traceGrouped = [
...new Set(
trace.map((file: string) => {
if (file.startsWith('/node_modules/next/')) {
......@@ -228,8 +233,33 @@ const isReact18 = parseInt(process.env.NEXT_TEST_REACT_VERSION) === 18
`)
})
it('should not include .next directory in traces despite dynamic fs operations', async () => {
// This test verifies that the denied_path feature prevents the .next directory
// from being included in traces. The app/dynamic-read page uses dynamic fs.readFileSync
// with path.join(process.cwd(), ...) which could theoretically read any file.
// Check the page-specific trace that has the dynamic fs operations
const pageTrace = await readNormalizedNFT(
'.next/server/app/dynamic-read/page.js.nft.json'
)
// Snapshot the non-node_modules and non-chunks files to see what's being traced
// We also filter out chunks because their names change with every build
const nonNodeModulesFiles = pageTrace.filter(
(file: string) =>
!file.includes('/node_modules/') && !file.includes('/chunks/')
)
expect(nonNodeModulesFiles).toMatchInlineSnapshot(`
[
"./page/react-loadable-manifest.json",
"./page_client-reference-manifest.js",
]
`)
})
it('should not trace too many files in next-minimal-server.js.nft.json', async () => {
let trace = await readNormalizedNFT(
const trace = await readNormalizedNFT(
'.next/next-minimal-server.js.nft.json'
)
expect(trace).toMatchInlineSnapshot(`
......
This diff is collapsed.
......@@ -28,20 +28,21 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
.context("Expected root to have a DiskFileSystem")?
.await?;
let path = match path {
Some(path) => root.join(path)?,
None => root,
};
let sys_path = root_fs.to_sys_path(&path);
let sys_path = sys_path.to_string_lossy();
Ok(format!(
"file://{}",
&sys_to_unix(
&root_fs
.to_sys_path(match path {
Some(path) => root.join(path)?,
None => root,
})?
.to_string_lossy()
)
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
sys_to_unix(&sys_path)
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
))
}
......@@ -52,10 +53,11 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
.context("Expected root to have a DiskFileSystem")?
.await?;
let sys_path = root_fs.to_sys_path(match path {
let sys_path = match path {
Some(path) => root.join(path.into())?,
None => root,
})?;
};
let sys_path = root_fs.to_sys_path(&sys_path);
let raw_path = sys_path.to_string_lossy().to_string();
let normalized_path = raw_path.replace('\\', "/");
......
......@@ -34,6 +34,7 @@ pub fn unix_to_sys(path: &str) -> Cow<'_, str> {
/// Paths are concatenated with /.
///
/// see also [normalize_path] for normalization.
/// Returns `None` if the joined path would leave the filesystem root.
pub fn join_path(fs_path: &str, join: &str) -> Option<String> {
// Paths that we join are written as source code (eg, `join_path(fs_path, "foo/bar.js")`) and
// it's expected that they will never contain a backslash.
......
......@@ -53,6 +53,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-backend = { workspace = true }
turbo-tasks-env = { workspace = true }
turbo-unix-path = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-tasks-malloc = { workspace = true, default-features = false }
turbopack = { workspace = true }
......
......@@ -14,6 +14,7 @@ use turbo_tasks_backend::{
BackendOptions, NoopBackingStorage, TurboTasksBackend, noop_backing_storage,
};
use turbo_tasks_fs::FileSystem;
use turbo_unix_path::join_path;
use turbopack::{
css::chunk::CssChunkType, ecmascript::chunk::EcmascriptChunkType,
global_module_ids::get_global_module_id_strategy,
......@@ -195,21 +196,28 @@ async fn build_internal(
scope_hoist: bool,
) -> Result<Vc<()>> {
let output_fs = output_fs(project_dir.clone());
let project_fs = project_fs(root_dir.clone(), /* watch= */ false);
const OUTPUT_DIR: &str = "dist";
let project_relative = project_dir.strip_prefix(&*root_dir).unwrap();
let project_relative: RcStr = project_relative
.strip_prefix(MAIN_SEPARATOR)
.unwrap_or(project_relative)
.replace(MAIN_SEPARATOR, "/")
.into();
let project_fs = project_fs(
root_dir.clone(),
/* watch= */ false,
join_path(project_relative.as_str(), OUTPUT_DIR)
.unwrap()
.into(),
);
let root_path = project_fs.root().owned().await?;
let project_path = root_path.join(&project_relative)?;
let build_output_root = output_fs.root().await?.join("dist")?;
let build_output_root = output_fs.root().await?.join(OUTPUT_DIR)?;
let node_env = NodeEnv::Production.cell();
let build_output_root_to_root_path = project_path
.join("dist")?
.join(OUTPUT_DIR)?
.get_relative_path_to(&root_path)
.context("Project path is in root path")?;
......
......@@ -22,6 +22,7 @@ use turbo_tasks_backend::{
};
use turbo_tasks_fs::FileSystem;
use turbo_tasks_malloc::TurboMalloc;
use turbo_unix_path::join_path;
use turbopack::evaluate_context::node_build_environment;
use turbopack_cli_utils::issue::{ConsoleUi, LogOptions};
use turbopack_core::{
......@@ -265,15 +266,22 @@ async fn source(
.into();
let output_fs = output_fs(project_dir);
let fs: Vc<Box<dyn FileSystem>> = project_fs(root_dir, /* watch= */ true);
const OUTPUT_DIR: &str = ".turbopack/build";
let fs: Vc<Box<dyn FileSystem>> = project_fs(
root_dir,
/* watch= */ true,
join_path(project_relative.as_str(), OUTPUT_DIR)
.unwrap()
.into(),
);
let root_path = fs.root().owned().await?;
let project_path = root_path.join(&project_relative)?;
let env = load_env(root_path.clone());
let build_output_root = output_fs.root().await?.join(".turbopack/build")?;
let build_output_root = output_fs.root().await?.join(OUTPUT_DIR)?;
let build_output_root_to_root_path = project_path
.join(".turbopack/build")?
.join(OUTPUT_DIR)?
.get_relative_path_to(&root_path)
.context("Project path is in root path")?;
let build_output_root_to_root_path = build_output_root_to_root_path;
......
......@@ -60,8 +60,13 @@ pub fn normalize_entries(entries: &Option<Vec<String>>) -> Vec<RcStr> {
}
#[turbo_tasks::function]
pub async fn project_fs(project_dir: RcStr, watch: bool) -> Result<Vc<Box<dyn FileSystem>>> {
let disk_fs = DiskFileSystem::new(rcstr!("project"), project_dir);
pub async fn project_fs(
project_dir: RcStr,
watch: bool,
denied_root_path: RcStr,
) -> Result<Vc<Box<dyn FileSystem>>> {
let disk_fs =
DiskFileSystem::new_with_denied_path(rcstr!("project"), project_dir, denied_root_path);
if watch {
disk_fs.await?.start_watching(None).await?;
}
......
......@@ -102,7 +102,7 @@ async fn node_file_trace_operation(
.cell(),
ResolveOptionsContext {
enable_node_native_modules: true,
enable_node_modules: Some(input_dir.clone()),
enable_node_modules: Some(input_dir),
custom_conditions: vec![rcstr!("node")],
loose_errors: true,
collect_affecting_sources: true,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment