Unverified Commit 5fc17bbd authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbopack: avoid location-dependent filesystem and tsconfig task outputs

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent f20f4548
...@@ -72,7 +72,6 @@ describe('turbopack-portable-build-cache', () => { ...@@ -72,7 +72,6 @@ describe('turbopack-portable-build-cache', () => {
const sameDirExecutions = await tracedBuild() const sameDirExecutions = await tracedBuild()
expect(executedFunctionNames(sameDirExecutions)).toMatchInlineSnapshot(` expect(executedFunctionNames(sameDirExecutions)).toMatchInlineSnapshot(`
[ [
"<DiskFileSystem as FileSystem>::metadata",
"<DiskFileSystem as FileSystem>::raw_read_dir", "<DiskFileSystem as FileSystem>::raw_read_dir",
"<DiskFileSystem as FileSystem>::read", "<DiskFileSystem as FileSystem>::read",
"<DiskFileSystem as FileSystem>::read_link", "<DiskFileSystem as FileSystem>::read_link",
...@@ -133,7 +132,6 @@ describe('turbopack-portable-build-cache', () => { ...@@ -133,7 +132,6 @@ describe('turbopack-portable-build-cache', () => {
const movedExecutions = await tracedBuild() const movedExecutions = await tracedBuild()
expect(executedFunctionNames(movedExecutions)).toMatchInlineSnapshot(` expect(executedFunctionNames(movedExecutions)).toMatchInlineSnapshot(`
[ [
"<DiskFileSystem as FileSystem>::metadata",
"<DiskFileSystem as FileSystem>::raw_read_dir", "<DiskFileSystem as FileSystem>::raw_read_dir",
"<DiskFileSystem as FileSystem>::read", "<DiskFileSystem as FileSystem>::read",
"<DiskFileSystem as FileSystem>::read_link", "<DiskFileSystem as FileSystem>::read_link",
...@@ -142,7 +140,6 @@ describe('turbopack-portable-build-cache', () => { ...@@ -142,7 +140,6 @@ describe('turbopack-portable-build-cache', () => {
"<NftJsonAsset as Asset>::content", "<NftJsonAsset as Asset>::content",
"<ServerNftJsonAsset as Asset>::content", "<ServerNftJsonAsset as Asset>::content",
"<SourceMapReference as GenerateSourceMap>::generate_source_map", "<SourceMapReference as GenerateSourceMap>::generate_source_map",
"AssetContent::hash",
"DiskFileSystem::new_internal", "DiskFileSystem::new_internal",
"EcmascriptClientReferenceModule::proxy_module", "EcmascriptClientReferenceModule::proxy_module",
"EcmascriptModuleContent::new", "EcmascriptModuleContent::new",
...@@ -279,7 +276,6 @@ describe('turbopack-portable-build-cache', () => { ...@@ -279,7 +276,6 @@ describe('turbopack-portable-build-cache', () => {
"parse", "parse",
"parse_actions", "parse_actions",
"project_root_path_operation", "project_root_path_operation",
"resolve_options",
"styled_jsx_transform_plugin", "styled_jsx_transform_plugin",
"uppercase_env_map", "uppercase_env_map",
] ]
......
...@@ -111,6 +111,8 @@ pub(super) fn make_task_dirty_without_scheduling( ...@@ -111,6 +111,8 @@ pub(super) fn make_task_dirty_without_scheduling(
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
let task_name = task.get_task_name(); let task_name = task.get_task_name();
#[cfg(feature = "trace_task_dirty")]
let task_id = task.id();
if make_stale if make_stale
&& let Some(InProgressState::InProgress(InProgressStateInner { stale, .. })) = && let Some(InProgressState::InProgress(InProgressStateInner { stale, .. })) =
task.get_in_progress_mut() task.get_in_progress_mut()
......
...@@ -277,11 +277,11 @@ impl WriteLinkContent { ...@@ -277,11 +277,11 @@ impl WriteLinkContent {
pub async fn hash(&self, salt: &RcStr, algorithm: HashAlgorithm) -> Result<RcStr> { pub async fn hash(&self, salt: &RcStr, algorithm: HashAlgorithm) -> Result<RcStr> {
// convert the fs Vc (which is not stable across cold builds therefore cannot implement // convert the fs Vc (which is not stable across cold builds therefore cannot implement
// DeterministicHash) to the configured name, which should be globally unique and stable // DeterministicHash) to the configured name, which should be globally unique and stable
// across cold builds. // across cold builds. Only read the name: the whole filesystem also contains its absolute
let target_fs = ResolvedVc::try_downcast_type::<DiskFileSystem>(self.target.fs) // root, which would make this depend on the project location.
.context("link target must use a disk filesystem")? ResolvedVc::try_downcast_type::<DiskFileSystem>(self.target.fs)
.await?; .context("link target must use a disk filesystem")?;
let target_fs_name = target_fs.name(); let target_fs_name = &*self.target.fs.to_string().await?;
Ok(RcStr::from(deterministic_hash( Ok(RcStr::from(deterministic_hash(
salt, salt,
......
...@@ -27,7 +27,7 @@ use turbo_rcstr::{RcStr, rcstr}; ...@@ -27,7 +27,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
CapturedEffect, Completion, Effect, EffectExt, EffectStateStorage, InvalidationReason, CapturedEffect, Completion, Effect, EffectExt, EffectStateStorage, InvalidationReason,
Invalidator, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TurboTasksApi, ValueToString, Vc, Invalidator, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TurboTasksApi, ValueToString, Vc,
debug::ValueDebugFormat, turbo_tasks_weak, turbobail, debug::ValueDebugFormat, turbo_tasks_weak, turbobail, turbofmt,
}; };
use turbo_tasks_hash::{hash_xxh3_hash64, hash_xxh3_hash128}; use turbo_tasks_hash::{hash_xxh3_hash64, hash_xxh3_hash128};
use turbo_unix_path::{normalize_path, sys_to_unix, unix_to_sys}; use turbo_unix_path::{normalize_path, sys_to_unix, unix_to_sys};
...@@ -1711,11 +1711,19 @@ impl FileSystem for DiskFileSystem { ...@@ -1711,11 +1711,19 @@ impl FileSystem for DiskFileSystem {
self.inner.register_read_invalidator(&full_path).await?; self.inner.register_read_invalidator(&full_path).await?;
let _lock = self.inner.lock_path(full_path.clone()).await; let _lock = self.inner.lock_path(full_path.clone()).await;
let meta = retry_blocking(|| std::fs::metadata(&**full_path)) let meta = match retry_blocking(|| std::fs::metadata(&**full_path))
.instrument(tracing::info_span!("read metadata", name = ?full_path)) .instrument(tracing::info_span!("read metadata", name = ?full_path))
.concurrency_limited(&self.inner.read_semaphore) .concurrency_limited(&self.inner.read_semaphore)
.await .await
.with_context(|| format!("reading metadata for {:?}", full_path))?; {
Ok(meta) => meta,
// The error is part of this task's output. Use the location-independent path instead
// of the absolute one, so that the output doesn't change when the project is moved.
Err(err) => {
return Err(anyhow::Error::new(err)
.context(turbofmt!("reading metadata for {fs_path}").await?));
}
};
Ok(FileMeta::cell(meta.into())) Ok(FileMeta::cell(meta.into()))
} }
...@@ -1877,6 +1885,47 @@ mod tests { ...@@ -1877,6 +1885,47 @@ mod tests {
Ok(()) Ok(())
} }
/// The error of a failed `metadata` call is part of the task output, so it must not contain
/// the absolute root path. Otherwise moving the project would invalidate every dependent.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_metadata_error_is_location_independent() {
let root = tempfile::tempdir().unwrap();
let root = RcStr::from(root.path().to_str().unwrap());
let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
BackendOptions::default(),
noop_backing_storage(),
));
tt.run_once(async move {
let error = metadata_error_operation(root.clone())
.read_strongly_consistent()
.await?;
assert!(
error.contains("reading metadata for [project]/missing.json"),
"unexpected error: {error}"
);
assert!(
!error.contains(root.as_str()),
"error contains the absolute root path: {error}"
);
anyhow::Ok(())
})
.await
.unwrap();
}
#[turbo_tasks::function(operation, root)]
async fn metadata_error_operation(sys_root: RcStr) -> anyhow::Result<Vc<RcStr>> {
let fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(sys_root));
let path = fs.root().await?.join("missing.json")?;
let error = path
.metadata()
.await
.err()
.context("expected reading the metadata of a missing file to fail")?;
Ok(Vc::cell(format!("{error:#}").into()))
}
#[cfg(test)] #[cfg(test)]
mod symlink_tests { mod symlink_tests {
use std::{ use std::{
......
...@@ -2,7 +2,7 @@ use anyhow::Result; ...@@ -2,7 +2,7 @@ use anyhow::Result;
use next_taskless::{BUN_EXTERNALS, EDGE_NODE_EXTERNALS, NODE_EXTERNALS}; use next_taskless::{BUN_EXTERNALS, EDGE_NODE_EXTERNALS, NODE_EXTERNALS};
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::Vc; use turbo_tasks::Vc;
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::{FileSystemEntryType, FileSystemPath};
use turbopack_core::resolve::{ use turbopack_core::resolve::{
AliasMap, AliasPattern, ExternalTraced, ExternalType, FindContextFileResult, find_context_file, AliasMap, AliasPattern, ExternalTraced, ExternalType, FindContextFileResult, find_context_file,
options::{ options::{
...@@ -246,8 +246,18 @@ pub async fn resolve_options( ...@@ -246,8 +246,18 @@ pub async fn resolve_options(
TsConfigHandling::Disabled => resolve_options, TsConfigHandling::Disabled => resolve_options,
TsConfigHandling::ContextFile => find_tsconfig().await?, TsConfigHandling::ContextFile => find_tsconfig().await?,
TsConfigHandling::Fixed(tsconfig_path) => { TsConfigHandling::Fixed(tsconfig_path) => {
let meta = tsconfig_path.metadata().await; // Check existence via the entry type (derived from the parent directory listing)
if meta.is_ok() { // instead of reading the file's metadata. This is cheaper, and doesn't depend on
// the error that reading the metadata of a missing file returns.
//
// A symlink counts as existing, even if it's dangling: reading it will fail
// loudly instead of silently ignoring it. A failure to read the parent directory
// is treated like a missing file.
let exists = matches!(
tsconfig_path.get_type().await.as_deref(),
Ok(FileSystemEntryType::File | FileSystemEntryType::Symlink)
);
if exists {
// If the file exists, use it. // If the file exists, use it.
apply_tsconfig_resolve_options( apply_tsconfig_resolve_options(
resolve_options, resolve_options,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment