Unverified Commit 5e8f31f7 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Improve how DiskWatcher is configured and fix polling watcher bugs (#96440)

- Adds a `DiskWatcherConfig` object to centralize the recursive mode,
polling, and invalidation reason options.
- **Polling Fix:** Use the non-recursive mode if the polling watcher is
selected, ignoring the OS/platform.
- **Polling Fix:** Treat mtime events as data update events if the
polling watcher is selected (this is what
https://github.com/vercel/next.js/pull/96288 was doing, but this PR only
does it if polling is enabled)
- Only treat the `MetadataKind::Any` event as a data change on macOS.

Fixes #80665
Closes #96288
parent a13e48dd
......@@ -45,7 +45,7 @@ use turbo_tasks::{
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystem, FileSystemPath, VirtualFileSystem,
DiskFileSystem, DiskWatcherConfig, FileContent, FileSystem, FileSystemPath, VirtualFileSystem,
canonicalize_to_rcstr, invalidation,
};
use turbo_unix_path::join_path;
......@@ -631,9 +631,7 @@ impl ProjectContainer {
.read_strongly_consistent()
.await?;
if watch.enable {
project_fs
.start_watching_with_invalidation_reason(watch.poll_interval)
.await?;
project_fs.start_watching().await?;
} else {
project_fs.invalidate_with_reason(|path| invalidation::Initialize {
// this path is just used for display purposes
......@@ -776,9 +774,7 @@ impl ProjectContainer {
if !ReadRef::ptr_eq(&prev_project_fs, &project_fs) {
if watch.enable {
// TODO stop watching: prev_project_fs.stop_watching()?;
project_fs
.start_watching_with_invalidation_reason(watch.poll_interval)
.await?;
project_fs.start_watching().await?;
} else {
project_fs.invalidate_with_reason(|path| invalidation::Initialize {
// this path is just used for display purposes
......@@ -1087,10 +1083,16 @@ impl Project {
.unwrap()
.into();
Ok(DiskFileSystem::new_with_denied_paths(
Ok(DiskFileSystem::new_with_options(
PROJECT_FILESYSTEM_NAME,
*self.root_path,
vec![denied_path, denied_profiles_path],
DiskWatcherConfig {
recursive_mode: None,
poll_interval: self.watch.poll_interval,
// the dev server reports these to the user
report_invalidation_reason: true,
},
))
}
......
......@@ -29,7 +29,7 @@ async fn main() -> Result<()> {
Box::pin(async {
let root = RcStr::from(current_dir().unwrap().to_str().unwrap());
let disk_fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(root));
disk_fs.await?.start_watching(None).await?;
disk_fs.await?.start_watching().await?;
// Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
......
......@@ -30,7 +30,7 @@ async fn main() -> Result<()> {
Box::pin(async {
let root = RcStr::from(current_dir().unwrap().to_str().unwrap());
let disk_fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(root));
disk_fs.await?.start_watching(None).await?;
disk_fs.await?.start_watching().await?;
// Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
......
......@@ -9,7 +9,6 @@ use std::{
mem::take,
path::{MAIN_SEPARATOR, Path, PathBuf},
sync::{Arc, LazyLock, Weak},
time::Duration,
};
use anyhow::{Context, Result, anyhow, bail};
......@@ -42,7 +41,7 @@ use crate::{
mutex_map::MutexMap,
path_map::OrderedPathMapExt,
retry::{can_retry, retry_blocking, retry_blocking_custom},
watcher::DiskWatcher,
watcher::{DiskWatcher, DiskWatcherConfig},
};
/// Validate the path, returning the valid path, a modified-but-now-valid path, or bailing with an
......@@ -445,11 +444,7 @@ impl DiskFileSystemInner {
}
#[tracing::instrument(level = "info", name = "start filesystem watching", skip_all, fields(path = %self.root))]
async fn start_watching_internal(
self: &Arc<Self>,
report_invalidation_reason: bool,
poll_interval: Option<Duration>,
) -> Result<()> {
async fn start_watching_internal(self: &Arc<Self>) -> Result<()> {
let root_path = self.root_path().to_path_buf();
// create the directory for the filesystem on disk, if it doesn't exist
......@@ -458,8 +453,7 @@ impl DiskFileSystemInner {
.concurrency_limited(&self.write_semaphore)
.await?;
DiskWatcher::start_watching(self.clone(), report_invalidation_reason, poll_interval)
.await?;
DiskWatcher::start_watching(self.clone()).await?;
Ok(())
}
......@@ -506,19 +500,8 @@ impl DiskFileSystem {
.invalidate_path_and_children_with_reason(paths, reason);
}
pub async fn start_watching(&self, poll_interval: Option<Duration>) -> Result<()> {
self.inner
.start_watching_internal(false, poll_interval)
.await
}
pub async fn start_watching_with_invalidation_reason(
&self,
poll_interval: Option<Duration>,
) -> Result<()> {
self.inner
.start_watching_internal(true, poll_interval)
.await
pub async fn start_watching(&self) -> Result<()> {
self.inner.start_watching_internal().await
}
pub async fn stop_watching(&self) {
......@@ -719,24 +702,32 @@ pub(crate) fn format_absolute_fs_path(path: &Path, name: &str, root_path: &Path)
}
impl DiskFileSystem {
/// Create a new instance of `DiskFileSystem`. `name` is a display name for the filesystem. This
/// should be unique. `root` is the [canonicalized][std::fs::canonicalize] root of the
/// filesystem.
/// Create a new instance of `DiskFileSystem`.
///
/// `name` is a display name for the filesystem. This should be unique. `root` is the
/// [canonicalized][std::fs::canonicalize] root of the filesystem.
///
/// This API does not canonicalize itself, as that requires IO operations (e.g. symlink
/// resolution) which should (ideally) not be cached.
pub fn new(name: RcStr, root: Vc<RcStr>) -> Vc<Self> {
Self::new_internal(name, root, Vec::new())
Self::new_internal(name, root, Vec::new(), DiskWatcherConfig::default())
}
/// Create a new instance of `DiskFileSystem`. This is the same as [`DiskFileSystem::new`], but
/// with the option to mark certain paths as not allowed to be accessed or navigated to.
/// Create a new instance of `DiskFileSystem`.
///
/// `name` is a display name for the filesystem. This should be unique. `root` is the
/// [canonicalized][std::fs::canonicalize] root of the filesystem.
///
/// This API does not canonicalize itself, as that requires IO operations (e.g. symlink
/// resolution) which should (ideally) not be cached.
///
/// `denied_paths` must be normalized unix-style paths, non-empty and relative to the fs root.
pub fn new_with_denied_paths(
/// `denied_paths` is a list of paths that are not allowed to be accessed or navigated to. These
/// must be normalized unix-style paths, non-empty and relative to the fs root.
pub fn new_with_options(
name: RcStr,
root: Vc<RcStr>,
denied_paths: Vec<RcStr>,
watcher_config: DiskWatcherConfig,
) -> Vc<Self> {
for denied_path in &denied_paths {
debug_assert!(!denied_path.is_empty(), "denied_path must not be empty");
......@@ -745,7 +736,7 @@ impl DiskFileSystem {
"denied_path must be normalized: {denied_path:?}"
);
}
Self::new_internal(name, root, denied_paths)
Self::new_internal(name, root, denied_paths, watcher_config)
}
}
......@@ -756,6 +747,7 @@ impl DiskFileSystem {
name: RcStr,
root: Vc<RcStr>,
denied_paths: Vec<RcStr>,
watcher_config: DiskWatcherConfig,
) -> Result<Vc<Self>> {
let root = root.owned().await?;
let instance = DiskFileSystem {
......@@ -768,7 +760,7 @@ impl DiskFileSystem {
dir_invalidator_map: InvalidatorMap::new(),
read_semaphore: create_read_semaphore(),
write_semaphore: create_write_semaphore(),
watcher: DiskWatcher::new(),
watcher: DiskWatcher::new(watcher_config),
denied_paths,
turbo_tasks: turbo_tasks_weak(),
tokio_handle: Handle::current(),
......@@ -2063,8 +2055,8 @@ mod tests {
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use crate::{
DirectoryContent, DiskFileSystem, File as TurboFile, FileContent, FileSystem,
FileSystemPath,
DirectoryContent, DiskFileSystem, DiskWatcherConfig, File as TurboFile, FileContent,
FileSystem, FileSystemPath,
glob::{Glob, GlobOptions},
};
......@@ -2117,10 +2109,11 @@ mod tests {
async fn test_denied_path_read() {
#[turbo_tasks::function(operation, root)]
async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths(
let fs = DiskFileSystem::new_with_options(
rcstr!("test"),
Vc::cell(root),
vec![denied_path],
DiskWatcherConfig::default(),
);
let root_path = fs.root().await?;
......@@ -2180,10 +2173,11 @@ mod tests {
async fn test_denied_path_read_dir() {
#[turbo_tasks::function(operation, root)]
async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths(
let fs = DiskFileSystem::new_with_options(
rcstr!("test"),
Vc::cell(root),
vec![denied_path],
DiskWatcherConfig::default(),
);
let root_path = fs.root().await?;
......@@ -2242,10 +2236,11 @@ mod tests {
async fn test_denied_path_read_glob() {
#[turbo_tasks::function(operation, root)]
async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths(
let fs = DiskFileSystem::new_with_options(
rcstr!("test"),
Vc::cell(root),
vec![denied_path],
DiskWatcherConfig::default(),
);
let root_path = fs.root().await?;
......@@ -2328,10 +2323,11 @@ mod tests {
file_path: RcStr,
contents: RcStr,
) -> anyhow::Result<Vc<Effects>> {
let fs = DiskFileSystem::new_with_denied_paths(
let fs = DiskFileSystem::new_with_options(
rcstr!("test"),
Vc::cell(root),
vec![denied_path],
DiskWatcherConfig::default(),
);
let root_path = fs.root().await?;
let allowed_file = root_path.join(&file_path)?;
......@@ -2347,10 +2343,11 @@ mod tests {
denied_file: RcStr,
nested_denied_file: RcStr,
) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths(
let fs = DiskFileSystem::new_with_options(
rcstr!("test"),
Vc::cell(root),
vec![denied_path],
DiskWatcherConfig::default(),
);
let root_path = fs.root().await?;
......
......@@ -19,7 +19,7 @@ pub async fn directory_from_relative_path(
// development and the root directory is unlikely to move.
let root = canonicalize_to_rcstr(Path::new(&*path))?;
let disk_fs = DiskFileSystem::new(name, Vc::cell(root));
disk_fs.await?.start_watching(None).await?;
disk_fs.await?.start_watching().await?;
Ok(Vc::upcast(disk_fs))
}
......
......@@ -60,6 +60,7 @@ pub use crate::{
path::{FileSystemPath, FileSystemPathOption, RealPathResult, RealPathResultError, rebase},
read_glob::ReadGlobResult,
virtual_fs::VirtualFileSystem,
watcher::{DiskWatcherConfig, DiskWatcherRecursiveMode},
windows::to_verbatim_with_case_folded_disk,
};
......
......@@ -15,7 +15,7 @@ use turbo_tasks::{
use crate::{
invalidator_map::InvalidatorMap,
watcher::{DiskFileSystemWatcherApi, DiskWatcher},
watcher::{DiskFileSystemWatcherApi, DiskWatcher, DiskWatcherConfig},
};
/// A stand-in for [`DiskFileSystemInner`], carrying only the state [`DiskWatcher`] touches.
......@@ -41,13 +41,13 @@ impl TraceRawVcs for MockFsHandle {
}
impl MockFileSystem {
pub fn new() -> Arc<MockFileSystem> {
pub fn new(config: DiskWatcherConfig) -> Arc<MockFileSystem> {
let temp_dir = TempDir::new().unwrap();
let root_path = canonicalize(temp_dir.path()).unwrap();
Arc::new_cyclic(|weak| MockFileSystem {
name: rcstr!("mock"),
root_path,
watcher: DiskWatcher::new(),
watcher: DiskWatcher::new(config),
turbo_tasks: turbo_tasks::turbo_tasks_weak(),
tokio_handle: Handle::current(),
invalidator_map: InvalidatorMap::new(),
......
......@@ -129,7 +129,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
};
if !args.start_watching_late {
project_fs.await?.start_watching(None).await?;
project_fs.await?.start_watching().await?;
}
let symlink_count = if args.symlinks.is_some() {
......@@ -175,7 +175,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
invalidations.0.lock().unwrap().clear();
if args.start_watching_late {
project_fs.await?.start_watching(None).await?;
project_fs.await?.start_watching().await?;
}
let mut rand_buf = [0; 16];
......
......@@ -4,7 +4,7 @@ use anyhow::{Context, Result};
use bincode::{Decode, Encode};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Vc, trace::TraceRawVcs};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, canonicalize_to_rcstr};
use turbo_tasks_fs::{DiskFileSystem, DiskWatcherConfig, FileSystem, canonicalize_to_rcstr};
#[turbo_tasks::task_input]
#[derive(Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, Encode, Decode)]
......@@ -56,13 +56,14 @@ pub async fn project_fs(
watch: bool,
denied_root_path: RcStr,
) -> Result<Vc<Box<dyn FileSystem>>> {
let disk_fs = DiskFileSystem::new_with_denied_paths(
let disk_fs = DiskFileSystem::new_with_options(
rcstr!("project"),
Vc::cell(project_dir),
vec![denied_root_path],
DiskWatcherConfig::default(),
);
if watch {
disk_fs.await?.start_watching(None).await?;
disk_fs.await?.start_watching().await?;
}
Ok(Vc::upcast(disk_fs))
}
......
......@@ -37,7 +37,7 @@ async fn main() -> Result<()> {
Box::pin(async {
let root: RcStr = current_dir().unwrap().to_str().unwrap().into();
let disk_fs = DiskFileSystem::new(PROJECT_FILESYSTEM_NAME, Vc::cell(root));
disk_fs.await?.start_watching(None).await?;
disk_fs.await?.start_watching().await?;
// Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment