Unverified Commit 5e8f31f7 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Improve how DiskWatcher is configured and fix polling watcher bugs (#96440)

- Adds a `DiskWatcherConfig` object to centralize the recursive mode,
polling, and invalidation reason options.
- **Polling Fix:** Use the non-recursive mode if the polling watcher is
selected, ignoring the OS/platform.
- **Polling Fix:** Treat mtime events as data update events if the
polling watcher is selected (this is what
https://github.com/vercel/next.js/pull/96288 was doing, but this PR only
does it if polling is enabled)
- Only treat the `MetadataKind::Any` event as a data change on macOS.

Fixes #80665
Closes #96288
parent a13e48dd
...@@ -45,7 +45,7 @@ use turbo_tasks::{ ...@@ -45,7 +45,7 @@ use turbo_tasks::{
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystem, FileSystemPath, VirtualFileSystem, DiskFileSystem, DiskWatcherConfig, FileContent, FileSystem, FileSystemPath, VirtualFileSystem,
canonicalize_to_rcstr, invalidation, canonicalize_to_rcstr, invalidation,
}; };
use turbo_unix_path::join_path; use turbo_unix_path::join_path;
...@@ -631,9 +631,7 @@ impl ProjectContainer { ...@@ -631,9 +631,7 @@ impl ProjectContainer {
.read_strongly_consistent() .read_strongly_consistent()
.await?; .await?;
if watch.enable { if watch.enable {
project_fs project_fs.start_watching().await?;
.start_watching_with_invalidation_reason(watch.poll_interval)
.await?;
} else { } else {
project_fs.invalidate_with_reason(|path| invalidation::Initialize { project_fs.invalidate_with_reason(|path| invalidation::Initialize {
// this path is just used for display purposes // this path is just used for display purposes
...@@ -776,9 +774,7 @@ impl ProjectContainer { ...@@ -776,9 +774,7 @@ impl ProjectContainer {
if !ReadRef::ptr_eq(&prev_project_fs, &project_fs) { if !ReadRef::ptr_eq(&prev_project_fs, &project_fs) {
if watch.enable { if watch.enable {
// TODO stop watching: prev_project_fs.stop_watching()?; // TODO stop watching: prev_project_fs.stop_watching()?;
project_fs project_fs.start_watching().await?;
.start_watching_with_invalidation_reason(watch.poll_interval)
.await?;
} else { } else {
project_fs.invalidate_with_reason(|path| invalidation::Initialize { project_fs.invalidate_with_reason(|path| invalidation::Initialize {
// this path is just used for display purposes // this path is just used for display purposes
...@@ -1087,10 +1083,16 @@ impl Project { ...@@ -1087,10 +1083,16 @@ impl Project {
.unwrap() .unwrap()
.into(); .into();
Ok(DiskFileSystem::new_with_denied_paths( Ok(DiskFileSystem::new_with_options(
PROJECT_FILESYSTEM_NAME, PROJECT_FILESYSTEM_NAME,
*self.root_path, *self.root_path,
vec![denied_path, denied_profiles_path], vec![denied_path, denied_profiles_path],
DiskWatcherConfig {
recursive_mode: None,
poll_interval: self.watch.poll_interval,
// the dev server reports these to the user
report_invalidation_reason: true,
},
)) ))
} }
......
...@@ -29,7 +29,7 @@ async fn main() -> Result<()> { ...@@ -29,7 +29,7 @@ async fn main() -> Result<()> {
Box::pin(async { Box::pin(async {
let root = RcStr::from(current_dir().unwrap().to_str().unwrap()); let root = RcStr::from(current_dir().unwrap().to_str().unwrap());
let disk_fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(root)); let disk_fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(root));
disk_fs.await?.start_watching(None).await?; disk_fs.await?.start_watching().await?;
// Smart Pointer cast // Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs); let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
......
...@@ -30,7 +30,7 @@ async fn main() -> Result<()> { ...@@ -30,7 +30,7 @@ async fn main() -> Result<()> {
Box::pin(async { Box::pin(async {
let root = RcStr::from(current_dir().unwrap().to_str().unwrap()); let root = RcStr::from(current_dir().unwrap().to_str().unwrap());
let disk_fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(root)); let disk_fs = DiskFileSystem::new(rcstr!("project"), Vc::cell(root));
disk_fs.await?.start_watching(None).await?; disk_fs.await?.start_watching().await?;
// Smart Pointer cast // Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs); let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
......
...@@ -9,7 +9,6 @@ use std::{ ...@@ -9,7 +9,6 @@ use std::{
mem::take, mem::take,
path::{MAIN_SEPARATOR, Path, PathBuf}, path::{MAIN_SEPARATOR, Path, PathBuf},
sync::{Arc, LazyLock, Weak}, sync::{Arc, LazyLock, Weak},
time::Duration,
}; };
use anyhow::{Context, Result, anyhow, bail}; use anyhow::{Context, Result, anyhow, bail};
...@@ -42,7 +41,7 @@ use crate::{ ...@@ -42,7 +41,7 @@ use crate::{
mutex_map::MutexMap, mutex_map::MutexMap,
path_map::OrderedPathMapExt, path_map::OrderedPathMapExt,
retry::{can_retry, retry_blocking, retry_blocking_custom}, retry::{can_retry, retry_blocking, retry_blocking_custom},
watcher::DiskWatcher, watcher::{DiskWatcher, DiskWatcherConfig},
}; };
/// Validate the path, returning the valid path, a modified-but-now-valid path, or bailing with an /// Validate the path, returning the valid path, a modified-but-now-valid path, or bailing with an
...@@ -445,11 +444,7 @@ impl DiskFileSystemInner { ...@@ -445,11 +444,7 @@ impl DiskFileSystemInner {
} }
#[tracing::instrument(level = "info", name = "start filesystem watching", skip_all, fields(path = %self.root))] #[tracing::instrument(level = "info", name = "start filesystem watching", skip_all, fields(path = %self.root))]
async fn start_watching_internal( async fn start_watching_internal(self: &Arc<Self>) -> Result<()> {
self: &Arc<Self>,
report_invalidation_reason: bool,
poll_interval: Option<Duration>,
) -> Result<()> {
let root_path = self.root_path().to_path_buf(); let root_path = self.root_path().to_path_buf();
// create the directory for the filesystem on disk, if it doesn't exist // create the directory for the filesystem on disk, if it doesn't exist
...@@ -458,8 +453,7 @@ impl DiskFileSystemInner { ...@@ -458,8 +453,7 @@ impl DiskFileSystemInner {
.concurrency_limited(&self.write_semaphore) .concurrency_limited(&self.write_semaphore)
.await?; .await?;
DiskWatcher::start_watching(self.clone(), report_invalidation_reason, poll_interval) DiskWatcher::start_watching(self.clone()).await?;
.await?;
Ok(()) Ok(())
} }
...@@ -506,19 +500,8 @@ impl DiskFileSystem { ...@@ -506,19 +500,8 @@ impl DiskFileSystem {
.invalidate_path_and_children_with_reason(paths, reason); .invalidate_path_and_children_with_reason(paths, reason);
} }
pub async fn start_watching(&self, poll_interval: Option<Duration>) -> Result<()> { pub async fn start_watching(&self) -> Result<()> {
self.inner self.inner.start_watching_internal().await
.start_watching_internal(false, poll_interval)
.await
}
pub async fn start_watching_with_invalidation_reason(
&self,
poll_interval: Option<Duration>,
) -> Result<()> {
self.inner
.start_watching_internal(true, poll_interval)
.await
} }
pub async fn stop_watching(&self) { pub async fn stop_watching(&self) {
...@@ -719,24 +702,32 @@ pub(crate) fn format_absolute_fs_path(path: &Path, name: &str, root_path: &Path) ...@@ -719,24 +702,32 @@ pub(crate) fn format_absolute_fs_path(path: &Path, name: &str, root_path: &Path)
} }
impl DiskFileSystem { impl DiskFileSystem {
/// Create a new instance of `DiskFileSystem`. `name` is a display name for the filesystem. This /// Create a new instance of `DiskFileSystem`.
/// should be unique. `root` is the [canonicalized][std::fs::canonicalize] root of the ///
/// filesystem. /// `name` is a display name for the filesystem. This should be unique. `root` is the
/// [canonicalized][std::fs::canonicalize] root of the filesystem.
/// ///
/// This API does not canonicalize itself, as that requires IO operations (e.g. symlink /// This API does not canonicalize itself, as that requires IO operations (e.g. symlink
/// resolution) which should (ideally) not be cached. /// resolution) which should (ideally) not be cached.
pub fn new(name: RcStr, root: Vc<RcStr>) -> Vc<Self> { pub fn new(name: RcStr, root: Vc<RcStr>) -> Vc<Self> {
Self::new_internal(name, root, Vec::new()) Self::new_internal(name, root, Vec::new(), DiskWatcherConfig::default())
} }
/// Create a new instance of `DiskFileSystem`. This is the same as [`DiskFileSystem::new`], but /// Create a new instance of `DiskFileSystem`.
/// with the option to mark certain paths as not allowed to be accessed or navigated to. ///
/// `name` is a display name for the filesystem. This should be unique. `root` is the
/// [canonicalized][std::fs::canonicalize] root of the filesystem.
///
/// This API does not canonicalize itself, as that requires IO operations (e.g. symlink
/// resolution) which should (ideally) not be cached.
/// ///
/// `denied_paths` must be normalized unix-style paths, non-empty and relative to the fs root. /// `denied_paths` is a list of paths that are not allowed to be accessed or navigated to. These
pub fn new_with_denied_paths( /// must be normalized unix-style paths, non-empty and relative to the fs root.
pub fn new_with_options(
name: RcStr, name: RcStr,
root: Vc<RcStr>, root: Vc<RcStr>,
denied_paths: Vec<RcStr>, denied_paths: Vec<RcStr>,
watcher_config: DiskWatcherConfig,
) -> Vc<Self> { ) -> Vc<Self> {
for denied_path in &denied_paths { for denied_path in &denied_paths {
debug_assert!(!denied_path.is_empty(), "denied_path must not be empty"); debug_assert!(!denied_path.is_empty(), "denied_path must not be empty");
...@@ -745,7 +736,7 @@ impl DiskFileSystem { ...@@ -745,7 +736,7 @@ impl DiskFileSystem {
"denied_path must be normalized: {denied_path:?}" "denied_path must be normalized: {denied_path:?}"
); );
} }
Self::new_internal(name, root, denied_paths) Self::new_internal(name, root, denied_paths, watcher_config)
} }
} }
...@@ -756,6 +747,7 @@ impl DiskFileSystem { ...@@ -756,6 +747,7 @@ impl DiskFileSystem {
name: RcStr, name: RcStr,
root: Vc<RcStr>, root: Vc<RcStr>,
denied_paths: Vec<RcStr>, denied_paths: Vec<RcStr>,
watcher_config: DiskWatcherConfig,
) -> Result<Vc<Self>> { ) -> Result<Vc<Self>> {
let root = root.owned().await?; let root = root.owned().await?;
let instance = DiskFileSystem { let instance = DiskFileSystem {
...@@ -768,7 +760,7 @@ impl DiskFileSystem { ...@@ -768,7 +760,7 @@ impl DiskFileSystem {
dir_invalidator_map: InvalidatorMap::new(), dir_invalidator_map: InvalidatorMap::new(),
read_semaphore: create_read_semaphore(), read_semaphore: create_read_semaphore(),
write_semaphore: create_write_semaphore(), write_semaphore: create_write_semaphore(),
watcher: DiskWatcher::new(), watcher: DiskWatcher::new(watcher_config),
denied_paths, denied_paths,
turbo_tasks: turbo_tasks_weak(), turbo_tasks: turbo_tasks_weak(),
tokio_handle: Handle::current(), tokio_handle: Handle::current(),
...@@ -2063,8 +2055,8 @@ mod tests { ...@@ -2063,8 +2055,8 @@ mod tests {
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
use crate::{ use crate::{
DirectoryContent, DiskFileSystem, File as TurboFile, FileContent, FileSystem, DirectoryContent, DiskFileSystem, DiskWatcherConfig, File as TurboFile, FileContent,
FileSystemPath, FileSystem, FileSystemPath,
glob::{Glob, GlobOptions}, glob::{Glob, GlobOptions},
}; };
...@@ -2117,10 +2109,11 @@ mod tests { ...@@ -2117,10 +2109,11 @@ mod tests {
async fn test_denied_path_read() { async fn test_denied_path_read() {
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> { async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths( let fs = DiskFileSystem::new_with_options(
rcstr!("test"), rcstr!("test"),
Vc::cell(root), Vc::cell(root),
vec![denied_path], vec![denied_path],
DiskWatcherConfig::default(),
); );
let root_path = fs.root().await?; let root_path = fs.root().await?;
...@@ -2180,10 +2173,11 @@ mod tests { ...@@ -2180,10 +2173,11 @@ mod tests {
async fn test_denied_path_read_dir() { async fn test_denied_path_read_dir() {
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> { async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths( let fs = DiskFileSystem::new_with_options(
rcstr!("test"), rcstr!("test"),
Vc::cell(root), Vc::cell(root),
vec![denied_path], vec![denied_path],
DiskWatcherConfig::default(),
); );
let root_path = fs.root().await?; let root_path = fs.root().await?;
...@@ -2242,10 +2236,11 @@ mod tests { ...@@ -2242,10 +2236,11 @@ mod tests {
async fn test_denied_path_read_glob() { async fn test_denied_path_read_glob() {
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> { async fn test_operation(root: RcStr, denied_path: RcStr) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths( let fs = DiskFileSystem::new_with_options(
rcstr!("test"), rcstr!("test"),
Vc::cell(root), Vc::cell(root),
vec![denied_path], vec![denied_path],
DiskWatcherConfig::default(),
); );
let root_path = fs.root().await?; let root_path = fs.root().await?;
...@@ -2328,10 +2323,11 @@ mod tests { ...@@ -2328,10 +2323,11 @@ mod tests {
file_path: RcStr, file_path: RcStr,
contents: RcStr, contents: RcStr,
) -> anyhow::Result<Vc<Effects>> { ) -> anyhow::Result<Vc<Effects>> {
let fs = DiskFileSystem::new_with_denied_paths( let fs = DiskFileSystem::new_with_options(
rcstr!("test"), rcstr!("test"),
Vc::cell(root), Vc::cell(root),
vec![denied_path], vec![denied_path],
DiskWatcherConfig::default(),
); );
let root_path = fs.root().await?; let root_path = fs.root().await?;
let allowed_file = root_path.join(&file_path)?; let allowed_file = root_path.join(&file_path)?;
...@@ -2347,10 +2343,11 @@ mod tests { ...@@ -2347,10 +2343,11 @@ mod tests {
denied_file: RcStr, denied_file: RcStr,
nested_denied_file: RcStr, nested_denied_file: RcStr,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let fs = DiskFileSystem::new_with_denied_paths( let fs = DiskFileSystem::new_with_options(
rcstr!("test"), rcstr!("test"),
Vc::cell(root), Vc::cell(root),
vec![denied_path], vec![denied_path],
DiskWatcherConfig::default(),
); );
let root_path = fs.root().await?; let root_path = fs.root().await?;
......
...@@ -19,7 +19,7 @@ pub async fn directory_from_relative_path( ...@@ -19,7 +19,7 @@ pub async fn directory_from_relative_path(
// development and the root directory is unlikely to move. // development and the root directory is unlikely to move.
let root = canonicalize_to_rcstr(Path::new(&*path))?; let root = canonicalize_to_rcstr(Path::new(&*path))?;
let disk_fs = DiskFileSystem::new(name, Vc::cell(root)); let disk_fs = DiskFileSystem::new(name, Vc::cell(root));
disk_fs.await?.start_watching(None).await?; disk_fs.await?.start_watching().await?;
Ok(Vc::upcast(disk_fs)) Ok(Vc::upcast(disk_fs))
} }
......
...@@ -60,6 +60,7 @@ pub use crate::{ ...@@ -60,6 +60,7 @@ pub use crate::{
path::{FileSystemPath, FileSystemPathOption, RealPathResult, RealPathResultError, rebase}, path::{FileSystemPath, FileSystemPathOption, RealPathResult, RealPathResultError, rebase},
read_glob::ReadGlobResult, read_glob::ReadGlobResult,
virtual_fs::VirtualFileSystem, virtual_fs::VirtualFileSystem,
watcher::{DiskWatcherConfig, DiskWatcherRecursiveMode},
windows::to_verbatim_with_case_folded_disk, windows::to_verbatim_with_case_folded_disk,
}; };
......
...@@ -15,7 +15,7 @@ use turbo_tasks::{ ...@@ -15,7 +15,7 @@ use turbo_tasks::{
use crate::{ use crate::{
invalidator_map::InvalidatorMap, invalidator_map::InvalidatorMap,
watcher::{DiskFileSystemWatcherApi, DiskWatcher}, watcher::{DiskFileSystemWatcherApi, DiskWatcher, DiskWatcherConfig},
}; };
/// A stand-in for [`DiskFileSystemInner`], carrying only the state [`DiskWatcher`] touches. /// A stand-in for [`DiskFileSystemInner`], carrying only the state [`DiskWatcher`] touches.
...@@ -41,13 +41,13 @@ impl TraceRawVcs for MockFsHandle { ...@@ -41,13 +41,13 @@ impl TraceRawVcs for MockFsHandle {
} }
impl MockFileSystem { impl MockFileSystem {
pub fn new() -> Arc<MockFileSystem> { pub fn new(config: DiskWatcherConfig) -> Arc<MockFileSystem> {
let temp_dir = TempDir::new().unwrap(); let temp_dir = TempDir::new().unwrap();
let root_path = canonicalize(temp_dir.path()).unwrap(); let root_path = canonicalize(temp_dir.path()).unwrap();
Arc::new_cyclic(|weak| MockFileSystem { Arc::new_cyclic(|weak| MockFileSystem {
name: rcstr!("mock"), name: rcstr!("mock"),
root_path, root_path,
watcher: DiskWatcher::new(), watcher: DiskWatcher::new(config),
turbo_tasks: turbo_tasks::turbo_tasks_weak(), turbo_tasks: turbo_tasks::turbo_tasks_weak(),
tokio_handle: Handle::current(), tokio_handle: Handle::current(),
invalidator_map: InvalidatorMap::new(), invalidator_map: InvalidatorMap::new(),
......
...@@ -129,7 +129,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> { ...@@ -129,7 +129,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
}; };
if !args.start_watching_late { if !args.start_watching_late {
project_fs.await?.start_watching(None).await?; project_fs.await?.start_watching().await?;
} }
let symlink_count = if args.symlinks.is_some() { let symlink_count = if args.symlinks.is_some() {
...@@ -175,7 +175,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> { ...@@ -175,7 +175,7 @@ pub async fn run(args: FsWatcher) -> anyhow::Result<()> {
invalidations.0.lock().unwrap().clear(); invalidations.0.lock().unwrap().clear();
if args.start_watching_late { if args.start_watching_late {
project_fs.await?.start_watching(None).await?; project_fs.await?.start_watching().await?;
} }
let mut rand_buf = [0; 16]; let mut rand_buf = [0; 16];
......
...@@ -4,7 +4,7 @@ use anyhow::{Context, Result}; ...@@ -4,7 +4,7 @@ use anyhow::{Context, Result};
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Vc, trace::TraceRawVcs}; use turbo_tasks::{Vc, trace::TraceRawVcs};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, canonicalize_to_rcstr}; use turbo_tasks_fs::{DiskFileSystem, DiskWatcherConfig, FileSystem, canonicalize_to_rcstr};
#[turbo_tasks::task_input] #[turbo_tasks::task_input]
#[derive(Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, Encode, Decode)] #[derive(Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, Encode, Decode)]
...@@ -56,13 +56,14 @@ pub async fn project_fs( ...@@ -56,13 +56,14 @@ pub async fn project_fs(
watch: bool, watch: bool,
denied_root_path: RcStr, denied_root_path: RcStr,
) -> Result<Vc<Box<dyn FileSystem>>> { ) -> Result<Vc<Box<dyn FileSystem>>> {
let disk_fs = DiskFileSystem::new_with_denied_paths( let disk_fs = DiskFileSystem::new_with_options(
rcstr!("project"), rcstr!("project"),
Vc::cell(project_dir), Vc::cell(project_dir),
vec![denied_root_path], vec![denied_root_path],
DiskWatcherConfig::default(),
); );
if watch { if watch {
disk_fs.await?.start_watching(None).await?; disk_fs.await?.start_watching().await?;
} }
Ok(Vc::upcast(disk_fs)) Ok(Vc::upcast(disk_fs))
} }
......
...@@ -37,7 +37,7 @@ async fn main() -> Result<()> { ...@@ -37,7 +37,7 @@ async fn main() -> Result<()> {
Box::pin(async { Box::pin(async {
let root: RcStr = current_dir().unwrap().to_str().unwrap().into(); let root: RcStr = current_dir().unwrap().to_str().unwrap().into();
let disk_fs = DiskFileSystem::new(PROJECT_FILESYSTEM_NAME, Vc::cell(root)); let disk_fs = DiskFileSystem::new(PROJECT_FILESYSTEM_NAME, Vc::cell(root));
disk_fs.await?.start_watching(None).await?; disk_fs.await?.start_watching().await?;
// Smart Pointer cast // Smart Pointer cast
let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs); let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment