Unverified Commit 253980da authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Watch the root and every parent directory in non-recursive mode (#82129)

This only impacts the non-recursive codepath.

The notify-rs docs explain that you need to watch every parent of the file/directory you care about: https://docs.rs/notify/latest/notify/#parent-folder-deletion

This reduces the missing invalidations on Linux detected by my fuzzer, but does not eliminate them:

```
rm -rf /tmp/fuzz && cargo run --release -p turbo-tasks-fuzz -- fs-watcher --fs-root /tmp/fuzz
```
parent 4e355ccb
...@@ -10,7 +10,7 @@ use std::{ ...@@ -10,7 +10,7 @@ use std::{
time::Duration, time::Duration,
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result, anyhow};
use dashmap::DashSet; use dashmap::DashSet;
use notify::{ use notify::{
Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher, Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher,
...@@ -105,6 +105,11 @@ impl Default for DiskWatcher { ...@@ -105,6 +105,11 @@ impl Default for DiskWatcher {
/// [`RecursiveMode::NonRecursive`] (default on Linux). /// [`RecursiveMode::NonRecursive`] (default on Linux).
pub(crate) struct NonRecursiveDiskWatcherState { pub(crate) struct NonRecursiveDiskWatcherState {
/// Keeps track of which directories are currently (or were previously) watched. /// Keeps track of which directories are currently (or were previously) watched.
///
/// Invariants:
/// - Never contains `root_path`. A watcher for `root_path` is implicitly set up during
/// [`DiskWatcher::start_watching`].
/// - Contains all parent directories up to `root_path` for every entry.
watching: DashSet<PathBuf>, watching: DashSet<PathBuf>,
} }
...@@ -127,28 +132,31 @@ impl NonRecursiveDiskWatcherState { ...@@ -127,28 +132,31 @@ impl NonRecursiveDiskWatcherState {
} }
} }
/// Called when a new directory is found in a parent directory we're watching. /// Called when a new directory is found in a parent directory we're watching. Restores the
/// watcher if we were previously watching it.
pub(crate) fn restore_if_watching( pub(crate) fn restore_if_watching(
&self, &self,
watcher: &DiskWatcher, watcher: &DiskWatcher,
dir_path: &Path, dir_path: &Path,
root_path: &Path, root_path: &Path,
) -> Result<()> { ) -> Result<()> {
if self.watching.contains(dir_path) { if dir_path == root_path || !self.watching.contains(dir_path) {
return Ok(());
}
let mut internal = watcher.internal.lock().unwrap(); let mut internal = watcher.internal.lock().unwrap();
// TODO: Also restore any watchers for children of this directory // TODO: Also restore any watchers for children of this directory
self.start_watching_dir(&mut internal, dir_path, root_path)?; self.start_watching_dir(&mut internal, dir_path, root_path)
}
Ok(())
} }
/// Called when a file in `dir_path` or `dir_path` itself is read or written. Adds a new watcher
/// if we're not already watching the directory.
pub(crate) fn ensure_watching( pub(crate) fn ensure_watching(
&self, &self,
watcher: &DiskWatcher, watcher: &DiskWatcher,
dir_path: &Path, dir_path: &Path,
root_path: &Path, root_path: &Path,
) -> Result<()> { ) -> Result<()> {
if self.watching.contains(dir_path) { if dir_path == root_path || self.watching.contains(dir_path) {
return Ok(()); return Ok(());
} }
let mut internal = watcher.internal.lock().unwrap(); let mut internal = watcher.internal.lock().unwrap();
...@@ -165,42 +173,48 @@ impl NonRecursiveDiskWatcherState { ...@@ -165,42 +173,48 @@ impl NonRecursiveDiskWatcherState {
dir_path: &Path, dir_path: &Path,
root_path: &Path, root_path: &Path,
) -> Result<()> { ) -> Result<()> {
if let Some(watcher_internal_guard) = watcher_internal_guard.as_mut() { debug_assert_ne!(dir_path, root_path);
let Some(watcher_internal_guard) = watcher_internal_guard.as_mut() else {
return Ok(());
};
let mut path = dir_path; let mut path = dir_path;
let err_with_context = |err| { let err_with_context = |err: anyhow::Error| {
return Err(err).context(format!( return Err(err).context(format!(
"Unable to watch {} (tried up to {})", "Unable to watch {} (tried up to {})",
dir_path.display(), dir_path.display(),
path.display() path.display()
)); ));
}; };
while let Err(err) = watcher_internal_guard.watch(path, RecursiveMode::NonRecursive) {
match err { // watch every parent: https://docs.rs/notify/latest/notify/#parent-folder-deletion
notify::Error { loop {
kind: notify::ErrorKind::PathNotFound, match watcher_internal_guard.watch(path, RecursiveMode::NonRecursive) {
.. res @ Ok(())
} => { | res @ Err(notify::Error {
// The path was probably deleted before we could process the event. That's // The path was probably deleted before we could process the event. That's
// okay, just make sure we're watching the parent directory, so we can know // okay, just make sure we're watching the parent directory, so we can know
// if it gets recreated. // if it gets recreated.
kind: notify::ErrorKind::PathNotFound,
..
}) => {
let Some(parent_path) = path.parent() else { let Some(parent_path) = path.parent() else {
// this should never happen as we break before we reach the root path // this should never happen as we break before we reach the root path
return err_with_context(err); return err_with_context(res.err().map_or_else(
|| anyhow!("failed to compute parent path"),
|err| err.into(),
));
}; };
if parent_path == root_path { if parent_path == root_path || !self.watching.insert(parent_path.to_path_buf())
// assume there's already a root watcher {
break;
}
if !self.watching.insert(parent_path.to_owned()) {
// we're already watching the parent path!
break; break;
} }
path = parent_path; path = parent_path;
} }
_ => return err_with_context(err), Err(err) => return err_with_context(err.into()),
}
} }
} }
Ok(()) Ok(())
} }
} }
...@@ -256,6 +270,7 @@ impl DiskWatcher { ...@@ -256,6 +270,7 @@ impl DiskWatcher {
}; };
if let Some(non_recursive) = &self.non_recursive_state { if let Some(non_recursive) = &self.non_recursive_state {
internal.watch(fs_inner.root_path(), RecursiveMode::NonRecursive)?;
for dir_path in non_recursive.watching.iter() { for dir_path in non_recursive.watching.iter() {
internal.watch(&dir_path, RecursiveMode::NonRecursive)?; internal.watch(&dir_path, RecursiveMode::NonRecursive)?;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment