Unverified Commit 13899e5f authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Restore watchers for children when using non-recursive FS watcher (#82130)

**This should be merged alongside #82258, as this introduces a potentially high time complexity operation that #82258 solves.**

With this PR and the previous one, my fuzzer shows that we capture every (currently tested) file change correctly:

```
rm -rf /tmp/fuzz && cargo run --release -p turbo-tasks-fuzz -- fs-watcher --fs-root /tmp/fuzz
```

However, this does add an `O(n)` iteration of the `watching` map that I'm working on cleaning up (why this is still a draft).

I'll do something similar to https://github.com/vercel/next.js/pull/82133, but it's a bit more complicated to switch to a `BTreeSet` because it's currently using a `DashSet`, and I don't want to introduce more locking than I have to.
parent 2d3cdb01
...@@ -4,13 +4,13 @@ use std::{ ...@@ -4,13 +4,13 @@ use std::{
mem::take, mem::take,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::{ sync::{
Arc, LazyLock, Mutex, MutexGuard, Arc, LazyLock, Mutex,
mpsc::{Receiver, TryRecvError, channel}, mpsc::{Receiver, TryRecvError, channel},
}, },
time::Duration, time::Duration,
}; };
use anyhow::{Context, Result, anyhow}; use anyhow::{Context, Result};
use dashmap::DashSet; use dashmap::DashSet;
use notify::{ use notify::{
Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher, Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher,
...@@ -78,6 +78,8 @@ impl DiskWatcherInternal { ...@@ -78,6 +78,8 @@ impl DiskWatcherInternal {
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub(crate) struct DiskWatcher { pub(crate) struct DiskWatcher {
/// This value is [`None`] when the watcher has been stopped (see
/// [`DiskWatcher::stop_watching`]).
#[serde(skip)] #[serde(skip)]
internal: Mutex<Option<DiskWatcherInternal>>, internal: Mutex<Option<DiskWatcherInternal>>,
...@@ -118,10 +120,17 @@ impl NonRecursiveDiskWatcherState { ...@@ -118,10 +120,17 @@ impl NonRecursiveDiskWatcherState {
/// Called after a rescan in case a previously watched-but-deleted directory was recreated. /// Called after a rescan in case a previously watched-but-deleted directory was recreated.
pub(crate) fn restore_all_watching(&self, watcher: &DiskWatcher, root_path: &Path) { pub(crate) fn restore_all_watching(&self, watcher: &DiskWatcher, root_path: &Path) {
let mut internal = watcher.internal.lock().unwrap(); let mut internal_guard = watcher.internal.lock().unwrap();
let Some(internal) = &mut *internal_guard else {
return;
};
for dir_path in self.watching.iter() { for dir_path in self.watching.iter() {
// TODO: Report diagnostics if this error happens // TODO: Report diagnostics if this error happens
let _ = self.start_watching_dir(&mut internal, &dir_path, root_path); //
// Don't watch the parents, because those are already included in `self.watching` (so
// it'd be redundant), but also because this could deadlock, since we'd try to modify
// `self.watching` while iterating over it (write lock overlapping with a read lock).
let _ = self.start_watching_dir(internal, &dir_path, root_path);
} }
} }
...@@ -136,9 +145,24 @@ impl NonRecursiveDiskWatcherState { ...@@ -136,9 +145,24 @@ impl NonRecursiveDiskWatcherState {
if dir_path == root_path || !self.watching.contains(dir_path) { if dir_path == root_path || !self.watching.contains(dir_path) {
return Ok(()); return Ok(());
} }
let mut internal = watcher.internal.lock().unwrap(); let mut internal_guard = watcher.internal.lock().unwrap();
// TODO: Also restore any watchers for children of this directory let Some(internal) = &mut *internal_guard else {
self.start_watching_dir(&mut internal, dir_path, root_path) return Ok(());
};
// watch the new directory
self.start_watching_dir(internal, dir_path, root_path)?;
// Also try to restore any watchers for children of this directory
for child_path in self
.watching
.iter()
.filter(|p| p.key().starts_with(dir_path) && **p != dir_path)
{
// Don't watch the parents -- see the comment on `restore_all_watching`
self.start_watching_dir(internal, child_path.key(), root_path)?;
}
Ok(())
} }
/// Called when a file in `dir_path` or `dir_path` itself is read or written. Adds a new watcher /// Called when a file in `dir_path` or `dir_path` itself is read or written. Adds a new watcher
...@@ -152,60 +176,72 @@ impl NonRecursiveDiskWatcherState { ...@@ -152,60 +176,72 @@ impl NonRecursiveDiskWatcherState {
if dir_path == root_path || self.watching.contains(dir_path) { if dir_path == root_path || self.watching.contains(dir_path) {
return Ok(()); return Ok(());
} }
let mut internal = watcher.internal.lock().unwrap(); let mut internal_guard = watcher.internal.lock().unwrap();
let Some(internal) = &mut *internal_guard else {
return Ok(());
};
if self.watching.insert(dir_path.to_path_buf()) { if self.watching.insert(dir_path.to_path_buf()) {
self.start_watching_dir(&mut internal, dir_path, root_path)?; self.start_watching_dir_and_parents(internal, dir_path, root_path)?;
} }
Ok(()) Ok(())
} }
/// Private helper, assumes that the path has already been added to `self.watching`. /// Private helper, assumes that `dir_path` has already been added to `self.watching`.
///
/// This does not watch any of the parent directories. For that, use
/// [`start_watching_dir_and_parents`]. Use this method when iterating over previously-watched
/// values in `self.watching`.
fn start_watching_dir( fn start_watching_dir(
&self, &self,
watcher_internal_guard: &mut MutexGuard<Option<DiskWatcherInternal>>, watcher_internal: &mut DiskWatcherInternal,
dir_path: &Path, dir_path: &Path,
root_path: &Path, root_path: &Path,
) -> Result<()> { ) -> Result<()> {
debug_assert_ne!(dir_path, root_path); debug_assert_ne!(dir_path, root_path);
let Some(watcher_internal_guard) = watcher_internal_guard.as_mut() else {
return Ok(());
};
let mut path = dir_path; match watcher_internal.watch(dir_path, RecursiveMode::NonRecursive) {
let err_with_context = |err: anyhow::Error| { Ok(())
return Err(err).context(format!( | Err(notify::Error {
"Unable to watch {} (tried up to {})", // The path was probably deleted before we could process the event, but the parent
dir_path.display(), // should still be watched. The codepaths that care about this either call
path.display() // `start_watching_dir_and_parents` or handle the parents themselves.
)); kind: notify::ErrorKind::PathNotFound,
}; ..
}) => Ok(()),
Err(err) => {
return Err(err).context(format!("Unable to watch {}", dir_path.display(),));
}
}
}
// watch every parent: https://docs.rs/notify/latest/notify/#parent-folder-deletion /// Private helper, assumes that `dir_path` has already been added to `self.watching`.
///
/// Watches the given `dir_path` and every parent up to `root_path`. Parents must be recursively
/// watched in case any of them change:
/// https://docs.rs/notify/latest/notify/#parent-folder-deletion
fn start_watching_dir_and_parents(
&self,
watcher_internal: &mut DiskWatcherInternal,
dir_path: &Path,
root_path: &Path,
) -> Result<()> {
let mut cur_path = dir_path;
loop { loop {
match watcher_internal_guard.watch(path, RecursiveMode::NonRecursive) { self.start_watching_dir(watcher_internal, cur_path, root_path)?;
res @ Ok(())
| res @ Err(notify::Error { let Some(parent_path) = cur_path.parent() else {
// The path was probably deleted before we could process the event. That's // this should never happen as we break before we reach the root path
// okay, just make sure we're watching the parent directory, so we can know anyhow::bail!(
// if it gets recreated. "failed to compute parent path of {cur_path:?} while watching {dir_path:?} in \
kind: notify::ErrorKind::PathNotFound, root {root_path:?}"
.. );
}) => { };
let Some(parent_path) = path.parent() else {
// this should never happen as we break before we reach the root path if parent_path == root_path || !self.watching.insert(parent_path.to_path_buf()) {
return err_with_context(res.err().map_or_else( break;
|| anyhow!("failed to compute parent path"),
|err| err.into(),
));
};
if parent_path == root_path || !self.watching.insert(parent_path.to_path_buf())
{
break;
}
path = parent_path;
}
Err(err) => return err_with_context(err.into()),
} }
cur_path = parent_path;
} }
Ok(()) Ok(())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment