Unverified Commit e9982de2 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

fix(turbo-tasks-fs): Handle filesystem watcher rescan events (#78045)

The notify crate requires handling rescan events with the event queue overflows.

Context on this issue in the notify crate:
- https://github.com/notify-rs/notify/issues/412
- https://github.com/notify-rs/notify/issues/434
- https://github.com/notify-rs/notify/pull/435

In our case, that means we should invalidate every file we've read in `turbo-tasks-fs`.

We should also attempt to re-add filesystem watchers for every path, since some of those paths could've been removed and re-added.

Tested this on Linux by lowering the inotify queue to something tiny to trigger frequent rescan events:

```
echo 3 | sudo tee /proc/sys/fs/inotify/max_queued_events
rm -rf /tmp/fuzz && RUST_BACKTRACE=1 cargo run -p turbo-tasks-fuzz -- fs-watcher --fs-root /tmp/fuzz --notify-timeout-ms 100 --directory-modifications 0
echo 16384 | sudo tee /proc/sys/fs/inotify/max_queued_events  # restore back to default
```

As a sanity check, I also tried running the fuzzer on macos, though it doesn't seem like there's as easy of a way to trigger rescan events on that platform (?).
parent 0ecb7e45
...@@ -33,7 +33,7 @@ use turbo_tasks::{ ...@@ -33,7 +33,7 @@ use turbo_tasks::{
ReadRef, ResolvedVc, State, TaskInput, TransientInstance, TryFlatJoinIterExt, Value, Vc, ReadRef, ResolvedVc, State, TaskInput, TransientInstance, TryFlatJoinIterExt, Value, Vc,
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem}; use turbo_tasks_fs::{invalidation, DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem};
use turbopack::{ use turbopack::{
evaluate_context::node_build_environment, global_module_ids::get_global_module_id_strategy, evaluate_context::node_build_environment, global_module_ids::get_global_module_id_strategy,
transition::TransitionOptions, ModuleAssetContext, transition::TransitionOptions, ModuleAssetContext,
...@@ -308,12 +308,16 @@ impl ProjectContainer { ...@@ -308,12 +308,16 @@ impl ProjectContainer {
.start_watching_with_invalidation_reason(watch.poll_interval) .start_watching_with_invalidation_reason(watch.poll_interval)
.await?; .await?;
} else { } else {
project_fs.invalidate_with_reason(); project_fs.invalidate_with_reason(|path| invalidation::Initialize {
path: RcStr::from(path),
});
} }
let output_fs = output_fs_operation(project) let output_fs = output_fs_operation(project)
.read_strongly_consistent() .read_strongly_consistent()
.await?; .await?;
output_fs.invalidate_with_reason(); output_fs.invalidate_with_reason(|path| invalidation::Initialize {
path: RcStr::from(path),
});
Ok(()) Ok(())
} }
...@@ -402,11 +406,15 @@ impl ProjectContainer { ...@@ -402,11 +406,15 @@ impl ProjectContainer {
.start_watching_with_invalidation_reason(watch.poll_interval) .start_watching_with_invalidation_reason(watch.poll_interval)
.await?; .await?;
} else { } else {
project_fs.invalidate_with_reason(); project_fs.invalidate_with_reason(|path| invalidation::Initialize {
path: RcStr::from(path),
});
} }
} }
if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) { if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) {
prev_output_fs.invalidate_with_reason(); prev_output_fs.invalidate_with_reason(|path| invalidation::Initialize {
path: RcStr::from(path),
});
} }
Ok(()) Ok(())
......
...@@ -8,7 +8,7 @@ use turbo_tasks::{util::StaticOrArc, FxIndexSet, InvalidationReason, Invalidatio ...@@ -8,7 +8,7 @@ use turbo_tasks::{util::StaticOrArc, FxIndexSet, InvalidationReason, Invalidatio
/// Invalidation was caused by a file change detected by the file watcher /// Invalidation was caused by a file change detected by the file watcher
#[derive(PartialEq, Eq, Hash)] #[derive(PartialEq, Eq, Hash)]
pub struct WatchChange { pub(crate) struct WatchChange {
pub path: String, pub path: String,
} }
...@@ -52,7 +52,7 @@ impl InvalidationReasonKind for WatchChangeKind { ...@@ -52,7 +52,7 @@ impl InvalidationReasonKind for WatchChangeKind {
/// Invalidation was caused by a directory starting to watch from which was read /// Invalidation was caused by a directory starting to watch from which was read
/// before. /// before.
#[derive(PartialEq, Eq, Hash, Clone)] #[derive(PartialEq, Eq, Hash, Clone)]
pub struct WatchStart { pub(crate) struct WatchStart {
pub name: RcStr, pub name: RcStr,
pub path: RcStr, pub path: RcStr,
} }
...@@ -84,7 +84,7 @@ impl InvalidationReasonKind for WatchStartKind { ...@@ -84,7 +84,7 @@ impl InvalidationReasonKind for WatchStartKind {
let example = reasons[0].as_any().downcast_ref::<WatchStart>().unwrap(); let example = reasons[0].as_any().downcast_ref::<WatchStart>().unwrap();
write!( write!(
f, f,
"{} items started watching (e. g. {} in {})", "{} items started watching (e.g. {} in {})",
reasons.len(), reasons.len(),
example.path, example.path,
example.name example.name
...@@ -92,70 +92,75 @@ impl InvalidationReasonKind for WatchStartKind { ...@@ -92,70 +92,75 @@ impl InvalidationReasonKind for WatchStartKind {
} }
} }
/// Invalidation was caused by a write operation on the filesystem /// Invalidation was caused by initialization of a project or filesystem.
#[derive(PartialEq, Eq, Hash)] #[derive(PartialEq, Eq, Hash, Clone)]
pub struct Write { pub struct Initialize {
pub path: String, pub path: RcStr,
} }
impl InvalidationReason for Write { impl InvalidationReason for Initialize {
fn kind(&self) -> Option<StaticOrArc<dyn InvalidationReasonKind>> { fn kind(&self) -> Option<StaticOrArc<dyn InvalidationReasonKind>> {
Some(StaticOrArc::Static(&WRITE_KIND)) Some(StaticOrArc::Static(&INITIALIZE_KIND))
} }
} }
impl Display for Write { impl Display for Initialize {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} written", self.path) write!(
f,
"initialized project or filesystem with path {}",
self.path
)
} }
} }
/// Invalidation kind for [Write] /// [Invalidation kind][InvalidationReasonKind] for [`Initialize`].
#[derive(PartialEq, Eq, Hash)] #[derive(PartialEq, Eq, Hash)]
struct WriteKind; struct InitializeKind;
static WRITE_KIND: WriteKind = WriteKind; static INITIALIZE_KIND: InitializeKind = InitializeKind;
impl InvalidationReasonKind for WriteKind { impl InvalidationReasonKind for InitializeKind {
fn fmt( fn fmt(
&self, &self,
reasons: &FxIndexSet<StaticOrArc<dyn InvalidationReason>>, reasons: &FxIndexSet<StaticOrArc<dyn InvalidationReason>>,
f: &mut Formatter<'_>, f: &mut Formatter<'_>,
) -> std::fmt::Result { ) -> std::fmt::Result {
let example = reasons[0].as_any().downcast_ref::<Initialize>().unwrap();
write!( write!(
f, f,
"{} files written ({}, ...)", "{} items invalidated as part of project or filesystem initialization ({}, ...)",
reasons.len(), reasons.len(),
reasons[0].as_any().downcast_ref::<Write>().unwrap().path example.path,
) )
} }
} }
/// Invalidation was caused by a invalidate operation on the filesystem /// Invalidation was caused by a write operation on the filesystem
#[derive(Clone, PartialEq, Eq, Hash)] #[derive(PartialEq, Eq, Hash)]
pub struct InvalidateFilesystem { pub(crate) struct Write {
pub path: RcStr, pub path: String,
} }
impl InvalidationReason for InvalidateFilesystem { impl InvalidationReason for Write {
fn kind(&self) -> Option<StaticOrArc<dyn InvalidationReasonKind>> { fn kind(&self) -> Option<StaticOrArc<dyn InvalidationReasonKind>> {
Some(StaticOrArc::Static(&INVALIDATE_FILESYSTEM_KIND)) Some(StaticOrArc::Static(&WRITE_KIND))
} }
} }
impl Display for InvalidateFilesystem { impl Display for Write {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} in filesystem invalidated", self.path) write!(f, "{} written", self.path)
} }
} }
/// Invalidation kind for [InvalidateFilesystem] /// Invalidation kind for [Write]
#[derive(PartialEq, Eq, Hash)] #[derive(PartialEq, Eq, Hash)]
struct InvalidateFilesystemKind; struct WriteKind;
static INVALIDATE_FILESYSTEM_KIND: InvalidateFilesystemKind = InvalidateFilesystemKind; static WRITE_KIND: WriteKind = WriteKind;
impl InvalidationReasonKind for InvalidateFilesystemKind { impl InvalidationReasonKind for WriteKind {
fn fmt( fn fmt(
&self, &self,
reasons: &FxIndexSet<StaticOrArc<dyn InvalidationReason>>, reasons: &FxIndexSet<StaticOrArc<dyn InvalidationReason>>,
...@@ -163,13 +168,9 @@ impl InvalidationReasonKind for InvalidateFilesystemKind { ...@@ -163,13 +168,9 @@ impl InvalidationReasonKind for InvalidateFilesystemKind {
) -> std::fmt::Result { ) -> std::fmt::Result {
write!( write!(
f, f,
"{} items in filesystem invalidated ({}, ...)", "{} files written ({}, ...)",
reasons.len(), reasons.len(),
reasons[0] reasons[0].as_any().downcast_ref::<Write>().unwrap().path
.as_any()
.downcast_ref::<InvalidateFilesystem>()
.unwrap()
.path
) )
} }
} }
...@@ -42,14 +42,22 @@ impl InvalidatorMap { ...@@ -42,14 +42,22 @@ impl InvalidatorMap {
Ok(guard) Ok(guard)
} }
#[allow(unused_must_use)]
pub fn insert( pub fn insert(
&self, &self,
key: String, key: String,
invalidator: Invalidator, invalidator: Invalidator,
write_content: Option<WriteContent>, write_content: Option<WriteContent>,
) { ) {
self.queue.push((key, invalidator, write_content)); self.queue
.push((key, invalidator, write_content))
.unwrap_or_else(|err| {
let (key, ..) = err.into_inner();
// PushError<T> is not Debug
panic!(
"failed to push {key:?} queue push should never fail, queue is unbounded and \
never closed"
)
});
} }
} }
......
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
pub mod attach; pub mod attach;
pub mod embed; pub mod embed;
pub mod glob; pub mod glob;
mod invalidation; pub mod invalidation;
mod invalidator_map; mod invalidator_map;
pub mod json; pub mod json;
mod mutex_map; mod mutex_map;
...@@ -42,7 +42,6 @@ use bitflags::bitflags; ...@@ -42,7 +42,6 @@ use bitflags::bitflags;
use dunce::simplified; use dunce::simplified;
use glob::Glob; use glob::Glob;
use indexmap::IndexSet; use indexmap::IndexSet;
use invalidation::InvalidateFilesystem;
use invalidator_map::InvalidatorMap; use invalidator_map::InvalidatorMap;
use jsonc_parser::{parse_to_serde_value, ParseOptions}; use jsonc_parser::{parse_to_serde_value, ParseOptions};
use mime::Mime; use mime::Mime;
...@@ -60,7 +59,8 @@ use tracing::Instrument; ...@@ -60,7 +59,8 @@ use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
debug::ValueDebugFormat, effect, mark_session_dependent, mark_stateful, trace::TraceRawVcs, debug::ValueDebugFormat, effect, mark_session_dependent, mark_stateful, trace::TraceRawVcs,
Completion, Invalidator, NonLocalValue, ReadRef, ResolvedVc, ValueToString, Vc, Completion, InvalidationReason, Invalidator, NonLocalValue, ReadRef, ResolvedVc, ValueToString,
Vc,
}; };
use turbo_tasks_hash::{ use turbo_tasks_hash::{
hash_xxh3_hash128, hash_xxh3_hash64, DeterministicHash, DeterministicHasher, hash_xxh3_hash128, hash_xxh3_hash64, DeterministicHash, DeterministicHasher,
...@@ -318,7 +318,13 @@ impl DiskFileSystemInner { ...@@ -318,7 +318,13 @@ impl DiskFileSystemInner {
}); });
} }
fn invalidate_with_reason(&self) { /// Invalidates every tracked file in the filesystem.
///
/// Calls the given
fn invalidate_with_reason<R: InvalidationReason + Clone>(
&self,
reason: impl Fn(String) -> R + Sync,
) {
let _span = tracing::info_span!("invalidate filesystem", path = &*self.root).entered(); let _span = tracing::info_span!("invalidate filesystem", path = &*self.root).entered();
let span = tracing::Span::current(); let span = tracing::Span::current();
let handle = tokio::runtime::Handle::current(); let handle = tokio::runtime::Handle::current();
...@@ -329,10 +335,10 @@ impl DiskFileSystemInner { ...@@ -329,10 +335,10 @@ impl DiskFileSystemInner {
.chain(dir_invalidator_map.into_par_iter()) .chain(dir_invalidator_map.into_par_iter())
.flat_map(|(path, invalidators)| { .flat_map(|(path, invalidators)| {
let _span = span.clone().entered(); let _span = span.clone().entered();
let reason = InvalidateFilesystem { path: path.into() }; let reason_for_path = reason(path);
invalidators invalidators
.into_par_iter() .into_par_iter()
.map(move |i| (reason.clone(), i)) .map(move |i| (reason_for_path.clone(), i))
}); });
iter.for_each(|(reason, (invalidator, _))| { iter.for_each(|(reason, (invalidator, _))| {
let _span = span.clone().entered(); let _span = span.clone().entered();
...@@ -408,8 +414,11 @@ impl DiskFileSystem { ...@@ -408,8 +414,11 @@ impl DiskFileSystem {
self.inner.invalidate(); self.inner.invalidate();
} }
pub fn invalidate_with_reason(&self) { pub fn invalidate_with_reason<R: InvalidationReason + Clone>(
self.inner.invalidate_with_reason(); &self,
reason: impl Fn(String) -> R + Sync,
) {
self.inner.invalidate_with_reason(reason);
} }
pub async fn start_watching(&self, poll_interval: Option<Duration>) -> Result<()> { pub async fn start_watching(&self, poll_interval: Option<Duration>) -> Result<()> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment