Unverified Commit ee690921 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Show last modified file when waiting for the filesystem to settle (#97648)

This is a follow-up to https://github.com/vercel/next.js/pull/96116

If we're waiting a long time for your filesystem to settle, we should
show you the last modified path, so at least you can have an idea of why
this is happening.
parent 38c38991
...@@ -33,6 +33,7 @@ import path from 'path' ...@@ -33,6 +33,7 @@ import path from 'path'
() => { () => {
const output = stripAnsi(next.cliOutput.slice(outputIndex)) const output = stripAnsi(next.cliOutput.slice(outputIndex))
expect(output).toContain('waiting for the filesystem to settle') expect(output).toContain('waiting for the filesystem to settle')
expect(output).toContain(pkgFile)
}, },
// The event fires after ~5s; allow a generous window to avoid flakes. // The event fires after ~5s; allow a generous window to avoid flakes.
15000, 15000,
......
...@@ -7,9 +7,13 @@ use std::{ ...@@ -7,9 +7,13 @@ use std::{
}; };
use serde::Serialize; use serde::Serialize;
use turbo_rcstr::RcStr;
use turbo_tasks::message_queue::{CompilationEvent, Severity}; use turbo_tasks::message_queue::{CompilationEvent, Severity};
use crate::{DiskWatcherConfig, watcher::fs_api::DiskFileSystemWatcherApi}; use crate::{
DiskWatcherConfig,
watcher::{BatchedInvalidations, fs_api::DiskFileSystemWatcherApi},
};
/// Decides how long a batch of watcher events stays open, and emits a repeated /// Decides how long a batch of watcher events stays open, and emits a repeated
/// [`FilesystemSettlingEvent`] for as long as it does. /// [`FilesystemSettlingEvent`] for as long as it does.
...@@ -45,7 +49,9 @@ impl BatchSchedule { ...@@ -45,7 +49,9 @@ impl BatchSchedule {
let now = Instant::now(); let now = Instant::now();
let deadline = now.checked_add(delay).unwrap_or_else(far_future); let deadline = now.checked_add(delay).unwrap_or_else(far_future);
match &mut self.pending { match &mut self.pending {
Some(pending) => pending.deadline = pending.deadline.max(deadline), Some(pending) => {
pending.deadline = pending.deadline.max(deadline);
}
None => { None => {
self.pending = Some(PendingBatch { self.pending = Some(PendingBatch {
started: now, started: now,
...@@ -68,6 +74,7 @@ impl BatchSchedule { ...@@ -68,6 +74,7 @@ impl BatchSchedule {
&mut self, &mut self,
rx: &Receiver<notify::Result<notify::Event>>, rx: &Receiver<notify::Result<notify::Event>>,
fs: &FsApi, fs: &FsApi,
batch: &BatchedInvalidations,
) -> Result<notify::Result<notify::Event>, RecvTimeoutError> { ) -> Result<notify::Result<notify::Event>, RecvTimeoutError> {
let max_event_delay = self.settling_event_max_delay; let max_event_delay = self.settling_event_max_delay;
loop { loop {
...@@ -78,7 +85,7 @@ impl BatchSchedule { ...@@ -78,7 +85,7 @@ impl BatchSchedule {
let now = Instant::now(); let now = Instant::now();
if now >= pending.settling_event_next_at { if now >= pending.settling_event_next_at {
pending.emit_settling_event(fs, now, max_event_delay); pending.emit_settling_event(fs, batch, now, max_event_delay);
} }
let timeout = pending let timeout = pending
...@@ -112,6 +119,7 @@ impl PendingBatch { ...@@ -112,6 +119,7 @@ impl PendingBatch {
fn emit_settling_event<FsApi: DiskFileSystemWatcherApi>( fn emit_settling_event<FsApi: DiskFileSystemWatcherApi>(
&mut self, &mut self,
fs: &FsApi, fs: &FsApi,
batch: &BatchedInvalidations,
now: Instant, now: Instant,
max_event_delay: Duration, max_event_delay: Duration,
) { ) {
...@@ -119,6 +127,9 @@ impl PendingBatch { ...@@ -119,6 +127,9 @@ impl PendingBatch {
if let Some(turbo_tasks) = fs.turbo_tasks() { if let Some(turbo_tasks) = fs.turbo_tasks() {
turbo_tasks.send_compilation_event(Arc::new(FilesystemSettlingEvent { turbo_tasks.send_compilation_event(Arc::new(FilesystemSettlingEvent {
elapsed_secs: (now - self.started).as_secs_f64(), elapsed_secs: (now - self.started).as_secs_f64(),
last_modified_path: batch
.last_updated_path()
.map(|path| RcStr::from(format!("{path:?}"))),
})); }));
} }
self.event_interval = self.event_interval.saturating_mul(2).min(max_event_delay); self.event_interval = self.event_interval.saturating_mul(2).min(max_event_delay);
...@@ -137,6 +148,8 @@ impl PendingBatch { ...@@ -137,6 +148,8 @@ impl PendingBatch {
pub struct FilesystemSettlingEvent { pub struct FilesystemSettlingEvent {
/// How long the current batch has been held open, in seconds. /// How long the current batch has been held open, in seconds.
pub elapsed_secs: f64, pub elapsed_secs: f64,
/// The most recently changed path in the current batch.
pub last_modified_path: Option<RcStr>,
} }
impl CompilationEvent for FilesystemSettlingEvent { impl CompilationEvent for FilesystemSettlingEvent {
...@@ -149,9 +162,14 @@ impl CompilationEvent for FilesystemSettlingEvent { ...@@ -149,9 +162,14 @@ impl CompilationEvent for FilesystemSettlingEvent {
} }
fn message(&self) -> String { fn message(&self) -> String {
let last_modified = self
.last_modified_path
.as_deref()
.map(|path| format!("; last modified: {path}"))
.unwrap_or_default();
format!( format!(
"Turbopack has seen frequent file updates and is waiting for the filesystem to settle \ "Turbopack has seen frequent file updates and is waiting for the filesystem to settle \
({:.1}s elapsed so far).", ({:.1}s elapsed so far{last_modified}).",
self.elapsed_secs self.elapsed_secs
) )
} }
......
...@@ -23,17 +23,18 @@ use bincode::{ ...@@ -23,17 +23,18 @@ use bincode::{
error::{DecodeError, EncodeError}, error::{DecodeError, EncodeError},
}; };
use bitflags::bitflags; use bitflags::bitflags;
use indexmap::map::Entry;
use notify::{ use notify::{
Config, EventKind, PollWatcher, RecommendedWatcher, Watcher, Config, EventKind, PollWatcher, RecommendedWatcher, Watcher,
event::{MetadataKind, ModifyKind, RenameMode}, event::{MetadataKind, ModifyKind, RenameMode},
}; };
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::FxHashSet;
use tokio::sync::{RwLock, RwLockWriteGuard}; use tokio::sync::{RwLock, RwLockWriteGuard};
use tracing::instrument; use tracing::instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, ResolvedVc, TraitRef, FxIndexMap, FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, ResolvedVc,
TurboTasksApi, spawn_thread, trace::TraceRawVcs, util::StaticOrArc, TraitRef, TurboTasksApi, spawn_thread, trace::TraceRawVcs, util::StaticOrArc,
}; };
use crate::{ use crate::{
...@@ -607,7 +608,7 @@ impl DiskWatcher { ...@@ -607,7 +608,7 @@ impl DiskWatcher {
'outer: loop { 'outer: loop {
loop { loop {
match schedule.recv_event(&rx, &*fs) { match schedule.recv_event(&rx, &*fs, &batch) {
Ok(Ok(event)) => { Ok(Ok(event)) => {
// TODO: We might benefit from some user-facing diagnostics if it rescans // TODO: We might benefit from some user-facing diagnostics if it rescans
// occur frequently (i.e. more than X times in Y minutes) // occur frequently (i.e. more than X times in Y minutes)
...@@ -675,15 +676,7 @@ impl DiskWatcher { ...@@ -675,15 +676,7 @@ impl DiskWatcher {
Ok(Err(notify::Error { kind, paths })) => { Ok(Err(notify::Error { kind, paths })) => {
println!("watch error ({paths:?}): {kind:?} "); println!("watch error ({paths:?}): {kind:?} ");
let flags = InvalidationFlags::PATH_AND_CHILDREN batch.add_error(paths, fs.root_path());
| InvalidationFlags::PATH_AND_CHILDREN_DIR;
if paths.is_empty() {
batch.mark(Box::from(fs.root_path()), flags);
} else {
for path in paths {
batch.mark(path.into_boxed_path(), flags);
}
}
schedule.extend(config.batch_delay); schedule.extend(config.batch_delay);
} }
Err(RecvTimeoutError::Timeout) => { Err(RecvTimeoutError::Timeout) => {
...@@ -781,7 +774,9 @@ bitflags! { ...@@ -781,7 +774,9 @@ bitflags! {
/// needs to happen for each, rather than in several separate sets. This avoids cloning each /// needs to happen for each, rather than in several separate sets. This avoids cloning each
/// `PathBuf` into multiple collections. /// `PathBuf` into multiple collections.
struct BatchedInvalidations { struct BatchedInvalidations {
paths: FxHashMap<Box<Path>, InvalidationFlags>, paths: FxIndexMap<Box<Path>, InvalidationFlags>,
/// The most recently updated entry in [`Self::paths`].
last_updated_index: Option<usize>,
/// See [`Self::new_paths`]). Stored as [`None`] in non-recursive mode. /// See [`Self::new_paths`]). Stored as [`None`] in non-recursive mode.
new_paths: Option<FxHashSet<Box<Path>>>, new_paths: Option<FxHashSet<Box<Path>>>,
/// Whether events are coming from [`PollWatcher`] instead of [`RecommendedWatcher`], which /// Whether events are coming from [`PollWatcher`] instead of [`RecommendedWatcher`], which
...@@ -792,7 +787,8 @@ struct BatchedInvalidations { ...@@ -792,7 +787,8 @@ struct BatchedInvalidations {
impl BatchedInvalidations { impl BatchedInvalidations {
fn new(recursive_mode: DiskWatcherRecursiveMode, polling: bool) -> Self { fn new(recursive_mode: DiskWatcherRecursiveMode, polling: bool) -> Self {
Self { Self {
paths: FxHashMap::default(), paths: FxIndexMap::default(),
last_updated_index: None,
new_paths: match recursive_mode { new_paths: match recursive_mode {
DiskWatcherRecursiveMode::NonRecursive => Some(FxHashSet::default()), DiskWatcherRecursiveMode::NonRecursive => Some(FxHashSet::default()),
DiskWatcherRecursiveMode::Recursive => None, DiskWatcherRecursiveMode::Recursive => None,
...@@ -830,6 +826,7 @@ impl BatchedInvalidations { ...@@ -830,6 +826,7 @@ impl BatchedInvalidations {
fn clear(&mut self) { fn clear(&mut self) {
self.paths.clear(); self.paths.clear();
self.last_updated_index = None;
if let Some(new_paths) = &mut self.new_paths { if let Some(new_paths) = &mut self.new_paths {
new_paths.clear(); new_paths.clear();
} }
...@@ -843,8 +840,25 @@ impl BatchedInvalidations { ...@@ -843,8 +840,25 @@ impl BatchedInvalidations {
} }
} }
fn mark(&mut self, path: Box<Path>, flags: InvalidationFlags) { /// Sets the `flags` for `path`. Returns the index that was modified.
*self.paths.entry(path).or_insert(InvalidationFlags::empty()) |= flags; fn mark(&mut self, path: Box<Path>, flags: InvalidationFlags) -> usize {
match self.paths.entry(path) {
Entry::Occupied(mut entry) => {
*entry.get_mut() |= flags;
entry.index()
}
Entry::Vacant(entry) => {
let index = entry.index();
entry.insert(flags);
index
}
}
}
fn last_updated_path(&self) -> Option<&Path> {
self.last_updated_index
.and_then(|index| self.paths.get_index(index))
.map(|(path, _)| &**path)
} }
fn mark_parent_dir(&mut self, path: &Path) { fn mark_parent_dir(&mut self, path: &Path) {
...@@ -863,49 +877,45 @@ impl BatchedInvalidations { ...@@ -863,49 +877,45 @@ impl BatchedInvalidations {
/// Updates the batch to contain updated paths from the given event. Does not perform any /// Updates the batch to contain updated paths from the given event. Does not perform any
/// invalidations. /// invalidations.
/// ///
/// Returns `true` if the event contained relevant events, or `false` if it was filtered out. /// Returns whether the event contained relevant events.
#[must_use] #[must_use]
fn add_event(&mut self, event: notify::Event) -> bool { fn add_event(&mut self, event: notify::Event) -> bool {
let paths: Vec<PathBuf> = event.paths; let paths: Vec<PathBuf> = event.paths;
if paths.is_empty() { let mut last_updated_index = None;
return false;
}
match event.kind { match event.kind {
EventKind::Modify(ModifyKind::Data(_)) => { EventKind::Modify(ModifyKind::Data(_)) => {
for path in paths { for path in paths {
self.mark(path.into_boxed_path(), InvalidationFlags::PATH); last_updated_index =
Some(self.mark(path.into_boxed_path(), InvalidationFlags::PATH));
} }
true
} }
// Some backends (fsevents, polling) can report metadata events for file content changes // Some backends (fsevents, polling) can report metadata events for file content changes
EventKind::Modify(ModifyKind::Metadata(kind)) if self.is_content_change(kind) => { EventKind::Modify(ModifyKind::Metadata(kind)) if self.is_content_change(kind) => {
for path in paths { for path in paths {
self.mark(path.into_boxed_path(), InvalidationFlags::PATH); last_updated_index =
Some(self.mark(path.into_boxed_path(), InvalidationFlags::PATH));
} }
true
} }
EventKind::Create(_) => { EventKind::Create(_) => {
for path in paths { for path in paths {
self.mark_parent_dir(&path); self.mark_parent_dir(&path);
self.mark_new_path(&path); self.mark_new_path(&path);
self.mark( last_updated_index = Some(self.mark(
path.into_boxed_path(), path.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN InvalidationFlags::PATH_AND_CHILDREN
| InvalidationFlags::PATH_AND_CHILDREN_DIR, | InvalidationFlags::PATH_AND_CHILDREN_DIR,
); ));
} }
true
} }
EventKind::Remove(_) => { EventKind::Remove(_) => {
for path in paths { for path in paths {
self.mark_parent_dir(&path); self.mark_parent_dir(&path);
self.mark( last_updated_index = Some(self.mark(
path.into_boxed_path(), path.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN InvalidationFlags::PATH_AND_CHILDREN
| InvalidationFlags::PATH_AND_CHILDREN_DIR, | InvalidationFlags::PATH_AND_CHILDREN_DIR,
); ));
} }
true
} }
// A single event emitted with both the `From` and `To` paths. // A single event emitted with both the `From` and `To` paths.
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
...@@ -918,11 +928,10 @@ impl BatchedInvalidations { ...@@ -918,11 +928,10 @@ impl BatchedInvalidations {
); );
self.mark_parent_dir(&destination); self.mark_parent_dir(&destination);
self.mark_new_path(&destination); self.mark_new_path(&destination);
self.mark( last_updated_index = Some(self.mark(
destination.into_boxed_path(), destination.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN, InvalidationFlags::PATH_AND_CHILDREN,
); ));
true
} }
// We expect `RenameMode::Both` to cover most of the cases we need to invalidate, // We expect `RenameMode::Both` to cover most of the cases we need to invalidate,
// but we also check other RenameModes to cover cases where notify couldn't match the // but we also check other RenameModes to cover cases where notify couldn't match the
...@@ -930,19 +939,33 @@ impl BatchedInvalidations { ...@@ -930,19 +939,33 @@ impl BatchedInvalidations {
EventKind::Any | EventKind::Modify(ModifyKind::Any | ModifyKind::Name(..)) => { EventKind::Any | EventKind::Modify(ModifyKind::Any | ModifyKind::Name(..)) => {
for path in paths { for path in paths {
self.mark_parent_dir(&path); self.mark_parent_dir(&path);
self.mark( last_updated_index = Some(self.mark(
path.into_boxed_path(), path.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN InvalidationFlags::PATH_AND_CHILDREN
| InvalidationFlags::PATH_AND_CHILDREN_DIR, | InvalidationFlags::PATH_AND_CHILDREN_DIR,
); ));
} }
true
} }
EventKind::Modify(ModifyKind::Metadata(..) | ModifyKind::Other) EventKind::Modify(ModifyKind::Metadata(..) | ModifyKind::Other)
| EventKind::Access(_) | EventKind::Access(_)
| EventKind::Other => { | EventKind::Other => {}
// ignored }
false if let Some(index) = last_updated_index {
self.last_updated_index = Some(index);
true
} else {
false
}
}
/// Updates the batch to invalidate paths associated with a watcher error.
fn add_error(&mut self, paths: Vec<PathBuf>, root_path: &Path) {
let flags = InvalidationFlags::PATH_AND_CHILDREN | InvalidationFlags::PATH_AND_CHILDREN_DIR;
if paths.is_empty() {
self.last_updated_index = Some(self.mark(Box::from(root_path), flags));
} else {
for path in paths {
self.last_updated_index = Some(self.mark(path.into_boxed_path(), flags));
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment