Unverified Commit b677feb0 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: More aggressively debounce filesystem watch events if we detected…

Turbopack: More aggressively debounce filesystem watch events if we detected changes to node_modules (#96116)

Previously, we were debouncing update by sleeping 1ms at a time on macos
and windows, and 10ms at a time on Linux.

During a slow `pnpm install`, or a `git checkout`, this could cause us
to do a bunch of extra throwaway work.

Changes:
- Increase the debounce interval to a consistent 10ms everywhere. This
should still be small enough that it's not noticable on macos or
windows.
- If an event touches `node_modules`, there's a good chance that a
package manager is running and many other files will be modified, so
extend the batch deadline by 200ms instead of 10ms.
- Because there's a chance that the batch deadline could get extended
indefinitely (this was always possible, just more likely now) include a
compilation event that gets logged after 5 seconds.
parent dc5fe225
...@@ -93,7 +93,7 @@ use turbopack_node::child_process_backend; ...@@ -93,7 +93,7 @@ use turbopack_node::child_process_backend;
use turbopack_node::execution_context::ExecutionContext; use turbopack_node::execution_context::ExecutionContext;
#[cfg(feature = "worker_pool")] #[cfg(feature = "worker_pool")]
use turbopack_node::worker_threads_backend; use turbopack_node::worker_threads_backend;
use turbopack_nodejs::NodeJsChunkingContext; use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher};
use crate::{ use crate::{
aggregate_hmr::{AggregateHmrVersion, ChunkListUpdateBuilder, DiffResult, diff_chunks_against}, aggregate_hmr::{AggregateHmrVersion, ChunkListUpdateBuilder, DiffResult, diff_chunks_against},
...@@ -1094,10 +1094,13 @@ impl Project { ...@@ -1094,10 +1094,13 @@ impl Project {
*self.root_path, *self.root_path,
vec![denied_path, denied_profiles_path], vec![denied_path, denied_profiles_path],
DiskWatcherConfig { DiskWatcherConfig {
recursive_mode: None,
poll_interval: self.watch.poll_interval, poll_interval: self.watch.poll_interval,
// the dev server reports these to the user // the dev server reports these to the user
report_invalidation_reason: true, report_invalidation_reason: true,
extended_batch_delay_matcher: Some(ResolvedVc::upcast(
NodeModulesPathMatcher.resolved_cell(),
)),
..Default::default()
}, },
)) ))
} }
......
...@@ -496,6 +496,7 @@ export async function createHotReloaderTurbopack( ...@@ -496,6 +496,7 @@ export async function createHotReloaderTurbopack(
'StartupCacheInvalidationEvent', 'StartupCacheInvalidationEvent',
'TimingEvent', 'TimingEvent',
'SlowFilesystemEvent', 'SlowFilesystemEvent',
'FilesystemSettlingEvent',
'TraceEvent', 'TraceEvent',
], ],
parentSpan: hotReloaderSpan, parentSpan: hotReloaderSpan,
......
export default function RootLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html>
<body>{children}</body>
</html>
)
}
// Importing this package makes Turbopack read (and therefore watch) the file
// inside `node_modules`, so the writes the test performs generate watcher
// events. This matters on Linux, where the watcher is non-recursive and only
// watches directories it has been asked to read.
import counter from 'fs-settling-fixture-pkg'
export default function Page() {
return <p>counter: {counter}</p>
}
import { nextTestSetup } from 'e2e-utils'
import { retry } from 'next-test-utils'
import stripAnsi from 'strip-ansi'
import fs from 'fs'
import path from 'path'
// The `FilesystemSettlingEvent` compilation event is Turbopack-only.
;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)(
'fs-settling-event',
() => {
const { next } = nextTestSetup({ files: __dirname })
it('logs a settling event during sustained node_modules churn', async () => {
// Compile the page first so the imported `node_modules` file is watched.
await next.render('/')
const pkgFile = path.join(
next.testDir,
'node_modules/fs-settling-fixture-pkg/index.js'
)
const outputIndex = next.cliOutput.length
// Rewrite the imported module every 20ms. Since 20ms is well below the
// extended `node_modules` batch delay (200ms), the watcher keeps a single
// batch of events open, which triggers the settling event after ~5s.
let i = 0
const interval = setInterval(() => {
fs.writeFileSync(pkgFile, `export default ${i++}\n`)
}, 20)
try {
await retry(
() => {
const output = stripAnsi(next.cliOutput.slice(outputIndex))
expect(output).toContain('waiting for the filesystem to settle')
},
// The event fires after ~5s; allow a generous window to avoid flakes.
15000,
500
)
} finally {
clearInterval(interval)
}
})
}
)
{
"name": "fs-settling-fixture-pkg",
"version": "1.0.0",
"main": "index.js"
}
...@@ -60,7 +60,7 @@ pub use crate::{ ...@@ -60,7 +60,7 @@ pub use crate::{
path::{FileSystemPath, FileSystemPathOption, RealPathResult, RealPathResultError, rebase}, path::{FileSystemPath, FileSystemPathOption, RealPathResult, RealPathResultError, rebase},
read_glob::ReadGlobResult, read_glob::ReadGlobResult,
virtual_fs::VirtualFileSystem, virtual_fs::VirtualFileSystem,
watcher::{DiskWatcherConfig, DiskWatcherRecursiveMode}, watcher::{DiskWatcherConfig, DiskWatcherPathMatcher, DiskWatcherRecursiveMode},
windows::to_verbatim_with_case_folded_disk, windows::to_verbatim_with_case_folded_disk,
}; };
......
use std::{
sync::{
Arc,
mpsc::{Receiver, RecvTimeoutError},
},
time::{Duration, Instant},
};
use serde::Serialize;
use turbo_tasks::message_queue::{CompilationEvent, Severity};
use crate::{DiskWatcherConfig, watcher::fs_api::DiskFileSystemWatcherApi};
/// Decides how long a batch of watcher events stays open, and emits a repeated
/// [`FilesystemSettlingEvent`] for as long as it does.
pub struct BatchSchedule {
settling_event_initial_delay: Duration,
settling_event_max_delay: Duration,
pending: Option<PendingBatch>,
}
/// A batch that has at least one event in it and hasn't been flushed yet.
struct PendingBatch {
started: Instant,
/// The batch is flushed once this passes without any further events.
deadline: Instant,
/// When to emit the next [`FilesystemSettlingEvent`].
settling_event_next_at: Instant,
/// Grows exponentially (up to [`BatchSchedule::settling_event_max_delay`]) so that a writer
/// holding a batch open for minutes doesn't flood the compilation event queue.
event_interval: Duration,
}
impl BatchSchedule {
pub fn new(config: &DiskWatcherConfig) -> Self {
Self {
settling_event_initial_delay: config.settling_event_initial_delay,
settling_event_max_delay: config.settling_event_max_delay,
pending: None,
}
}
/// Keeps the batch open for at least `delay` from now, opening a new batch if there isn't one.
pub fn extend(&mut self, delay: Duration) {
let now = Instant::now();
let deadline = now.checked_add(delay).unwrap_or_else(far_future);
match &mut self.pending {
Some(pending) => pending.deadline = pending.deadline.max(deadline),
None => {
self.pending = Some(PendingBatch {
started: now,
deadline,
settling_event_next_at: now
.checked_add(self.settling_event_initial_delay)
.unwrap_or_else(far_future),
event_interval: self.settling_event_initial_delay,
})
}
}
}
/// Waits for the next watcher event, emitting [`FilesystemSettlingEvent`]s while the pending
/// batch keeps growing. If no batch is pending, this blocks until an event arrives.
///
/// [`RecvTimeoutError::Timeout`] means the pending batch's deadline has passed *and* nothing
/// more is queued, so the batch is complete and should be flushed.
pub fn recv_event<FsApi: DiskFileSystemWatcherApi>(
&mut self,
rx: &Receiver<notify::Result<notify::Event>>,
fs: &FsApi,
) -> Result<notify::Result<notify::Event>, RecvTimeoutError> {
let max_event_delay = self.settling_event_max_delay;
loop {
let Some(pending) = &mut self.pending else {
// no pending batch: wait indefinitely
return rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
};
let now = Instant::now();
if now >= pending.settling_event_next_at {
pending.emit_settling_event(fs, now, max_event_delay);
}
let timeout = pending
.deadline
.min(pending.settling_event_next_at)
.saturating_duration_since(now);
match rx.recv_timeout(timeout) {
Ok(event) => {
return Ok(event);
}
Err(RecvTimeoutError::Timeout) => {
if Instant::now() >= pending.deadline {
self.pending = None;
return Err(RecvTimeoutError::Timeout);
}
continue;
}
Err(err) => return Err(err),
}
}
}
/// Closes the pending batch, used when a rescan happens.
pub fn reset(&mut self) {
self.pending = None;
}
}
impl PendingBatch {
fn emit_settling_event<FsApi: DiskFileSystemWatcherApi>(
&mut self,
fs: &FsApi,
now: Instant,
max_event_delay: Duration,
) {
let _guard = fs.tokio_handle().enter();
if let Some(turbo_tasks) = fs.turbo_tasks() {
turbo_tasks.send_compilation_event(Arc::new(FilesystemSettlingEvent {
elapsed_secs: (now - self.started).as_secs_f64(),
}));
}
self.event_interval = self.event_interval.saturating_mul(2).min(max_event_delay);
// Schedule from "now" instead of accumulating intervals, so that emitting late (e.g. under
// heavy load) doesn't produce a catch-up burst of events.
self.settling_event_next_at = now
.checked_add(self.event_interval)
.unwrap_or_else(far_future);
}
}
/// Emitted when frequent filesystem updates cause us to keep a batch open for an extended period of
/// time. Informing the user when this happens may help them understand what's happening, and that
/// Turbopack is not stalled.
#[derive(Debug, Clone, Serialize)]
pub struct FilesystemSettlingEvent {
/// How long the current batch has been held open, in seconds.
pub elapsed_secs: f64,
}
impl CompilationEvent for FilesystemSettlingEvent {
fn type_name(&self) -> &'static str {
"FilesystemSettlingEvent"
}
fn severity(&self) -> Severity {
Severity::Info
}
fn message(&self) -> String {
format!(
"Turbopack has seen frequent file updates and is waiting for the filesystem to settle \
({:.1}s elapsed so far).",
self.elapsed_secs
)
}
fn to_json(&self) -> String {
serde_json::to_string(self).unwrap()
}
}
// from https://github.com/tokio-rs/tokio/blob/29cd6ec1ec6f90a7ee1ad641c03e0e00badbcb0e/tokio/src/time/instant.rs#L57-L63
fn far_future() -> Instant {
// Roughly 30 years from now.
// API does not provide a way to obtain max `Instant`
// or convert specific date in the future to instant.
// 1000 years overflows on macOS, 100 years overflows on FreeBSD.
Instant::now() + Duration::from_secs(86400 * 365 * 30)
}
use std::{
ffi::OsStr,
path::{Component, Path},
};
use turbo_tasks_fs::DiskWatcherPathMatcher;
/// Matches anything inside of a `node_modules` directory.
///
/// Package managers churn `node_modules` heavily while the dev server is running. More aggressively
/// batching these may reduce system load during an installation.
#[turbo_tasks::value(shared)]
pub struct NodeModulesPathMatcher;
#[turbo_tasks::value_impl]
impl DiskWatcherPathMatcher for NodeModulesPathMatcher {
fn match_path(&self, path: &Path) -> bool {
path.components()
.any(|component| component == Component::Normal(OsStr::new("node_modules")))
}
}
...@@ -3,5 +3,6 @@ ...@@ -3,5 +3,6 @@
pub(crate) mod chunking_context; pub(crate) mod chunking_context;
pub mod ecmascript; pub mod ecmascript;
pub mod fs;
pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder}; pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment