Unverified Commit 49bfb650 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Avoid cloning paths in fs watcher if the path is already in a map (#97655)

This is a spiritual continuation of
https://github.com/vercel/next.js/pull/96114 . See
https://github.com/vercel/next.js/pull/94735 for motivation.

This codepath should already be pretty cheap, but in some cases (notably
inotify on Linux where updates aren't debounced as aggressively by the
OS) it could become somewhat hot and waste CPU while other
somewhat-CPU-heavy work is happening in other software (e.g. `pnpm`).

This removes two more sources of clones:
- If we only have a `&Path`, but it's already in the `paths` map, we can
use the raw entry API to avoid cloning it.
- `new_paths` can use `usize` indexes into the `FxIndexMap` instead of
storing another clone of the path. We could also encode this information
as flags on `path`, but the iteration over `new_paths` must happen as a
separate loop before the normal batch execution loop (near the bottom of
`DiskWatcher::watch_thread`), so I'd like to keep it as a separate data
structure to avoid having to filter out a bunch of unrelated keys
without the flag when finally processing the batch.
parent d7aa66c3
This diff is collapsed.
...@@ -281,7 +281,7 @@ futures-retry = "0.6.0" ...@@ -281,7 +281,7 @@ futures-retry = "0.6.0"
futures-util = "0.3.31" futures-util = "0.3.31"
hashbrown = "0.14.5" hashbrown = "0.14.5"
image = { version = "0.25.8", default-features = false } image = { version = "0.25.8", default-features = false }
indexmap = "2.13.0" indexmap = "2.14.0"
indoc = "2.0.0" indoc = "2.0.0"
itertools = "0.10.5" itertools = "0.10.5"
lightningcss = { version = "1.0.0-alpha.72", features = [ lightningcss = { version = "1.0.0-alpha.72", features = [
......
...@@ -5,6 +5,7 @@ mod mock_fs_api; ...@@ -5,6 +5,7 @@ mod mock_fs_api;
use std::{ use std::{
any::Any, any::Any,
borrow::Cow,
collections::BTreeSet, collections::BTreeSet,
env, fmt, env, fmt,
path::{Path, PathBuf}, path::{Path, PathBuf},
...@@ -23,7 +24,7 @@ use bincode::{ ...@@ -23,7 +24,7 @@ use bincode::{
error::{DecodeError, EncodeError}, error::{DecodeError, EncodeError},
}; };
use bitflags::bitflags; use bitflags::bitflags;
use indexmap::map::Entry; use indexmap::map::{RawEntryApiV1, raw_entry_v1::RawEntryMut};
use notify::{ use notify::{
Config, EventKind, PollWatcher, RecommendedWatcher, Watcher, Config, EventKind, PollWatcher, RecommendedWatcher, Watcher,
event::{MetadataKind, ModifyKind, RenameMode}, event::{MetadataKind, ModifyKind, RenameMode},
...@@ -777,8 +778,8 @@ struct BatchedInvalidations { ...@@ -777,8 +778,8 @@ struct BatchedInvalidations {
paths: FxIndexMap<Box<Path>, InvalidationFlags>, paths: FxIndexMap<Box<Path>, InvalidationFlags>,
/// The most recently updated entry in [`Self::paths`]. /// The most recently updated entry in [`Self::paths`].
last_updated_index: Option<usize>, last_updated_index: Option<usize>,
/// See [`Self::new_paths`]). Stored as [`None`] in non-recursive mode. /// See [`Self::new_paths`]. Stored as [`None`] in recursive mode.
new_paths: Option<FxHashSet<Box<Path>>>, new_paths: Option<FxHashSet<usize>>,
/// Whether events are coming from [`PollWatcher`] instead of [`RecommendedWatcher`], which /// Whether events are coming from [`PollWatcher`] instead of [`RecommendedWatcher`], which
/// changes how a file content change is reported. See [`Self::is_content_change`]. /// changes how a file content change is reported. See [`Self::is_content_change`].
polling: bool, polling: bool,
...@@ -832,24 +833,24 @@ impl BatchedInvalidations { ...@@ -832,24 +833,24 @@ impl BatchedInvalidations {
} }
} }
/// Records `path` as newly-created so its watch can be (re-)established. No-op in recursive /// Records `index` as newly-created so its watch can be (re-)established. No-op in recursive
/// watching mode. /// watching mode.
fn mark_new_path(&mut self, path: &Path) { fn mark_new_path(&mut self, index: usize) {
if let Some(new_paths) = &mut self.new_paths { if let Some(new_paths) = &mut self.new_paths {
new_paths.insert(Box::from(path)); new_paths.insert(index);
} }
} }
/// Sets the `flags` for `path`. Returns the index that was modified. /// Sets the `flags` for `path`. Returns the index that was modified.
fn mark(&mut self, path: Box<Path>, flags: InvalidationFlags) -> usize { fn mark(&mut self, path: Cow<'_, Path>, flags: InvalidationFlags) -> usize {
match self.paths.entry(path) { match self.paths.raw_entry_mut_v1().from_key(path.as_ref()) {
Entry::Occupied(mut entry) => { RawEntryMut::Occupied(mut entry) => {
*entry.get_mut() |= flags; *entry.get_mut() |= flags;
entry.index() entry.index()
} }
Entry::Vacant(entry) => { RawEntryMut::Vacant(entry) => {
let index = entry.index(); let index = entry.index();
entry.insert(flags); entry.insert(path.into_owned().into_boxed_path(), flags);
index index
} }
} }
...@@ -863,7 +864,7 @@ impl BatchedInvalidations { ...@@ -863,7 +864,7 @@ impl BatchedInvalidations {
fn mark_parent_dir(&mut self, path: &Path) { fn mark_parent_dir(&mut self, path: &Path) {
if let Some(parent) = path.parent() { if let Some(parent) = path.parent() {
self.mark(Box::from(parent), InvalidationFlags::PATH_DIR); self.mark(Cow::Borrowed(parent), InvalidationFlags::PATH_DIR);
} }
} }
...@@ -871,7 +872,10 @@ impl BatchedInvalidations { ...@@ -871,7 +872,10 @@ impl BatchedInvalidations {
/// must have their watches (re-)established before [`Self::execute`] is called (see the note /// must have their watches (re-)established before [`Self::execute`] is called (see the note
/// there). Always empty in recursive mode. /// there). Always empty in recursive mode.
fn new_paths(&self) -> impl Iterator<Item = &Path> { fn new_paths(&self) -> impl Iterator<Item = &Path> {
self.new_paths.iter().flatten().map(|path| &**path) self.new_paths
.iter()
.flatten()
.map(|&index| self.paths.get_index(index).unwrap().0.as_ref())
} }
/// Updates the batch to contain updated paths from the given event. Does not perform any /// Updates the batch to contain updated paths from the given event. Does not perform any
...@@ -885,33 +889,32 @@ impl BatchedInvalidations { ...@@ -885,33 +889,32 @@ impl BatchedInvalidations {
match event.kind { match event.kind {
EventKind::Modify(ModifyKind::Data(_)) => { EventKind::Modify(ModifyKind::Data(_)) => {
for path in paths { for path in paths {
last_updated_index = last_updated_index = Some(self.mark(Cow::Owned(path), InvalidationFlags::PATH));
Some(self.mark(path.into_boxed_path(), InvalidationFlags::PATH));
} }
} }
// Some backends (fsevents, polling) can report metadata events for file content changes // Some backends (fsevents, polling) can report metadata events for file content changes
EventKind::Modify(ModifyKind::Metadata(kind)) if self.is_content_change(kind) => { EventKind::Modify(ModifyKind::Metadata(kind)) if self.is_content_change(kind) => {
for path in paths { for path in paths {
last_updated_index = last_updated_index = Some(self.mark(Cow::Owned(path), InvalidationFlags::PATH));
Some(self.mark(path.into_boxed_path(), InvalidationFlags::PATH));
} }
} }
EventKind::Create(_) => { EventKind::Create(_) => {
for path in paths { for path in paths {
self.mark_parent_dir(&path); self.mark_parent_dir(&path);
self.mark_new_path(&path); let index = self.mark(
last_updated_index = Some(self.mark( Cow::Owned(path),
path.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN InvalidationFlags::PATH_AND_CHILDREN
| InvalidationFlags::PATH_AND_CHILDREN_DIR, | InvalidationFlags::PATH_AND_CHILDREN_DIR,
)); );
self.mark_new_path(index);
last_updated_index = Some(index);
} }
} }
EventKind::Remove(_) => { EventKind::Remove(_) => {
for path in paths { for path in paths {
self.mark_parent_dir(&path); self.mark_parent_dir(&path);
last_updated_index = Some(self.mark( last_updated_index = Some(self.mark(
path.into_boxed_path(), Cow::Owned(path),
InvalidationFlags::PATH_AND_CHILDREN InvalidationFlags::PATH_AND_CHILDREN
| InvalidationFlags::PATH_AND_CHILDREN_DIR, | InvalidationFlags::PATH_AND_CHILDREN_DIR,
)); ));
...@@ -921,17 +924,17 @@ impl BatchedInvalidations { ...@@ -921,17 +924,17 @@ impl BatchedInvalidations {
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
let [source, destination] = <[PathBuf; 2]>::try_from(paths) let [source, destination] = <[PathBuf; 2]>::try_from(paths)
.expect("RenameMode::Both event must contain exactly two paths"); .expect("RenameMode::Both event must contain exactly two paths");
self.mark_parent_dir(&source); self.mark_parent_dir(&source);
self.mark( self.mark(Cow::Owned(source), InvalidationFlags::PATH_AND_CHILDREN);
source.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN,
);
self.mark_parent_dir(&destination); self.mark_parent_dir(&destination);
self.mark_new_path(&destination); let index = self.mark(
last_updated_index = Some(self.mark( Cow::Owned(destination),
destination.into_boxed_path(),
InvalidationFlags::PATH_AND_CHILDREN, InvalidationFlags::PATH_AND_CHILDREN,
)); );
self.mark_new_path(index);
last_updated_index = Some(index);
} }
// We expect `RenameMode::Both` to cover most of the cases we need to invalidate, // We expect `RenameMode::Both` to cover most of the cases we need to invalidate,
// but we also check other RenameModes to cover cases where notify couldn't match the // but we also check other RenameModes to cover cases where notify couldn't match the
...@@ -940,7 +943,7 @@ impl BatchedInvalidations { ...@@ -940,7 +943,7 @@ impl BatchedInvalidations {
for path in paths { for path in paths {
self.mark_parent_dir(&path); self.mark_parent_dir(&path);
last_updated_index = Some(self.mark( last_updated_index = Some(self.mark(
path.into_boxed_path(), Cow::Owned(path),
InvalidationFlags::PATH_AND_CHILDREN InvalidationFlags::PATH_AND_CHILDREN
| InvalidationFlags::PATH_AND_CHILDREN_DIR, | InvalidationFlags::PATH_AND_CHILDREN_DIR,
)); ));
...@@ -962,10 +965,10 @@ impl BatchedInvalidations { ...@@ -962,10 +965,10 @@ impl BatchedInvalidations {
fn add_error(&mut self, paths: Vec<PathBuf>, root_path: &Path) { fn add_error(&mut self, paths: Vec<PathBuf>, root_path: &Path) {
let flags = InvalidationFlags::PATH_AND_CHILDREN | InvalidationFlags::PATH_AND_CHILDREN_DIR; let flags = InvalidationFlags::PATH_AND_CHILDREN | InvalidationFlags::PATH_AND_CHILDREN_DIR;
if paths.is_empty() { if paths.is_empty() {
self.last_updated_index = Some(self.mark(Box::from(root_path), flags)); self.last_updated_index = Some(self.mark(Cow::Borrowed(root_path), flags));
} else { } else {
for path in paths { for path in paths {
self.last_updated_index = Some(self.mark(path.into_boxed_path(), flags)); self.last_updated_index = Some(self.mark(Cow::Owned(path), flags));
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment