Unverified Commit fa2ccaf5 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] only invalidate fs writes with a different content (#75936)

### What?

There are cases where two write tasks write to the same file. This can happen e. g. when two different image source files with the same content are used and end up with the same content hashed filename.

But since dependency tracking is disabled for one off builds, this results in an invalidation of the first write task when the second write task effect is executed. And an invalidation will panic in dependency-tracking-disabled mode.

This change only invalidates other write tasks when they write a different content and keeps writes with the same content intact.

This should avoid the panic as no task is invalidated.
parent f9fcae17
use std::sync::{LockResult, Mutex, MutexGuard};
use concurrent_queue::ConcurrentQueue;
use rustc_hash::{FxHashMap, FxHashSet};
use rustc_hash::FxHashMap;
use serde::{de::Visitor, Deserialize, Serialize};
use turbo_tasks::Invalidator;
use turbo_tasks::{Invalidator, ReadRef};
use crate::{FileContent, LinkContent};
#[derive(Serialize, Deserialize, PartialEq, Eq)]
pub enum WriteContent {
File(ReadRef<FileContent>),
Link(ReadRef<LinkContent>),
}
type InnerMap = FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>;
pub struct InvalidatorMap {
queue: ConcurrentQueue<(String, Invalidator)>,
map: Mutex<FxHashMap<String, FxHashSet<Invalidator>>>,
queue: ConcurrentQueue<(String, Invalidator, Option<WriteContent>)>,
map: Mutex<InnerMap>,
}
impl Default for InvalidatorMap {
......@@ -24,17 +34,22 @@ impl InvalidatorMap {
Self::default()
}
pub fn lock(&self) -> LockResult<MutexGuard<'_, FxHashMap<String, FxHashSet<Invalidator>>>> {
pub fn lock(&self) -> LockResult<MutexGuard<'_, InnerMap>> {
let mut guard = self.map.lock()?;
while let Ok((key, value)) = self.queue.pop() {
guard.entry(key).or_default().insert(value);
while let Ok((key, value, write_content)) = self.queue.pop() {
guard.entry(key).or_default().insert(value, write_content);
}
Ok(guard)
}
#[allow(unused_must_use)]
pub fn insert(&self, key: String, invalidator: Invalidator) {
self.queue.push((key, invalidator));
pub fn insert(
&self,
key: String,
invalidator: Invalidator,
write_content: Option<WriteContent>,
) {
self.queue.push((key, invalidator, write_content));
}
}
......
......@@ -49,7 +49,6 @@ use mime::Mime;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use read_glob::read_glob;
pub use read_glob::ReadGlobResult;
use rustc_hash::FxHashSet;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::{
......@@ -73,6 +72,7 @@ use watcher::DiskWatcher;
use self::{invalidation::Write, json::UnparseableJson, mutex_map::MutexMap};
use crate::{
attach::AttachedFileSystem,
invalidator_map::WriteContent,
retry::{retry_blocking, retry_future},
rope::{Rope, RopeReader},
};
......@@ -244,9 +244,10 @@ impl DiskFileSystemInner {
/// registers the path as an invalidator for the current task,
/// has to be called within a turbo-tasks function
fn register_invalidator(&self, path: &Path) -> Result<()> {
fn register_read_invalidator(&self, path: &Path) -> Result<()> {
let invalidator = turbo_tasks::get_invalidator();
self.invalidator_map.insert(path_to_key(path), invalidator);
self.invalidator_map
.insert(path_to_key(path), invalidator, None);
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
if let Some(dir) = path.parent() {
self.watcher.ensure_watching(dir, self.root_path())?;
......@@ -257,20 +258,30 @@ impl DiskFileSystemInner {
/// registers the path as an invalidator for the current task,
/// has to be called within a turbo-tasks function. It removes and returns
/// the current list of invalidators.
fn register_sole_invalidator(
fn register_write_invalidator(
&self,
path: &Path,
invalidator: Invalidator,
) -> Result<FxHashSet<Invalidator>> {
write_content: WriteContent,
) -> Result<Vec<(Invalidator, Option<WriteContent>)>> {
let mut invalidator_map = self.invalidator_map.lock().unwrap();
let old_invalidators =
invalidator_map.insert(path_to_key(path), FxHashSet::from_iter([invalidator]));
let invalidators = invalidator_map.entry(path_to_key(path)).or_default();
let old_invalidators = invalidators
.extract_if(|i, old_write_content| {
i == &invalidator
|| old_write_content
.as_ref()
.is_none_or(|old| old != &write_content)
})
.filter(|(i, _)| i != &invalidator)
.collect::<Vec<_>>();
invalidators.insert(invalidator, Some(write_content));
drop(invalidator_map);
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
if let Some(dir) = path.parent() {
self.watcher.ensure_watching(dir, self.root_path())?;
}
Ok(old_invalidators.unwrap_or_default())
Ok(old_invalidators)
}
/// registers the path as an invalidator for the current task,
......@@ -278,7 +289,7 @@ impl DiskFileSystemInner {
fn register_dir_invalidator(&self, path: &Path) -> Result<()> {
let invalidator = turbo_tasks::get_invalidator();
self.dir_invalidator_map
.insert(path_to_key(path), invalidator);
.insert(path_to_key(path), invalidator, None);
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
self.watcher.ensure_watching(path, self.root_path())?;
Ok(())
......@@ -300,7 +311,7 @@ impl DiskFileSystemInner {
.into_par_iter()
.chain(dir_invalidator_map.into_par_iter())
.flat_map(|(_, invalidators)| invalidators.into_par_iter());
iter.for_each(|i| {
iter.for_each(|(i, _)| {
let _span = span.clone().entered();
let _guard = handle.enter();
i.invalidate()
......@@ -323,26 +334,30 @@ impl DiskFileSystemInner {
.into_par_iter()
.map(move |i| (reason.clone(), i))
});
iter.for_each(|(reason, invalidator)| {
iter.for_each(|(reason, (invalidator, _))| {
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate_with_reason(reason)
});
}
fn invalidate_from_write(&self, full_path: &Path, invalidators: FxHashSet<Invalidator>) {
fn invalidate_from_write(
&self,
full_path: &Path,
invalidators: Vec<(Invalidator, Option<WriteContent>)>,
) {
if !invalidators.is_empty() {
if let Some(path) = format_absolute_fs_path(full_path, &self.name, self.root_path()) {
if invalidators.len() == 1 {
let invalidator = invalidators.into_iter().next().unwrap();
let (invalidator, _) = invalidators.into_iter().next().unwrap();
invalidator.invalidate_with_reason(Write { path });
} else {
invalidators.into_iter().for_each(|invalidator| {
invalidators.into_iter().for_each(|(invalidator, _)| {
invalidator.invalidate_with_reason(Write { path: path.clone() });
});
}
} else {
invalidators.into_iter().for_each(|invalidator| {
invalidators.into_iter().for_each(|(invalidator, _)| {
invalidator.invalidate();
});
}
......@@ -498,7 +513,7 @@ impl FileSystem for DiskFileSystem {
async fn read(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileContent>> {
mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?;
self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await;
let content = match retry_future(|| File::from_path(full_path.clone()))
......@@ -579,7 +594,7 @@ impl FileSystem for DiskFileSystem {
async fn read_link(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<LinkContent>> {
mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?;
self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await;
let link_path = match retry_future(|| fs::read_link(&full_path))
......@@ -665,7 +680,7 @@ impl FileSystem for DiskFileSystem {
async fn track(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<Completion>> {
mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?;
self.inner.register_read_invalidator(&full_path)?;
Ok(Completion::new())
}
......@@ -683,7 +698,11 @@ impl FileSystem for DiskFileSystem {
let _lock = inner.lock_path(&full_path).await;
// Track the file, so that we will rewrite it if it ever changes.
let old_invalidators = inner.register_sole_invalidator(&full_path, invalidator)?;
let old_invalidators = inner.register_write_invalidator(
&full_path,
invalidator,
WriteContent::File(content.clone()),
)?;
// We perform an untracked comparison here, so that this write is not dependent
// on a read's Vc<FileContent> (and the memory it holds). Our untracked read can
......@@ -701,8 +720,10 @@ impl FileSystem for DiskFileSystem {
if compare == FileComparison::Equal {
if !old_invalidators.is_empty() {
let key = path_to_key(&full_path);
for i in old_invalidators {
inner.invalidator_map.insert(key.clone(), i);
for (invalidator, write_content) in old_invalidators {
inner
.invalidator_map
.insert(key.clone(), invalidator, write_content);
}
}
return Ok(());
......@@ -806,7 +827,11 @@ impl FileSystem for DiskFileSystem {
let _lock = inner.lock_path(&full_path).await;
let old_invalidators = inner.register_sole_invalidator(&full_path, invalidator)?;
let old_invalidators = inner.register_write_invalidator(
&full_path,
invalidator,
WriteContent::Link(content.clone()),
)?;
// TODO(sokra) preform a untracked read here, register an invalidator and get
// all existing invalidators
......@@ -832,8 +857,10 @@ impl FileSystem for DiskFileSystem {
if is_equal {
if !old_invalidators.is_empty() {
let key = path_to_key(&full_path);
for i in old_invalidators {
inner.invalidator_map.insert(key.clone(), i);
for (invalidator, write_content) in old_invalidators {
inner
.invalidator_map
.insert(key.clone(), invalidator, write_content);
}
}
return Ok(());
......@@ -919,7 +946,7 @@ impl FileSystem for DiskFileSystem {
async fn metadata(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileMeta>> {
mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?;
self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await;
let meta = retry_blocking(&full_path, |path| std::fs::metadata(path))
......
......@@ -22,6 +22,7 @@ use turbo_tasks::{spawn_thread, Invalidator};
use crate::{
format_absolute_fs_path,
invalidation::{WatchChange, WatchStart},
invalidator_map::WriteContent,
path_to_key, DiskFileSystemInner,
};
......@@ -192,7 +193,7 @@ impl DiskWatcher {
.into_par_iter()
.map(move |i| (reason.clone(), i))
})
.for_each(|(reason, invalidator)| {
.for_each(|(reason, (invalidator, _))| {
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate_with_reason(reason)
......@@ -202,7 +203,7 @@ impl DiskWatcher {
let _span = span.clone().entered();
invalidators.into_par_iter().map(move |i| i)
})
.for_each(|invalidator| {
.for_each(|(invalidator, _)| {
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate()
......@@ -459,7 +460,7 @@ fn invalidate(
fn invalidate_path(
inner: &DiskFileSystemInner,
report_invalidation_reason: bool,
invalidator_map: &mut FxHashMap<String, FxHashSet<Invalidator>>,
invalidator_map: &mut FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>,
paths: impl Iterator<Item = PathBuf>,
) {
for path in paths {
......@@ -467,7 +468,7 @@ fn invalidate_path(
if let Some(invalidators) = invalidator_map.remove(&key) {
invalidators
.into_iter()
.for_each(|i| invalidate(inner, report_invalidation_reason, &path, i));
.for_each(|(i, _)| invalidate(inner, report_invalidation_reason, &path, i));
}
}
}
......@@ -475,7 +476,7 @@ fn invalidate_path(
fn invalidate_path_and_children_execute(
inner: &DiskFileSystemInner,
report_invalidation_reason: bool,
invalidator_map: &mut FxHashMap<String, FxHashSet<Invalidator>>,
invalidator_map: &mut FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>,
paths: impl Iterator<Item = PathBuf>,
) {
for path in paths {
......@@ -483,7 +484,7 @@ fn invalidate_path_and_children_execute(
for (_, invalidators) in invalidator_map.extract_if(|key, _| key.starts_with(&path_key)) {
invalidators
.into_iter()
.for_each(|i| invalidate(inner, report_invalidation_reason, &path, i));
.for_each(|(i, _)| invalidate(inner, report_invalidation_reason, &path, i));
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment