Unverified Commit fa2ccaf5 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] only invalidate fs writes with a different content (#75936)

### What?

There are cases where two write tasks write to the same file. This can happen e. g. when two different image source files with the same content are used and end up with the same content hashed filename.

But since dependency tracking is disabled for one off builds, this results in an invalidation of the first write task when the second write task effect is executed. And an invalidation will panic in dependency-tracking-disabled mode.

This change only invalidates other write tasks when they write a different content and keeps writes with the same content intact.

This should avoid the panic as no task is invalidated.
parent f9fcae17
use std::sync::{LockResult, Mutex, MutexGuard}; use std::sync::{LockResult, Mutex, MutexGuard};
use concurrent_queue::ConcurrentQueue; use concurrent_queue::ConcurrentQueue;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::FxHashMap;
use serde::{de::Visitor, Deserialize, Serialize}; use serde::{de::Visitor, Deserialize, Serialize};
use turbo_tasks::Invalidator; use turbo_tasks::{Invalidator, ReadRef};
use crate::{FileContent, LinkContent};
#[derive(Serialize, Deserialize, PartialEq, Eq)]
pub enum WriteContent {
File(ReadRef<FileContent>),
Link(ReadRef<LinkContent>),
}
type InnerMap = FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>;
pub struct InvalidatorMap { pub struct InvalidatorMap {
queue: ConcurrentQueue<(String, Invalidator)>, queue: ConcurrentQueue<(String, Invalidator, Option<WriteContent>)>,
map: Mutex<FxHashMap<String, FxHashSet<Invalidator>>>, map: Mutex<InnerMap>,
} }
impl Default for InvalidatorMap { impl Default for InvalidatorMap {
...@@ -24,17 +34,22 @@ impl InvalidatorMap { ...@@ -24,17 +34,22 @@ impl InvalidatorMap {
Self::default() Self::default()
} }
pub fn lock(&self) -> LockResult<MutexGuard<'_, FxHashMap<String, FxHashSet<Invalidator>>>> { pub fn lock(&self) -> LockResult<MutexGuard<'_, InnerMap>> {
let mut guard = self.map.lock()?; let mut guard = self.map.lock()?;
while let Ok((key, value)) = self.queue.pop() { while let Ok((key, value, write_content)) = self.queue.pop() {
guard.entry(key).or_default().insert(value); guard.entry(key).or_default().insert(value, write_content);
} }
Ok(guard) Ok(guard)
} }
#[allow(unused_must_use)] #[allow(unused_must_use)]
pub fn insert(&self, key: String, invalidator: Invalidator) { pub fn insert(
self.queue.push((key, invalidator)); &self,
key: String,
invalidator: Invalidator,
write_content: Option<WriteContent>,
) {
self.queue.push((key, invalidator, write_content));
} }
} }
......
...@@ -49,7 +49,6 @@ use mime::Mime; ...@@ -49,7 +49,6 @@ use mime::Mime;
use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator};
use read_glob::read_glob; use read_glob::read_glob;
pub use read_glob::ReadGlobResult; pub use read_glob::ReadGlobResult;
use rustc_hash::FxHashSet;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use tokio::{ use tokio::{
...@@ -73,6 +72,7 @@ use watcher::DiskWatcher; ...@@ -73,6 +72,7 @@ use watcher::DiskWatcher;
use self::{invalidation::Write, json::UnparseableJson, mutex_map::MutexMap}; use self::{invalidation::Write, json::UnparseableJson, mutex_map::MutexMap};
use crate::{ use crate::{
attach::AttachedFileSystem, attach::AttachedFileSystem,
invalidator_map::WriteContent,
retry::{retry_blocking, retry_future}, retry::{retry_blocking, retry_future},
rope::{Rope, RopeReader}, rope::{Rope, RopeReader},
}; };
...@@ -244,9 +244,10 @@ impl DiskFileSystemInner { ...@@ -244,9 +244,10 @@ impl DiskFileSystemInner {
/// registers the path as an invalidator for the current task, /// registers the path as an invalidator for the current task,
/// has to be called within a turbo-tasks function /// has to be called within a turbo-tasks function
fn register_invalidator(&self, path: &Path) -> Result<()> { fn register_read_invalidator(&self, path: &Path) -> Result<()> {
let invalidator = turbo_tasks::get_invalidator(); let invalidator = turbo_tasks::get_invalidator();
self.invalidator_map.insert(path_to_key(path), invalidator); self.invalidator_map
.insert(path_to_key(path), invalidator, None);
#[cfg(not(any(target_os = "macos", target_os = "windows")))] #[cfg(not(any(target_os = "macos", target_os = "windows")))]
if let Some(dir) = path.parent() { if let Some(dir) = path.parent() {
self.watcher.ensure_watching(dir, self.root_path())?; self.watcher.ensure_watching(dir, self.root_path())?;
...@@ -257,20 +258,30 @@ impl DiskFileSystemInner { ...@@ -257,20 +258,30 @@ impl DiskFileSystemInner {
/// registers the path as an invalidator for the current task, /// registers the path as an invalidator for the current task,
/// has to be called within a turbo-tasks function. It removes and returns /// has to be called within a turbo-tasks function. It removes and returns
/// the current list of invalidators. /// the current list of invalidators.
fn register_sole_invalidator( fn register_write_invalidator(
&self, &self,
path: &Path, path: &Path,
invalidator: Invalidator, invalidator: Invalidator,
) -> Result<FxHashSet<Invalidator>> { write_content: WriteContent,
) -> Result<Vec<(Invalidator, Option<WriteContent>)>> {
let mut invalidator_map = self.invalidator_map.lock().unwrap(); let mut invalidator_map = self.invalidator_map.lock().unwrap();
let old_invalidators = let invalidators = invalidator_map.entry(path_to_key(path)).or_default();
invalidator_map.insert(path_to_key(path), FxHashSet::from_iter([invalidator])); let old_invalidators = invalidators
.extract_if(|i, old_write_content| {
i == &invalidator
|| old_write_content
.as_ref()
.is_none_or(|old| old != &write_content)
})
.filter(|(i, _)| i != &invalidator)
.collect::<Vec<_>>();
invalidators.insert(invalidator, Some(write_content));
drop(invalidator_map); drop(invalidator_map);
#[cfg(not(any(target_os = "macos", target_os = "windows")))] #[cfg(not(any(target_os = "macos", target_os = "windows")))]
if let Some(dir) = path.parent() { if let Some(dir) = path.parent() {
self.watcher.ensure_watching(dir, self.root_path())?; self.watcher.ensure_watching(dir, self.root_path())?;
} }
Ok(old_invalidators.unwrap_or_default()) Ok(old_invalidators)
} }
/// registers the path as an invalidator for the current task, /// registers the path as an invalidator for the current task,
...@@ -278,7 +289,7 @@ impl DiskFileSystemInner { ...@@ -278,7 +289,7 @@ impl DiskFileSystemInner {
fn register_dir_invalidator(&self, path: &Path) -> Result<()> { fn register_dir_invalidator(&self, path: &Path) -> Result<()> {
let invalidator = turbo_tasks::get_invalidator(); let invalidator = turbo_tasks::get_invalidator();
self.dir_invalidator_map self.dir_invalidator_map
.insert(path_to_key(path), invalidator); .insert(path_to_key(path), invalidator, None);
#[cfg(not(any(target_os = "macos", target_os = "windows")))] #[cfg(not(any(target_os = "macos", target_os = "windows")))]
self.watcher.ensure_watching(path, self.root_path())?; self.watcher.ensure_watching(path, self.root_path())?;
Ok(()) Ok(())
...@@ -300,7 +311,7 @@ impl DiskFileSystemInner { ...@@ -300,7 +311,7 @@ impl DiskFileSystemInner {
.into_par_iter() .into_par_iter()
.chain(dir_invalidator_map.into_par_iter()) .chain(dir_invalidator_map.into_par_iter())
.flat_map(|(_, invalidators)| invalidators.into_par_iter()); .flat_map(|(_, invalidators)| invalidators.into_par_iter());
iter.for_each(|i| { iter.for_each(|(i, _)| {
let _span = span.clone().entered(); let _span = span.clone().entered();
let _guard = handle.enter(); let _guard = handle.enter();
i.invalidate() i.invalidate()
...@@ -323,26 +334,30 @@ impl DiskFileSystemInner { ...@@ -323,26 +334,30 @@ impl DiskFileSystemInner {
.into_par_iter() .into_par_iter()
.map(move |i| (reason.clone(), i)) .map(move |i| (reason.clone(), i))
}); });
iter.for_each(|(reason, invalidator)| { iter.for_each(|(reason, (invalidator, _))| {
let _span = span.clone().entered(); let _span = span.clone().entered();
let _guard = handle.enter(); let _guard = handle.enter();
invalidator.invalidate_with_reason(reason) invalidator.invalidate_with_reason(reason)
}); });
} }
fn invalidate_from_write(&self, full_path: &Path, invalidators: FxHashSet<Invalidator>) { fn invalidate_from_write(
&self,
full_path: &Path,
invalidators: Vec<(Invalidator, Option<WriteContent>)>,
) {
if !invalidators.is_empty() { if !invalidators.is_empty() {
if let Some(path) = format_absolute_fs_path(full_path, &self.name, self.root_path()) { if let Some(path) = format_absolute_fs_path(full_path, &self.name, self.root_path()) {
if invalidators.len() == 1 { if invalidators.len() == 1 {
let invalidator = invalidators.into_iter().next().unwrap(); let (invalidator, _) = invalidators.into_iter().next().unwrap();
invalidator.invalidate_with_reason(Write { path }); invalidator.invalidate_with_reason(Write { path });
} else { } else {
invalidators.into_iter().for_each(|invalidator| { invalidators.into_iter().for_each(|(invalidator, _)| {
invalidator.invalidate_with_reason(Write { path: path.clone() }); invalidator.invalidate_with_reason(Write { path: path.clone() });
}); });
} }
} else { } else {
invalidators.into_iter().for_each(|invalidator| { invalidators.into_iter().for_each(|(invalidator, _)| {
invalidator.invalidate(); invalidator.invalidate();
}); });
} }
...@@ -498,7 +513,7 @@ impl FileSystem for DiskFileSystem { ...@@ -498,7 +513,7 @@ impl FileSystem for DiskFileSystem {
async fn read(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileContent>> { async fn read(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileContent>> {
mark_session_dependent(); mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?; let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
let content = match retry_future(|| File::from_path(full_path.clone())) let content = match retry_future(|| File::from_path(full_path.clone()))
...@@ -579,7 +594,7 @@ impl FileSystem for DiskFileSystem { ...@@ -579,7 +594,7 @@ impl FileSystem for DiskFileSystem {
async fn read_link(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<LinkContent>> { async fn read_link(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<LinkContent>> {
mark_session_dependent(); mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?; let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
let link_path = match retry_future(|| fs::read_link(&full_path)) let link_path = match retry_future(|| fs::read_link(&full_path))
...@@ -665,7 +680,7 @@ impl FileSystem for DiskFileSystem { ...@@ -665,7 +680,7 @@ impl FileSystem for DiskFileSystem {
async fn track(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<Completion>> { async fn track(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<Completion>> {
mark_session_dependent(); mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?; let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
Ok(Completion::new()) Ok(Completion::new())
} }
...@@ -683,7 +698,11 @@ impl FileSystem for DiskFileSystem { ...@@ -683,7 +698,11 @@ impl FileSystem for DiskFileSystem {
let _lock = inner.lock_path(&full_path).await; let _lock = inner.lock_path(&full_path).await;
// Track the file, so that we will rewrite it if it ever changes. // Track the file, so that we will rewrite it if it ever changes.
let old_invalidators = inner.register_sole_invalidator(&full_path, invalidator)?; let old_invalidators = inner.register_write_invalidator(
&full_path,
invalidator,
WriteContent::File(content.clone()),
)?;
// We perform an untracked comparison here, so that this write is not dependent // We perform an untracked comparison here, so that this write is not dependent
// on a read's Vc<FileContent> (and the memory it holds). Our untracked read can // on a read's Vc<FileContent> (and the memory it holds). Our untracked read can
...@@ -701,8 +720,10 @@ impl FileSystem for DiskFileSystem { ...@@ -701,8 +720,10 @@ impl FileSystem for DiskFileSystem {
if compare == FileComparison::Equal { if compare == FileComparison::Equal {
if !old_invalidators.is_empty() { if !old_invalidators.is_empty() {
let key = path_to_key(&full_path); let key = path_to_key(&full_path);
for i in old_invalidators { for (invalidator, write_content) in old_invalidators {
inner.invalidator_map.insert(key.clone(), i); inner
.invalidator_map
.insert(key.clone(), invalidator, write_content);
} }
} }
return Ok(()); return Ok(());
...@@ -806,7 +827,11 @@ impl FileSystem for DiskFileSystem { ...@@ -806,7 +827,11 @@ impl FileSystem for DiskFileSystem {
let _lock = inner.lock_path(&full_path).await; let _lock = inner.lock_path(&full_path).await;
let old_invalidators = inner.register_sole_invalidator(&full_path, invalidator)?; let old_invalidators = inner.register_write_invalidator(
&full_path,
invalidator,
WriteContent::Link(content.clone()),
)?;
// TODO(sokra) preform a untracked read here, register an invalidator and get // TODO(sokra) preform a untracked read here, register an invalidator and get
// all existing invalidators // all existing invalidators
...@@ -832,8 +857,10 @@ impl FileSystem for DiskFileSystem { ...@@ -832,8 +857,10 @@ impl FileSystem for DiskFileSystem {
if is_equal { if is_equal {
if !old_invalidators.is_empty() { if !old_invalidators.is_empty() {
let key = path_to_key(&full_path); let key = path_to_key(&full_path);
for i in old_invalidators { for (invalidator, write_content) in old_invalidators {
inner.invalidator_map.insert(key.clone(), i); inner
.invalidator_map
.insert(key.clone(), invalidator, write_content);
} }
} }
return Ok(()); return Ok(());
...@@ -919,7 +946,7 @@ impl FileSystem for DiskFileSystem { ...@@ -919,7 +946,7 @@ impl FileSystem for DiskFileSystem {
async fn metadata(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileMeta>> { async fn metadata(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<FileMeta>> {
mark_session_dependent(); mark_session_dependent();
let full_path = self.to_sys_path(fs_path).await?; let full_path = self.to_sys_path(fs_path).await?;
self.inner.register_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
let meta = retry_blocking(&full_path, |path| std::fs::metadata(path)) let meta = retry_blocking(&full_path, |path| std::fs::metadata(path))
......
...@@ -22,6 +22,7 @@ use turbo_tasks::{spawn_thread, Invalidator}; ...@@ -22,6 +22,7 @@ use turbo_tasks::{spawn_thread, Invalidator};
use crate::{ use crate::{
format_absolute_fs_path, format_absolute_fs_path,
invalidation::{WatchChange, WatchStart}, invalidation::{WatchChange, WatchStart},
invalidator_map::WriteContent,
path_to_key, DiskFileSystemInner, path_to_key, DiskFileSystemInner,
}; };
...@@ -192,7 +193,7 @@ impl DiskWatcher { ...@@ -192,7 +193,7 @@ impl DiskWatcher {
.into_par_iter() .into_par_iter()
.map(move |i| (reason.clone(), i)) .map(move |i| (reason.clone(), i))
}) })
.for_each(|(reason, invalidator)| { .for_each(|(reason, (invalidator, _))| {
let _span = span.clone().entered(); let _span = span.clone().entered();
let _guard = handle.enter(); let _guard = handle.enter();
invalidator.invalidate_with_reason(reason) invalidator.invalidate_with_reason(reason)
...@@ -202,7 +203,7 @@ impl DiskWatcher { ...@@ -202,7 +203,7 @@ impl DiskWatcher {
let _span = span.clone().entered(); let _span = span.clone().entered();
invalidators.into_par_iter().map(move |i| i) invalidators.into_par_iter().map(move |i| i)
}) })
.for_each(|invalidator| { .for_each(|(invalidator, _)| {
let _span = span.clone().entered(); let _span = span.clone().entered();
let _guard = handle.enter(); let _guard = handle.enter();
invalidator.invalidate() invalidator.invalidate()
...@@ -459,7 +460,7 @@ fn invalidate( ...@@ -459,7 +460,7 @@ fn invalidate(
fn invalidate_path( fn invalidate_path(
inner: &DiskFileSystemInner, inner: &DiskFileSystemInner,
report_invalidation_reason: bool, report_invalidation_reason: bool,
invalidator_map: &mut FxHashMap<String, FxHashSet<Invalidator>>, invalidator_map: &mut FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>,
paths: impl Iterator<Item = PathBuf>, paths: impl Iterator<Item = PathBuf>,
) { ) {
for path in paths { for path in paths {
...@@ -467,7 +468,7 @@ fn invalidate_path( ...@@ -467,7 +468,7 @@ fn invalidate_path(
if let Some(invalidators) = invalidator_map.remove(&key) { if let Some(invalidators) = invalidator_map.remove(&key) {
invalidators invalidators
.into_iter() .into_iter()
.for_each(|i| invalidate(inner, report_invalidation_reason, &path, i)); .for_each(|(i, _)| invalidate(inner, report_invalidation_reason, &path, i));
} }
} }
} }
...@@ -475,7 +476,7 @@ fn invalidate_path( ...@@ -475,7 +476,7 @@ fn invalidate_path(
fn invalidate_path_and_children_execute( fn invalidate_path_and_children_execute(
inner: &DiskFileSystemInner, inner: &DiskFileSystemInner,
report_invalidation_reason: bool, report_invalidation_reason: bool,
invalidator_map: &mut FxHashMap<String, FxHashSet<Invalidator>>, invalidator_map: &mut FxHashMap<String, FxHashMap<Invalidator, Option<WriteContent>>>,
paths: impl Iterator<Item = PathBuf>, paths: impl Iterator<Item = PathBuf>,
) { ) {
for path in paths { for path in paths {
...@@ -483,7 +484,7 @@ fn invalidate_path_and_children_execute( ...@@ -483,7 +484,7 @@ fn invalidate_path_and_children_execute(
for (_, invalidators) in invalidator_map.extract_if(|key, _| key.starts_with(&path_key)) { for (_, invalidators) in invalidator_map.extract_if(|key, _| key.starts_with(&path_key)) {
invalidators invalidators
.into_iter() .into_iter()
.for_each(|i| invalidate(inner, report_invalidation_reason, &path, i)); .for_each(|(i, _)| invalidate(inner, report_invalidation_reason, &path, i));
} }
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment