Unverified Commit 3d0e4721 authored by Donny/강동윤's avatar Donny/강동윤 Committed by GitHub

perf(turbopack): Drop `tokio::fs` usages (#80674)

### What?

`std::fs` with manual `spawn_blocking` is better than using `tokio::fs`

### Why?

`tokio::fs` inevitably calls `spawn_blocking` internally, but it calls `spawn_blocking` way too much time

---------
Co-authored-by: 's avatargraphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <tobias.koppers@googlemail.com>
parent 3327565c
...@@ -29,7 +29,7 @@ use std::{ ...@@ -29,7 +29,7 @@ use std::{
fmt::{self, Debug, Display, Formatter}, fmt::{self, Debug, Display, Formatter},
fs::FileType, fs::FileType,
future::Future, future::Future,
io::{self, BufRead, ErrorKind}, io::{self, BufRead, BufReader, ErrorKind, Read},
mem::take, mem::take,
path::{MAIN_SEPARATOR, Path, PathBuf}, path::{MAIN_SEPARATOR, Path, PathBuf},
sync::Arc, sync::Arc,
...@@ -51,11 +51,7 @@ use read_glob::{read_glob, track_glob}; ...@@ -51,11 +51,7 @@ use read_glob::{read_glob, track_glob};
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use tokio::{ use tokio::sync::{RwLock, RwLockReadGuard};
fs,
io::{AsyncBufReadExt, AsyncReadExt, BufReader},
sync::{RwLock, RwLockReadGuard},
};
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
...@@ -72,7 +68,7 @@ use self::{invalidation::Write, json::UnparseableJson, mutex_map::MutexMap}; ...@@ -72,7 +68,7 @@ use self::{invalidation::Write, json::UnparseableJson, mutex_map::MutexMap};
use crate::{ use crate::{
attach::AttachedFileSystem, attach::AttachedFileSystem,
invalidator_map::WriteContent, invalidator_map::WriteContent,
retry::{retry_blocking, retry_future}, retry::retry_blocking,
rope::{Rope, RopeReader}, rope::{Rope, RopeReader},
}; };
...@@ -390,12 +386,12 @@ impl DiskFileSystemInner { ...@@ -390,12 +386,12 @@ impl DiskFileSystemInner {
let root_path = self.root_path().to_path_buf(); let root_path = self.root_path().to_path_buf();
// create the directory for the filesystem on disk, if it doesn't exist // create the directory for the filesystem on disk, if it doesn't exist
retry_future(|| { retry_blocking(root_path.clone(), move |path| {
let path = root_path.as_path(); let _tracing =
fs::create_dir_all(&root_path).instrument(tracing::info_span!( tracing::info_span!("create root directory", path = display(path.display()))
"create root directory", .entered();
path = display(path.display())
)) std::fs::create_dir_all(path)
}) })
.concurrency_limited(&self.semaphore) .concurrency_limited(&self.semaphore)
.await?; .await?;
...@@ -413,7 +409,7 @@ impl DiskFileSystemInner { ...@@ -413,7 +409,7 @@ impl DiskFileSystemInner {
); );
if !already_created { if !already_created {
let func = |p: &Path| std::fs::create_dir_all(p); let func = |p: &Path| std::fs::create_dir_all(p);
retry_blocking(directory, func) retry_blocking(directory.to_path_buf(), func)
.concurrency_limited(&self.semaphore) .concurrency_limited(&self.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"create directory", "create directory",
...@@ -559,7 +555,7 @@ impl FileSystem for DiskFileSystem { ...@@ -559,7 +555,7 @@ impl FileSystem for DiskFileSystem {
self.inner.register_read_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
let content = match retry_future(|| File::from_path(full_path.clone())) let content = match retry_blocking(full_path.clone(), |path: &Path| File::from_path(path))
.concurrency_limited(&self.inner.semaphore) .concurrency_limited(&self.inner.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"read file", "read file",
...@@ -586,7 +582,7 @@ impl FileSystem for DiskFileSystem { ...@@ -586,7 +582,7 @@ impl FileSystem for DiskFileSystem {
// we use the sync std function here as it's a lot faster (600%) in // we use the sync std function here as it's a lot faster (600%) in
// node-file-trace // node-file-trace
let read_dir = match retry_blocking(&full_path, |path| { let read_dir = match retry_blocking(full_path.clone(), |path| {
let _span = let _span =
tracing::info_span!("read directory", path = display(path.display())).entered(); tracing::info_span!("read directory", path = display(path.display())).entered();
std::fs::read_dir(path) std::fs::read_dir(path)
...@@ -640,7 +636,8 @@ impl FileSystem for DiskFileSystem { ...@@ -640,7 +636,8 @@ impl FileSystem for DiskFileSystem {
self.inner.register_read_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
let link_path = match retry_future(|| fs::read_link(&full_path)) let link_path =
match retry_blocking(full_path.clone(), |path: &Path| std::fs::read_link(path))
.concurrency_limited(&self.inner.semaphore) .concurrency_limited(&self.inner.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"read symlink", "read symlink",
...@@ -779,7 +776,7 @@ impl FileSystem for DiskFileSystem { ...@@ -779,7 +776,7 @@ impl FileSystem for DiskFileSystem {
let full_path_to_write = full_path.clone(); let full_path_to_write = full_path.clone();
let content = content.clone(); let content = content.clone();
retry_blocking(&full_path_to_write, move |full_path| { retry_blocking(full_path_to_write.into_owned(), move |full_path| {
use std::io::Write; use std::io::Write;
let mut f = std::fs::File::create(full_path)?; let mut f = std::fs::File::create(full_path)?;
...@@ -818,7 +815,9 @@ impl FileSystem for DiskFileSystem { ...@@ -818,7 +815,9 @@ impl FileSystem for DiskFileSystem {
.with_context(|| format!("failed to write to {}", full_path.display()))?; .with_context(|| format!("failed to write to {}", full_path.display()))?;
} }
FileContent::NotFound => { FileContent::NotFound => {
retry_blocking(&full_path, |path| std::fs::remove_file(path)) retry_blocking(full_path.clone().into_owned(), |path| {
std::fs::remove_file(path)
})
.concurrency_limited(&inner.semaphore) .concurrency_limited(&inner.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"remove file", "remove file",
...@@ -865,7 +864,9 @@ impl FileSystem for DiskFileSystem { ...@@ -865,7 +864,9 @@ impl FileSystem for DiskFileSystem {
// TODO(sokra) preform a untracked read here, register an invalidator and get // TODO(sokra) preform a untracked read here, register an invalidator and get
// all existing invalidators // all existing invalidators
let old_content = match retry_blocking(&full_path, |path| std::fs::read_link(path)) let old_content = match retry_blocking(full_path.clone().into_owned(), |path| {
std::fs::read_link(path)
})
.concurrency_limited(&inner.semaphore) .concurrency_limited(&inner.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"read symlink before write", "read symlink before write",
...@@ -916,7 +917,7 @@ impl FileSystem for DiskFileSystem { ...@@ -916,7 +917,7 @@ impl FileSystem for DiskFileSystem {
PathBuf::from(unix_to_sys(target).as_ref()) PathBuf::from(unix_to_sys(target).as_ref())
}; };
let full_path = full_path.into_owned(); let full_path = full_path.into_owned();
retry_blocking(&target_path, move |target_path| { retry_blocking(target_path, move |target_path| {
let _span = tracing::info_span!( let _span = tracing::info_span!(
"write symlink", "write symlink",
path = display(target_path.display()) path = display(target_path.display())
...@@ -944,7 +945,9 @@ impl FileSystem for DiskFileSystem { ...@@ -944,7 +945,9 @@ impl FileSystem for DiskFileSystem {
anyhow::bail!("invalid symlink target: {}", full_path.display()) anyhow::bail!("invalid symlink target: {}", full_path.display())
} }
LinkContent::NotFound => { LinkContent::NotFound => {
retry_blocking(&full_path, |path| std::fs::remove_file(path)) retry_blocking(full_path.clone().into_owned(), |path| {
std::fs::remove_file(path)
})
.concurrency_limited(&inner.semaphore) .concurrency_limited(&inner.semaphore)
.await .await
.or_else(|err| { .or_else(|err| {
...@@ -970,7 +973,7 @@ impl FileSystem for DiskFileSystem { ...@@ -970,7 +973,7 @@ impl FileSystem for DiskFileSystem {
self.inner.register_read_invalidator(&full_path)?; self.inner.register_read_invalidator(&full_path)?;
let _lock = self.inner.lock_path(&full_path).await; let _lock = self.inner.lock_path(&full_path).await;
let meta = retry_blocking(&full_path, |path| std::fs::metadata(path)) let meta = retry_blocking(full_path.clone(), |path| std::fs::metadata(path))
.concurrency_limited(&self.inner.semaphore) .concurrency_limited(&self.inner.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
"read metadata", "read metadata",
...@@ -1708,8 +1711,11 @@ impl FileContent { ...@@ -1708,8 +1711,11 @@ impl FileContent {
/// Performs a comparison of self's data against a disk file's streamed /// Performs a comparison of self's data against a disk file's streamed
/// read. /// read.
async fn streaming_compare(&self, path: &Path) -> Result<FileComparison> { async fn streaming_compare(&self, path: &Path) -> Result<FileComparison> {
let old_file = extract_disk_access(retry_future(|| fs::File::open(path)).await, path)?; let old_file = extract_disk_access(
let Some(mut old_file) = old_file else { retry_blocking(path.to_path_buf(), |path| std::fs::File::open(path)).await,
path,
)?;
let Some(old_file) = old_file else {
return Ok(match self { return Ok(match self {
FileContent::NotFound => FileComparison::Equal, FileContent::NotFound => FileComparison::Equal,
_ => FileComparison::Create, _ => FileComparison::Create,
...@@ -1720,7 +1726,14 @@ impl FileContent { ...@@ -1720,7 +1726,14 @@ impl FileContent {
return Ok(FileComparison::NotEqual); return Ok(FileComparison::NotEqual);
}; };
let old_meta = extract_disk_access(retry_future(|| old_file.metadata()).await, path)?; let old_meta = extract_disk_access(
retry_blocking(path.to_path_buf(), {
let file_for_metadata = old_file.try_clone()?;
move |_| file_for_metadata.metadata()
})
.await,
path,
)?;
let Some(old_meta) = old_meta else { let Some(old_meta) = old_meta else {
// If we failed to get meta, then the old file has been deleted between the // If we failed to get meta, then the old file has been deleted between the
// handle open. In which case, we just pretend the file never // handle open. In which case, we just pretend the file never
...@@ -1735,10 +1748,10 @@ impl FileContent { ...@@ -1735,10 +1748,10 @@ impl FileContent {
// So meta matches, and we have a file handle. Let's stream the contents to see // So meta matches, and we have a file handle. Let's stream the contents to see
// if they match. // if they match.
let mut new_contents = new_file.read(); let mut new_contents = new_file.read();
let mut old_contents = BufReader::new(&mut old_file); let mut old_contents = BufReader::new(old_file);
Ok(loop { Ok(loop {
let new_chunk = new_contents.fill_buf()?; let new_chunk = new_contents.fill_buf()?;
let Ok(old_chunk) = old_contents.fill_buf().await else { let Ok(old_chunk) = old_contents.fill_buf() else {
break FileComparison::NotEqual; break FileComparison::NotEqual;
}; };
...@@ -1793,12 +1806,12 @@ pub struct File { ...@@ -1793,12 +1806,12 @@ pub struct File {
impl File { impl File {
/// Reads a [File] from the given path /// Reads a [File] from the given path
async fn from_path(p: PathBuf) -> io::Result<Self> { fn from_path(p: &Path) -> io::Result<Self> {
let mut file = fs::File::open(p).await?; let mut file = std::fs::File::open(p)?;
let metadata = file.metadata().await?; let metadata = file.metadata()?;
let mut output = Vec::with_capacity(metadata.len() as usize); let mut output = Vec::with_capacity(metadata.len() as usize);
file.read_to_end(&mut output).await?; file.read_to_end(&mut output)?;
Ok(File { Ok(File {
meta: metadata.into(), meta: metadata.into(),
......
use std::{future::Future, io, io::ErrorKind, path::Path, thread::sleep, time::Duration}; use std::{
io::{self, ErrorKind},
use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy}; path::{Path, PathBuf},
thread::sleep,
time::Duration,
};
const MAX_RETRY_ATTEMPTS: usize = 10; const MAX_RETRY_ATTEMPTS: usize = 10;
pub(crate) async fn retry_future<'a, R, F, Fut>(func: F) -> io::Result<R> pub(crate) async fn retry_blocking<R, F>(path: PathBuf, func: F) -> io::Result<R>
where
F: FnMut() -> Fut + Unpin,
Fut: Future<Output = io::Result<R>> + 'a,
{
match FutureRetry::new(
func,
FsRetryHandler {
max_attempts: MAX_RETRY_ATTEMPTS,
},
)
.await
{
Ok((r, _attempts)) => Ok(r),
Err((err, _attempts)) => Err(err),
}
}
pub(crate) async fn retry_blocking<R, F>(path: impl AsRef<Path>, func: F) -> io::Result<R>
where where
F: Fn(&Path) -> io::Result<R> + Send + 'static, F: Fn(&Path) -> io::Result<R> + Send + 'static,
R: Send + 'static, R: Send + 'static,
{ {
let path = path.as_ref().to_owned();
turbo_tasks::spawn_blocking(move || { turbo_tasks::spawn_blocking(move || {
let mut attempt = 1; let mut attempt = 1;
...@@ -60,19 +43,3 @@ fn can_retry(err: &io::Error) -> bool { ...@@ -60,19 +43,3 @@ fn can_retry(err: &io::Error) -> bool {
fn get_retry_wait_time(attempt: usize) -> Duration { fn get_retry_wait_time(attempt: usize) -> Duration {
Duration::from_millis((attempt as u64) * 100) Duration::from_millis((attempt as u64) * 100)
} }
struct FsRetryHandler {
max_attempts: usize,
}
impl ErrorHandler<io::Error> for FsRetryHandler {
type OutError = io::Error;
fn handle(&mut self, attempt: usize, e: io::Error) -> RetryPolicy<io::Error> {
if attempt == self.max_attempts || !can_retry(&e) {
RetryPolicy::ForwardError(e)
} else {
RetryPolicy::WaitRetry(get_retry_wait_time(attempt))
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment