Unverified Commit ef39ba42 authored by Donny/강동윤's avatar Donny/강동윤 Committed by GitHub

perf(turbopack): Spawn only one thread for one `write file` effect (#79318)

### What?

Replace `retry_future` call with `retry_blocking`.

### Why?

`tokio::fs` is a trap for this kind of usage. Each function call will spawn one thread until it reaches the thread pool limit. Our previous code could spawn at most eight threads to handle one `write file` effect.

The numbers at x-ref: https://vercel.slack.com/archives/C06PPGZ0FD3/p1747437041264999
parent 64ecb48d
...@@ -52,7 +52,7 @@ use serde::{Deserialize, Serialize}; ...@@ -52,7 +52,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use tokio::{ use tokio::{
fs, fs,
io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, io::{AsyncBufReadExt, AsyncReadExt, BufReader},
sync::{RwLock, RwLockReadGuard}, sync::{RwLock, RwLockReadGuard},
}; };
use tracing::Instrument; use tracing::Instrument;
...@@ -757,7 +757,7 @@ impl FileSystem for DiskFileSystem { ...@@ -757,7 +757,7 @@ impl FileSystem for DiskFileSystem {
} }
match &*content { match &*content {
FileContent::Content(file) => { FileContent::Content(..) => {
let create_directory = compare == FileComparison::Create; let create_directory = compare == FileComparison::Create;
if create_directory { if create_directory {
if let Some(parent) = full_path.parent() { if let Some(parent) = full_path.parent() {
...@@ -770,15 +770,20 @@ impl FileSystem for DiskFileSystem { ...@@ -770,15 +770,20 @@ impl FileSystem for DiskFileSystem {
})?; })?;
} }
} }
let full_path_to_write = full_path.clone(); let full_path_to_write = full_path.clone();
retry_future(move || { let content = content.clone();
let full_path = full_path_to_write.clone(); retry_blocking(&full_path_to_write, move |full_path| {
async move { use std::io::Write;
let mut f = fs::File::create(&full_path).await?;
tokio::io::copy(&mut file.read(), &mut f).await?; let mut f = std::fs::File::create(full_path)?;
let FileContent::Content(file) = &*content else {
unreachable!()
};
std::io::copy(&mut file.read(), &mut f)?;
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
f.set_permissions(file.meta.permissions.into()).await?; f.set_permissions(file.meta.permissions.into())?;
f.flush().await?; f.flush()?;
#[cfg(feature = "write_version")] #[cfg(feature = "write_version")]
{ {
let mut full_path = full_path.into_owned(); let mut full_path = full_path.into_owned();
...@@ -790,14 +795,13 @@ impl FileSystem for DiskFileSystem { ...@@ -790,14 +795,13 @@ impl FileSystem for DiskFileSystem {
format!("{:016x}", hash) format!("{:016x}", hash)
}; };
full_path.set_extension(ext); full_path.set_extension(ext);
let mut f = fs::File::create(&full_path).await?; let mut f = std::fs::File::create(&full_path)?;
tokio::io::copy(&mut file.read(), &mut f).await?; std::io::copy(&mut file.read(), &mut f)?;
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
f.set_permissions(file.meta.permissions.into()).await?; f.set_permissions(file.meta.permissions.into())?;
f.flush().await?; f.flush()?;
} }
Ok::<(), io::Error>(()) Ok::<(), io::Error>(())
}
}) })
.concurrency_limited(&inner.semaphore) .concurrency_limited(&inner.semaphore)
.instrument(tracing::info_span!( .instrument(tracing::info_span!(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment