Unverified Commit ef39ba42 authored by Donny/강동윤's avatar Donny/강동윤 Committed by GitHub

perf(turbopack): Spawn only one thread for one `write file` effect (#79318)

### What?

Replace `retry_future` call with `retry_blocking`.

### Why?

`tokio::fs` is a trap for this kind of usage. Each function call will spawn one thread until it reaches the thread pool limit. Our previous code could spawn at most eight threads to handle one `write file` effect.

The numbers at x-ref: https://vercel.slack.com/archives/C06PPGZ0FD3/p1747437041264999
parent 64ecb48d
......@@ -52,7 +52,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::{
fs,
io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
io::{AsyncBufReadExt, AsyncReadExt, BufReader},
sync::{RwLock, RwLockReadGuard},
};
use tracing::Instrument;
......@@ -757,7 +757,7 @@ impl FileSystem for DiskFileSystem {
}
match &*content {
FileContent::Content(file) => {
FileContent::Content(..) => {
let create_directory = compare == FileComparison::Create;
if create_directory {
if let Some(parent) = full_path.parent() {
......@@ -770,34 +770,38 @@ impl FileSystem for DiskFileSystem {
})?;
}
}
let full_path_to_write = full_path.clone();
retry_future(move || {
let full_path = full_path_to_write.clone();
async move {
let mut f = fs::File::create(&full_path).await?;
tokio::io::copy(&mut file.read(), &mut f).await?;
let content = content.clone();
retry_blocking(&full_path_to_write, move |full_path| {
use std::io::Write;
let mut f = std::fs::File::create(full_path)?;
let FileContent::Content(file) = &*content else {
unreachable!()
};
std::io::copy(&mut file.read(), &mut f)?;
#[cfg(target_family = "unix")]
f.set_permissions(file.meta.permissions.into())?;
f.flush()?;
#[cfg(feature = "write_version")]
{
let mut full_path = full_path.into_owned();
let hash = hash_xxh3_hash64(file);
let ext = full_path.extension();
let ext = if let Some(ext) = ext {
format!("{:016x}.{}", hash, ext.to_string_lossy())
} else {
format!("{:016x}", hash)
};
full_path.set_extension(ext);
let mut f = std::fs::File::create(&full_path)?;
std::io::copy(&mut file.read(), &mut f)?;
#[cfg(target_family = "unix")]
f.set_permissions(file.meta.permissions.into()).await?;
f.flush().await?;
#[cfg(feature = "write_version")]
{
let mut full_path = full_path.into_owned();
let hash = hash_xxh3_hash64(file);
let ext = full_path.extension();
let ext = if let Some(ext) = ext {
format!("{:016x}.{}", hash, ext.to_string_lossy())
} else {
format!("{:016x}", hash)
};
full_path.set_extension(ext);
let mut f = fs::File::create(&full_path).await?;
tokio::io::copy(&mut file.read(), &mut f).await?;
#[cfg(target_family = "unix")]
f.set_permissions(file.meta.permissions.into()).await?;
f.flush().await?;
}
Ok::<(), io::Error>(())
f.set_permissions(file.meta.permissions.into())?;
f.flush()?;
}
Ok::<(), io::Error>(())
})
.concurrency_limited(&inner.semaphore)
.instrument(tracing::info_span!(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment