Unverified Commit 69310a84 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: use block in place for db writes (#82380)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the PR.
- Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to understand the PR)
- When linking to a Slack thread, you might want to share details of the conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->
parent 450b07e8
pub trait ParallelScheduler: Clone + Sync + Send { pub trait ParallelScheduler: Clone + Sync + Send {
fn block_in_place<R>(&self, f: impl FnOnce() -> R + Send) -> R
where
R: Send;
fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync) fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync)
where where
T: Sync; T: Sync;
...@@ -55,6 +59,13 @@ pub trait ParallelScheduler: Clone + Sync + Send { ...@@ -55,6 +59,13 @@ pub trait ParallelScheduler: Clone + Sync + Send {
pub struct SerialScheduler; pub struct SerialScheduler;
impl ParallelScheduler for SerialScheduler { impl ParallelScheduler for SerialScheduler {
fn block_in_place<R>(&self, f: impl FnOnce() -> R + Send) -> R
where
R: Send,
{
f()
}
fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync) fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync)
where where
T: Sync, T: Sync,
......
...@@ -14,6 +14,13 @@ use crate::{ ...@@ -14,6 +14,13 @@ use crate::{
struct RayonParallelScheduler; struct RayonParallelScheduler;
impl ParallelScheduler for RayonParallelScheduler { impl ParallelScheduler for RayonParallelScheduler {
fn block_in_place<R>(&self, f: impl FnOnce() -> R + Send) -> R
where
R: Send,
{
f()
}
fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync) fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync)
where where
T: Sync, T: Sync,
......
...@@ -413,9 +413,12 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize> ...@@ -413,9 +413,12 @@ impl<K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize>
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1; let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let path = self.db_path.join(format!("{seq:08}.sst")); let path = self.db_path.join(format!("{seq:08}.sst"));
let (meta, file) = let (meta, file) = self
write_static_stored_file(entries, total_key_size, total_value_size, &path) .parallel_scheduler
.with_context(|| format!("Unable to write SST file {seq:08}.sst"))?; .block_in_place(|| {
write_static_stored_file(entries, total_key_size, total_value_size, &path)
})
.with_context(|| format!("Unable to write SST file {seq:08}.sst"))?;
#[cfg(feature = "verify_sst_content")] #[cfg(feature = "verify_sst_content")]
{ {
......
use turbo_persistence::ParallelScheduler; use turbo_persistence::ParallelScheduler;
use turbo_tasks::parallel; use turbo_tasks::{block_in_place, parallel};
#[derive(Clone, Copy, Default)] #[derive(Clone, Copy, Default)]
pub struct TurboTasksParallelScheduler; pub struct TurboTasksParallelScheduler;
impl ParallelScheduler for TurboTasksParallelScheduler { impl ParallelScheduler for TurboTasksParallelScheduler {
fn block_in_place<R>(&self, f: impl FnOnce() -> R + Send) -> R
where
R: Send,
{
block_in_place(f)
}
fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync) fn parallel_for_each<T>(&self, items: &[T], f: impl Fn(&T) + Send + Sync)
where where
T: Sync, T: Sync,
......
use std::{ use std::{
any::{Any, TypeId}, any::{Any, TypeId},
borrow::Cow,
future::Future, future::Future,
mem::replace, mem::replace,
panic, panic,
...@@ -8,20 +7,20 @@ use std::{ ...@@ -8,20 +7,20 @@ use std::{
sync::Arc, sync::Arc,
}; };
use anyhow::{Result, anyhow}; use anyhow::Result;
use auto_hash_map::AutoSet; use auto_hash_map::AutoSet;
use futures::{StreamExt, TryStreamExt}; use futures::{StreamExt, TryStreamExt};
use parking_lot::Mutex; use parking_lot::Mutex;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use tokio::task_local; use tokio::task_local;
use tracing::{Instrument, Span}; use tracing::Instrument;
use crate::{ use crate::{
self as turbo_tasks, CollectiblesSource, NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt, self as turbo_tasks, CollectiblesSource, NonLocalValue, ReadRef, ResolvedVc, TryJoinIterExt,
debug::ValueDebugFormat, debug::ValueDebugFormat,
emit, emit,
event::{Event, EventListener}, event::{Event, EventListener},
manager::turbo_tasks_future_scope, spawn,
trace::TraceRawVcs, trace::TraceRawVcs,
util::SharedError, util::SharedError,
}; };
...@@ -98,28 +97,10 @@ impl EffectInstance { ...@@ -98,28 +97,10 @@ impl EffectInstance {
listener.await; listener.await;
} }
State::NotStarted(EffectInner { future }) => { State::NotStarted(EffectInner { future }) => {
let join_handle = tokio::spawn(ApplyEffectsContext::in_current_scope( let join_handle = spawn(ApplyEffectsContext::in_current_scope(future));
turbo_tasks_future_scope(turbo_tasks::turbo_tasks(), future)
.instrument(Span::current()),
));
let result = match join_handle.await { let result = match join_handle.await {
Ok(Err(err)) => Err(SharedError::new(err)), Err(err) => Err(SharedError::new(err)),
Err(err) => { Ok(()) => Ok(()),
let any = err.into_panic();
let panic = match any.downcast::<String>() {
Ok(owned) => Some(Cow::Owned(*owned)),
Err(any) => match any.downcast::<&'static str>() {
Ok(str) => Some(Cow::Borrowed(*str)),
Err(_) => None,
},
};
Err(SharedError::new(if let Some(panic) = panic {
anyhow!("Task effect panicked: {panic}")
} else {
anyhow!("Task effect panicked")
}))
}
Ok(Ok(())) => Ok(()),
}; };
let event = { let event = {
let mut guard = self.inner.lock(); let mut guard = self.inner.lock();
......
...@@ -120,7 +120,9 @@ pub use read_ref::ReadRef; ...@@ -120,7 +120,9 @@ pub use read_ref::ReadRef;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
pub use serialization_invalidation::SerializationInvalidator; pub use serialization_invalidation::SerializationInvalidator;
pub use shrink_to_fit::ShrinkToFit; pub use shrink_to_fit::ShrinkToFit;
pub use spawn::{JoinHandle, block_for_future, spawn, spawn_blocking, spawn_thread}; pub use spawn::{
JoinHandle, block_for_future, block_in_place, spawn, spawn_blocking, spawn_thread,
};
pub use state::{State, TransientState}; pub use state::{State, TransientState};
pub use task::{SharedReference, TypedSharedReference, task_input::TaskInput}; pub use task::{SharedReference, TypedSharedReference, task_input::TaskInput};
pub use task_execution_reason::TaskExecutionReason; pub use task_execution_reason::TaskExecutionReason;
......
...@@ -8,7 +8,7 @@ use std::{ ...@@ -8,7 +8,7 @@ use std::{
use anyhow::Result; use anyhow::Result;
use futures::{FutureExt, ready}; use futures::{FutureExt, ready};
use tokio::{runtime::Handle, task::block_in_place}; use tokio::runtime::Handle;
use tracing::{Instrument, Span, info_span}; use tracing::{Instrument, Span, info_span};
use turbo_tasks_malloc::{AllocationInfo, TurboMalloc}; use turbo_tasks_malloc::{AllocationInfo, TurboMalloc};
...@@ -94,6 +94,15 @@ pub fn spawn_thread(func: impl FnOnce() + Send + 'static) { ...@@ -94,6 +94,15 @@ pub fn spawn_thread(func: impl FnOnce() + Send + 'static) {
}); });
} }
/// Tells the scheduler about blocking work happening in the current thread.
/// It will make sure to allocate extra threads for the pool.
pub fn block_in_place<R>(f: impl FnOnce() -> R + Send) -> R
where
R: Send,
{
tokio::task::block_in_place(f)
}
/// Blocking waits for a future to complete. This blocks the current thread potentially staling /// Blocking waits for a future to complete. This blocks the current thread potentially staling
/// other concurrent futures (but not other concurrent tasks). Try to avoid this method infavor of /// other concurrent futures (but not other concurrent tasks). Try to avoid this method infavor of
/// awaiting the future instead. /// awaiting the future instead.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment