Unverified Commit 1c86b8a4 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks: add scope_unbounded, a scoped execution primitive that allows more…

turbo-tasks: add scope_unbounded, a scoped execution primitive that allows more work to be discovered (#95974)

## What

Adds `scope_unbounded `, a parallel scope to turbo-tasks: jobs run on a
shared work-queue, any job may enqueue more, and the pool is drained by
the runtime worker threads plus the calling thread until empty.

## Why

The garbage collector (later in this stack) needs to process a pool of
work that *discovers more work as it runs* — collecting a task cascades
into collecting newly-parentless children. A fixed `parallel::for_each`
can't express that. `scope_self_feeding` is the general primitive for
"parallel pool where jobs spawn jobs".

## Design notes

- **Deadlock-safe on thread-limited / contended runtimes.** Helper
workers are a pure optimization: the calling thread always makes
progress on the shared queue on its own, so the pool completes even when
no worker threads are available.
- Supports growing/shrinking the set of workers pulling from the queue
based on the amount of work available.
- Supports a simple mostly lock free way to aggregate data from the
tasks
- Supports a way for tasks to abort the whole queue, which is important
for making GC interruptible

## Testing

There are a fair number of new unit tests and i have done some runs
under miri
parent 8482890a
...@@ -10083,6 +10083,7 @@ dependencies = [ ...@@ -10083,6 +10083,7 @@ dependencies = [
"either", "either",
"erased-serde", "erased-serde",
"event-listener", "event-listener",
"fixedbitset",
"futures", "futures",
"indexmap 2.13.0", "indexmap 2.13.0",
"parking_lot", "parking_lot",
......
...@@ -274,6 +274,7 @@ dhat = { version = "0.3.2" } ...@@ -274,6 +274,7 @@ dhat = { version = "0.3.2" }
dunce = "1.0.3" dunce = "1.0.3"
either = "1.15.0" either = "1.15.0"
erased-serde = "0.4.5" erased-serde = "0.4.5"
fixedbitset = "0.5.7"
flate2 = "1.0.28" flate2 = "1.0.28"
fs-err = "3.1.1" fs-err = "3.1.1"
futures = "0.3.31" futures = "0.3.31"
......
...@@ -44,7 +44,7 @@ use turbo_tasks::{ ...@@ -44,7 +44,7 @@ use turbo_tasks::{
macro_helpers::NativeFunction, macro_helpers::NativeFunction,
message_queue::{TimingEvent, TraceEvent}, message_queue::{TimingEvent, TraceEvent},
registry::get_value_type, registry::get_value_type,
scope::scope_and_block, scope_bounded::scope_bounded,
task_statistics::TaskStatisticsApi, task_statistics::TaskStatisticsApi,
trace::TraceRawVcs, trace::TraceRawVcs,
util::{IdFactoryWithReuse, good_chunk_size, into_chunks}, util::{IdFactoryWithReuse, good_chunk_size, into_chunks},
...@@ -2522,7 +2522,7 @@ impl TurboTasksBackend { ...@@ -2522,7 +2522,7 @@ impl TurboTasksBackend {
if output_dependent_tasks.len() > DEPENDENT_TASKS_DIRTY_PARALLELIZATION_THRESHOLD { if output_dependent_tasks.len() > DEPENDENT_TASKS_DIRTY_PARALLELIZATION_THRESHOLD {
let chunk_size = good_chunk_size(output_dependent_tasks.len()); let chunk_size = good_chunk_size(output_dependent_tasks.len());
let chunks = into_chunks(output_dependent_tasks.to_vec(), chunk_size); let chunks = into_chunks(output_dependent_tasks.to_vec(), chunk_size);
let _ = scope_and_block(chunks.len(), |scope| { let _ = scope_bounded(chunks.len(), |scope| {
for chunk in chunks { for chunk in chunks {
let child_ctx = ctx.child_context(); let child_ctx = ctx.child_context();
#[cfg(feature = "task_dirty_cause")] #[cfg(feature = "task_dirty_cause")]
......
...@@ -2,7 +2,7 @@ use rustc_hash::FxHashSet; ...@@ -2,7 +2,7 @@ use rustc_hash::FxHashSet;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_tasks::{ use turbo_tasks::{
TaskId, TaskId,
scope::scope_and_block, scope_bounded::scope_bounded,
util::{good_chunk_size, into_chunks}, util::{good_chunk_size, into_chunks},
}; };
...@@ -139,7 +139,7 @@ pub fn connect_children( ...@@ -139,7 +139,7 @@ pub fn connect_children(
if len >= CONNECT_CHILDREN_PARALLIZATION_THRESHOLD { if len >= CONNECT_CHILDREN_PARALLIZATION_THRESHOLD {
let new_follower_ids = new_follower_ids.into_vec(); let new_follower_ids = new_follower_ids.into_vec();
let chunk_size = good_chunk_size(len); let chunk_size = good_chunk_size(len);
let _ = scope_and_block(len.div_ceil(chunk_size), |scope| { let _ = scope_bounded(len.div_ceil(chunk_size), |scope| {
for chunk in into_chunks(new_follower_ids, chunk_size) { for chunk in into_chunks(new_follower_ids, chunk_size) {
let upper_ids = &upper_ids; let upper_ids = &upper_ids;
let child_ctx = ctx.child_context(); let child_ctx = ctx.child_context();
......
...@@ -33,6 +33,7 @@ dashmap = { workspace = true } ...@@ -33,6 +33,7 @@ dashmap = { workspace = true }
either = { workspace = true } either = { workspace = true }
erased-serde = { workspace = true } erased-serde = { workspace = true }
event-listener = "5.4.0" event-listener = "5.4.0"
fixedbitset = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
indexmap = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] }
parking_lot = { workspace = true, features = ["serde"]} parking_lot = { workspace = true, features = ["serde"]}
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
#![feature(arbitrary_self_types)] #![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
#![feature(ptr_metadata)] #![feature(ptr_metadata)]
#![feature(exclusive_wrapper)]
#![feature(sync_unsafe_cell)] #![feature(sync_unsafe_cell)]
#![feature(async_fn_traits)] #![feature(async_fn_traits)]
#![feature(impl_trait_in_assoc_type)] #![feature(impl_trait_in_assoc_type)]
...@@ -47,7 +48,8 @@ mod priority_runner; ...@@ -47,7 +48,8 @@ mod priority_runner;
mod read_options; mod read_options;
mod read_ref; mod read_ref;
pub mod registry; pub mod registry;
pub mod scope; pub mod scope_bounded;
pub mod scope_unbounded;
mod serialization_invalidation; mod serialization_invalidation;
pub mod small_duration; pub mod small_duration;
mod spawn; mod spawn;
......
...@@ -13,7 +13,7 @@ use std::{ ...@@ -13,7 +13,7 @@ use std::{
}; };
use crate::{ use crate::{
scope::scope_and_block, scope_bounded::scope_bounded,
util::{Chunk, good_chunk_size, into_chunks}, util::{Chunk, good_chunk_size, into_chunks},
}; };
...@@ -76,7 +76,7 @@ where ...@@ -76,7 +76,7 @@ where
return; return;
}; };
let f = &f; let f = &f;
let _results = scope_and_block(chunk_count, |scope| { let _results = scope_bounded(chunk_count, |scope| {
for chunk in items.chunks(chunk_size) { for chunk in items.chunks(chunk_size) {
scope.spawn(move || { scope.spawn(move || {
for item in chunk { for item in chunk {
...@@ -102,7 +102,7 @@ where ...@@ -102,7 +102,7 @@ where
return; return;
}; };
let f = &f; let f = &f;
let _results = scope_and_block(chunk_count, |scope| { let _results = scope_bounded(chunk_count, |scope| {
for chunk in into_chunks(items, chunk_size) { for chunk in into_chunks(items, chunk_size) {
scope.spawn(move || { scope.spawn(move || {
// SAFETY: Even when f() panics we drop all items in the chunk. // SAFETY: Even when f() panics we drop all items in the chunk.
...@@ -133,7 +133,7 @@ where ...@@ -133,7 +133,7 @@ where
return Ok(()); return Ok(());
}; };
let f = &f; let f = &f;
scope_and_block(chunk_count, |scope| { scope_bounded(chunk_count, |scope| {
for chunk in items.chunks(chunk_size) { for chunk in items.chunks(chunk_size) {
scope.spawn(move || { scope.spawn(move || {
for item in chunk { for item in chunk {
...@@ -165,7 +165,7 @@ where ...@@ -165,7 +165,7 @@ where
return Ok(()); return Ok(());
}; };
let f = &f; let f = &f;
scope_and_block(chunk_count, |scope| { scope_bounded(chunk_count, |scope| {
for chunk in items.chunks_mut(chunk_size) { for chunk in items.chunks_mut(chunk_size) {
scope.spawn(move || { scope.spawn(move || {
for item in chunk { for item in chunk {
...@@ -197,7 +197,7 @@ where ...@@ -197,7 +197,7 @@ where
return Ok(()); return Ok(());
}; };
let f = &f; let f = &f;
scope_and_block(chunk_count, |scope| { scope_bounded(chunk_count, |scope| {
for chunk in into_chunks(items, chunk_size) { for chunk in into_chunks(items, chunk_size) {
scope.spawn(move || { scope.spawn(move || {
for item in chunk { for item in chunk {
...@@ -227,7 +227,7 @@ where ...@@ -227,7 +227,7 @@ where
return Result::from_iter(items.iter().map(f)); return Result::from_iter(items.iter().map(f));
}; };
let f = &f; let f = &f;
scope_and_block(chunk_count, |scope| { scope_bounded(chunk_count, |scope| {
for chunk in items.chunks(chunk_size) { for chunk in items.chunks(chunk_size) {
scope.spawn(move || chunk.iter().map(f).collect::<Vec<_>>()) scope.spawn(move || chunk.iter().map(f).collect::<Vec<_>>())
} }
...@@ -253,7 +253,7 @@ where ...@@ -253,7 +253,7 @@ where
return Result::from_iter(items.into_iter().map(f)); return Result::from_iter(items.into_iter().map(f));
}; };
let f = &f; let f = &f;
scope_and_block(chunk_count, |scope| { scope_bounded(chunk_count, |scope| {
for chunk in into_chunks(items, chunk_size) { for chunk in into_chunks(items, chunk_size) {
scope.spawn(move || chunk.map(f).collect::<Vec<_>>()) scope.spawn(move || chunk.map(f).collect::<Vec<_>>())
} }
...@@ -280,7 +280,7 @@ where ...@@ -280,7 +280,7 @@ where
return Result::from_iter(into_chunks(items, len).map(f)); return Result::from_iter(into_chunks(items, len).map(f));
}; };
let f = &f; let f = &f;
scope_and_block(chunk_count, |scope| { scope_bounded(chunk_count, |scope| {
for chunk in into_chunks(items, chunk_size) { for chunk in into_chunks(items, chunk_size) {
scope.spawn(move || f(chunk)) scope.spawn(move || f(chunk))
} }
......
//! A scoped tokio spawn implementation that allow a non-'static lifetime for tasks. //! Bounded scoped parallelism: the number of tasks is known before the scope starts.
//!
//! [`scope_bounded`] takes that count up front, hands the caller a [`Scope`] to spawn each task
//! onto, and returns their results as an iterator. Tasks are closures and may borrow from the
//! enclosing scope (`'env`); the scope blocks until every one has finished, which is what makes
//! those borrows sound.
//!
//! Use [`scope_unbounded`](crate::scope_unbounded::scope_unbounded) instead when a running job can
//! discover more work, so the total isn't known up front.
use std::{ use std::{
any::Any, any::Any,
...@@ -238,8 +246,8 @@ impl<'scope, 'env: 'scope, R: Send + 'env> Drop for Scope<'scope, 'env, R> { ...@@ -238,8 +246,8 @@ impl<'scope, 'env: 'scope, R: Send + 'env> Drop for Scope<'scope, 'env, R> {
/// Be aware that although this function avoids starving other independently spawned tasks, any /// Be aware that although this function avoids starving other independently spawned tasks, any
/// other code running concurrently in the same task will be suspended during the call to /// other code running concurrently in the same task will be suspended during the call to
/// block_in_place. This can happen e.g. when using the `join!` macro. To avoid this issue, call /// block_in_place. This can happen e.g. when using the `join!` macro. To avoid this issue, call
/// `scope_and_block` in `spawn_blocking`. /// `scope_bounded` in `spawn_blocking`.
pub fn scope_and_block<'env, F, R>(number_of_tasks: usize, f: F) -> impl Iterator<Item = R> pub fn scope_bounded<'env, F, R>(number_of_tasks: usize, f: F) -> impl Iterator<Item = R>
where where
R: Send + 'env, R: Send + 'env,
F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, R>) + 'env, F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, R>) + 'env,
...@@ -266,7 +274,10 @@ where ...@@ -266,7 +274,10 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::panic::{AssertUnwindSafe, catch_unwind}; use std::{
panic::{AssertUnwindSafe, catch_unwind},
sync::atomic::AtomicUsize,
};
use super::*; use super::*;
...@@ -300,7 +311,7 @@ mod tests { ...@@ -300,7 +311,7 @@ mod tests {
let started = Instant::now(); let started = Instant::now();
let results = tokio::task::spawn_blocking(move || { let results = tokio::task::spawn_blocking(move || {
scope_and_block(JOBS, |scope| { scope_bounded(JOBS, |scope| {
for i in 0..JOBS { for i in 0..JOBS {
scope.spawn(move || i); scope.spawn(move || i);
} }
...@@ -317,7 +328,7 @@ mod tests { ...@@ -317,7 +328,7 @@ mod tests {
}); });
assert!( assert!(
elapsed < RELEASE_AFTER / 2, elapsed < RELEASE_AFTER / 2,
"scope_and_block took {elapsed:?}; it should not depend on an occupied worker thread \ "scope_bounded took {elapsed:?}; it should not depend on an occupied worker thread \
freeing up" freeing up"
); );
...@@ -331,7 +342,7 @@ mod tests { ...@@ -331,7 +342,7 @@ mod tests {
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn test_scope_current_thread_runtime() { async fn test_scope_current_thread_runtime() {
let results = tokio::task::spawn_blocking(|| { let results = tokio::task::spawn_blocking(|| {
scope_and_block(16, |scope| { scope_bounded(16, |scope| {
for i in 0..16 { for i in 0..16 {
scope.spawn(move || i); scope.spawn(move || i);
} }
...@@ -354,7 +365,7 @@ mod tests { ...@@ -354,7 +365,7 @@ mod tests {
const PER_JOB: Duration = Duration::from_millis(50); const PER_JOB: Duration = Duration::from_millis(50);
let started = Instant::now(); let started = Instant::now();
let results = tokio::task::spawn_blocking(|| { let results = tokio::task::spawn_blocking(|| {
scope_and_block(JOBS, |scope| { scope_bounded(JOBS, |scope| {
for i in 0..JOBS { for i in 0..JOBS {
scope.spawn(move || { scope.spawn(move || {
thread::sleep(PER_JOB); thread::sleep(PER_JOB);
...@@ -372,13 +383,13 @@ mod tests { ...@@ -372,13 +383,13 @@ mod tests {
// so a slow machine won't make this flaky. // so a slow machine won't make this flaky.
assert!( assert!(
elapsed < (JOBS as u32 * PER_JOB) / 2, elapsed < (JOBS as u32 * PER_JOB) / 2,
"scope_and_block took {elapsed:?}; expected parallel speedup across worker threads" "scope_bounded took {elapsed:?}; expected parallel speedup across worker threads"
); );
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_scope() { async fn test_scope() {
let results = scope_and_block(1000, |scope| { let results = scope_bounded(1000, |scope| {
for i in 0..1000 { for i in 0..1000 {
scope.spawn(move || i); scope.spawn(move || i);
} }
...@@ -392,7 +403,7 @@ mod tests { ...@@ -392,7 +403,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_empty_scope() { async fn test_empty_scope() {
let results = scope_and_block(0, |scope| { let results = scope_bounded(0, |scope| {
if false { if false {
scope.spawn(|| 42); scope.spawn(|| 42);
} }
...@@ -402,7 +413,7 @@ mod tests { ...@@ -402,7 +413,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_single_task() { async fn test_single_task() {
let results = scope_and_block(1, |scope| { let results = scope_bounded(1, |scope| {
scope.spawn(|| 42); scope.spawn(|| 42);
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
...@@ -411,7 +422,7 @@ mod tests { ...@@ -411,7 +422,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_task_finish_before_scope() { async fn test_task_finish_before_scope() {
let results = scope_and_block(1, |scope| { let results = scope_bounded(1, |scope| {
scope.spawn(|| 42); scope.spawn(|| 42);
thread::sleep(std::time::Duration::from_millis(100)); thread::sleep(std::time::Duration::from_millis(100));
}) })
...@@ -421,7 +432,7 @@ mod tests { ...@@ -421,7 +432,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_task_finish_after_scope() { async fn test_task_finish_after_scope() {
let results = scope_and_block(1, |scope| { let results = scope_bounded(1, |scope| {
scope.spawn(|| { scope.spawn(|| {
thread::sleep(std::time::Duration::from_millis(100)); thread::sleep(std::time::Duration::from_millis(100));
42 42
...@@ -434,7 +445,7 @@ mod tests { ...@@ -434,7 +445,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_panic_in_scope_factory() { async fn test_panic_in_scope_factory() {
let result = catch_unwind(AssertUnwindSafe(|| { let result = catch_unwind(AssertUnwindSafe(|| {
let _results = scope_and_block(1000, |scope| { let _results = scope_bounded(1000, |scope| {
for i in 0..500 { for i in 0..500 {
scope.spawn(move || i); scope.spawn(move || i);
} }
...@@ -452,7 +463,7 @@ mod tests { ...@@ -452,7 +463,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_panic_in_scope_task() { async fn test_panic_in_scope_task() {
let result = catch_unwind(AssertUnwindSafe(|| { let result = catch_unwind(AssertUnwindSafe(|| {
let _results = scope_and_block(1000, |scope| { let _results = scope_bounded(1000, |scope| {
for i in 0..1000 { for i in 0..1000 {
scope.spawn(move || { scope.spawn(move || {
if i == 500 { if i == 500 {
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment