Unverified Commit ba0939fc authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-tasks-backend: track early aggregation removals as follower debt

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 00161ce1
//! Benchmarks adding and removing edges of the aggregation graph.
//!
//! - `ordered`: one parent alternates between connecting all of its children and none of them.
//! Every removal is processed after the corresponding addition.
//! - `concurrent`: many roots and parents share children and change which children they connect on
//! every iteration, so additions and removals of the same edges run concurrently and removals can
//! overtake additions.
//!
//! The graph is built once per configuration. Each iteration sets the state and reads all roots
//! strongly consistently.
//!
//! ```bash
//! cargo bench -p turbo-tasks-backend --bench mod -- aggregation_churn
//! ```
use std::time::{Duration, Instant};
use anyhow::Result;
use criterion::{BenchmarkId, Criterion, Throughput};
use turbo_tasks::{OperationVc, ResolvedVc, State, TryJoinIterExt, TurboTasks, Vc};
use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
/// Length of the chain below each child. Chains increase the aggregation numbers of the tasks.
const CHAIN: u32 = 4;
#[turbo_tasks::value(transparent)]
struct Generation(State<u32>);
#[turbo_tasks::function(operation, root)]
fn create_generation() -> Vc<Generation> {
Generation(State::new(0)).cell()
}
/// Deterministically decides if `from` connects to `to` in `generation`.
fn connected(generation: u32, salt: u32, from: u32, to: u32) -> bool {
let mut x = generation
.wrapping_mul(0x9e37_79b9)
.wrapping_add(salt.wrapping_mul(0x85eb_ca6b))
.wrapping_add(from.wrapping_mul(0xc2b2_ae35))
.wrapping_add(to.wrapping_mul(0x27d4_eb2f));
x ^= x >> 15;
x = x.wrapping_mul(0x2c1b_3c6d);
x ^= x >> 12;
x.is_multiple_of(3)
}
#[turbo_tasks::task_input]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]
struct Shape {
parents: u32,
children: u32,
/// When `false`, every parent connects all children in even and none in odd generations.
concurrent: bool,
}
#[turbo_tasks::function(operation, root)]
async fn root_operation(
generation: ResolvedVc<Generation>,
shape: Shape,
root: u32,
) -> Result<Vc<()>> {
let current = *generation.await?.get();
(0..shape.parents)
.filter(|&parent_index| !shape.concurrent || connected(current, 0, root, parent_index))
.map(|parent_index| parent(*generation, shape, parent_index))
.map(|parent| async move { parent.await })
.try_join()
.await?;
Ok(Vc::cell(()))
}
#[turbo_tasks::function]
async fn parent(generation: Vc<Generation>, shape: Shape, index: u32) -> Result<Vc<()>> {
let current = *generation.await?.get();
(0..shape.children)
.filter(|&child_index| {
if shape.concurrent {
connected(current, 1, index, child_index)
} else {
current.is_multiple_of(2)
}
})
.map(|child_index| child(child_index, CHAIN))
.map(|child| async move { child.await })
.try_join()
.await?;
Ok(Vc::cell(()))
}
#[turbo_tasks::function]
async fn child(index: u32, depth: u32) -> Result<Vc<()>> {
if depth > 0 {
child(index, depth - 1).await?;
}
Ok(Vc::cell(()))
}
async fn set_and_read(
generation_op: OperationVc<Generation>,
shape: Shape,
roots: u32,
value: u32,
) -> Result<Duration> {
let generation = generation_op.resolve().strongly_consistent().await?;
let state = generation_op.read_strongly_consistent().await?;
let start = Instant::now();
state.set(value);
(0..roots)
.map(|root| async move {
root_operation(generation, shape, root)
.read_strongly_consistent()
.await
})
.try_join()
.await?;
Ok(start.elapsed())
}
pub fn aggregation_churn(c: &mut Criterion) {
let mut group = c.benchmark_group("turbo_tasks_backend_aggregation_churn");
group.sample_size(20);
group.measurement_time(Duration::from_secs(10));
let configs = [
(
"ordered",
1,
Shape {
parents: 1,
children: 1_000,
concurrent: false,
},
),
(
"concurrent",
8,
Shape {
parents: 64,
children: 256,
concurrent: true,
},
),
];
for (name, roots, shape) in configs {
group.throughput(Throughput::Elements(shape.children as u64));
group.bench_with_input(
BenchmarkId::new(name, format_args!("{} x {}", shape.parents, shape.children)),
&shape,
|b, &shape| {
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
// `TurboTasks::new` needs the runtime context.
let tt = rt.block_on(async {
TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
storage_mode: None,
..Default::default()
},
noop_backing_storage(),
))
});
// Build the graph once (not timed).
let generation_op = rt
.block_on(tt.run_once(async move {
let generation_op = create_generation();
set_and_read(generation_op, shape, roots, 0).await?;
Ok(generation_op)
}))
.unwrap();
let mut next_value = 1u32;
b.iter_custom(|iters| {
let mut total = Duration::ZERO;
for _ in 0..iters {
let value = next_value;
next_value = next_value.wrapping_add(1);
total += rt
.block_on(tt.run_once(async move {
set_and_read(generation_op, shape, roots, value).await
}))
.unwrap();
}
total
});
rt.block_on(async { tt.stop_and_wait().await });
},
);
}
group.finish();
}
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
use criterion::{Criterion, criterion_group, criterion_main}; use criterion::{Criterion, criterion_group, criterion_main};
pub(crate) mod aggregation_churn;
pub(crate) mod batch_lookup; pub(crate) mod batch_lookup;
pub(crate) mod cell_fan_out; pub(crate) mod cell_fan_out;
pub(crate) mod collectibles; pub(crate) mod collectibles;
...@@ -17,6 +18,6 @@ criterion_group!( ...@@ -17,6 +18,6 @@ criterion_group!(
config = Criterion::default(); config = Criterion::default();
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc, targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc,
cell_fan_out::cell_fan_out, state_fan_out::state_fan_out, collectibles::collectibles, cell_fan_out::cell_fan_out, state_fan_out::state_fan_out, collectibles::collectibles,
batch_lookup::batch_lookup batch_lookup::batch_lookup, aggregation_churn::aggregation_churn
); );
criterion_main!(turbo_tasks_backend_stress); criterion_main!(turbo_tasks_backend_stress);
...@@ -16,7 +16,8 @@ libfuzzer-sys = "0.4.9" ...@@ -16,7 +16,8 @@ libfuzzer-sys = "0.4.9"
serde = { workspace = true } serde = { workspace = true }
tokio = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["full"] }
turbo-tasks = { workspace = true, features = ["non_operation_vc_strongly_consistent"] } turbo-tasks = { workspace = true, features = ["non_operation_vc_strongly_consistent"] }
turbo-tasks-backend = { workspace = true } # Verify the aggregation graph and its edges (e.g. that no follower debt is left) on stop.
turbo-tasks-backend = { workspace = true, features = ["verify_aggregation_graph"] }
turbo-tasks-malloc = { workspace = true , features = ["custom_allocator"]} turbo-tasks-malloc = { workspace = true , features = ["custom_allocator"]}
[[bin]] [[bin]]
......
...@@ -3,7 +3,7 @@ use std::sync::{Arc, LazyLock}; ...@@ -3,7 +3,7 @@ use std::sync::{Arc, LazyLock};
use anyhow::Result; use anyhow::Result;
use arbitrary::Arbitrary; use arbitrary::Arbitrary;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use turbo_tasks::{self, State, TurboTasks, Vc}; use turbo_tasks::{self, State, TryJoinIterExt, TurboTasks, Vc};
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
#[turbo_tasks::task_input] #[turbo_tasks::task_input]
...@@ -58,6 +58,9 @@ impl<'a> Iterator for Iter<'a> { ...@@ -58,6 +58,9 @@ impl<'a> Iterator for Iter<'a> {
} }
} }
/// The first tasks of the spec are run as concurrent roots.
const MAX_CONCURRENT_ROOTS: usize = 3;
static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| { static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
tokio::runtime::Builder::new_multi_thread() tokio::runtime::Builder::new_multi_thread()
.enable_all() .enable_all()
...@@ -124,16 +127,31 @@ fn actual_operation(spec: Arc<Vec<TaskSpec>>, iterations: usize) { ...@@ -124,16 +127,31 @@ fn actual_operation(spec: Arc<Vec<TaskSpec>>, iterations: usize) {
}, },
turbo_tasks_backend::noop_backing_storage(), turbo_tasks_backend::noop_backing_storage(),
)); ));
let roots = spec.len().min(MAX_CONCURRENT_ROOTS) as u16;
for i in 0..iterations { for i in 0..iterations {
let spec = spec.clone(); // Run multiple roots concurrently. They share subgraphs, so edges of the
tt.run(async move { // aggregation graph are added and removed concurrently.
let it = *create_state().to_resolved().await?; (0..roots)
it.await?.set(i); .map(|root| {
let task = run_task(spec.clone(), it, 0); let spec = spec.clone();
task.strongly_consistent().await?; let tt = &tt;
Ok(()) async move {
}) anyhow::Ok(
.await?; tt.run(async move {
// Top-level tasks must only use strongly consistent reads.
let it_op = create_state();
let it = it_op.resolve().strongly_consistent().await?;
it_op.read_strongly_consistent().await?.set(i);
let task = run_task(spec.clone(), *it, root);
task.strongly_consistent().await?;
Ok(())
})
.await?,
)
}
})
.try_join()
.await?;
} }
tt.stop_and_wait().await; tt.stop_and_wait().await;
drop(tt); drop(tt);
...@@ -142,7 +160,7 @@ fn actual_operation(spec: Arc<Vec<TaskSpec>>, iterations: usize) { ...@@ -142,7 +160,7 @@ fn actual_operation(spec: Arc<Vec<TaskSpec>>, iterations: usize) {
.unwrap(); .unwrap();
} }
#[turbo_tasks::function] #[turbo_tasks::function(operation, root)]
fn create_state() -> Vc<Iteration> { fn create_state() -> Vc<Iteration> {
Vc::cell(State::new(0)) Vc::cell(State::new(0))
} }
......
...@@ -1684,6 +1684,8 @@ impl TurboTasksBackend { ...@@ -1684,6 +1684,8 @@ impl TurboTasksBackend {
{ {
self.is_idle.store(false, Ordering::Release); self.is_idle.store(false, Ordering::Release);
self.verify_aggregation_graph(turbo_tasks, false); self.verify_aggregation_graph(turbo_tasks, false);
// Storage is still alive here and no new aggregation updates can start.
self.verify_aggregation_edges();
} }
// eagerly drop the task cache before persisting // eagerly drop the task cache before persisting
self.storage.drop_task_cache(); self.storage.drop_task_cache();
...@@ -3598,6 +3600,30 @@ impl TurboTasksBackend { ...@@ -3598,6 +3600,30 @@ impl TurboTasksBackend {
} }
} }
/// Verifies the aggregation edges of all resident tasks (see
/// [`operation::verify_aggregation_edges`]). Must only be called when no aggregation updates
/// run.
#[cfg(any(test, feature = "verify_aggregation_graph"))]
pub(crate) fn verify_aggregation_edges(&self) {
if std::env::var("TURBO_ENGINE_VERIFY_GRAPH").ok().as_deref() == Some("0") {
return;
}
let violations =
operation::verify_aggregation_edges(&self.storage.aggregation_edges_for_verification());
if !violations.is_empty() {
panic!(
"The aggregation edges are inconsistent ({} violations):\n{}",
violations.len(),
violations
.iter()
.take(20)
.map(String::as_str)
.collect::<Vec<_>>()
.join("\n")
);
}
}
#[cfg(feature = "verify_aggregation_graph")] #[cfg(feature = "verify_aggregation_graph")]
fn verify_aggregation_graph(&self, turbo_tasks: &TurboTasks<TurboTasksBackend>, idle: bool) { fn verify_aggregation_graph(&self, turbo_tasks: &TurboTasks<TurboTasksBackend>, idle: bool) {
if env::var("TURBO_ENGINE_VERIFY_GRAPH").ok().as_deref() == Some("0") { if env::var("TURBO_ENGINE_VERIFY_GRAPH").ok().as_deref() == Some("0") {
......
This source diff could not be displayed because it is too large. You can view the blob instead.
...@@ -202,7 +202,6 @@ fn cleanup_old_edges_inner( ...@@ -202,7 +202,6 @@ fn cleanup_old_edges_inner(
queue.push(AggregationUpdateJob::InnerOfUpperLostFollowers { queue.push(AggregationUpdateJob::InnerOfUpperLostFollowers {
upper_id: task_id, upper_id: task_id,
lost_follower_ids: children, lost_follower_ids: children,
retry: 0,
}); });
} else { } else {
let upper_ids = get_uppers(&task); let upper_ids = get_uppers(&task);
......
mod aggregation_edge;
mod aggregation_update; mod aggregation_update;
mod cleanup_old_edges; mod cleanup_old_edges;
mod connect_child; mod connect_child;
...@@ -24,6 +25,8 @@ use turbo_tasks::{ ...@@ -24,6 +25,8 @@ use turbo_tasks::{
macro_helpers::NativeFunction, macro_helpers::NativeFunction,
}; };
#[cfg(any(test, feature = "verify_aggregation_graph"))]
pub use self::aggregation_edge::{TaskAggregationEdges, verify_aggregation_edges};
pub use self::aggregation_update::{AggregationUpdateQueue, ComputeDirtyAndCleanUpdate}; pub use self::aggregation_update::{AggregationUpdateQueue, ComputeDirtyAndCleanUpdate};
use crate::{ use crate::{
backend::{ backend::{
...@@ -160,6 +163,20 @@ impl<'e> ExecuteContext<'e> { ...@@ -160,6 +163,20 @@ impl<'e> ExecuteContext<'e> {
} }
} }
/// Like [`ExecuteContext::new`], but reports tasks that may have become collectible to
/// `gc_collectible`, like the context of the garbage collector does.
#[cfg(test)]
pub(super) fn new_with_gc_collectible_for_testing(
backend: &'e TurboTasksBackend,
turbo_tasks: &'e TurboTasks<TurboTasksBackend>,
gc_collectible: &'e dyn Fn(TaskId),
) -> Self {
Self {
gc_collectible: Some(gc_collectible),
..Self::new(backend, turbo_tasks)
}
}
/// Constructs a context that does NOT take an operation guard, for use by the garbage /// Constructs a context that does NOT take an operation guard, for use by the garbage
/// collector while it holds the coordinator's exclusion phase. /// collector while it holds the coordinator's exclusion phase.
/// ///
......
...@@ -662,6 +662,22 @@ impl Storage { ...@@ -662,6 +662,22 @@ impl Storage {
per_shard.into_iter().flatten() per_shard.into_iter().flatten()
} }
/// Captures the aggregation edges of all resident tasks, for verification of the aggregation
/// graph. Only meaningful when no aggregation updates run concurrently.
#[cfg(any(test, feature = "verify_aggregation_graph"))]
pub fn aggregation_edges_for_verification(
&self,
) -> Vec<crate::backend::operation::TaskAggregationEdges> {
let mut result = Vec::new();
for shard in self.map.shards() {
let shard = shard.read();
for (task_id, task) in shard.iter() {
result.extend(task.aggregation_edges_for_verification(*task_id));
}
}
result
}
pub fn access_pair_mut( pub fn access_pair_mut(
&self, &self,
key1: TaskId, key1: TaskId,
......
...@@ -252,8 +252,12 @@ struct TaskStorageSchema { ...@@ -252,8 +252,12 @@ struct TaskStorageSchema {
children: AutoSet<TaskId, 6>, children: AutoSet<TaskId, 6>,
/// Follower nodes in the aggregation tree (reference counted). /// Follower nodes in the aggregation tree (reference counted).
///
/// Signed: a negative count is "debt", recorded when the removal of a follower edge overtakes
/// its addition. Only counts `> 0` are real edges. See the module docs of
/// `operation::aggregation_update` for details.
#[field(storage = "counter_map", category = "meta", filter_transient)] #[field(storage = "counter_map", category = "meta", filter_transient)]
followers: CounterMap<TaskId, u32, 3>, followers: CounterMap<TaskId, i32, 3>,
// ========================================================================= // =========================================================================
// DEPENDENCIES (data) // DEPENDENCIES (data)
...@@ -812,6 +816,30 @@ impl TaskStorage { ...@@ -812,6 +816,30 @@ impl TaskStorage {
}); });
} }
/// Captures the aggregation edges of this task for verification of the aggregation graph.
/// Returns `None` when the task has no aggregation edges.
#[cfg(any(test, feature = "verify_aggregation_graph"))]
pub fn aggregation_edges_for_verification(
&self,
task_id: TaskId,
) -> Option<crate::backend::operation::TaskAggregationEdges> {
let upper: Vec<_> = self.upper().iter().map(|(&id, &c)| (id, c)).collect();
let followers: Vec<_> = self
.followers()
.into_iter()
.flat_map(|f| f.iter().map(|(&id, &c)| (id, c)))
.collect();
if upper.is_empty() && followers.is_empty() {
return None;
}
Some(crate::backend::operation::TaskAggregationEdges {
task_id,
aggregation_number: self.get_aggregation_number().map_or(0, |a| a.effective),
upper,
followers,
})
}
/// Returns counts for aggregation tree and collectibles fields. /// Returns counts for aggregation tree and collectibles fields.
/// Used for cache size statistics. /// Used for cache size statistics.
#[cfg(feature = "print_cache_item_size")] #[cfg(feature = "print_cache_item_size")]
......
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
//! Stress test for the aggregation graph: many tasks concurrently add and remove edges to shared
//! children on every change of a state, so removals of edges race with their additions.
//!
//! Run it with `--features verify_aggregation_graph` to verify the aggregation graph when the
//! backend stops: aggregated dirty containers and collectibles, and that the aggregation edges are
//! consistent (e.g. no follower debt is left over).
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, State, TryJoinIterExt, ValueToString, Vc, emit};
use turbo_tasks_testing::{Registration, register, run_once};
static REGISTRATION: Registration = register!();
/// Number of root operations.
const ROOTS: u32 = 4;
/// Number of parents shared by the roots.
const PARENTS: u32 = 16;
/// Number of children shared by the parents.
const CHILDREN: u32 = 24;
/// Length of the chain below each child. Chains increase the aggregation numbers of the tasks.
const CHAIN: u32 = 4;
fn rounds() -> u32 {
std::env::var("TURBOPACK_TEST_AGGREGATION_CHURN_ROUNDS")
.map(|rounds| rounds.parse().unwrap())
.unwrap_or(20)
}
/// Deterministically decides if `from` connects to `to` in `generation`.
fn connected(generation: u32, salt: u32, from: u32, to: u32) -> bool {
let mut x = generation
.wrapping_mul(0x9e37_79b9)
.wrapping_add(salt.wrapping_mul(0x85eb_ca6b))
.wrapping_add(from.wrapping_mul(0xc2b2_ae35))
.wrapping_add(to.wrapping_mul(0x27d4_eb2f));
x ^= x >> 15;
x = x.wrapping_mul(0x2c1b_3c6d);
x ^= x >> 12;
x.is_multiple_of(3)
}
/// The number of children all parents of `root` connect in `generation`.
fn expected_connected_children(generation: u32, root: u32) -> u32 {
(0..PARENTS)
.filter(|&parent| connected(generation, 0, root, parent))
.map(|parent| {
(0..CHILDREN)
.filter(|&child| connected(generation, 1, parent, child))
.count() as u32
})
.sum()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn aggregation_churn() {
run_once(&REGISTRATION, async || {
let generation_op = make_generation_operation();
let generation_vc = generation_op.resolve().strongly_consistent().await?;
let generation = generation_op.read_strongly_consistent().await?;
for round in 0..rounds() {
generation.state.set(round);
(0..ROOTS)
.map(|root| async move {
let connected_children = *root_operation(generation_vc, root)
.read_strongly_consistent()
.await?;
assert_eq!(
connected_children,
expected_connected_children(round, root),
"connected children of root {root} in round {round}"
);
anyhow::Ok(())
})
.try_join()
.await?;
}
anyhow::Ok(())
})
.await
.unwrap()
}
#[turbo_tasks::value]
struct Generation {
state: State<u32>,
}
#[turbo_tasks::function(operation, root)]
fn make_generation_operation() -> Vc<Generation> {
Generation {
state: State::new(0),
}
.cell()
}
#[turbo_tasks::value(transparent)]
struct Count(u32);
/// Returns the number of children connected by the parents of `root`.
#[turbo_tasks::function(operation, root)]
async fn root_operation(generation: ResolvedVc<Generation>, root: u32) -> Result<Vc<Count>> {
let current = *generation.await?.state.get();
let counts = (0..PARENTS)
.filter(|&parent_index| connected(current, 0, root, parent_index))
.map(|parent_index| parent(*generation, parent_index))
.map(|parent| async move { parent.await })
.try_join()
.await?;
Ok(Vc::cell(counts.iter().map(|count| **count).sum()))
}
/// Connects a subset of the children, depending on the generation. Returns their number.
#[turbo_tasks::function]
async fn parent(generation: Vc<Generation>, index: u32) -> Result<Vc<Count>> {
let current = *generation.await?.state.get();
let children = (0..CHILDREN)
.filter(|&child_index| connected(current, 1, index, child_index))
.map(|child_index| child(child_index, CHAIN))
.map(|child| async move { child.await })
.try_join()
.await?;
Ok(Vc::cell(children.len() as u32))
}
#[turbo_tasks::function]
async fn child(index: u32, depth: u32) -> Result<Vc<()>> {
if depth > 0 {
child(index, depth - 1).await?;
} else {
emit(ResolvedVc::upcast::<Box<dyn ValueToString>>(Thing::new(
index,
)));
}
Ok(Vc::cell(()))
}
#[turbo_tasks::value(shared)]
struct Thing(u32);
impl Thing {
fn new(v: u32) -> ResolvedVc<Self> {
Self::resolved_cell(Thing(v))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for Thing {
#[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> {
Vc::cell(self.0.to_string().into())
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment