Unverified Commit 4b239e45 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

perf(turbo-tasks): coalesce active count updates

Co-authored-by: 's avatarJimmy Miller <6071338+jimmyhmiller@users.noreply.github.com>
parent 5b8c866c
...@@ -77,6 +77,23 @@ const AGGREGATION_UPDATE_CATEGORY: TaskDataCategory = TaskDataCategory::All; ...@@ -77,6 +77,23 @@ const AGGREGATION_UPDATE_CATEGORY: TaskDataCategory = TaskDataCategory::All;
type TaskIdVec = SmallVec<[TaskId; 4]>; type TaskIdVec = SmallVec<[TaskId; 4]>;
type TaskIdWithCountVec = SmallVec<[(TaskId, u32); 2]>; type TaskIdWithCountVec = SmallVec<[(TaskId, u32); 2]>;
#[derive(Debug, PartialEq, Eq)]
enum ActiveCountTransition {
None,
Activated,
Deactivated,
}
fn apply_active_count_delta(state: &mut ActivenessState, delta: i32) -> ActiveCountTransition {
let was_active = state.active_counter > 0;
state.active_counter += delta;
match (was_active, state.active_counter > 0) {
(false, true) => ActiveCountTransition::Activated,
(true, false) => ActiveCountTransition::Deactivated,
_ => ActiveCountTransition::None,
}
}
/// Returns true, when a node is aggregating its children and a partial subgraph. /// Returns true, when a node is aggregating its children and a partial subgraph.
pub fn is_aggregating_node(aggregation_number: u32) -> bool { pub fn is_aggregating_node(aggregation_number: u32) -> bool {
aggregation_number >= LEAF_NUMBER aggregation_number >= LEAF_NUMBER
...@@ -944,6 +961,13 @@ mod encode_jobs { ...@@ -944,6 +961,13 @@ mod encode_jobs {
pub struct AggregationUpdateQueue { pub struct AggregationUpdateQueue {
#[bincode(with = "encode_jobs")] #[bincode(with = "encode_jobs")]
jobs: VecDeque<AggregationUpdateJobItem>, jobs: VecDeque<AggregationUpdateJobItem>,
/// Pending active-counter changes, coalesced until the regular job queue is drained.
///
/// Activeness propagation is deferred maintenance: only the counter state at this drain
/// boundary is observable. Keeping these updates transient also matches the active-count jobs
/// that were previously filtered out while serializing `jobs`.
#[bincode(skip, default = "FxIndexMap::default")]
active_count_updates: FxIndexMap<TaskId, i32>,
#[bincode(with = "turbo_bincode::indexmap")] #[bincode(with = "turbo_bincode::indexmap")]
aggregation_number_updates: FxIndexMap<TaskId, AggregationNumberUpdate>, aggregation_number_updates: FxIndexMap<TaskId, AggregationNumberUpdate>,
done_aggregation_number_updates: FxHashMap<TaskId, AggregationNumberUpdate>, done_aggregation_number_updates: FxHashMap<TaskId, AggregationNumberUpdate>,
...@@ -970,6 +994,7 @@ impl AggregationUpdateQueue { ...@@ -970,6 +994,7 @@ impl AggregationUpdateQueue {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
jobs: VecDeque::with_capacity(0), jobs: VecDeque::with_capacity(0),
active_count_updates: FxIndexMap::default(),
aggregation_number_updates: FxIndexMap::default(), aggregation_number_updates: FxIndexMap::default(),
done_aggregation_number_updates: FxHashMap::default(), done_aggregation_number_updates: FxHashMap::default(),
find_and_schedule: FxRingSet::default(), find_and_schedule: FxRingSet::default(),
...@@ -986,6 +1011,7 @@ impl AggregationUpdateQueue { ...@@ -986,6 +1011,7 @@ impl AggregationUpdateQueue {
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
let Self { let Self {
jobs, jobs,
active_count_updates,
aggregation_number_updates, aggregation_number_updates,
find_and_schedule, find_and_schedule,
balance_queue, balance_queue,
...@@ -997,6 +1023,7 @@ impl AggregationUpdateQueue { ...@@ -997,6 +1023,7 @@ impl AggregationUpdateQueue {
stats: _, stats: _,
} = self; } = self;
jobs.is_empty() jobs.is_empty()
&& active_count_updates.is_empty()
&& aggregation_number_updates.is_empty() && aggregation_number_updates.is_empty()
&& find_and_schedule.is_empty() && find_and_schedule.is_empty()
&& balance_queue.is_empty() && balance_queue.is_empty()
...@@ -1057,12 +1084,43 @@ impl AggregationUpdateQueue { ...@@ -1057,12 +1084,43 @@ impl AggregationUpdateQueue {
self.balance_queue self.balance_queue
.push_back(BalanceJob::new(upper_id, task_id)); .push_back(BalanceJob::new(upper_id, task_id));
} }
AggregationUpdateJob::IncreaseActiveCount { task } => {
self.push_active_count_update(task, 1);
}
AggregationUpdateJob::IncreaseActiveCounts { task_ids } => {
for task_id in task_ids {
self.push_active_count_update(task_id, 1);
}
}
AggregationUpdateJob::DecreaseActiveCount { task } => {
self.push_active_count_update(task, -1);
}
AggregationUpdateJob::DecreaseActiveCounts { task_ids } => {
for task_id in task_ids {
self.push_active_count_update(task_id, -1);
}
}
_ => { _ => {
self.jobs.push_back(AggregationUpdateJobItem::new(job)); self.jobs.push_back(AggregationUpdateJobItem::new(job));
} }
} }
} }
fn push_active_count_update(&mut self, task_id: TaskId, delta: i32) {
match self.active_count_updates.entry(task_id) {
Entry::Occupied(mut entry) => {
*entry.get_mut() += delta;
if *entry.get() == 0 {
// Preserve the relative order of all other pending task updates.
entry.shift_remove();
}
}
Entry::Vacant(entry) => {
entry.insert(delta);
}
}
}
/// Extends the queue with multiple jobs. /// Extends the queue with multiple jobs.
pub fn extend(&mut self, jobs: impl IntoIterator<Item = AggregationUpdateJob>) { pub fn extend(&mut self, jobs: impl IntoIterator<Item = AggregationUpdateJob>) {
for job in jobs { for job in jobs {
...@@ -1485,31 +1543,23 @@ impl AggregationUpdateQueue { ...@@ -1485,31 +1543,23 @@ impl AggregationUpdateQueue {
} }
}); });
} }
AggregationUpdateJob::DecreaseActiveCount { task } => { AggregationUpdateJob::DecreaseActiveCount { .. }
self.decrease_active_count(ctx, task); | AggregationUpdateJob::DecreaseActiveCounts { .. }
} | AggregationUpdateJob::IncreaseActiveCount { .. }
AggregationUpdateJob::DecreaseActiveCounts { mut task_ids } => { | AggregationUpdateJob::IncreaseActiveCounts { .. } => {
if let Some(task_id) = task_ids.pop() { // `push` always redirects these jobs into `active_count_updates`.
self.decrease_active_count(ctx, task_id); unreachable!();
if !task_ids.is_empty() {
self.jobs.push_front(AggregationUpdateJobItem::new(
AggregationUpdateJob::DecreaseActiveCounts { task_ids },
));
}
}
}
AggregationUpdateJob::IncreaseActiveCount { task } => {
self.increase_active_count(ctx, task);
} }
AggregationUpdateJob::IncreaseActiveCounts { mut task_ids } => { }
if let Some(task_id) = task_ids.pop() { false
self.increase_active_count(ctx, task_id); } else if !self.active_count_updates.is_empty() {
if !task_ids.is_empty() { let mut remaining = MAX_COUNT_BEFORE_YIELD;
self.jobs.push_front(AggregationUpdateJobItem::new( while remaining > 0 {
AggregationUpdateJob::IncreaseActiveCounts { task_ids }, if let Some((task_id, delta)) = self.active_count_updates.pop() {
)); self.update_active_count(ctx, task_id, delta);
} remaining -= 1;
} } else {
break;
} }
} }
false false
...@@ -3168,95 +3218,84 @@ impl AggregationUpdateQueue { ...@@ -3168,95 +3218,84 @@ impl AggregationUpdateQueue {
} }
} }
/// Decreases the active count of a task. /// Applies a coalesced active-count delta to a task.
/// ///
/// Only used when activeness is tracked. /// Only used when activeness is tracked. Positive net deltas retain the old increment path's
fn decrease_active_count(&mut self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) { /// GC resurrection behavior; non-positive net outcomes intentionally do not resurrect a task
/// that remains inactive. Negative deltas retain the rare absent-state scheduling behavior.
fn update_active_count(
&mut self,
ctx: &mut impl ExecuteContext<'_>,
task_id: TaskId,
delta: i32,
) {
debug_assert_ne!(delta, 0);
#[cfg(feature = "trace_aggregation_update")] #[cfg(feature = "trace_aggregation_update")]
let _span = trace_span!("decrease active count").entered(); let _span = trace_span!("update active count", delta).entered();
let mut task = ctx.task( let task = ctx.task(
task_id, task_id,
// For performance reasons this should stay `Meta` and not `All` // For performance reasons this should stay Meta and not All.
// persistent_task_type is now set eagerly in initialize_new_task.
AGGREGATION_UPDATE_CATEGORY, AGGREGATION_UPDATE_CATEGORY,
); );
let mut task = if delta > 0 {
// Revive the task if GC soft-deleted it.
resurrect_deleted(task, task_id, self, ctx)
} else {
task
};
self.check_optimization_pending(&task); self.check_optimization_pending(&task);
let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id)); let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id));
let is_new = state.is_empty(); let is_new = state.is_empty();
let is_zero = state.decrement_active_counter(); let transition = apply_active_count_delta(state, delta);
let is_active = state.active_counter > 0;
let is_empty = state.is_empty(); let is_empty = state.is_empty();
if is_empty { if is_empty {
task.take_activeness(); task.take_activeness();
} }
debug_assert!(
delta >= 0 || !(is_new && is_empty),
"A new Activeness will never be empty after a negative delta"
);
debug_assert!( debug_assert!(
!(is_new && is_zero), delta <= 0 || !is_new || is_active,
// This allows us to but the `if is_zero` block in the else branch of the `if is_new` "A new Activeness will always be active after a positive delta"
// block below for fewer checks and less problems with the borrow checker.
"A new Activeness will never be zero after decrementing"
); );
if is_new {
if is_new && delta < 0 {
// A task is considered "active" purely by the existence of an `Activeness` item, even // A task is considered "active" purely by the existence of an `Activeness` item, even
// if that item has an negative active counter. So we need to make sure to // if that item has a negative active counter. Schedule this rare race exactly as the
// schedule it here. That case is pretty rare and only happens under extreme race // old single-decrement path did.
// conditions.
self.find_and_schedule_dirty_internal(task_id, task, ctx); self.find_and_schedule_dirty_internal(task_id, task, ctx);
} else if is_zero { return;
let followers = get_followers(&task);
drop(task);
if !followers.is_empty() {
self.push(AggregationUpdateJob::DecreaseActiveCounts {
task_ids: followers,
});
}
} else {
drop(task);
} }
}
/// Increases the active count of a task. match transition {
/// ActiveCountTransition::Activated => {
/// Only used when activeness is tracked. let followers = get_followers(&task);
fn increase_active_count(&mut self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) { if is_new {
#[cfg(feature = "trace_aggregation_update")] self.find_and_schedule_dirty_internal(task_id, task, ctx);
let _span = trace_span!("increase active count").entered(); } else {
drop(task);
let task = ctx.task( }
task_id, if !followers.is_empty() {
// For performance reasons this should stay Meta and not All. self.push(AggregationUpdateJob::IncreaseActiveCounts {
// persistent_task_type is now set eagerly in initialize_new_task. task_ids: followers,
AGGREGATION_UPDATE_CATEGORY, });
); }
// Revive the task if GC soft-deleted it.
let mut task = resurrect_deleted(task, task_id, self, ctx);
self.check_optimization_pending(&task);
let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id));
let is_new = state.is_empty();
let is_positive_now = state.increment_active_counter();
let is_empty = state.is_empty();
// This can happen if active count was negative before
if is_empty {
task.take_activeness();
}
debug_assert!(
!is_new || is_positive_now,
// This allows us to nest the `if is_new` block below `if is_positive_now` for fewer
// checks.
"A new Activeness will always be positive after incrementing"
);
if is_positive_now {
let followers = get_followers(&task);
if is_new {
// Fast path to schedule
self.find_and_schedule_dirty_internal(task_id, task, ctx);
} else {
drop(task);
} }
ActiveCountTransition::Deactivated => {
if !followers.is_empty() { let followers = get_followers(&task);
self.push(AggregationUpdateJob::IncreaseActiveCounts { drop(task);
task_ids: followers, if !followers.is_empty() {
}); self.push(AggregationUpdateJob::DecreaseActiveCounts {
task_ids: followers,
});
}
} }
ActiveCountTransition::None => drop(task),
} }
} }
...@@ -3598,3 +3637,138 @@ fn retry_loop(mut retry: u16, mut f: impl FnMut() -> ControlFlow<()>) -> Result< ...@@ -3598,3 +3637,138 @@ fn retry_loop(mut retry: u16, mut f: impl FnMut() -> ControlFlow<()>) -> Result<
} }
} }
} }
#[cfg(test)]
mod tests {
use super::*;
fn task_id(id: u32) -> TaskId {
TaskId::new(id).unwrap()
}
#[test]
fn coalesces_active_count_updates_by_task() {
let first = task_id(1);
let second = task_id(2);
let mut queue = AggregationUpdateQueue::new();
queue.push(AggregationUpdateJob::IncreaseActiveCount { task: first });
queue.push(AggregationUpdateJob::IncreaseActiveCounts {
task_ids: smallvec![first, second, first],
});
queue.push(AggregationUpdateJob::DecreaseActiveCount { task: second });
assert_eq!(queue.active_count_updates.len(), 1);
assert_eq!(queue.active_count_updates.get(&first), Some(&3));
assert!(!queue.is_empty());
}
#[test]
fn coalesces_active_count_decrements() {
let task = task_id(1);
let mut queue = AggregationUpdateQueue::new();
queue.push(AggregationUpdateJob::DecreaseActiveCount { task });
queue.push(AggregationUpdateJob::DecreaseActiveCounts {
task_ids: smallvec![task, task],
});
assert_eq!(queue.active_count_updates.get(&task), Some(&-3));
}
#[test]
fn removes_zero_active_count_updates_without_changing_other_order() {
let first = task_id(1);
let cancelled = task_id(2);
let last = task_id(3);
let mut queue = AggregationUpdateQueue::new();
for task in [first, cancelled, last] {
queue.push(AggregationUpdateJob::IncreaseActiveCount { task });
}
queue.push(AggregationUpdateJob::DecreaseActiveCount { task: cancelled });
assert_eq!(
queue
.active_count_updates
.iter()
.map(|(&task, &delta)| (task, delta))
.collect::<Vec<_>>(),
vec![(first, 1), (last, 1)]
);
queue.push(AggregationUpdateJob::DecreaseActiveCount { task: first });
queue.push(AggregationUpdateJob::DecreaseActiveCount { task: last });
assert!(queue.is_empty());
}
#[test]
fn regular_jobs_stay_ahead_of_coalesced_active_count_updates() {
let task = task_id(1);
let mut queue = AggregationUpdateQueue::new();
queue.push(AggregationUpdateJob::IncreaseActiveCount { task });
queue.push(AggregationUpdateJob::Noop);
queue.push(AggregationUpdateJob::DecreaseActiveCount { task });
assert_eq!(queue.jobs.len(), 1);
assert!(queue.active_count_updates.is_empty());
assert!(!queue.is_empty());
}
#[test]
fn active_count_delta_reports_only_boundary_crossings() {
let task = task_id(1);
let mut state = ActivenessState::new(task);
assert_eq!(
apply_active_count_delta(&mut state, 3),
ActiveCountTransition::Activated
);
assert_eq!(state.active_counter, 3);
assert_eq!(
apply_active_count_delta(&mut state, 2),
ActiveCountTransition::None
);
assert_eq!(
apply_active_count_delta(&mut state, -6),
ActiveCountTransition::Deactivated
);
assert_eq!(state.active_counter, -1);
assert_eq!(
apply_active_count_delta(&mut state, -2),
ActiveCountTransition::None
);
assert_eq!(
apply_active_count_delta(&mut state, 4),
ActiveCountTransition::Activated
);
assert_eq!(state.active_counter, 1);
}
#[test]
fn active_count_delta_preserves_single_step_semantics() {
let task = task_id(1);
let mut state = ActivenessState::new(task);
assert_eq!(
apply_active_count_delta(&mut state, -1),
ActiveCountTransition::None
);
assert_eq!(state.active_counter, -1);
assert_eq!(
apply_active_count_delta(&mut state, 1),
ActiveCountTransition::None
);
assert!(state.is_empty());
assert_eq!(
apply_active_count_delta(&mut state, 1),
ActiveCountTransition::Activated
);
assert_eq!(
apply_active_count_delta(&mut state, -1),
ActiveCountTransition::Deactivated
);
assert!(state.is_empty());
}
}
...@@ -145,18 +145,6 @@ impl ActivenessState { ...@@ -145,18 +145,6 @@ impl ActivenessState {
self.active_until_clean = true; self.active_until_clean = true;
} }
/// Increment the active counter and return true if the counter was 0 before.
pub fn increment_active_counter(&mut self) -> bool {
self.active_counter += 1;
self.active_counter == 1
}
/// Decrement the active counter and return true if the counter is 0 after.
pub fn decrement_active_counter(&mut self) -> bool {
self.active_counter -= 1;
self.active_counter == 0
}
pub fn unset_root_type(&mut self) { pub fn unset_root_type(&mut self) {
self.root_ty = None; self.root_ty = None;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment