Unverified Commit eaa2f075 authored by Jimmy Miller's avatar Jimmy Miller Committed by GitHub

Remove suspend during operations (#99383)

## What

Remove the ability to suspend in flight operations to allow for
snapshotting and GC

## Why

Exstensive testing with GC has revealed a number of bugs latent in this
system

* transient task ids are getting serialized as part of operations, this
confuses our new stricter task access semantics in fresh sessions
* child execution contexts don't properly participate, they both need to
suspend when requested to avoid exceeding retries since the task they
are waiting for has already suspended, and cannot suspend because that
could deadlock the scope_bounded call

Instead we just don't suspend. This simplifies a lot, should be a small
perf win (purely from not touching the atomics as often)

The major risk is that waiting for everything to settle adds latency. To
mitigate that:

* idle snapshots don't wait for operations to settle, they just bail out
instead
* other snapshots do wait but log warnings if it takes more than 10s,
this should never happen but if it does we will have a smoking gun

---------
Co-authored-by: 's avatarLuke Sandberg <lukesandberg@users.noreply.github.com>
parent c62c51e3
...@@ -9051,7 +9051,6 @@ dependencies = [ ...@@ -9051,7 +9051,6 @@ dependencies = [
"either", "either",
"indexmap 2.14.0", "indexmap 2.14.0",
"mime", "mime",
"ringmap",
"serde", "serde",
"smallvec", "smallvec",
"turbo-tasks-hash", "turbo-tasks-hash",
......
...@@ -15,7 +15,6 @@ bincode = { workspace = true } ...@@ -15,7 +15,6 @@ bincode = { workspace = true }
either = { workspace = true } either = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
mime = { workspace = true } mime = { workspace = true }
ringmap = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true }
turbo-tasks-hash = { workspace = true } turbo-tasks-hash = { workspace = true }
......
...@@ -375,83 +375,6 @@ pub mod indexset { ...@@ -375,83 +375,6 @@ pub mod indexset {
} }
} }
pub mod ringset {
use std::hash::{BuildHasher, Hash};
use ::ringmap::RingSet;
use super::*;
pub fn encode<E, T, S>(set: &RingSet<T, S>, encoder: &mut E) -> Result<(), EncodeError>
where
E: Encoder,
T: Encode,
{
usize::encode(&set.len(), encoder)?;
for item in set {
T::encode(item, encoder)?;
}
Ok(())
}
pub fn decode<Context, D, T, S>(decoder: &mut D) -> Result<RingSet<T, S>, DecodeError>
where
D: Decoder<Context = Context>,
T: Decode<Context> + Eq + Hash,
S: BuildHasher + Default,
{
let len = usize::decode(decoder)?;
let mut set = RingSet::with_capacity_and_hasher(len, Default::default());
for _i in 0..len {
set.insert(T::decode(decoder)?);
}
Ok(set)
}
pub fn borrow_decode<'de, Context, D, T, S>(
decoder: &mut D,
) -> Result<RingSet<T, S>, DecodeError>
where
D: BorrowDecoder<'de, Context = Context>,
T: BorrowDecode<'de, Context> + Eq + Hash,
S: BuildHasher + Default,
{
let len = usize::decode(decoder)?;
let mut set = RingSet::with_capacity_and_hasher(len, Default::default());
for _i in 0..len {
set.insert(T::borrow_decode(decoder)?);
}
Ok(set)
}
#[cfg(test)]
mod tests {
use bincode::{decode_from_slice, encode_to_vec};
use super::*;
#[test]
fn test_roundtrip() {
let cfg = bincode::config::standard();
#[derive(Encode, Decode)]
struct Wrapper(#[bincode(with = "crate::ringset")] RingSet<String>);
let set1 = Wrapper(RingSet::from([
"value1".to_string(),
"value2".to_string(),
"value3".to_string(),
]));
let set2: Wrapper = decode_from_slice(&encode_to_vec(&set1, cfg).unwrap(), cfg)
.unwrap()
.0;
assert_eq!(set1.0, set2.0);
}
}
}
pub mod mime_option { pub mod mime_option {
use std::str::FromStr; use std::str::FromStr;
......
...@@ -29,10 +29,10 @@ use turbo_tasks::{TaskId, TurboTasks, scope_unbounded::scope_unbounded_with}; ...@@ -29,10 +29,10 @@ use turbo_tasks::{TaskId, TurboTasks, scope_unbounded::scope_unbounded_with};
use crate::{ use crate::{
backend::{ backend::{
AnyOperation, TurboTasksBackend, TurboTasksBackend,
operation::{ operation::{
AggregationUpdateJob, AggregationUpdateQueue, CleanupOldEdgesOperation, ExecuteContext, AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext, ExecuteContextImpl,
ExecuteContextImpl, TaskGuard, capture_all_edges, TaskGuard, capture_all_edges, cleanup_old_edges_deletions_only,
}, },
snapshot_coordinator::SnapshotPhase, snapshot_coordinator::SnapshotPhase,
storage::{SpecificTaskDataCategory, TaskDataCategory}, storage::{SpecificTaskDataCategory, TaskDataCategory},
...@@ -73,7 +73,7 @@ enum GcJob { ...@@ -73,7 +73,7 @@ enum GcJob {
/// Decides when a GC pass should stop early because it is delaying real work. /// Decides when a GC pass should stop early because it is delaying real work.
struct GcBudget<'a> { struct GcBudget<'a> {
phase: &'a SnapshotPhase<'a, AnyOperation>, phase: &'a SnapshotPhase<'a>,
started: Instant, started: Instant,
/// The minimum quantum of work this pass does before any interrupt is honoured. /// The minimum quantum of work this pass does before any interrupt is honoured.
min_progress: Duration, min_progress: Duration,
...@@ -208,7 +208,7 @@ impl TurboTasksBackend { ...@@ -208,7 +208,7 @@ impl TurboTasksBackend {
pub(crate) fn gc_collect( pub(crate) fn gc_collect(
&self, &self,
turbo_tasks: &TurboTasks<TurboTasksBackend>, turbo_tasks: &TurboTasks<TurboTasksBackend>,
phase: &SnapshotPhase<'_, AnyOperation>, phase: &SnapshotPhase<'_>,
interruptible: bool, interruptible: bool,
) -> (GcStats, GcPassResult, Option<Vec<(TaskId, TtlCounter)>>) { ) -> (GcStats, GcPassResult, Option<Vec<(TaskId, TtlCounter)>>) {
// Record the time at the beginning of the loop to have a consistent timestamp for the roots // Record the time at the beginning of the loop to have a consistent timestamp for the roots
...@@ -288,7 +288,7 @@ impl TurboTasksBackend { ...@@ -288,7 +288,7 @@ impl TurboTasksBackend {
// It is almost certainly already marked modified, so this is mostly a no-op. // It is almost certainly already marked modified, so this is mostly a no-op.
let _ = task.track_modification(SpecificTaskDataCategory::Meta, "gc_deleted"); let _ = task.track_modification(SpecificTaskDataCategory::Meta, "gc_deleted");
} }
drop(task); // drop the lock so CleanupOldEdgesOperation can run drop(task); // drop the lock so edge cleanup can run
stats.collected += 1; stats.collected += 1;
stats.edges_deleted += old_edges.len(); stats.edges_deleted += old_edges.len();
// If we happened to delete a known root at this point record it so we can reconcile // If we happened to delete a known root at this point record it so we can reconcile
...@@ -299,8 +299,7 @@ impl TurboTasksBackend { ...@@ -299,8 +299,7 @@ impl TurboTasksBackend {
// Delete outgoing edges but don't update the aggregation graph yet. // Delete outgoing edges but don't update the aggregation graph yet.
// To avoid accidentally rebalancing on deleted tasks due to racing deletions, // To avoid accidentally rebalancing on deleted tasks due to racing deletions,
// we defer all rebalancing to the end // we defer all rebalancing to the end
let deferred = let deferred = cleanup_old_edges_deletions_only(task_id, old_edges, &mut ctx);
CleanupOldEdgesOperation::run_edge_deletions_only(task_id, old_edges, &mut ctx);
result.deferred_balance_edges.extend(deferred.balance_edges); result.deferred_balance_edges.extend(deferred.balance_edges);
result result
.deferred_dirty_dependents .deferred_dirty_dependents
...@@ -451,18 +450,18 @@ impl TurboTasksBackend { ...@@ -451,18 +450,18 @@ impl TurboTasksBackend {
"gc_for_testing requires a GC-enabled backend: set `BackendOptions::gc = Some(true)`" "gc_for_testing requires a GC-enabled backend: set `BackendOptions::gc = Some(true)`"
); );
let _serialize = self.snapshot_in_progress.lock(); let _serialize = self.snapshot_in_progress.lock();
let phase = self.snapshot_coord.begin_snapshot(); let phase = self
.snapshot_coord
.begin_snapshot(|slow| Self::report_slow_settle_for_snapshot(turbo_tasks, slow));
let (stats, _result, roots) = let (stats, _result, roots) =
self.gc_collect(turbo_tasks, &phase, /* interruptible= */ false); self.gc_collect(turbo_tasks, &phase, /* interruptible= */ false);
// Persist the roots map this pass produced. Some tests query the roots set and GC itself // Persist the roots map this pass produced. Some tests query the roots set and GC itself
// does as well, this ensures it is available to the next cycle. // does as well, this ensures it is available to the next cycle.
if let Some(roots) = roots if let Some(roots) = roots
&& let Err(err) = self.backing_storage.save_snapshot( && let Err(err) = self
Vec::new(), .backing_storage
Some(roots), .save_snapshot(Some(roots), Vec::<Vec<SnapshotItem>>::new())
Vec::<Vec<SnapshotItem>>::new(),
)
{ {
panic!("gc_for_testing: failed to persist GC roots: {err:?}"); panic!("gc_for_testing: failed to persist GC roots: {err:?}");
} }
......
...@@ -11,9 +11,8 @@ use turbo_tasks::{ ...@@ -11,9 +11,8 @@ use turbo_tasks::{
use crate::backend::{ use crate::backend::{
operation::{ operation::{
AggregationUpdateJob, AggregationUpdateQueue, ChildExecuteContext, ExecuteContext, AggregationUpdateJob, AggregationUpdateQueue, ChildExecuteContext, ExecuteContext,
Operation, TaskGuard, aggregation_update::InnerOfUppersHasNewFollowersJob, TaskGuard, aggregation_update::InnerOfUppersHasNewFollowersJob, get_aggregation_number,
get_aggregation_number, get_uppers, invalidate::make_task_dirty_internal, get_uppers, invalidate::make_task_dirty_internal, is_aggregating_node,
is_aggregating_node,
}, },
storage_schema::TaskStorageAccessors, storage_schema::TaskStorageAccessors,
}; };
...@@ -60,8 +59,7 @@ pub fn connect_children( ...@@ -60,8 +59,7 @@ pub fn connect_children(
// Single pass over the newly-connected children, two things per child under one guard: // Single pass over the newly-connected children, two things per child under one guard:
// //
// 1. Bump the child-side parent reference count. it is important to do this before any // 1. Bump the child-side parent reference count before graph propagation.
// suspend points persistence/GC cannot run
// //
// 2. Make any child that has not produced output yet dirty, so it gets scheduled and // 2. Make any child that has not produced output yet dirty, so it gets scheduled and
// computes. // computes.
...@@ -151,21 +149,19 @@ pub fn connect_children( ...@@ -151,21 +149,19 @@ pub fn connect_children(
}); });
} }
#[cfg(any(
feature = "trace_task_completion",
feature = "trace_aggregation_update_stats"
))]
let _span =
tracing::trace_span!("connect new children", stats = tracing::field::Empty).entered();
#[cfg(feature = "trace_aggregation_update_stats")]
{ {
#[cfg(any( let stats = queue.execute_with_stats(ctx);
feature = "trace_task_completion", _span.record("stats", tracing::field::debug(stats));
feature = "trace_aggregation_update_stats"
))]
let _span = tracing::trace_span!("connect new children", stats = tracing::field::Empty)
.entered();
#[cfg(feature = "trace_aggregation_update_stats")]
{
let stats = queue.execute_with_stats(ctx);
_span.record("stats", tracing::field::debug(stats));
}
#[cfg(not(feature = "trace_aggregation_update_stats"))]
queue.execute(ctx);
} }
#[cfg(not(feature = "trace_aggregation_update_stats"))]
queue.execute(ctx);
} }
// Connecting a child varies a lot, but it's in the range of 10-30µs. // Connecting a child varies a lot, but it's in the range of 10-30µs.
......
use bincode::{Decode, Encode};
use smallvec::SmallVec; use smallvec::SmallVec;
#[cfg(feature = "task_dirty_cause")] #[cfg(feature = "task_dirty_cause")]
use turbo_tasks::TaskDirtyCause; use turbo_tasks::TaskDirtyCause;
...@@ -8,7 +7,7 @@ use crate::{ ...@@ -8,7 +7,7 @@ use crate::{
backend::{ backend::{
TaskDataCategory, TaskDataCategory,
operation::{ operation::{
ExecuteContext, Operation, TaskGuard, ExecuteContext, TaskGuard,
aggregation_update::{ aggregation_update::{
AggregationUpdateJob, AggregationUpdateQueue, ComputeDirtyAndCleanUpdate, AggregationUpdateJob, AggregationUpdateQueue, ComputeDirtyAndCleanUpdate,
}, },
...@@ -18,74 +17,22 @@ use crate::{ ...@@ -18,74 +17,22 @@ use crate::{
data::{Dirtyness, InProgressState, InProgressStateInner}, data::{Dirtyness, InProgressState, InProgressStateInner},
}; };
#[derive(Encode, Decode, Clone, Default)] pub fn invalidate(
#[allow(clippy::large_enum_variant)] task_ids: SmallVec<[TaskId; 4]>,
pub enum InvalidateOperation { #[cfg(feature = "task_dirty_cause")] cause: TaskDirtyCause,
MakeDirty { mut ctx: impl ExecuteContext<'_>,
task_ids: SmallVec<[TaskId; 4]>, ) {
#[cfg(feature = "task_dirty_cause")] let mut queue = AggregationUpdateQueue::new();
cause: TaskDirtyCause, for task_id in task_ids {
}, try_make_task_dirty(
AggregationUpdate { task_id,
queue: AggregationUpdateQueue,
},
#[default]
Done,
}
impl InvalidateOperation {
pub fn run(
task_ids: SmallVec<[TaskId; 4]>,
#[cfg(feature = "task_dirty_cause")] cause: TaskDirtyCause,
mut ctx: impl ExecuteContext<'_>,
) {
InvalidateOperation::MakeDirty {
task_ids,
#[cfg(feature = "task_dirty_cause")] #[cfg(feature = "task_dirty_cause")]
cause, cause.clone(),
} &mut queue,
.execute(&mut ctx) &mut ctx,
} );
}
impl Operation for InvalidateOperation {
fn execute(mut self, ctx: &mut impl ExecuteContext<'_>) {
loop {
ctx.operation_suspend_point(&self);
match self {
InvalidateOperation::MakeDirty {
task_ids,
#[cfg(feature = "task_dirty_cause")]
cause,
} => {
let mut queue = AggregationUpdateQueue::new();
for task_id in task_ids {
try_make_task_dirty(
task_id,
#[cfg(feature = "task_dirty_cause")]
cause.clone(),
&mut queue,
ctx,
);
}
if queue.is_empty() {
self = InvalidateOperation::Done
} else {
self = InvalidateOperation::AggregationUpdate { queue }
}
continue;
}
InvalidateOperation::AggregationUpdate { ref mut queue } => {
if queue.process(ctx) {
self = InvalidateOperation::Done
}
}
InvalidateOperation::Done => {
return;
}
}
}
} }
queue.execute(&mut ctx);
} }
/// Marks a task dirty. The task must exist. /// Marks a task dirty. The task must exist.
......
...@@ -3,21 +3,16 @@ use std::{ ...@@ -3,21 +3,16 @@ use std::{
collections::{BinaryHeap, hash_map::Entry}, collections::{BinaryHeap, hash_map::Entry},
}; };
use bincode::{Decode, Encode};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
#[cfg(feature = "trace_leaf_distance_update")] #[cfg(feature = "trace_leaf_distance_update")]
use tracing::{span::Span, trace_span}; use tracing::{span::Span, trace_span};
use turbo_tasks::TaskId; use turbo_tasks::TaskId;
use crate::backend::{ use crate::backend::{
TaskDataCategory, TaskDataCategory, operation::ExecuteContext, storage_schema::TaskStorageAccessors,
operation::{ExecuteContext, Operation},
storage_schema::TaskStorageAccessors,
}; };
/// The maximum number of leaf distance updates to process before yielding back to the executor. /// The maximum number of leaf distance updates processed in one step.
/// This prevents long blocking operations and allows to interrupt the processing for persistent
/// caching.
const MAX_COUNT_BEFORE_YIELD: usize = 1000; const MAX_COUNT_BEFORE_YIELD: usize = 1000;
/// We avoid incrementing the leaf distance by 1 each time to avoid frequent updates. /// We avoid incrementing the leaf distance by 1 each time to avoid frequent updates.
...@@ -26,13 +21,11 @@ const MAX_COUNT_BEFORE_YIELD: usize = 1000; ...@@ -26,13 +21,11 @@ const MAX_COUNT_BEFORE_YIELD: usize = 1000;
const BASE_LEAF_DISTANCE_BUFFER: u32 = 128; const BASE_LEAF_DISTANCE_BUFFER: u32 = 128;
/// An leaf distance update job that is enqueued. /// An leaf distance update job that is enqueued.
#[derive(Encode, Decode, Clone)]
struct LeafDistanceUpdate { struct LeafDistanceUpdate {
dependencies_distance: u32, dependencies_distance: u32,
dependencies_max_distance_in_buffer: u32, dependencies_max_distance_in_buffer: u32,
done: bool, done: bool,
#[cfg(feature = "trace_leaf_distance_update")] #[cfg(feature = "trace_leaf_distance_update")]
#[bincode(skip)]
span: Option<Span>, span: Option<Span>,
} }
...@@ -48,7 +41,7 @@ impl LeafDistanceUpdate { ...@@ -48,7 +41,7 @@ impl LeafDistanceUpdate {
/// A queue of leaf distance update jobs. /// A queue of leaf distance update jobs.
/// It will execute these jobs in order of their minimum dependency leaf distance. /// It will execute these jobs in order of their minimum dependency leaf distance.
/// This ensures that we never have to re-process a task. /// This ensures that we never have to re-process a task.
#[derive(Default, Encode, Decode, Clone)] #[derive(Default)]
pub struct LeafDistanceUpdateQueue { pub struct LeafDistanceUpdateQueue {
queue: BinaryHeap<(Reverse<u32>, TaskId)>, queue: BinaryHeap<(Reverse<u32>, TaskId)>,
leaf_distance_updates: FxHashMap<TaskId, LeafDistanceUpdate>, leaf_distance_updates: FxHashMap<TaskId, LeafDistanceUpdate>,
...@@ -175,18 +168,11 @@ impl LeafDistanceUpdateQueue { ...@@ -175,18 +168,11 @@ impl LeafDistanceUpdateQueue {
} }
task.set_leaf_distance(leaf_distance); task.set_leaf_distance(leaf_distance);
} }
}
impl Operation for LeafDistanceUpdateQueue { pub fn execute(&mut self, ctx: &mut impl ExecuteContext<'_>) {
fn execute(mut self, ctx: &mut impl ExecuteContext<'_>) {
if self.is_empty() { if self.is_empty() {
return; return;
} }
loop { while !self.process(ctx) {}
ctx.operation_suspend_point(&self);
if self.process(ctx) {
return;
}
}
} }
} }
...@@ -14,7 +14,6 @@ use std::{ ...@@ -14,7 +14,6 @@ use std::{
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use bincode::{Decode, Encode};
use parking_lot::RwLockReadGuard; use parking_lot::RwLockReadGuard;
use tracing::info_span; use tracing::info_span;
#[cfg(feature = "trace_prepare_tasks")] #[cfg(feature = "trace_prepare_tasks")]
...@@ -25,7 +24,7 @@ use turbo_tasks::{ ...@@ -25,7 +24,7 @@ use turbo_tasks::{
macro_helpers::NativeFunction, macro_helpers::NativeFunction,
}; };
pub use self::aggregation_update::ComputeDirtyAndCleanUpdate; pub use self::aggregation_update::{AggregationUpdateQueue, ComputeDirtyAndCleanUpdate};
use crate::{ use crate::{
backend::{ backend::{
EventDescription, TaskDataCategory, TurboTasksBackend, EventDescription, TaskDataCategory, TurboTasksBackend,
...@@ -37,10 +36,6 @@ use crate::{ ...@@ -37,10 +36,6 @@ use crate::{
data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState, TransientTask}, data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState, TransientTask},
}; };
pub trait Operation: Encode + Decode<()> + Default + TryFrom<AnyOperation, Error = ()> {
fn execute(self, ctx: &mut impl ExecuteContext<'_>);
}
/// The task storage `open_task` is working with, which may or may not still own its map entry. /// The task storage `open_task` is working with, which may or may not still own its map entry.
enum OpenedTask<'a> { enum OpenedTask<'a> {
Owned(TaskEntryGuard<'a>), Owned(TaskEntryGuard<'a>),
...@@ -181,9 +176,6 @@ pub trait ExecuteContext<'e>: Sized { ...@@ -181,9 +176,6 @@ pub trait ExecuteContext<'e>: Sized {
) -> (Self::TaskGuardImpl, Self::TaskGuardImpl); ) -> (Self::TaskGuardImpl, Self::TaskGuardImpl);
fn schedule_task(&self, task: &Self::TaskGuardImpl, parent_priority: TaskPriority); fn schedule_task(&self, task: &Self::TaskGuardImpl, parent_priority: TaskPriority);
fn get_current_task_priority(&self) -> TaskPriority; fn get_current_task_priority(&self) -> TaskPriority;
fn operation_suspend_point<T>(&mut self, op: &T)
where
T: Clone + Into<AnyOperation>;
/// Record `task` as a GC candidate **if it is in fact collectible**. /// Record `task` as a GC candidate **if it is in fact collectible**.
/// ///
/// Call this wherever an operation removes the last reference of some kind to a task. This may /// Call this wherever an operation removes the last reference of some kind to a task. This may
...@@ -262,18 +254,12 @@ impl TaskLockCounter { ...@@ -262,18 +254,12 @@ impl TaskLockCounter {
} }
} }
enum ExecutePhase<'e> {
Normal {
guard: Option<OperationGuard<'e, AnyOperation>>,
},
Child,
Gc(&'e dyn Fn(TaskId)),
}
pub struct ExecuteContextImpl<'e> { pub struct ExecuteContextImpl<'e> {
backend: &'e TurboTasksBackend, backend: &'e TurboTasksBackend,
turbo_tasks: &'e TurboTasks<TurboTasksBackend>, turbo_tasks: &'e TurboTasks<TurboTasksBackend>,
phase: ExecutePhase<'e>, gc_collectible: Option<&'e dyn Fn(TaskId)>,
/// Held until all graph changes made by this context have completed.
_operation_guard: Option<OperationGuard<'e>>,
/// Held by contexts built through `TurboTasksBackend::try_execute_context`, so that storage /// Held by contexts built through `TurboTasksBackend::try_execute_context`, so that storage
/// teardown in `stop()` waits for this context to be dropped. /// teardown in `stop()` waits for this context to be dropped.
_shutdown_guard: Option<RwLockReadGuard<'e, bool>>, _shutdown_guard: Option<RwLockReadGuard<'e, bool>>,
...@@ -288,9 +274,8 @@ impl<'e> ExecuteContextImpl<'e> { ...@@ -288,9 +274,8 @@ impl<'e> ExecuteContextImpl<'e> {
Self { Self {
backend, backend,
turbo_tasks, turbo_tasks,
phase: ExecutePhase::Normal { gc_collectible: None,
guard: backend.start_operation(), _operation_guard: backend.start_operation(),
},
_shutdown_guard: None, _shutdown_guard: None,
task_lock_counter: TaskLockCounter::new(), task_lock_counter: TaskLockCounter::new(),
} }
...@@ -307,9 +292,8 @@ impl<'e> ExecuteContextImpl<'e> { ...@@ -307,9 +292,8 @@ impl<'e> ExecuteContextImpl<'e> {
Self { Self {
backend, backend,
turbo_tasks, turbo_tasks,
phase: ExecutePhase::Normal { gc_collectible: None,
guard: backend.start_operation(), _operation_guard: backend.start_operation(),
},
_shutdown_guard: Some(shutdown_guard), _shutdown_guard: Some(shutdown_guard),
task_lock_counter: TaskLockCounter::new(), task_lock_counter: TaskLockCounter::new(),
} }
...@@ -319,18 +303,19 @@ impl<'e> ExecuteContextImpl<'e> { ...@@ -319,18 +303,19 @@ impl<'e> ExecuteContextImpl<'e> {
/// collector while it holds the coordinator's exclusion phase. /// collector while it holds the coordinator's exclusion phase.
/// ///
/// The exclusion excludes all concurrent operations and task execution, so taking an operation /// The exclusion excludes all concurrent operations and task execution, so taking an operation
/// guard here would deadlock. Requiring `&ExclusionPhase` makes that a type-level obligation: /// guard here would deadlock. Requiring `&SnapshotPhase` makes that a type-level obligation:
/// the caller cannot construct this context without actually holding the exclusion. /// the caller cannot construct this context without actually holding the exclusion.
pub(super) fn new_for_gc( pub(super) fn new_for_gc(
backend: &'e TurboTasksBackend, backend: &'e TurboTasksBackend,
turbo_tasks: &'e TurboTasks<TurboTasksBackend>, turbo_tasks: &'e TurboTasks<TurboTasksBackend>,
_phase: &'e SnapshotPhase<'_, AnyOperation>, _phase: &'e SnapshotPhase<'_>,
gc_collectible: &'e dyn Fn(TaskId), gc_collectible: &'e dyn Fn(TaskId),
) -> Self { ) -> Self {
Self { Self {
backend, backend,
turbo_tasks, turbo_tasks,
phase: ExecutePhase::Gc(gc_collectible), gc_collectible: Some(gc_collectible),
_operation_guard: None,
_shutdown_guard: None, _shutdown_guard: None,
task_lock_counter: TaskLockCounter::new(), task_lock_counter: TaskLockCounter::new(),
} }
...@@ -1364,15 +1349,8 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> { ...@@ -1364,15 +1349,8 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> {
self.turbo_tasks.get_current_task_priority() self.turbo_tasks.get_current_task_priority()
} }
fn operation_suspend_point<T: Clone + Into<AnyOperation>>(&mut self, op: &T) {
let ExecutePhase::Normal { guard: Some(guard) } = &mut self.phase else {
return;
};
guard.suspend_point(|| op.clone().into());
}
fn note_maybe_collectible(&mut self, task: &impl TaskGuard) { fn note_maybe_collectible(&mut self, task: &impl TaskGuard) {
if let ExecutePhase::Gc(collector) = self.phase if let Some(collector) = self.gc_collectible
&& task.is_gc_collectible() && task.is_gc_collectible()
{ {
collector(task.id()); collector(task.id());
...@@ -1435,7 +1413,8 @@ impl<'e> ChildExecuteContext<'e> for ChildExecuteContextImpl<'e> { ...@@ -1435,7 +1413,8 @@ impl<'e> ChildExecuteContext<'e> for ChildExecuteContextImpl<'e> {
ExecuteContextImpl { ExecuteContextImpl {
backend: self.backend, backend: self.backend,
turbo_tasks: self.turbo_tasks, turbo_tasks: self.turbo_tasks,
phase: ExecutePhase::Child, gc_collectible: None,
_operation_guard: None,
// A child context runs inside its parent's execution, which the foreground drain // A child context runs inside its parent's execution, which the foreground drain
// already waits for, so it needs no shutdown guard of its own. // already waits for, so it needs no shutdown guard of its own.
_shutdown_guard: None, _shutdown_guard: None,
...@@ -2025,74 +2004,20 @@ impl TaskStorageAccessors for TaskGuardImpl<'_> { ...@@ -2025,74 +2004,20 @@ impl TaskStorageAccessors for TaskGuardImpl<'_> {
} }
} }
macro_rules! impl_operation {
($name:ident $type_path:path) => {
impl From<$type_path> for AnyOperation {
fn from(op: $type_path) -> Self {
AnyOperation::$name(op)
}
}
impl TryFrom<AnyOperation> for $type_path {
type Error = ();
fn try_from(op: AnyOperation) -> Result<Self, Self::Error> {
match op {
AnyOperation::$name(op) => Ok(op),
_ => Err(()),
}
}
}
pub use $type_path;
};
}
#[derive(Encode, Decode, Clone)]
pub enum AnyOperation {
ConnectChild(connect_child::ConnectChildOperation),
Invalidate(invalidate::InvalidateOperation),
UpdateCell(update_cell::UpdateCellOperation),
CleanupOldEdges(cleanup_old_edges::CleanupOldEdgesOperation),
AggregationUpdate(aggregation_update::AggregationUpdateQueue),
LeafDistanceUpdate(leaf_distance_update::LeafDistanceUpdateQueue),
Nested(Vec<AnyOperation>),
}
impl AnyOperation {
pub fn execute(self, ctx: &mut impl ExecuteContext<'_>) {
match self {
AnyOperation::ConnectChild(op) => op.execute(ctx),
AnyOperation::Invalidate(op) => op.execute(ctx),
AnyOperation::UpdateCell(op) => op.execute(ctx),
AnyOperation::CleanupOldEdges(op) => op.execute(ctx),
AnyOperation::AggregationUpdate(op) => op.execute(ctx),
AnyOperation::LeafDistanceUpdate(op) => op.execute(ctx),
AnyOperation::Nested(ops) => {
for op in ops {
op.execute(ctx);
}
}
}
}
}
impl_operation!(ConnectChild connect_child::ConnectChildOperation);
impl_operation!(Invalidate invalidate::InvalidateOperation);
impl_operation!(UpdateCell update_cell::UpdateCellOperation);
impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation);
impl_operation!(AggregationUpdate aggregation_update::AggregationUpdateQueue);
impl_operation!(LeafDistanceUpdate leaf_distance_update::LeafDistanceUpdateQueue);
pub use self::{ pub use self::{
aggregation_update::{ aggregation_update::{
AggregatedDataUpdate, AggregationUpdateJob, get_aggregation_number, get_uppers, AggregatedDataUpdate, AggregationUpdateJob, get_aggregation_number, get_uppers,
is_aggregating_node, is_root_node, is_aggregating_node, is_root_node,
}, },
cleanup_old_edges::{OutdatedEdge, capture_all_edges}, cleanup_old_edges::{
OutdatedEdge, capture_all_edges, cleanup_old_edges, cleanup_old_edges_deletions_only,
},
connect_child::connect_child,
connect_children::connect_children, connect_children::connect_children,
invalidate::make_task_dirty_internal, invalidate::{invalidate, make_task_dirty_internal},
leaf_distance_update::LeafDistanceUpdateQueue,
prepare_new_children::prepare_new_children, prepare_new_children::prepare_new_children,
update_cell::update_cell,
update_collectible::UpdateCollectibleOperation, update_collectible::UpdateCollectibleOperation,
}; };
......
...@@ -8,7 +8,6 @@ use crate::{ ...@@ -8,7 +8,6 @@ use crate::{
TaskDataCategory, TaskDataCategory,
operation::{ operation::{
AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext, AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue, ExecuteContext,
Operation,
}, },
storage_schema::TaskStorageAccessors, storage_schema::TaskStorageAccessors,
}, },
......
...@@ -18,7 +18,7 @@ use turbo_tasks::{ ...@@ -18,7 +18,7 @@ use turbo_tasks::{
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, TtlCounter, storage_schema::TaskStorage}, backend::{SpecificTaskDataCategory, TtlCounter, storage_schema::TaskStorage},
backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash_from_components}, backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash_from_components},
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
...@@ -34,9 +34,8 @@ use crate::{ ...@@ -34,9 +34,8 @@ use crate::{
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
#[repr(u8)] #[repr(u8)]
enum InfraKey { enum InfraKey {
Operations = 0, NextFreeTaskId = 0,
NextFreeTaskId = 1, GcRoots = 1,
GcRoots = 2,
} }
impl InfraKey { impl InfraKey {
...@@ -237,19 +236,6 @@ impl TurboBackingStorage { ...@@ -237,19 +236,6 @@ impl TurboBackingStorage {
.map_or(Ok(TaskId::MIN), TaskId::try_from)?) .map_or(Ok(TaskId::MIN), TaskId::try_from)?)
} }
pub(crate) fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> {
fn get(database: &TurboKeyValueDatabase) -> Result<Vec<AnyOperation>> {
let Some(operations) =
database.get(KeySpace::Infra, InfraKey::Operations.key().as_ref())?
else {
return Ok(Vec::new());
};
let operations = turbo_bincode_decode(operations.borrow())?;
Ok(operations)
}
get(&self.inner.database).context("Unable to read uncompleted operations from database")
}
/// Reads the persisted GC roots set (see [`InfraKey::GcRoots`]). Empty on a fresh database. /// Reads the persisted GC roots set (see [`InfraKey::GcRoots`]). Empty on a fresh database.
pub(crate) fn roots(&self) -> Result<Vec<(TaskId, TtlCounter)>> { pub(crate) fn roots(&self) -> Result<Vec<(TaskId, TtlCounter)>> {
fn get(database: &TurboKeyValueDatabase) -> Result<Vec<(TaskId, TtlCounter)>> { fn get(database: &TurboKeyValueDatabase) -> Result<Vec<(TaskId, TtlCounter)>> {
...@@ -265,14 +251,13 @@ impl TurboBackingStorage { ...@@ -265,14 +251,13 @@ impl TurboBackingStorage {
pub(crate) fn save_snapshot<I>( pub(crate) fn save_snapshot<I>(
&self, &self,
operations: Vec<Arc<AnyOperation>>,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<SnapshotMeta> ) -> Result<SnapshotMeta>
where where
I: IntoIterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{ {
let _span = tracing::info_span!("save snapshot", operations = operations.len()).entered(); let _span = tracing::info_span!("save snapshot").entered();
let batch = self.inner.database.write_batch()?; let batch = self.inner.database.write_batch()?;
{ {
...@@ -367,7 +352,7 @@ impl TurboBackingStorage { ...@@ -367,7 +352,7 @@ impl TurboBackingStorage {
let mut next_task_id = get_next_free_task_id(&batch)?; let mut next_task_id = get_next_free_task_id(&batch)?;
next_task_id = next_task_id.max(snapshot_meta.max_next_task_id + 1); next_task_id = next_task_id.max(snapshot_meta.max_next_task_id + 1);
save_infra(&batch, next_task_id, operations, roots)?; save_infra(&batch, next_task_id, roots)?;
{ {
let _span = tracing::trace_span!("commit").entered(); let _span = tracing::trace_span!("commit").entered();
// Byte totals are the physical on-disk bytes (post-compression, including .sst / // Byte totals are the physical on-disk bytes (post-compression, including .sst /
...@@ -494,7 +479,6 @@ fn get_next_free_task_id(batch: &TurboWriteBatch<'_>) -> Result<u32, anyhow::Err ...@@ -494,7 +479,6 @@ fn get_next_free_task_id(batch: &TurboWriteBatch<'_>) -> Result<u32, anyhow::Err
fn save_infra( fn save_infra(
batch: &TurboWriteBatch<'_>, batch: &TurboWriteBatch<'_>,
next_task_id: u32, next_task_id: u32,
operations: Vec<Arc<AnyOperation>>,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
) -> Result<(), anyhow::Error> { ) -> Result<(), anyhow::Error> {
batch batch
...@@ -504,19 +488,6 @@ fn save_infra( ...@@ -504,19 +488,6 @@ fn save_infra(
WriteBuffer::Borrowed(&next_task_id.to_le_bytes()), WriteBuffer::Borrowed(&next_task_id.to_le_bytes()),
) )
.context("Unable to write next free task id")?; .context("Unable to write next free task id")?;
{
let _span =
tracing::trace_span!("update operations", operations = operations.len()).entered();
let operations =
turbo_bincode_encode(&operations).context("Unable to serialize operations")?;
batch
.put(
KeySpace::Infra,
WriteBuffer::Borrowed(InfraKey::Operations.key().as_ref()),
WriteBuffer::SmallVec(operations),
)
.context("Unable to write operations")?;
}
if let Some(roots) = roots { if let Some(roots) = roots {
let _span = tracing::trace_span!("update roots", roots = roots.len()).entered(); let _span = tracing::trace_span!("update roots", roots = roots.len()).entered();
let roots = turbo_bincode_encode(&roots).context("Unable to serialize GC roots")?; let roots = turbo_bincode_encode(&roots).context("Unable to serialize GC roots")?;
...@@ -731,7 +702,6 @@ mod tests { ...@@ -731,7 +702,6 @@ mod tests {
// Snapshot with no task data, just the one deletion. // Snapshot with no task data, just the one deletion.
storage.save_snapshot( storage.save_snapshot(
Vec::new(),
None, None,
vec![vec![SnapshotItem::Delete { vec![vec![SnapshotItem::Delete {
task_id: deleted_id, task_id: deleted_id,
......
...@@ -2,7 +2,6 @@ pub mod dash_map_drop_contents; ...@@ -2,7 +2,6 @@ pub mod dash_map_drop_contents;
pub mod dash_map_entry; pub mod dash_map_entry;
pub mod dash_map_multi; pub mod dash_map_multi;
pub mod markdown_table; pub mod markdown_table;
pub mod ptr_eq_arc;
pub mod shard_amount; pub mod shard_amount;
pub mod stopwatch; pub mod stopwatch;
pub mod swap_retain; pub mod swap_retain;
......
use std::{
hash::{Hash, Hasher},
ops::Deref,
sync::Arc,
};
pub struct PtrEqArc<T>(Arc<T>);
impl<T> PtrEqArc<T> {
pub fn arc(&self) -> &Arc<T> {
&self.0
}
}
impl<T> From<Arc<T>> for PtrEqArc<T> {
fn from(value: Arc<T>) -> Self {
Self(value)
}
}
impl<T> Deref for PtrEqArc<T> {
type Target = Arc<T>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> Clone for PtrEqArc<T> {
fn clone(&self) -> Self {
Self(self.0.clone())
}
}
impl<T> PartialEq for PtrEqArc<T> {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.0, &other.0)
}
}
impl<T> Eq for PtrEqArc<T> {}
impl<T> Hash for PtrEqArc<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
Arc::as_ptr(&self.0).hash(state)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment