Unverified Commit 0fd2df41 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-tasks: unify keyed cell dependencies

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent b3a13b48
......@@ -3,7 +3,7 @@ use std::mem::take;
use rustc_hash::FxHashSet;
use smallvec::SmallVec;
use turbo_tasks::TaskId;
use turbo_tasks::{CellKey, TaskId};
use crate::{
backend::{
......@@ -29,16 +29,11 @@ pub enum OutdatedEdge {
dependent: TaskId,
cell: CellRef,
},
HashedCellDependency {
dependent: TaskId,
cell: CellRef,
key: u64,
},
/// A `Vc::keyed()` read dependency (`TrackKey::Keyed(id)`).
/// `dependent` reads `cell` with a [`CellKey`] (a hashed sub-value or a `Vc::keyed()` read).
KeyedCellDependency {
dependent: TaskId,
cell: CellRef,
id: u32,
key: CellKey,
},
/// `dependent` reads the output of `output_task`.
OutputDependency {
......@@ -66,18 +61,11 @@ pub fn capture_all_edges(task: &TaskGuard<'_>) -> Vec<OutdatedEdge> {
cell,
}),
);
old_edges.extend(task.iter_cell_dependencies_hashed().map(|(cell, key)| {
OutdatedEdge::HashedCellDependency {
dependent: task_id,
cell,
key,
}
}));
old_edges.extend(task.iter_cell_dependencies_keyed().map(|(cell, id)| {
old_edges.extend(task.iter_cell_dependencies_keyed().map(|(cell, key)| {
OutdatedEdge::KeyedCellDependency {
dependent: task_id,
cell,
id,
key,
}
}));
old_edges.extend(
......@@ -96,24 +84,14 @@ pub fn capture_all_edges(task: &TaskGuard<'_>) -> Vec<OutdatedEdge> {
},
}),
);
old_edges.extend(task.iter_cell_dependents_hashed().map(|(entry, key)| {
OutdatedEdge::HashedCellDependency {
dependent: entry.task,
cell: CellRef {
task: task_id,
cell: entry.cell,
},
key,
}
}));
old_edges.extend(task.iter_cell_dependents_keyed().map(|(entry, id)| {
old_edges.extend(task.iter_cell_dependents_keyed().map(|(entry, key)| {
OutdatedEdge::KeyedCellDependency {
dependent: entry.task,
cell: CellRef {
task: task_id,
cell: entry.cell,
},
id,
key,
}
}));
old_edges.extend(task.iter_output_dependent().map(|dependent| {
......@@ -297,32 +275,10 @@ fn cleanup_old_edges_inner(
task.remove_outdated_cell_dependencies(&cell);
}
}
OutdatedEdge::HashedCellDependency {
dependent,
cell,
key,
} => {
// Same as above but in the `_hashed` sets.
{
let mut task = ctx.task(cell.task, TaskDataCategory::Data);
task.remove_cell_dependents_hashed(&(
CellRef {
task: dependent,
cell: cell.cell,
},
key,
));
}
{
let mut task = ctx.task(dependent, TaskDataCategory::Data);
task.remove_cell_dependencies_hashed(&(cell, key));
task.remove_outdated_cell_dependencies_hashed(&(cell, key));
}
}
OutdatedEdge::KeyedCellDependency {
dependent,
cell,
id,
key,
} => {
// Same as above but in the `_keyed` sets.
{
......@@ -332,13 +288,13 @@ fn cleanup_old_edges_inner(
task: dependent,
cell: cell.cell,
},
id,
key,
));
}
{
let mut task = ctx.task(dependent, TaskDataCategory::Data);
task.remove_cell_dependencies_keyed(&(cell, id));
task.remove_outdated_cell_dependencies_keyed(&(cell, id));
task.remove_cell_dependencies_keyed(&(cell, key));
task.remove_outdated_cell_dependencies_keyed(&(cell, key));
}
}
OutdatedEdge::OutputDependency {
......
......@@ -1775,10 +1775,6 @@ impl<'a> TaskGuard<'a> {
self.iter_cell_dependencies()
.map(|r| (r.task, TaskDataCategory::All)),
)
.chain(
self.iter_cell_dependencies_hashed()
.map(|(r, _)| (r.task, TaskDataCategory::All)),
)
.chain(
self.iter_cell_dependencies_keyed()
.map(|(r, _)| (r.task, TaskDataCategory::All)),
......
......@@ -25,7 +25,7 @@ use std::{
use parking_lot::Mutex;
use rustc_hash::FxHasher;
use turbo_tasks::{
CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
CellId, CellKey, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
backend::{CachedTaskTypeArc, CellHash, TransientTaskType},
event::Event,
task_storage,
......@@ -267,9 +267,9 @@ struct TaskStorageSchema {
)]
output_dependencies: AutoSet<TaskId, 6>,
/// Cells this task depends on as a whole (keyless `CellDependency::All`). The common case;
/// kept separate from `cell_dependencies_hashed` so it stores a bare `CellRef` instead of the
/// wider `CellDependency` enum (no tag, no `u64`).
/// Cells this task depends on as a whole (keyless). The common case; kept separate from
/// `cell_dependencies_keyed` so it stores a bare `CellRef` instead of a `(CellRef, CellKey)`
/// pair.
#[field(
storage = "auto_set",
category = "data",
......@@ -279,7 +279,13 @@ struct TaskStorageSchema {
)]
cell_dependencies: AutoSet<CellRef, 3>,
/// Cells this task depends on, narrowed to a hashed sub-value (`CellDependency::Hash`). Rare.
/// Cells this task depends on with a `CellKey`. Rare.
/// - `CellKey::Hash`: narrowed to a hashed sub-value, only invalidated when it changes.
/// - `CellKey::Keyed`: a `Vc::keyed()` read that hasn't been converted to `CellKey::Hash`
/// dependencies yet, identified by the `keyed()` call counter of the execution. Invalidated
/// like `cell_dependencies`. Short-lived: the `KeyedReadRef` converts it to `CellKey::Hash`
/// entries when dropped, and any remaining ones are converted to `cell_dependencies` when
/// the execution completes.
#[field(
storage = "auto_set",
category = "data",
......@@ -287,20 +293,7 @@ struct TaskStorageSchema {
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Cells this task depends on via a `Vc::keyed()` read that hasn't been converted to hashed
/// dependencies (`TrackKey::Keyed`), identified by the `keyed()` call counter of the
/// execution. Invalidated like `cell_dependencies`. Rare and usually short-lived: the
/// `KeyedReadRef` converts it to `cell_dependencies_hashed` when dropped.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies_keyed: AutoSet<(CellRef, u32), 1>,
cell_dependencies_keyed: AutoSet<(CellRef, CellKey), 1>,
/// Collectibles this task depends on.
#[field(
......@@ -320,13 +313,9 @@ struct TaskStorageSchema {
#[field(storage = "auto_set", category = "transient", drop_on_completion)]
outdated_cell_dependencies: AutoSet<CellRef, 3>,
/// Outdated hashed cell dependencies to be cleaned up (transient).
/// Outdated cell dependencies with a `CellKey` to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", drop_on_completion)]
outdated_cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Outdated keyed-read cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", drop_on_completion)]
outdated_cell_dependencies_keyed: AutoSet<(CellRef, u32), 1>,
outdated_cell_dependencies_keyed: AutoSet<(CellRef, CellKey), 1>,
/// Outdated collectibles dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", drop_on_completion)]
......@@ -346,25 +335,16 @@ struct TaskStorageSchema {
)]
cell_dependents: AutoSet<CellRef, 3>,
/// Tasks that depend on a hashed sub-value of this task's cells. Reverse of
/// `cell_dependencies_hashed`.
/// Tasks that depend on this task's cells with a `CellKey`. Reverse of
/// `cell_dependencies_keyed`. `CellKey::Hash` entries are only invalidated when that sub-value
/// changes, `CellKey::Keyed` entries like `cell_dependents`.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents_hashed: AutoSet<(CellRef, u64), 1>,
/// Tasks that depend on this task's cells via a `Vc::keyed()` read. Reverse of
/// `cell_dependencies_keyed`. Invalidated like `cell_dependents`.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents_keyed: AutoSet<(CellRef, u32), 1>,
cell_dependents_keyed: AutoSet<(CellRef, CellKey), 1>,
/// Tasks that depend on collectibles of a specific type from this task.
/// Maps TraitTypeId -> Set<TaskId>
......@@ -879,7 +859,7 @@ impl TaskStorage {
///
/// The `Data`-category dependent sets are deliberately not consulted:
///
/// - `cell_dependents` / `cell_dependents_hashed` are redundant with ancestry. A cell dependent
/// - `cell_dependents` / `cell_dependents_keyed` are redundant with ancestry. A cell dependent
/// is either a child, whose child edge already orders the teardown, or a sibling reached by
/// passing a `ResolvedVc` laterally, which needs a common ancestor that collects both in the
/// same pass. Counting them deadlocked the caller/callee cycle `NftJsonAsset::content` ->
......@@ -1005,12 +985,7 @@ impl IsTransient for CellRef {
CellRef::is_transient(self)
}
}
impl IsTransient for (CellRef, u64) {
fn is_transient(&self) -> bool {
self.0.is_transient()
}
}
impl IsTransient for (CellRef, u32) {
impl IsTransient for (CellRef, CellKey) {
fn is_transient(&self) -> bool {
self.0.is_transient()
}
......
......@@ -625,6 +625,11 @@ pub trait Backend: Sized + Sync + Send {
/// Called when a task's execution finishes.
///
/// `has_keyed_reads` is true when the execution made [`Vc::keyed`][crate::Vc::keyed] reads.
/// The backend must then convert all [`CellKey::Keyed`][crate::CellKey::Keyed] dependencies of
/// the task that are still present into normal (full) cell dependencies, as their ids are only
/// meaningful within this execution.
///
/// Returns `Some(priority)` if the task was invalidated again while executing and must be
/// re-run. The caller is responsible for re-scheduling the task at the returned priority
/// (typically lower than the priority of the just-finished run).
......@@ -633,6 +638,7 @@ pub trait Backend: Sized + Sync + Send {
task: TaskId,
result: Result<RawVc, TurboTasksExecutionError>,
cell_counters: &AutoMap<ValueTypeId, u32, BuildHasherDefault<FxHasher>, 8>,
has_keyed_reads: bool,
#[cfg(feature = "verify_determinism")] stateful: bool,
has_invalidator: bool,
turbo_tasks: &TurboTasks<Self>,
......@@ -667,9 +673,9 @@ pub trait Backend: Sized + Sync + Send {
turbo_tasks: &TurboTasks<Self>,
) -> Result<ReadOutcome<TypedCellContent>>;
/// Replaces the [`TrackKey::Keyed(id)`][crate::TrackKey::Keyed] dependency of `reader` on the
/// Replaces the [`CellKey::Keyed(id)`][crate::CellKey::Keyed] dependency of `reader` on the
/// cell `index` of `task`, which was created by a [`Vc::keyed`][crate::Vc::keyed] read, with
/// [`TrackKey::Key`][crate::TrackKey::Key] dependencies on each of the `key_hashes`, in a
/// [`CellKey::Hash`][crate::CellKey::Hash] dependencies on each of the `key_hashes`, in a
/// single operation.
///
/// Must only be called from within the same execution of `reader` that created the
......
......@@ -100,8 +100,8 @@ pub use crate::{
},
join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt},
manager::{
CurrentCellRef, GcRoot, InputResolution, ReadCellTracking, ReadConsistency, ReadTracking,
ScheduleKey, TaskPersistence, TaskPriority, TrackKey, TurboTasks, TurboTasksApi,
CellKey, CurrentCellRef, GcRoot, InputResolution, ReadCellTracking, ReadConsistency,
ReadTracking, ScheduleKey, TaskPersistence, TaskPriority, TurboTasks, TurboTasksApi,
TurboTasksCallApi, Unused, UpdateInfo, dynamic_call, emit, get_serialization_invalidator,
mark_finished, mark_stateful, mark_top_level_task, prevent_gc, run, run_once,
run_once_with_reason, trait_call, try_turbo_tasks, turbo_tasks, turbo_tasks_scope,
......
......@@ -150,8 +150,8 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send {
options: ReadCellOptions,
) -> Result<ReadOutcome<TypedCellContent>>;
/// Replaces the [`TrackKey::Keyed(id)`][TrackKey::Keyed] dependency of the current task
/// (`reader`) on the cell `index` of `task` with [`TrackKey::Key`] dependencies on each of the
/// Replaces the [`CellKey::Keyed(id)`][CellKey::Keyed] dependency of the current task
/// (`reader`) on the cell `index` of `task` with [`CellKey::Hash`] dependencies on each of the
/// `key_hashes`. See [`Backend::convert_keyed_cell_dependency`].
fn convert_keyed_cell_dependency(
&self,
......@@ -351,21 +351,21 @@ pub enum ReadConsistency {
Strong,
}
/// Identifies which part of a cell a tracked cell read depends on.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Default)]
pub enum TrackKey {
/// The read depends on the whole cell content.
#[default]
None,
/// Identifies the part of a cell a tracked cell read depends on, when it doesn't depend on the
/// whole cell content (which is represented by `None` in an `Option<CellKey>`).
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Encode, Decode)]
pub enum CellKey {
/// The read only depends on the sub-value identified by this key hash (see
/// [`KeyedEq`]). The dependency is only invalidated when the value of this key changes.
Key(u64),
/// The read is a [`Vc::keyed`](crate::Vc::keyed) read. For invalidation it behaves like
/// [`TrackKey::None`] (invalidated by any change of the cell), but it is a separate dependency
/// that is uniquely identified by this counter of `keyed()` calls within the current task
/// execution. This allows the [`KeyedReadRef`](crate::KeyedReadRef) to replace it with
/// [`TrackKey::Key`] dependencies of the accessed keys once it's dropped, without affecting
/// other (full) dependencies on the same cell.
Hash(u64),
/// The read is a [`Vc::keyed`](crate::Vc::keyed) read. For invalidation it behaves like a
/// dependency on the whole cell (invalidated by any change of the cell), but it is a separate
/// dependency that is uniquely identified by this counter of `keyed()` calls within the
/// current task execution. This allows the [`KeyedReadRef`](crate::KeyedReadRef) to replace it
/// with [`CellKey::Hash`] dependencies of the accessed keys once it's dropped, without
/// affecting other (full) dependencies on the same cell. `Keyed` dependencies that are still
/// present when the execution completes are converted to normal (full) dependencies, so they
/// never outlive the execution that created them.
Keyed(u32),
}
......@@ -373,8 +373,8 @@ pub enum TrackKey {
pub enum ReadCellTracking {
/// Reads are tracked as dependencies of the current task.
Tracked {
/// The key used for the dependency
key: TrackKey,
/// The part of the cell the dependency is on. `None` depends on the whole cell.
key: Option<CellKey>,
},
/// The read is only tracked when there is an error, otherwise it is untracked.
///
......@@ -397,34 +397,30 @@ impl ReadCellTracking {
}
}
pub fn key(&self) -> TrackKey {
pub fn key(&self) -> Option<CellKey> {
match self {
ReadCellTracking::Tracked { key } => *key,
ReadCellTracking::TrackOnlyError => TrackKey::None,
ReadCellTracking::Untracked => TrackKey::None,
ReadCellTracking::TrackOnlyError => None,
ReadCellTracking::Untracked => None,
}
}
}
impl Default for ReadCellTracking {
fn default() -> Self {
ReadCellTracking::Tracked {
key: TrackKey::None,
}
ReadCellTracking::Tracked { key: None }
}
}
impl Display for ReadCellTracking {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ReadCellTracking::Tracked { key: None } => write!(f, "tracked"),
ReadCellTracking::Tracked {
key: TrackKey::None,
} => write!(f, "tracked"),
ReadCellTracking::Tracked {
key: TrackKey::Key(key),
key: Some(CellKey::Hash(key)),
} => write!(f, "tracked with key {key}"),
ReadCellTracking::Tracked {
key: TrackKey::Keyed(id),
key: Some(CellKey::Keyed(id)),
} => write!(f, "tracked as keyed read {id}"),
ReadCellTracking::TrackOnlyError => write!(f, "track only error"),
ReadCellTracking::Untracked => write!(f, "untracked"),
......@@ -864,7 +860,7 @@ struct CurrentTaskStateHandle {
struct CurrentTaskStateInner {
current_task_id: Option<TaskId>,
/// Counter of [`Vc::keyed`](crate::Vc::keyed) reads in this execution, used to give every
/// [`TrackKey::Keyed`] dependency a unique id.
/// [`CellKey::Keyed`] dependency a unique id.
keyed_read_counter: AtomicU32,
state: RwLock<CurrentTaskState>,
}
......@@ -885,7 +881,7 @@ impl CurrentTaskStateHandle {
}
}
/// Identifies the [`TrackKey::Keyed`] dependency of a [`Vc::keyed`](crate::Vc::keyed) read and the
/// Identifies the [`CellKey::Keyed`] dependency of a [`Vc::keyed`](crate::Vc::keyed) read and the
/// task execution that created it.
pub(crate) struct KeyedReadDependency {
/// The execution state of the reading task. Only used for identity (a `Weak` keeps the
......@@ -896,7 +892,7 @@ pub(crate) struct KeyedReadDependency {
}
impl KeyedReadDependency {
/// Allocates a new [`TrackKey::Keyed`] dependency id for the current task execution. Returns
/// Allocates a new [`CellKey::Keyed`] dependency id for the current task execution. Returns
/// `None` when not called from within a task execution (e.g. a top-level `run`), where reads
/// are not tracked anyway, or when the ids of this execution are exhausted. In that case the
/// read is tracked as a normal (full) dependency.
......@@ -1745,12 +1741,23 @@ impl<B: Backend> Executor<TurboTasks<B>, ScheduledTask, TaskPriority> for TurboT
};
let finished_state = this.finish_current_task_state();
let cell_counters = CURRENT_TASK_STATE
.with(|ts| ts.write().unwrap().cell_counters.take().unwrap());
let (cell_counters, has_keyed_reads) =
CURRENT_TASK_STATE.with(|ts| {
let cell_counters =
ts.write().unwrap().cell_counters.take().unwrap();
// All reads of this execution are done (local tasks
// were awaited above), so the backend only needs to
// look for remaining `CellKey::Keyed` dependencies when
// `keyed()` was used.
let has_keyed_reads =
ts.inner.keyed_read_counter.load(Ordering::Relaxed) > 0;
(cell_counters, has_keyed_reads)
});
this.backend.task_execution_completed(
task_id,
result,
&cell_counters,
has_keyed_reads,
#[cfg(feature = "verify_determinism")]
finished_state.stateful,
finished_state.has_invalidator,
......
......@@ -26,9 +26,9 @@ use crate::{
/// change. The hashes of the accessed keys are recorded. When it is dropped within the task
/// execution that read it, that dependency is replaced (in a single batch operation) with
/// dependencies on only the accessed keys, so the task is only invalidated when one of the
/// accessed keys changes, like with [`Vc::get`](crate::Vc::get). When it is dropped outside of
/// that task execution (e.g. moved to another thread or task), the dependency on the whole cell
/// is kept.
/// accessed keys changes, like with [`Vc::get`](crate::Vc::get). When it is still alive at the end
/// of that task execution or dropped outside of it (e.g. moved to another thread or task), the
/// dependency is converted to a normal dependency on the whole cell when the execution completes.
///
/// Note: Like with [`Vc::contains_key`](crate::Vc::contains_key), a
/// [`contains_key`](KeyedReadRef::contains_key) access is also invalidated when the value of the
......
......@@ -25,9 +25,9 @@ use crate::{
event::EventListener,
id::{ExecutionId, LocalTaskId, TASK_ID_MAX},
manager::{
ReadCellTracking, ReadTracking, SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK,
ScheduleKey, TrackKey, TurboTasksApi, execute_read_target_inline, read_local_output,
with_turbo_tasks,
CellKey, ReadCellTracking, ReadTracking,
SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK, ScheduleKey, TurboTasksApi,
execute_read_target_inline, read_local_output, with_turbo_tasks,
},
read_options::ReadOutcome,
registry::get_value_type,
......@@ -770,12 +770,12 @@ impl ReadRawVcFuture {
/// Track the value as a dependency with an key.
pub fn track_with_key(mut self, key: u64) -> Self {
self.read_cell_options.tracking = ReadCellTracking::Tracked {
key: TrackKey::Key(key),
key: Some(CellKey::Hash(key)),
};
self.map_resolve(|r| r.track_with_key())
}
/// Track the value as a [`TrackKey::Keyed`] dependency of a [`Vc::keyed`](crate::Vc::keyed)
/// Track the value as a [`CellKey::Keyed`] dependency of a [`Vc::keyed`](crate::Vc::keyed)
/// read. Like [`ReadRawVcFuture::track_with_key`], this also tracks the reads of task outputs
/// while resolving the [`RawVc`].
///
......@@ -787,20 +787,18 @@ impl ReadRawVcFuture {
};
resolve.read_output_options.tracking = ReadTracking::Tracked;
self.read_cell_options.tracking = ReadCellTracking::Tracked {
key: TrackKey::Keyed(id),
key: Some(CellKey::Keyed(id)),
};
}
/// Track the remaining reads as normal full dependencies, e.g. when a [`TrackKey::Keyed`] read
/// Track the remaining reads as normal full dependencies, e.g. when a [`CellKey::Keyed`] read
/// continues in a different task execution than the one that allocated its id. Can be called
/// at any time before the future completes.
pub(crate) fn set_track_full(&mut self) {
if let ReadRawVcState::Resolving(resolve) = &mut self.state {
resolve.read_output_options.tracking = ReadTracking::Tracked;
}
self.read_cell_options.tracking = ReadCellTracking::Tracked {
key: TrackKey::None,
};
self.read_cell_options.tracking = ReadCellTracking::Tracked { key: None };
}
/// The task cell the [`RawVc`] was resolved to. Only available once the future has completed
......
......@@ -349,7 +349,7 @@ where
/// created the future).
enum KeyedDependencyState {
NotStarted,
/// `None` if the read isn't a [`TrackKey::Keyed`](crate::TrackKey::Keyed) read (no current
/// `None` if the read isn't a [`CellKey::Keyed`](crate::CellKey::Keyed) read (no current
/// task, or the dependency was already handed to the [`KeyedReadRef`]).
Started(Option<KeyedReadDependency>),
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment