Unverified Commit f2bfe7fc authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Optimize cell_dependency representation (#94717)

Split the representtation of cell dependencies to split `hashed` and
`non-hashed` dependencies into separate structures

Hashed deps are rare but the bloat the size of the edge representation
so a distinct field can optimize for the common case.


| | canary | branch | delta | persistence disabled | gap |
-------------------------|----------|----------|------------------|
---------------------- | ---------|
| maxRSS median | 16.74 GB | 15.69 GB | −1.05 GB (−6.2%) | 13.86 GB |
1.83 GB |
| maxRSS max (worst run) | 18.14 GB | 16.19 GB | −1.95 GB | 14.16 GB |
2.03 GB |

This closes 1/3 of the gap from enabling persistence. 

Time metrics were ~neutral but i didn't take the time to quiet my
machine. I would assume there is a small perf benefit from this
approach, but memory is the goal
parent 658d3786
......@@ -70,8 +70,8 @@ use crate::{
},
backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash},
data::{
ActivenessState, CellDependency, CellRef, CollectibleRef, CollectiblesRef, Dirtyness,
InProgressCellState, InProgressState, InProgressStateInner, OutputValue, TransientTask,
ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressState, InProgressStateInner, OutputValue, TransientTask,
},
error::TaskError,
kv_backing_storage::TurboBackingStorage,
......@@ -453,7 +453,6 @@ impl TurboTasksBackend {
let mut ctx = self.execute_context(turbo_tasks);
let need_reader_task = if self.should_track_dependencies()
&& !matches!(options.tracking, ReadTracking::Untracked)
&& reader.is_some_and(|reader_id| reader_id != task_id)
&& let Some(reader_id) = reader
&& reader_id != task_id
{
......@@ -794,8 +793,12 @@ impl TurboTasksBackend {
&& (!task.immutable() || cfg!(feature = "verify_immutable"))
{
let reader = reader.unwrap();
let _ = task
.add_cell_dependents(CellDependency::new(CellRef { task: reader, cell }, key));
let reverse = CellRef { task: reader, cell };
if let Some(k) = key {
let _ = task.add_cell_dependents_hashed((reverse, k));
} else {
let _ = task.add_cell_dependents(reverse);
}
drop(task);
// Note: We use `task_pair` earlier to lock the task and its reader at the same
......@@ -807,9 +810,12 @@ impl TurboTasksBackend {
task: task_id,
cell,
};
let dep = CellDependency::new(target, key);
if !reader_task.remove_outdated_cell_dependencies(&dep) {
let _ = reader_task.add_cell_dependencies(dep);
if let Some(k) = key {
if !reader_task.remove_outdated_cell_dependencies_hashed(&(target, k)) {
let _ = reader_task.add_cell_dependencies_hashed((target, k));
}
} else if !reader_task.remove_outdated_cell_dependencies(&target) {
let _ = reader_task.add_cell_dependencies(target);
}
drop(reader_task);
}
......@@ -1922,8 +1928,13 @@ impl TurboTasksBackend {
if self.should_track_dependencies() {
// Make all dependencies outdated
// We copy to outdated which allows us to detect when we drop a dependency, but we
// keep existing dependencies in place to avoid redoing the
// dependency registration work if we re-read them
let cell_dependencies = task.iter_cell_dependencies().collect();
task.set_outdated_cell_dependencies(cell_dependencies);
let cell_dependencies_hashed = task.iter_cell_dependencies_hashed().collect();
task.set_outdated_cell_dependencies_hashed(cell_dependencies_hashed);
let outdated_output_dependencies = task.iter_output_dependencies().collect();
task.set_outdated_output_dependencies(outdated_output_dependencies);
......@@ -2259,9 +2270,10 @@ impl TurboTasksBackend {
&& task.is_collectibles_dependencies_empty()
{
Some(
// Collect all dependencies on tasks to check if all dependencies are immutable
// Collect all dependencies on tasks to check if all dependencies are immutable.
task.iter_output_dependencies()
.chain(task.iter_cell_dependencies().map(|dep| dep.cell_ref().task))
.chain(task.iter_cell_dependencies().map(|r| r.task))
.chain(task.iter_cell_dependencies_hashed().map(|(r, _)| r.task))
.collect::<FxHashSet<_>>(),
)
} else {
......@@ -2302,6 +2314,10 @@ impl TurboTasksBackend {
task.iter_outdated_cell_dependencies()
.map(OutdatedEdge::CellDependency),
);
old_edges.extend(
task.iter_outdated_cell_dependencies_hashed()
.map(|(r, k)| OutdatedEdge::HashedCellDependency(r, k)),
);
old_edges.extend(
task.iter_outdated_output_dependencies()
.map(OutdatedEdge::OutputDependency),
......
......@@ -17,7 +17,7 @@ use crate::{
},
storage_schema::TaskStorageAccessors,
},
data::{CellDependency, CellRef, CollectibleRef, CollectiblesRef},
data::{CellRef, CollectibleRef, CollectiblesRef},
};
#[derive(Encode, Decode, Clone)]
......@@ -48,7 +48,8 @@ impl Default for CleanupOldEdgesOperation {
pub enum OutdatedEdge {
Child(TaskId),
Collectible(CollectibleRef, i32),
CellDependency(CellDependency),
CellDependency(CellRef),
HashedCellDependency(CellRef, u64),
OutputDependency(TaskId),
CollectiblesDependency(CollectiblesRef),
}
......@@ -166,17 +167,32 @@ impl CleanupOldEdgesOperation {
AggregatedDataUpdate::new().collectibles_update(collectibles),
));
}
OutdatedEdge::CellDependency(dep) => {
let (
CellRef {
task: cell_task_id,
OutdatedEdge::CellDependency(forward) => {
let CellRef {
task: cell_task_id,
cell,
} = forward;
{
let mut task = ctx.task(cell_task_id, TaskDataCategory::Data);
task.remove_cell_dependents(&CellRef {
task: task_id,
cell,
},
key,
) = dep.into_parts();
});
}
{
let mut task = ctx.task(task_id, TaskDataCategory::Data);
task.remove_cell_dependencies(&forward);
}
}
OutdatedEdge::HashedCellDependency(forward, key) => {
// ame as above but in the `_hashed` sets.
let CellRef {
task: cell_task_id,
cell,
} = forward;
{
let mut task = ctx.task(cell_task_id, TaskDataCategory::Data);
task.remove_cell_dependents(&CellDependency::new(
task.remove_cell_dependents_hashed(&(
CellRef {
task: task_id,
cell,
......@@ -186,7 +202,7 @@ impl CleanupOldEdgesOperation {
}
{
let mut task = ctx.task(task_id, TaskDataCategory::Data);
task.remove_cell_dependencies(&dep);
task.remove_cell_dependencies_hashed(&(forward, key));
}
}
OutdatedEdge::OutputDependency(output_task_id) => {
......
......@@ -1382,7 +1382,11 @@ impl TaskGuard for TaskGuardImpl<'_> {
.map(|target| (target, TaskDataCategory::Meta))
.chain(
self.iter_cell_dependencies()
.map(|dep| (dep.cell_ref().task, TaskDataCategory::All)),
.map(|r| (r.task, TaskDataCategory::All)),
)
.chain(
self.iter_cell_dependencies_hashed()
.map(|(r, _)| (r.task, TaskDataCategory::All)),
)
.chain(
self.iter_collectibles_dependencies()
......
......@@ -20,7 +20,7 @@ use crate::{
},
storage_schema::TaskStorageAccessors,
},
data::{CellDependency, CellRef},
data::CellRef,
};
#[derive(Encode, Decode, Clone, Default)]
......@@ -132,29 +132,43 @@ impl UpdateCellOperation {
});
// Collect dependent tasks only when not skipping invalidation.
// The iterator borrows from `task`, so it must be scoped to drop before
// The iterators borrow from `task`, so they must be scoped to drop before
// we mutably borrow `task` again in the fast path.
let mut dependent_tasks: FxIndexMap<TaskId, SmallVec<[Option<u64>; 2]>> =
FxIndexMap::default();
if !skip_invalidation {
let tasks_with_keys = task.iter_cell_dependents().filter_map(|dep| {
let (
CellRef {
task: dependent_task,
cell: dependent_cell,
},
key,
) = dep.into_parts();
(dependent_cell == cell
&& key.is_none_or(|key_hash| {
updated_key_hashes_set
.as_ref()
.is_none_or(|set| set.contains(&key_hash))
}))
.then_some((dependent_task, key))
});
for (task, key) in tasks_with_keys {
dependent_tasks.entry(task).or_default().push(key);
// Keyless dependents: always invalidate when the cell matches.
for CellRef {
task: dependent_task,
cell: dependent_cell,
} in task.iter_cell_dependents()
{
if dependent_cell == cell {
dependent_tasks
.entry(dependent_task)
.or_default()
.push(None);
}
}
// Keyed dependents: invalidate only when the changed sub-value (key) matches.
for (
CellRef {
task: dependent_task,
cell: dependent_cell,
},
key_hash,
) in task.iter_cell_dependents_hashed()
{
if dependent_cell == cell
&& updated_key_hashes_set
.as_ref()
.is_none_or(|set| set.contains(&key_hash))
{
dependent_tasks
.entry(dependent_task)
.or_default()
.push(Some(key_hash));
}
}
}
......@@ -281,15 +295,27 @@ impl Operation for UpdateCellOperation {
let mut make_stale = false;
let dependent = ctx.task(dependent_task_id, TaskDataCategory::All);
for key in keys.iter().copied() {
let dep = CellDependency::new(cell_ref, key);
if dependent.outdated_cell_dependencies_contains(&dep) {
let (in_outdated, in_current) = if let Some(k) = key {
let in_outdated = dependent
.outdated_cell_dependencies_hashed_contains(&(cell_ref, k));
let in_current = !in_outdated
&& dependent.cell_dependencies_hashed_contains(&(cell_ref, k));
(in_outdated, in_current)
} else {
let in_outdated =
dependent.outdated_cell_dependencies_contains(&cell_ref);
let in_current =
!in_outdated && dependent.cell_dependencies_contains(&cell_ref);
(in_outdated, in_current)
};
if in_outdated {
// cell dependency is outdated, so it hasn't read the cell yet
// and doesn't need to be invalidated.
// We do not need to make the task stale in this case.
// But importantly we still need to make the task dirty as it should
// no longer be considered as
// "recomputation".
} else if !dependent.cell_dependencies_contains(&dep) {
} else if !in_current {
// cell dependency has been removed, so the task doesn't depend on
// the cell anymore and doesn't need
// to be invalidated
......
......@@ -34,9 +34,8 @@ use turbo_tasks::{
use crate::{
backend::{cell_data::CellData, counter_map::CounterMap},
data::{
ActivenessState, AggregationNumber, CellDependency, CollectibleRef, CollectiblesRef,
Dirtyness, InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType,
TransientTask,
ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness,
InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask,
},
};
......@@ -245,7 +244,9 @@ struct TaskStorageSchema {
)]
output_dependencies: AutoSet<TaskId, 6>,
/// Cells this task depends on.
/// Cells this task depends on as a whole (keyless `CellDependency::All`). The common case;
/// kept separate from `cell_dependencies_hashed` so it stores a bare `CellRef` instead of the
/// wider `CellDependency` enum (no tag, no `u64`).
#[field(
storage = "auto_set",
category = "data",
......@@ -253,7 +254,17 @@ struct TaskStorageSchema {
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies: AutoSet<CellDependency, 1>,
cell_dependencies: AutoSet<CellRef, 2>,
/// Cells this task depends on, narrowed to a hashed sub-value (`CellDependency::Hash`). Rare.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Collectibles this task depends on.
#[field(
......@@ -269,9 +280,13 @@ struct TaskStorageSchema {
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_output_dependencies: AutoSet<TaskId, 6>,
/// Outdated cell dependencies to be cleaned up (transient).
/// Outdated keyless cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_cell_dependencies: AutoSet<CellRef, 2>,
/// Outdated hashed cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_cell_dependencies: AutoSet<CellDependency, 1>,
outdated_cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Outdated collectibles dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
......@@ -280,13 +295,26 @@ struct TaskStorageSchema {
// =========================================================================
// DEPENDENTS - Tasks that depend on this task's cells
// =========================================================================
/// Tasks that depend on this task's cells as a whole (keyless). Reverse of
/// `cell_dependencies`. In a `cell_dependents` entry the `CellRef.task` field holds the
/// DEPENDENT task's id and `CellRef.cell` is this task's cell (see `CellDependency` docs).
#[field(
storage = "auto_set",
category = "data",
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents: AutoSet<CellRef, 2>,
/// Tasks that depend on a hashed sub-value of this task's cells. Reverse of
/// `cell_dependencies_hashed`.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents: AutoSet<CellDependency, 1>,
cell_dependents_hashed: AutoSet<(CellRef, u64), 1>,
/// Tasks that depend on collectibles of a specific type from this task.
/// Maps TraitTypeId -> Set<TaskId>
......@@ -850,9 +878,14 @@ impl IsTransient for (TraitTypeId, TaskId) {
self.1.is_transient()
}
}
impl IsTransient for CellDependency {
impl IsTransient for CellRef {
fn is_transient(&self) -> bool {
CellRef::is_transient(self)
}
}
impl IsTransient for (CellRef, u64) {
fn is_transient(&self) -> bool {
CellDependency::is_transient(self)
self.0.is_transient()
}
}
......@@ -957,7 +990,7 @@ mod tests {
use turbo_tasks::{CellId, TaskId};
use super::*;
use crate::data::{AggregationNumber, CellDependency, CellRef, Dirtyness, OutputValue};
use crate::data::{AggregationNumber, CellRef, Dirtyness, OutputValue};
#[test]
fn test_accessors() {
......@@ -1243,15 +1276,13 @@ mod tests {
original
.output_dependencies_mut()
.insert(TaskId::new(200).unwrap());
original
.cell_dependencies_mut()
.insert(CellDependency::All(CellRef {
task: TaskId::new(1).unwrap(),
cell: CellId {
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
index: 0,
},
}));
original.cell_dependencies_mut().insert(CellRef {
task: TaskId::new(1).unwrap(),
cell: CellId {
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
index: 0,
},
});
// Set lazy data transient field (should NOT be serialized)
original
......@@ -1390,15 +1421,13 @@ mod tests {
storage.output_dependent_mut().insert(transient_task(3));
// Lazy filter_transient data field.
storage
.cell_dependencies_mut()
.insert(CellDependency::All(CellRef {
task: persistent_task(10),
cell: CellId {
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
index: 0,
},
}));
storage.cell_dependencies_mut().insert(CellRef {
task: persistent_task(10),
cell: CellId {
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
index: 0,
},
});
// Mark as restored so the task is eligible for dropping.
storage.flags.set_data_restored(true);
......
......@@ -79,49 +79,6 @@ impl CollectiblesRef {
}
}
/// An edge between a [`CellRef`] and a task, optionally narrowed by a hashed sub-key.
///
/// Used both as a forward and reverse edge:
/// - In `cell_dependencies`, the [`CellRef`] is the cell another task owns that this task depends
/// on.
/// - In `cell_dependents`, the [`CellRef`]'s `task` is the dependent task and `cell` is the cell of
/// the storing task; the `task` field is reused as the dependent's id rather than the cell's
/// owning task. The fields encode the same bits either way.
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Encode, Decode)]
pub enum CellDependency {
/// Depend on the cell as a whole.
All(CellRef),
/// Depend only on the sub-value identified by this hash key.
Hash(CellRef, u64),
}
impl CellDependency {
pub fn cell_ref(&self) -> CellRef {
match *self {
CellDependency::All(c) | CellDependency::Hash(c, _) => c,
}
}
/// Decompose into the underlying `(CellRef, Option<u64>)` in a single match.
pub fn into_parts(self) -> (CellRef, Option<u64>) {
match self {
CellDependency::All(c) => (c, None),
CellDependency::Hash(c, k) => (c, Some(k)),
}
}
pub fn new(cell_ref: CellRef, key: Option<u64>) -> Self {
match key {
None => CellDependency::All(cell_ref),
Some(k) => CellDependency::Hash(cell_ref, k),
}
}
pub fn is_transient(&self) -> bool {
self.cell_ref().is_transient()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
pub enum OutputValue {
Cell(CellRef),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment