Unverified Commit f2bfe7fc authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Optimize cell_dependency representation (#94717)

Split the representtation of cell dependencies to split `hashed` and
`non-hashed` dependencies into separate structures

Hashed deps are rare but the bloat the size of the edge representation
so a distinct field can optimize for the common case.


| | canary | branch | delta | persistence disabled | gap |
-------------------------|----------|----------|------------------|
---------------------- | ---------|
| maxRSS median | 16.74 GB | 15.69 GB | −1.05 GB (−6.2%) | 13.86 GB |
1.83 GB |
| maxRSS max (worst run) | 18.14 GB | 16.19 GB | −1.95 GB | 14.16 GB |
2.03 GB |

This closes 1/3 of the gap from enabling persistence. 

Time metrics were ~neutral but i didn't take the time to quiet my
machine. I would assume there is a small perf benefit from this
approach, but memory is the goal
parent 658d3786
...@@ -70,8 +70,8 @@ use crate::{ ...@@ -70,8 +70,8 @@ use crate::{
}, },
backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash}, backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash},
data::{ data::{
ActivenessState, CellDependency, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressCellState, InProgressState, InProgressStateInner, OutputValue, TransientTask, InProgressState, InProgressStateInner, OutputValue, TransientTask,
}, },
error::TaskError, error::TaskError,
kv_backing_storage::TurboBackingStorage, kv_backing_storage::TurboBackingStorage,
...@@ -453,7 +453,6 @@ impl TurboTasksBackend { ...@@ -453,7 +453,6 @@ impl TurboTasksBackend {
let mut ctx = self.execute_context(turbo_tasks); let mut ctx = self.execute_context(turbo_tasks);
let need_reader_task = if self.should_track_dependencies() let need_reader_task = if self.should_track_dependencies()
&& !matches!(options.tracking, ReadTracking::Untracked) && !matches!(options.tracking, ReadTracking::Untracked)
&& reader.is_some_and(|reader_id| reader_id != task_id)
&& let Some(reader_id) = reader && let Some(reader_id) = reader
&& reader_id != task_id && reader_id != task_id
{ {
...@@ -794,8 +793,12 @@ impl TurboTasksBackend { ...@@ -794,8 +793,12 @@ impl TurboTasksBackend {
&& (!task.immutable() || cfg!(feature = "verify_immutable")) && (!task.immutable() || cfg!(feature = "verify_immutable"))
{ {
let reader = reader.unwrap(); let reader = reader.unwrap();
let _ = task let reverse = CellRef { task: reader, cell };
.add_cell_dependents(CellDependency::new(CellRef { task: reader, cell }, key)); if let Some(k) = key {
let _ = task.add_cell_dependents_hashed((reverse, k));
} else {
let _ = task.add_cell_dependents(reverse);
}
drop(task); drop(task);
// Note: We use `task_pair` earlier to lock the task and its reader at the same // Note: We use `task_pair` earlier to lock the task and its reader at the same
...@@ -807,9 +810,12 @@ impl TurboTasksBackend { ...@@ -807,9 +810,12 @@ impl TurboTasksBackend {
task: task_id, task: task_id,
cell, cell,
}; };
let dep = CellDependency::new(target, key); if let Some(k) = key {
if !reader_task.remove_outdated_cell_dependencies(&dep) { if !reader_task.remove_outdated_cell_dependencies_hashed(&(target, k)) {
let _ = reader_task.add_cell_dependencies(dep); let _ = reader_task.add_cell_dependencies_hashed((target, k));
}
} else if !reader_task.remove_outdated_cell_dependencies(&target) {
let _ = reader_task.add_cell_dependencies(target);
} }
drop(reader_task); drop(reader_task);
} }
...@@ -1922,8 +1928,13 @@ impl TurboTasksBackend { ...@@ -1922,8 +1928,13 @@ impl TurboTasksBackend {
if self.should_track_dependencies() { if self.should_track_dependencies() {
// Make all dependencies outdated // Make all dependencies outdated
// We copy to outdated which allows us to detect when we drop a dependency, but we
// keep existing dependencies in place to avoid redoing the
// dependency registration work if we re-read them
let cell_dependencies = task.iter_cell_dependencies().collect(); let cell_dependencies = task.iter_cell_dependencies().collect();
task.set_outdated_cell_dependencies(cell_dependencies); task.set_outdated_cell_dependencies(cell_dependencies);
let cell_dependencies_hashed = task.iter_cell_dependencies_hashed().collect();
task.set_outdated_cell_dependencies_hashed(cell_dependencies_hashed);
let outdated_output_dependencies = task.iter_output_dependencies().collect(); let outdated_output_dependencies = task.iter_output_dependencies().collect();
task.set_outdated_output_dependencies(outdated_output_dependencies); task.set_outdated_output_dependencies(outdated_output_dependencies);
...@@ -2259,9 +2270,10 @@ impl TurboTasksBackend { ...@@ -2259,9 +2270,10 @@ impl TurboTasksBackend {
&& task.is_collectibles_dependencies_empty() && task.is_collectibles_dependencies_empty()
{ {
Some( Some(
// Collect all dependencies on tasks to check if all dependencies are immutable // Collect all dependencies on tasks to check if all dependencies are immutable.
task.iter_output_dependencies() task.iter_output_dependencies()
.chain(task.iter_cell_dependencies().map(|dep| dep.cell_ref().task)) .chain(task.iter_cell_dependencies().map(|r| r.task))
.chain(task.iter_cell_dependencies_hashed().map(|(r, _)| r.task))
.collect::<FxHashSet<_>>(), .collect::<FxHashSet<_>>(),
) )
} else { } else {
...@@ -2302,6 +2314,10 @@ impl TurboTasksBackend { ...@@ -2302,6 +2314,10 @@ impl TurboTasksBackend {
task.iter_outdated_cell_dependencies() task.iter_outdated_cell_dependencies()
.map(OutdatedEdge::CellDependency), .map(OutdatedEdge::CellDependency),
); );
old_edges.extend(
task.iter_outdated_cell_dependencies_hashed()
.map(|(r, k)| OutdatedEdge::HashedCellDependency(r, k)),
);
old_edges.extend( old_edges.extend(
task.iter_outdated_output_dependencies() task.iter_outdated_output_dependencies()
.map(OutdatedEdge::OutputDependency), .map(OutdatedEdge::OutputDependency),
......
...@@ -17,7 +17,7 @@ use crate::{ ...@@ -17,7 +17,7 @@ use crate::{
}, },
storage_schema::TaskStorageAccessors, storage_schema::TaskStorageAccessors,
}, },
data::{CellDependency, CellRef, CollectibleRef, CollectiblesRef}, data::{CellRef, CollectibleRef, CollectiblesRef},
}; };
#[derive(Encode, Decode, Clone)] #[derive(Encode, Decode, Clone)]
...@@ -48,7 +48,8 @@ impl Default for CleanupOldEdgesOperation { ...@@ -48,7 +48,8 @@ impl Default for CleanupOldEdgesOperation {
pub enum OutdatedEdge { pub enum OutdatedEdge {
Child(TaskId), Child(TaskId),
Collectible(CollectibleRef, i32), Collectible(CollectibleRef, i32),
CellDependency(CellDependency), CellDependency(CellRef),
HashedCellDependency(CellRef, u64),
OutputDependency(TaskId), OutputDependency(TaskId),
CollectiblesDependency(CollectiblesRef), CollectiblesDependency(CollectiblesRef),
} }
...@@ -166,17 +167,32 @@ impl CleanupOldEdgesOperation { ...@@ -166,17 +167,32 @@ impl CleanupOldEdgesOperation {
AggregatedDataUpdate::new().collectibles_update(collectibles), AggregatedDataUpdate::new().collectibles_update(collectibles),
)); ));
} }
OutdatedEdge::CellDependency(dep) => { OutdatedEdge::CellDependency(forward) => {
let ( let CellRef {
CellRef { task: cell_task_id,
task: cell_task_id, cell,
} = forward;
{
let mut task = ctx.task(cell_task_id, TaskDataCategory::Data);
task.remove_cell_dependents(&CellRef {
task: task_id,
cell, cell,
}, });
key, }
) = dep.into_parts(); {
let mut task = ctx.task(task_id, TaskDataCategory::Data);
task.remove_cell_dependencies(&forward);
}
}
OutdatedEdge::HashedCellDependency(forward, key) => {
// ame as above but in the `_hashed` sets.
let CellRef {
task: cell_task_id,
cell,
} = forward;
{ {
let mut task = ctx.task(cell_task_id, TaskDataCategory::Data); let mut task = ctx.task(cell_task_id, TaskDataCategory::Data);
task.remove_cell_dependents(&CellDependency::new( task.remove_cell_dependents_hashed(&(
CellRef { CellRef {
task: task_id, task: task_id,
cell, cell,
...@@ -186,7 +202,7 @@ impl CleanupOldEdgesOperation { ...@@ -186,7 +202,7 @@ impl CleanupOldEdgesOperation {
} }
{ {
let mut task = ctx.task(task_id, TaskDataCategory::Data); let mut task = ctx.task(task_id, TaskDataCategory::Data);
task.remove_cell_dependencies(&dep); task.remove_cell_dependencies_hashed(&(forward, key));
} }
} }
OutdatedEdge::OutputDependency(output_task_id) => { OutdatedEdge::OutputDependency(output_task_id) => {
......
...@@ -1382,7 +1382,11 @@ impl TaskGuard for TaskGuardImpl<'_> { ...@@ -1382,7 +1382,11 @@ impl TaskGuard for TaskGuardImpl<'_> {
.map(|target| (target, TaskDataCategory::Meta)) .map(|target| (target, TaskDataCategory::Meta))
.chain( .chain(
self.iter_cell_dependencies() self.iter_cell_dependencies()
.map(|dep| (dep.cell_ref().task, TaskDataCategory::All)), .map(|r| (r.task, TaskDataCategory::All)),
)
.chain(
self.iter_cell_dependencies_hashed()
.map(|(r, _)| (r.task, TaskDataCategory::All)),
) )
.chain( .chain(
self.iter_collectibles_dependencies() self.iter_collectibles_dependencies()
......
...@@ -20,7 +20,7 @@ use crate::{ ...@@ -20,7 +20,7 @@ use crate::{
}, },
storage_schema::TaskStorageAccessors, storage_schema::TaskStorageAccessors,
}, },
data::{CellDependency, CellRef}, data::CellRef,
}; };
#[derive(Encode, Decode, Clone, Default)] #[derive(Encode, Decode, Clone, Default)]
...@@ -132,29 +132,43 @@ impl UpdateCellOperation { ...@@ -132,29 +132,43 @@ impl UpdateCellOperation {
}); });
// Collect dependent tasks only when not skipping invalidation. // Collect dependent tasks only when not skipping invalidation.
// The iterator borrows from `task`, so it must be scoped to drop before // The iterators borrow from `task`, so they must be scoped to drop before
// we mutably borrow `task` again in the fast path. // we mutably borrow `task` again in the fast path.
let mut dependent_tasks: FxIndexMap<TaskId, SmallVec<[Option<u64>; 2]>> = let mut dependent_tasks: FxIndexMap<TaskId, SmallVec<[Option<u64>; 2]>> =
FxIndexMap::default(); FxIndexMap::default();
if !skip_invalidation { if !skip_invalidation {
let tasks_with_keys = task.iter_cell_dependents().filter_map(|dep| { // Keyless dependents: always invalidate when the cell matches.
let ( for CellRef {
CellRef { task: dependent_task,
task: dependent_task, cell: dependent_cell,
cell: dependent_cell, } in task.iter_cell_dependents()
}, {
key, if dependent_cell == cell {
) = dep.into_parts(); dependent_tasks
(dependent_cell == cell .entry(dependent_task)
&& key.is_none_or(|key_hash| { .or_default()
updated_key_hashes_set .push(None);
.as_ref() }
.is_none_or(|set| set.contains(&key_hash)) }
})) // Keyed dependents: invalidate only when the changed sub-value (key) matches.
.then_some((dependent_task, key)) for (
}); CellRef {
for (task, key) in tasks_with_keys { task: dependent_task,
dependent_tasks.entry(task).or_default().push(key); cell: dependent_cell,
},
key_hash,
) in task.iter_cell_dependents_hashed()
{
if dependent_cell == cell
&& updated_key_hashes_set
.as_ref()
.is_none_or(|set| set.contains(&key_hash))
{
dependent_tasks
.entry(dependent_task)
.or_default()
.push(Some(key_hash));
}
} }
} }
...@@ -281,15 +295,27 @@ impl Operation for UpdateCellOperation { ...@@ -281,15 +295,27 @@ impl Operation for UpdateCellOperation {
let mut make_stale = false; let mut make_stale = false;
let dependent = ctx.task(dependent_task_id, TaskDataCategory::All); let dependent = ctx.task(dependent_task_id, TaskDataCategory::All);
for key in keys.iter().copied() { for key in keys.iter().copied() {
let dep = CellDependency::new(cell_ref, key); let (in_outdated, in_current) = if let Some(k) = key {
if dependent.outdated_cell_dependencies_contains(&dep) { let in_outdated = dependent
.outdated_cell_dependencies_hashed_contains(&(cell_ref, k));
let in_current = !in_outdated
&& dependent.cell_dependencies_hashed_contains(&(cell_ref, k));
(in_outdated, in_current)
} else {
let in_outdated =
dependent.outdated_cell_dependencies_contains(&cell_ref);
let in_current =
!in_outdated && dependent.cell_dependencies_contains(&cell_ref);
(in_outdated, in_current)
};
if in_outdated {
// cell dependency is outdated, so it hasn't read the cell yet // cell dependency is outdated, so it hasn't read the cell yet
// and doesn't need to be invalidated. // and doesn't need to be invalidated.
// We do not need to make the task stale in this case. // We do not need to make the task stale in this case.
// But importantly we still need to make the task dirty as it should // But importantly we still need to make the task dirty as it should
// no longer be considered as // no longer be considered as
// "recomputation". // "recomputation".
} else if !dependent.cell_dependencies_contains(&dep) { } else if !in_current {
// cell dependency has been removed, so the task doesn't depend on // cell dependency has been removed, so the task doesn't depend on
// the cell anymore and doesn't need // the cell anymore and doesn't need
// to be invalidated // to be invalidated
......
...@@ -34,9 +34,8 @@ use turbo_tasks::{ ...@@ -34,9 +34,8 @@ use turbo_tasks::{
use crate::{ use crate::{
backend::{cell_data::CellData, counter_map::CounterMap}, backend::{cell_data::CellData, counter_map::CounterMap},
data::{ data::{
ActivenessState, AggregationNumber, CellDependency, CollectibleRef, CollectiblesRef, ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness,
Dirtyness, InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask,
TransientTask,
}, },
}; };
...@@ -245,7 +244,9 @@ struct TaskStorageSchema { ...@@ -245,7 +244,9 @@ struct TaskStorageSchema {
)] )]
output_dependencies: AutoSet<TaskId, 6>, output_dependencies: AutoSet<TaskId, 6>,
/// Cells this task depends on. /// Cells this task depends on as a whole (keyless `CellDependency::All`). The common case;
/// kept separate from `cell_dependencies_hashed` so it stores a bare `CellRef` instead of the
/// wider `CellDependency` enum (no tag, no `u64`).
#[field( #[field(
storage = "auto_set", storage = "auto_set",
category = "data", category = "data",
...@@ -253,7 +254,17 @@ struct TaskStorageSchema { ...@@ -253,7 +254,17 @@ struct TaskStorageSchema {
shrink_on_completion, shrink_on_completion,
drop_on_completion_if_immutable drop_on_completion_if_immutable
)] )]
cell_dependencies: AutoSet<CellDependency, 1>, cell_dependencies: AutoSet<CellRef, 2>,
/// Cells this task depends on, narrowed to a hashed sub-value (`CellDependency::Hash`). Rare.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Collectibles this task depends on. /// Collectibles this task depends on.
#[field( #[field(
...@@ -269,9 +280,13 @@ struct TaskStorageSchema { ...@@ -269,9 +280,13 @@ struct TaskStorageSchema {
#[field(storage = "auto_set", category = "transient", shrink_on_completion)] #[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_output_dependencies: AutoSet<TaskId, 6>, outdated_output_dependencies: AutoSet<TaskId, 6>,
/// Outdated cell dependencies to be cleaned up (transient). /// Outdated keyless cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_cell_dependencies: AutoSet<CellRef, 2>,
/// Outdated hashed cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)] #[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_cell_dependencies: AutoSet<CellDependency, 1>, outdated_cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Outdated collectibles dependencies to be cleaned up (transient). /// Outdated collectibles dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", shrink_on_completion)] #[field(storage = "auto_set", category = "transient", shrink_on_completion)]
...@@ -280,13 +295,26 @@ struct TaskStorageSchema { ...@@ -280,13 +295,26 @@ struct TaskStorageSchema {
// ========================================================================= // =========================================================================
// DEPENDENTS - Tasks that depend on this task's cells // DEPENDENTS - Tasks that depend on this task's cells
// ========================================================================= // =========================================================================
/// Tasks that depend on this task's cells as a whole (keyless). Reverse of
/// `cell_dependencies`. In a `cell_dependents` entry the `CellRef.task` field holds the
/// DEPENDENT task's id and `CellRef.cell` is this task's cell (see `CellDependency` docs).
#[field(
storage = "auto_set",
category = "data",
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents: AutoSet<CellRef, 2>,
/// Tasks that depend on a hashed sub-value of this task's cells. Reverse of
/// `cell_dependencies_hashed`.
#[field( #[field(
storage = "auto_set", storage = "auto_set",
category = "data", category = "data",
filter_transient, filter_transient,
drop_on_completion_if_immutable drop_on_completion_if_immutable
)] )]
cell_dependents: AutoSet<CellDependency, 1>, cell_dependents_hashed: AutoSet<(CellRef, u64), 1>,
/// Tasks that depend on collectibles of a specific type from this task. /// Tasks that depend on collectibles of a specific type from this task.
/// Maps TraitTypeId -> Set<TaskId> /// Maps TraitTypeId -> Set<TaskId>
...@@ -850,9 +878,14 @@ impl IsTransient for (TraitTypeId, TaskId) { ...@@ -850,9 +878,14 @@ impl IsTransient for (TraitTypeId, TaskId) {
self.1.is_transient() self.1.is_transient()
} }
} }
impl IsTransient for CellDependency { impl IsTransient for CellRef {
fn is_transient(&self) -> bool {
CellRef::is_transient(self)
}
}
impl IsTransient for (CellRef, u64) {
fn is_transient(&self) -> bool { fn is_transient(&self) -> bool {
CellDependency::is_transient(self) self.0.is_transient()
} }
} }
...@@ -957,7 +990,7 @@ mod tests { ...@@ -957,7 +990,7 @@ mod tests {
use turbo_tasks::{CellId, TaskId}; use turbo_tasks::{CellId, TaskId};
use super::*; use super::*;
use crate::data::{AggregationNumber, CellDependency, CellRef, Dirtyness, OutputValue}; use crate::data::{AggregationNumber, CellRef, Dirtyness, OutputValue};
#[test] #[test]
fn test_accessors() { fn test_accessors() {
...@@ -1243,15 +1276,13 @@ mod tests { ...@@ -1243,15 +1276,13 @@ mod tests {
original original
.output_dependencies_mut() .output_dependencies_mut()
.insert(TaskId::new(200).unwrap()); .insert(TaskId::new(200).unwrap());
original original.cell_dependencies_mut().insert(CellRef {
.cell_dependencies_mut() task: TaskId::new(1).unwrap(),
.insert(CellDependency::All(CellRef { cell: CellId {
task: TaskId::new(1).unwrap(), type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
cell: CellId { index: 0,
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) }, },
index: 0, });
},
}));
// Set lazy data transient field (should NOT be serialized) // Set lazy data transient field (should NOT be serialized)
original original
...@@ -1390,15 +1421,13 @@ mod tests { ...@@ -1390,15 +1421,13 @@ mod tests {
storage.output_dependent_mut().insert(transient_task(3)); storage.output_dependent_mut().insert(transient_task(3));
// Lazy filter_transient data field. // Lazy filter_transient data field.
storage storage.cell_dependencies_mut().insert(CellRef {
.cell_dependencies_mut() task: persistent_task(10),
.insert(CellDependency::All(CellRef { cell: CellId {
task: persistent_task(10), type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
cell: CellId { index: 0,
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) }, },
index: 0, });
},
}));
// Mark as restored so the task is eligible for dropping. // Mark as restored so the task is eligible for dropping.
storage.flags.set_data_restored(true); storage.flags.set_data_restored(true);
......
...@@ -79,49 +79,6 @@ impl CollectiblesRef { ...@@ -79,49 +79,6 @@ impl CollectiblesRef {
} }
} }
/// An edge between a [`CellRef`] and a task, optionally narrowed by a hashed sub-key.
///
/// Used both as a forward and reverse edge:
/// - In `cell_dependencies`, the [`CellRef`] is the cell another task owns that this task depends
/// on.
/// - In `cell_dependents`, the [`CellRef`]'s `task` is the dependent task and `cell` is the cell of
/// the storing task; the `task` field is reused as the dependent's id rather than the cell's
/// owning task. The fields encode the same bits either way.
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Encode, Decode)]
pub enum CellDependency {
/// Depend on the cell as a whole.
All(CellRef),
/// Depend only on the sub-value identified by this hash key.
Hash(CellRef, u64),
}
impl CellDependency {
pub fn cell_ref(&self) -> CellRef {
match *self {
CellDependency::All(c) | CellDependency::Hash(c, _) => c,
}
}
/// Decompose into the underlying `(CellRef, Option<u64>)` in a single match.
pub fn into_parts(self) -> (CellRef, Option<u64>) {
match self {
CellDependency::All(c) => (c, None),
CellDependency::Hash(c, k) => (c, Some(k)),
}
}
pub fn new(cell_ref: CellRef, key: Option<u64>) -> Self {
match key {
None => CellDependency::All(cell_ref),
Some(k) => CellDependency::Hash(cell_ref, k),
}
}
pub fn is_transient(&self) -> bool {
self.cell_ref().is_transient()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)] #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
pub enum OutputValue { pub enum OutputValue {
Cell(CellRef), Cell(CellRef),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment