Unverified Commit 08c4e58a authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Improve memory usage of the new backend (#74740)

### What?

Improves memory usage by changing the way data is stored in the new backend. Instead of keeping a map `enum CachedDataItemKey` -> `enum CachedDataItemValue`, it uses a Vec `enum CachedDataItemStorage` which keeps a single storage for every `enum CachedDataItemType`. The storage is strongly types and e. g. stores a map `TaskId` -> `()`, which makes it more memory efficient while still not wasting memory for unused types.

It also calls `shrink_to_fit` after task completion and `shrink_amortized` after removing items, to reduce the unnecessary capacity.
parent dd791274
pub trait Indexed {
type Index: Clone + PartialEq + Eq + std::hash::Hash;
fn index(&self) -> Self::Index;
}
......@@ -51,9 +51,9 @@ fn get_followers_with_aggregation_number(
aggregation_number: u32,
) -> Vec<TaskId> {
if is_aggregating_node(aggregation_number) {
get_many!(task, Follower { task } count if *count > 0 => *task)
get_many!(task, Follower { task } count if *count > 0 => task)
} else {
get_many!(task, Child { task } => *task)
get_many!(task, Child { task } => task)
}
}
......@@ -66,12 +66,12 @@ fn get_followers(task: &impl TaskGuard) -> Vec<TaskId> {
/// Returns a list of tasks that are considered as "upper" tasks of the task. The upper tasks are
/// aggregating over the task.
pub fn get_uppers(task: &impl TaskGuard) -> Vec<TaskId> {
get_many!(task, Upper { task } count if *count > 0 => *task)
get_many!(task, Upper { task } count if *count > 0 => task)
}
/// Returns an iterator of tasks that are considered as "upper" tasks of the task. See `get_uppers`
fn iter_uppers<'a>(task: &'a (impl TaskGuard + 'a)) -> impl Iterator<Item = TaskId> + 'a {
iter_many!(task, Upper { task } count if *count > 0 => *task)
iter_many!(task, Upper { task } count if *count > 0 => task)
}
/// Returns the aggregation number of the task.
......@@ -84,7 +84,7 @@ pub fn get_aggregation_number(task: &impl TaskGuard) -> u32 {
/// A job in the job queue for updating something in the aggregated graph.
#[derive(Serialize, Deserialize, Clone, Debug)]
pub enum AggregationUpdateJob {
/// Update the aggregation number of a task. This might result in balancing needed to update
/// Update the aggregation number of a task. This might result in balancing needed to update
/// "upper" and "follower" edges.
UpdateAggregationNumber {
task_id: TaskId,
......@@ -175,7 +175,7 @@ impl AggregatedDataUpdate {
let aggregation = get_aggregation_number(task);
let mut dirty_container_count = Default::default();
let mut collectibles_update: Vec<_> =
get_many!(task, Collectible { collectible } count => (*collectible, *count));
get_many!(task, Collectible { collectible } count => (collectible, *count));
if is_aggregating_node(aggregation) {
dirty_container_count = get!(task, AggregatedDirtyContainerCount)
.cloned()
......@@ -185,7 +185,7 @@ impl AggregatedDataUpdate {
AggregatedCollectible {
collectible
} count if *count > 0 => {
*collectible
collectible
}
);
for collectible in collectibles {
......@@ -321,8 +321,8 @@ impl AggregatedDataUpdate {
CollectiblesDependent {
collectible_type,
task,
} if *collectible_type == ty => {
*task
} if collectible_type == ty => {
task
}
);
if !dependent.is_empty() {
......@@ -1027,7 +1027,7 @@ impl AggregationUpdateQueue {
// if it has an `AggregateRoot` we can skip visiting the nested nodes since
// this would already be scheduled by the `AggregateRoot`
if !task.has_key(&CachedDataItemKey::AggregateRoot {}) {
let dirty_containers: Vec<_> = get_many!(task, AggregatedDirtyContainer { task } count if count.get(session_id) > 0 => *task);
let dirty_containers: Vec<_> = get_many!(task, AggregatedDirtyContainer { task } count if count.get(session_id) > 0 => task);
if !dirty_containers.is_empty() || dirty {
task.insert(CachedDataItem::AggregateRoot {
value: RootState::new(ActiveType::CachedActiveUntilClean, task_id),
......@@ -1613,7 +1613,7 @@ impl AggregationUpdateQueue {
if !is_aggregating_node(old) && is_aggregating_node(aggregation_number) {
// When converted from leaf to aggregating node, all children become
// followers
let children: Vec<_> = get_many!(task, Child { task } => *task);
let children: Vec<_> = get_many!(task, Child { task } => task);
for child_id in children {
task.add_new(CachedDataItem::Follower {
task: child_id,
......@@ -1625,7 +1625,7 @@ impl AggregationUpdateQueue {
if is_aggregating_node(aggregation_number) {
// followers might become inner nodes when the aggregation number is
// increased
let followers = iter_many!(task, Follower { task } count if *count > 0 => *task);
let followers = iter_many!(task, Follower { task } count if *count > 0 => task);
for follower_id in followers {
self.push(AggregationUpdateJob::BalanceEdge {
upper_id: task_id,
......@@ -1637,7 +1637,7 @@ impl AggregationUpdateQueue {
self.push(AggregationUpdateJob::BalanceEdge { upper_id, task_id });
}
} else {
let children = iter_many!(task, Child { task } => *task);
let children = iter_many!(task, Child { task } => task);
for child_id in children {
self.push(AggregationUpdateJob::UpdateAggregationNumber {
task_id: child_id,
......
......@@ -21,7 +21,10 @@ use crate::{
TurboTasksBackend, TurboTasksBackendInner,
},
backing_storage::BackingStorage,
data::{CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue},
data::{
CachedDataItem, CachedDataItemKey, CachedDataItemType, CachedDataItemValue,
CachedDataItemValueRef, CachedDataItemValueRefMut,
},
};
pub trait Operation:
......@@ -361,35 +364,31 @@ pub trait TaskGuard: Debug {
fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue>;
fn update(
&mut self,
key: &CachedDataItemKey,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
);
fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue>;
fn get(&self, key: &CachedDataItemKey) -> Option<&CachedDataItemValue>;
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<&mut CachedDataItemValue>;
fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>>;
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>>;
fn has_key(&self, key: &CachedDataItemKey) -> bool;
fn is_indexed(&self) -> bool;
fn iter(
&self,
index: CachedDataItemIndex,
) -> impl Iterator<Item = (&CachedDataItemKey, &CachedDataItemValue)>;
fn iter_all(&self) -> impl Iterator<Item = (&CachedDataItemKey, &CachedDataItemValue)>;
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)>;
fn shrink_to_fit(&mut self, ty: CachedDataItemType);
fn extract_if<'l, F>(
&'l mut self,
index: CachedDataItemIndex,
ty: CachedDataItemType,
f: F,
) -> impl Iterator<Item = CachedDataItem>
where
F: for<'a, 'b> FnMut(&'a CachedDataItemKey, &'b CachedDataItemValue) -> bool + 'l;
fn extract_if_all<'l, F>(&'l mut self, f: F) -> impl Iterator<Item = CachedDataItem>
where
F: for<'a, 'b> FnMut(&'a CachedDataItemKey, &'b CachedDataItemValue) -> bool + 'l;
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l;
fn invalidate_serialization(&mut self);
}
struct TaskGuardImpl<'a, B: BackingStorage> {
task_id: TaskId,
task: StorageWriteGuard<'a, TaskId, CachedDataItem>,
task: StorageWriteGuard<'a>,
backend: &'a TurboTasksBackendInner<B>,
#[cfg(debug_assertions)]
category: TaskDataCategory,
......@@ -474,10 +473,9 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task
.insert(CachedDataItem::from_key_and_value(key, value))
} else if value.is_persistent() {
let old = self.task.insert(CachedDataItem::from_key_and_value(
key.clone(),
value.clone(),
));
let old = self
.task
.insert(CachedDataItem::from_key_and_value(key, value.clone()));
self.task.persistance_state_mut().add_persisting_item();
self.backend
.persisted_storage_log(key.category())
......@@ -491,7 +489,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
);
old
} else {
let item = CachedDataItem::from_key_and_value(key.clone(), value);
let item = CachedDataItem::from_key_and_value(key, value);
if let Some(old) = self.task.insert(item) {
if old.is_persistent() {
self.task.persistance_state_mut().add_persisting_item();
......@@ -509,7 +507,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
fn update(
&mut self,
key: &CachedDataItemKey,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
) {
self.check_access(key.category());
......@@ -538,7 +536,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
add_persisting_item = true;
backend.persisted_storage_log(key.category()).unwrap().push(
*task_id,
key.clone(),
key,
Some(old_value),
None,
);
......@@ -547,7 +545,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
add_persisting_item = true;
backend.persisted_storage_log(key.category()).unwrap().push(
*task_id,
key.clone(),
key,
old_value,
new.clone(),
);
......@@ -570,14 +568,13 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
&& key.is_persistent()
&& value.is_persistent()
{
let key = key.clone();
self.task.persistance_state_mut().add_persisting_item();
self.backend
.persisted_storage_log(key.category())
.unwrap()
.push(
self.task_id,
key,
*key,
value.is_persistent().then(|| value.clone()),
None,
);
......@@ -588,12 +585,12 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
}
}
fn get(&self, key: &CachedDataItemKey) -> Option<&CachedDataItemValue> {
fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>> {
self.check_access(key.category());
self.task.get(key)
}
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<&mut CachedDataItemValue> {
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
self.check_access(key.category());
self.task.get_mut(key)
}
......@@ -603,52 +600,29 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.has_key(key)
}
fn is_indexed(&self) -> bool {
self.task.is_indexed()
}
fn iter(
&self,
index: CachedDataItemIndex,
) -> impl Iterator<Item = (&CachedDataItemKey, &CachedDataItemValue)> {
self.task.iter(Some(index))
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.task.iter(ty)
}
fn iter_all(&self) -> impl Iterator<Item = (&CachedDataItemKey, &CachedDataItemValue)> {
self.task.iter_all()
fn shrink_to_fit(&mut self, ty: CachedDataItemType) {
self.task.shrink_to_fit(ty)
}
fn extract_if<'l, F>(
&'l mut self,
index: CachedDataItemIndex,
ty: CachedDataItemType,
f: F,
) -> impl Iterator<Item = CachedDataItem>
where
F: for<'a, 'b> FnMut(&'a CachedDataItemKey, &'b CachedDataItemValue) -> bool + 'l,
{
if !self.backend.should_persist() || self.task_id.is_transient() {
return Either::Left(self.task.extract_if(Some(index), f));
}
Either::Right(self.task.extract_if(Some(index), f).inspect(|item| {
if item.is_persistent() {
let key = item.key();
let value = item.value();
self.backend
.persisted_storage_log(key.category())
.unwrap()
.push(self.task_id, key, Some(value), None);
}
}))
}
fn extract_if_all<'l, F>(&'l mut self, f: F) -> impl Iterator<Item = CachedDataItem>
where
F: for<'a, 'b> FnMut(&'a CachedDataItemKey, &'b CachedDataItemValue) -> bool + 'l,
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{
if !self.backend.should_persist() || self.task_id.is_transient() {
return Either::Left(self.task.extract_if_all(f));
return Either::Left(self.task.extract_if(ty, f));
}
Either::Right(self.task.extract_if_all(f).inspect(|item| {
Either::Right(self.task.extract_if(ty, f).inspect(|item| {
if item.is_persistent() {
let key = item.key();
let value = item.value();
......@@ -666,12 +640,15 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
}
let mut count = 0;
let cell_data = self
.iter(CachedDataItemIndex::CellData)
.iter(CachedDataItemType::CellData)
.filter_map(|(key, value)| match (key, value) {
(CachedDataItemKey::CellData { cell }, CachedDataItemValue::CellData { value }) => {
(
CachedDataItemKey::CellData { cell },
CachedDataItemValueRef::CellData { value },
) => {
count += 1;
Some(CachedDataItem::CellData {
cell: *cell,
cell,
value: value.clone(),
})
}
......
......@@ -41,8 +41,8 @@ impl UpdateCellOperation {
let dependent = get_many!(
task,
CellDependent { cell: dependent_cell, task }
if *dependent_cell == cell
=> *task
if dependent_cell == cell
=> task
);
drop(task);
......
......@@ -13,10 +13,7 @@ use crate::{
storage::{get, get_many},
TaskDataCategory,
},
data::{
CachedDataItem, CachedDataItemKey, CachedDataItemValue, CellRef, InProgressState,
OutputValue,
},
data::{CachedDataItem, CachedDataItemKey, CellRef, InProgressState, OutputValue},
};
#[derive(Serialize, Deserialize, Clone, Default)]
......@@ -50,13 +47,10 @@ impl UpdateOutputOperation {
return;
}
let old_error = task.remove(&CachedDataItemKey::Error {});
let current_output = task.get(&CachedDataItemKey::Output {});
let current_output = get!(task, Output);
let output_value = match output {
Ok(Ok(RawVc::TaskOutput(output_task_id))) => {
if let Some(CachedDataItemValue::Output {
value: OutputValue::Output(current_task_id),
}) = current_output
{
if let Some(OutputValue::Output(current_task_id)) = current_output {
if *current_task_id == output_task_id {
return;
}
......@@ -64,13 +58,10 @@ impl UpdateOutputOperation {
OutputValue::Output(output_task_id)
}
Ok(Ok(RawVc::TaskCell(output_task_id, cell))) => {
if let Some(CachedDataItemValue::Output {
value:
OutputValue::Cell(CellRef {
task: current_task_id,
cell: current_cell,
}),
}) = current_output
if let Some(OutputValue::Cell(CellRef {
task: current_task_id,
cell: current_cell,
})) = current_output
{
if *current_task_id == output_task_id && *current_cell == cell {
return;
......@@ -113,11 +104,11 @@ impl UpdateOutputOperation {
let dependent_tasks = ctx
.should_track_dependencies()
.then(|| get_many!(task, OutputDependent { task } => *task))
.then(|| get_many!(task, OutputDependent { task } => task))
.unwrap_or_default();
let children = ctx
.should_track_children()
.then(|| get_many!(task, Child { task } => *task))
.then(|| get_many!(task, Child { task } => task))
.unwrap_or_default();
let mut queue = AggregationUpdateQueue::new();
......
......@@ -8,7 +8,7 @@ use turbo_tasks::{
CellId, KeyValuePair, SessionId, TaskId, TraitTypeId, TypedSharedReference, ValueTypeId,
};
use crate::backend::{indexed::Indexed, TaskDataCategory};
use crate::backend::TaskDataCategory;
// this traits are needed for the transient variants of `CachedDataItem`
// transient variants are never cloned or compared
......@@ -623,100 +623,6 @@ pub mod allow_mut_access {
pub const AggregateRoot: () = ();
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum CachedDataItemIndex {
Children,
Collectibles,
Follower,
Upper,
AggregatedDirtyContainer,
AggregatedCollectible,
CellData,
CellTypeMaxIndex,
CellDependent,
OutputDependent,
CollectiblesDependent,
Dependencies,
InProgressCell,
}
#[allow(non_upper_case_globals, dead_code)]
pub mod indicies {
use super::CachedDataItemIndex;
pub const Child: CachedDataItemIndex = CachedDataItemIndex::Children;
pub const OutdatedChild: CachedDataItemIndex = CachedDataItemIndex::Children;
pub const Collectible: CachedDataItemIndex = CachedDataItemIndex::Collectibles;
pub const OutdatedCollectible: CachedDataItemIndex = CachedDataItemIndex::Collectibles;
pub const Follower: CachedDataItemIndex = CachedDataItemIndex::Follower;
pub const Upper: CachedDataItemIndex = CachedDataItemIndex::Upper;
pub const AggregatedDirtyContainer: CachedDataItemIndex =
CachedDataItemIndex::AggregatedDirtyContainer;
pub const AggregatedCollectible: CachedDataItemIndex =
CachedDataItemIndex::AggregatedCollectible;
pub const CellData: CachedDataItemIndex = CachedDataItemIndex::CellData;
pub const CellTypeMaxIndex: CachedDataItemIndex = CachedDataItemIndex::CellTypeMaxIndex;
pub const CellDependent: CachedDataItemIndex = CachedDataItemIndex::CellDependent;
pub const OutputDependent: CachedDataItemIndex = CachedDataItemIndex::OutputDependent;
pub const CollectiblesDependent: CachedDataItemIndex =
CachedDataItemIndex::CollectiblesDependent;
pub const OutputDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies;
pub const CellDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies;
pub const CollectibleDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies;
pub const OutdatedOutputDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies;
pub const OutdatedCellDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies;
pub const OutdatedCollectiblesDependency: CachedDataItemIndex =
CachedDataItemIndex::Dependencies;
pub const OutdatedCollectibleDependency: CachedDataItemIndex =
CachedDataItemIndex::Dependencies;
pub const InProgressCell: CachedDataItemIndex = CachedDataItemIndex::InProgressCell;
}
impl Indexed for CachedDataItemKey {
type Index = Option<CachedDataItemIndex>;
fn index(&self) -> Option<CachedDataItemIndex> {
match self {
CachedDataItemKey::Child { .. } => Some(CachedDataItemIndex::Children),
CachedDataItemKey::OutdatedChild { .. } => Some(CachedDataItemIndex::Children),
CachedDataItemKey::Collectible { .. } => Some(CachedDataItemIndex::Collectibles),
CachedDataItemKey::OutdatedCollectible { .. } => {
Some(CachedDataItemIndex::Collectibles)
}
CachedDataItemKey::Follower { .. } => Some(CachedDataItemIndex::Follower),
CachedDataItemKey::Upper { .. } => Some(CachedDataItemIndex::Upper),
CachedDataItemKey::AggregatedDirtyContainer { .. } => {
Some(CachedDataItemIndex::AggregatedDirtyContainer)
}
CachedDataItemKey::AggregatedCollectible { .. } => {
Some(CachedDataItemIndex::AggregatedCollectible)
}
CachedDataItemKey::CellData { .. } => Some(CachedDataItemIndex::CellData),
CachedDataItemKey::CellTypeMaxIndex { .. } => {
Some(CachedDataItemIndex::CellTypeMaxIndex)
}
CachedDataItemKey::CellDependent { .. } => Some(CachedDataItemIndex::CellDependent),
CachedDataItemKey::OutputDependent { .. } => Some(CachedDataItemIndex::OutputDependent),
CachedDataItemKey::OutputDependency { .. } => Some(CachedDataItemIndex::Dependencies),
CachedDataItemKey::CellDependency { .. } => Some(CachedDataItemIndex::Dependencies),
CachedDataItemKey::CollectiblesDependency { .. } => {
Some(CachedDataItemIndex::Dependencies)
}
CachedDataItemKey::OutdatedOutputDependency { .. } => {
Some(CachedDataItemIndex::Dependencies)
}
CachedDataItemKey::OutdatedCellDependency { .. } => {
Some(CachedDataItemIndex::Dependencies)
}
CachedDataItemKey::OutdatedCollectiblesDependency { .. } => {
Some(CachedDataItemIndex::Dependencies)
}
CachedDataItemKey::InProgressCell { .. } => Some(CachedDataItemIndex::InProgressCell),
_ => None,
}
}
}
impl CachedDataItemValue {
pub fn is_persistent(&self) -> bool {
match self {
......@@ -742,3 +648,15 @@ pub enum CachedDataUpdate {
/// An item was replaced. This is step 2 and tells about the new value.
Replace2 { value: CachedDataItemValue },
}
#[cfg(test)]
mod tests {
#[test]
fn test_sizes() {
assert_eq!(std::mem::size_of::<super::CachedDataItem>(), 40);
assert_eq!(std::mem::size_of::<super::CachedDataItemKey>(), 20);
assert_eq!(std::mem::size_of::<super::CachedDataItemValue>(), 32);
assert_eq!(std::mem::size_of::<super::CachedDataItemStorage>(), 48);
assert_eq!(std::mem::size_of::<super::CachedDataUpdate>(), 48);
}
}
......@@ -812,7 +812,8 @@ struct SerializeLikeCachedDataItem<'l>(&'l CachedDataItemKey, &'l CachedDataItem
impl Serialize for SerializeLikeCachedDataItem<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let item = CachedDataItem::from_key_and_value(self.0.clone(), self.1.clone());
// TODO add CachedDataItemRef to avoid cloning
let item = CachedDataItem::from_key_and_value(*self.0, self.1.clone());
item.serialize(serializer)
}
}
use std::fmt::Debug;
pub trait KeyValuePair {
type Type: Debug + Copy + Clone + PartialEq + Eq + std::hash::Hash;
type Key: Debug + Clone + PartialEq + Eq + std::hash::Hash;
type Value: Debug + Clone + Default + PartialEq + Eq;
type ValueRef<'l>: Debug + Copy + Clone + PartialEq + Eq + 'l
where
Self: 'l;
type ValueRefMut<'l>: Debug + PartialEq + Eq + 'l
where
Self: 'l;
fn ty(&self) -> Self::Type;
fn key(&self) -> Self::Key;
fn value(&self) -> Self::Value;
fn value_ref(&self) -> Self::ValueRef<'_>;
fn value_mut(&mut self) -> Self::ValueRefMut<'_>;
fn from_key_and_value(key: Self::Key, value: Self::Value) -> Self;
fn into_key_and_value(self) -> (Self::Key, Self::Value);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment