Unverified Commit 21fcfb01 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

feat: Replace InnerStorage with the generated TaskStorage struct (#88355)

## What

Replace `InnerStorage` with `TaskStorage` generated by the new derive macro from #88338 . 

Extend the TaskStorage derive macro to generate `CachedDataItem` adapter methods on the `TaskStorageAccessors` trait so we can maintain API compatibility. 

Replace the serialization layer to use the native TaskStorage representations now that CachedDataitem is only used for API access

## Why

TaskStorage will enable more performant serialization and access patterns.  From testing on vercel-site this saves ~1% of persistent cache size.  For this PR there is no benefit to access pattern performance (in fact i would anticipate a small regression due to the adaptor layer)

## How

1. **Extended macro attribute parsing** - Added `variant`, and `key_field`` attributes to specify CachedDataItem variant mapping per field

2. **Generated adapter methods on TaskStorageAccessors trait**:
   - `insert_kv(item)` - Insert a CachedDataItem into typed storage
   - `get(key)` - Look up by CachedDataItemKey
   - `remove(key)` - Remove by key
   - `get_mut(key)` - Mutable access (returns None for flags/sets/multimaps)
   - `iter(type)` - Iterate over a specific variant type
  - ...

   This enables current access patterns to keep working

3. **Migrated serialization and deserialization flow**:
   - use `TaskStorage` throughout the serialization flow using apis like `restore_from` and `clone_data` to create targeted copies for lock free IO.
    
4. **Removed new dead code** - Delete the `Storage` layer now that it is dead.

On its own this will be a small savings in serialized data size (saves 20Mb for vercel-site) and is slightly faster to snapshot/restore.  Access performance should be nearly equivalent in performance.
parent 8fe8ff79
...@@ -21,11 +21,11 @@ pub type TurboBincodeDecoder<'a> = ...@@ -21,11 +21,11 @@ pub type TurboBincodeDecoder<'a> =
pub type AnyEncodeFn = fn(&dyn Any, &mut TurboBincodeEncoder<'_>) -> Result<(), EncodeError>; pub type AnyEncodeFn = fn(&dyn Any, &mut TurboBincodeEncoder<'_>) -> Result<(), EncodeError>;
pub type AnyDecodeFn<T> = fn(&mut TurboBincodeDecoder<'_>) -> Result<T, DecodeError>; pub type AnyDecodeFn<T> = fn(&mut TurboBincodeDecoder<'_>) -> Result<T, DecodeError>;
fn new_turbo_bincode_encoder(buf: &mut TurboBincodeBuffer) -> TurboBincodeEncoder<'_> { pub fn new_turbo_bincode_encoder(buf: &mut TurboBincodeBuffer) -> TurboBincodeEncoder<'_> {
EncoderImpl::new(TurboBincodeWriter::new(buf), TURBO_BINCODE_CONFIG) EncoderImpl::new(TurboBincodeWriter::new(buf), TURBO_BINCODE_CONFIG)
} }
fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> { pub fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> {
DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ()) DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ())
} }
......
// TODO(PR 2): Remove this once the storage schema is integrated with the rest of the codebase.
// This module is scaffolding for the TaskStorage macro and is not yet used.
#![allow(dead_code)]
use std::{ use std::{
hash::{BuildHasherDefault, Hash}, hash::{BuildHasherDefault, Hash},
ops::{Add, AddAssign, Sub}, ops::{Add, AddAssign, Sub},
...@@ -84,7 +81,7 @@ impl CounterValue for i32 { ...@@ -84,7 +81,7 @@ impl CounterValue for i32 {
} }
impl<K, V> CounterMap<K, V> { impl<K, V> CounterMap<K, V> {
fn new() -> Self { pub fn new() -> Self {
Self(AutoMap::default()) Self(AutoMap::default())
} }
...@@ -92,6 +89,13 @@ impl<K, V> CounterMap<K, V> { ...@@ -92,6 +89,13 @@ impl<K, V> CounterMap<K, V> {
self.0.is_empty() self.0.is_empty()
} }
pub fn shrink_to_fit(&mut self)
where
K: Eq + Hash,
{
self.0.shrink_to_fit();
}
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.0.len() self.0.len()
} }
...@@ -102,7 +106,15 @@ impl<K, V> CounterMap<K, V> { ...@@ -102,7 +106,15 @@ impl<K, V> CounterMap<K, V> {
{ {
self.0.get(key) self.0.get(key)
} }
// TODO(lukesandberg): this is just here for the CachedDataItem adaptor layer, can be removed
// once that is gone.
#[doc(hidden)]
pub fn get_mut(&mut self, _key: &K) -> Option<&mut V>
where
K: Eq + Hash,
{
unreachable!("This should never be called, please insert instead")
}
pub fn iter(&self) -> impl Iterator<Item = (&K, &V)> { pub fn iter(&self) -> impl Iterator<Item = (&K, &V)> {
self.0.iter() self.0.iter()
} }
...@@ -117,7 +129,6 @@ impl<K, V> CounterMap<K, V> { ...@@ -117,7 +129,6 @@ impl<K, V> CounterMap<K, V> {
impl<K: Hash + Eq, V: CounterValue> CounterMap<K, V> { impl<K: Hash + Eq, V: CounterValue> CounterMap<K, V> {
/// Insert a key-value pair. Panics if value is zero (invariant: zero values are not stored). /// Insert a key-value pair. Panics if value is zero (invariant: zero values are not stored).
#[cfg(test)]
pub fn insert(&mut self, key: K, value: V) -> Option<V> { pub fn insert(&mut self, key: K, value: V) -> Option<V> {
debug_assert!( debug_assert!(
!value.is_zero(), !value.is_zero(),
......
use either::Either;
use turbo_tasks::KeyValuePair;
use crate::data::{
CachedDataItem, CachedDataItemKey, CachedDataItemStorage, CachedDataItemType,
CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut,
};
#[derive(Debug, Clone)]
pub struct DynamicStorage {
map: Vec<CachedDataItemStorage>,
}
impl DynamicStorage {
pub fn new() -> Self {
Self {
map: Default::default(),
}
}
fn get_or_create_map_mut(&mut self, ty: CachedDataItemType) -> &mut CachedDataItemStorage {
let i = self.map.iter().position(|m| m.ty() == ty);
if let Some(i) = i {
&mut self.map[i]
} else {
self.map.reserve_exact(1);
self.map.push(CachedDataItemStorage::new(ty));
self.map.last_mut().unwrap()
}
}
fn get_map_mut(&mut self, ty: CachedDataItemType) -> Option<&mut CachedDataItemStorage> {
self.map.iter_mut().find(|m| m.ty() == ty)
}
fn get_map_index(&mut self, ty: CachedDataItemType) -> Option<usize> {
self.map.iter_mut().position(|m| m.ty() == ty)
}
fn get_or_create_map_index(&mut self, ty: CachedDataItemType) -> usize {
let i = self.map.iter().position(|m| m.ty() == ty);
if let Some(i) = i {
i
} else {
let i = self.map.len();
self.map.reserve_exact(1);
self.map.push(CachedDataItemStorage::new(ty));
i
}
}
fn get_map(&self, ty: CachedDataItemType) -> Option<&CachedDataItemStorage> {
self.map.iter().find(|m| m.ty() == ty)
}
pub fn add(&mut self, item: CachedDataItem) -> bool {
let ty = item.ty();
self.get_or_create_map_mut(ty).add(item)
}
pub fn extend(
&mut self,
ty: CachedDataItemType,
iterator: impl Iterator<Item = CachedDataItem>,
) -> bool {
self.get_or_create_map_mut(ty).extend(iterator)
}
pub fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> {
let ty = item.ty();
self.get_or_create_map_mut(ty).insert(item)
}
pub fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> {
self.get_map_index(key.ty()).and_then(|i| {
let storage = &mut self.map[i];
let result = storage.remove(key);
if result.is_some() && storage.is_empty() {
self.map.swap_remove(i);
self.map.shrink_to_fit();
}
result
})
}
pub fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>> {
self.get_map(key.ty()).and_then(|m| m.get(key))
}
pub fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
self.get_map_mut(key.ty()).and_then(|m| m.get_mut(key))
}
pub fn get_mut_or_insert_with(
&mut self,
key: CachedDataItemKey,
f: impl FnOnce() -> CachedDataItemValue,
) -> CachedDataItemValueRefMut<'_> {
self.get_or_create_map_mut(key.ty())
.get_mut_or_insert_with(key, f)
}
pub fn contains_key(&self, key: &CachedDataItemKey) -> bool {
self.get_map(key.ty())
.map(|m| m.contains_key(key))
.unwrap_or_default()
}
pub fn count(&self, ty: CachedDataItemType) -> usize {
self.get_map(ty).map(|m| m.len()).unwrap_or_default()
}
pub fn iter(
&self,
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.get_map(ty).map(|m| m.iter()).into_iter().flatten()
}
pub fn iter_all(
&self,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.map.iter().flat_map(|m| m.iter())
}
pub fn extract_if<'l, F>(
&'l mut self,
ty: CachedDataItemType,
mut f: F,
) -> impl Iterator<Item = CachedDataItem> + use<'l, F>
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{
// TODO this could be more efficient when the storage would support extract_if directly.
// This requires some macro magic to make it work...
// But we could potentially avoid the two temporary Vecs.
let Some(i) = self.get_map_index(ty) else {
return Either::Left(std::iter::empty());
};
let storage = &mut self.map[i];
let items_to_extract = storage
.iter()
.filter(|(k, v)| f(*k, *v))
.map(|(key, _)| key)
.collect::<Vec<_>>();
let items = items_to_extract
.into_iter()
.map(move |key| {
let value = storage.remove(&key).unwrap();
CachedDataItem::from_key_and_value(key, value)
})
.collect::<Vec<_>>();
if self.map[i].is_empty() {
self.map.swap_remove(i);
self.map.shrink_to_fit();
}
Either::Right(items.into_iter())
}
pub fn update(
&mut self,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
) {
let i = self.get_or_create_map_index(key.ty());
let map = &mut self.map[i];
map.update(key, update);
if map.is_empty() {
self.map.swap_remove(i);
self.map.shrink_to_fit();
}
}
pub fn len(&self) -> usize {
self.map.iter().map(|m| m.len()).sum()
}
pub fn shrink_to_fit(&mut self, ty: CachedDataItemType) {
if let Some(map) = self.get_map_mut(ty) {
map.shrink_to_fit();
}
}
pub fn snapshot_for_persisting(&self) -> Self {
Self {
map: self
.map
.iter()
.filter(|m| m.ty().is_persistent())
.cloned()
.collect(),
}
}
}
mod counter_map; mod counter_map;
mod dynamic_storage;
mod operation; mod operation;
mod storage; mod storage;
mod storage_schema; pub mod storage_schema;
use std::{ use std::{
borrow::Cow, borrow::Cow,
...@@ -26,7 +25,7 @@ use smallvec::{SmallVec, smallvec}; ...@@ -26,7 +25,7 @@ use smallvec::{SmallVec, smallvec};
use tokio::time::{Duration, Instant}; use tokio::time::{Duration, Instant};
use tracing::{Span, trace_span}; use tracing::{Span, trace_span};
use turbo_tasks::{ use turbo_tasks::{
CellId, FxDashMap, KeyValuePair, RawVc, ReadCellOptions, ReadCellTracking, ReadConsistency, CellId, FxDashMap, RawVc, ReadCellOptions, ReadCellTracking, ReadConsistency,
ReadOutputOptions, ReadTracking, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TaskPriority, ReadOutputOptions, ReadTracking, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TaskPriority,
TraitTypeId, TurboTasksBackendApi, ValueTypeId, TraitTypeId, TurboTasksBackendApi, ValueTypeId,
backend::{ backend::{
...@@ -43,7 +42,10 @@ use turbo_tasks::{ ...@@ -43,7 +42,10 @@ use turbo_tasks::{
util::{IdFactoryWithReuse, good_chunk_size, into_chunks}, util::{IdFactoryWithReuse, good_chunk_size, into_chunks},
}; };
pub use self::{operation::AnyOperation, storage::TaskDataCategory}; pub use self::{
operation::AnyOperation,
storage::{SpecificTaskDataCategory, TaskDataCategory},
};
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
use crate::backend::operation::TaskDirtyCause; use crate::backend::operation::TaskDirtyCause;
use crate::{ use crate::{
...@@ -56,9 +58,9 @@ use crate::{ ...@@ -56,9 +58,9 @@ use crate::{
make_task_dirty_internal, prepare_new_children, make_task_dirty_internal, prepare_new_children,
}, },
storage::{ storage::{
InnerStorageSnapshot, Storage, count, get, get_many, get_mut, get_mut_or_insert_with, Storage, count, get, get_many, get_mut, get_mut_or_insert_with, iter_many, remove,
iter_many, remove,
}, },
storage_schema::{TaskStorage, TaskStorageAccessors},
}, },
backing_storage::BackingStorage, backing_storage::BackingStorage,
data::{ data::{
...@@ -1066,75 +1068,54 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1066,75 +1068,54 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let snapshot_time = Instant::now(); let snapshot_time = Instant::now();
drop(snapshot_request); drop(snapshot_request);
let preprocess = |task_id: TaskId, inner: &storage::InnerStorage| { let preprocess = |task_id: TaskId, inner: &TaskStorage| {
if task_id.is_transient() { if task_id.is_transient() {
return (None, None); return (None, None);
} }
let len = inner.len();
let meta_restored = inner.state().meta_restored(); let meta_restored = inner.flags.meta_restored();
let data_restored = inner.state().data_restored(); let data_restored = inner.flags.data_restored();
let mut meta = meta_restored.then(|| Vec::with_capacity(len)); // Encode meta/data directly from TaskStorage
let mut data = data_restored.then(|| Vec::with_capacity(len)); let meta = meta_restored.then(|| inner.clone_meta_snapshot());
for (key, value) in inner.iter_all() { let data = data_restored.then(|| inner.clone_data_snapshot());
if key.is_persistent() && value.is_persistent() {
match key.category() {
TaskDataCategory::Meta => {
if let Some(meta) = &mut meta {
meta.push(CachedDataItem::from_key_and_value_ref(key, value))
}
}
TaskDataCategory::Data => {
if let Some(data) = &mut data {
data.push(CachedDataItem::from_key_and_value_ref(key, value))
}
}
_ => {}
}
}
}
(meta, data) (meta, data)
}; };
let process = |task_id: TaskId, (meta, data): (Option<Vec<_>>, Option<Vec<_>>)| { let process =
// TODO: perf: Instead of returning a `Vec` of individually allocated `SmallVec`s, it'd |task_id: TaskId, (meta, data): (Option<TaskStorage>, Option<TaskStorage>)| {
// be better to append everything to a flat per-task or per-shard `Vec<u8>`, and have // TODO: perf: Instead of returning a `Vec` of individually allocated `SmallVec`s,
// each `serialize` call return `(start_idx, end_idx)`. // it'd be better to append everything to a flat per-task or
( // per-shard `Vec<u8>`, and have each `serialize` call return
task_id, // `(start_idx, end_idx)`.
meta.map(|d| self.backing_storage.serialize(task_id, &d)), (
data.map(|d| self.backing_storage.serialize(task_id, &d)), task_id,
) meta.map(|d| {
}; self.backing_storage
let process_snapshot = |task_id: TaskId, inner: Box<InnerStorageSnapshot>| { .serialize(task_id, &d, SpecificTaskDataCategory::Meta)
}),
data.map(|d| {
self.backing_storage
.serialize(task_id, &d, SpecificTaskDataCategory::Data)
}),
)
};
let process_snapshot = |task_id: TaskId, inner: Box<TaskStorage>| {
if task_id.is_transient() { if task_id.is_transient() {
return (task_id, None, None); return (task_id, None, None);
} }
let len = inner.len();
let mut meta = inner.meta_modified.then(|| Vec::with_capacity(len)); // Encode meta/data directly from TaskStorage snapshot
let mut data = inner.data_modified.then(|| Vec::with_capacity(len));
for (key, value) in inner.iter_all() {
if key.is_persistent() && value.is_persistent() {
match key.category() {
TaskDataCategory::Meta => {
if let Some(meta) = &mut meta {
meta.push(CachedDataItem::from_key_and_value_ref(key, value));
}
}
TaskDataCategory::Data => {
if let Some(data) = &mut data {
data.push(CachedDataItem::from_key_and_value_ref(key, value));
}
}
_ => {}
}
}
}
( (
task_id, task_id,
meta.map(|meta| self.backing_storage.serialize(task_id, &meta)), inner.flags.meta_modified().then(|| {
data.map(|data| self.backing_storage.serialize(task_id, &data)), self.backing_storage
.serialize(task_id, &inner, SpecificTaskDataCategory::Meta)
}),
inner.flags.data_modified().then(|| {
self.backing_storage
.serialize(task_id, &inner, SpecificTaskDataCategory::Data)
}),
) )
}; };
...@@ -2599,17 +2580,20 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2599,17 +2580,20 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
} }
} }
// Shrink memory usage // Shrink memory usage for collections that are only mutated during/after execution
task.shrink_to_fit(CachedDataItemType::CellData); task.shrink_cell_data();
task.shrink_to_fit(CachedDataItemType::TransientCellData); task.shrink_transient_cell_data();
task.shrink_to_fit(CachedDataItemType::CellTypeMaxIndex); task.shrink_cell_type_max_index();
task.shrink_to_fit(CachedDataItemType::CellDependency); task.shrink_cell_dependencies();
task.shrink_to_fit(CachedDataItemType::OutputDependency); task.shrink_output_dependencies();
task.shrink_to_fit(CachedDataItemType::CollectiblesDependency); task.shrink_collectibles_dependencies();
task.shrink_to_fit(CachedDataItemType::OutdatedOutputDependency); task.shrink_outdated_cell_dependencies();
task.shrink_to_fit(CachedDataItemType::OutdatedCellDependency); task.shrink_outdated_output_dependencies();
task.shrink_to_fit(CachedDataItemType::OutdatedCollectiblesDependency); task.shrink_outdated_collectibles();
task.shrink_to_fit(CachedDataItemType::OutdatedCollectible); task.shrink_outdated_collectibles_dependencies();
task.shrink_children();
task.shrink_collectibles();
task.shrink_in_progress_cells();
drop(task); drop(task);
} }
......
...@@ -33,6 +33,7 @@ use crate::{ ...@@ -33,6 +33,7 @@ use crate::{
storage::{ storage::{
count, get, get_many, iter_many, remove, update, update_count, update_count_and_get, count, get, get_many, iter_many, remove, update, update_count, update_count_and_get,
}, },
storage_schema::TaskStorageAccessors,
}, },
data::{ data::{
ActivenessState, AggregationNumber, CachedDataItem, CachedDataItemKey, CachedDataItemType, ActivenessState, AggregationNumber, CachedDataItem, CachedDataItemKey, CachedDataItemType,
......
...@@ -9,13 +9,14 @@ use crate::{ ...@@ -9,13 +9,14 @@ use crate::{
backend::{ backend::{
TaskDataCategory, get, get_many, TaskDataCategory, get, get_many,
operation::{ operation::{
AggregatedDataUpdate, ExecuteContext, Operation, TaskGuard, AggregatedDataUpdate, ExecuteContext, Operation,
aggregation_update::{ aggregation_update::{
AggregationUpdateJob, AggregationUpdateQueue, InnerOfUppersLostFollowersJob, AggregationUpdateJob, AggregationUpdateQueue, InnerOfUppersLostFollowersJob,
get_aggregation_number, get_uppers, is_aggregating_node, get_aggregation_number, get_uppers, is_aggregating_node,
}, },
}, },
storage::update_count, storage::update_count,
storage_schema::TaskStorageAccessors,
}, },
data::{CachedDataItemKey, CellRef, CollectibleRef, CollectiblesRef}, data::{CachedDataItemKey, CellRef, CollectibleRef, CollectiblesRef},
}; };
......
...@@ -5,9 +5,10 @@ use crate::{ ...@@ -5,9 +5,10 @@ use crate::{
backend::{ backend::{
TaskDataCategory, get_mut, TaskDataCategory, get_mut,
operation::{ operation::{
ExecuteContext, Operation, TaskGuard, ExecuteContext, Operation,
aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue},
}, },
storage_schema::TaskStorageAccessors,
}, },
data::{CachedDataItem, CachedDataItemKey, InProgressState, InProgressStateInner}, data::{CachedDataItem, CachedDataItemKey, InProgressState, InProgressStateInner},
}; };
...@@ -26,7 +27,7 @@ impl ConnectChildOperation { ...@@ -26,7 +27,7 @@ impl ConnectChildOperation {
pub fn run( pub fn run(
parent_task_id: Option<TaskId>, parent_task_id: Option<TaskId>,
child_task_id: TaskId, child_task_id: TaskId,
mut ctx: impl ExecuteContext, mut ctx: impl ExecuteContext<'_>,
) { ) {
if let Some(parent_task_id) = parent_task_id { if let Some(parent_task_id) = parent_task_id {
let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::All); let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::All);
......
...@@ -12,8 +12,9 @@ use turbo_tasks::TaskId; ...@@ -12,8 +12,9 @@ use turbo_tasks::TaskId;
use crate::{ use crate::{
backend::{ backend::{
TaskDataCategory, TaskDataCategory,
operation::{ExecuteContext, Operation, TaskGuard}, operation::{ExecuteContext, Operation},
storage::{get, iter_many}, storage::{get, iter_many},
storage_schema::TaskStorageAccessors,
}, },
data::CachedDataItem, data::CachedDataItem,
}; };
...@@ -184,7 +185,7 @@ impl LeafDistanceUpdateQueue { ...@@ -184,7 +185,7 @@ impl LeafDistanceUpdateQueue {
} }
impl Operation for LeafDistanceUpdateQueue { impl Operation for LeafDistanceUpdateQueue {
fn execute(mut self, ctx: &mut impl ExecuteContext) { fn execute(mut self, ctx: &mut impl ExecuteContext<'_>) {
if self.is_empty() { if self.is_empty() {
return; return;
} }
......
...@@ -16,26 +16,23 @@ use std::{ ...@@ -16,26 +16,23 @@ use std::{
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use turbo_tasks::{ use turbo_tasks::{
CellId, FxIndexMap, KeyValuePair, TaskId, TaskPriority, TurboTasksBackendApi, CellId, FxIndexMap, TaskId, TaskPriority, TurboTasksBackendApi, TypedSharedReference,
TypedSharedReference,
}; };
use crate::{ use crate::{
backend::{ backend::{
OperationGuard, TaskDataCategory, TransientTask, TurboTasksBackend, TurboTasksBackendInner, OperationGuard, TaskDataCategory, TransientTask, TurboTasksBackend, TurboTasksBackendInner,
storage::{SpecificTaskDataCategory, StorageWriteGuard, get, iter_many, remove}, storage::{SpecificTaskDataCategory, StorageWriteGuard, get, iter_many, remove},
storage_schema::{TaskStorage, TaskStorageAccessors},
}, },
backing_storage::{BackingStorage, BackingStorageSealed}, backing_storage::{BackingStorage, BackingStorageSealed},
data::{ data::{CachedDataItemKey, Dirtyness},
CachedDataItem, CachedDataItemKey, CachedDataItemType, CachedDataItemValue,
CachedDataItemValueRef, CachedDataItemValueRefMut, Dirtyness,
},
}; };
pub trait Operation: pub trait Operation:
Encode + Decode<()> + Default + TryFrom<AnyOperation, Error = ()> + Into<AnyOperation> Encode + Decode<()> + Default + TryFrom<AnyOperation, Error = ()> + Into<AnyOperation>
{ {
fn execute(self, ctx: &mut impl ExecuteContext); fn execute(self, ctx: &mut impl ExecuteContext<'_>);
} }
#[derive(Copy, Clone)] #[derive(Copy, Clone)]
...@@ -162,21 +159,23 @@ where ...@@ -162,21 +159,23 @@ where
fn restore_task_data( fn restore_task_data(
&mut self, &mut self,
task_id: TaskId, task_id: TaskId,
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Vec<CachedDataItem> { ) -> TaskStorage {
if !self.ensure_transaction() { if !self.ensure_transaction() {
// If we don't need to restore, we can just return an empty vector // If we don't need to restore, we can just return an empty storage
return Vec::new(); return TaskStorage::default();
} }
let tx = self.get_tx(); let tx = self.get_tx();
let mut storage = TaskStorage::default();
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`. // Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
let result = unsafe { let result = unsafe {
self.backend self.backend
.backing_storage .backing_storage
.lookup_data(tx, task_id, category) .lookup_data(tx, task_id, category, &mut storage)
}; };
match result { match result {
Ok(data) => data, Ok(()) => storage,
Err(e) => { Err(e) => {
let task_name = self.backend.get_task_description(task_id); let task_name = self.backend.get_task_description(task_id);
panic!( panic!(
...@@ -190,9 +189,12 @@ where ...@@ -190,9 +189,12 @@ where
fn restore_task_data_batch( fn restore_task_data_batch(
&mut self, &mut self,
task_ids: &[TaskId], task_ids: &[TaskId],
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Option<Vec<Vec<CachedDataItem>>> { ) -> Option<Vec<TaskStorage>> {
debug_assert!(task_ids.len() > 1, "Use restore_task_data for single task"); debug_assert!(
task_ids.len() > 1,
"Use restore_task_data_typed for single task"
);
if !self.ensure_transaction() { if !self.ensure_transaction() {
// If we don't need to restore, we return None // If we don't need to restore, we return None
return None; return None;
...@@ -248,8 +250,8 @@ where ...@@ -248,8 +250,8 @@ where
let mut task = self.backend.storage.access_mut(id); let mut task = self.backend.storage.access_mut(id);
// TODO add is_restoring and avoid concurrent restores and duplicates tasks // TODO add is_restoring and avoid concurrent restores and duplicates tasks
// ids in `task_ids` // ids in `task_ids`
if !task.state().is_restored(category) { if !task.flags.is_restored(category) {
task.state_mut().set_restored(TaskDataCategory::All); task.flags.set_restored(TaskDataCategory::All);
} }
prepared_task_callback(self, id, category, task); prepared_task_callback(self, id, category, task);
} }
...@@ -284,14 +286,14 @@ where ...@@ -284,14 +286,14 @@ where
let task = self.backend.storage.access_mut(task_id); let task = self.backend.storage.access_mut(task_id);
let mut ready = true; let mut ready = true;
if matches!(category, TaskDataCategory::Data | TaskDataCategory::All) if matches!(category, TaskDataCategory::Data | TaskDataCategory::All)
&& !task.state().is_restored(TaskDataCategory::Data) && !task.flags.is_restored(TaskDataCategory::Data)
{ {
tasks_to_restore_for_data.push(task_id); tasks_to_restore_for_data.push(task_id);
tasks_to_restore_for_data_indicies.push(i); tasks_to_restore_for_data_indicies.push(i);
ready = false; ready = false;
} }
if matches!(category, TaskDataCategory::Meta | TaskDataCategory::All) if matches!(category, TaskDataCategory::Meta | TaskDataCategory::All)
&& !task.state().is_restored(TaskDataCategory::Meta) && !task.flags.is_restored(TaskDataCategory::Meta)
{ {
tasks_to_restore_for_meta.push(task_id); tasks_to_restore_for_meta.push(task_id);
tasks_to_restore_for_meta_indicies.push(i); tasks_to_restore_for_meta_indicies.push(i);
...@@ -311,22 +313,23 @@ where ...@@ -311,22 +313,23 @@ where
0 => {} 0 => {}
1 => { 1 => {
let task_id = tasks_to_restore_for_data[0]; let task_id = tasks_to_restore_for_data[0];
let data = self.restore_task_data(task_id, TaskDataCategory::Data); let data = self.restore_task_data(task_id, SpecificTaskDataCategory::Data);
let idx = tasks_to_restore_for_data_indicies[0]; let idx = tasks_to_restore_for_data_indicies[0];
tasks[idx].2 = Some(data); tasks[idx].2 = Some(data);
} }
_ => { _ => {
if let Some(data) = if let Some(data) = self.restore_task_data_batch(
self.restore_task_data_batch(&tasks_to_restore_for_data, TaskDataCategory::Data) &tasks_to_restore_for_data,
{ SpecificTaskDataCategory::Data,
) {
data.into_iter() data.into_iter()
.zip(tasks_to_restore_for_data_indicies) .zip(tasks_to_restore_for_data_indicies)
.for_each(|(items, idx)| { .for_each(|(item, idx)| {
tasks[idx].2 = Some(items); tasks[idx].2 = Some(item);
}); });
} else { } else {
for idx in tasks_to_restore_for_data_indicies { for idx in tasks_to_restore_for_data_indicies {
tasks[idx].2 = Some(Vec::new()); tasks[idx].2 = Some(TaskStorage::default());
} }
} }
} }
...@@ -335,29 +338,30 @@ where ...@@ -335,29 +338,30 @@ where
0 => {} 0 => {}
1 => { 1 => {
let task_id = tasks_to_restore_for_meta[0]; let task_id = tasks_to_restore_for_meta[0];
let data = self.restore_task_data(task_id, TaskDataCategory::Meta); let data = self.restore_task_data(task_id, SpecificTaskDataCategory::Meta);
let idx = tasks_to_restore_for_meta_indicies[0]; let idx = tasks_to_restore_for_meta_indicies[0];
tasks[idx].3 = Some(data); tasks[idx].3 = Some(data);
} }
_ => { _ => {
if let Some(data) = if let Some(data) = self.restore_task_data_batch(
self.restore_task_data_batch(&tasks_to_restore_for_meta, TaskDataCategory::Meta) &tasks_to_restore_for_meta,
{ SpecificTaskDataCategory::Meta,
) {
data.into_iter() data.into_iter()
.zip(tasks_to_restore_for_meta_indicies) .zip(tasks_to_restore_for_meta_indicies)
.for_each(|(items, idx)| { .for_each(|(item, idx)| {
tasks[idx].3 = Some(items); tasks[idx].3 = Some(item);
}); });
} else { } else {
for idx in tasks_to_restore_for_meta_indicies { for idx in tasks_to_restore_for_meta_indicies {
tasks[idx].3 = Some(Vec::new()); tasks[idx].3 = Some(TaskStorage::default());
} }
} }
} }
} }
for (task_id, category, items_for_data, items_for_meta) in tasks { for (task_id, category, storage_for_data, storage_for_meta) in tasks {
if items_for_data.is_none() && items_for_meta.is_none() { if storage_for_data.is_none() && storage_for_meta.is_none() {
continue; continue;
} }
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
...@@ -369,23 +373,17 @@ where ...@@ -369,23 +373,17 @@ where
} }
let mut task = self.backend.storage.access_mut(task_id); let mut task = self.backend.storage.access_mut(task_id);
if let Some(items) = items_for_data if let Some(storage) = storage_for_data
&& !task.state().is_restored(TaskDataCategory::Data) && !task.flags.is_restored(TaskDataCategory::Data)
{ {
// TODO store items groups by type to be able to use extend here task.restore_from(storage, TaskDataCategory::Data);
for item in items { task.flags.set_restored(TaskDataCategory::Data);
task.add(item);
}
task.state_mut().set_restored(TaskDataCategory::Data);
} }
if let Some(items) = items_for_meta if let Some(storage) = storage_for_meta
&& !task.state().is_restored(TaskDataCategory::Meta) && !task.flags.is_restored(TaskDataCategory::Meta)
{ {
// TODO store items groups by type to be able to use extend here task.restore_from(storage, TaskDataCategory::Meta);
for item in items { task.flags.set_restored(TaskDataCategory::Meta);
task.add(item);
}
task.state_mut().set_restored(TaskDataCategory::Meta);
} }
prepared_task_callback(self, task_id, category, task); prepared_task_callback(self, task_id, category, task);
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
...@@ -420,23 +418,40 @@ where ...@@ -420,23 +418,40 @@ where
} }
let mut task = self.backend.storage.access_mut(task_id); let mut task = self.backend.storage.access_mut(task_id);
if !task.state().is_restored(category) { if !task.flags.is_restored(category) {
if task_id.is_transient() { if task_id.is_transient() {
task.state_mut().set_restored(TaskDataCategory::All); task.flags.set_restored(TaskDataCategory::All);
} else { } else {
for category in category { // Collect which categories need restoring while we have the lock
if !task.state().is_restored(category) { let needs_data =
// Avoid holding the lock too long since this can also affect other tasks category.includes_data() && !task.flags.is_restored(TaskDataCategory::Data);
drop(task); let needs_meta =
category.includes_meta() && !task.flags.is_restored(TaskDataCategory::Meta);
let items = self.restore_task_data(task_id, category);
task = self.backend.storage.access_mut(task_id); if needs_data || needs_meta {
if !task.state().is_restored(category) { // Avoid holding the lock too long since this can also affect other tasks
for item in items { // Drop lock once, do all I/O, then re-acquire once
task.add(item); drop(task);
}
task.state_mut().set_restored(category); let storage_data = needs_data
} .then(|| self.restore_task_data(task_id, SpecificTaskDataCategory::Data));
let storage_meta = needs_meta
.then(|| self.restore_task_data(task_id, SpecificTaskDataCategory::Meta));
task = self.backend.storage.access_mut(task_id);
// Handle race conditions and merge
if let Some(storage) = storage_data
&& !task.flags.is_restored(TaskDataCategory::Data)
{
task.restore_from(storage, TaskDataCategory::Data);
task.flags.set_restored(TaskDataCategory::Data);
}
if let Some(storage) = storage_meta
&& !task.flags.is_restored(TaskDataCategory::Meta)
{
task.restore_from(storage, TaskDataCategory::Meta);
task.flags.set_restored(TaskDataCategory::Meta);
} }
} }
} }
...@@ -510,32 +525,60 @@ where ...@@ -510,32 +525,60 @@ where
} }
let (mut task1, mut task2) = self.backend.storage.access_pair_mut(task_id1, task_id2); let (mut task1, mut task2) = self.backend.storage.access_pair_mut(task_id1, task_id2);
let is_restored1 = task1.state().is_restored(category);
let is_restored2 = task2.state().is_restored(category); // Collect what needs restoring for each task
if !is_restored1 || !is_restored2 { let needs_data1 =
for category in category { category.includes_data() && !task1.flags.is_restored(TaskDataCategory::Data);
// Avoid holding the lock too long since this can also affect other tasks let needs_meta1 =
drop(task1); category.includes_meta() && !task1.flags.is_restored(TaskDataCategory::Meta);
drop(task2); let needs_data2 =
category.includes_data() && !task2.flags.is_restored(TaskDataCategory::Data);
let items1 = (!is_restored1).then(|| self.restore_task_data(task_id1, category)); let needs_meta2 =
let items2 = (!is_restored2).then(|| self.restore_task_data(task_id2, category)); category.includes_meta() && !task2.flags.is_restored(TaskDataCategory::Meta);
let (t1, t2) = self.backend.storage.access_pair_mut(task_id1, task_id2); if needs_data1 || needs_meta1 || needs_data2 || needs_meta2 {
task1 = t1; // Avoid holding the lock too long since this can also affect other tasks
task2 = t2; // Drop locks once, do all I/O, then re-acquire once
if !task1.state().is_restored(category) { drop(task1);
for item in items1.unwrap() { drop(task2);
task1.add(item);
} let storage_data1 = needs_data1
task1.state_mut().set_restored(category); .then(|| self.restore_task_data(task_id1, SpecificTaskDataCategory::Data));
} let storage_meta1 = needs_meta1
if !task2.state().is_restored(category) { .then(|| self.restore_task_data(task_id1, SpecificTaskDataCategory::Meta));
for item in items2.unwrap() { let storage_data2 = needs_data2
task2.add(item); .then(|| self.restore_task_data(task_id2, SpecificTaskDataCategory::Data));
} let storage_meta2 = needs_meta2
task2.state_mut().set_restored(category); .then(|| self.restore_task_data(task_id2, SpecificTaskDataCategory::Meta));
}
let (t1, t2) = self.backend.storage.access_pair_mut(task_id1, task_id2);
task1 = t1;
task2 = t2;
// Merge results, handling race conditions
if let Some(storage) = storage_data1
&& !task1.flags.is_restored(TaskDataCategory::Data)
{
task1.restore_from(storage, TaskDataCategory::Data);
task1.flags.set_restored(TaskDataCategory::Data);
}
if let Some(storage) = storage_meta1
&& !task1.flags.is_restored(TaskDataCategory::Meta)
{
task1.restore_from(storage, TaskDataCategory::Meta);
task1.flags.set_restored(TaskDataCategory::Meta);
}
if let Some(storage) = storage_data2
&& !task2.flags.is_restored(TaskDataCategory::Data)
{
task2.restore_from(storage, TaskDataCategory::Data);
task2.flags.set_restored(TaskDataCategory::Data);
}
if let Some(storage) = storage_meta2
&& !task2.flags.is_restored(TaskDataCategory::Meta)
{
task2.restore_from(storage, TaskDataCategory::Meta);
task2.flags.set_restored(TaskDataCategory::Meta);
} }
} }
( (
...@@ -627,56 +670,9 @@ impl<'e, B: BackingStorage> ChildExecuteContext<'e> for ChildExecuteContextImpl< ...@@ -627,56 +670,9 @@ impl<'e, B: BackingStorage> ChildExecuteContext<'e> for ChildExecuteContextImpl<
} }
} }
pub trait TaskGuard: Debug { pub trait TaskGuard: Debug + TaskStorageAccessors {
fn id(&self) -> TaskId; fn id(&self) -> TaskId;
/// Adds a new item to the task if the key is not already present.
/// Returns `true` if the item was added.
/// Returns `false` if an item with the same key was already present.
#[must_use]
fn add(&mut self, item: CachedDataItem) -> bool;
/// Adds a new item to the task. The key must not be already present.
/// Might panic if the key is already present.
fn add_new(&mut self, item: CachedDataItem);
/// Extends the task with items from the iterator.
/// Overwrites existing keys.
/// Returns `true` if all items were new and added.
/// Returns `false` if any item had a key that was already present.
fn extend(
&mut self,
ty: CachedDataItemType,
items: impl Iterator<Item = CachedDataItem>,
) -> bool;
/// Extends the task with items from the iterator.
/// Might panic if any item has a key that is already present.
fn extend_new(&mut self, ty: CachedDataItemType, items: impl Iterator<Item = CachedDataItem>);
fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue>;
fn update(
&mut self,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
);
fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue>;
fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>>;
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>>;
fn get_mut_or_insert_with(
&mut self,
key: CachedDataItemKey,
insert: impl FnOnce() -> CachedDataItemValue,
) -> CachedDataItemValueRefMut<'_>;
fn has_key(&self, key: &CachedDataItemKey) -> bool;
fn count(&self, ty: CachedDataItemType) -> usize;
fn iter(
&self,
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)>;
fn shrink_to_fit(&mut self, ty: CachedDataItemType);
fn extract_if<'l, F>(
&'l mut self,
ty: CachedDataItemType,
f: F,
) -> impl Iterator<Item = CachedDataItem>
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l;
fn invalidate_serialization(&mut self); fn invalidate_serialization(&mut self);
/// Determine which tasks to prefetch for a task. /// Determine which tasks to prefetch for a task.
/// Only returns Some once per task. /// Only returns Some once per task.
...@@ -841,9 +837,7 @@ impl<B: BackingStorage> Debug for TaskGuardImpl<'_, B> { ...@@ -841,9 +837,7 @@ impl<B: BackingStorage> Debug for TaskGuardImpl<'_, B> {
if let Some(task_type) = self.backend.task_cache.lookup_reverse(&self.task_id) { if let Some(task_type) = self.backend.task_cache.lookup_reverse(&self.task_id) {
d.field("task_type", &task_type); d.field("task_type", &task_type);
}; };
for (key, value) in self.task.iter_all() { d.field("storage", &*self.task);
d.field(&format!("{key:?}"), &value);
}
d.finish() d.finish()
} }
} }
...@@ -853,167 +847,6 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -853,167 +847,6 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task_id self.task_id
} }
#[track_caller]
fn add(&mut self, item: CachedDataItem) -> bool {
let category = item.category();
self.check_access(category);
if !self.task_id.is_transient() && item.is_persistent() {
if self.task.contains_key(&item.key()) {
return false;
}
self.task.track_modification(category.into_specific());
}
self.task.add(item)
}
#[track_caller]
fn add_new(&mut self, item: CachedDataItem) {
let category = item.category();
self.check_access(category);
if !self.task_id.is_transient() && item.is_persistent() {
self.task.track_modification(category.into_specific());
}
let added = self.task.add(item);
assert!(added, "Item already exists");
}
#[track_caller]
fn extend(
&mut self,
ty: CachedDataItemType,
items: impl Iterator<Item = CachedDataItem>,
) -> bool {
let category = ty.category();
self.check_access(category);
if !self.task_id.is_transient() && ty.is_persistent() {
let mut items = items.peekable();
// Check if the iterator is empty
if items.peek().is_none() {
return true;
}
// TODO this is not optimal as we always track a modification even if nothing is changed
self.task.track_modification(category.into_specific());
self.task.extend(ty, items)
} else {
self.task.extend(ty, items)
}
}
#[track_caller]
fn extend_new(&mut self, ty: CachedDataItemType, items: impl Iterator<Item = CachedDataItem>) {
let category = ty.category();
self.check_access(category);
if !self.task_id.is_transient() && ty.is_persistent() {
self.task.track_modification(category.into_specific());
}
let added = self.task.extend(ty, items);
assert!(added, "At least one item already exists");
}
#[track_caller]
fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> {
let category = item.category();
self.check_access(category);
if !self.task_id.is_transient() && item.is_persistent() {
self.task.track_modification(category.into_specific());
}
self.task.insert(item)
}
#[track_caller]
fn update(
&mut self,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
) {
let category = key.category();
self.check_access(category);
if !self.task_id.is_transient() && key.is_persistent() {
self.task.track_modification(category.into_specific());
}
self.task.update(key, update);
}
#[track_caller]
fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> {
let category = key.category();
self.check_access(category);
if !self.task_id.is_transient() && key.is_persistent() {
self.task.track_modification(category.into_specific());
}
self.task.remove(key)
}
fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>> {
self.check_access(key.category());
self.task.get(key)
}
#[track_caller]
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
let category = key.category();
self.check_access(category);
if !self.task_id.is_transient() && key.is_persistent() {
self.task.track_modification(category.into_specific());
}
self.task.get_mut(key)
}
#[track_caller]
fn get_mut_or_insert_with(
&mut self,
key: CachedDataItemKey,
insert: impl FnOnce() -> CachedDataItemValue,
) -> CachedDataItemValueRefMut<'_> {
let category = key.category();
self.check_access(category);
if !self.task_id.is_transient() && key.is_persistent() {
self.task.track_modification(category.into_specific());
}
self.task.get_mut_or_insert_with(key, insert)
}
#[track_caller]
fn has_key(&self, key: &CachedDataItemKey) -> bool {
self.check_access(key.category());
self.task.contains_key(key)
}
#[track_caller]
fn count(&self, ty: CachedDataItemType) -> usize {
self.check_access(ty.category());
self.task.count(ty)
}
fn iter(
&self,
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.check_access(ty.category());
self.task.iter(ty)
}
fn shrink_to_fit(&mut self, ty: CachedDataItemType) {
self.task.shrink_to_fit(ty)
}
#[track_caller]
fn extract_if<'l, F>(
&'l mut self,
ty: CachedDataItemType,
f: F,
) -> impl Iterator<Item = CachedDataItem>
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{
self.check_access(ty.category());
if !self.task_id.is_transient() && ty.is_persistent() {
self.task.track_modification(ty.category().into_specific());
}
self.task.extract_if(ty, f)
}
fn invalidate_serialization(&mut self) { fn invalidate_serialization(&mut self) {
// TODO this causes race conditions, since we never know when a value is changed. We can't // TODO this causes race conditions, since we never know when a value is changed. We can't
// "snapshot" the value correctly. // "snapshot" the value correctly.
...@@ -1024,10 +857,10 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -1024,10 +857,10 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
} }
fn prefetch(&mut self) -> Option<FxIndexMap<TaskId, TaskDataCategory>> { fn prefetch(&mut self) -> Option<FxIndexMap<TaskId, TaskDataCategory>> {
if self.task.state().prefetched() { if self.task.flags.prefetched() {
return None; return None;
} }
self.task.state_mut().set_prefetched(true); self.task.flags.set_prefetched(true);
let map = iter_many!(self, OutputDependency { target } => (target, TaskDataCategory::Meta)) let map = iter_many!(self, OutputDependency { target } => (target, TaskDataCategory::Meta))
.chain(iter_many!(self, CellDependency { target, key: _ } => (target.task, TaskDataCategory::All))) .chain(iter_many!(self, CellDependency { target, key: _ } => (target.task, TaskDataCategory::All)))
.chain(iter_many!(self, CollectiblesDependency { target } => (target.task, TaskDataCategory::All))) .chain(iter_many!(self, CollectiblesDependency { target } => (target.task, TaskDataCategory::All)))
...@@ -1037,6 +870,26 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -1037,6 +870,26 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
} }
} }
impl<'a, B: BackingStorage> TaskStorageAccessors for TaskGuardImpl<'a, B> {
fn typed(&self) -> &TaskStorage {
&self.task
}
fn typed_mut(&mut self) -> &mut TaskStorage {
&mut self.task
}
fn track_modification(&mut self, category: crate::backend::storage::SpecificTaskDataCategory) {
if !self.task_id.is_transient() {
self.task.track_modification(category);
}
}
fn check_access(&self, category: crate::backend::TaskDataCategory) {
self.check_access(category);
}
}
macro_rules! impl_operation { macro_rules! impl_operation {
($name:ident $type_path:path) => { ($name:ident $type_path:path) => {
impl From<$type_path> for AnyOperation { impl From<$type_path> for AnyOperation {
...@@ -1072,7 +925,7 @@ pub enum AnyOperation { ...@@ -1072,7 +925,7 @@ pub enum AnyOperation {
} }
impl AnyOperation { impl AnyOperation {
pub fn execute(self, ctx: &mut impl ExecuteContext) { pub fn execute(self, ctx: &mut impl ExecuteContext<'_>) {
match self { match self {
AnyOperation::ConnectChild(op) => op.execute(ctx), AnyOperation::ConnectChild(op) => op.execute(ctx),
AnyOperation::Invalidate(op) => op.execute(ctx), AnyOperation::Invalidate(op) => op.execute(ctx),
......
...@@ -12,7 +12,7 @@ use turbo_tasks::{CellId, FxIndexMap, TaskId, TypedSharedReference, backend::Cel ...@@ -12,7 +12,7 @@ use turbo_tasks::{CellId, FxIndexMap, TaskId, TypedSharedReference, backend::Cel
use crate::backend::operation::invalidate::TaskDirtyCause; use crate::backend::operation::invalidate::TaskDirtyCause;
use crate::{ use crate::{
backend::{ backend::{
TaskDataCategory, TaskDataCategory, TaskStorageAccessors,
operation::{ operation::{
AggregationUpdateQueue, ExecuteContext, Operation, TaskGuard, AggregationUpdateQueue, ExecuteContext, Operation, TaskGuard,
invalidate::make_task_dirty_internal, invalidate::make_task_dirty_internal,
......
...@@ -4,18 +4,12 @@ use std::{ ...@@ -4,18 +4,12 @@ use std::{
sync::{Arc, atomic::AtomicBool}, sync::{Arc, atomic::AtomicBool},
}; };
use bitfield::bitfield;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_tasks::{FxDashMap, TaskId, parallel}; use turbo_tasks::{FxDashMap, TaskId, parallel};
use crate::{ use crate::{
backend::dynamic_storage::DynamicStorage, backend::storage_schema::TaskStorage,
data::{ database::key_value_database::KeySpace,
AggregationNumber, CachedDataItem, CachedDataItemKey, CachedDataItemType,
CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut, LeafDistance,
OutputValue,
},
data_storage::{AutoMapStorage, DefaultStorage, OptionStorage},
utils::{ utils::{
dash_map_drop_contents::drop_contents, dash_map_drop_contents::drop_contents,
dash_map_multi::{RefMut, get_multiple_mut}, dash_map_multi::{RefMut, get_multiple_mut},
...@@ -37,621 +31,30 @@ impl TaskDataCategory { ...@@ -37,621 +31,30 @@ impl TaskDataCategory {
TaskDataCategory::All => unreachable!(), TaskDataCategory::All => unreachable!(),
} }
} }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SpecificTaskDataCategory {
Meta,
Data,
}
impl IntoIterator for TaskDataCategory { pub fn includes_data(self) -> bool {
type Item = TaskDataCategory; matches!(self, TaskDataCategory::Data | TaskDataCategory::All)
}
type IntoIter = TaskDataCategoryIterator;
fn into_iter(self) -> Self::IntoIter { pub fn includes_meta(self) -> bool {
match self { matches!(self, TaskDataCategory::Meta | TaskDataCategory::All)
TaskDataCategory::Meta => TaskDataCategoryIterator::Meta,
TaskDataCategory::Data => TaskDataCategoryIterator::Data,
TaskDataCategory::All => TaskDataCategoryIterator::All,
}
} }
} }
pub enum TaskDataCategoryIterator { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
All, pub enum SpecificTaskDataCategory {
Meta, Meta,
Data, Data,
None,
} }
impl Iterator for TaskDataCategoryIterator { impl SpecificTaskDataCategory {
type Item = TaskDataCategory; /// Returns the KeySpace for storing data of this category
pub fn key_space(self) -> KeySpace {
fn next(&mut self) -> Option<Self::Item> {
match self { match self {
TaskDataCategoryIterator::All => { SpecificTaskDataCategory::Meta => KeySpace::TaskMeta,
*self = TaskDataCategoryIterator::Data; SpecificTaskDataCategory::Data => KeySpace::TaskData,
Some(TaskDataCategory::Meta)
}
TaskDataCategoryIterator::Meta => {
*self = TaskDataCategoryIterator::None;
Some(TaskDataCategory::Meta)
}
TaskDataCategoryIterator::Data => {
*self = TaskDataCategoryIterator::None;
Some(TaskDataCategory::Data)
}
TaskDataCategoryIterator::None => None,
}
}
}
bitfield! {
// Note: Due to alignment in InnerStorage it doesn't matter if this struct is 1 or 4 bytes.
#[derive(Clone, Default)]
pub struct InnerStorageState(u32);
impl Debug;
pub meta_restored, set_meta_restored: 0;
pub data_restored, set_data_restored: 1;
/// Item was modified before snapshot mode was entered.
pub meta_modified, set_meta_modified: 2;
pub data_modified, set_data_modified: 3;
/// Item was modified after snapshot mode was entered. A snapshot was taken.
pub meta_snapshot, set_meta_snapshot: 4;
pub data_snapshot, set_data_snapshot: 5;
/// Prefetched dependencies
pub prefetched, set_prefetched: 6;
}
impl InnerStorageState {
pub fn set_restored(&mut self, category: TaskDataCategory) {
match category {
TaskDataCategory::Meta => {
self.set_meta_restored(true);
}
TaskDataCategory::Data => {
self.set_data_restored(true);
}
TaskDataCategory::All => {
self.set_meta_restored(true);
self.set_data_restored(true);
}
}
}
pub fn is_restored(&self, category: TaskDataCategory) -> bool {
match category {
TaskDataCategory::Meta => self.meta_restored(),
TaskDataCategory::Data => self.data_restored(),
TaskDataCategory::All => self.meta_restored() && self.data_restored(),
}
}
pub fn any_snapshot(&self) -> bool {
self.meta_snapshot() || self.data_snapshot()
}
pub fn any_modified(&self) -> bool {
self.meta_modified() || self.data_modified()
}
}
pub struct InnerStorageSnapshot {
leaf_distance: DefaultStorage<LeafDistance>,
aggregation_number: DefaultStorage<AggregationNumber>,
output_dependent: AutoMapStorage<TaskId, ()>,
output: OptionStorage<OutputValue>,
upper: AutoMapStorage<TaskId, u32>,
dynamic: DynamicStorage,
pub meta_modified: bool,
pub data_modified: bool,
}
impl From<&InnerStorage> for InnerStorageSnapshot {
fn from(inner: &InnerStorage) -> Self {
Self {
leaf_distance: inner.leaf_distance.clone(),
aggregation_number: inner.aggregation_number.clone(),
output_dependent: inner.output_dependent.clone(),
output: inner.output.clone(),
upper: inner.upper.clone(),
dynamic: inner.dynamic.snapshot_for_persisting(),
meta_modified: inner.state.meta_modified(),
data_modified: inner.state.data_modified(),
}
}
}
impl InnerStorageSnapshot {
pub fn iter_all(
&self,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
use crate::data_storage::Storage;
self.dynamic
.iter_all()
.chain(self.leaf_distance.iter().map(|(_, value)| {
(
CachedDataItemKey::LeafDistance {},
CachedDataItemValueRef::LeafDistance { value },
)
}))
.chain(self.aggregation_number.iter().map(|(_, value)| {
(
CachedDataItemKey::AggregationNumber {},
CachedDataItemValueRef::AggregationNumber { value },
)
}))
.chain(self.output.iter().map(|(_, value)| {
(
CachedDataItemKey::Output {},
CachedDataItemValueRef::Output { value },
)
}))
.chain(self.upper.iter().map(|(k, value)| {
(
CachedDataItemKey::Upper { task: *k },
CachedDataItemValueRef::Upper { value },
)
}))
.chain(self.output_dependent.iter().map(|(k, value)| {
(
CachedDataItemKey::OutputDependent { task: *k },
CachedDataItemValueRef::OutputDependent { value },
)
}))
}
pub fn len(&self) -> usize {
use crate::data_storage::Storage;
self.dynamic.len()
+ self.leaf_distance.len()
+ self.aggregation_number.len()
+ self.output.len()
+ self.upper.len()
+ self.output_dependent.len()
}
}
#[derive(Debug, Clone)]
pub struct InnerStorage {
leaf_distance: DefaultStorage<LeafDistance>,
aggregation_number: DefaultStorage<AggregationNumber>,
output_dependent: AutoMapStorage<TaskId, ()>,
output: OptionStorage<OutputValue>,
upper: AutoMapStorage<TaskId, u32>,
dynamic: DynamicStorage,
state: InnerStorageState,
}
impl InnerStorage {
fn new() -> Self {
Self {
leaf_distance: Default::default(),
aggregation_number: Default::default(),
output_dependent: Default::default(),
output: Default::default(),
upper: Default::default(),
dynamic: DynamicStorage::new(),
state: InnerStorageState::default(),
} }
} }
pub fn state(&self) -> &InnerStorageState {
&self.state
}
pub fn state_mut(&mut self) -> &mut InnerStorageState {
&mut self.state
}
}
#[macro_export]
macro_rules! generate_inner_storage_internal {
// Matching on CachedDataItem with a $value
(CachedDataItem: $self:ident, $item:ident, $value:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItem::$tag { $key_field, $value } = $item {
let result = $self.$field.$fn($key_field, $($args)*);
return $crate::generate_inner_storage_internal!(return_value: result, $return_ty: $tag $key_field => $field);
}
};
(CachedDataItem: $self:ident, $item:ident, $value:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident => $field:ident,) => {
if let CachedDataItem::$tag { $value } = $item {
let result = $self.$field.$fn((), $($args)*);
return $crate::generate_inner_storage_internal!(return_value: result, $return_ty: $tag => $field);
}
};
(CachedDataItem: $self:ident, $item:ident, $value:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(CachedDataItem: $self, $item, $value, $return_ty, $fn($($args)*): $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(CachedDataItem: $self, $item, $value, $return_ty, $fn($($args)*): $($config)+)
};
// Matching on CachedDataItemKey without a $value
(CachedDataItemKey: $self:ident, $item:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItemKey::$tag { $key_field } = $item {
let result = $self.$field.$fn($key_field, $($args)*);
return $crate::generate_inner_storage_internal!(return_value: result, $return_ty: $tag $key_field => $field);
}
};
(CachedDataItemKey: $self:ident, $item:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident => $field:ident,) => {
if let CachedDataItemKey::$tag { } = $item {
let result = $self.$field.$fn(&(), $($args)*);
return $crate::generate_inner_storage_internal!(return_value: result, $return_ty: $tag => $field);
}
};
(CachedDataItemKey: $self:ident, $item:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(CachedDataItemKey: $self, $item, $return_ty, $fn($($args)*): $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(CachedDataItemKey: $self, $item, $return_ty, $fn($($args)*): $($config)+)
};
// Matching on CachedDataItemType without a $value
(CachedDataItemType: $self:ident, $item:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident $($key_field:ident)? => $field:ident,) => {
if let CachedDataItemType::$tag = $item {
let result = $self.$field.$fn($($args)*);
return $crate::generate_inner_storage_internal!(return_value: result, $return_ty: $tag $($key_field)? => $field);
}
};
(CachedDataItemType: $self:ident, $item:ident, $return_ty:tt, $fn:ident($($args:tt)*): $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(CachedDataItemType: $self, $item, $return_ty, $fn($($args)*): $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(CachedDataItemType: $self, $item, $return_ty, $fn($($args)*): $($config)+)
};
// fn update
(update: $self:ident, $key:ident, $update:ident: $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItemKey::$tag { $key_field } = $key {
$self.$field.update($key_field, |old| {
let old = old.map(|old| CachedDataItemValue::$tag { value: old });
let new = $update(old);
new.map(|new| if let CachedDataItemValue::$tag { value } = new {
value
} else {
unreachable!()
})
});
return;
}
};
(update: $self:ident, $key:ident, $update:ident: $tag:ident => $field:ident,) => {
if let CachedDataItemKey::$tag { } = $key {
$self.$field.update((), |old| {
let old = old.map(|old| CachedDataItemValue::$tag { value: old });
let new = $update(old);
new.map(|new| if let CachedDataItemValue::$tag { value } = new {
value
} else {
unreachable!()
})
});
return;
}
};
(update: $self:ident, $key:ident, $update:ident: $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(update: $self, $key, $update: $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(update: $self, $key, $update: $($config)+)
};
// fn extend
(extend: $self:ident, $ty:ident, $items:ident: $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItemType::$tag = $ty {
return $self.$field.extend($items.map(|item| {
let pair = turbo_tasks::KeyValuePair::into_key_and_value(item);
if let (CachedDataItemKey::$tag { $key_field }, CachedDataItemValue::$tag { value }) = pair {
($key_field, value)
} else {
unreachable!()
}
}));
}
};
(extend: $self:ident, $ty:ident, $items:ident: $tag:ident => $field:ident,) => {
if let CachedDataItemType::$tag = $ty {
return $self.$field.extend($items.map(|item| {
let pair = turbo_tasks::KeyValuePair::into_key_and_value(item);
if let (_, CachedDataItemValue::$tag { value }) = pair {
((), value)
} else {
unreachable!()
}
}));
}
};
(extend: $self:ident, $ty:ident, $items:ident: $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(extend: $self, $ty, $items: $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(extend: $self, $ty, $items: $($config)+)
};
// fn get_mut_or_insert_with
(get_mut_or_insert_with: $self:ident, $key:ident, $insert_with:ident: $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItemKey::$tag { $key_field } = $key {
let value = $self.$field.get_mut_or_insert_with($key_field, || {
let value = $insert_with();
if let CachedDataItemValue::$tag { value } = value {
value
} else {
unreachable!()
}
});
return CachedDataItemValueRefMut::$tag { value };
}
};
(get_mut_or_insert_with: $self:ident, $key:ident, $insert_with:ident: $tag:ident => $field:ident,) => {
if let CachedDataItemKey::$tag { } = $key {
let value = $self.$field.get_mut_or_insert_with((), || {
let value = $insert_with();
if let CachedDataItemValue::$tag { value } = value {
value
} else {
unreachable!()
}
});
return CachedDataItemValueRefMut::$tag { value };
}
};
(get_mut_or_insert_with: $self:ident, $key:ident, $insert_with:ident: $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(get_mut_or_insert_with: $self, $key, $insert_with: $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(get_mut_or_insert_with: $self, $key, $insert_with: $($config)+)
};
// fn extract_if
(extract_if: $self:ident, $ty:ident, $f:ident: $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItemType::$tag = $ty {
let iter = $self.$field.extract_if(move |key, value| {
$f(CachedDataItemKey::$tag { $key_field: *key }, CachedDataItemValueRef::$tag { value })
}).map(|($key_field, value)| CachedDataItem::$tag { $key_field, value });
return InnerStorageIter::$tag(iter);
}
};
(extract_if: $self:ident, $ty:ident, $f:ident: $tag:ident => $field:ident,) => {
if let CachedDataItemType::$tag = $ty {
let iter = $self.$field.extract_if(move |_, value| {
$f(CachedDataItemKey::$tag { }, CachedDataItemValueRef::$tag { value })
}).map(|(_, value)| CachedDataItem::$tag { value });
return InnerStorageIter::$tag(iter);
}
};
(extract_if: $self:ident, $ty:ident, $f:ident: $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(extract_if: $self, $ty, $f: $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(extract_if: $self, $ty, $f: $($config)+)
};
// fn iter
(iter: $self:ident, $ty:ident: $tag:ident $key_field:ident => $field:ident,) => {
if let CachedDataItemType::$tag = $ty {
let iter = $self.$field.iter().map(|($key_field, value)| (CachedDataItemKey::$tag { $key_field: *$key_field }, CachedDataItemValueRef::$tag { value }));
return InnerStorageIter::$tag(iter);
}
};
(iter: $self:ident, $ty:ident: $tag:ident => $field:ident,) => {
if let CachedDataItemType::$tag = $ty {
let iter = $self.$field.iter().map(|(_, value)| (CachedDataItemKey::$tag { }, CachedDataItemValueRef::$tag { value }));
return InnerStorageIter::$tag(iter);
}
};
(iter: $self:ident, $ty:ident: $tag:ident $($key_field:ident)? => $field:ident, $($config:tt)+) => {
$crate::generate_inner_storage_internal!(iter: $self, $ty: $tag $($key_field)? => $field,);
$crate::generate_inner_storage_internal!(iter: $self, $ty: $($config)+)
};
// Return value handling
(return_value: $result:ident, none: $($more:tt)*) => {
$result
};
(return_value: $result:ident, option_value: $tag:ident $($more:tt)*) => {
$result.map(|value| CachedDataItemValue::$tag { value })
};
(return_value: $result:ident, option_ref: $tag:ident $($more:tt)*) => {
$result.map(|value| CachedDataItemValueRef::$tag { value })
};
(return_value: $result:ident, option_ref_mut: $tag:ident $($more:tt)*) => {
$result.map(|value| CachedDataItemValueRefMut::$tag { value })
};
// Input value handling
(input_value: $input:ident, option_value: $tag:ident $($more:tt)*) => {
$input.map(|value| {
if let CachedDataItemValue::$tag { value } = value {
value
} else {
unreachable!()
}
})
};
}
macro_rules! generate_inner_storage {
($($config:tt)*) => {
impl InnerStorage {
pub fn add(&mut self, item: CachedDataItem) -> bool {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItem: self, item, value, none, add(value): $($config)*);
self.dynamic.add(item)
}
pub fn extend(&mut self, ty: CachedDataItemType, items: impl Iterator<Item = CachedDataItem>) -> bool {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(extend: self, ty, items: $($config)*);
self.dynamic.extend(ty, items)
}
pub fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItem: self, item, value, option_value, insert(value): $($config)*);
self.dynamic.insert(item)
}
pub fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItemKey: self, key, option_value, remove(): $($config)*);
self.dynamic.remove(key)
}
pub fn count(&self, ty: CachedDataItemType) -> usize {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItemType: self, ty, none, len(): $($config)*);
self.dynamic.count(ty)
}
pub fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>> {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItemKey: self, key, option_ref, get(): $($config)*);
self.dynamic.get(key)
}
pub fn contains_key(&self, key: &CachedDataItemKey) -> bool {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItemKey: self, key, none, contains_key(): $($config)*);
self.dynamic.contains_key(key)
}
pub fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItemKey: self, key, option_ref_mut, get_mut(): $($config)*);
self.dynamic.get_mut(key)
}
pub fn shrink_to_fit(&mut self, ty: CachedDataItemType) {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(CachedDataItemType: self, ty, none, shrink_to_fit(): $($config)*);
self.dynamic.shrink_to_fit(ty)
}
pub fn update(
&mut self,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
) {
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(update: self, key, update: $($config)*);
self.dynamic.update(key, update)
}
pub fn extract_if<'l, F>(
&'l mut self,
ty: CachedDataItemType,
mut f: F,
) -> impl Iterator<Item = CachedDataItem> + use<'l, F>
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(extract_if: self, ty, f: $($config)*);
InnerStorageIter::Dynamic(self.dynamic.extract_if(ty, f))
}
pub fn get_mut_or_insert_with(
&mut self,
key: CachedDataItemKey,
f: impl FnOnce() -> CachedDataItemValue,
) -> CachedDataItemValueRefMut<'_>
{
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(get_mut_or_insert_with: self, key, f: $($config)*);
self.dynamic.get_mut_or_insert_with(key, f)
}
pub fn iter(
&self,
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)>
{
use crate::data_storage::Storage;
$crate::generate_inner_storage_internal!(iter: self, ty: $($config)*);
InnerStorageIter::Dynamic(self.dynamic.iter(ty))
}
}
};
}
generate_inner_storage!(
LeafDistance => leaf_distance,
AggregationNumber => aggregation_number,
OutputDependent task => output_dependent,
Output => output,
Upper task => upper,
);
enum InnerStorageIter<A, B, C, D, E, F> {
LeafDistance(A),
AggregationNumber(B),
OutputDependent(C),
Output(D),
Upper(E),
Dynamic(F),
}
impl<T, A, B, C, D, E, F> Iterator for InnerStorageIter<A, B, C, D, E, F>
where
A: Iterator<Item = T>,
B: Iterator<Item = T>,
C: Iterator<Item = T>,
D: Iterator<Item = T>,
E: Iterator<Item = T>,
F: Iterator<Item = T>,
{
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
match self {
InnerStorageIter::LeafDistance(iter) => iter.next(),
InnerStorageIter::AggregationNumber(iter) => iter.next(),
InnerStorageIter::OutputDependent(iter) => iter.next(),
InnerStorageIter::Output(iter) => iter.next(),
InnerStorageIter::Upper(iter) => iter.next(),
InnerStorageIter::Dynamic(iter) => iter.next(),
}
}
}
impl InnerStorage {
pub fn iter_all(
&self,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
use crate::data_storage::Storage;
self.dynamic
.iter_all()
.chain(self.leaf_distance.iter().map(|(_, value)| {
(
CachedDataItemKey::LeafDistance {},
CachedDataItemValueRef::LeafDistance { value },
)
}))
.chain(self.aggregation_number.iter().map(|(_, value)| {
(
CachedDataItemKey::AggregationNumber {},
CachedDataItemValueRef::AggregationNumber { value },
)
}))
.chain(self.output.iter().map(|(_, value)| {
(
CachedDataItemKey::Output {},
CachedDataItemValueRef::Output { value },
)
}))
.chain(self.upper.iter().map(|(k, value)| {
(
CachedDataItemKey::Upper { task: *k },
CachedDataItemValueRef::Upper { value },
)
}))
.chain(self.output_dependent.iter().map(|(k, value)| {
(
CachedDataItemKey::OutputDependent { task: *k },
CachedDataItemValueRef::OutputDependent { value },
)
}))
}
pub fn len(&self) -> usize {
use crate::data_storage::Storage;
self.dynamic.len()
+ self.leaf_distance.len()
+ self.aggregation_number.len()
+ self.output.len()
+ self.upper.len()
+ self.output_dependent.len()
}
} }
enum ModifiedState { enum ModifiedState {
...@@ -664,13 +67,13 @@ enum ModifiedState { ...@@ -664,13 +67,13 @@ enum ModifiedState {
/// Snapshot(None): /// Snapshot(None):
/// It was not modified before snapshot mode was entered, but it was accessed during snapshot /// It was not modified before snapshot mode was entered, but it was accessed during snapshot
/// mode. Or the snapshot was already taken out by the snapshot operation. /// mode. Or the snapshot was already taken out by the snapshot operation.
Snapshot(Option<Box<InnerStorageSnapshot>>), Snapshot(Option<Box<TaskStorage>>),
} }
pub struct Storage { pub struct Storage {
snapshot_mode: AtomicBool, snapshot_mode: AtomicBool,
modified: FxDashMap<TaskId, ModifiedState>, modified: FxDashMap<TaskId, ModifiedState>,
map: FxDashMap<TaskId, Box<InnerStorage>>, map: FxDashMap<TaskId, Box<TaskStorage>>,
} }
impl Storage { impl Storage {
...@@ -705,9 +108,9 @@ impl Storage { ...@@ -705,9 +108,9 @@ impl Storage {
'l, 'l,
T, T,
R, R,
PP: for<'a> Fn(TaskId, &'a InnerStorage) -> T + Sync, PP: for<'a> Fn(TaskId, &'a TaskStorage) -> T + Sync,
P: Fn(TaskId, T) -> R + Sync, P: Fn(TaskId, T) -> R + Sync,
PS: Fn(TaskId, Box<InnerStorageSnapshot>) -> R + Sync, PS: Fn(TaskId, Box<TaskStorage>) -> R + Sync,
>( >(
&'l self, &'l self,
preprocess: &'l PP, preprocess: &'l PP,
...@@ -723,7 +126,7 @@ impl Storage { ...@@ -723,7 +126,7 @@ impl Storage {
// The number of shards is much larger than the number of threads, so the effect of the // The number of shards is much larger than the number of threads, so the effect of the
// locks held is negligible. // locks held is negligible.
parallel::map_collect::<_, _, Vec<_>>(self.modified.shards(), |shard| { parallel::map_collect::<_, _, Vec<_>>(self.modified.shards(), |shard| {
let mut direct_snapshots: Vec<(TaskId, Box<InnerStorageSnapshot>)> = Vec::new(); let mut direct_snapshots: Vec<(TaskId, Box<TaskStorage>)> = Vec::new();
let mut modified: SmallVec<[TaskId; 4]> = SmallVec::new(); let mut modified: SmallVec<[TaskId; 4]> = SmallVec::new();
{ {
// Take the snapshots from the modified map // Take the snapshots from the modified map
...@@ -787,9 +190,8 @@ impl Storage { ...@@ -787,9 +190,8 @@ impl Storage {
// We also need to unset all the modified flags. // We also need to unset all the modified flags.
for key in removed_modified { for key in removed_modified {
if let Some(mut inner) = self.map.get_mut(&key) { if let Some(mut inner) = self.map.get_mut(&key) {
let state = inner.state_mut(); inner.flags.set_data_modified(false);
state.set_data_modified(false); inner.flags.set_meta_modified(false);
state.set_meta_modified(false);
} }
} }
...@@ -816,14 +218,13 @@ impl Storage { ...@@ -816,14 +218,13 @@ impl Storage {
// And update the flags // And update the flags
for key in removed_snapshots { for key in removed_snapshots {
if let Some(mut inner) = self.map.get_mut(&key) { if let Some(mut inner) = self.map.get_mut(&key) {
let state = inner.state_mut(); if inner.flags.meta_snapshot() {
if state.meta_snapshot() { inner.flags.set_meta_snapshot(false);
state.set_meta_snapshot(false); inner.flags.set_meta_modified(true);
state.set_meta_modified(true);
} }
if state.data_snapshot() { if inner.flags.data_snapshot() {
state.set_data_snapshot(false); inner.flags.set_data_snapshot(false);
state.set_data_modified(true); inner.flags.set_data_modified(true);
} }
} }
} }
...@@ -840,7 +241,7 @@ impl Storage { ...@@ -840,7 +241,7 @@ impl Storage {
pub fn access_mut(&self, key: TaskId) -> StorageWriteGuard<'_> { pub fn access_mut(&self, key: TaskId) -> StorageWriteGuard<'_> {
let inner = match self.map.entry(key) { let inner = match self.map.entry(key) {
dashmap::mapref::entry::Entry::Occupied(e) => e.into_ref(), dashmap::mapref::entry::Entry::Occupied(e) => e.into_ref(),
dashmap::mapref::entry::Entry::Vacant(e) => e.insert(Box::new(InnerStorage::new())), dashmap::mapref::entry::Entry::Vacant(e) => e.insert(Box::new(TaskStorage::new())),
}; };
StorageWriteGuard { StorageWriteGuard {
storage: self, storage: self,
...@@ -853,7 +254,7 @@ impl Storage { ...@@ -853,7 +254,7 @@ impl Storage {
key1: TaskId, key1: TaskId,
key2: TaskId, key2: TaskId,
) -> (StorageWriteGuard<'_>, StorageWriteGuard<'_>) { ) -> (StorageWriteGuard<'_>, StorageWriteGuard<'_>) {
let (a, b) = get_multiple_mut(&self.map, key1, key2, || Box::new(InnerStorage::new())); let (a, b) = get_multiple_mut(&self.map, key1, key2, || Box::new(TaskStorage::new()));
( (
StorageWriteGuard { StorageWriteGuard {
storage: self, storage: self,
...@@ -874,75 +275,61 @@ impl Storage { ...@@ -874,75 +275,61 @@ impl Storage {
pub struct StorageWriteGuard<'a> { pub struct StorageWriteGuard<'a> {
storage: &'a Storage, storage: &'a Storage,
inner: RefMut<'a, TaskId, Box<InnerStorage>>, inner: RefMut<'a, TaskId, Box<TaskStorage>>,
} }
impl StorageWriteGuard<'_> { impl StorageWriteGuard<'_> {
/// Tracks mutation of this task /// Tracks mutation of this task
pub fn track_modification(&mut self, category: SpecificTaskDataCategory) { pub fn track_modification(&mut self, category: SpecificTaskDataCategory) {
let state = self.inner.state(); let flags = &self.inner.flags;
let snapshot = match category { if flags.is_snapshot(category) {
SpecificTaskDataCategory::Meta => state.meta_snapshot(), return;
SpecificTaskDataCategory::Data => state.data_snapshot(), }
}; let modified = flags.is_modified(category);
if !snapshot { match (self.storage.snapshot_mode(), modified) {
let modified = match category { (false, false) => {
SpecificTaskDataCategory::Meta => state.meta_modified(), // Not in snapshot mode and item is unmodified
SpecificTaskDataCategory::Data => state.data_modified(), if !flags.any_snapshot() && !flags.any_modified() {
}; self.storage
match (self.storage.snapshot_mode(), modified) { .modified
(false, false) => { .insert(*self.inner.key(), ModifiedState::Modified);
// Not in snapshot mode and item is unmodified
if !state.any_snapshot() && !state.any_modified() {
self.storage
.modified
.insert(*self.inner.key(), ModifiedState::Modified);
}
let state = self.inner.state_mut();
match category {
SpecificTaskDataCategory::Meta => state.set_meta_modified(true),
SpecificTaskDataCategory::Data => state.set_data_modified(true),
}
} }
(false, true) => { self.inner.flags.set_modified(category, true);
// Not in snapshot mode and item is already modified }
// Do nothing (false, true) => {
// Not in snapshot mode and item is already modified
// Do nothing
}
(true, false) => {
// In snapshot mode and item is unmodified (so it's not part of the snapshot)
if !flags.any_snapshot() {
self.storage
.modified
.insert(*self.inner.key(), ModifiedState::Snapshot(None));
} }
(true, false) => { self.inner.flags.set_snapshot(category, true);
// In snapshot mode and item is unmodified (so it's not part of the snapshot) }
if !state.any_snapshot() { (true, true) => {
self.storage // In snapshot mode and item is modified (so it's part of the snapshot)
.modified // We need to store the original version that is part of the snapshot
.insert(*self.inner.key(), ModifiedState::Snapshot(None)); if !flags.any_snapshot() {
} // Snapshot all non-transient fields but keep the modified bits.
let state = self.inner.state_mut(); let mut snapshot = self.inner.clone_snapshot();
match category { snapshot.flags.set_data_modified(flags.data_modified());
SpecificTaskDataCategory::Meta => state.set_meta_snapshot(true), snapshot.flags.set_meta_modified(flags.meta_modified());
SpecificTaskDataCategory::Data => state.set_data_snapshot(true), self.storage.modified.insert(
} *self.inner.key(),
} ModifiedState::Snapshot(Some(Box::new(snapshot))),
(true, true) => { );
// In snapshot mode and item is modified (so it's part of the snapshot)
// We need to store the original version that is part of the snapshot
if !state.any_snapshot() {
self.storage.modified.insert(
*self.inner.key(),
ModifiedState::Snapshot(Some(Box::new((&**self.inner).into()))),
);
}
let state = self.inner.state_mut();
match category {
SpecificTaskDataCategory::Meta => state.set_meta_snapshot(true),
SpecificTaskDataCategory::Data => state.set_data_snapshot(true),
}
} }
self.inner.flags.set_snapshot(category, true);
} }
} }
} }
} }
impl Deref for StorageWriteGuard<'_> { impl Deref for StorageWriteGuard<'_> {
type Target = InnerStorage; type Target = TaskStorage;
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
&self.inner &self.inner
...@@ -955,6 +342,27 @@ impl DerefMut for StorageWriteGuard<'_> { ...@@ -955,6 +342,27 @@ impl DerefMut for StorageWriteGuard<'_> {
} }
} }
// TODO: this implementation is only needed to bootstrap new tasks
impl super::storage_schema::TaskStorageAccessors for StorageWriteGuard<'_> {
fn typed(&self) -> &super::storage_schema::TaskStorage {
&self.inner
}
fn typed_mut(&mut self) -> &mut super::storage_schema::TaskStorage {
&mut self.inner
}
fn track_modification(&mut self, category: SpecificTaskDataCategory) {
// Delegate to the existing track_modification method
StorageWriteGuard::track_modification(self, category)
}
fn check_access(&self, _category: super::TaskDataCategory) {
// StorageWriteGuard doesn't have category tracking - that's handled by TaskGuardImpl.
// This is a no-op for Stor
}
}
macro_rules! count { macro_rules! count {
($task:ident, $key:ident) => {{ $task.count($crate::data::CachedDataItemType::$key) }}; ($task:ident, $key:ident) => {{ $task.count($crate::data::CachedDataItemType::$key) }};
} }
...@@ -962,7 +370,7 @@ macro_rules! count { ...@@ -962,7 +370,7 @@ macro_rules! count {
macro_rules! get { macro_rules! get {
($task:ident, $key:ident $input:tt) => {{ ($task:ident, $key:ident $input:tt) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use $crate::backend::operation::TaskGuard; use $crate::backend::storage_schema::TaskStorageAccessors;
if let Some($crate::data::CachedDataItemValueRef::$key { if let Some($crate::data::CachedDataItemValueRef::$key {
value, value,
}) = $task.get(&$crate::data::CachedDataItemKey::$key $input) { }) = $task.get(&$crate::data::CachedDataItemKey::$key $input) {
...@@ -979,7 +387,7 @@ macro_rules! get { ...@@ -979,7 +387,7 @@ macro_rules! get {
macro_rules! get_mut { macro_rules! get_mut {
($task:ident, $key:ident $input:tt) => {{ ($task:ident, $key:ident $input:tt) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use $crate::backend::operation::TaskGuard; use $crate::backend::storage_schema::TaskStorageAccessors;
if let Some($crate::data::CachedDataItemValueRefMut::$key { if let Some($crate::data::CachedDataItemValueRefMut::$key {
value, value,
}) = $task.get_mut(&$crate::data::CachedDataItemKey::$key $input) { }) = $task.get_mut(&$crate::data::CachedDataItemKey::$key $input) {
...@@ -1020,7 +428,7 @@ macro_rules! get_mut_or_insert_with { ...@@ -1020,7 +428,7 @@ macro_rules! get_mut_or_insert_with {
macro_rules! iter_many { macro_rules! iter_many {
($task:ident, $key:ident $key_pattern:tt $(if $cond:expr)? => $iter_item:expr) => {{ ($task:ident, $key:ident $key_pattern:tt $(if $cond:expr)? => $iter_item:expr) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use $crate::backend::operation::TaskGuard; use $crate::backend::storage_schema::TaskStorageAccessors;
$task $task
.iter($crate::data::CachedDataItemType::$key) .iter($crate::data::CachedDataItemType::$key)
.filter_map(|(key, _)| match key { .filter_map(|(key, _)| match key {
...@@ -1032,7 +440,7 @@ macro_rules! iter_many { ...@@ -1032,7 +440,7 @@ macro_rules! iter_many {
}}; }};
($task:ident, $key:ident $input:tt $value_pattern:tt $(if $cond:expr)? => $iter_item:expr) => {{ ($task:ident, $key:ident $input:tt $value_pattern:tt $(if $cond:expr)? => $iter_item:expr) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use $crate::backend::operation::TaskGuard; use $crate::backend::storage_schema::TaskStorageAccessors;
$task $task
.iter($crate::data::CachedDataItemType::$key) .iter($crate::data::CachedDataItemType::$key)
.filter_map(|(key, value)| match (key, value) { .filter_map(|(key, value)| match (key, value) {
...@@ -1058,7 +466,7 @@ macro_rules! get_many { ...@@ -1058,7 +466,7 @@ macro_rules! get_many {
macro_rules! update { macro_rules! update {
($task:ident, $key:ident $input:tt, $update:expr) => {{ ($task:ident, $key:ident $input:tt, $update:expr) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use $crate::backend::operation::TaskGuard; use $crate::backend::storage_schema::TaskStorageAccessors;
#[allow(unused_mut)] #[allow(unused_mut)]
let mut update = $update; let mut update = $update;
$task.update($crate::data::CachedDataItemKey::$key $input, |old| { $task.update($crate::data::CachedDataItemKey::$key $input, |old| {
...@@ -1161,7 +569,7 @@ macro_rules! update_count_and_get { ...@@ -1161,7 +569,7 @@ macro_rules! update_count_and_get {
macro_rules! remove { macro_rules! remove {
($task:ident, $key:ident $input:tt) => {{ ($task:ident, $key:ident $input:tt) => {{
#[allow(unused_imports)] #[allow(unused_imports)]
use $crate::backend::operation::TaskGuard; use $crate::backend::storage_schema::TaskStorageAccessors;
if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.remove( if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.remove(
&$crate::data::CachedDataItemKey::$key $input &$crate::data::CachedDataItemKey::$key $input
) { ) {
...@@ -1197,7 +605,7 @@ impl Drop for SnapshotGuard<'_> { ...@@ -1197,7 +605,7 @@ impl Drop for SnapshotGuard<'_> {
} }
pub struct SnapshotShard<'l, PP, P, PS> { pub struct SnapshotShard<'l, PP, P, PS> {
direct_snapshots: Vec<(TaskId, Box<InnerStorageSnapshot>)>, direct_snapshots: Vec<(TaskId, Box<TaskStorage>)>,
modified: SmallVec<[TaskId; 4]>, modified: SmallVec<[TaskId; 4]>,
storage: &'l Storage, storage: &'l Storage,
guard: Option<Arc<SnapshotGuard<'l>>>, guard: Option<Arc<SnapshotGuard<'l>>>,
...@@ -1208,9 +616,9 @@ pub struct SnapshotShard<'l, PP, P, PS> { ...@@ -1208,9 +616,9 @@ pub struct SnapshotShard<'l, PP, P, PS> {
impl<'l, T, R, PP, P, PS> Iterator for SnapshotShard<'l, PP, P, PS> impl<'l, T, R, PP, P, PS> Iterator for SnapshotShard<'l, PP, P, PS>
where where
PP: for<'a> Fn(TaskId, &'a InnerStorage) -> T + Sync, PP: for<'a> Fn(TaskId, &'a TaskStorage) -> T + Sync,
P: Fn(TaskId, T) -> R + Sync, P: Fn(TaskId, T) -> R + Sync,
PS: Fn(TaskId, Box<InnerStorageSnapshot>) -> R + Sync, PS: Fn(TaskId, Box<TaskStorage>) -> R + Sync,
{ {
type Item = R; type Item = R;
...@@ -1220,8 +628,7 @@ where ...@@ -1220,8 +628,7 @@ where
} }
while let Some(task_id) = self.modified.pop() { while let Some(task_id) = self.modified.pop() {
let inner = self.storage.map.get(&task_id).unwrap(); let inner = self.storage.map.get(&task_id).unwrap();
let state = inner.state(); if !inner.flags.any_snapshot() {
if !state.any_snapshot() {
let preprocessed = (self.preprocess)(task_id, &inner); let preprocessed = (self.preprocess)(task_id, &inner);
drop(inner); drop(inner);
return Some((self.process)(task_id, preprocessed)); return Some((self.process)(task_id, preprocessed));
...@@ -1243,24 +650,3 @@ where ...@@ -1243,24 +650,3 @@ where
None None
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_inner_storage_size() {
// InnerStorage size affects memory usage per task.
// We track this to catch unexpected bloat from type changes.
assert_eq!(
std::mem::size_of::<InnerStorage>(),
136,
"InnerStorage size changed - please review if this is intentional"
);
assert_eq!(
std::mem::size_of::<InnerStorageSnapshot>(),
136,
"InnerStorageSnapshot size changed - please review if this is intentional"
);
}
}
...@@ -17,26 +17,22 @@ ...@@ -17,26 +17,22 @@
//! - `data` - Frequently changed bulk data (dependencies, cell data) //! - `data` - Frequently changed bulk data (dependencies, cell data)
//! - `meta` - Rarely changed metadata (output, aggregation, flags) //! - `meta` - Rarely changed metadata (output, aggregation, flags)
//! - `transient` - Not serialized, only exists in memory //! - `transient` - Not serialized, only exists in memory
// TODO(PR 2): Remove this once the storage schema is integrated with the rest of the codebase.
// This module is scaffolding for the TaskStorage macro and is not yet used.
#![allow(dead_code)]
use turbo_tasks::{ use turbo_tasks::{
CellId, SharedReference, TaskId, TraitTypeId, TypedSharedReference, ValueTypeId, task_storage, CellId, SharedReference, TaskId, TraitTypeId, TypedSharedReference, ValueTypeId, task_storage,
}; };
use crate::data::{ use crate::{
ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, backend::counter_map::CounterMap,
InProgressCellState, InProgressState, OutputValue, data::{
ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness,
InProgressCellState, InProgressState, LeafDistance, OutputValue,
},
}; };
/// Auto-set storage for small sets of keys with unit values. /// Auto-set storage for small sets of keys with unit values.
/// Optimized for small collections (< 8 items use SmallVec inline). /// Optimized for small collections (< 8 items use SmallVec inline).
type AutoSet<K> = auto_hash_map::AutoSet<K, std::hash::BuildHasherDefault<rustc_hash::FxHasher>, 1>; type AutoSet<K> = auto_hash_map::AutoSet<K, std::hash::BuildHasherDefault<rustc_hash::FxHasher>, 1>;
use super::counter_map::CounterMap;
/// Auto-map storage for key-value pairs. /// Auto-map storage for key-value pairs.
type AutoMap<K, V> = type AutoMap<K, V> =
auto_hash_map::AutoMap<K, V, std::hash::BuildHasherDefault<rustc_hash::FxHasher>, 1>; auto_hash_map::AutoMap<K, V, std::hash::BuildHasherDefault<rustc_hash::FxHasher>, 1>;
...@@ -59,37 +55,92 @@ struct TaskStorageSchema { ...@@ -59,37 +55,92 @@ struct TaskStorageSchema {
// ========================================================================= // =========================================================================
// INLINE FIELDS (hot path, always allocated inline) // INLINE FIELDS (hot path, always allocated inline)
// ========================================================================= // =========================================================================
/// The task's distance for prioritizing invalidation execution
#[field(
storage = "direct",
category = "data",
inline,
default,
variant = "LeafDistance"
)]
pub leaf_distance: LeafDistance,
/// The task's aggregation number for the aggregation tree. /// The task's aggregation number for the aggregation tree.
/// Uses Default::default() semantics - a zero aggregation number means "not set". /// Uses Default::default() semantics - a zero aggregation number means "not set".
#[field(storage = "direct", category = "meta", inline, default)]
#[field(
storage = "direct",
category = "meta",
inline,
default,
variant = "AggregationNumber"
)]
pub aggregation_number: AggregationNumber, pub aggregation_number: AggregationNumber,
/// Tasks that depend on this task's output. /// Tasks that depend on this task's output.
#[field(storage = "auto_set", category = "data", inline, filter_transient)]
#[field(
storage = "auto_set",
category = "data",
inline,
filter_transient,
variant = "OutputDependent",
key_field = "task"
)]
pub output_dependent: AutoSet<TaskId>, pub output_dependent: AutoSet<TaskId>,
/// The task's output value. /// The task's output value.
/// Filtered during serialization to skip transient outputs (referencing transient tasks). /// Filtered during serialization to skip transient outputs (referencing transient tasks).
#[field(storage = "direct", category = "meta", inline, filter_transient)] #[field(
storage = "direct",
category = "meta",
inline,
filter_transient,
variant = "Output"
)]
pub output: Option<OutputValue>, pub output: Option<OutputValue>,
/// Upper nodes in the aggregation tree (reference counted). /// Upper nodes in the aggregation tree (reference counted).
#[field(storage = "counter_map", category = "meta", inline, filter_transient)] #[field(
storage = "counter_map",
category = "meta",
inline,
filter_transient,
variant = "Upper",
key_field = "task"
)]
pub upper: CounterMap<TaskId, u32>, pub upper: CounterMap<TaskId, u32>,
// ========================================================================= // =========================================================================
// COLLECTIBLES (meta) // COLLECTIBLES (meta)
// ========================================================================= // =========================================================================
/// Collectibles emitted by this task (reference counted). /// Collectibles emitted by this task (reference counted).
#[field(storage = "counter_map", category = "meta", filter_transient)] #[field(
storage = "counter_map",
category = "meta",
filter_transient,
variant = "Collectible",
key_field = "collectible"
)]
pub collectibles: CounterMap<CollectibleRef, i32>, pub collectibles: CounterMap<CollectibleRef, i32>,
/// Aggregated collectibles from the subgraph. /// Aggregated collectibles from the subgraph.
#[field(storage = "counter_map", category = "meta", filter_transient)] #[field(
storage = "counter_map",
category = "meta",
filter_transient,
variant = "AggregatedCollectible",
key_field = "collectible"
)]
pub aggregated_collectibles: CounterMap<CollectibleRef, i32>, pub aggregated_collectibles: CounterMap<CollectibleRef, i32>,
/// Outdated collectibles to be cleaned up (transient). /// Outdated collectibles to be cleaned up (transient).
#[field(storage = "counter_map", category = "transient")] #[field(
storage = "counter_map",
category = "transient",
variant = "OutdatedCollectible",
key_field = "collectible"
)]
pub outdated_collectibles: CounterMap<CollectibleRef, i32>, pub outdated_collectibles: CounterMap<CollectibleRef, i32>,
// ========================================================================= // =========================================================================
...@@ -98,25 +149,44 @@ struct TaskStorageSchema { ...@@ -98,25 +149,44 @@ struct TaskStorageSchema {
// ========================================================================= // =========================================================================
/// Whether the task is dirty (needs re-execution). /// Whether the task is dirty (needs re-execution).
/// Absent = clean, present = dirty with the specified Dirtyness state. /// Absent = clean, present = dirty with the specified Dirtyness state.
#[field(storage = "direct", category = "meta")] #[field(storage = "direct", category = "meta", variant = "Dirty")]
pub dirty: Dirtyness, pub dirty: Dirtyness,
/// Count of dirty containers in the aggregated subgraph. /// Count of dirty containers in the aggregated subgraph.
/// Absent = 0, present = actual count. /// Absent = 0, present = actual count.
#[field(storage = "direct", category = "meta")] #[field(
storage = "direct",
category = "meta",
variant = "AggregatedDirtyContainerCount"
)]
pub aggregated_dirty_container_count: i32, pub aggregated_dirty_container_count: i32,
/// Individual dirty containers in the aggregated subgraph. /// Individual dirty containers in the aggregated subgraph.
#[field(storage = "counter_map", category = "meta", filter_transient)] #[field(
storage = "counter_map",
category = "meta",
filter_transient,
variant = "AggregatedDirtyContainer",
key_field = "task"
)]
pub aggregated_dirty_containers: CounterMap<TaskId, i32>, pub aggregated_dirty_containers: CounterMap<TaskId, i32>,
/// Count of clean containers in current session (transient). /// Count of clean containers in current session (transient).
/// Absent = 0, present = actual count. /// Absent = 0, present = actual count.
#[field(storage = "direct", category = "transient")] #[field(
storage = "direct",
category = "transient",
variant = "AggregatedCurrentSessionCleanContainerCount"
)]
pub aggregated_current_session_clean_container_count: i32, pub aggregated_current_session_clean_container_count: i32,
/// Individual clean containers in current session (transient). /// Individual clean containers in current session (transient).
#[field(storage = "counter_map", category = "transient")] #[field(
storage = "counter_map",
category = "transient",
variant = "AggregatedCurrentSessionCleanContainer",
key_field = "task"
)]
pub aggregated_current_session_clean_containers: CounterMap<TaskId, i32>, pub aggregated_current_session_clean_containers: CounterMap<TaskId, i32>,
// ========================================================================= // =========================================================================
...@@ -124,15 +194,19 @@ struct TaskStorageSchema { ...@@ -124,15 +194,19 @@ struct TaskStorageSchema {
// Persisted flags come first, then transient flags. // Persisted flags come first, then transient flags.
// ========================================================================= // =========================================================================
/// Whether the task has an invalidator. /// Whether the task has an invalidator.
#[field(storage = "flag", category = "meta")] #[field(storage = "flag", category = "meta", variant = "HasInvalidator")]
pub invalidator: bool, pub invalidator: bool,
/// Whether the task output is immutable (persisted). /// Whether the task output is immutable (persisted).
#[field(storage = "flag", category = "meta")] #[field(storage = "flag", category = "meta", variant = "Immutable")]
pub immutable: bool, pub immutable: bool,
/// Whether clean in current session (transient flag). /// Whether clean in current session (transient flag).
#[field(storage = "flag", category = "transient")] #[field(
storage = "flag",
category = "transient",
variant = "CurrentSessionClean"
)]
pub current_session_clean: bool, pub current_session_clean: bool,
// ========================================================================= // =========================================================================
...@@ -171,85 +245,156 @@ struct TaskStorageSchema { ...@@ -171,85 +245,156 @@ struct TaskStorageSchema {
// CHILDREN & AGGREGATION (meta) // CHILDREN & AGGREGATION (meta)
// ========================================================================= // =========================================================================
/// Child tasks of this task. /// Child tasks of this task.
#[field(storage = "auto_set", category = "meta", filter_transient)] #[field(
storage = "auto_set",
category = "meta",
filter_transient,
variant = "Child",
key_field = "task"
)]
pub children: AutoSet<TaskId>, pub children: AutoSet<TaskId>,
/// Follower nodes in the aggregation tree (reference counted). /// Follower nodes in the aggregation tree (reference counted).
#[field(storage = "counter_map", category = "meta", filter_transient)] #[field(
storage = "counter_map",
category = "meta",
filter_transient,
variant = "Follower",
key_field = "task"
)]
pub followers: CounterMap<TaskId, u32>, pub followers: CounterMap<TaskId, u32>,
// ========================================================================= // =========================================================================
// DEPENDENCIES (data) // DEPENDENCIES (data)
// ========================================================================= // =========================================================================
/// Tasks whose output this task depends on. #[field(
#[field(storage = "auto_set", category = "data", filter_transient)] storage = "auto_set",
category = "data",
filter_transient,
variant = "OutputDependency",
key_field = "target"
)]
pub output_dependencies: AutoSet<TaskId>, pub output_dependencies: AutoSet<TaskId>,
/// Cells this task depends on. /// Cells this task depends on.
#[field(storage = "auto_set", category = "data", filter_transient)] #[field(
pub cell_dependencies: AutoSet<CellRef>, storage = "auto_set",
category = "data",
filter_transient,
variant = "CellDependency",
key_field = "target, key"
)]
pub cell_dependencies: AutoSet<(CellRef, Option<u64>)>,
/// Collectibles this task depends on. /// Collectibles this task depends on.
#[field(storage = "auto_set", category = "data", filter_transient)] #[field(
storage = "auto_set",
category = "data",
filter_transient,
variant = "CollectiblesDependency",
key_field = "target"
)]
pub collectibles_dependencies: AutoSet<CollectiblesRef>, pub collectibles_dependencies: AutoSet<CollectiblesRef>,
/// Outdated output dependencies to be cleaned up (transient). /// Outdated output dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient")] #[field(
storage = "auto_set",
category = "transient",
variant = "OutdatedOutputDependency",
key_field = "target"
)]
pub outdated_output_dependencies: AutoSet<TaskId>, pub outdated_output_dependencies: AutoSet<TaskId>,
/// Outdated cell dependencies to be cleaned up (transient). /// Outdated cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient")] #[field(
pub outdated_cell_dependencies: AutoSet<CellRef>, storage = "auto_set",
category = "transient",
variant = "OutdatedCellDependency",
key_field = "target, key"
)]
pub outdated_cell_dependencies: AutoSet<(CellRef, Option<u64>)>,
/// Outdated collectibles dependencies to be cleaned up (transient). /// Outdated collectibles dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient")] #[field(
storage = "auto_set",
category = "transient",
variant = "OutdatedCollectiblesDependency",
key_field = "target"
)]
pub outdated_collectibles_dependencies: AutoSet<CollectiblesRef>, pub outdated_collectibles_dependencies: AutoSet<CollectiblesRef>,
// ========================================================================= // =========================================================================
// DEPENDENTS - Tasks that depend on this task's cells // DEPENDENTS - Tasks that depend on this task's cells
// ========================================================================= // =========================================================================
/// Tasks that depend on specific cells of this task. #[field(
/// Maps CellId -> Set<TaskId> storage = "auto_set",
#[field(storage = "auto_set", category = "data", filter_transient)] category = "data",
variant = "CellDependent",
key_field = "cell, key, task",
filter_transient
)]
pub cell_dependents: AutoSet<(CellId, Option<u64>, TaskId)>, pub cell_dependents: AutoSet<(CellId, Option<u64>, TaskId)>,
/// Tasks that depend on collectibles of a specific type from this task. /// Tasks that depend on collectibles of a specific type from this task.
/// Maps TraitTypeId -> Set<TaskId> /// Maps TraitTypeId -> Set<TaskId>
#[field(storage = "auto_set", category = "meta", filter_transient)]
#[field(
storage = "auto_set",
category = "meta",
variant = "CollectiblesDependent",
key_field = "collectible_type, task",
filter_transient
)]
pub collectibles_dependents: AutoSet<(TraitTypeId, TaskId)>, pub collectibles_dependents: AutoSet<(TraitTypeId, TaskId)>,
// ========================================================================= // =========================================================================
// CELL DATA (data) // CELL DATA (data)
// ========================================================================= // =========================================================================
/// Persistent cell data (serializable). /// Persistent cell data (serializable).
#[field(storage = "auto_map", category = "data")] #[field(
storage = "auto_map",
category = "data",
variant = "CellData",
key_field = "cell"
)]
pub cell_data: AutoMap<CellId, TypedSharedReference>, pub cell_data: AutoMap<CellId, TypedSharedReference>,
/// Transient cell data (not serializable). /// Transient cell data (not serializable).
#[field(storage = "auto_map", category = "transient")] #[field(
storage = "auto_map",
category = "transient",
variant = "TransientCellData",
key_field = "cell"
)]
pub transient_cell_data: AutoMap<CellId, SharedReference>, pub transient_cell_data: AutoMap<CellId, SharedReference>,
/// Maximum cell index per cell type. /// Maximum cell index per cell type.
#[field(storage = "auto_map", category = "data")] #[field(
storage = "auto_map",
category = "data",
variant = "CellTypeMaxIndex",
key_field = "cell_type"
)]
pub cell_type_max_index: AutoMap<ValueTypeId, u32>, pub cell_type_max_index: AutoMap<ValueTypeId, u32>,
// ========================================================================= // =========================================================================
// TRANSIENT EXECUTION STATE (transient) // TRANSIENT EXECUTION STATE (transient)
// ========================================================================= // =========================================================================
/// Activeness state for root/once tasks (transient). /// Activeness state for root/once tasks (transient).
/// Note: Lazy storage provides natural optionality - #[field(storage = "direct", category = "transient", variant = "Activeness")]
/// presence in Vec<LazyField> = Some, absence = None. No Option wrapper needed.
#[field(storage = "direct", category = "transient")]
pub activeness: ActivenessState, pub activeness: ActivenessState,
/// In-progress execution state (transient). /// In-progress execution state (transient).
/// Note: Lazy storage provides natural optionality - #[field(storage = "direct", category = "transient", variant = "InProgress")]
/// presence in Vec<LazyField> = Some, absence = None. No Option wrapper needed.
#[field(storage = "direct", category = "transient")]
pub in_progress: InProgressState, pub in_progress: InProgressState,
/// In-progress cell state for cells being computed (transient). /// In-progress cell state for cells being computed (transient).
#[field(storage = "auto_map", category = "transient")] #[field(
storage = "auto_map",
category = "transient",
variant = "InProgressCell",
key_field = "cell"
)]
pub in_progress_cells: AutoMap<CellId, InProgressCellState>, pub in_progress_cells: AutoMap<CellId, InProgressCellState>,
} }
...@@ -257,7 +402,7 @@ struct TaskStorageSchema { ...@@ -257,7 +402,7 @@ struct TaskStorageSchema {
// TaskFlags helper methods (for InnerStorageState compatibility) // TaskFlags helper methods (for InnerStorageState compatibility)
// ============================================================================= // =============================================================================
use crate::backend::TaskDataCategory; use crate::backend::{TaskDataCategory, storage::SpecificTaskDataCategory};
impl TaskFlags { impl TaskFlags {
/// Set restored flags based on category /// Set restored flags based on category
...@@ -294,6 +439,38 @@ impl TaskFlags { ...@@ -294,6 +439,38 @@ impl TaskFlags {
pub fn any_modified(&self) -> bool { pub fn any_modified(&self) -> bool {
self.meta_modified() || self.data_modified() self.meta_modified() || self.data_modified()
} }
/// Check if the specified category is modified
pub fn is_modified(&self, category: SpecificTaskDataCategory) -> bool {
match category {
SpecificTaskDataCategory::Meta => self.meta_modified(),
SpecificTaskDataCategory::Data => self.data_modified(),
}
}
/// Set the modified flag for the specified category
pub fn set_modified(&mut self, category: SpecificTaskDataCategory, value: bool) {
match category {
SpecificTaskDataCategory::Meta => self.set_meta_modified(value),
SpecificTaskDataCategory::Data => self.set_data_modified(value),
}
}
/// Check if the specified category has a snapshot
pub fn is_snapshot(&self, category: SpecificTaskDataCategory) -> bool {
match category {
SpecificTaskDataCategory::Meta => self.meta_snapshot(),
SpecificTaskDataCategory::Data => self.data_snapshot(),
}
}
/// Set the snapshot flag for the specified category
pub fn set_snapshot(&mut self, category: SpecificTaskDataCategory, value: bool) {
match category {
SpecificTaskDataCategory::Meta => self.set_meta_snapshot(value),
SpecificTaskDataCategory::Data => self.set_data_snapshot(value),
}
}
} }
// ============================================================================= // =============================================================================
...@@ -388,6 +565,38 @@ impl TaskStorage { ...@@ -388,6 +565,38 @@ impl TaskStorage {
None None
} }
} }
/// Encode fields for the specified category
pub fn encode<E: bincode::enc::Encoder>(
&self,
category: SpecificTaskDataCategory,
encoder: &mut E,
) -> Result<(), bincode::error::EncodeError> {
match category {
SpecificTaskDataCategory::Meta => self.encode_meta(encoder),
SpecificTaskDataCategory::Data => self.encode_data(encoder),
}
}
/// Decode fields for the specified category
pub fn decode<D: bincode::de::Decoder>(
&mut self,
category: SpecificTaskDataCategory,
decoder: &mut D,
) -> Result<(), bincode::error::DecodeError> {
match category {
SpecificTaskDataCategory::Meta => self.decode_meta(decoder),
SpecificTaskDataCategory::Data => self.decode_data(decoder),
}
}
/// Clone only the fields for the specified category
pub fn clone_category_snapshot(&self, category: SpecificTaskDataCategory) -> TaskStorage {
match category {
SpecificTaskDataCategory::Meta => self.clone_meta_snapshot(),
SpecificTaskDataCategory::Data => self.clone_data_snapshot(),
}
}
} }
// Support serialization filtering for CellDependents and CollectibleDependents // Support serialization filtering for CellDependents and CollectibleDependents
...@@ -406,6 +615,12 @@ impl IsTransient for (CellId, Option<u64>, TaskId) { ...@@ -406,6 +615,12 @@ impl IsTransient for (CellId, Option<u64>, TaskId) {
self.2.is_transient() self.2.is_transient()
} }
} }
impl IsTransient for (CellRef, Option<u64>) {
fn is_transient(&self) -> bool {
self.0.task.is_transient()
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::mem::size_of; use std::mem::size_of;
...@@ -413,7 +628,38 @@ mod tests { ...@@ -413,7 +628,38 @@ mod tests {
use turbo_tasks::{CellId, TaskId}; use turbo_tasks::{CellId, TaskId};
use super::*; use super::*;
use crate::data::{AggregationNumber, CellRef, Dirtyness, OutputValue}; use crate::{
backend::storage::SpecificTaskDataCategory,
data::{AggregationNumber, CellRef, Dirtyness, OutputValue},
};
/// Test wrapper that implements TaskStorageAccessors for testing the CachedDataItem adapter.
/// This wrapper doesn't track modifications since we're just testing functionality.
struct TestStorage(TaskStorage);
impl TestStorage {
fn new() -> Self {
Self(TaskStorage::new())
}
}
impl TaskStorageAccessors for TestStorage {
fn typed(&self) -> &TaskStorage {
&self.0
}
fn typed_mut(&mut self) -> &mut TaskStorage {
&mut self.0
}
fn track_modification(&mut self, _category: SpecificTaskDataCategory) {
// No-op for tests
}
fn check_access(&self, _category: crate::backend::TaskDataCategory) {
// No-op for tests
}
}
#[test] #[test]
fn test_accessors() { fn test_accessors() {
...@@ -698,13 +944,16 @@ mod tests { ...@@ -698,13 +944,16 @@ mod tests {
original original
.output_dependencies_mut() .output_dependencies_mut()
.insert(TaskId::new(200).unwrap()); .insert(TaskId::new(200).unwrap());
original.cell_dependencies_mut().insert(CellRef { original.cell_dependencies_mut().insert((
task: TaskId::new(1).unwrap(), CellRef {
cell: CellId { task: TaskId::new(1).unwrap(),
type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) }, cell: CellId {
index: 0, type_id: unsafe { turbo_tasks::ValueTypeId::new_unchecked(1) },
index: 0,
},
}, },
}); None,
));
// Set lazy data transient field (should NOT be serialized) // Set lazy data transient field (should NOT be serialized)
original original
...@@ -815,23 +1064,185 @@ mod tests { ...@@ -815,23 +1064,185 @@ mod tests {
#[test] #[test]
#[cfg(target_pointer_width = "64")] #[cfg(target_pointer_width = "64")]
fn test_schema_size() { fn test_schema_size() {
// TaskStorage uses lazy storage for most fields, keeping inline storage minimal.
// Current layout (128 bytes):
// - output_dependent (AutoSet<TaskId>): 24 bytes
// - aggregation_number (AggregationNumber with default): 12 bytes
// - output (Option<OutputValue>): 32 bytes
// - upper (CounterMap<TaskId, u32>): 24 bytes
// - flags (TaskFlags): 8 bytes
// - lazy (Vec<LazyField>): 24 bytes
// - padding: 4 bytes
//
// Use exact size check to catch regressions in either direction.
assert_eq!( assert_eq!(
size_of::<TaskStorage>(), size_of::<TaskStorage>(),
128, 136,
"TaskStorage size changed! Was 128 bytes, now {} bytes. If this is intentional, \ "TaskStorage size changed! If this is intentional, update this test."
update this test.",
size_of::<TaskStorage>()
); );
} }
// ==========================================================================
// CachedDataItem Adapter Tests
// ==========================================================================
#[test]
fn test_adapter_basic() {
use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue};
let mut storage = TestStorage::new();
let task1 = unsafe { TaskId::new_unchecked(1) };
let task2 = unsafe { TaskId::new_unchecked(2) };
// Test add for inline direct field (Output)
let output_item = CachedDataItem::Output {
value: OutputValue::Output(task1),
};
assert!(storage.add(output_item.clone()));
assert!(!storage.add(output_item)); // Second add should return false
// Test get for Output
let key = CachedDataItemKey::Output {};
assert!(storage.get(&key).is_some());
assert!(storage.contains_key(&key));
// Test remove for Output
let removed = storage.remove(&key);
assert!(matches!(removed, Some(CachedDataItemValue::Output { .. })));
assert!(storage.get(&key).is_none());
// Test add for inline AutoSet field (OutputDependent)
let dep_item = CachedDataItem::OutputDependent {
task: task1,
value: (),
};
assert!(storage.add(dep_item.clone()));
assert!(!storage.add(dep_item)); // Already exists
// Verify via typed accessor
assert!(storage.typed().output_dependent().contains(&task1));
// Test add another
let dep_item2 = CachedDataItem::OutputDependent {
task: task2,
value: (),
};
assert!(storage.add(dep_item2));
assert_eq!(storage.typed().output_dependent().len(), 2);
// Test remove OutputDependent
let key = CachedDataItemKey::OutputDependent { task: task1 };
let removed = storage.remove(&key);
assert!(matches!(
removed,
Some(CachedDataItemValue::OutputDependent { value: () })
));
assert_eq!(storage.typed().output_dependent().len(), 1);
assert!(!storage.typed().output_dependent().contains(&task1));
assert!(storage.typed().output_dependent().contains(&task2));
}
#[test]
fn test_adapter_flags() {
use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue};
let mut storage = TestStorage::new();
// Test flag field (Immutable)
let immutable_item = CachedDataItem::Immutable { value: () };
assert!(storage.add(immutable_item.clone()));
assert!(storage.typed().flags.immutable());
assert!(!storage.add(immutable_item)); // Already set
// Test get for flag
let key = CachedDataItemKey::Immutable {};
assert!(storage.get(&key).is_some());
// Test remove for flag
let removed = storage.remove(&key);
assert!(matches!(
removed,
Some(CachedDataItemValue::Immutable { value: () })
));
assert!(!storage.typed().flags.immutable());
// Removing again should return None
assert!(storage.remove(&key).is_none());
}
#[test]
fn test_adapter_counter_map() {
use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue};
let mut storage = TestStorage::new();
let task1 = unsafe { TaskId::new_unchecked(1) };
let task2 = unsafe { TaskId::new_unchecked(2) };
// Test inline CounterMap field (Upper)
let upper_item = CachedDataItem::Upper {
task: task1,
value: 5,
};
assert!(storage.add(upper_item));
assert_eq!(storage.upper().get(&task1), Some(&5));
// Update the value (insert returns old value)
let upper_update = CachedDataItem::Upper {
task: task1,
value: 10,
};
let old = storage.insert(upper_update);
assert!(matches!(old, Some(CachedDataItemValue::Upper { value: 5 })));
assert_eq!(storage.upper().get(&task1), Some(&10));
// Test lazy CounterMap field (Follower)
let follower_item = CachedDataItem::Follower {
task: task2,
value: 3,
};
assert!(storage.add(follower_item));
assert_eq!(storage.followers().unwrap().get(&task2), Some(&3));
// Test remove Follower
let key = CachedDataItemKey::Follower { task: task2 };
let removed = storage.remove(&key);
assert!(matches!(
removed,
Some(CachedDataItemValue::Follower { value: 3 })
));
// After removal, followers map might be empty or the key just removed
assert!(storage.followers().is_none_or(|f| f.get(&task2).is_none()));
}
#[test]
fn test_adapter_lazy_autoset() {
use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue};
let mut storage = TestStorage::new();
let task1 = unsafe { TaskId::new_unchecked(1) };
let task2 = unsafe { TaskId::new_unchecked(2) };
// Test lazy AutoSet field (Child)
let child_item = CachedDataItem::Child {
task: task1,
value: (),
};
assert!(storage.add(child_item));
assert!(storage.children().is_some());
assert!(storage.children().unwrap().contains(&task1));
// Add another child
let child_item2 = CachedDataItem::Child {
task: task2,
value: (),
};
assert!(storage.add(child_item2));
assert_eq!(storage.children().unwrap().len(), 2);
// Test get
let key = CachedDataItemKey::Child { task: task1 };
assert!(storage.get(&key).is_some());
let key_missing = CachedDataItemKey::Child {
task: unsafe { TaskId::new_unchecked(999) },
};
assert!(storage.get(&key_missing).is_none());
// Test remove
let removed = storage.remove(&key);
assert!(matches!(
removed,
Some(CachedDataItemValue::Child { value: () })
));
assert_eq!(storage.children().unwrap().len(), 1);
assert!(!storage.children().unwrap().contains(&task1));
}
} }
...@@ -6,8 +6,7 @@ use smallvec::SmallVec; ...@@ -6,8 +6,7 @@ use smallvec::SmallVec;
use turbo_tasks::{TaskId, backend::CachedTaskType}; use turbo_tasks::{TaskId, backend::CachedTaskType};
use crate::{ use crate::{
backend::{AnyOperation, TaskDataCategory}, backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
data::CachedDataItem,
utils::chunked_vec::ChunkedVec, utils::chunked_vec::ChunkedVec,
}; };
...@@ -44,8 +43,13 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -44,8 +43,13 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
type ReadTransaction<'l>; type ReadTransaction<'l>;
fn next_free_task_id(&self) -> Result<TaskId>; fn next_free_task_id(&self) -> Result<TaskId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>; fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
#[allow(clippy::ptr_arg)] fn serialize(
fn serialize(&self, task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>>; &self,
task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<SmallVec<[u8; 16]>>;
fn save_snapshot<I>( fn save_snapshot<I>(
&self, &self,
operations: Vec<Arc<AnyOperation>>, operations: Vec<Arc<AnyOperation>>,
...@@ -78,6 +82,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -78,6 +82,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId, task_id: TaskId,
) -> Result<Option<Arc<CachedTaskType>>>; ) -> Result<Option<Arc<CachedTaskType>>>;
/// Lookup and decode fields directly into TaskStorage.
/// # Safety /// # Safety
/// ///
/// `tx` must be a transaction from this BackingStorage instance. /// `tx` must be a transaction from this BackingStorage instance.
...@@ -85,8 +91,12 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -85,8 +91,12 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId, task_id: TaskId,
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<CachedDataItem>>; storage: &mut TaskStorage,
) -> Result<()>;
/// Batch lookup and decode data for multiple tasks directly into TaskStorage instances.
/// Returns a vector of TaskStorage, one for each task_id in the input slice.
/// # Safety /// # Safety
/// ///
/// `tx` must be a transaction from this BackingStorage instance. /// `tx` must be a transaction from this BackingStorage instance.
...@@ -94,16 +104,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -94,16 +104,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId], task_ids: &[TaskId],
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> { ) -> Result<Vec<TaskStorage>>;
let mut results = Vec::with_capacity(task_ids.len());
for &task_id in task_ids {
// TODO more efficient batch implementation
let data = unsafe { self.lookup_data(tx, task_id, category)? };
results.push(data);
}
Ok(results)
}
fn shutdown(&self) -> Result<()> { fn shutdown(&self) -> Result<()> {
Ok(()) Ok(())
...@@ -134,11 +136,14 @@ where ...@@ -134,11 +136,14 @@ where
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> { fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> {
either::for_both!(self, this => this.uncompleted_operations()) either::for_both!(self, this => this.uncompleted_operations())
} }
fn serialize(
fn serialize(&self, task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>> { &self,
either::for_both!(self, this => this.serialize(task, data)) task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<SmallVec<[u8; 16]>> {
either::for_both!(self, this => this.serialize(task, data, category))
} }
fn save_snapshot<I>( fn save_snapshot<I>(
&self, &self,
operations: Vec<Arc<AnyOperation>>, operations: Vec<Arc<AnyOperation>>,
...@@ -207,16 +212,17 @@ where ...@@ -207,16 +212,17 @@ where
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId, task_id: TaskId,
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<CachedDataItem>> { storage: &mut TaskStorage,
) -> Result<()> {
match self { match self {
Either::Left(this) => { Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref())); let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
unsafe { this.lookup_data(tx, task_id, category) } unsafe { this.lookup_data(tx, task_id, category, storage) }
} }
Either::Right(this) => { Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref())); let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
unsafe { this.lookup_data(tx, task_id, category) } unsafe { this.lookup_data(tx, task_id, category, storage) }
} }
} }
} }
...@@ -225,8 +231,8 @@ where ...@@ -225,8 +231,8 @@ where
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId], task_ids: &[TaskId],
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> { ) -> Result<Vec<TaskStorage>> {
match self { match self {
Either::Left(this) => { Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref())); let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
......
...@@ -7,10 +7,7 @@ use turbo_tasks::{ ...@@ -7,10 +7,7 @@ use turbo_tasks::{
event::{Event, EventListener}, event::{Event, EventListener},
}; };
use crate::{ use crate::backend::TaskDataCategory;
backend::TaskDataCategory,
data_storage::{AutoMapStorage, OptionStorage, Storage},
};
// this traits are needed for the transient variants of `CachedDataItem` // this traits are needed for the transient variants of `CachedDataItem`
// transient variants are never cloned or compared // transient variants are never cloned or compared
...@@ -733,6 +730,5 @@ mod tests { ...@@ -733,6 +730,5 @@ mod tests {
assert_eq!(std::mem::size_of::<super::CachedDataItem>(), 40); assert_eq!(std::mem::size_of::<super::CachedDataItem>(), 40);
assert_eq!(std::mem::size_of::<super::CachedDataItemKey>(), 32); assert_eq!(std::mem::size_of::<super::CachedDataItemKey>(), 32);
assert_eq!(std::mem::size_of::<super::CachedDataItemValue>(), 32); assert_eq!(std::mem::size_of::<super::CachedDataItemValue>(), 32);
assert_eq!(std::mem::size_of::<super::CachedDataItemStorage>(), 56);
} }
} }
use std::hash::{BuildHasherDefault, Hash};
use auto_hash_map::{AutoMap, map::Entry};
use rustc_hash::FxHasher;
pub trait Storage {
type K;
type V;
type Iterator<'l>: Iterator<Item = (&'l Self::K, &'l Self::V)> + 'l
where
Self: 'l;
/// Adds a key-value pair to the storage, if the key is not already present.
/// Returns `true` if the key was not present and the value was added.
/// Returns `false` if the key was already present.
fn add(&mut self, key: Self::K, value: Self::V) -> bool;
/// Extends the storage with key-value pairs from the iterator.
/// Overwrites existing keys.
/// Returns `true` if all keys were new and added.
/// Returns `false` if any key was already present.
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool;
fn insert(&mut self, key: Self::K, value: Self::V) -> Option<Self::V>;
fn remove(&mut self, key: &Self::K) -> Option<Self::V>;
fn contains_key(&self, key: &Self::K) -> bool;
fn get(&self, key: &Self::K) -> Option<&Self::V>;
fn get_mut(&mut self, key: &Self::K) -> Option<&mut Self::V>;
fn get_mut_or_insert_with(&mut self, key: Self::K, f: impl FnOnce() -> Self::V)
-> &mut Self::V;
fn extract_if<'l, F>(&'l mut self, f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l;
fn update(&mut self, key: Self::K, update: impl FnOnce(Option<Self::V>) -> Option<Self::V>);
fn shrink_to_fit(&mut self);
fn is_empty(&self) -> bool;
fn len(&self) -> usize;
fn iter(&self) -> Self::Iterator<'_>;
}
fn value_to_key_value<V>(value: &V) -> (&(), &V) {
(&(), value)
}
#[derive(Debug, Clone)]
pub struct OptionStorage<V> {
value: Option<V>,
}
impl<V> Default for OptionStorage<V> {
fn default() -> Self {
Self { value: None }
}
}
impl<V> Storage for OptionStorage<V> {
type K = ();
type V = V;
type Iterator<'l>
= std::option::IntoIter<(&'l (), &'l V)>
where
Self: 'l;
fn add(&mut self, _: (), value: V) -> bool {
if self.value.is_none() {
self.value = Some(value);
true
} else {
false
}
}
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool {
let mut added = true;
for (_, value) in items {
added = self.insert((), value).is_none() && added;
}
added
}
fn insert(&mut self, _: (), value: V) -> Option<V> {
self.value.replace(value)
}
fn remove(&mut self, _: &()) -> Option<V> {
self.value.take()
}
fn contains_key(&self, _: &()) -> bool {
self.value.is_some()
}
fn get(&self, _: &()) -> Option<&V> {
self.value.as_ref()
}
fn get_mut(&mut self, _: &()) -> Option<&mut V> {
self.value.as_mut()
}
fn get_mut_or_insert_with(&mut self, _: (), f: impl FnOnce() -> V) -> &mut V {
self.value.get_or_insert_with(f)
}
fn shrink_to_fit(&mut self) {
// Nothing to do
}
fn is_empty(&self) -> bool {
self.value.is_none()
}
fn len(&self) -> usize {
if self.value.is_some() { 1 } else { 0 }
}
fn iter(&self) -> Self::Iterator<'_> {
self.value.as_ref().map(value_to_key_value).into_iter()
}
fn extract_if<'l, F>(&'l mut self, mut f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l,
{
if let Some(value) = self.value.as_mut()
&& f(&(), value)
{
return self.value.take().map(|v| ((), v)).into_iter();
}
None.into_iter()
}
fn update(&mut self, _: (), update: impl FnOnce(Option<V>) -> Option<V>) {
self.value = update(self.value.take());
}
}
/// Storage for a single value that uses `Default::default()` to initialize the value "empty".
#[derive(Debug, Clone)]
pub struct DefaultStorage<V> {
value: V,
}
impl<V: Default> Default for DefaultStorage<V> {
fn default() -> Self {
Self {
value: V::default(),
}
}
}
impl<V: Default + PartialEq> Storage for DefaultStorage<V> {
type K = ();
type V = V;
type Iterator<'l>
= std::option::IntoIter<(&'l (), &'l V)>
where
Self: 'l;
fn add(&mut self, _: (), value: V) -> bool {
if self.value == V::default() {
self.value = value;
true
} else {
false
}
}
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool {
let mut added = true;
for (_, value) in items {
added = self.insert((), value).is_none() && added;
}
added
}
fn insert(&mut self, _: (), value: V) -> Option<V> {
let old = std::mem::replace(&mut self.value, value);
if old == V::default() { None } else { Some(old) }
}
fn remove(&mut self, _: &()) -> Option<V> {
let old = std::mem::take(&mut self.value);
if old == V::default() { None } else { Some(old) }
}
fn contains_key(&self, _: &()) -> bool {
self.value != V::default()
}
fn get(&self, _: &()) -> Option<&V> {
if self.value != V::default() {
Some(&self.value)
} else {
None
}
}
fn get_mut(&mut self, _: &()) -> Option<&mut V> {
if self.value != V::default() {
Some(&mut self.value)
} else {
None
}
}
fn get_mut_or_insert_with(&mut self, _: (), f: impl FnOnce() -> V) -> &mut V {
if self.value == V::default() {
self.value = f();
}
&mut self.value
}
fn shrink_to_fit(&mut self) {
// Nothing to do
}
fn is_empty(&self) -> bool {
self.value == V::default()
}
fn len(&self) -> usize {
if self.value != V::default() { 1 } else { 0 }
}
fn iter(&self) -> Self::Iterator<'_> {
if self.value != V::default() {
Some(value_to_key_value(&self.value)).into_iter()
} else {
None.into_iter()
}
}
fn extract_if<'l, F>(&'l mut self, mut f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l,
{
if self.value != V::default() && f(&(), &mut self.value) {
let old = std::mem::take(&mut self.value);
return Some(((), old)).into_iter();
}
None.into_iter()
}
fn update(&mut self, _: (), update: impl FnOnce(Option<V>) -> Option<V>) {
let old = std::mem::take(&mut self.value);
let old_opt = if old == V::default() { None } else { Some(old) };
self.value = update(old_opt).unwrap_or_default();
}
}
#[derive(Debug, Clone)]
pub struct AutoMapStorage<K, V> {
map: AutoMap<K, V, BuildHasherDefault<FxHasher>, 1>,
}
impl<K, V> Default for AutoMapStorage<K, V> {
fn default() -> Self {
Self {
map: AutoMap::default(),
}
}
}
impl<K: Hash + Eq, V> Storage for AutoMapStorage<K, V> {
type K = K;
type V = V;
type Iterator<'l>
= auto_hash_map::map::Iter<'l, K, V>
where
Self: 'l;
fn add(&mut self, key: K, value: V) -> bool {
match self.map.entry(key) {
auto_hash_map::map::Entry::Vacant(entry) => {
entry.insert(value);
true
}
auto_hash_map::map::Entry::Occupied(_) => false,
}
}
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool {
let len = self.map.len();
let mut count = 0;
self.map.extend(items.inspect(|_| count += 1));
self.map.len() == len + count
}
fn insert(&mut self, key: K, value: V) -> Option<V> {
self.map.insert(key, value)
}
fn remove(&mut self, key: &K) -> Option<V> {
self.map
.remove(key)
.inspect(|_| self.map.shrink_amortized())
}
fn contains_key(&self, key: &K) -> bool {
self.map.contains_key(key)
}
fn get(&self, key: &K) -> Option<&V> {
self.map.get(key)
}
fn get_mut(&mut self, key: &K) -> Option<&mut V> {
self.map.get_mut(key)
}
fn get_mut_or_insert_with(&mut self, key: K, f: impl FnOnce() -> V) -> &mut V {
self.map.entry(key).or_insert_with(f)
}
fn shrink_to_fit(&mut self) {
self.map.shrink_to_fit()
}
fn is_empty(&self) -> bool {
self.map.is_empty()
}
fn len(&self) -> usize {
self.map.len()
}
fn iter(&self) -> Self::Iterator<'_> {
self.map.iter()
}
fn extract_if<'l, F>(&'l mut self, f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l,
{
self.map.extract_if(f)
}
fn update(&mut self, key: K, update: impl FnOnce(Option<V>) -> Option<V>) {
match self.map.entry(key) {
Entry::Vacant(e) => {
if let Some(new) = update(None) {
e.insert(new);
}
}
Entry::Occupied(e) => e.replace_entry_with(move |_, v| update(Some(v))),
}
}
}
...@@ -7,7 +7,8 @@ use std::{ ...@@ -7,7 +7,8 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use turbo_bincode::{ use turbo_bincode::{
TurboBincodeBuffer, turbo_bincode_decode, turbo_bincode_encode, turbo_bincode_encode_into, TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder, turbo_bincode_decode,
turbo_bincode_encode, turbo_bincode_encode_into,
}; };
use turbo_tasks::{ use turbo_tasks::{
TaskId, TaskId,
...@@ -18,9 +19,8 @@ use turbo_tasks::{ ...@@ -18,9 +19,8 @@ use turbo_tasks::{
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{AnyOperation, TaskDataCategory}, backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backing_storage::{BackingStorage, BackingStorageSealed}, backing_storage::{BackingStorage, BackingStorageSealed},
data::CachedDataItem,
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
db_versioning::handle_db_versioning, db_versioning::handle_db_versioning,
...@@ -250,8 +250,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -250,8 +250,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
get(&self.inner.database).context("Unable to read uncompleted operations from database") get(&self.inner.database).context("Unable to read uncompleted operations from database")
} }
fn serialize(&self, task: TaskId, data: &Vec<CachedDataItem>) -> Result<TurboBincodeBuffer> { fn serialize(
encode_task_data(task, data) &self,
task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<TurboBincodeBuffer> {
encode_task_data(task, data, category)
} }
fn save_snapshot<I>( fn save_snapshot<I>(
...@@ -510,64 +515,61 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -510,64 +515,61 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&self, &self,
tx: Option<&T::ReadTransaction<'_>>, tx: Option<&T::ReadTransaction<'_>>,
task_id: TaskId, task_id: TaskId,
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<CachedDataItem>> { storage: &mut TaskStorage,
) -> Result<()> {
let inner = &*self.inner; let inner = &*self.inner;
fn lookup<D: KeyValueDatabase>( fn lookup<D: KeyValueDatabase>(
database: &D, database: &D,
tx: &D::ReadTransaction<'_>, tx: &D::ReadTransaction<'_>,
task_id: TaskId, task_id: TaskId,
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<CachedDataItem>> { storage: &mut TaskStorage,
let Some(bytes) = database.get( ) -> Result<()> {
tx, let Some(bytes) =
category_to_key_space(category), database.get(tx, category.key_space(), IntKey::new(*task_id).as_ref())?
IntKey::new(*task_id).as_ref(),
)?
else { else {
return Ok(Vec::new()); return Ok(());
}; };
let result: Vec<CachedDataItem> = turbo_bincode_decode(bytes.borrow())?; let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
Ok(result) storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))
} }
inner inner
.with_tx(tx, |tx| lookup(&inner.database, tx, task_id, category)) .with_tx(tx, |tx| {
.with_context(|| format!("Looking up data for {task_id} from database failed")) lookup(&inner.database, tx, task_id, category, storage)
})
.with_context(|| format!("Looking up task storage for {task_id} from database failed"))
} }
unsafe fn batch_lookup_data( unsafe fn batch_lookup_data(
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId], task_ids: &[TaskId],
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> { ) -> Result<Vec<TaskStorage>> {
let inner = &*self.inner; let inner = &*self.inner;
fn lookup<D: KeyValueDatabase>( fn lookup<D: KeyValueDatabase>(
database: &D, database: &D,
tx: &D::ReadTransaction<'_>, tx: &D::ReadTransaction<'_>,
task_ids: &[TaskId], task_ids: &[TaskId],
category: TaskDataCategory, category: SpecificTaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> { ) -> Result<Vec<TaskStorage>> {
let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect(); let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect();
let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>(); let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>();
let bytes = database.batch_get( let bytes = database.batch_get(tx, category.key_space(), &keys)?;
tx,
match category {
TaskDataCategory::Meta => KeySpace::TaskMeta,
TaskDataCategory::Data => KeySpace::TaskData,
TaskDataCategory::All => unreachable!(),
},
&keys,
)?;
bytes bytes
.into_iter() .into_iter()
.map(|opt_bytes| { .map(|opt_bytes| {
let mut storage = TaskStorage::new();
if let Some(bytes) = opt_bytes { if let Some(bytes) = opt_bytes {
let result: Vec<CachedDataItem> = turbo_bincode_decode(bytes.borrow())?; let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
Ok(result) storage
} else { .decode(category, &mut decoder)
Ok(Vec::new()) .map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))?;
} }
Ok(storage)
}) })
.collect::<Result<Vec<_>>>() .collect::<Result<Vec<_>>>()
} }
...@@ -575,7 +577,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -575,7 +577,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
.with_tx(tx, |tx| lookup(&inner.database, tx, task_ids, category)) .with_tx(tx, |tx| lookup(&inner.database, tx, task_ids, category))
.with_context(|| { .with_context(|| {
format!( format!(
"Looking up data for {} tasks from database failed", "Looking up typed data for {} tasks from database failed",
task_ids.len() task_ids.len()
) )
}) })
...@@ -761,55 +763,29 @@ where ...@@ -761,55 +763,29 @@ where
}) })
} }
fn encode_task_data(task: TaskId, data: &Vec<CachedDataItem>) -> Result<TurboBincodeBuffer> { fn encode_task_data(
let orig_result = turbo_bincode_encode(data); task: TaskId,
if !cfg!(feature = "verify_serialization") data: &TaskStorage,
&& let Ok(value) = orig_result category: SpecificTaskDataCategory,
{ ) -> Result<TurboBincodeBuffer> {
return Ok(value); // TODO: see if the caller can pass us a buffer instead of us allocating a new one.
// This should be possible and save a lot of small allocations.
let mut buffer = TurboBincodeBuffer::new();
let mut encoder = new_turbo_bincode_encoder(&mut buffer);
data.encode(category, &mut encoder)?;
if !cfg!(feature = "verify_serialization") {
return Ok(buffer);
} }
let mut error = Ok(()); TaskStorage::new()
let mut filtered_data = data.clone(); .decode(category, &mut new_turbo_bincode_decoder(buffer.borrow()))
filtered_data.retain(|item| match turbo_bincode_encode(&item) { .with_context(|| {
Ok(buf) => { format!(
if cfg!(feature = "verify_serialization") { "expected to be able to decode serialized data for '{category:?}' information for \
let deserialized = turbo_bincode_decode::<CachedDataItem>(&buf); {task}"
if let Err(err) = deserialized { )
println!("Data item would not be deserializable {task}: {err:?}\n{item:?}"); })?;
return false;
}
}
true
}
Err(err) => {
if item.is_optional() {
if cfg!(feature = "verify_serialization") {
println!(
"Skipping non-encodable optional item for {task}: {item:?} due to {err}"
);
}
} else {
error =
Err(err).context(format!("Unable to encode data item for {task}: {item:?}"));
}
false
}
});
error?;
(if filtered_data.len() == data.len() {
orig_result
} else {
turbo_bincode_encode(&filtered_data)
})
.with_context(|| format!("Unable to serialize data items for {task}: {filtered_data:#?}"))
}
fn category_to_key_space(category: TaskDataCategory) -> KeySpace { Ok(buffer)
match category {
TaskDataCategory::Meta => KeySpace::TaskMeta,
TaskDataCategory::Data => KeySpace::TaskData,
TaskDataCategory::All => unreachable!(),
}
} }
...@@ -6,7 +6,6 @@ ...@@ -6,7 +6,6 @@
mod backend; mod backend;
mod backing_storage; mod backing_storage;
mod data; mod data;
mod data_storage;
mod database; mod database;
mod kv_backing_storage; mod kv_backing_storage;
mod utils; mod utils;
......
...@@ -12,8 +12,6 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { ...@@ -12,8 +12,6 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream {
let value_name = Ident::new(&format!("{}Value", input.ident), input.ident.span()); let value_name = Ident::new(&format!("{}Value", input.ident), input.ident.span());
let value_ref_name = Ident::new(&format!("{}ValueRef", input.ident), input.ident.span()); let value_ref_name = Ident::new(&format!("{}ValueRef", input.ident), input.ident.span());
let value_ref_mut_name = Ident::new(&format!("{}ValueRefMut", input.ident), input.ident.span()); let value_ref_mut_name = Ident::new(&format!("{}ValueRefMut", input.ident), input.ident.span());
let iter_name = Ident::new(&format!("{}Iter", input.ident), input.ident.span());
let storage_name = Ident::new(&format!("{}Storage", input.ident), input.ident.span());
let variant_names = input let variant_names = input
.variants .variants
...@@ -67,101 +65,6 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { ...@@ -67,101 +65,6 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream {
let value_ref_mut_decl = mut_ref_field_declarations(&value_fields); let value_ref_mut_decl = mut_ref_field_declarations(&value_fields);
let value_ref_fields = ref_fields(&value_fields); let value_ref_fields = ref_fields(&value_fields);
let storage = key_fields
.iter()
.zip(value_fields.iter())
.zip(variant_names.iter())
.map(|((key_fields, value_fields), variant_name)| {
let value_types = value_fields
.iter()
.map(|field| {
let ty = &field.ty;
quote! {
#ty
}
})
.collect::<Vec<_>>();
let key_types = key_fields
.iter()
.map(|field| {
let ty = &field.ty;
quote! {
#ty
}
})
.collect::<Vec<_>>();
let value_fields = value_fields
.iter()
.map(|field| {
let ident = field.ident.as_ref().unwrap();
quote! {
#ident
}
})
.collect::<Vec<_>>();
let key_fields = key_fields
.iter()
.map(|field| {
let ident = field.ident.as_ref().unwrap();
quote! {
#ident
}
})
.collect::<Vec<_>>();
match (key_types.len(), value_types.len()) {
(0, 1) => StorageDecl {
decl: quote! {
OptionStorage<#(#value_types)*>
},
key: quote! {
()
},
pattern: quote! {
#ident::#variant_name { #(#value_fields)* }
},
key_pattern: quote! {},
},
(1, 1) => StorageDecl {
decl: quote! {
AutoMapStorage<#(#key_types)*, #(#value_types)*>
},
key: quote! {
#(#key_fields)*
},
pattern: quote! {
#ident::#variant_name { #(#key_fields)*, #(#value_fields)* }
},
key_pattern: quote! {
#(#key_fields)*
},
},
(_, 1) => StorageDecl {
decl: quote! {
AutoMapStorage<(#(#key_types),*), #(#value_types)*>
},
key: quote! {
(#(#key_fields),*)
},
pattern: quote! {
#ident::#variant_name { #(#key_fields),*, #(#value_fields)* }
},
key_pattern: quote! {
#(#key_fields),*
},
},
_ => unreachable!(),
}
})
.collect::<Vec<_>>();
let storage_decl = storage.iter().map(|decl| &decl.decl).collect::<Vec<_>>();
let storage_key = storage.iter().map(|decl| &decl.key).collect::<Vec<_>>();
let storage_pattern = storage.iter().map(|decl| &decl.pattern).collect::<Vec<_>>();
let storage_key_pattern = storage
.iter()
.map(|decl| &decl.key_pattern)
.collect::<Vec<_>>();
quote! { quote! {
#[automatically_derived] #[automatically_derived]
impl turbo_tasks::KeyValuePair for #ident { impl turbo_tasks::KeyValuePair for #ident {
...@@ -315,214 +218,6 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { ...@@ -315,214 +218,6 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream {
} }
} }
#[derive(Debug, Clone)]
#vis enum #storage_name {
#(
#variant_names {
storage: #storage_decl
},
)*
}
#[automatically_derived]
impl #storage_name {
pub fn new(ty: #type_name) -> Self {
match ty {
#(
#type_name::#variant_names => #storage_name::#variant_names { storage: Default::default() },
)*
}
}
pub fn ty(&self) -> #type_name {
match self {
#(
#storage_name::#variant_names { .. } => #type_name::#variant_names,
)*
}
}
pub fn add(&mut self, item: #ident) -> bool {
match (self, item) {
#(
(#storage_name::#variant_names { storage }, #storage_pattern) => {
storage.add(#storage_key, value)
}
)*
_ => unreachable!(),
}
}
pub fn insert(&mut self, item: #ident) -> Option<#value_name> {
match (self, item) {
#(
(#storage_name::#variant_names { storage }, #storage_pattern) => {
storage.insert(#storage_key, value).map(|value| #value_name::#variant_names { value })
}
)*
_ => unreachable!(),
}
}
pub fn extend(&mut self, iterator: impl Iterator<Item = #ident>) -> bool {
match self {
#(
#storage_name::#variant_names { storage } => {
storage.extend(iterator.map(|item| match item {
#storage_pattern => (#storage_key, value),
_ => unreachable!(),
}))
}
)*
_ => unreachable!(),
}
}
pub fn remove(&mut self, key: &#key_name) -> Option<#value_name> {
match (self, *key) {
#(
(#storage_name::#variant_names { storage }, #key_name::#variant_names { #storage_key_pattern }) => {
storage.remove(&#storage_key).map(|value| #value_name::#variant_names { value })
}
)*
_ => unreachable!(),
}
}
pub fn contains_key(&self, key: &#key_name) -> bool {
match (self, *key) {
#(
(#storage_name::#variant_names { storage }, #key_name::#variant_names { #storage_key_pattern }) => {
storage.contains_key(&#storage_key)
}
)*
_ => unreachable!(),
}
}
pub fn get(&self, key: &#key_name) -> Option<#value_ref_name> {
match (self, *key) {
#(
(#storage_name::#variant_names { storage }, #key_name::#variant_names { #storage_key_pattern }) => {
storage.get(&#storage_key).map(|value| #value_ref_name::#variant_names { value })
}
)*
_ => unreachable!(),
}
}
pub fn get_mut(&mut self, key: &#key_name) -> Option<#value_ref_mut_name> {
match (self, *key) {
#(
(#storage_name::#variant_names { storage }, #key_name::#variant_names { #storage_key_pattern }) => {
storage.get_mut(&#storage_key).map(|value| #value_ref_mut_name::#variant_names { value })
}
)*
_ => unreachable!(),
}
}
pub fn get_mut_or_insert_with(&mut self, key: #key_name, insert_with: impl FnOnce() -> #value_name) -> #value_ref_mut_name {
match (self, key) {
#(
(#storage_name::#variant_names { storage }, #key_name::#variant_names { #storage_key_pattern }) => {
#value_ref_mut_name::#variant_names { value: storage
.get_mut_or_insert_with(#storage_key, || {
let #value_name::#variant_names { value } = insert_with() else {
unreachable!();
};
value
})
}
}
)*
_ => unreachable!(),
}
}
pub fn update(&mut self, key: #key_name, update: impl FnOnce(Option<#value_name>) -> Option<#value_name>) {
match (self, key) {
#(
(#storage_name::#variant_names { storage }, #key_name::#variant_names { #storage_key_pattern }) => {
storage.update(#storage_key, |old| {
let old = old.map(|value| #value_name::#variant_names { value });
if let Some(#value_name::#variant_names { value }) = update(old) {
Some(value)
} else {
None
}
})
}
)*
_ => unreachable!(),
}
}
pub fn shrink_to_fit(&mut self) {
match self {
#(
#storage_name::#variant_names { storage } => {
storage.shrink_to_fit();
}
)*
}
}
pub fn is_empty(&self) -> bool {
match self {
#(
#storage_name::#variant_names { storage } => {
storage.is_empty()
}
)*
}
}
pub fn len(&self) -> usize {
match self {
#(
#storage_name::#variant_names { storage } => {
storage.len()
}
)*
}
}
pub fn iter(&self) -> #iter_name {
match self {
#(
#storage_name::#variant_names { storage } => {
#iter_name::#variant_names(storage.iter())
}
)*
}
}
}
#vis enum #iter_name<'l> {
#(
#variant_names(<#storage_decl as Storage>::Iterator<'l>),
)*
}
#[automatically_derived]
impl<'l> Iterator for #iter_name<'l> {
type Item = (#key_name, #value_ref_name<'l>);
fn next(&mut self) -> Option<Self::Item> {
match self {
#(
#iter_name::#variant_names(iter) => {
iter.next()
.map(|(&#storage_key, value)| (
#key_name::#variant_names { #storage_key_pattern },
#value_ref_name::#variant_names { value }
))
}
)*
}
}
}
} }
.into() .into()
} }
...@@ -653,10 +348,3 @@ fn mut_ref_field_declarations(fields: &[Vec<&syn::Field>]) -> Vec<proc_macro2::T ...@@ -653,10 +348,3 @@ fn mut_ref_field_declarations(fields: &[Vec<&syn::Field>]) -> Vec<proc_macro2::T
}) })
.collect::<Vec<_>>() .collect::<Vec<_>>()
} }
struct StorageDecl {
decl: proc_macro2::TokenStream,
key: proc_macro2::TokenStream,
pattern: proc_macro2::TokenStream,
key_pattern: proc_macro2::TokenStream,
}
...@@ -55,11 +55,16 @@ struct FieldInfo { ...@@ -55,11 +55,16 @@ struct FieldInfo {
/// If true, filter out values that reference transient tasks during encoding. /// If true, filter out values that reference transient tasks during encoding.
/// For direct fields: skip encoding if value.is_transient() returns true. /// For direct fields: skip encoding if value.is_transient() returns true.
/// For collections: filter out entries where key/value is_transient() returns true. /// For collections: filter out entries where key/value is_transient() returns true.
/// For AutoMultimap: filter is always applied to inner set values automatically.
filter_transient: bool, filter_transient: bool,
/// If true, use Default::default() semantics instead of Option for inline direct fields. /// If true, use Default::default() semantics instead of Option for inline direct fields.
/// The field type should be T (not Option<T>), and empty is represented by T::default(). /// The field type should be T (not Option<T>), and empty is represented by T::default().
use_default: bool, use_default: bool,
/// The CachedDataItem variant name for adapter code generation.
/// If None, no adapter code is generated for this field.
cached_data_variant: Option<Ident>,
/// The key field names in CachedDataItemKey for collection types.
/// E.g., ["task"], ["target", "key"], ["cell", "key", "task"].
key_fields: Option<Vec<Ident>>,
} }
impl FieldInfo { impl FieldInfo {
...@@ -68,6 +73,16 @@ impl FieldInfo { ...@@ -68,6 +73,16 @@ impl FieldInfo {
self.storage_type == StorageType::Flag self.storage_type == StorageType::Flag
} }
/// Whether this field has a CachedDataItem variant (for adapter code generation).
fn has_cached_data_variant(&self) -> bool {
self.cached_data_variant.is_some()
}
/// Get the CachedDataItem variant identifier (panics if none).
fn cached_data_variant_ident(&self) -> &Ident {
self.cached_data_variant.as_ref().unwrap()
}
/// Whether this field is transient (not serialized, in-memory only). /// Whether this field is transient (not serialized, in-memory only).
fn is_transient(&self) -> bool { fn is_transient(&self) -> bool {
self.category == Category::Transient self.category == Category::Transient
...@@ -290,6 +305,92 @@ impl FieldInfo { ...@@ -290,6 +305,92 @@ impl FieldInfo {
proc_macro2::Span::call_site(), proc_macro2::Span::call_site(),
) )
} }
fn shrink_ident(&self) -> syn::Ident {
self.prefixed_ident("shrink")
}
// =========================================================================
// Key Fields Helpers (for CachedDataItem adapter code generation)
// =========================================================================
/// Generate the key pattern for match arms in CachedDataItemKey.
///
/// For single key field: `{ task }`
/// For multiple key fields: `{ cell, key, task }`
fn key_pattern(&self) -> TokenStream {
match &self.key_fields {
None => quote! {},
Some(fields) => {
quote! { #(#fields),* }
}
}
}
/// Generate the key expression for use in set/map insert operations (moves values).
///
/// For single key field: `task`
/// For multiple key fields: `(cell, key, task)`
fn key_expr(&self) -> TokenStream {
match &self.key_fields {
None => quote! {},
Some(fields) if fields.len() == 1 => {
let f = &fields[0];
quote! { #f }
}
Some(fields) => {
quote! { (#(#fields),*) }
}
}
}
/// Generate the key expression for use when fields are bound by reference (e.g., in
/// get/contains). Clones the values to construct a tuple by value.
///
/// For single key field: `target.clone()`
/// For multiple key fields: `(target.clone(), key.clone())`
fn key_expr_cloned(&self) -> TokenStream {
match &self.key_fields {
None => quote! {},
Some(fields) if fields.len() == 1 => {
let f = &fields[0];
quote! { #f.clone() }
}
Some(fields) => {
let cloned: Vec<_> = fields.iter().map(|f| quote! { #f.clone() }).collect();
quote! { (#(#cloned),*) }
}
}
}
/// Generate the key expression with clone/copy for constructing CachedDataItemKey.
///
/// For single key field: `task: *task`
/// For multiple key fields: `cell: *cell, key: *key, task: *task`
fn key_construction(&self) -> TokenStream {
match &self.key_fields {
None => quote! {},
Some(fields) => {
let constructs: Vec<_> = fields.iter().map(|f| quote! { #f: *#f }).collect();
quote! { #(#constructs),* }
}
}
}
/// Generate key construction for owned values (no dereference).
///
/// For single key field: `task: task`
/// For multiple key fields: `cell: cell, key: key, task: task`
///
/// Used when the key is already owned (e.g., from iter methods that return values not refs).
fn key_construction_owned(&self) -> TokenStream {
match &self.key_fields {
None => quote! {},
Some(fields) => {
let constructs: Vec<_> = fields.iter().map(|f| quote! { #f: #f }).collect();
quote! { #(#constructs),* }
}
}
}
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
...@@ -338,6 +439,8 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { ...@@ -338,6 +439,8 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo {
let mut inline = false; // Default is lazy (not inline) let mut inline = false; // Default is lazy (not inline)
let mut filter_transient = false; let mut filter_transient = false;
let mut use_default = false; let mut use_default = false;
let mut cached_data_variant: Option<Ident> = None;
let mut key_fields: Option<Vec<Ident>> = None;
// Find and parse the field attribute // Find and parse the field attribute
if let Some(attr) = field.attrs.iter().find(|attr| { if let Some(attr) = field.attrs.iter().find(|attr| {
...@@ -412,6 +515,31 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { ...@@ -412,6 +515,31 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo {
}); });
} }
} }
"variant" => {
if let syn::Expr::Lit(syn::ExprLit {
lit: syn::Lit::Str(lit_str),
..
}) = &nv.value
{
cached_data_variant =
Some(Ident::new(&lit_str.value(), lit_str.span()));
}
}
"key_field" => {
if let syn::Expr::Lit(syn::ExprLit {
lit: syn::Lit::Str(lit_str),
..
}) = &nv.value
{
let fields: Vec<Ident> = lit_str
.value()
.split(',')
.map(|s| Ident::new(s.trim(), lit_str.span()))
.collect();
key_fields = Some(fields);
}
}
other => { other => {
meta.span() meta.span()
.unwrap() .unwrap()
...@@ -420,7 +548,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { ...@@ -420,7 +548,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo {
)) ))
.emit(); .emit();
} }
} };
} }
Meta::Path(path) => { Meta::Path(path) => {
let Some(ident) = path.get_ident() else { let Some(ident) = path.get_ident() else {
...@@ -515,6 +643,8 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo { ...@@ -515,6 +643,8 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo {
lazy: !inline, // Default is lazy; inline = true means lazy = false lazy: !inline, // Default is lazy; inline = true means lazy = false
filter_transient, filter_transient,
use_default, use_default,
cached_data_variant,
key_fields,
} }
} }
...@@ -604,6 +734,21 @@ impl GroupedFields { ...@@ -604,6 +734,21 @@ impl GroupedFields {
.iter() .iter()
.filter(move |f| !f.is_flag() && f.lazy && !f.is_transient() && f.category == category) .filter(move |f| !f.is_flag() && f.lazy && !f.is_transient() && f.category == category)
} }
// =========================================================================
// CachedDataItem adapter iterators
// =========================================================================
/// Returns an iterator over all fields that have a CachedDataItem variant.
/// These fields will generate adapter code for the CachedDataItem API.
fn fields_with_variant(&self) -> impl Iterator<Item = &FieldInfo> {
self.fields.iter().filter(|f| f.has_cached_data_variant())
}
/// Returns true if any fields have a CachedDataItem variant.
fn has_fields_with_variant(&self) -> bool {
self.fields.iter().any(|f| f.has_cached_data_variant())
}
} }
// ============================================================================= // =============================================================================
...@@ -922,7 +1067,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -922,7 +1067,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
let lazy_field = if has_lazy { let lazy_field = if has_lazy {
quote! { quote! {
/// Lazily-allocated fields stored in a single Vec for memory efficiency /// Lazily-allocated fields stored in a single Vec for memory efficiency
pub(crate) lazy: Vec<LazyField>, lazy: Vec<LazyField>,
} }
} else { } else {
quote! {} quote! {}
...@@ -938,7 +1083,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -938,7 +1083,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
/// Unified typed storage containing all task fields. /// Unified typed storage containing all task fields.
/// This is designed to be embedded in the actual InnerStorage for incremental migration. /// This is designed to be embedded in the actual InnerStorage for incremental migration.
#[automatically_derived] #[automatically_derived]
#[derive(Debug, Clone, Default, PartialEq, turbo_tasks::ShrinkToFit)] #[derive(Debug, Default, PartialEq, turbo_tasks::ShrinkToFit)]
#[shrink_to_fit(crate = "turbo_tasks::macro_helpers::shrink_to_fit")] #[shrink_to_fit(crate = "turbo_tasks::macro_helpers::shrink_to_fit")]
pub struct TaskStorage { pub struct TaskStorage {
#(#field_defs,)* #(#field_defs,)*
...@@ -1147,6 +1292,9 @@ fn generate_task_storage_accessors_trait(grouped_fields: &GroupedFields) -> Toke ...@@ -1147,6 +1292,9 @@ fn generate_task_storage_accessors_trait(grouped_fields: &GroupedFields) -> Toke
trait_methods.extend(generate_flag_trait_accessor_methods(field)); trait_methods.extend(generate_flag_trait_accessor_methods(field));
} }
// Generate CachedDataItem adapter methods
let adapter_methods = generate_cached_data_adapter_trait_methods(grouped_fields);
quote! { quote! {
/// Trait for typed storage accessors. /// Trait for typed storage accessors.
/// ///
...@@ -1198,6 +1346,8 @@ fn generate_task_storage_accessors_trait(grouped_fields: &GroupedFields) -> Toke ...@@ -1198,6 +1346,8 @@ fn generate_task_storage_accessors_trait(grouped_fields: &GroupedFields) -> Toke
} }
#trait_methods #trait_methods
#adapter_methods
} }
} }
} }
...@@ -1220,8 +1370,10 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream { ...@@ -1220,8 +1370,10 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream {
generate_direct_accessors(field) generate_direct_accessors(field)
} }
StorageType::AutoSet => { StorageType::AutoSet => {
// For AutoSet types, generate read-only accessor plus add/remove/has/iter/len/is_empty // For AutoSet types, generate read-only accessor, mutable accessor, and
// add/remove/has/iter/len/is_empty
let ref_name = field.ref_ident(); let ref_name = field.ref_ident();
let mut_name = field.mut_ident();
let (return_type, doc_comment) = if is_option { let (return_type, doc_comment) = if is_option {
( (
...@@ -1236,24 +1388,40 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream { ...@@ -1236,24 +1388,40 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream {
) )
}; };
let track_modification = field.track_modification_call();
let base_accessor = quote! { let base_accessor = quote! {
#[doc = #doc_comment] #[doc = #doc_comment]
fn #ref_name(&self) -> #return_type { fn #ref_name(&self) -> #return_type {
#check_access #check_access
#ref_expr #ref_expr
} }
/// Get a mutable reference to the collection (allocates if needed for lazy fields).
///
/// Tracks modification pessimistically - assumes caller will mutate.
/// TODO: try to remove this method, tracking mutations before the mutation interferes with snapshotting logic
fn #mut_name(&mut self) -> &mut #field_type {
#check_access
#track_modification
#mut_expr
}
}; };
let set_ops = generate_autoset_ops(field); let set_ops = generate_autoset_ops(field);
let shrink_accessor = generate_shrink_accessor(field);
quote! { quote! {
#base_accessor #base_accessor
#set_ops #set_ops
#shrink_accessor
} }
} }
StorageType::CounterMap => { StorageType::CounterMap => {
// For CounterMap types, generate read-only accessor plus mutation methods // For CounterMap types, generate read-only accessor, mutable accessor, and typed
// mutation methods
let ref_name = field.ref_ident(); let ref_name = field.ref_ident();
let mut_name = field.mut_ident();
let (return_type, doc_comment) = if is_option { let (return_type, doc_comment) = if is_option {
( (
...@@ -1268,19 +1436,33 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream { ...@@ -1268,19 +1436,33 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream {
) )
}; };
let track_modification = field.track_modification_call();
let base_accessor = quote! { let base_accessor = quote! {
#[doc = #doc_comment] #[doc = #doc_comment]
fn #ref_name(&self) -> #return_type { fn #ref_name(&self) -> #return_type {
#check_access #check_access
#ref_expr #ref_expr
} }
/// Get a mutable reference to the collection (allocates if needed for lazy fields).
///
/// Tracks modification pessimistically - assumes caller will mutate.
/// TODO: try to remove this method, tracking mutations before the mutation interferes with snapshotting logic
fn #mut_name(&mut self) -> &mut #field_type {
#check_access
#track_modification
#mut_expr
}
}; };
let countermap_ops = generate_countermap_ops(field); let countermap_ops = generate_countermap_ops(field);
let shrink_accessor = generate_shrink_accessor(field);
quote! { quote! {
#base_accessor #base_accessor
#countermap_ops #countermap_ops
#shrink_accessor
} }
} }
StorageType::AutoMap => { StorageType::AutoMap => {
...@@ -1300,6 +1482,8 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream { ...@@ -1300,6 +1482,8 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream {
) )
}; };
let track_modification = field.track_modification_call();
let base_accessor = quote! { let base_accessor = quote! {
#[doc = #ref_doc] #[doc = #ref_doc]
fn #ref_name(&self) -> #return_type { fn #ref_name(&self) -> #return_type {
...@@ -1309,19 +1493,22 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream { ...@@ -1309,19 +1493,22 @@ fn generate_trait_accessor_methods(field: &FieldInfo) -> TokenStream {
/// Get a mutable reference to the collection (allocates if needed for lazy fields). /// Get a mutable reference to the collection (allocates if needed for lazy fields).
/// ///
/// Note: This does NOT track modifications. Call `track_modification` after /// Tracks modification pessimistically - assumes caller will mutate.
/// making changes to ensure persistence. /// TODO: try to remove this method, tracking mutations before the mutation interferes with snapshotting logic
fn #mut_name(&mut self) -> &mut #field_type { fn #mut_name(&mut self) -> &mut #field_type {
#check_access #check_access
#track_modification
#mut_expr #mut_expr
} }
}; };
let automap_ops = generate_automap_ops(field); let automap_ops = generate_automap_ops(field);
let shrink_accessor = generate_shrink_accessor(field);
quote! { quote! {
#base_accessor #base_accessor
#automap_ops #automap_ops
#shrink_accessor
} }
} }
StorageType::Flag => { StorageType::Flag => {
...@@ -1366,23 +1553,54 @@ fn generate_direct_accessors(field: &FieldInfo) -> TokenStream { ...@@ -1366,23 +1553,54 @@ fn generate_direct_accessors(field: &FieldInfo) -> TokenStream {
quote! { #field_type } quote! { #field_type }
}; };
// Generate get_mut accessor only for lazy fields // Generate get_mut accessor for all direct fields
// (for inline fields, use set/take instead) let get_mut_accessor = {
let get_mut_accessor = if !field.is_inline() {
let get_mut_name = field.get_mut_ident(); let get_mut_name = field.get_mut_ident();
let get_mut_expr = field.direct_get_mut_expr(); if field.is_inline() {
quote! { // For inline fields, access the field directly
/// Get a mutable reference to the field value (if present). let field_name = &field.field_name;
/// if field.use_default {
/// Note: This does NOT track modifications. Call `track_modification` after // For fields with default semantics, always return Some(&mut self.field)
/// making changes to ensure persistence. quote! {
fn #get_mut_name(&mut self) -> Option<&mut #value_type> { /// Get a mutable reference to the field value.
#check_access ///
#get_mut_expr /// Tracks modification pessimistically - assumes caller will mutate.
/// TODO: try to remove this method, tracking mutations before the mutation interferes with snapshotting logic.
/// If this is needed then we need to use a guard struct that tracks `DerefMut` calls and calls track_modification
/// in Drop
fn #get_mut_name(&mut self) -> Option<&mut #value_type> {
#check_access
#track_modification
Some(&mut self.typed_mut().#field_name)
}
}
} else {
// For Option fields, return as_mut()
quote! {
/// Get a mutable reference to the field value (if present).
///
/// Tracks modification pessimistically - assumes caller will mutate.
fn #get_mut_name(&mut self) -> Option<&mut #value_type> {
#check_access
#track_modification
self.typed_mut().#field_name.as_mut()
}
}
}
} else {
// For lazy fields, use the existing get_mut expression
let get_mut_expr = field.direct_get_mut_expr();
quote! {
/// Get a mutable reference to the field value (if present).
///
/// Tracks modification pessimistically - assumes caller will mutate.
fn #get_mut_name(&mut self) -> Option<&mut #value_type> {
#check_access
#track_modification
#get_mut_expr
}
} }
} }
} else {
quote! {}
}; };
quote! { quote! {
...@@ -1887,6 +2105,45 @@ fn generate_automap_ops(field: &FieldInfo) -> TokenStream { ...@@ -1887,6 +2105,45 @@ fn generate_automap_ops(field: &FieldInfo) -> TokenStream {
} }
} }
/// Generate shrink_to_fit accessor for a collection field.
///
/// For collection fields (AutoSet, AutoMap, CounterMap), generates a `shrink_{field_name}()`
/// method that calls `shrink_to_fit()` on the underlying collection.
///
/// This does NOT track modifications or check access since shrinking doesn't
/// semantically change the data - it only reduces memory usage.
///
/// For lazy fields, avoids allocation if the collection doesn't exist.
fn generate_shrink_accessor(field: &FieldInfo) -> TokenStream {
let shrink_name = field.shrink_ident();
let field_name = &field.field_name;
let is_lazy = field.lazy;
let shrink_body = if is_lazy {
// For lazy fields, use find_lazy_mut to avoid allocating
let extractor = field.lazy_extractor_closure();
quote! {
if let Some(collection) = self.typed_mut().find_lazy_mut(#extractor) {
collection.shrink_to_fit();
}
}
} else {
// For inline fields, access the field directly
quote! {
self.typed_mut().#field_name.shrink_to_fit();
}
};
quote! {
/// Shrink the collection to fit its current contents, releasing excess memory.
///
/// This does NOT track modifications since it doesn't change the data semantically.
fn #shrink_name(&mut self) {
#shrink_body
}
}
}
/// Extract the inner type from Option<T>, or return the type as-is if not Option /// Extract the inner type from Option<T>, or return the type as-is if not Option
fn extract_option_inner_type(ty: &Type) -> TokenStream { fn extract_option_inner_type(ty: &Type) -> TokenStream {
// Try to parse as Option<T> and extract T // Try to parse as Option<T> and extract T
...@@ -2597,3 +2854,1124 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr ...@@ -2597,3 +2854,1124 @@ fn generate_snapshot_restore_methods(grouped_fields: &GroupedFields) -> TokenStr
} }
} }
} }
// =============================================================================
// CachedDataItem Adapter Code Generation
// =============================================================================
/// Generate the CachedDataItem adapter methods for the TaskStorageAccessors trait.
///
/// These methods provide the CachedDataItem API (add, insert, get, remove, etc.)
/// using the typed accessor methods from the trait. This enables proper access
/// tracking via check_access() and track_modification().
fn generate_cached_data_adapter_trait_methods(
grouped_fields: &GroupedFields,
) -> proc_macro2::TokenStream {
// Only generate if there are fields with variants
if !grouped_fields.has_fields_with_variant() {
return quote! {};
}
// Generate match arms for each method
let insert_kv_arms = generate_insert_kv_arms(grouped_fields);
let add_arms = generate_add_arms(grouped_fields);
let get_arms = generate_get_arms(grouped_fields);
let remove_arms = generate_remove_arms(grouped_fields);
let get_mut_arms = generate_get_mut_arms(grouped_fields);
let iter_arms = generate_iter_arms(grouped_fields);
let count_arms = generate_count_arms(grouped_fields);
let extend_arms = generate_extend_arms(grouped_fields);
let extract_if_arms = generate_extract_if_arms(grouped_fields);
quote! {
// =========================================================================
// CachedDataItem Adapter Methods
//
// These methods provide backward compatibility with the CachedDataItem API
// while using typed accessors internally for proper access tracking.
// =========================================================================
/// Add a CachedDataItem to storage.
///
/// Returns `true` if the item was newly added, `false` if it already existed.
/// Does NOT overwrite if the key already exists.
fn add(&mut self, item: crate::data::CachedDataItem) -> bool {
use crate::data::{CachedDataItemKey, CachedDataItemValue};
use turbo_tasks::KeyValuePair;
let (key, value) = item.into_key_and_value();
match (key, value) {
#add_arms
// Catch-all for mismatched key/value types
#[allow(unreachable_patterns)]
(key, value) => {
panic!(
"Mismatched CachedDataItem key/value types: key={key:?}, value={value:?}"
);
}
}
}
/// Insert a CachedDataItem, returning the old value if present.
fn insert(
&mut self,
item: crate::data::CachedDataItem,
) -> Option<crate::data::CachedDataItemValue> {
use turbo_tasks::KeyValuePair;
let (key, value) = item.into_key_and_value();
self.insert_kv(key, value)
}
/// Insert a key-value pair, returning the old value if present.
fn insert_kv(
&mut self,
key: crate::data::CachedDataItemKey,
value: crate::data::CachedDataItemValue,
) -> Option<crate::data::CachedDataItemValue> {
use crate::data::{CachedDataItemKey, CachedDataItemValue};
match (key, value) {
#insert_kv_arms
// Catch-all for mismatched key/value types
#[allow(unreachable_patterns)]
(key, value) => {
panic!(
"Mismatched CachedDataItem key/value types: key={key:?}, value={value:?}"
);
}
}
}
/// Check if a key exists in storage.
fn contains_key(&self, key: &crate::data::CachedDataItemKey) -> bool {
self.get(key).is_some()
}
/// Check if a key exists in storage (alias for contains_key for backward compatibility).
fn has_key(&self, key: &crate::data::CachedDataItemKey) -> bool {
self.contains_key(key)
}
/// Get a reference to a CachedDataItem value by key.
fn get(
&self,
key: &crate::data::CachedDataItemKey,
) -> Option<crate::data::CachedDataItemValueRef<'_>> {
use crate::data::{CachedDataItemKey, CachedDataItemValueRef};
match key {
#get_arms
}
}
/// Remove a CachedDataItem by key, returning the value if present.
fn remove(
&mut self,
key: &crate::data::CachedDataItemKey,
) -> Option<crate::data::CachedDataItemValue> {
use crate::data::{CachedDataItemKey, CachedDataItemValue};
match key {
#remove_arms
}
}
/// Get a mutable reference to a CachedDataItem value by key.
fn get_mut(
&mut self,
key: &crate::data::CachedDataItemKey,
) -> Option<crate::data::CachedDataItemValueRefMut<'_>> {
use crate::data::{CachedDataItemKey, CachedDataItemValueRefMut};
match key {
#get_mut_arms
}
}
/// Update a value in-place, creating it if it doesn't exist.
fn update(
&mut self,
key: crate::data::CachedDataItemKey,
update: impl FnOnce(Option<crate::data::CachedDataItemValue>) -> Option<crate::data::CachedDataItemValue>,
) {
use turbo_tasks::KeyValuePair;
let old_value = self.remove(&key);
if let Some(new_value) = update(old_value) {
let item = crate::data::CachedDataItem::from_key_and_value(key, new_value);
self.add(item);
}
}
/// Get a mutable reference or insert a value created by the given closure.
fn get_mut_or_insert_with(
&mut self,
key: crate::data::CachedDataItemKey,
insert: impl FnOnce() -> crate::data::CachedDataItemValue,
) -> crate::data::CachedDataItemValueRefMut<'_> {
use turbo_tasks::KeyValuePair;
if self.get(&key).is_none() {
let value = insert();
self.insert_kv(key.clone(), value);
}
self.get_mut(&key).expect("just inserted")
}
/// Count items of a specific type.
fn count(&self, ty: crate::data::CachedDataItemType) -> usize {
use crate::data::CachedDataItemType;
match ty {
#count_arms
}
}
/// Iterate over items of a specific type.
fn iter(
&self,
ty: crate::data::CachedDataItemType,
) -> Box<dyn Iterator<Item = (crate::data::CachedDataItemKey, crate::data::CachedDataItemValueRef<'_>)> + '_> {
use crate::data::{CachedDataItemKey, CachedDataItemType, CachedDataItemValueRef};
match ty {
#iter_arms
}
}
/// Extend storage with items from an iterator.
/// Returns `true` if all items were newly added, `false` if any already existed.
fn extend(
&mut self,
ty: crate::data::CachedDataItemType,
items: impl IntoIterator<Item = crate::data::CachedDataItem>,
) -> bool {
use crate::data::{CachedDataItemKey, CachedDataItemType, CachedDataItemValue};
use turbo_tasks::KeyValuePair;
match ty {
#extend_arms
}
}
/// Add a CachedDataItem that must not already exist.
/// Panics if the item already exists.
fn add_new(&mut self, item: crate::data::CachedDataItem) {
if !self.add(item) {
panic!("add_new: item already exists");
}
}
/// Extend with items that must all be new.
/// Panics if any item already exists.
fn extend_new(
&mut self,
_ty: crate::data::CachedDataItemType,
items: impl IntoIterator<Item = crate::data::CachedDataItem>,
) {
for item in items {
self.add_new(item);
}
}
/// Remove items matching a predicate.
fn extract_if<'a, F>(
&'a mut self,
ty: crate::data::CachedDataItemType,
mut predicate: F,
) -> Vec<crate::data::CachedDataItem>
where
F: for<'b> FnMut(crate::data::CachedDataItemKey, crate::data::CachedDataItemValueRef<'b>) -> bool + 'a,
{
use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemType, CachedDataItemValue, CachedDataItemValueRef};
use turbo_tasks::KeyValuePair;
match ty {
#extract_if_arms
}
}
}
}
/// Generate insert_kv match arms for all fields with variants.
fn generate_insert_kv_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_insert_kv_arm)
.collect();
quote! { #(#arms)* }
}
/// Generate a single insert_kv match arm for a field.
fn generate_insert_kv_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_insert_kv_arm_direct(field, variant),
StorageType::AutoSet => generate_insert_kv_arm_auto_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => {
generate_insert_kv_arm_map(field, variant)
}
StorageType::Flag => generate_insert_kv_arm_flag(field, variant),
}
}
fn generate_insert_kv_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let set_name = field.set_ident();
quote! {
(
CachedDataItemKey::#variant {},
CachedDataItemValue::#variant { value },
) => self
.#set_name(value)
.map(|v| CachedDataItemValue::#variant { value: v }),
}
}
fn generate_insert_kv_arm_auto_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
let key_expr = field.key_expr();
quote! {
(
CachedDataItemKey::#variant { #key_pattern },
CachedDataItemValue::#variant { value: () },
) => {
let existed = !self.#field_mut().insert(#key_expr);
if existed {
Some(CachedDataItemValue::#variant { value: () })
} else {
None
}
},
}
}
fn generate_insert_kv_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
let key_expr = field.key_expr();
quote! {
(
CachedDataItemKey::#variant { #key_pattern },
CachedDataItemValue::#variant { value },
) => self
.#field_mut()
.insert(#key_expr, value)
.map(|v| CachedDataItemValue::#variant { value: v }),
}
}
fn generate_insert_kv_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let set_name = field.set_ident();
quote! {
(
CachedDataItemKey::#variant {},
CachedDataItemValue::#variant { value: () },
) => {
let existed = self.typed().flags.#field_name();
self.typed_mut().flags.#set_name(true);
if existed {
Some(CachedDataItemValue::#variant { value: () })
} else {
None
}
},
}
}
/// Generate get match arms for all fields with variants.
fn generate_get_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_get_arm)
.collect();
quote! { #(#arms)* }
}
/// Generate a single get match arm for a field.
fn generate_get_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_get_arm_direct(field, variant),
StorageType::AutoSet => generate_get_arm_auto_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_get_arm_map(field, variant),
StorageType::Flag => generate_get_arm_flag(field, variant),
}
}
fn generate_get_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let get_name = field.get_ident();
quote! {
CachedDataItemKey::#variant {} => self
.#get_name()
.map(|value| CachedDataItemValueRef::#variant { value }),
}
}
fn generate_get_arm_auto_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let key_pattern = field.key_pattern();
// Use cloned key expr since the key is passed by reference
let key_expr = field.key_expr_cloned();
if field.is_inline() {
quote! {
CachedDataItemKey::#variant { #key_pattern } => {
if self.#field_name().contains(&#key_expr) {
Some(CachedDataItemValueRef::#variant { value: &() })
} else {
None
}
},
}
} else {
quote! {
CachedDataItemKey::#variant { #key_pattern } => self.#field_name().and_then(|set| {
if set.contains(&#key_expr) {
Some(CachedDataItemValueRef::#variant { value: &() })
} else {
None
}
}),
}
}
}
fn generate_get_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let key_pattern = field.key_pattern();
// Use cloned key expr since the key is passed by reference
let key_expr = field.key_expr_cloned();
if field.is_inline() {
quote! {
CachedDataItemKey::#variant { #key_pattern } => self
.#field_name()
.get(&#key_expr)
.map(|value| CachedDataItemValueRef::#variant { value }),
}
} else {
quote! {
CachedDataItemKey::#variant { #key_pattern } => self
.#field_name()
.and_then(|map| map.get(&#key_expr))
.map(|value| CachedDataItemValueRef::#variant { value }),
}
}
}
fn generate_get_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
quote! {
CachedDataItemKey::#variant {} => {
if self.typed().flags.#field_name() {
Some(CachedDataItemValueRef::#variant { value: &() })
} else {
None
}
},
}
}
/// Generate remove match arms for all fields with variants.
fn generate_remove_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_remove_arm)
.collect();
quote! { #(#arms)* }
}
/// Generate a single remove match arm for a field.
fn generate_remove_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_remove_arm_direct(field, variant),
StorageType::AutoSet => generate_remove_arm_auto_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_remove_arm_map(field, variant),
StorageType::Flag => generate_remove_arm_flag(field, variant),
}
}
fn generate_remove_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let take_name = field.take_ident();
quote! {
CachedDataItemKey::#variant {} => self
.#take_name()
.map(|value| CachedDataItemValue::#variant { value }),
}
}
fn generate_remove_arm_auto_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
// Use cloned key expr since the key is passed by reference
let key_expr = field.key_expr_cloned();
quote! {
CachedDataItemKey::#variant { #key_pattern } => {
if self.#field_mut().remove(&#key_expr) {
Some(CachedDataItemValue::#variant { value: () })
} else {
None
}
},
}
}
fn generate_remove_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
// Use cloned key expr since the key is passed by reference
let key_expr = field.key_expr_cloned();
quote! {
CachedDataItemKey::#variant { #key_pattern } => self
.#field_mut()
.remove(&#key_expr)
.map(|value| CachedDataItemValue::#variant { value }),
}
}
fn generate_remove_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let set_name = field.set_ident();
quote! {
CachedDataItemKey::#variant {} => {
let existed = self.typed().flags.#field_name();
self.typed_mut().flags.#set_name(false);
if existed {
Some(CachedDataItemValue::#variant { value: () })
} else {
None
}
},
}
}
/// Generate get_mut match arms for all fields with variants.
fn generate_get_mut_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_get_mut_arm)
.collect();
quote! { #(#arms)* }
}
/// Generate a single get_mut match arm for a field.
/// For types that don't support mutable access (flags, sets, multimaps), generates an arm
/// that returns None.
fn generate_get_mut_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_get_mut_arm_direct(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_get_mut_arm_map(field, variant),
// AutoSet, and Flag don't support get_mut (value is always ())
// But we need to include the match arm to make the match exhaustive
StorageType::AutoSet => generate_get_mut_arm_set_none(field, variant),
StorageType::Flag => generate_get_mut_arm_flag_none(field, variant),
}
}
fn generate_get_mut_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let get_mut_name = field.get_mut_ident();
quote! {
CachedDataItemKey::#variant {} => self
.#get_mut_name()
.map(|value| CachedDataItemValueRefMut::#variant { value }),
}
}
fn generate_get_mut_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
// Use cloned key expr since the key is passed by reference
let key_expr = field.key_expr_cloned();
quote! {
CachedDataItemKey::#variant { #key_pattern } => self
.#field_mut()
.get_mut(&#key_expr)
.map(|value| CachedDataItemValueRefMut::#variant { value }),
}
}
/// Generate a get_mut arm for AutoSet types that returns None.
/// AutoSet values are always () so mutable access is not meaningful.
fn generate_get_mut_arm_set_none(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
// Ignore all key fields with _
let ignored_fields: Vec<_> = field
.key_fields
.as_ref()
.map(|fields| fields.iter().map(|f| quote! { #f: _ }).collect())
.unwrap_or_default();
quote! {
CachedDataItemKey::#variant { #(#ignored_fields),* } => None,
}
}
/// Generate a get_mut arm for Flag types that returns None.
/// Flag values are booleans so mutable access is not meaningful.
fn generate_get_mut_arm_flag_none(_field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
quote! {
CachedDataItemKey::#variant {} => None,
}
}
/// Generate count match arms for all fields with variants.
fn generate_count_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_count_arm)
.collect();
quote! { #(#arms)* }
}
/// Generate a single count match arm for a field.
fn generate_count_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_count_arm_direct(field, variant),
StorageType::AutoSet => generate_count_arm_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_count_arm_map(field, variant),
StorageType::Flag => generate_count_arm_flag(field, variant),
}
}
fn generate_count_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let get_name = field.get_ident();
quote! {
CachedDataItemType::#variant => if self.#get_name().is_some() { 1 } else { 0 },
}
}
fn generate_count_arm_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
if field.is_inline() {
quote! {
CachedDataItemType::#variant => self.#field_name().len(),
}
} else {
quote! {
CachedDataItemType::#variant => self.#field_name().map(|s| s.len()).unwrap_or(0),
}
}
}
fn generate_count_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
if field.is_inline() {
quote! {
CachedDataItemType::#variant => self.#field_name().len(),
}
} else {
quote! {
CachedDataItemType::#variant => self.#field_name().map(|m| m.len()).unwrap_or(0),
}
}
}
fn generate_count_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
quote! {
CachedDataItemType::#variant => if self.typed().flags.#field_name() { 1 } else { 0 },
}
}
/// Generate iter match arms for all fields with variants.
fn generate_iter_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_iter_arm)
.collect();
quote! { #(#arms)* }
}
/// Generate a single iter match arm for a field.
fn generate_iter_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_iter_arm_direct(field, variant),
StorageType::AutoSet => generate_iter_arm_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_iter_arm_map(field, variant),
StorageType::Flag => generate_iter_arm_flag(field, variant),
}
}
fn generate_iter_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let get_name = field.get_ident();
quote! {
CachedDataItemType::#variant => Box::new(
self.#get_name()
.into_iter()
.map(|value| (CachedDataItemKey::#variant {}, CachedDataItemValueRef::#variant { value })),
),
}
}
fn generate_iter_arm_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let key_expr = field.key_expr();
let key_construction = field.key_construction();
if field.is_inline() {
quote! {
CachedDataItemType::#variant => Box::new(self.#field_name().iter().map(|#key_expr| {
(
CachedDataItemKey::#variant { #key_construction },
CachedDataItemValueRef::#variant { value: &() },
)
})),
}
} else {
quote! {
CachedDataItemType::#variant => Box::new(
self.#field_name()
.into_iter()
.flat_map(|set| set.iter())
.map(|#key_expr| {
(
CachedDataItemKey::#variant { #key_construction },
CachedDataItemValueRef::#variant { value: &() },
)
}),
),
}
}
}
fn generate_iter_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let key_expr = field.key_expr();
let key_construction = field.key_construction();
if field.is_inline() {
quote! {
CachedDataItemType::#variant => Box::new(self.#field_name().iter().map(|(#key_expr, value)| {
(
CachedDataItemKey::#variant { #key_construction },
CachedDataItemValueRef::#variant { value },
)
})),
}
} else {
quote! {
CachedDataItemType::#variant => Box::new(
self.#field_name()
.into_iter()
.flat_map(|m| m.iter())
.map(|(#key_expr, value)| {
(
CachedDataItemKey::#variant { #key_construction },
CachedDataItemValueRef::#variant { value },
)
}),
),
}
}
}
fn generate_iter_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
quote! {
CachedDataItemType::#variant => Box::new(
self.typed()
.flags
.#field_name()
.then_some((
CachedDataItemKey::#variant {},
CachedDataItemValueRef::#variant { value: &() },
))
.into_iter(),
),
}
}
// =============================================================================
// Optimized add() match arms
// =============================================================================
/// Generate add match arms for all fields with variants.
/// Delegates to typed methods for single check_access + conditional track_modification.
fn generate_add_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_add_arm)
.collect();
quote! { #(#arms)* }
}
fn generate_add_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_add_arm_direct(field, variant),
StorageType::AutoSet => generate_add_arm_auto_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_add_arm_map(field, variant),
StorageType::Flag => generate_add_arm_flag(field, variant),
}
}
fn generate_add_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let has_name = field.has_ident();
let set_name = field.set_ident();
quote! {
(
CachedDataItemKey::#variant {},
CachedDataItemValue::#variant { value },
) => {
if self.#has_name() {
false
} else {
self.#set_name(value);
true
}
},
}
}
fn generate_add_arm_auto_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let add_name = field.prefixed_ident("add");
let key_pattern = field.key_pattern();
let key_expr = field.key_expr();
quote! {
(
CachedDataItemKey::#variant { #key_pattern },
CachedDataItemValue::#variant { value: () },
) => self.#add_name(#key_expr),
}
}
fn generate_add_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
let key_expr = field.key_expr();
if field.is_inline() {
quote! {
(
CachedDataItemKey::#variant { #key_pattern },
CachedDataItemValue::#variant { value },
) => {
if self.#field_name().get(&#key_expr).is_some() {
false
} else {
self.#field_mut().insert(#key_expr, value);
true
}
},
}
} else {
quote! {
(
CachedDataItemKey::#variant { #key_pattern },
CachedDataItemValue::#variant { value },
) => {
if self.#field_name().is_some_and(|m| m.get(&#key_expr).is_some()) {
false
} else {
self.#field_mut().insert(#key_expr, value);
true
}
},
}
}
}
fn generate_add_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let set_name = field.set_ident();
quote! {
(
CachedDataItemKey::#variant {},
CachedDataItemValue::#variant { value: () },
) => {
if self.typed().flags.#field_name() {
false
} else {
self.typed_mut().flags.#set_name(true);
true
}
},
}
}
// =============================================================================
// Optimized extend() match arms
// =============================================================================
/// Generate extend match arms for all fields with variants.
/// Uses typed batch methods for single check_access + single track_modification.
fn generate_extend_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_extend_arm)
.collect();
quote! { #(#arms)* }
}
fn generate_extend_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_extend_arm_direct(field, variant),
StorageType::AutoSet => generate_extend_arm_auto_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => generate_extend_arm_map(field, variant),
StorageType::Flag => generate_extend_arm_flag(field, variant),
}
}
fn generate_extend_arm_direct(_field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
// Direct fields can only have one value, so just delegate to add() for the first item
quote! {
CachedDataItemType::#variant => {
for item in items {
return self.add(item);
}
true
},
}
}
fn generate_extend_arm_auto_set(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let extend_name = field.suffixed_ident("extend");
let key_pattern = field.key_pattern();
let key_expr = field.key_expr();
quote! {
CachedDataItemType::#variant => {
// Delegate to typed extend method which batches check_access + track_modification
self.#extend_name(items.into_iter().filter_map(|item| {
match item.into_key_and_value() {
(CachedDataItemKey::#variant { #key_pattern }, _) => Some(#key_expr),
_ => None,
}
}));
true // AutoSet extend doesn't track per-item success
},
}
}
fn generate_extend_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_mut = field.mut_ident();
let key_pattern = field.key_pattern();
let key_expr = field.key_expr();
quote! {
CachedDataItemType::#variant => {
let mut all_new = true;
// Get mutable reference once (single check_access + track_modification)
let map = self.#field_mut();
for item in items {
if let (CachedDataItemKey::#variant { #key_pattern }, CachedDataItemValue::#variant { value })
= item.into_key_and_value()
{
if map.insert(#key_expr, value).is_some() {
all_new = false;
}
}
}
all_new
},
}
}
fn generate_extend_arm_flag(_field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
// Flags can only have one value, so just delegate to add() for the first item
quote! {
CachedDataItemType::#variant => {
for item in items {
return self.add(item);
}
true
},
}
}
// =============================================================================
// Optimized extract_if() match arms
// =============================================================================
/// Generate extract_if match arms for all fields with variants.
/// Uses retain pattern for single check_access + single track_modification.
fn generate_extract_if_arms(grouped_fields: &GroupedFields) -> proc_macro2::TokenStream {
let arms: Vec<_> = grouped_fields
.fields_with_variant()
.map(generate_extract_if_arm)
.collect();
quote! { #(#arms)* }
}
fn generate_extract_if_arm(field: &FieldInfo) -> proc_macro2::TokenStream {
let variant = field.cached_data_variant_ident();
match &field.storage_type {
StorageType::Direct => generate_extract_if_arm_direct(field, variant),
StorageType::AutoSet => generate_extract_if_arm_auto_set(field, variant),
StorageType::CounterMap | StorageType::AutoMap => {
generate_extract_if_arm_map(field, variant)
}
StorageType::Flag => generate_extract_if_arm_flag(field, variant),
}
}
fn generate_extract_if_arm_direct(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let take_name = field.take_ident();
let get_name = field.get_ident();
quote! {
CachedDataItemType::#variant => {
let key = CachedDataItemKey::#variant {};
if let Some(value_ref) = self.#get_name() {
if predicate(key.clone(), CachedDataItemValueRef::#variant { value: value_ref }) {
if let Some(value) = self.#take_name() {
return vec![CachedDataItem::from_key_and_value(key, CachedDataItemValue::#variant { value })];
}
}
}
Vec::new()
},
}
}
fn generate_extract_if_arm_auto_set(
field: &FieldInfo,
variant: &Ident,
) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let key_expr = field.key_expr();
let key_construction = field.key_construction();
// AutoSet doesn't have retain, so we collect keys first and then remove them.
// The typed iter_X and remove_X methods handle check_access/track_modification efficiently.
if field.is_inline() {
quote! {
CachedDataItemType::#variant => {
// First pass: collect keys to remove (single check_access via iter)
let keys_to_remove: Vec<_> = self.#field_name().iter().filter_map(|#key_expr| {
let key = CachedDataItemKey::#variant { #key_construction };
let value_ref = CachedDataItemValueRef::#variant { value: &() };
if predicate(key.clone(), value_ref) {
Some(key)
} else {
None
}
}).collect();
// Second pass: remove items using the adapter's remove method
keys_to_remove.into_iter().filter_map(|key| {
self.remove(&key).map(|value| {
CachedDataItem::from_key_and_value(key, value)
})
}).collect()
},
}
} else {
// Lazy autoset iter methods return owned values (via .copied()), so use
// key_construction_owned
let iter_name = field.iter_ident();
let key_construction_owned = field.key_construction_owned();
quote! {
CachedDataItemType::#variant => {
// First pass: collect keys to remove (single check_access via iter)
let keys_to_remove: Vec<_> = self.#iter_name().filter_map(|#key_expr| {
let key = CachedDataItemKey::#variant { #key_construction_owned };
let value_ref = CachedDataItemValueRef::#variant { value: &() };
if predicate(key.clone(), value_ref) {
Some(key)
} else {
None
}
}).collect();
// Second pass: remove items using the adapter's remove method
keys_to_remove.into_iter().filter_map(|key| {
self.remove(&key).map(|value| {
CachedDataItem::from_key_and_value(key, value)
})
}).collect()
},
}
}
}
fn generate_extract_if_arm_map(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let key_expr = field.key_expr();
let key_construction = field.key_construction();
// Maps (CounterMap/AutoMap) don't have retain with the signature we need,
// so we collect keys first and then remove them.
// Use the adapter's remove method which handles track_modification.
if field.is_inline() {
quote! {
CachedDataItemType::#variant => {
// First pass: collect keys to remove
let keys_to_remove: Vec<_> = self.#field_name().iter().filter_map(|(#key_expr, value)| {
let key = CachedDataItemKey::#variant { #key_construction };
let value_ref = CachedDataItemValueRef::#variant { value };
if predicate(key.clone(), value_ref) {
Some(key)
} else {
None
}
}).collect();
// Second pass: remove and collect using the adapter's remove method
keys_to_remove.into_iter().filter_map(|key| {
self.remove(&key).map(|value| {
CachedDataItem::from_key_and_value(key, value)
})
}).collect()
},
}
} else {
// For lazy map fields, use iter_X_entries which iterates over (key, value) pairs
let iter_entries_name = field.infixed_ident("iter", "entries");
quote! {
CachedDataItemType::#variant => {
// First pass: collect keys to remove
let keys_to_remove: Vec<_> = self.#iter_entries_name().filter_map(|(#key_expr, value)| {
let key = CachedDataItemKey::#variant { #key_construction };
let value_ref = CachedDataItemValueRef::#variant { value };
if predicate(key.clone(), value_ref) {
Some(key)
} else {
None
}
}).collect();
// Second pass: remove and collect using the adapter's remove method
keys_to_remove.into_iter().filter_map(|key| {
self.remove(&key).map(|value| {
CachedDataItem::from_key_and_value(key, value)
})
}).collect()
},
}
}
}
fn generate_extract_if_arm_flag(field: &FieldInfo, variant: &Ident) -> proc_macro2::TokenStream {
let field_name = &field.field_name;
let set_name = field.set_ident();
quote! {
CachedDataItemType::#variant => {
let key = CachedDataItemKey::#variant {};
if self.typed().flags.#field_name() {
let value_ref = CachedDataItemValueRef::#variant { value: &() };
if predicate(key.clone(), value_ref) {
self.typed_mut().flags.#set_name(false);
return vec![CachedDataItem::from_key_and_value(key, CachedDataItemValue::#variant { value: () })];
}
}
Vec::new()
},
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment