Unverified Commit 21fcfb01 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

feat: Replace InnerStorage with the generated TaskStorage struct (#88355)

## What

Replace `InnerStorage` with `TaskStorage` generated by the new derive macro from #88338 . 

Extend the TaskStorage derive macro to generate `CachedDataItem` adapter methods on the `TaskStorageAccessors` trait so we can maintain API compatibility. 

Replace the serialization layer to use the native TaskStorage representations now that CachedDataitem is only used for API access

## Why

TaskStorage will enable more performant serialization and access patterns.  From testing on vercel-site this saves ~1% of persistent cache size.  For this PR there is no benefit to access pattern performance (in fact i would anticipate a small regression due to the adaptor layer)

## How

1. **Extended macro attribute parsing** - Added `variant`, and `key_field`` attributes to specify CachedDataItem variant mapping per field

2. **Generated adapter methods on TaskStorageAccessors trait**:
   - `insert_kv(item)` - Insert a CachedDataItem into typed storage
   - `get(key)` - Look up by CachedDataItemKey
   - `remove(key)` - Remove by key
   - `get_mut(key)` - Mutable access (returns None for flags/sets/multimaps)
   - `iter(type)` - Iterate over a specific variant type
  - ...

   This enables current access patterns to keep working

3. **Migrated serialization and deserialization flow**:
   - use `TaskStorage` throughout the serialization flow using apis like `restore_from` and `clone_data` to create targeted copies for lock free IO.
    
4. **Removed new dead code** - Delete the `Storage` layer now that it is dead.

On its own this will be a small savings in serialized data size (saves 20Mb for vercel-site) and is slightly faster to snapshot/restore.  Access performance should be nearly equivalent in performance.
parent 8fe8ff79
......@@ -21,11 +21,11 @@ pub type TurboBincodeDecoder<'a> =
pub type AnyEncodeFn = fn(&dyn Any, &mut TurboBincodeEncoder<'_>) -> Result<(), EncodeError>;
pub type AnyDecodeFn<T> = fn(&mut TurboBincodeDecoder<'_>) -> Result<T, DecodeError>;
fn new_turbo_bincode_encoder(buf: &mut TurboBincodeBuffer) -> TurboBincodeEncoder<'_> {
pub fn new_turbo_bincode_encoder(buf: &mut TurboBincodeBuffer) -> TurboBincodeEncoder<'_> {
EncoderImpl::new(TurboBincodeWriter::new(buf), TURBO_BINCODE_CONFIG)
}
fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> {
pub fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> {
DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ())
}
......
// TODO(PR 2): Remove this once the storage schema is integrated with the rest of the codebase.
// This module is scaffolding for the TaskStorage macro and is not yet used.
#![allow(dead_code)]
use std::{
hash::{BuildHasherDefault, Hash},
ops::{Add, AddAssign, Sub},
......@@ -84,7 +81,7 @@ impl CounterValue for i32 {
}
impl<K, V> CounterMap<K, V> {
fn new() -> Self {
pub fn new() -> Self {
Self(AutoMap::default())
}
......@@ -92,6 +89,13 @@ impl<K, V> CounterMap<K, V> {
self.0.is_empty()
}
pub fn shrink_to_fit(&mut self)
where
K: Eq + Hash,
{
self.0.shrink_to_fit();
}
pub fn len(&self) -> usize {
self.0.len()
}
......@@ -102,7 +106,15 @@ impl<K, V> CounterMap<K, V> {
{
self.0.get(key)
}
// TODO(lukesandberg): this is just here for the CachedDataItem adaptor layer, can be removed
// once that is gone.
#[doc(hidden)]
pub fn get_mut(&mut self, _key: &K) -> Option<&mut V>
where
K: Eq + Hash,
{
unreachable!("This should never be called, please insert instead")
}
pub fn iter(&self) -> impl Iterator<Item = (&K, &V)> {
self.0.iter()
}
......@@ -117,7 +129,6 @@ impl<K, V> CounterMap<K, V> {
impl<K: Hash + Eq, V: CounterValue> CounterMap<K, V> {
/// Insert a key-value pair. Panics if value is zero (invariant: zero values are not stored).
#[cfg(test)]
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
debug_assert!(
!value.is_zero(),
......
use either::Either;
use turbo_tasks::KeyValuePair;
use crate::data::{
CachedDataItem, CachedDataItemKey, CachedDataItemStorage, CachedDataItemType,
CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut,
};
#[derive(Debug, Clone)]
pub struct DynamicStorage {
map: Vec<CachedDataItemStorage>,
}
impl DynamicStorage {
pub fn new() -> Self {
Self {
map: Default::default(),
}
}
fn get_or_create_map_mut(&mut self, ty: CachedDataItemType) -> &mut CachedDataItemStorage {
let i = self.map.iter().position(|m| m.ty() == ty);
if let Some(i) = i {
&mut self.map[i]
} else {
self.map.reserve_exact(1);
self.map.push(CachedDataItemStorage::new(ty));
self.map.last_mut().unwrap()
}
}
fn get_map_mut(&mut self, ty: CachedDataItemType) -> Option<&mut CachedDataItemStorage> {
self.map.iter_mut().find(|m| m.ty() == ty)
}
fn get_map_index(&mut self, ty: CachedDataItemType) -> Option<usize> {
self.map.iter_mut().position(|m| m.ty() == ty)
}
fn get_or_create_map_index(&mut self, ty: CachedDataItemType) -> usize {
let i = self.map.iter().position(|m| m.ty() == ty);
if let Some(i) = i {
i
} else {
let i = self.map.len();
self.map.reserve_exact(1);
self.map.push(CachedDataItemStorage::new(ty));
i
}
}
fn get_map(&self, ty: CachedDataItemType) -> Option<&CachedDataItemStorage> {
self.map.iter().find(|m| m.ty() == ty)
}
pub fn add(&mut self, item: CachedDataItem) -> bool {
let ty = item.ty();
self.get_or_create_map_mut(ty).add(item)
}
pub fn extend(
&mut self,
ty: CachedDataItemType,
iterator: impl Iterator<Item = CachedDataItem>,
) -> bool {
self.get_or_create_map_mut(ty).extend(iterator)
}
pub fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> {
let ty = item.ty();
self.get_or_create_map_mut(ty).insert(item)
}
pub fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> {
self.get_map_index(key.ty()).and_then(|i| {
let storage = &mut self.map[i];
let result = storage.remove(key);
if result.is_some() && storage.is_empty() {
self.map.swap_remove(i);
self.map.shrink_to_fit();
}
result
})
}
pub fn get(&self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRef<'_>> {
self.get_map(key.ty()).and_then(|m| m.get(key))
}
pub fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
self.get_map_mut(key.ty()).and_then(|m| m.get_mut(key))
}
pub fn get_mut_or_insert_with(
&mut self,
key: CachedDataItemKey,
f: impl FnOnce() -> CachedDataItemValue,
) -> CachedDataItemValueRefMut<'_> {
self.get_or_create_map_mut(key.ty())
.get_mut_or_insert_with(key, f)
}
pub fn contains_key(&self, key: &CachedDataItemKey) -> bool {
self.get_map(key.ty())
.map(|m| m.contains_key(key))
.unwrap_or_default()
}
pub fn count(&self, ty: CachedDataItemType) -> usize {
self.get_map(ty).map(|m| m.len()).unwrap_or_default()
}
pub fn iter(
&self,
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.get_map(ty).map(|m| m.iter()).into_iter().flatten()
}
pub fn iter_all(
&self,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.map.iter().flat_map(|m| m.iter())
}
pub fn extract_if<'l, F>(
&'l mut self,
ty: CachedDataItemType,
mut f: F,
) -> impl Iterator<Item = CachedDataItem> + use<'l, F>
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{
// TODO this could be more efficient when the storage would support extract_if directly.
// This requires some macro magic to make it work...
// But we could potentially avoid the two temporary Vecs.
let Some(i) = self.get_map_index(ty) else {
return Either::Left(std::iter::empty());
};
let storage = &mut self.map[i];
let items_to_extract = storage
.iter()
.filter(|(k, v)| f(*k, *v))
.map(|(key, _)| key)
.collect::<Vec<_>>();
let items = items_to_extract
.into_iter()
.map(move |key| {
let value = storage.remove(&key).unwrap();
CachedDataItem::from_key_and_value(key, value)
})
.collect::<Vec<_>>();
if self.map[i].is_empty() {
self.map.swap_remove(i);
self.map.shrink_to_fit();
}
Either::Right(items.into_iter())
}
pub fn update(
&mut self,
key: CachedDataItemKey,
update: impl FnOnce(Option<CachedDataItemValue>) -> Option<CachedDataItemValue>,
) {
let i = self.get_or_create_map_index(key.ty());
let map = &mut self.map[i];
map.update(key, update);
if map.is_empty() {
self.map.swap_remove(i);
self.map.shrink_to_fit();
}
}
pub fn len(&self) -> usize {
self.map.iter().map(|m| m.len()).sum()
}
pub fn shrink_to_fit(&mut self, ty: CachedDataItemType) {
if let Some(map) = self.get_map_mut(ty) {
map.shrink_to_fit();
}
}
pub fn snapshot_for_persisting(&self) -> Self {
Self {
map: self
.map
.iter()
.filter(|m| m.ty().is_persistent())
.cloned()
.collect(),
}
}
}
mod counter_map;
mod dynamic_storage;
mod operation;
mod storage;
mod storage_schema;
pub mod storage_schema;
use std::{
borrow::Cow,
......@@ -26,7 +25,7 @@ use smallvec::{SmallVec, smallvec};
use tokio::time::{Duration, Instant};
use tracing::{Span, trace_span};
use turbo_tasks::{
CellId, FxDashMap, KeyValuePair, RawVc, ReadCellOptions, ReadCellTracking, ReadConsistency,
CellId, FxDashMap, RawVc, ReadCellOptions, ReadCellTracking, ReadConsistency,
ReadOutputOptions, ReadTracking, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TaskPriority,
TraitTypeId, TurboTasksBackendApi, ValueTypeId,
backend::{
......@@ -43,7 +42,10 @@ use turbo_tasks::{
util::{IdFactoryWithReuse, good_chunk_size, into_chunks},
};
pub use self::{operation::AnyOperation, storage::TaskDataCategory};
pub use self::{
operation::AnyOperation,
storage::{SpecificTaskDataCategory, TaskDataCategory},
};
#[cfg(feature = "trace_task_dirty")]
use crate::backend::operation::TaskDirtyCause;
use crate::{
......@@ -56,9 +58,9 @@ use crate::{
make_task_dirty_internal, prepare_new_children,
},
storage::{
InnerStorageSnapshot, Storage, count, get, get_many, get_mut, get_mut_or_insert_with,
iter_many, remove,
Storage, count, get, get_many, get_mut, get_mut_or_insert_with, iter_many, remove,
},
storage_schema::{TaskStorage, TaskStorageAccessors},
},
backing_storage::BackingStorage,
data::{
......@@ -1066,75 +1068,54 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let snapshot_time = Instant::now();
drop(snapshot_request);
let preprocess = |task_id: TaskId, inner: &storage::InnerStorage| {
let preprocess = |task_id: TaskId, inner: &TaskStorage| {
if task_id.is_transient() {
return (None, None);
}
let len = inner.len();
let meta_restored = inner.state().meta_restored();
let data_restored = inner.state().data_restored();
let meta_restored = inner.flags.meta_restored();
let data_restored = inner.flags.data_restored();
let mut meta = meta_restored.then(|| Vec::with_capacity(len));
let mut data = data_restored.then(|| Vec::with_capacity(len));
for (key, value) in inner.iter_all() {
if key.is_persistent() && value.is_persistent() {
match key.category() {
TaskDataCategory::Meta => {
if let Some(meta) = &mut meta {
meta.push(CachedDataItem::from_key_and_value_ref(key, value))
}
}
TaskDataCategory::Data => {
if let Some(data) = &mut data {
data.push(CachedDataItem::from_key_and_value_ref(key, value))
}
}
_ => {}
}
}
}
// Encode meta/data directly from TaskStorage
let meta = meta_restored.then(|| inner.clone_meta_snapshot());
let data = data_restored.then(|| inner.clone_data_snapshot());
(meta, data)
};
let process = |task_id: TaskId, (meta, data): (Option<Vec<_>>, Option<Vec<_>>)| {
// TODO: perf: Instead of returning a `Vec` of individually allocated `SmallVec`s, it'd
// be better to append everything to a flat per-task or per-shard `Vec<u8>`, and have
// each `serialize` call return `(start_idx, end_idx)`.
(
task_id,
meta.map(|d| self.backing_storage.serialize(task_id, &d)),
data.map(|d| self.backing_storage.serialize(task_id, &d)),
)
};
let process_snapshot = |task_id: TaskId, inner: Box<InnerStorageSnapshot>| {
let process =
|task_id: TaskId, (meta, data): (Option<TaskStorage>, Option<TaskStorage>)| {
// TODO: perf: Instead of returning a `Vec` of individually allocated `SmallVec`s,
// it'd be better to append everything to a flat per-task or
// per-shard `Vec<u8>`, and have each `serialize` call return
// `(start_idx, end_idx)`.
(
task_id,
meta.map(|d| {
self.backing_storage
.serialize(task_id, &d, SpecificTaskDataCategory::Meta)
}),
data.map(|d| {
self.backing_storage
.serialize(task_id, &d, SpecificTaskDataCategory::Data)
}),
)
};
let process_snapshot = |task_id: TaskId, inner: Box<TaskStorage>| {
if task_id.is_transient() {
return (task_id, None, None);
}
let len = inner.len();
let mut meta = inner.meta_modified.then(|| Vec::with_capacity(len));
let mut data = inner.data_modified.then(|| Vec::with_capacity(len));
for (key, value) in inner.iter_all() {
if key.is_persistent() && value.is_persistent() {
match key.category() {
TaskDataCategory::Meta => {
if let Some(meta) = &mut meta {
meta.push(CachedDataItem::from_key_and_value_ref(key, value));
}
}
TaskDataCategory::Data => {
if let Some(data) = &mut data {
data.push(CachedDataItem::from_key_and_value_ref(key, value));
}
}
_ => {}
}
}
}
// Encode meta/data directly from TaskStorage snapshot
(
task_id,
meta.map(|meta| self.backing_storage.serialize(task_id, &meta)),
data.map(|data| self.backing_storage.serialize(task_id, &data)),
inner.flags.meta_modified().then(|| {
self.backing_storage
.serialize(task_id, &inner, SpecificTaskDataCategory::Meta)
}),
inner.flags.data_modified().then(|| {
self.backing_storage
.serialize(task_id, &inner, SpecificTaskDataCategory::Data)
}),
)
};
......@@ -2599,17 +2580,20 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
}
}
// Shrink memory usage
task.shrink_to_fit(CachedDataItemType::CellData);
task.shrink_to_fit(CachedDataItemType::TransientCellData);
task.shrink_to_fit(CachedDataItemType::CellTypeMaxIndex);
task.shrink_to_fit(CachedDataItemType::CellDependency);
task.shrink_to_fit(CachedDataItemType::OutputDependency);
task.shrink_to_fit(CachedDataItemType::CollectiblesDependency);
task.shrink_to_fit(CachedDataItemType::OutdatedOutputDependency);
task.shrink_to_fit(CachedDataItemType::OutdatedCellDependency);
task.shrink_to_fit(CachedDataItemType::OutdatedCollectiblesDependency);
task.shrink_to_fit(CachedDataItemType::OutdatedCollectible);
// Shrink memory usage for collections that are only mutated during/after execution
task.shrink_cell_data();
task.shrink_transient_cell_data();
task.shrink_cell_type_max_index();
task.shrink_cell_dependencies();
task.shrink_output_dependencies();
task.shrink_collectibles_dependencies();
task.shrink_outdated_cell_dependencies();
task.shrink_outdated_output_dependencies();
task.shrink_outdated_collectibles();
task.shrink_outdated_collectibles_dependencies();
task.shrink_children();
task.shrink_collectibles();
task.shrink_in_progress_cells();
drop(task);
}
......
......@@ -33,6 +33,7 @@ use crate::{
storage::{
count, get, get_many, iter_many, remove, update, update_count, update_count_and_get,
},
storage_schema::TaskStorageAccessors,
},
data::{
ActivenessState, AggregationNumber, CachedDataItem, CachedDataItemKey, CachedDataItemType,
......
......@@ -9,13 +9,14 @@ use crate::{
backend::{
TaskDataCategory, get, get_many,
operation::{
AggregatedDataUpdate, ExecuteContext, Operation, TaskGuard,
AggregatedDataUpdate, ExecuteContext, Operation,
aggregation_update::{
AggregationUpdateJob, AggregationUpdateQueue, InnerOfUppersLostFollowersJob,
get_aggregation_number, get_uppers, is_aggregating_node,
},
},
storage::update_count,
storage_schema::TaskStorageAccessors,
},
data::{CachedDataItemKey, CellRef, CollectibleRef, CollectiblesRef},
};
......
......@@ -5,9 +5,10 @@ use crate::{
backend::{
TaskDataCategory, get_mut,
operation::{
ExecuteContext, Operation, TaskGuard,
ExecuteContext, Operation,
aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue},
},
storage_schema::TaskStorageAccessors,
},
data::{CachedDataItem, CachedDataItemKey, InProgressState, InProgressStateInner},
};
......@@ -26,7 +27,7 @@ impl ConnectChildOperation {
pub fn run(
parent_task_id: Option<TaskId>,
child_task_id: TaskId,
mut ctx: impl ExecuteContext,
mut ctx: impl ExecuteContext<'_>,
) {
if let Some(parent_task_id) = parent_task_id {
let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::All);
......
......@@ -12,8 +12,9 @@ use turbo_tasks::TaskId;
use crate::{
backend::{
TaskDataCategory,
operation::{ExecuteContext, Operation, TaskGuard},
operation::{ExecuteContext, Operation},
storage::{get, iter_many},
storage_schema::TaskStorageAccessors,
},
data::CachedDataItem,
};
......@@ -184,7 +185,7 @@ impl LeafDistanceUpdateQueue {
}
impl Operation for LeafDistanceUpdateQueue {
fn execute(mut self, ctx: &mut impl ExecuteContext) {
fn execute(mut self, ctx: &mut impl ExecuteContext<'_>) {
if self.is_empty() {
return;
}
......
......@@ -12,7 +12,7 @@ use turbo_tasks::{CellId, FxIndexMap, TaskId, TypedSharedReference, backend::Cel
use crate::backend::operation::invalidate::TaskDirtyCause;
use crate::{
backend::{
TaskDataCategory,
TaskDataCategory, TaskStorageAccessors,
operation::{
AggregationUpdateQueue, ExecuteContext, Operation, TaskGuard,
invalidate::make_task_dirty_internal,
......
......@@ -6,8 +6,7 @@ use smallvec::SmallVec;
use turbo_tasks::{TaskId, backend::CachedTaskType};
use crate::{
backend::{AnyOperation, TaskDataCategory},
data::CachedDataItem,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
utils::chunked_vec::ChunkedVec,
};
......@@ -44,8 +43,13 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
type ReadTransaction<'l>;
fn next_free_task_id(&self) -> Result<TaskId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
#[allow(clippy::ptr_arg)]
fn serialize(&self, task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>>;
fn serialize(
&self,
task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<SmallVec<[u8; 16]>>;
fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
......@@ -78,6 +82,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
) -> Result<Option<Arc<CachedTaskType>>>;
/// Lookup and decode fields directly into TaskStorage.
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
......@@ -85,8 +91,12 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
category: TaskDataCategory,
) -> Result<Vec<CachedDataItem>>;
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()>;
/// Batch lookup and decode data for multiple tasks directly into TaskStorage instances.
/// Returns a vector of TaskStorage, one for each task_id in the input slice.
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
......@@ -94,16 +104,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId],
category: TaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> {
let mut results = Vec::with_capacity(task_ids.len());
for &task_id in task_ids {
// TODO more efficient batch implementation
let data = unsafe { self.lookup_data(tx, task_id, category)? };
results.push(data);
}
Ok(results)
}
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>>;
fn shutdown(&self) -> Result<()> {
Ok(())
......@@ -134,11 +136,14 @@ where
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> {
either::for_both!(self, this => this.uncompleted_operations())
}
fn serialize(&self, task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>> {
either::for_both!(self, this => this.serialize(task, data))
fn serialize(
&self,
task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<SmallVec<[u8; 16]>> {
either::for_both!(self, this => this.serialize(task, data, category))
}
fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
......@@ -207,16 +212,17 @@ where
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
category: TaskDataCategory,
) -> Result<Vec<CachedDataItem>> {
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
unsafe { this.lookup_data(tx, task_id, category) }
unsafe { this.lookup_data(tx, task_id, category, storage) }
}
Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
unsafe { this.lookup_data(tx, task_id, category) }
unsafe { this.lookup_data(tx, task_id, category, storage) }
}
}
}
......@@ -225,8 +231,8 @@ where
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId],
category: TaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> {
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
......
......@@ -7,10 +7,7 @@ use turbo_tasks::{
event::{Event, EventListener},
};
use crate::{
backend::TaskDataCategory,
data_storage::{AutoMapStorage, OptionStorage, Storage},
};
use crate::backend::TaskDataCategory;
// this traits are needed for the transient variants of `CachedDataItem`
// transient variants are never cloned or compared
......@@ -733,6 +730,5 @@ mod tests {
assert_eq!(std::mem::size_of::<super::CachedDataItem>(), 40);
assert_eq!(std::mem::size_of::<super::CachedDataItemKey>(), 32);
assert_eq!(std::mem::size_of::<super::CachedDataItemValue>(), 32);
assert_eq!(std::mem::size_of::<super::CachedDataItemStorage>(), 56);
}
}
use std::hash::{BuildHasherDefault, Hash};
use auto_hash_map::{AutoMap, map::Entry};
use rustc_hash::FxHasher;
pub trait Storage {
type K;
type V;
type Iterator<'l>: Iterator<Item = (&'l Self::K, &'l Self::V)> + 'l
where
Self: 'l;
/// Adds a key-value pair to the storage, if the key is not already present.
/// Returns `true` if the key was not present and the value was added.
/// Returns `false` if the key was already present.
fn add(&mut self, key: Self::K, value: Self::V) -> bool;
/// Extends the storage with key-value pairs from the iterator.
/// Overwrites existing keys.
/// Returns `true` if all keys were new and added.
/// Returns `false` if any key was already present.
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool;
fn insert(&mut self, key: Self::K, value: Self::V) -> Option<Self::V>;
fn remove(&mut self, key: &Self::K) -> Option<Self::V>;
fn contains_key(&self, key: &Self::K) -> bool;
fn get(&self, key: &Self::K) -> Option<&Self::V>;
fn get_mut(&mut self, key: &Self::K) -> Option<&mut Self::V>;
fn get_mut_or_insert_with(&mut self, key: Self::K, f: impl FnOnce() -> Self::V)
-> &mut Self::V;
fn extract_if<'l, F>(&'l mut self, f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l;
fn update(&mut self, key: Self::K, update: impl FnOnce(Option<Self::V>) -> Option<Self::V>);
fn shrink_to_fit(&mut self);
fn is_empty(&self) -> bool;
fn len(&self) -> usize;
fn iter(&self) -> Self::Iterator<'_>;
}
fn value_to_key_value<V>(value: &V) -> (&(), &V) {
(&(), value)
}
#[derive(Debug, Clone)]
pub struct OptionStorage<V> {
value: Option<V>,
}
impl<V> Default for OptionStorage<V> {
fn default() -> Self {
Self { value: None }
}
}
impl<V> Storage for OptionStorage<V> {
type K = ();
type V = V;
type Iterator<'l>
= std::option::IntoIter<(&'l (), &'l V)>
where
Self: 'l;
fn add(&mut self, _: (), value: V) -> bool {
if self.value.is_none() {
self.value = Some(value);
true
} else {
false
}
}
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool {
let mut added = true;
for (_, value) in items {
added = self.insert((), value).is_none() && added;
}
added
}
fn insert(&mut self, _: (), value: V) -> Option<V> {
self.value.replace(value)
}
fn remove(&mut self, _: &()) -> Option<V> {
self.value.take()
}
fn contains_key(&self, _: &()) -> bool {
self.value.is_some()
}
fn get(&self, _: &()) -> Option<&V> {
self.value.as_ref()
}
fn get_mut(&mut self, _: &()) -> Option<&mut V> {
self.value.as_mut()
}
fn get_mut_or_insert_with(&mut self, _: (), f: impl FnOnce() -> V) -> &mut V {
self.value.get_or_insert_with(f)
}
fn shrink_to_fit(&mut self) {
// Nothing to do
}
fn is_empty(&self) -> bool {
self.value.is_none()
}
fn len(&self) -> usize {
if self.value.is_some() { 1 } else { 0 }
}
fn iter(&self) -> Self::Iterator<'_> {
self.value.as_ref().map(value_to_key_value).into_iter()
}
fn extract_if<'l, F>(&'l mut self, mut f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l,
{
if let Some(value) = self.value.as_mut()
&& f(&(), value)
{
return self.value.take().map(|v| ((), v)).into_iter();
}
None.into_iter()
}
fn update(&mut self, _: (), update: impl FnOnce(Option<V>) -> Option<V>) {
self.value = update(self.value.take());
}
}
/// Storage for a single value that uses `Default::default()` to initialize the value "empty".
#[derive(Debug, Clone)]
pub struct DefaultStorage<V> {
value: V,
}
impl<V: Default> Default for DefaultStorage<V> {
fn default() -> Self {
Self {
value: V::default(),
}
}
}
impl<V: Default + PartialEq> Storage for DefaultStorage<V> {
type K = ();
type V = V;
type Iterator<'l>
= std::option::IntoIter<(&'l (), &'l V)>
where
Self: 'l;
fn add(&mut self, _: (), value: V) -> bool {
if self.value == V::default() {
self.value = value;
true
} else {
false
}
}
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool {
let mut added = true;
for (_, value) in items {
added = self.insert((), value).is_none() && added;
}
added
}
fn insert(&mut self, _: (), value: V) -> Option<V> {
let old = std::mem::replace(&mut self.value, value);
if old == V::default() { None } else { Some(old) }
}
fn remove(&mut self, _: &()) -> Option<V> {
let old = std::mem::take(&mut self.value);
if old == V::default() { None } else { Some(old) }
}
fn contains_key(&self, _: &()) -> bool {
self.value != V::default()
}
fn get(&self, _: &()) -> Option<&V> {
if self.value != V::default() {
Some(&self.value)
} else {
None
}
}
fn get_mut(&mut self, _: &()) -> Option<&mut V> {
if self.value != V::default() {
Some(&mut self.value)
} else {
None
}
}
fn get_mut_or_insert_with(&mut self, _: (), f: impl FnOnce() -> V) -> &mut V {
if self.value == V::default() {
self.value = f();
}
&mut self.value
}
fn shrink_to_fit(&mut self) {
// Nothing to do
}
fn is_empty(&self) -> bool {
self.value == V::default()
}
fn len(&self) -> usize {
if self.value != V::default() { 1 } else { 0 }
}
fn iter(&self) -> Self::Iterator<'_> {
if self.value != V::default() {
Some(value_to_key_value(&self.value)).into_iter()
} else {
None.into_iter()
}
}
fn extract_if<'l, F>(&'l mut self, mut f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l,
{
if self.value != V::default() && f(&(), &mut self.value) {
let old = std::mem::take(&mut self.value);
return Some(((), old)).into_iter();
}
None.into_iter()
}
fn update(&mut self, _: (), update: impl FnOnce(Option<V>) -> Option<V>) {
let old = std::mem::take(&mut self.value);
let old_opt = if old == V::default() { None } else { Some(old) };
self.value = update(old_opt).unwrap_or_default();
}
}
#[derive(Debug, Clone)]
pub struct AutoMapStorage<K, V> {
map: AutoMap<K, V, BuildHasherDefault<FxHasher>, 1>,
}
impl<K, V> Default for AutoMapStorage<K, V> {
fn default() -> Self {
Self {
map: AutoMap::default(),
}
}
}
impl<K: Hash + Eq, V> Storage for AutoMapStorage<K, V> {
type K = K;
type V = V;
type Iterator<'l>
= auto_hash_map::map::Iter<'l, K, V>
where
Self: 'l;
fn add(&mut self, key: K, value: V) -> bool {
match self.map.entry(key) {
auto_hash_map::map::Entry::Vacant(entry) => {
entry.insert(value);
true
}
auto_hash_map::map::Entry::Occupied(_) => false,
}
}
fn extend(&mut self, items: impl Iterator<Item = (Self::K, Self::V)>) -> bool {
let len = self.map.len();
let mut count = 0;
self.map.extend(items.inspect(|_| count += 1));
self.map.len() == len + count
}
fn insert(&mut self, key: K, value: V) -> Option<V> {
self.map.insert(key, value)
}
fn remove(&mut self, key: &K) -> Option<V> {
self.map
.remove(key)
.inspect(|_| self.map.shrink_amortized())
}
fn contains_key(&self, key: &K) -> bool {
self.map.contains_key(key)
}
fn get(&self, key: &K) -> Option<&V> {
self.map.get(key)
}
fn get_mut(&mut self, key: &K) -> Option<&mut V> {
self.map.get_mut(key)
}
fn get_mut_or_insert_with(&mut self, key: K, f: impl FnOnce() -> V) -> &mut V {
self.map.entry(key).or_insert_with(f)
}
fn shrink_to_fit(&mut self) {
self.map.shrink_to_fit()
}
fn is_empty(&self) -> bool {
self.map.is_empty()
}
fn len(&self) -> usize {
self.map.len()
}
fn iter(&self) -> Self::Iterator<'_> {
self.map.iter()
}
fn extract_if<'l, F>(&'l mut self, f: F) -> impl Iterator<Item = (Self::K, Self::V)>
where
F: for<'a, 'b> FnMut(&'a Self::K, &'b mut Self::V) -> bool + 'l,
{
self.map.extract_if(f)
}
fn update(&mut self, key: K, update: impl FnOnce(Option<V>) -> Option<V>) {
match self.map.entry(key) {
Entry::Vacant(e) => {
if let Some(new) = update(None) {
e.insert(new);
}
}
Entry::Occupied(e) => e.replace_entry_with(move |_, v| update(Some(v))),
}
}
}
......@@ -7,7 +7,8 @@ use std::{
use anyhow::{Context, Result};
use turbo_bincode::{
TurboBincodeBuffer, turbo_bincode_decode, turbo_bincode_encode, turbo_bincode_encode_into,
TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder, turbo_bincode_decode,
turbo_bincode_encode, turbo_bincode_encode_into,
};
use turbo_tasks::{
TaskId,
......@@ -18,9 +19,8 @@ use turbo_tasks::{
use crate::{
GitVersionInfo,
backend::{AnyOperation, TaskDataCategory},
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backing_storage::{BackingStorage, BackingStorageSealed},
data::CachedDataItem,
database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
db_versioning::handle_db_versioning,
......@@ -250,8 +250,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
get(&self.inner.database).context("Unable to read uncompleted operations from database")
}
fn serialize(&self, task: TaskId, data: &Vec<CachedDataItem>) -> Result<TurboBincodeBuffer> {
encode_task_data(task, data)
fn serialize(
&self,
task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<TurboBincodeBuffer> {
encode_task_data(task, data, category)
}
fn save_snapshot<I>(
......@@ -510,64 +515,61 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&self,
tx: Option<&T::ReadTransaction<'_>>,
task_id: TaskId,
category: TaskDataCategory,
) -> Result<Vec<CachedDataItem>> {
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()> {
let inner = &*self.inner;
fn lookup<D: KeyValueDatabase>(
database: &D,
tx: &D::ReadTransaction<'_>,
task_id: TaskId,
category: TaskDataCategory,
) -> Result<Vec<CachedDataItem>> {
let Some(bytes) = database.get(
tx,
category_to_key_space(category),
IntKey::new(*task_id).as_ref(),
)?
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()> {
let Some(bytes) =
database.get(tx, category.key_space(), IntKey::new(*task_id).as_ref())?
else {
return Ok(Vec::new());
return Ok(());
};
let result: Vec<CachedDataItem> = turbo_bincode_decode(bytes.borrow())?;
Ok(result)
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))
}
inner
.with_tx(tx, |tx| lookup(&inner.database, tx, task_id, category))
.with_context(|| format!("Looking up data for {task_id} from database failed"))
.with_tx(tx, |tx| {
lookup(&inner.database, tx, task_id, category, storage)
})
.with_context(|| format!("Looking up task storage for {task_id} from database failed"))
}
unsafe fn batch_lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId],
category: TaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> {
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> {
let inner = &*self.inner;
fn lookup<D: KeyValueDatabase>(
database: &D,
tx: &D::ReadTransaction<'_>,
task_ids: &[TaskId],
category: TaskDataCategory,
) -> Result<Vec<Vec<CachedDataItem>>> {
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> {
let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect();
let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>();
let bytes = database.batch_get(
tx,
match category {
TaskDataCategory::Meta => KeySpace::TaskMeta,
TaskDataCategory::Data => KeySpace::TaskData,
TaskDataCategory::All => unreachable!(),
},
&keys,
)?;
let bytes = database.batch_get(tx, category.key_space(), &keys)?;
bytes
.into_iter()
.map(|opt_bytes| {
let mut storage = TaskStorage::new();
if let Some(bytes) = opt_bytes {
let result: Vec<CachedDataItem> = turbo_bincode_decode(bytes.borrow())?;
Ok(result)
} else {
Ok(Vec::new())
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))?;
}
Ok(storage)
})
.collect::<Result<Vec<_>>>()
}
......@@ -575,7 +577,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
.with_tx(tx, |tx| lookup(&inner.database, tx, task_ids, category))
.with_context(|| {
format!(
"Looking up data for {} tasks from database failed",
"Looking up typed data for {} tasks from database failed",
task_ids.len()
)
})
......@@ -761,55 +763,29 @@ where
})
}
fn encode_task_data(task: TaskId, data: &Vec<CachedDataItem>) -> Result<TurboBincodeBuffer> {
let orig_result = turbo_bincode_encode(data);
if !cfg!(feature = "verify_serialization")
&& let Ok(value) = orig_result
{
return Ok(value);
fn encode_task_data(
task: TaskId,
data: &TaskStorage,
category: SpecificTaskDataCategory,
) -> Result<TurboBincodeBuffer> {
// TODO: see if the caller can pass us a buffer instead of us allocating a new one.
// This should be possible and save a lot of small allocations.
let mut buffer = TurboBincodeBuffer::new();
let mut encoder = new_turbo_bincode_encoder(&mut buffer);
data.encode(category, &mut encoder)?;
if !cfg!(feature = "verify_serialization") {
return Ok(buffer);
}
let mut error = Ok(());
let mut filtered_data = data.clone();
filtered_data.retain(|item| match turbo_bincode_encode(&item) {
Ok(buf) => {
if cfg!(feature = "verify_serialization") {
let deserialized = turbo_bincode_decode::<CachedDataItem>(&buf);
if let Err(err) = deserialized {
println!("Data item would not be deserializable {task}: {err:?}\n{item:?}");
return false;
}
}
true
}
Err(err) => {
if item.is_optional() {
if cfg!(feature = "verify_serialization") {
println!(
"Skipping non-encodable optional item for {task}: {item:?} due to {err}"
);
}
} else {
error =
Err(err).context(format!("Unable to encode data item for {task}: {item:?}"));
}
false
}
});
error?;
(if filtered_data.len() == data.len() {
orig_result
} else {
turbo_bincode_encode(&filtered_data)
})
.with_context(|| format!("Unable to serialize data items for {task}: {filtered_data:#?}"))
}
TaskStorage::new()
.decode(category, &mut new_turbo_bincode_decoder(buffer.borrow()))
.with_context(|| {
format!(
"expected to be able to decode serialized data for '{category:?}' information for \
{task}"
)
})?;
fn category_to_key_space(category: TaskDataCategory) -> KeySpace {
match category {
TaskDataCategory::Meta => KeySpace::TaskMeta,
TaskDataCategory::Data => KeySpace::TaskData,
TaskDataCategory::All => unreachable!(),
}
Ok(buffer)
}
......@@ -6,7 +6,6 @@
mod backend;
mod backing_storage;
mod data;
mod data_storage;
mod database;
mod kv_backing_storage;
mod utils;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment