Unverified Commit 0dd0f20a authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] refactor persistent caching from log based to cow approach (#76234)

### What?

Instead of collecting all modifications to the task graph in a log
structure (which ends up taking a lot of memory), use a modified and
snapshot flag to avoid using more memory than needed.

This way we don't store extra memory before a snapshot is requested.
Once a snapshot is captured, we store at most twice the memory. We store
only the memory of the modified tasks.

It works this way:

Idle Phase: (before a snapshot is requested)

* When a task is modified, we only set the modified flag on that task.

Capturing a snapshot want to avoid locking all tasks. So it first
switches to the Snapshot Phase.

Snapshot Phase: (after a snapshot is captured)

* When a task is modified:
  * If there is already a snapshot, skip
* If the task has the modified flag set, clone the task and store the
snapshot.
* If the task doesn't have the modified flag set, store a None snapshot.

To actually capture the snapshot, go over all modified or snapshot tasks
(which is stored in a separate map):

* If the task has a snapshot (not None) stored, use that.
* If the task has only the modified flag set, read the task state and
capture a snapshot from that.
* There is a race condition here, when the task is modified inbetween.
We handle that by having a snapshot flag too.

We have all snapshot captured now. To leave Snapshot Phase, we 

* First unset all modified flags.
* Switch to Idle Phase.
* Change all stored snapshots to modified.
parent e9982de2
This diff is collapsed.
...@@ -332,6 +332,7 @@ async-compression = { version = "0.3.13", default-features = false, features = [ ...@@ -332,6 +332,7 @@ async-compression = { version = "0.3.13", default-features = false, features = [
] } ] }
async-trait = "0.1.64" async-trait = "0.1.64"
atty = "0.2.14" atty = "0.2.14"
bitfield = "0.18.0"
bytes = "1.1.0" bytes = "1.1.0"
chrono = "0.4.23" chrono = "0.4.23"
clap = { version = "4.5.2", features = ["derive"] } clap = { version = "4.5.2", features = ["derive"] }
......
...@@ -26,6 +26,7 @@ anyhow = { workspace = true } ...@@ -26,6 +26,7 @@ anyhow = { workspace = true }
arc-swap = { version = "1.7.1" } arc-swap = { version = "1.7.1" }
async-trait = { workspace = true } async-trait = { workspace = true }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
bitfield = { workspace = true }
byteorder = "1.5.0" byteorder = "1.5.0"
dashmap = { workspace = true, features = ["raw-api"]} dashmap = { workspace = true, features = ["raw-api"]}
either = { workspace = true } either = { workspace = true }
......
...@@ -6,6 +6,7 @@ use crate::data::{ ...@@ -6,6 +6,7 @@ use crate::data::{
CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut, CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut,
}; };
#[derive(Debug, Clone)]
pub struct DynamicStorage { pub struct DynamicStorage {
map: Vec<CachedDataItemStorage>, map: Vec<CachedDataItemStorage>,
} }
...@@ -162,9 +163,24 @@ impl DynamicStorage { ...@@ -162,9 +163,24 @@ impl DynamicStorage {
} }
} }
pub fn len(&self) -> usize {
self.map.iter().map(|m| m.len()).sum()
}
pub fn shrink_to_fit(&mut self, ty: CachedDataItemType) { pub fn shrink_to_fit(&mut self, ty: CachedDataItemType) {
if let Some(map) = self.get_map_mut(ty) { if let Some(map) = self.get_map_mut(ty) {
map.shrink_to_fit(); map.shrink_to_fit();
} }
} }
pub fn snapshot_for_persisting(&self) -> Self {
Self {
map: self
.map
.iter()
.filter(|m| m.ty().is_persistent())
.cloned()
.collect(),
}
}
} }
...@@ -30,7 +30,7 @@ use crate::{ ...@@ -30,7 +30,7 @@ use crate::{
ActivenessState, AggregationNumber, CachedDataItem, CachedDataItemKey, CollectibleRef, ActivenessState, AggregationNumber, CachedDataItem, CachedDataItemKey, CollectibleRef,
DirtyContainerCount, DirtyContainerCount,
}, },
utils::deque_set::DequeSet, utils::{deque_set::DequeSet, swap_retain},
}; };
pub const LEAF_NUMBER: u32 = 16; pub const LEAF_NUMBER: u32 = 16;
...@@ -1526,6 +1526,7 @@ impl AggregationUpdateQueue { ...@@ -1526,6 +1526,7 @@ impl AggregationUpdateQueue {
let mut upper_upper_ids_with_new_follower = SmallVec::new(); let mut upper_upper_ids_with_new_follower = SmallVec::new();
let mut tasks_for_which_increment_active_count = SmallVec::new(); let mut tasks_for_which_increment_active_count = SmallVec::new();
let mut is_active = false; let mut is_active = false;
swap_retain(&mut upper_ids, |&mut upper_id| { swap_retain(&mut upper_ids, |&mut upper_id| {
let mut upper = ctx.task( let mut upper = ctx.task(
upper_id, upper_id,
...@@ -2355,29 +2356,3 @@ impl Operation for AggregationUpdateQueue { ...@@ -2355,29 +2356,3 @@ impl Operation for AggregationUpdateQueue {
} }
} }
} }
fn swap_retain<T, const N: usize>(vec: &mut SmallVec<[T; N]>, mut f: impl FnMut(&mut T) -> bool) {
let mut i = 0;
while i < vec.len() {
if !f(&mut vec[i]) {
vec.swap_remove(i);
} else {
i += 1;
}
}
}
#[cfg(test)]
mod tests {
use smallvec::{smallvec, SmallVec};
use crate::backend::operation::aggregation_update::swap_retain;
#[test]
fn test_swap_retain() {
let mut vec: SmallVec<[i32; 4]> = smallvec![1, 2, 3, 4, 5];
swap_retain(&mut vec, |a| *a % 2 != 0);
let expected: SmallVec<[i32; 4]> = smallvec![1, 5, 3];
assert_eq!(vec, expected);
}
}
use turbo_tasks::{KeyValuePair, TaskId};
use crate::{
data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate},
utils::{chunked_vec::ChunkedVec, sharded::Sharded},
};
#[derive(Default)]
struct ShardData {
last_task: Option<TaskId>,
data: ChunkedVec<CachedDataUpdate>,
}
impl ShardData {
fn set_task(&mut self, task: TaskId) {
if self.last_task != Some(task) {
self.data.push(CachedDataUpdate::Task { task });
self.last_task = Some(task);
}
}
}
pub struct PersistedStorageLog {
data: Sharded<ShardData>,
}
impl PersistedStorageLog {
pub fn new(shard_amount: usize) -> Self {
Self {
data: Sharded::new(shard_amount),
}
}
pub fn push(
&self,
task: TaskId,
key: CachedDataItemKey,
old_value: Option<CachedDataItemValue>,
new_value: Option<CachedDataItemValue>,
) {
let mut guard = self.data.lock(task);
guard.set_task(task);
match (old_value, new_value) {
(None, None) => {}
(None, Some(new_value)) => guard.data.push(CachedDataUpdate::New {
item: CachedDataItem::from_key_and_value(key, new_value),
}),
(Some(old_value), None) => guard.data.push(CachedDataUpdate::Removed {
old_item: CachedDataItem::from_key_and_value(key, old_value),
}),
(Some(old_value), Some(new_value)) => {
guard.data.push(CachedDataUpdate::Replace1 {
old_item: CachedDataItem::from_key_and_value(key, old_value),
});
guard
.data
.push(CachedDataUpdate::Replace2 { value: new_value });
}
}
}
pub fn push_batch_insert(
&self,
task: TaskId,
updates: impl IntoIterator<Item = CachedDataItem>,
) {
let updates = updates
.into_iter()
.map(|item| CachedDataUpdate::New { item });
let mut guard = self.data.lock(task);
guard.set_task(task);
guard.data.extend(updates);
}
pub fn take(&self) -> Vec<ChunkedVec<CachedDataUpdate>> {
self.data.take(|shard| shard.data)
}
}
use std::sync::Arc; use std::sync::Arc;
use anyhow::Result; use anyhow::Result;
use smallvec::SmallVec;
use turbo_tasks::{backend::CachedTaskType, SessionId, TaskId}; use turbo_tasks::{backend::CachedTaskType, SessionId, TaskId};
use crate::{ use crate::{
backend::{AnyOperation, TaskDataCategory}, backend::{AnyOperation, TaskDataCategory},
data::{CachedDataItem, CachedDataUpdate}, data::CachedDataItem,
utils::chunked_vec::ChunkedVec, utils::chunked_vec::ChunkedVec,
}; };
...@@ -17,14 +18,24 @@ pub trait BackingStorage: 'static + Send + Sync { ...@@ -17,14 +18,24 @@ pub trait BackingStorage: 'static + Send + Sync {
fn next_free_task_id(&self) -> TaskId; fn next_free_task_id(&self) -> TaskId;
fn next_session_id(&self) -> SessionId; fn next_session_id(&self) -> SessionId;
fn uncompleted_operations(&self) -> Vec<AnyOperation>; fn uncompleted_operations(&self) -> Vec<AnyOperation>;
fn save_snapshot( #[allow(clippy::ptr_arg)]
fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>>;
fn save_snapshot<I>(
&self, &self,
session_id: SessionId, session_id: SessionId,
operations: Vec<Arc<AnyOperation>>, operations: Vec<Arc<AnyOperation>>,
task_cache_updates: Vec<ChunkedVec<(Arc<CachedTaskType>, TaskId)>>, task_cache_updates: Vec<ChunkedVec<(Arc<CachedTaskType>, TaskId)>>,
meta_updates: Vec<ChunkedVec<CachedDataUpdate>>, snapshots: Vec<I>,
data_updates: Vec<ChunkedVec<CachedDataUpdate>>, ) -> Result<()>
) -> Result<()>; where
I: Iterator<
Item = (
TaskId,
Option<SmallVec<[u8; 16]>>,
Option<SmallVec<[u8; 16]>>,
),
> + Send
+ Sync;
fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>; fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>;
/// # Safety /// # Safety
/// ///
......
...@@ -588,6 +588,10 @@ impl CachedDataItem { ...@@ -588,6 +588,10 @@ impl CachedDataItem {
| Self::Activeness { .. } => TaskDataCategory::All, | Self::Activeness { .. } => TaskDataCategory::All,
} }
} }
pub fn is_optional(&self) -> bool {
matches!(self, CachedDataItem::CellData { .. })
}
} }
impl CachedDataItemKey { impl CachedDataItemKey {
...@@ -627,10 +631,6 @@ impl CachedDataItemKey { ...@@ -627,10 +631,6 @@ impl CachedDataItemKey {
} }
} }
pub fn is_optional(&self) -> bool {
matches!(self, CachedDataItemKey::CellData { .. })
}
pub fn category(&self) -> TaskDataCategory { pub fn category(&self) -> TaskDataCategory {
self.ty().category() self.ty().category()
} }
...@@ -670,6 +670,39 @@ impl CachedDataItemType { ...@@ -670,6 +670,39 @@ impl CachedDataItemType {
| Self::Activeness { .. } => TaskDataCategory::All, | Self::Activeness { .. } => TaskDataCategory::All,
} }
} }
pub fn is_persistent(&self) -> bool {
match self {
Self::Output
| Self::Collectible
| Self::Dirty
| Self::Child
| Self::CellData
| Self::CellTypeMaxIndex
| Self::OutputDependency
| Self::CellDependency
| Self::CollectiblesDependency
| Self::OutputDependent
| Self::CellDependent
| Self::CollectiblesDependent
| Self::AggregationNumber
| Self::Follower
| Self::Upper
| Self::AggregatedDirtyContainer
| Self::AggregatedCollectible
| Self::AggregatedDirtyContainerCount
| Self::Stateful => true,
Self::Activeness
| Self::InProgress
| Self::InProgressCell
| Self::OutdatedCollectible
| Self::OutdatedOutputDependency
| Self::OutdatedCellDependency
| Self::OutdatedCollectiblesDependency
| Self::Error => false,
}
}
} }
/// Used by the [`get_mut`][crate::backend::storage::get_mut] macro to restrict mutable access to a /// Used by the [`get_mut`][crate::backend::storage::get_mut] macro to restrict mutable access to a
...@@ -681,11 +714,11 @@ pub mod allow_mut_access { ...@@ -681,11 +714,11 @@ pub mod allow_mut_access {
pub const Activeness: () = (); pub const Activeness: () = ();
} }
impl CachedDataItemValue { impl CachedDataItemValueRef<'_> {
pub fn is_persistent(&self) -> bool { pub fn is_persistent(&self) -> bool {
match self { match self {
CachedDataItemValue::Output { value } => !value.is_transient(), CachedDataItemValueRef::Output { value } => !value.is_transient(),
CachedDataItemValue::CellData { value } => { CachedDataItemValueRef::CellData { value } => {
registry::get_value_type(value.0).is_serializable() registry::get_value_type(value.0).is_serializable()
} }
_ => true, _ => true,
...@@ -693,20 +726,6 @@ impl CachedDataItemValue { ...@@ -693,20 +726,6 @@ impl CachedDataItemValue {
} }
} }
#[derive(Debug)]
pub enum CachedDataUpdate {
/// Sets the current task id.
Task { task: TaskId },
/// An item was added. There was no old value.
New { item: CachedDataItem },
/// An item was removed.
Removed { old_item: CachedDataItem },
/// An item was replaced. This is step 1 and tells about the key and the old value
Replace1 { old_item: CachedDataItem },
/// An item was replaced. This is step 2 and tells about the new value.
Replace2 { value: CachedDataItemValue },
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
...@@ -716,6 +735,5 @@ mod tests { ...@@ -716,6 +735,5 @@ mod tests {
assert_eq!(std::mem::size_of::<super::CachedDataItemKey>(), 20); assert_eq!(std::mem::size_of::<super::CachedDataItemKey>(), 20);
assert_eq!(std::mem::size_of::<super::CachedDataItemValue>(), 32); assert_eq!(std::mem::size_of::<super::CachedDataItemValue>(), 32);
assert_eq!(std::mem::size_of::<super::CachedDataItemStorage>(), 48); assert_eq!(std::mem::size_of::<super::CachedDataItemStorage>(), 48);
assert_eq!(std::mem::size_of::<super::CachedDataUpdate>(), 48);
} }
} }
...@@ -32,7 +32,7 @@ fn value_to_key_value<V>(value: &V) -> (&(), &V) { ...@@ -32,7 +32,7 @@ fn value_to_key_value<V>(value: &V) -> (&(), &V) {
(&(), value) (&(), value)
} }
#[derive(Debug)] #[derive(Debug, Clone)]
pub struct OptionStorage<V> { pub struct OptionStorage<V> {
value: Option<V>, value: Option<V>,
} }
...@@ -121,7 +121,7 @@ impl<V> Storage for OptionStorage<V> { ...@@ -121,7 +121,7 @@ impl<V> Storage for OptionStorage<V> {
} }
} }
#[derive(Debug)] #[derive(Debug, Clone)]
pub struct AutoMapStorage<K, V> { pub struct AutoMapStorage<K, V> {
map: AutoMap<K, V, BuildHasherDefault<FxHasher>, 1>, map: AutoMap<K, V, BuildHasherDefault<FxHasher>, 1>,
} }
......
...@@ -25,6 +25,10 @@ unsafe impl<K: Eq + Hash + Sync, V: Sync> Send for RefMut<'_, K, V> {} ...@@ -25,6 +25,10 @@ unsafe impl<K: Eq + Hash + Sync, V: Sync> Send for RefMut<'_, K, V> {}
unsafe impl<K: Eq + Hash + Sync, V: Sync> Sync for RefMut<'_, K, V> {} unsafe impl<K: Eq + Hash + Sync, V: Sync> Sync for RefMut<'_, K, V> {}
impl<K: Eq + Hash, V> RefMut<'_, K, V> { impl<K: Eq + Hash, V> RefMut<'_, K, V> {
pub fn key(&self) -> &K {
self.pair().0
}
pub fn value(&self) -> &V { pub fn value(&self) -> &V {
self.pair().1 self.pair().1
} }
......
...@@ -4,3 +4,6 @@ pub mod dash_map_multi; ...@@ -4,3 +4,6 @@ pub mod dash_map_multi;
pub mod deque_set; pub mod deque_set;
pub mod ptr_eq_arc; pub mod ptr_eq_arc;
pub mod sharded; pub mod sharded;
pub mod swap_retain;
pub use swap_retain::swap_retain;
use std::ops::{Index, IndexMut};
use smallvec::SmallVec;
pub trait VecLike<T>: Index<usize, Output = T> + IndexMut<usize, Output = T> {
fn len(&self) -> usize;
fn swap_remove(&mut self, index: usize);
}
impl<T> VecLike<T> for Vec<T> {
fn len(&self) -> usize {
Vec::len(self)
}
fn swap_remove(&mut self, index: usize) {
Vec::swap_remove(self, index);
}
}
impl<T, const N: usize> VecLike<T> for SmallVec<[T; N]> {
fn len(&self) -> usize {
SmallVec::len(self)
}
fn swap_remove(&mut self, index: usize) {
SmallVec::swap_remove(self, index);
}
}
pub fn swap_retain<T>(vec: &mut impl VecLike<T>, mut f: impl FnMut(&mut T) -> bool) {
let mut i = 0;
while i < vec.len() {
if !f(&mut vec[i]) {
vec.swap_remove(i);
} else {
i += 1;
}
}
}
#[cfg(test)]
mod tests {
use smallvec::{smallvec, SmallVec};
use super::swap_retain;
#[test]
fn test_swap_retain() {
let mut vec: SmallVec<[i32; 4]> = smallvec![1, 2, 3, 4, 5];
swap_retain(&mut vec, |a| *a % 2 != 0);
let expected: SmallVec<[i32; 4]> = smallvec![1, 5, 3];
assert_eq!(vec, expected);
}
}
...@@ -219,6 +219,15 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { ...@@ -219,6 +219,15 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream {
} }
} }
fn from_key_and_value_ref(key: #key_name, value_ref: #value_ref_name) -> Self {
match (key, value_ref) {
#(
(#key_name::#variant_names { #key_pat }, #value_ref_name::#variant_names { #value_pat }) => #ident::#variant_names { #key_pat #value_clone_fields },
)*
_ => panic!("Invalid key and value combination"),
}
}
fn into_key_and_value(self) -> (#key_name, #value_name) { fn into_key_and_value(self) -> (#key_name, #value_name) {
match self { match self {
#( #(
...@@ -303,6 +312,7 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { ...@@ -303,6 +312,7 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream {
} }
} }
#[derive(Debug, Clone)]
#vis enum #storage_name { #vis enum #storage_name {
#( #(
#variant_names { #variant_names {
......
...@@ -168,15 +168,18 @@ pub async fn apply_effects(source: impl CollectiblesSource) -> Result<()> { ...@@ -168,15 +168,18 @@ pub async fn apply_effects(source: impl CollectiblesSource) -> Result<()> {
} }
let span = tracing::info_span!("apply effects", count = effects.len()); let span = tracing::info_span!("apply effects", count = effects.len());
async move { async move {
let mut first_error = anyhow::Ok(()); effects
for effect in effects { .into_iter()
let Some(effect) = Vc::try_resolve_downcast_type::<EffectInstance>(effect).await? .map(async |effect| {
else { let Some(effect) = Vc::try_resolve_downcast_type::<EffectInstance>(effect).await?
panic!("Effect must only be implemented by EffectInstance"); else {
}; panic!("Effect must only be implemented by EffectInstance");
apply_effect(&effect.await?, &mut first_error).await; };
} effect.await?.apply().await
first_error })
.try_join()
.await?;
Ok(())
} }
.instrument(span) .instrument(span)
.await .await
...@@ -249,29 +252,18 @@ impl Effects { ...@@ -249,29 +252,18 @@ impl Effects {
pub async fn apply(&self) -> Result<()> { pub async fn apply(&self) -> Result<()> {
let span = tracing::info_span!("apply effects", count = self.effects.len()); let span = tracing::info_span!("apply effects", count = self.effects.len());
async move { async move {
let mut first_error = anyhow::Ok(()); self.effects
for effect in self.effects.iter() { .iter()
apply_effect(effect, &mut first_error).await; .map(async |effect| effect.apply().await)
} .try_join()
first_error .await?;
Ok(())
} }
.instrument(span) .instrument(span)
.await .await
} }
} }
async fn apply_effect(
effect: &ReadRef<EffectInstance>,
first_error: &mut std::result::Result<(), anyhow::Error>,
) {
match effect.apply().await {
Err(err) if first_error.is_ok() => {
*first_error = Err(err);
}
_ => {}
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::{apply_effects, get_effects, CollectiblesSource}; use crate::{apply_effects, get_effects, CollectiblesSource};
......
...@@ -16,5 +16,6 @@ pub trait KeyValuePair { ...@@ -16,5 +16,6 @@ pub trait KeyValuePair {
fn value_ref(&self) -> Self::ValueRef<'_>; fn value_ref(&self) -> Self::ValueRef<'_>;
fn value_mut(&mut self) -> Self::ValueRefMut<'_>; fn value_mut(&mut self) -> Self::ValueRefMut<'_>;
fn from_key_and_value(key: Self::Key, value: Self::Value) -> Self; fn from_key_and_value(key: Self::Key, value: Self::Value) -> Self;
fn from_key_and_value_ref(key: Self::Key, value_ref: Self::ValueRef<'_>) -> Self;
fn into_key_and_value(self) -> (Self::Key, Self::Value); fn into_key_and_value(self) -> (Self::Key, Self::Value);
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment