Unverified Commit a07cba75 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Add trace metadata about the persist size (#93339)

Record simple counters for how many items were persisted.

This should be cheap enough to always do
parent a9323da2
...@@ -68,7 +68,7 @@ use crate::{ ...@@ -68,7 +68,7 @@ use crate::{
storage::Storage, storage::Storage,
storage_schema::{TaskStorage, TaskStorageAccessors}, storage_schema::{TaskStorage, TaskStorageAccessors},
}, },
backing_storage::{BackingStorage, SnapshotItem, compute_task_type_hash}, backing_storage::{BackingStorage, SnapshotItem, SnapshotMeta, compute_task_type_hash},
data::{ data::{
ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState, ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressState, InProgressStateInner, OutputValue, TransientTask, InProgressState, InProgressStateInner, OutputValue, TransientTask,
...@@ -1233,13 +1233,32 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1233,13 +1233,32 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
} }
let persist_start = Instant::now(); let persist_start = Instant::now();
let _span = tracing::info_span!(parent: parent_span, "persist", reason = reason).entered(); let span = tracing::info_span!(
parent: parent_span,
"persist",
reason = reason,
data_items= tracing::field::Empty,
meta_items= tracing::field::Empty,
task_cache_items= tracing::field::Empty,
next_task_id= tracing::field::Empty,)
.entered();
{ {
// Tasks were already consumed by take_snapshot, so a future snapshot // Tasks were already consumed by take_snapshot, so a future snapshot
// would not re-persist them — returning an error signals to the caller // would not re-persist them — returning an error signals to the caller
// that further persist attempts would corrupt the task graph in storage. // that further persist attempts would corrupt the task graph in storage.
self.backing_storage let SnapshotMeta {
task_cache_items,
data_items,
meta_items,
max_next_task_id,
} = self
.backing_storage
.save_snapshot(suspended_operations, task_snapshots)?; .save_snapshot(suspended_operations, task_snapshots)?;
span.record("data_items", data_items);
span.record("meta_items", meta_items);
span.record("task_cache_items", task_cache_items);
span.record("next_task_id", max_next_task_id);
#[cfg(feature = "print_cache_item_size")] #[cfg(feature = "print_cache_item_size")]
{ {
let mut task_cache_stats = task_cache_stats let mut task_cache_stats = task_cache_stats
......
use std::sync::Arc; use std::{cmp::max, sync::Arc};
use anyhow::Result; use anyhow::Result;
use either::Either; use either::Either;
...@@ -82,6 +82,26 @@ pub trait BackingStorage: BackingStorageSealed { ...@@ -82,6 +82,26 @@ pub trait BackingStorage: BackingStorageSealed {
fn invalidate(&self, reason_code: &str) -> Result<()>; fn invalidate(&self, reason_code: &str) -> Result<()>;
} }
#[derive(Copy, Clone, Debug, Default)]
pub struct SnapshotMeta {
pub data_items: usize,
pub meta_items: usize,
pub task_cache_items: usize,
pub max_next_task_id: u32,
}
impl SnapshotMeta {
/// Merge two snapshots, summing the counts and `max`'ing the task id
pub fn merge(&self, rhs: Self) -> Self {
Self {
data_items: self.data_items + rhs.data_items,
meta_items: self.meta_items + rhs.meta_items,
task_cache_items: self.task_cache_items + rhs.task_cache_items,
max_next_task_id: max(self.max_next_task_id, rhs.max_next_task_id),
}
}
}
/// Private methods used by [`BackingStorage`]. This trait is `pub` (because of the sealed-trait /// Private methods used by [`BackingStorage`]. This trait is `pub` (because of the sealed-trait
/// pattern), but should not be exported outside of the crate. /// pattern), but should not be exported outside of the crate.
/// ///
...@@ -91,7 +111,11 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -91,7 +111,11 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
fn next_free_task_id(&self) -> Result<TaskId>; fn next_free_task_id(&self) -> Result<TaskId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>; fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
fn save_snapshot<I>(&self, operations: Vec<Arc<AnyOperation>>, snapshots: Vec<I>) -> Result<()> fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
snapshots: Vec<I>,
) -> Result<SnapshotMeta>
where where
I: IntoIterator<Item = SnapshotItem> + Send + Sync; I: IntoIterator<Item = SnapshotItem> + Send + Sync;
/// Returns all task IDs that match the given task type (hash collision candidates). /// Returns all task IDs that match the given task type (hash collision candidates).
...@@ -160,7 +184,11 @@ where ...@@ -160,7 +184,11 @@ where
either::for_both!(self, this => this.uncompleted_operations()) either::for_both!(self, this => this.uncompleted_operations())
} }
fn save_snapshot<I>(&self, operations: Vec<Arc<AnyOperation>>, snapshots: Vec<I>) -> Result<()> fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
snapshots: Vec<I>,
) -> Result<SnapshotMeta>
where where
I: IntoIterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{ {
......
...@@ -19,7 +19,8 @@ use crate::{ ...@@ -19,7 +19,8 @@ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage}, backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backing_storage::{ backing_storage::{
BackingStorage, BackingStorageSealed, SnapshotItem, compute_task_type_hash_from_components, BackingStorage, BackingStorageSealed, SnapshotItem, SnapshotMeta,
compute_task_type_hash_from_components,
}, },
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
...@@ -224,7 +225,11 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -224,7 +225,11 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
get(&self.inner.database).context("Unable to read uncompleted operations from database") get(&self.inner.database).context("Unable to read uncompleted operations from database")
} }
fn save_snapshot<I>(&self, operations: Vec<Arc<AnyOperation>>, snapshots: Vec<I>) -> Result<()> fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
snapshots: Vec<I>,
) -> Result<SnapshotMeta>
where where
I: IntoIterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{ {
...@@ -233,9 +238,12 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -233,9 +238,12 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
{ {
let _span = tracing::trace_span!("update task data").entered(); let _span = tracing::trace_span!("update task data").entered();
let max_new_task_id = let snapshot_meta =
parallel::map_collect_owned::<_, _, Result<Vec<_>>>(snapshots, |shard: I| { parallel::map_collect_owned::<_, _, Result<Vec<_>>>(snapshots, |shard: I| {
let mut max_new_task_id = 0; let mut max_new_task_id = 0;
let mut data_items = 0;
let mut meta_items = 0;
let mut task_cache_items = 0;
for SnapshotItem { for SnapshotItem {
task_id, task_id,
meta, meta,
...@@ -251,6 +259,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -251,6 +259,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
WriteBuffer::Borrowed(key), WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(meta), WriteBuffer::SmallVec(meta),
)?; )?;
meta_items += 1;
} }
if let Some(data) = data { if let Some(data) = data {
batch.put( batch.put(
...@@ -258,6 +267,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -258,6 +267,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
WriteBuffer::Borrowed(key), WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(data), WriteBuffer::SmallVec(data),
)?; )?;
data_items += 1;
} }
// Write task cache entry inline if this is a new task // Write task cache entry inline if this is a new task
if let Some(task_type_hash) = task_type_hash { if let Some(task_type_hash) = task_type_hash {
...@@ -266,13 +276,19 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -266,13 +276,19 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
WriteBuffer::Borrowed(&task_type_hash), WriteBuffer::Borrowed(&task_type_hash),
WriteBuffer::Borrowed(key), WriteBuffer::Borrowed(key),
)?; )?;
task_cache_items += 1;
max_new_task_id = max_new_task_id.max(*task_id); max_new_task_id = max_new_task_id.max(*task_id);
} }
} }
Ok(max_new_task_id) Ok(SnapshotMeta {
data_items,
meta_items,
task_cache_items,
max_next_task_id: max_new_task_id,
})
})? })?
.into_iter() .into_iter()
.max() .reduce(|t1, t2| t1.merge(t2))
.unwrap_or_default(); .unwrap_or_default();
let span = tracing::trace_span!("flush task data").entered(); let span = tracing::trace_span!("flush task data").entered();
...@@ -287,14 +303,14 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -287,14 +303,14 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
)?; )?;
let mut next_task_id = get_next_free_task_id(&batch)?; let mut next_task_id = get_next_free_task_id(&batch)?;
next_task_id = next_task_id.max(max_new_task_id + 1); next_task_id = next_task_id.max(snapshot_meta.max_next_task_id + 1);
save_infra(&batch, next_task_id, operations)?; save_infra(&batch, next_task_id, operations)?;
{ {
let _span = tracing::trace_span!("commit").entered(); let _span = tracing::trace_span!("commit").entered();
batch.commit().context("Unable to commit operations")?; batch.commit().context("Unable to commit operations")?;
} }
Ok(()) Ok(snapshot_meta)
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment