Unverified Commit a07cba75 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Add trace metadata about the persist size (#93339)

Record simple counters for how many items were persisted.

This should be cheap enough to always do
parent a9323da2
......@@ -68,7 +68,7 @@ use crate::{
storage::Storage,
storage_schema::{TaskStorage, TaskStorageAccessors},
},
backing_storage::{BackingStorage, SnapshotItem, compute_task_type_hash},
backing_storage::{BackingStorage, SnapshotItem, SnapshotMeta, compute_task_type_hash},
data::{
ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressState, InProgressStateInner, OutputValue, TransientTask,
......@@ -1233,13 +1233,32 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
}
let persist_start = Instant::now();
let _span = tracing::info_span!(parent: parent_span, "persist", reason = reason).entered();
let span = tracing::info_span!(
parent: parent_span,
"persist",
reason = reason,
data_items= tracing::field::Empty,
meta_items= tracing::field::Empty,
task_cache_items= tracing::field::Empty,
next_task_id= tracing::field::Empty,)
.entered();
{
// Tasks were already consumed by take_snapshot, so a future snapshot
// would not re-persist them — returning an error signals to the caller
// that further persist attempts would corrupt the task graph in storage.
self.backing_storage
let SnapshotMeta {
task_cache_items,
data_items,
meta_items,
max_next_task_id,
} = self
.backing_storage
.save_snapshot(suspended_operations, task_snapshots)?;
span.record("data_items", data_items);
span.record("meta_items", meta_items);
span.record("task_cache_items", task_cache_items);
span.record("next_task_id", max_next_task_id);
#[cfg(feature = "print_cache_item_size")]
{
let mut task_cache_stats = task_cache_stats
......
use std::sync::Arc;
use std::{cmp::max, sync::Arc};
use anyhow::Result;
use either::Either;
......@@ -82,6 +82,26 @@ pub trait BackingStorage: BackingStorageSealed {
fn invalidate(&self, reason_code: &str) -> Result<()>;
}
#[derive(Copy, Clone, Debug, Default)]
pub struct SnapshotMeta {
pub data_items: usize,
pub meta_items: usize,
pub task_cache_items: usize,
pub max_next_task_id: u32,
}
impl SnapshotMeta {
/// Merge two snapshots, summing the counts and `max`'ing the task id
pub fn merge(&self, rhs: Self) -> Self {
Self {
data_items: self.data_items + rhs.data_items,
meta_items: self.meta_items + rhs.meta_items,
task_cache_items: self.task_cache_items + rhs.task_cache_items,
max_next_task_id: max(self.max_next_task_id, rhs.max_next_task_id),
}
}
}
/// Private methods used by [`BackingStorage`]. This trait is `pub` (because of the sealed-trait
/// pattern), but should not be exported outside of the crate.
///
......@@ -91,7 +111,11 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
fn next_free_task_id(&self) -> Result<TaskId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
fn save_snapshot<I>(&self, operations: Vec<Arc<AnyOperation>>, snapshots: Vec<I>) -> Result<()>
fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
snapshots: Vec<I>,
) -> Result<SnapshotMeta>
where
I: IntoIterator<Item = SnapshotItem> + Send + Sync;
/// Returns all task IDs that match the given task type (hash collision candidates).
......@@ -160,7 +184,11 @@ where
either::for_both!(self, this => this.uncompleted_operations())
}
fn save_snapshot<I>(&self, operations: Vec<Arc<AnyOperation>>, snapshots: Vec<I>) -> Result<()>
fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
snapshots: Vec<I>,
) -> Result<SnapshotMeta>
where
I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{
......
......@@ -19,7 +19,8 @@ use crate::{
GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backing_storage::{
BackingStorage, BackingStorageSealed, SnapshotItem, compute_task_type_hash_from_components,
BackingStorage, BackingStorageSealed, SnapshotItem, SnapshotMeta,
compute_task_type_hash_from_components,
},
database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
......@@ -224,7 +225,11 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
get(&self.inner.database).context("Unable to read uncompleted operations from database")
}
fn save_snapshot<I>(&self, operations: Vec<Arc<AnyOperation>>, snapshots: Vec<I>) -> Result<()>
fn save_snapshot<I>(
&self,
operations: Vec<Arc<AnyOperation>>,
snapshots: Vec<I>,
) -> Result<SnapshotMeta>
where
I: IntoIterator<Item = SnapshotItem> + Send + Sync,
{
......@@ -233,9 +238,12 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
{
let _span = tracing::trace_span!("update task data").entered();
let max_new_task_id =
let snapshot_meta =
parallel::map_collect_owned::<_, _, Result<Vec<_>>>(snapshots, |shard: I| {
let mut max_new_task_id = 0;
let mut data_items = 0;
let mut meta_items = 0;
let mut task_cache_items = 0;
for SnapshotItem {
task_id,
meta,
......@@ -251,6 +259,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(meta),
)?;
meta_items += 1;
}
if let Some(data) = data {
batch.put(
......@@ -258,6 +267,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
WriteBuffer::Borrowed(key),
WriteBuffer::SmallVec(data),
)?;
data_items += 1;
}
// Write task cache entry inline if this is a new task
if let Some(task_type_hash) = task_type_hash {
......@@ -266,13 +276,19 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
WriteBuffer::Borrowed(&task_type_hash),
WriteBuffer::Borrowed(key),
)?;
task_cache_items += 1;
max_new_task_id = max_new_task_id.max(*task_id);
}
}
Ok(max_new_task_id)
Ok(SnapshotMeta {
data_items,
meta_items,
task_cache_items,
max_next_task_id: max_new_task_id,
})
})?
.into_iter()
.max()
.reduce(|t1, t2| t1.merge(t2))
.unwrap_or_default();
let span = tracing::trace_span!("flush task data").entered();
......@@ -287,14 +303,14 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
)?;
let mut next_task_id = get_next_free_task_id(&batch)?;
next_task_id = next_task_id.max(max_new_task_id + 1);
next_task_id = next_task_id.max(snapshot_meta.max_next_task_id + 1);
save_infra(&batch, next_task_id, operations)?;
{
let _span = tracing::trace_span!("commit").entered();
batch.commit().context("Unable to commit operations")?;
}
Ok(())
Ok(snapshot_meta)
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment