Unverified Commit 7f97a3e7 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Refine dense task chunk metadata

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 0221ecdc
...@@ -9,7 +9,7 @@ use std::{ ...@@ -9,7 +9,7 @@ use std::{
time::Instant, time::Instant,
}; };
use criterion::{BenchmarkId, Criterion, Throughput}; use criterion::{BatchSize, BenchmarkId, Criterion, Throughput};
use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, lock_api::RawMutex as _}; use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, lock_api::RawMutex as _};
use turbo_tasks::{FxDashMap, TaskId}; use turbo_tasks::{FxDashMap, TaskId};
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
...@@ -218,6 +218,50 @@ fn dense_map(count: u32, stride: u32) -> TaskMap<Payload> { ...@@ -218,6 +218,50 @@ fn dense_map(count: u32, stride: u32) -> TaskMap<Payload> {
map map
} }
fn dense_remove(map: &TaskMap<Payload>, id: TaskId) -> Option<Payload> {
map.get(id).map(|task| task.take_and_vacate())
}
fn dense_remove_discard(map: &TaskMap<Payload>, id: TaskId) -> bool {
let Some(task) = map.get(id) else {
return false;
};
task.vacate();
true
}
fn scan_dense(map: &TaskMap<Payload>, use_bitmap: bool) -> u64 {
let mut sum = 0;
for chunk in map.chunks() {
if use_bitmap {
chunk.for_each_mut(|value| sum ^= value.value());
} else {
chunk.for_each_all_mut(|value| sum ^= value.value());
}
}
sum
}
fn scan_dense_parallel(
runtime: &tokio::runtime::Runtime,
map: &TaskMap<Payload>,
use_bitmap: bool,
) -> u64 {
let chunks = map.chunks();
runtime.block_on(async {
let sums: Vec<u64> = turbo_tasks::parallel::map_collect(&chunks, |chunk| {
let mut sum = 0;
if use_bitmap {
chunk.for_each_mut(|value| sum ^= value.value());
} else {
chunk.for_each_all_mut(|value| sum ^= value.value());
}
sum
});
sums.into_iter().fold(0, |sum, value| sum ^ value)
})
}
fn memory_delta<M>(build: impl FnOnce() -> M) -> (i128, usize) { fn memory_delta<M>(build: impl FnOnce() -> M) -> (i128, usize) {
TurboMalloc::collect(true); TurboMalloc::collect(true);
let before = TurboMalloc::allocation_counters(); let before = TurboMalloc::allocation_counters();
...@@ -281,7 +325,7 @@ fn report_reclamation(count: u32) { ...@@ -281,7 +325,7 @@ fn report_reclamation(count: u32) {
let live = TurboMalloc::allocation_counters(); let live = TurboMalloc::allocation_counters();
let loaded_before = dense.loaded_chunk_count(); let loaded_before = dense.loaded_chunk_count();
for raw in 1..=count { for raw in 1..=count {
assert!(dense.remove_discard(task_id(raw))); assert!(dense_remove_discard(&dense, task_id(raw)));
} }
let retired = dense.retire_empty_chunks(); let retired = dense.retire_empty_chunks();
TurboMalloc::collect(true); TurboMalloc::collect(true);
...@@ -423,7 +467,7 @@ pub fn resident_storage(c: &mut Criterion) { ...@@ -423,7 +467,7 @@ pub fn resident_storage(c: &mut Criterion) {
b.iter(|| { b.iter(|| {
next = next % LOOKUP_TASKS + 1; next = next % LOOKUP_TASKS + 1;
let id = task_id(next); let id = task_id(next);
black_box(dense.remove(id)); black_box(dense_remove(&dense, id));
let mut item = dense.get_or_insert(id); let mut item = dense.get_or_insert(id);
item.set(next); item.set(next);
black_box(item.value()); black_box(item.value());
...@@ -448,7 +492,7 @@ pub fn resident_storage(c: &mut Criterion) { ...@@ -448,7 +492,7 @@ pub fn resident_storage(c: &mut Criterion) {
b.iter(|| { b.iter(|| {
next = next % LOOKUP_TASKS + 1; next = next % LOOKUP_TASKS + 1;
let id = task_id(next); let id = task_id(next);
black_box(dense.remove_discard(id)); black_box(dense_remove_discard(&dense, id));
dense.get_or_insert(id).set(next); dense.get_or_insert(id).set(next);
}) })
}); });
...@@ -466,7 +510,7 @@ pub fn resident_storage(c: &mut Criterion) { ...@@ -466,7 +510,7 @@ pub fn resident_storage(c: &mut Criterion) {
let dense = dense.clone(); let dense = dense.clone();
scope.spawn(move || { scope.spawn(move || {
for _ in 0..per_thread { for _ in 0..per_thread {
black_box(dense.remove_discard(id)); black_box(dense_remove_discard(&dense, id));
dense.get_or_insert(id).set(*id); dense.get_or_insert(id).set(*id);
} }
}); });
...@@ -623,6 +667,10 @@ pub fn resident_storage(c: &mut Criterion) { ...@@ -623,6 +667,10 @@ pub fn resident_storage(c: &mut Criterion) {
// models incremental scans after the first eviction/restoration pass established sparsity. // models incremental scans after the first eviction/restoration pass established sparsity.
for chunk in dense.chunks() { for chunk in dense.chunks() {
for offset in chunk.probably_occupied_offsets() { for offset in chunk.probably_occupied_offsets() {
// TaskId zero is reserved; the all-true first bitmap contains it as a stale hint.
if !chunk.transient && chunk.base_id + offset == 0 {
continue;
}
drop(chunk.get(offset)); drop(chunk.get(offset));
} }
} }
...@@ -646,4 +694,39 @@ pub fn resident_storage(c: &mut Criterion) { ...@@ -646,4 +694,39 @@ pub fn resident_storage(c: &mut Criterion) {
}) })
}); });
sparse_iteration.finish(); sparse_iteration.finish();
let mut bitmap = c.benchmark_group("resident_storage_bitmap");
for (shape, count, stride) in [
("dense", 65_536, 1),
("half", 32_768, 2),
("ten_percent", 6_553, 10),
("one_percent", 655, 100),
] {
for (strategy, use_bitmap) in [("bitmap", true), ("all_slots", false)] {
bitmap.bench_function(format!("{shape}/first_serial/{strategy}"), |b| {
b.iter_batched(
|| dense_map(count, stride),
|map| black_box(scan_dense(&map, use_bitmap)),
BatchSize::SmallInput,
)
});
bitmap.bench_function(format!("{shape}/first_parallel/{strategy}"), |b| {
b.iter_batched(
|| dense_map(count, stride),
|map| black_box(scan_dense_parallel(&runtime, &map, use_bitmap)),
BatchSize::SmallInput,
)
});
let map = dense_map(count, stride);
black_box(scan_dense(&map, true));
bitmap.bench_function(format!("{shape}/subsequent_serial/{strategy}"), |b| {
b.iter(|| black_box(scan_dense(&map, use_bitmap)))
});
bitmap.bench_function(format!("{shape}/subsequent_parallel/{strategy}"), |b| {
b.iter(|| black_box(scan_dense_parallel(&runtime, &map, use_bitmap)))
});
}
}
bitmap.finish();
} }
...@@ -5,7 +5,7 @@ use std::{ ...@@ -5,7 +5,7 @@ use std::{
ops::{Deref, DerefMut}, ops::{Deref, DerefMut},
sync::{ sync::{
Arc, Arc,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering},
}, },
}; };
...@@ -16,7 +16,7 @@ use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, p ...@@ -16,7 +16,7 @@ use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, p
use crate::{ use crate::{
backend::{ backend::{
dense_task_map::{TaskMap, TaskMapGuard}, dense_task_map::{CHUNK_SIZE, TaskMap, TaskMapGuard},
storage_schema::{ storage_schema::{
DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability, DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability,
}, },
...@@ -223,7 +223,7 @@ impl Storage { ...@@ -223,7 +223,7 @@ impl Storage {
/// This is used after persisting a snapshot: _during_snapshot flags represent changes /// This is used after persisting a snapshot: _during_snapshot flags represent changes
/// that occurred concurrently and were not included in the persisted snapshot, so they /// that occurred concurrently and were not included in the persisted snapshot, so they
/// must be carried forward as `modified` for the next snapshot cycle. /// must be carried forward as `modified` for the next snapshot cycle.
fn promote_during_snapshot_flags(&self, task: &mut TaskStorage, modified_count: &AtomicU64) { fn promote_during_snapshot_flags(&self, task: &mut TaskStorage, modified_count: &AtomicU8) {
let already_modified = task.flags.any_modified(); let already_modified = task.flags.any_modified();
let mut promoted = false; let mut promoted = false;
if task.flags.meta_modified_during_snapshot() { if task.flags.meta_modified_during_snapshot() {
...@@ -237,7 +237,8 @@ impl Storage { ...@@ -237,7 +237,8 @@ impl Storage {
promoted = true; promoted = true;
} }
if !already_modified && promoted { if !already_modified && promoted {
modified_count.fetch_add(1, Ordering::Relaxed); let previous = modified_count.fetch_add(1, Ordering::Relaxed);
debug_assert!(previous < CHUNK_SIZE as u8);
} }
} }
...@@ -291,7 +292,7 @@ impl Storage { ...@@ -291,7 +292,7 @@ impl Storage {
let shards: Vec<Option<SnapshotShard<'l, P>>> = self.map.parallel_collect(|chunk| { let shards: Vec<Option<SnapshotShard<'l, P>>> = self.map.parallel_collect(|chunk| {
// Once snapshot mode is active, new writes use the during-snapshot flags and do not // Once snapshot mode is active, new writes use the during-snapshot flags and do not
// increment this counter. Each chunk can therefore be claimed independently. // increment this counter. Each chunk can therefore be claimed independently.
let modified_count = chunk.swap_modified_count().unwrap_or(0); let modified_count = chunk.take_modified_count().unwrap_or(0);
if modified_count == 0 && !drain_entries { if modified_count == 0 && !drain_entries {
return None; return None;
} }
...@@ -740,7 +741,8 @@ impl StorageWriteGuard<'_> { ...@@ -740,7 +741,8 @@ impl StorageWriteGuard<'_> {
// Not in snapshot mode and item is unmodified // Not in snapshot mode and item is unmodified
let bumped = !flags.any_modified(); let bumped = !flags.any_modified();
if bumped { if bumped {
self.inner.modified_count().fetch_add(1, Ordering::Relaxed); let previous = self.inner.modified_count().fetch_add(1, Ordering::Relaxed);
debug_assert!(previous < CHUNK_SIZE as u8);
} }
self.inner.flags.set_modified(category, true); self.inner.flags.set_modified(category, true);
TrackOutcome::Tracked { category, bumped } TrackOutcome::Tracked { category, bumped }
...@@ -811,7 +813,8 @@ impl StorageWriteGuard<'_> { ...@@ -811,7 +813,8 @@ impl StorageWriteGuard<'_> {
TrackOutcome::Tracked { category, bumped } => { TrackOutcome::Tracked { category, bumped } => {
self.inner.flags.set_modified(category, false); self.inner.flags.set_modified(category, false);
if bumped { if bumped {
self.inner.modified_count().fetch_sub(1, Ordering::Relaxed); let previous = self.inner.modified_count().fetch_sub(1, Ordering::Relaxed);
debug_assert!(previous > 0);
} }
} }
TrackOutcome::TrackedDuringSnapshot { TrackOutcome::TrackedDuringSnapshot {
...@@ -840,7 +843,8 @@ impl StorageWriteGuard<'_> { ...@@ -840,7 +843,8 @@ impl StorageWriteGuard<'_> {
"only a never-persisted (new_task) collected task may be discarded this way" "only a never-persisted (new_task) collected task may be discarded this way"
); );
if self.inner.flags.any_modified() { if self.inner.flags.any_modified() {
self.inner.modified_count().fetch_sub(1, Ordering::Relaxed); let previous = self.inner.modified_count().fetch_sub(1, Ordering::Relaxed);
debug_assert!(previous > 0);
} }
self.inner.flags.set_meta_modified(false); self.inner.flags.set_meta_modified(false);
self.inner.flags.set_data_modified(false); self.inner.flags.set_data_modified(false);
......
...@@ -1973,9 +1973,9 @@ mod tests { ...@@ -1973,9 +1973,9 @@ mod tests {
#[test] #[test]
fn const_empty_slots_have_independent_unlocked_mutexes() { fn const_new_slots_have_independent_unlocked_mutexes() {
let empty_task = const { TaskStorage::empty_slot() }; let empty_task = const { TaskStorage::new() };
let slots = [const { TaskStorage::empty_slot() }; 2]; let slots = [const { TaskStorage::new() }; 2];
assert!(!empty_task.is_occupied()); assert!(!empty_task.is_occupied());
assert!(!slots[0].is_occupied()); assert!(!slots[0].is_occupied());
assert!(!slots[1].is_occupied()); assert!(!slots[1].is_occupied());
...@@ -2059,7 +2059,7 @@ mod tests { ...@@ -2059,7 +2059,7 @@ mod tests {
); );
assert_eq!( assert_eq!(
size_of::<TaskChunk<TaskStorage>>(), size_of::<TaskChunk<TaskStorage>>(),
24 + CHUNK_SIZE / 8, 16 + CHUNK_SIZE / 8,
"chunk counters and bitmap size changed" "chunk counters and bitmap size changed"
); );
assert_eq!( assert_eq!(
......
...@@ -1435,8 +1435,8 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1435,8 +1435,8 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#[automatically_derived] #[automatically_derived]
impl TaskStorage { impl TaskStorage {
#[doc = "Constructs fresh task payload with unlocked, vacant slot metadata."] #[doc = "Constructs fresh task payload with unlocked, vacant slot metadata."]
pub fn new() -> Self { pub const fn new() -> Self {
Self::default() Self::empty_slot()
} }
#[doc = "Moves payload into a fresh unlocked value and leaves this locked slot vacant."] #[doc = "Moves payload into a fresh unlocked value and leaves this locked slot vacant."]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment