Unverified Commit d2781957 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Use intrusive locks for task storage

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent fed3b4be
...@@ -6,11 +6,12 @@ use criterion::{Criterion, criterion_group, criterion_main}; ...@@ -6,11 +6,12 @@ use criterion::{Criterion, criterion_group, criterion_main};
pub(crate) mod gc; pub(crate) mod gc;
pub(crate) mod overhead; pub(crate) mod overhead;
pub(crate) mod scope_stress; pub(crate) mod scope_stress;
pub(crate) mod storage_locking;
pub(crate) mod stress; pub(crate) mod stress;
criterion_group!( criterion_group!(
name = turbo_tasks_backend_stress; name = turbo_tasks_backend_stress;
config = Criterion::default(); config = Criterion::default();
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc, storage_locking::storage_locking
); );
criterion_main!(turbo_tasks_backend_stress); criterion_main!(turbo_tasks_backend_stress);
use std::{cell::UnsafeCell, hash::BuildHasher, hint::black_box, sync::Arc, thread};
use criterion::{BatchSize, Criterion, Throughput};
use dashmap::DashMap;
use parking_lot::{RawMutex, lock_api::RawMutex as RawMutexTrait};
use rustc_hash::FxBuildHasher;
const SHARDS: usize = 64;
const TASKS: u32 = 4096;
#[repr(C)]
struct Payload {
lock: RawMutex,
value: UnsafeCell<u64>,
padding: [u64; 14],
}
impl Payload {
fn new(value: u64) -> Self {
Self {
lock: <RawMutex as RawMutexTrait>::INIT,
value: UnsafeCell::new(value),
padding: [0; 14],
}
}
fn with_mut<R>(&self, f: impl FnOnce(&mut u64) -> R) -> R {
self.lock.lock();
let guard = PayloadGuard(self);
// SAFETY: `guard` owns this payload's lock for the closure's lifetime.
let result = f(unsafe { &mut *self.value.get() });
drop(guard);
result
}
}
// SAFETY: `value` is accessed only while `lock` is held.
unsafe impl Sync for Payload {}
struct PayloadGuard<'a>(&'a Payload);
impl Drop for PayloadGuard<'_> {
fn drop(&mut self) {
// SAFETY: This guard is created only after acquiring this payload's lock.
unsafe { self.0.lock.unlock() };
}
}
type Map = DashMap<u32, Box<Payload>, FxBuildHasher>;
fn empty_map() -> Map {
DashMap::with_capacity_and_hasher_and_shard_amount(TASKS as usize, FxBuildHasher, SHARDS)
}
fn populated_map() -> Map {
let map = empty_map();
for id in 1..=TASKS {
map.insert(id, Box::new(Payload::new(id as u64)));
}
map
}
fn same_shard_keys(map: &Map, count: usize) -> Vec<u32> {
let mut keys = Vec::with_capacity(count);
let target = map.determine_shard(map.hash_usize(&1));
for key in 1.. {
if map.determine_shard(map.hash_usize(&key)) == target {
keys.push(key);
if keys.len() == count {
return keys;
}
}
}
unreachable!()
}
pub fn storage_locking(criterion: &mut Criterion) {
let map = populated_map();
let mut hit = criterion.benchmark_group("storage_locking_hit");
hit.throughput(Throughput::Elements(1));
hit.bench_function("dashmap_shard_write", |b| {
b.iter(|| {
let mut task = map.get_mut(black_box(&1)).unwrap();
task.value = UnsafeCell::new(black_box(1));
})
});
hit.bench_function("shard_read_intrusive", |b| {
b.iter(|| {
let task = map.get(black_box(&1)).unwrap();
task.with_mut(|value| *value = black_box(1));
})
});
hit.finish();
let mut insertion = criterion.benchmark_group("storage_locking_insert_roundtrip");
insertion.throughput(Throughput::Elements(1));
insertion.bench_function("dashmap_shard_write", |b| {
let map = empty_map();
let mut key = 1_u32;
b.iter(|| {
let current = key;
key = key.wrapping_add(1);
let mut task = map
.entry(current)
.or_insert_with(|| Box::new(Payload::new(0)));
*task.value.get_mut() = black_box(current as u64);
drop(task);
map.remove(&current);
})
});
insertion.bench_function("downgrade_then_intrusive", |b| {
let map = empty_map();
let mut key = 1_u32;
b.iter(|| {
let current = key;
key = key.wrapping_add(1);
let task = map
.entry(current)
.or_insert_with(|| Box::new(Payload::new(0)))
.downgrade();
task.with_mut(|value| *value = black_box(current as u64));
drop(task);
map.remove(&current);
})
});
insertion.finish();
let mut contention = criterion.benchmark_group("storage_locking_same_shard_parallel");
const THREADS: usize = 4;
const OPS: usize = 1000;
contention.throughput(Throughput::Elements((THREADS * OPS) as u64));
let keys = same_shard_keys(&map, THREADS);
contention.bench_function("dashmap_shard_write", |b| {
b.iter(|| {
thread::scope(|scope| {
for &key in &keys {
let map = &map;
scope.spawn(move || {
for _ in 0..OPS {
let mut task = map.get_mut(&key).unwrap();
*task.value.get_mut() = black_box(key as u64);
}
});
}
});
})
});
contention.bench_function("shard_read_intrusive", |b| {
b.iter(|| {
thread::scope(|scope| {
for &key in &keys {
let map = &map;
scope.spawn(move || {
for _ in 0..OPS {
let task = map.get(&key).unwrap();
task.with_mut(|value| *value = black_box(key as u64));
}
});
}
});
})
});
contention.finish();
let pair_keys = same_shard_keys(&map, 2);
let hash1 = map.hasher().hash_one(pair_keys[0]);
let hash2 = map.hasher().hash_one(pair_keys[1]);
let shard_index = map.determine_shard(hash1 as usize);
let mut pair = criterion.benchmark_group("storage_locking_same_shard_pair");
pair.throughput(Throughput::Elements(2));
pair.bench_function("dashmap_shard_write", |b| {
b.iter(|| {
let mut shard = map.shards()[shard_index].write();
let task1 = shard
.find_mut(hash1, |(key, _)| *key == pair_keys[0])
.unwrap();
*task1.1.value.get_mut() = black_box(1);
let task2 = shard
.find_mut(hash2, |(key, _)| *key == pair_keys[1])
.unwrap();
*task2.1.value.get_mut() = black_box(2);
})
});
pair.bench_function("shared_shard_read_intrusive", |b| {
b.iter(|| {
let shard = map.shards()[shard_index].read();
let task1 = shard.find(hash1, |(key, _)| *key == pair_keys[0]).unwrap();
let task2 = shard.find(hash2, |(key, _)| *key == pair_keys[1]).unwrap();
task1.1.with_mut(|value| *value = black_box(1));
task2.1.with_mut(|value| *value = black_box(2));
})
});
pair.finish();
let mut iteration = criterion.benchmark_group("storage_locking_iteration");
iteration.throughput(Throughput::Elements(TASKS as u64));
iteration.bench_function("shard_write", |b| {
b.iter(|| {
let mut sum = 0_u64;
for shard in map.shards() {
let shard = shard.write();
for (_, task) in shard.iter() {
// SAFETY: The shard write lock excludes every map accessor in this baseline.
sum = sum.wrapping_add(unsafe { *task.value.get() });
}
}
black_box(sum)
})
});
iteration.bench_function("shard_read_intrusive", |b| {
b.iter(|| {
let mut sum = 0_u64;
for shard in map.shards() {
let shard = shard.read();
for (_, task) in shard.iter() {
sum = sum.wrapping_add(task.with_mut(|value| *value));
}
}
black_box(sum)
})
});
iteration.finish();
let mut removal = criterion.benchmark_group("storage_locking_structural_remove");
removal.throughput(Throughput::Elements(TASKS as u64));
removal.bench_function("dashmap_remove", |b| {
b.iter_batched(
|| Arc::new(populated_map()),
|map| {
for key in 1..=TASKS {
black_box(map.remove(&key));
}
},
BatchSize::LargeInput,
)
});
removal.finish();
}
...@@ -18,14 +18,15 @@ ...@@ -18,14 +18,15 @@
//! - `meta` - Rarely changed metadata (output, aggregation, flags) //! - `meta` - Rarely changed metadata (output, aggregation, flags)
//! - `transient` - Not serialized, only exists in memory //! - `transient` - Not serialized, only exists in memory
use std::{ use std::{
fmt,
hash::{BuildHasherDefault, Hash}, hash::{BuildHasherDefault, Hash},
sync::Arc, sync::Arc,
}; };
use parking_lot::Mutex; use parking_lot::{Mutex, RawMutex, lock_api::RawMutex as RawMutexTrait};
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use turbo_tasks::{ use turbo_tasks::{
CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId, CellId, SharedReference, ShrinkToFit, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
backend::{CachedTaskTypeArc, CellHash, TransientTaskType}, backend::{CachedTaskTypeArc, CellHash, TransientTaskType},
event::Event, event::Event,
task_storage, task_storage,
...@@ -47,6 +48,61 @@ type AutoSet<K, const I: usize> = auto_hash_map::AutoSet<K, BuildHasherDefault<F ...@@ -47,6 +48,61 @@ type AutoSet<K, const I: usize> = auto_hash_map::AutoSet<K, BuildHasherDefault<F
/// See [`AutoSet`] for the meaning of `I`. /// See [`AutoSet`] for the meaning of `I`.
type AutoMap<K, V, const I: usize> = auto_hash_map::AutoMap<K, V, BuildHasherDefault<FxHasher>, I>; type AutoMap<K, V, const I: usize> = auto_hash_map::AutoMap<K, V, BuildHasherDefault<FxHasher>, I>;
/// One-byte parking_lot lock embedded in each task's storage.
///
/// The lock is synchronization metadata: snapshots create a fresh lock rather than copying its
/// state. This wrapper supplies the generated storage type's trait requirements.
pub(crate) struct IntrusiveTaskLock(RawMutex);
impl IntrusiveTaskLock {
const fn new() -> Self {
Self(<RawMutex as RawMutexTrait>::INIT)
}
fn lock(&self) {
self.0.lock();
}
/// # Safety
///
/// The current thread must own this lock and perform no protected access after this call.
unsafe fn unlock(&self) {
// SAFETY: Forwarded from the caller.
unsafe { self.0.unlock() };
}
#[cfg(test)]
fn is_locked(&self) -> bool {
self.0.is_locked()
}
}
struct IntrusiveTaskLockGuard<'a>(&'a IntrusiveTaskLock);
impl Drop for IntrusiveTaskLockGuard<'_> {
fn drop(&mut self) {
// SAFETY: This lexical guard is created only after acquiring this lock and drops before
// the protected reference can escape.
unsafe { self.0.unlock() };
}
}
impl Default for IntrusiveTaskLock {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for IntrusiveTaskLock {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("IntrusiveTaskLock").finish_non_exhaustive()
}
}
impl ShrinkToFit for IntrusiveTaskLock {
fn shrink_to_fit(&mut self) {}
}
/// The complete task storage schema. /// The complete task storage schema.
/// ///
/// This struct defines all storage fields for a task. The `#[task_storage]` macro /// This struct defines all storage fields for a task. The `#[task_storage]` macro
...@@ -550,6 +606,33 @@ pub enum KeyEvictability { ...@@ -550,6 +606,33 @@ pub enum KeyEvictability {
} }
impl TaskStorage { impl TaskStorage {
/// Lock this task's storage while a resident-map guard keeps its allocation alive.
pub(crate) fn lock(&self) {
self.lock.lock();
}
/// Unlock this task's storage.
///
/// # Safety
///
/// The current thread must own the task lock, hold the resident-map guard that keeps this
/// storage alive, and perform no protected access after this call.
pub(crate) unsafe fn unlock(&self) {
// SAFETY: Forwarded from the caller.
unsafe { self.lock.unlock() };
}
#[cfg(test)]
pub(crate) fn is_locked(&self) -> bool {
self.lock.is_locked()
}
pub(crate) fn with_lock<R>(&self, f: impl FnOnce(&Self) -> R) -> R {
self.lock();
let _guard = IntrusiveTaskLockGuard(&self.lock);
f(self)
}
/// Determine the evictability level of this task based on its flags. /// Determine the evictability level of this task based on its flags.
/// ///
/// This checks only the flags on the TaskStorage itself. The caller /// This checks only the flags on the TaskStorage itself. The caller
...@@ -1841,6 +1924,18 @@ mod tests { ...@@ -1841,6 +1924,18 @@ mod tests {
// `hanging_detection` adds an `Arc<dyn Fn() -> String>` description to every `Event`, which // `hanging_detection` adds an `Arc<dyn Fn() -> String>` description to every `Event`, which
// grows `LazyField` past the size asserted here. The feature is diagnostic-only, so the sizes // grows `LazyField` past the size asserted here. The feature is diagnostic-only, so the sizes
// are not meaningful under it. // are not meaningful under it.
#[test]
fn snapshot_clone_has_an_independent_unlocked_task_lock() {
let storage = TaskStorage::new();
storage.lock();
assert!(storage.is_locked());
let snapshot = storage.clone_snapshot();
assert!(storage.is_locked());
assert!(!snapshot.is_locked());
// SAFETY: This test acquired the source lock above and accesses it no further.
unsafe { storage.unlock() };
}
#[test] #[test]
#[cfg(all(target_pointer_width = "64", not(feature = "hanging_detection")))] #[cfg(all(target_pointer_width = "64", not(feature = "hanging_detection")))]
fn test_schema_size() { fn test_schema_size() {
......
...@@ -1406,6 +1406,8 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1406,6 +1406,8 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#(#field_defs,)* #(#field_defs,)*
#flags_field #flags_field
#lazy_field #lazy_field
#[doc = "Per-task lock used with a shared resident-map guard."]
pub(crate) lock: IntrusiveTaskLock,
} }
#[automatically_derived] #[automatically_derived]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment