Unverified Commit 6728df44 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Address intrusive task lock feedback

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent d2781957
use std::{cell::UnsafeCell, hash::BuildHasher, hint::black_box, sync::Arc, thread};
use criterion::{BatchSize, Criterion, Throughput};
use crossbeam_utils::CachePadded;
use dashmap::DashMap;
use parking_lot::{RawMutex, lock_api::RawMutex as RawMutexTrait};
use rustc_hash::FxBuildHasher;
......@@ -8,11 +9,9 @@ use rustc_hash::FxBuildHasher;
const SHARDS: usize = 64;
const TASKS: u32 = 4096;
#[repr(C)]
struct Payload {
lock: RawMutex,
value: UnsafeCell<u64>,
padding: [u64; 14],
}
impl Payload {
......@@ -20,7 +19,6 @@ impl Payload {
Self {
lock: <RawMutex as RawMutexTrait>::INIT,
value: UnsafeCell::new(value),
padding: [0; 14],
}
}
......@@ -46,7 +44,8 @@ impl Drop for PayloadGuard<'_> {
}
}
type Map = DashMap<u32, Box<Payload>, FxBuildHasher>;
// Keep independently locked tasks on separate cache lines without hand-written layout padding.
type Map = DashMap<u32, Box<CachePadded<Payload>>, FxBuildHasher>;
fn empty_map() -> Map {
DashMap::with_capacity_and_hasher_and_shard_amount(TASKS as usize, FxBuildHasher, SHARDS)
......@@ -55,7 +54,7 @@ fn empty_map() -> Map {
fn populated_map() -> Map {
let map = empty_map();
for id in 1..=TASKS {
map.insert(id, Box::new(Payload::new(id as u64)));
map.insert(id, Box::new(CachePadded::new(Payload::new(id as u64))));
}
map
}
......@@ -102,7 +101,7 @@ pub fn storage_locking(criterion: &mut Criterion) {
key = key.wrapping_add(1);
let mut task = map
.entry(current)
.or_insert_with(|| Box::new(Payload::new(0)));
.or_insert_with(|| Box::new(CachePadded::new(Payload::new(0))));
*task.value.get_mut() = black_box(current as u64);
drop(task);
map.remove(&current);
......@@ -116,7 +115,7 @@ pub fn storage_locking(criterion: &mut Criterion) {
key = key.wrapping_add(1);
let task = map
.entry(current)
.or_insert_with(|| Box::new(Payload::new(0)))
.or_insert_with(|| Box::new(CachePadded::new(Payload::new(0))))
.downgrade();
task.with_mut(|value| *value = black_box(current as u64));
drop(task);
......
......@@ -307,7 +307,8 @@ impl TurboTasksBackend {
}
pub fn new(mut options: BackendOptions, backing_storage: TurboBackingStorage) -> Self {
let shard_amount = compute_shard_amount(options.num_workers, options.small_preallocation);
let resident_shard_amount =
compute_shard_amount(options.num_workers, options.small_preallocation);
if !options.dependency_tracking {
options.active_tracking = false;
}
......@@ -343,7 +344,7 @@ impl TurboTasksBackend {
TaskId::try_from(TRANSIENT_TASK_BIT).unwrap(),
TaskId::MAX,
),
storage: Storage::new(shard_amount, small_preallocation),
storage: Storage::new(resident_shard_amount, small_preallocation),
snapshot_coord: SnapshotCoordinator::new(),
snapshot_in_progress: Mutex::new(()),
stopping: RwLock::new(false),
......
......@@ -19,7 +19,8 @@ use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, p
use crate::{
backend::storage_schema::{
DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability,
DropPartialOutcome, KeyEvictability, TaskSlot, TaskStorage, UnevictableReason,
ValueEvictability,
},
backing_storage::SnapshotItem,
database::key_value_database::KeySpace,
......@@ -175,9 +176,6 @@ pub struct Storage {
///
/// Indexed by `map.determine_shard(map.hash_usize(&key))` and guaranteed by construction so
/// that `shard_modified_counts.len()==map.shards().len()`
///
/// Should only be modified while holding either the task's intrusive lock and shard read
/// guard, or the corresponding DashMap shard write lock.
shard_modified_counts: Box<[CachePadded<AtomicU64>]>,
/// Stores snapshots of task state for tasks accessed during snapshot mode.
/// - `Some(snapshot)`: Task was modified before snapshot mode and accessed again during it.
......@@ -209,7 +207,7 @@ pub struct Storage {
/// while `SnapshotShardIter::next` holds the map shard write lock; both then take a
/// `snapshots` shard lock. `end_snapshot` must lock in the same order — see the shard-zipping
/// pattern there.
map: FxDashMap<TaskId, Box<TaskStorage>>,
map: FxDashMap<TaskId, Box<TaskSlot>>,
/// A shared event notified whenever any task finishes restoring (successfully or not).
///
/// Threads waiting for another thread's in-progress restore subscribe to this event,
......@@ -358,6 +356,7 @@ impl Storage {
let work = if drain_entries {
let mut shard_guard = shard.write();
shard_guard.retain(|(key, task)| {
let task = task.get_mut_exclusive();
let modified_task = task.flags.any_modified();
if modified_task {
debug_assert!(
......@@ -490,7 +489,7 @@ impl Storage {
// write lock and would also obscure the pairing.
let hash = self.map.hasher().hash_one(key);
if let Some((_, task)) = map_guard.find_mut(hash, |(k, _)| *k == key) {
self.promote_during_snapshot_flags(task, shard_idx);
self.promote_during_snapshot_flags(task.get_mut_exclusive(), shard_idx);
}
}
// If we are saving a non-trivial amount of memory just clear it out.
......@@ -516,7 +515,7 @@ impl Storage {
let inner = match self.map.entry(key) {
dashmap::mapref::entry::Entry::Occupied(entry) => entry.into_ref(),
dashmap::mapref::entry::Entry::Vacant(entry) => {
entry.insert(Box::new(TaskStorage::new()))
entry.insert(Box::new(TaskSlot::new()))
}
};
StorageWriteGuard::new(self, inner.downgrade().into())
......@@ -603,9 +602,13 @@ impl Storage {
};
// `get_disjoint` obtains all shard guards in shard-index order (sharing one guard for a
// same-shard pair) before task locks are taken here in TaskId order.
let (lower, upper) = get_disjoint(&self.map, lower_key, upper_key, || {
Box::new(TaskStorage::new())
});
let (lower, upper) =
get_disjoint(
&self.map,
lower_key,
upper_key,
|| Box::new(TaskSlot::new()),
);
let lower = StorageWriteGuard::new(self, lower);
let upper = StorageWriteGuard::new(self, upper);
if reverse {
......@@ -679,8 +682,8 @@ impl Storage {
}
};
shard.retain(|(task_id, task)| {
// Transient tasks can not be evicted at all, unless they are fully
// delete by the GC.
let task = task.get_mut_exclusive();
// Transient tasks cannot be evicted unless the GC has deleted them.
if task_id.is_transient() && !task.flags.deleted() {
evicted.unevictable_reasons[UnevictableReason::Transient.index()] += 1;
return true;
......@@ -823,24 +826,26 @@ impl DerefMut for TaskEntryGuard<'_> {
pub struct StorageWriteGuard<'a> {
storage: &'a Storage,
inner: Ref<'a, TaskId, Box<TaskStorage>>,
task: *mut TaskStorage,
inner: Ref<'a, TaskId, Box<TaskSlot>>,
// Match parking_lot's default guard behavior: task guards must not cross `.await`.
_not_send: PhantomData<*const ()>,
}
impl<'a> StorageWriteGuard<'a> {
fn new(storage: &'a Storage, inner: Ref<'a, TaskId, Box<TaskStorage>>) -> Self {
let task = inner.value().as_ref() as *const TaskStorage as *mut TaskStorage;
// SAFETY: The DashMap read guard keeps the boxed task alive at this stable address.
unsafe { &*task }.lock();
fn new(storage: &'a Storage, inner: Ref<'a, TaskId, Box<TaskSlot>>) -> Self {
// The map guard keeps the opaque slot alive while its raw lock is acquired.
inner.value().lock();
Self {
storage,
inner,
task,
_not_send: PhantomData,
}
}
fn task(&self) -> &TaskStorage {
// SAFETY: `new` acquired this slot's lock, and `inner` keeps it alive.
unsafe { self.inner.value().get() }
}
}
impl StorageWriteGuard<'_> {
......@@ -875,7 +880,7 @@ impl StorageWriteGuard<'_> {
"track_modification called on transient task {:?}",
self.inner.key()
);
let flags = &self.inner.flags;
let flags = &self.flags;
if flags.is_modified_during_snapshot(category) {
// We can early return since `end_snapshot` is responsible for reconciling.
return TrackOutcome::NoChange;
......@@ -921,7 +926,7 @@ impl StorageWriteGuard<'_> {
if inserted_snapshot {
// Snapshot all non-transient fields, carrying the modified bits into
// the copy so the iterator knows which categories to persist.
let mut snapshot = self.inner.clone_snapshot();
let mut snapshot = self.clone_snapshot();
snapshot.flags.set_data_modified(flags.data_modified());
snapshot.flags.set_meta_modified(flags.meta_modified());
snapshot.flags.set_new_task(flags.new_task());
......@@ -980,10 +985,10 @@ impl StorageWriteGuard<'_> {
"discard_modifications_for_gc_new_task must run before the snapshot starts"
);
debug_assert!(
self.inner.flags.new_task(),
self.flags.new_task(),
"only a never-persisted (new_task) collected task may be discarded this way"
);
if self.inner.flags.any_modified() {
if self.flags.any_modified() {
let shard_idx = self.storage.shard_index(self.inner.key());
self.storage.shard_modified_counts[shard_idx].fetch_sub(1, Ordering::Relaxed);
}
......@@ -997,15 +1002,14 @@ impl Deref for StorageWriteGuard<'_> {
type Target = TaskStorage;
fn deref(&self) -> &Self::Target {
// SAFETY: `new` locked this task, and `inner` keeps its Box alive until after `Drop`.
unsafe { &*self.task }
self.task()
}
}
impl DerefMut for StorageWriteGuard<'_> {
fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY: The task's intrusive lock gives this guard exclusive payload access.
unsafe { &mut *self.task }
// SAFETY: This guard owns the slot's task lock, and `inner` keeps it alive.
unsafe { &mut *self.inner.value().as_ptr() }
}
}
......@@ -1014,7 +1018,7 @@ impl Drop for StorageWriteGuard<'_> {
// SAFETY: This guard acquired the lock in `new`, still owns it, and performs no task
// access after unlocking. Rust drops `inner` only after this method returns, so the Box
// remains alive through the unlock.
unsafe { (&*self.task).unlock() };
unsafe { self.inner.value().unlock() };
}
}
......@@ -1106,7 +1110,7 @@ enum ShardWork {
/// (modified-only) shard table out of the map. The iterator owns that table and drains it
/// directly, freeing each task box as it is serialized. No second map lookup, no flag
/// bookkeeping (the whole map is discarded right after this snapshot).
Drain(hash_table::IntoIter<(TaskId, Box<TaskStorage>)>),
Drain(hash_table::IntoIter<(TaskId, Box<TaskSlot>)>),
}
pub struct SnapshotShard<'l, P> {
......@@ -1188,12 +1192,13 @@ where
}
ShardWork::Drain(entries) => {
// Shutdown only: the scan already moved this shard's modified entries out of the
// map, so we own each `Box<TaskStorage>` here. Serialize from a borrow of the owned
// box and let it drop at the end of this branch — freeing the task's memory as it
// map, so we own each `Box<TaskSlot>` here. Move out the storage, serialize it, and
// let it drop at the end of this branch — freeing the task's memory as it
// is persisted rather than after the whole batch is written. We skip the flag
// bookkeeping the normal path does, since the entire map is discarded right after
// this snapshot.
let (task_id, inner) = entries.next()?;
let inner = (*inner).into_inner();
Some(serialize_task(task_id, &inner))
// we don't need to update any bits because everything is getting dropped.
}
......
......@@ -18,6 +18,7 @@
//! - `meta` - Rarely changed metadata (output, aggregation, flags)
//! - `transient` - Not serialized, only exists in memory
use std::{
cell::UnsafeCell,
fmt,
hash::{BuildHasherDefault, Hash},
sync::Arc,
......@@ -26,7 +27,7 @@ use std::{
use parking_lot::{Mutex, RawMutex, lock_api::RawMutex as RawMutexTrait};
use rustc_hash::FxHasher;
use turbo_tasks::{
CellId, SharedReference, ShrinkToFit, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
backend::{CachedTaskTypeArc, CellHash, TransientTaskType},
event::Event,
task_storage,
......@@ -77,16 +78,6 @@ impl IntrusiveTaskLock {
}
}
struct IntrusiveTaskLockGuard<'a>(&'a IntrusiveTaskLock);
impl Drop for IntrusiveTaskLockGuard<'_> {
fn drop(&mut self) {
// SAFETY: This lexical guard is created only after acquiring this lock and drops before
// the protected reference can escape.
unsafe { self.0.unlock() };
}
}
impl Default for IntrusiveTaskLock {
fn default() -> Self {
Self::new()
......@@ -99,10 +90,82 @@ impl fmt::Debug for IntrusiveTaskLock {
}
}
impl ShrinkToFit for IntrusiveTaskLock {
fn shrink_to_fit(&mut self) {}
/// Opaque interior-mutable task slot used when a resident map guard provides lifetime but not
/// payload exclusion.
///
/// # Unsafe access protocol
///
/// - An unlocked slot never exposes a `TaskStorage` reference.
/// - `lock` acquires the embedded task lock through the raw slot pointer before references exist.
/// - Only a lock-owning storage guard may call `get` or materialize a mutable reference from
/// `as_ptr`.
/// - A structural map write guard may call `get_mut_exclusive` because it excludes all map readers.
/// - The task lock is released before the map guard that keeps this allocation alive.
#[repr(transparent)]
pub(crate) struct TaskSlot(UnsafeCell<TaskStorage>);
impl TaskSlot {
pub(crate) fn new() -> Self {
Self(UnsafeCell::new(TaskStorage::new()))
}
pub(crate) fn as_ptr(&self) -> *mut TaskStorage {
self.0.get()
}
pub(crate) fn lock(&self) {
// SAFETY: Project only the embedded lock from the opaque cell. No TaskStorage reference is
// materialized before exclusion is acquired.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).lock() };
}
/// # Safety
/// The caller must own this slot's task lock and perform no protected access after unlocking.
pub(crate) unsafe fn unlock(&self) {
// SAFETY: Forwarded from the caller; projection does not materialize TaskStorage.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).unlock() };
}
/// # Safety
/// The caller must own this slot's task lock.
pub(crate) unsafe fn get(&self) -> &TaskStorage {
// SAFETY: Forwarded from the caller.
unsafe { &*self.as_ptr() }
}
pub(crate) fn get_mut_exclusive(&mut self) -> &mut TaskStorage {
self.0.get_mut()
}
pub(crate) fn into_inner(self) -> TaskStorage {
self.0.into_inner()
}
#[cfg(test)]
pub(crate) fn is_locked(&self) -> bool {
// SAFETY: Reading the raw lock state does not expose the task payload.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).is_locked() }
}
pub(crate) fn with_lock<R>(&self, f: impl FnOnce(&TaskStorage) -> R) -> R {
self.lock();
struct Unlock<'a>(&'a TaskSlot);
impl Drop for Unlock<'_> {
fn drop(&mut self) {
// SAFETY: `with_lock` acquired this lock and the closure has returned/unwound.
unsafe { self.0.unlock() };
}
}
let _unlock = Unlock(self);
// SAFETY: The lock-owning lexical guard remains alive through the closure.
f(unsafe { self.get() })
}
}
// SAFETY: all shared payload access is governed by the protocol above. Structural mutation uses
// `&mut TaskSlot`, while ordinary mutation owns the embedded task lock.
unsafe impl Sync for TaskSlot {}
/// The complete task storage schema.
///
/// This struct defines all storage fields for a task. The `#[task_storage]` macro
......@@ -606,33 +669,6 @@ pub enum KeyEvictability {
}
impl TaskStorage {
/// Lock this task's storage while a resident-map guard keeps its allocation alive.
pub(crate) fn lock(&self) {
self.lock.lock();
}
/// Unlock this task's storage.
///
/// # Safety
///
/// The current thread must own the task lock, hold the resident-map guard that keeps this
/// storage alive, and perform no protected access after this call.
pub(crate) unsafe fn unlock(&self) {
// SAFETY: Forwarded from the caller.
unsafe { self.lock.unlock() };
}
#[cfg(test)]
pub(crate) fn is_locked(&self) -> bool {
self.lock.is_locked()
}
pub(crate) fn with_lock<R>(&self, f: impl FnOnce(&Self) -> R) -> R {
self.lock();
let _guard = IntrusiveTaskLockGuard(&self.lock);
f(self)
}
/// Determine the evictability level of this task based on its flags.
///
/// This checks only the flags on the TaskStorage itself. The caller
......@@ -1926,14 +1962,15 @@ mod tests {
// are not meaningful under it.
#[test]
fn snapshot_clone_has_an_independent_unlocked_task_lock() {
let storage = TaskStorage::new();
storage.lock();
assert!(storage.is_locked());
let snapshot = storage.clone_snapshot();
assert!(storage.is_locked());
assert!(!snapshot.is_locked());
let slot = TaskSlot::new();
slot.lock();
assert!(slot.is_locked());
// SAFETY: This test holds the slot lock until after cloning.
let snapshot = unsafe { slot.get() }.clone_snapshot();
assert!(slot.is_locked());
assert!(!snapshot.lock.is_locked());
// SAFETY: This test acquired the source lock above and accesses it no further.
unsafe { storage.unlock() };
unsafe { slot.unlock() };
}
#[test]
......@@ -1944,6 +1981,11 @@ mod tests {
128,
"TaskStorage size changed! Update this test."
);
assert_eq!(
size_of::<TaskSlot>(),
size_of::<TaskStorage>(),
"the transparent resident slot must add no per-task size"
);
// `LazyField` is 40 B = 32 B largest payload + 8 B discriminant.
assert_eq!(
size_of::<LazyField>(),
......
......@@ -25,12 +25,12 @@ pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: boo
// for some constant `k` the probability of at least one collision for large `N` can be
// approximated as: P = 1 - exp(-N^2 / (2*S)) = 1 - exp(-1/(2*k))
//
// For `k = 16` this results in a collision probability of about 3%.
// For `k = 1` this results in a collision probability of about 39%.
// For `k = 2` this results in a collision probability of about 22%. Fine-grained task locks
// keep a shard collision from serializing independent task mutations, so the lower shard count
// saves memory without recreating the old shard-write-lock bottleneck.
//
// We clamp the number of shards to 1 << 16 to avoid excessive memory usage in case of a very
// high number of worker threads. This case is hit with more than 64 worker threads for `k =
// 16` and more than 256 worker threads for `k = 1`.
// high number of worker threads. With `k = 2`, this case is hit above roughly 181 workers.
if small_preallocation {
// We also clamp the maximum number of workers to 256 so all following multiplications can't
......@@ -38,9 +38,8 @@ pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: boo
let num_workers = num_workers.min(256);
(num_workers * num_workers).next_power_of_two()
} else {
// We also clamp the maximum number of workers to 64 so all following multiplications can't
// overflow.
let num_workers = num_workers.min(64);
(num_workers * num_workers * 16).next_power_of_two()
// Clamp the worker count so the shard total stays at or below 1 << 16.
let num_workers = num_workers.min(181);
(num_workers * num_workers * 2).next_power_of_two()
}
}
......@@ -1407,7 +1407,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#flags_field
#lazy_field
#[doc = "Per-task lock used with a shared resident-map guard."]
pub(crate) lock: IntrusiveTaskLock,
lock: IntrusiveTaskLock,
}
#[automatically_derived]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment