Unverified Commit 6728df44 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Address intrusive task lock feedback

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent d2781957
use std::{cell::UnsafeCell, hash::BuildHasher, hint::black_box, sync::Arc, thread}; use std::{cell::UnsafeCell, hash::BuildHasher, hint::black_box, sync::Arc, thread};
use criterion::{BatchSize, Criterion, Throughput}; use criterion::{BatchSize, Criterion, Throughput};
use crossbeam_utils::CachePadded;
use dashmap::DashMap; use dashmap::DashMap;
use parking_lot::{RawMutex, lock_api::RawMutex as RawMutexTrait}; use parking_lot::{RawMutex, lock_api::RawMutex as RawMutexTrait};
use rustc_hash::FxBuildHasher; use rustc_hash::FxBuildHasher;
...@@ -8,11 +9,9 @@ use rustc_hash::FxBuildHasher; ...@@ -8,11 +9,9 @@ use rustc_hash::FxBuildHasher;
const SHARDS: usize = 64; const SHARDS: usize = 64;
const TASKS: u32 = 4096; const TASKS: u32 = 4096;
#[repr(C)]
struct Payload { struct Payload {
lock: RawMutex, lock: RawMutex,
value: UnsafeCell<u64>, value: UnsafeCell<u64>,
padding: [u64; 14],
} }
impl Payload { impl Payload {
...@@ -20,7 +19,6 @@ impl Payload { ...@@ -20,7 +19,6 @@ impl Payload {
Self { Self {
lock: <RawMutex as RawMutexTrait>::INIT, lock: <RawMutex as RawMutexTrait>::INIT,
value: UnsafeCell::new(value), value: UnsafeCell::new(value),
padding: [0; 14],
} }
} }
...@@ -46,7 +44,8 @@ impl Drop for PayloadGuard<'_> { ...@@ -46,7 +44,8 @@ impl Drop for PayloadGuard<'_> {
} }
} }
type Map = DashMap<u32, Box<Payload>, FxBuildHasher>; // Keep independently locked tasks on separate cache lines without hand-written layout padding.
type Map = DashMap<u32, Box<CachePadded<Payload>>, FxBuildHasher>;
fn empty_map() -> Map { fn empty_map() -> Map {
DashMap::with_capacity_and_hasher_and_shard_amount(TASKS as usize, FxBuildHasher, SHARDS) DashMap::with_capacity_and_hasher_and_shard_amount(TASKS as usize, FxBuildHasher, SHARDS)
...@@ -55,7 +54,7 @@ fn empty_map() -> Map { ...@@ -55,7 +54,7 @@ fn empty_map() -> Map {
fn populated_map() -> Map { fn populated_map() -> Map {
let map = empty_map(); let map = empty_map();
for id in 1..=TASKS { for id in 1..=TASKS {
map.insert(id, Box::new(Payload::new(id as u64))); map.insert(id, Box::new(CachePadded::new(Payload::new(id as u64))));
} }
map map
} }
...@@ -102,7 +101,7 @@ pub fn storage_locking(criterion: &mut Criterion) { ...@@ -102,7 +101,7 @@ pub fn storage_locking(criterion: &mut Criterion) {
key = key.wrapping_add(1); key = key.wrapping_add(1);
let mut task = map let mut task = map
.entry(current) .entry(current)
.or_insert_with(|| Box::new(Payload::new(0))); .or_insert_with(|| Box::new(CachePadded::new(Payload::new(0))));
*task.value.get_mut() = black_box(current as u64); *task.value.get_mut() = black_box(current as u64);
drop(task); drop(task);
map.remove(&current); map.remove(&current);
...@@ -116,7 +115,7 @@ pub fn storage_locking(criterion: &mut Criterion) { ...@@ -116,7 +115,7 @@ pub fn storage_locking(criterion: &mut Criterion) {
key = key.wrapping_add(1); key = key.wrapping_add(1);
let task = map let task = map
.entry(current) .entry(current)
.or_insert_with(|| Box::new(Payload::new(0))) .or_insert_with(|| Box::new(CachePadded::new(Payload::new(0))))
.downgrade(); .downgrade();
task.with_mut(|value| *value = black_box(current as u64)); task.with_mut(|value| *value = black_box(current as u64));
drop(task); drop(task);
......
...@@ -307,7 +307,8 @@ impl TurboTasksBackend { ...@@ -307,7 +307,8 @@ impl TurboTasksBackend {
} }
pub fn new(mut options: BackendOptions, backing_storage: TurboBackingStorage) -> Self { pub fn new(mut options: BackendOptions, backing_storage: TurboBackingStorage) -> Self {
let shard_amount = compute_shard_amount(options.num_workers, options.small_preallocation); let resident_shard_amount =
compute_shard_amount(options.num_workers, options.small_preallocation);
if !options.dependency_tracking { if !options.dependency_tracking {
options.active_tracking = false; options.active_tracking = false;
} }
...@@ -343,7 +344,7 @@ impl TurboTasksBackend { ...@@ -343,7 +344,7 @@ impl TurboTasksBackend {
TaskId::try_from(TRANSIENT_TASK_BIT).unwrap(), TaskId::try_from(TRANSIENT_TASK_BIT).unwrap(),
TaskId::MAX, TaskId::MAX,
), ),
storage: Storage::new(shard_amount, small_preallocation), storage: Storage::new(resident_shard_amount, small_preallocation),
snapshot_coord: SnapshotCoordinator::new(), snapshot_coord: SnapshotCoordinator::new(),
snapshot_in_progress: Mutex::new(()), snapshot_in_progress: Mutex::new(()),
stopping: RwLock::new(false), stopping: RwLock::new(false),
......
...@@ -19,7 +19,8 @@ use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, p ...@@ -19,7 +19,8 @@ use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskTypeArc, event::Event, p
use crate::{ use crate::{
backend::storage_schema::{ backend::storage_schema::{
DropPartialOutcome, KeyEvictability, TaskStorage, UnevictableReason, ValueEvictability, DropPartialOutcome, KeyEvictability, TaskSlot, TaskStorage, UnevictableReason,
ValueEvictability,
}, },
backing_storage::SnapshotItem, backing_storage::SnapshotItem,
database::key_value_database::KeySpace, database::key_value_database::KeySpace,
...@@ -175,9 +176,6 @@ pub struct Storage { ...@@ -175,9 +176,6 @@ pub struct Storage {
/// ///
/// Indexed by `map.determine_shard(map.hash_usize(&key))` and guaranteed by construction so /// Indexed by `map.determine_shard(map.hash_usize(&key))` and guaranteed by construction so
/// that `shard_modified_counts.len()==map.shards().len()` /// that `shard_modified_counts.len()==map.shards().len()`
///
/// Should only be modified while holding either the task's intrusive lock and shard read
/// guard, or the corresponding DashMap shard write lock.
shard_modified_counts: Box<[CachePadded<AtomicU64>]>, shard_modified_counts: Box<[CachePadded<AtomicU64>]>,
/// Stores snapshots of task state for tasks accessed during snapshot mode. /// Stores snapshots of task state for tasks accessed during snapshot mode.
/// - `Some(snapshot)`: Task was modified before snapshot mode and accessed again during it. /// - `Some(snapshot)`: Task was modified before snapshot mode and accessed again during it.
...@@ -209,7 +207,7 @@ pub struct Storage { ...@@ -209,7 +207,7 @@ pub struct Storage {
/// while `SnapshotShardIter::next` holds the map shard write lock; both then take a /// while `SnapshotShardIter::next` holds the map shard write lock; both then take a
/// `snapshots` shard lock. `end_snapshot` must lock in the same order — see the shard-zipping /// `snapshots` shard lock. `end_snapshot` must lock in the same order — see the shard-zipping
/// pattern there. /// pattern there.
map: FxDashMap<TaskId, Box<TaskStorage>>, map: FxDashMap<TaskId, Box<TaskSlot>>,
/// A shared event notified whenever any task finishes restoring (successfully or not). /// A shared event notified whenever any task finishes restoring (successfully or not).
/// ///
/// Threads waiting for another thread's in-progress restore subscribe to this event, /// Threads waiting for another thread's in-progress restore subscribe to this event,
...@@ -358,6 +356,7 @@ impl Storage { ...@@ -358,6 +356,7 @@ impl Storage {
let work = if drain_entries { let work = if drain_entries {
let mut shard_guard = shard.write(); let mut shard_guard = shard.write();
shard_guard.retain(|(key, task)| { shard_guard.retain(|(key, task)| {
let task = task.get_mut_exclusive();
let modified_task = task.flags.any_modified(); let modified_task = task.flags.any_modified();
if modified_task { if modified_task {
debug_assert!( debug_assert!(
...@@ -490,7 +489,7 @@ impl Storage { ...@@ -490,7 +489,7 @@ impl Storage {
// write lock and would also obscure the pairing. // write lock and would also obscure the pairing.
let hash = self.map.hasher().hash_one(key); let hash = self.map.hasher().hash_one(key);
if let Some((_, task)) = map_guard.find_mut(hash, |(k, _)| *k == key) { if let Some((_, task)) = map_guard.find_mut(hash, |(k, _)| *k == key) {
self.promote_during_snapshot_flags(task, shard_idx); self.promote_during_snapshot_flags(task.get_mut_exclusive(), shard_idx);
} }
} }
// If we are saving a non-trivial amount of memory just clear it out. // If we are saving a non-trivial amount of memory just clear it out.
...@@ -516,7 +515,7 @@ impl Storage { ...@@ -516,7 +515,7 @@ impl Storage {
let inner = match self.map.entry(key) { let inner = match self.map.entry(key) {
dashmap::mapref::entry::Entry::Occupied(entry) => entry.into_ref(), dashmap::mapref::entry::Entry::Occupied(entry) => entry.into_ref(),
dashmap::mapref::entry::Entry::Vacant(entry) => { dashmap::mapref::entry::Entry::Vacant(entry) => {
entry.insert(Box::new(TaskStorage::new())) entry.insert(Box::new(TaskSlot::new()))
} }
}; };
StorageWriteGuard::new(self, inner.downgrade().into()) StorageWriteGuard::new(self, inner.downgrade().into())
...@@ -603,9 +602,13 @@ impl Storage { ...@@ -603,9 +602,13 @@ impl Storage {
}; };
// `get_disjoint` obtains all shard guards in shard-index order (sharing one guard for a // `get_disjoint` obtains all shard guards in shard-index order (sharing one guard for a
// same-shard pair) before task locks are taken here in TaskId order. // same-shard pair) before task locks are taken here in TaskId order.
let (lower, upper) = get_disjoint(&self.map, lower_key, upper_key, || { let (lower, upper) =
Box::new(TaskStorage::new()) get_disjoint(
}); &self.map,
lower_key,
upper_key,
|| Box::new(TaskSlot::new()),
);
let lower = StorageWriteGuard::new(self, lower); let lower = StorageWriteGuard::new(self, lower);
let upper = StorageWriteGuard::new(self, upper); let upper = StorageWriteGuard::new(self, upper);
if reverse { if reverse {
...@@ -679,8 +682,8 @@ impl Storage { ...@@ -679,8 +682,8 @@ impl Storage {
} }
}; };
shard.retain(|(task_id, task)| { shard.retain(|(task_id, task)| {
// Transient tasks can not be evicted at all, unless they are fully let task = task.get_mut_exclusive();
// delete by the GC. // Transient tasks cannot be evicted unless the GC has deleted them.
if task_id.is_transient() && !task.flags.deleted() { if task_id.is_transient() && !task.flags.deleted() {
evicted.unevictable_reasons[UnevictableReason::Transient.index()] += 1; evicted.unevictable_reasons[UnevictableReason::Transient.index()] += 1;
return true; return true;
...@@ -823,24 +826,26 @@ impl DerefMut for TaskEntryGuard<'_> { ...@@ -823,24 +826,26 @@ impl DerefMut for TaskEntryGuard<'_> {
pub struct StorageWriteGuard<'a> { pub struct StorageWriteGuard<'a> {
storage: &'a Storage, storage: &'a Storage,
inner: Ref<'a, TaskId, Box<TaskStorage>>, inner: Ref<'a, TaskId, Box<TaskSlot>>,
task: *mut TaskStorage,
// Match parking_lot's default guard behavior: task guards must not cross `.await`. // Match parking_lot's default guard behavior: task guards must not cross `.await`.
_not_send: PhantomData<*const ()>, _not_send: PhantomData<*const ()>,
} }
impl<'a> StorageWriteGuard<'a> { impl<'a> StorageWriteGuard<'a> {
fn new(storage: &'a Storage, inner: Ref<'a, TaskId, Box<TaskStorage>>) -> Self { fn new(storage: &'a Storage, inner: Ref<'a, TaskId, Box<TaskSlot>>) -> Self {
let task = inner.value().as_ref() as *const TaskStorage as *mut TaskStorage; // The map guard keeps the opaque slot alive while its raw lock is acquired.
// SAFETY: The DashMap read guard keeps the boxed task alive at this stable address. inner.value().lock();
unsafe { &*task }.lock();
Self { Self {
storage, storage,
inner, inner,
task,
_not_send: PhantomData, _not_send: PhantomData,
} }
} }
fn task(&self) -> &TaskStorage {
// SAFETY: `new` acquired this slot's lock, and `inner` keeps it alive.
unsafe { self.inner.value().get() }
}
} }
impl StorageWriteGuard<'_> { impl StorageWriteGuard<'_> {
...@@ -875,7 +880,7 @@ impl StorageWriteGuard<'_> { ...@@ -875,7 +880,7 @@ impl StorageWriteGuard<'_> {
"track_modification called on transient task {:?}", "track_modification called on transient task {:?}",
self.inner.key() self.inner.key()
); );
let flags = &self.inner.flags; let flags = &self.flags;
if flags.is_modified_during_snapshot(category) { if flags.is_modified_during_snapshot(category) {
// We can early return since `end_snapshot` is responsible for reconciling. // We can early return since `end_snapshot` is responsible for reconciling.
return TrackOutcome::NoChange; return TrackOutcome::NoChange;
...@@ -921,7 +926,7 @@ impl StorageWriteGuard<'_> { ...@@ -921,7 +926,7 @@ impl StorageWriteGuard<'_> {
if inserted_snapshot { if inserted_snapshot {
// Snapshot all non-transient fields, carrying the modified bits into // Snapshot all non-transient fields, carrying the modified bits into
// the copy so the iterator knows which categories to persist. // the copy so the iterator knows which categories to persist.
let mut snapshot = self.inner.clone_snapshot(); let mut snapshot = self.clone_snapshot();
snapshot.flags.set_data_modified(flags.data_modified()); snapshot.flags.set_data_modified(flags.data_modified());
snapshot.flags.set_meta_modified(flags.meta_modified()); snapshot.flags.set_meta_modified(flags.meta_modified());
snapshot.flags.set_new_task(flags.new_task()); snapshot.flags.set_new_task(flags.new_task());
...@@ -980,10 +985,10 @@ impl StorageWriteGuard<'_> { ...@@ -980,10 +985,10 @@ impl StorageWriteGuard<'_> {
"discard_modifications_for_gc_new_task must run before the snapshot starts" "discard_modifications_for_gc_new_task must run before the snapshot starts"
); );
debug_assert!( debug_assert!(
self.inner.flags.new_task(), self.flags.new_task(),
"only a never-persisted (new_task) collected task may be discarded this way" "only a never-persisted (new_task) collected task may be discarded this way"
); );
if self.inner.flags.any_modified() { if self.flags.any_modified() {
let shard_idx = self.storage.shard_index(self.inner.key()); let shard_idx = self.storage.shard_index(self.inner.key());
self.storage.shard_modified_counts[shard_idx].fetch_sub(1, Ordering::Relaxed); self.storage.shard_modified_counts[shard_idx].fetch_sub(1, Ordering::Relaxed);
} }
...@@ -997,15 +1002,14 @@ impl Deref for StorageWriteGuard<'_> { ...@@ -997,15 +1002,14 @@ impl Deref for StorageWriteGuard<'_> {
type Target = TaskStorage; type Target = TaskStorage;
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
// SAFETY: `new` locked this task, and `inner` keeps its Box alive until after `Drop`. self.task()
unsafe { &*self.task }
} }
} }
impl DerefMut for StorageWriteGuard<'_> { impl DerefMut for StorageWriteGuard<'_> {
fn deref_mut(&mut self) -> &mut Self::Target { fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY: The task's intrusive lock gives this guard exclusive payload access. // SAFETY: This guard owns the slot's task lock, and `inner` keeps it alive.
unsafe { &mut *self.task } unsafe { &mut *self.inner.value().as_ptr() }
} }
} }
...@@ -1014,7 +1018,7 @@ impl Drop for StorageWriteGuard<'_> { ...@@ -1014,7 +1018,7 @@ impl Drop for StorageWriteGuard<'_> {
// SAFETY: This guard acquired the lock in `new`, still owns it, and performs no task // SAFETY: This guard acquired the lock in `new`, still owns it, and performs no task
// access after unlocking. Rust drops `inner` only after this method returns, so the Box // access after unlocking. Rust drops `inner` only after this method returns, so the Box
// remains alive through the unlock. // remains alive through the unlock.
unsafe { (&*self.task).unlock() }; unsafe { self.inner.value().unlock() };
} }
} }
...@@ -1106,7 +1110,7 @@ enum ShardWork { ...@@ -1106,7 +1110,7 @@ enum ShardWork {
/// (modified-only) shard table out of the map. The iterator owns that table and drains it /// (modified-only) shard table out of the map. The iterator owns that table and drains it
/// directly, freeing each task box as it is serialized. No second map lookup, no flag /// directly, freeing each task box as it is serialized. No second map lookup, no flag
/// bookkeeping (the whole map is discarded right after this snapshot). /// bookkeeping (the whole map is discarded right after this snapshot).
Drain(hash_table::IntoIter<(TaskId, Box<TaskStorage>)>), Drain(hash_table::IntoIter<(TaskId, Box<TaskSlot>)>),
} }
pub struct SnapshotShard<'l, P> { pub struct SnapshotShard<'l, P> {
...@@ -1188,12 +1192,13 @@ where ...@@ -1188,12 +1192,13 @@ where
} }
ShardWork::Drain(entries) => { ShardWork::Drain(entries) => {
// Shutdown only: the scan already moved this shard's modified entries out of the // Shutdown only: the scan already moved this shard's modified entries out of the
// map, so we own each `Box<TaskStorage>` here. Serialize from a borrow of the owned // map, so we own each `Box<TaskSlot>` here. Move out the storage, serialize it, and
// box and let it drop at the end of this branch — freeing the task's memory as it // let it drop at the end of this branch — freeing the task's memory as it
// is persisted rather than after the whole batch is written. We skip the flag // is persisted rather than after the whole batch is written. We skip the flag
// bookkeeping the normal path does, since the entire map is discarded right after // bookkeeping the normal path does, since the entire map is discarded right after
// this snapshot. // this snapshot.
let (task_id, inner) = entries.next()?; let (task_id, inner) = entries.next()?;
let inner = (*inner).into_inner();
Some(serialize_task(task_id, &inner)) Some(serialize_task(task_id, &inner))
// we don't need to update any bits because everything is getting dropped. // we don't need to update any bits because everything is getting dropped.
} }
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
//! - `meta` - Rarely changed metadata (output, aggregation, flags) //! - `meta` - Rarely changed metadata (output, aggregation, flags)
//! - `transient` - Not serialized, only exists in memory //! - `transient` - Not serialized, only exists in memory
use std::{ use std::{
cell::UnsafeCell,
fmt, fmt,
hash::{BuildHasherDefault, Hash}, hash::{BuildHasherDefault, Hash},
sync::Arc, sync::Arc,
...@@ -26,7 +27,7 @@ use std::{ ...@@ -26,7 +27,7 @@ use std::{
use parking_lot::{Mutex, RawMutex, lock_api::RawMutex as RawMutexTrait}; use parking_lot::{Mutex, RawMutex, lock_api::RawMutex as RawMutexTrait};
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use turbo_tasks::{ use turbo_tasks::{
CellId, SharedReference, ShrinkToFit, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId, CellId, SharedReference, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
backend::{CachedTaskTypeArc, CellHash, TransientTaskType}, backend::{CachedTaskTypeArc, CellHash, TransientTaskType},
event::Event, event::Event,
task_storage, task_storage,
...@@ -77,16 +78,6 @@ impl IntrusiveTaskLock { ...@@ -77,16 +78,6 @@ impl IntrusiveTaskLock {
} }
} }
struct IntrusiveTaskLockGuard<'a>(&'a IntrusiveTaskLock);
impl Drop for IntrusiveTaskLockGuard<'_> {
fn drop(&mut self) {
// SAFETY: This lexical guard is created only after acquiring this lock and drops before
// the protected reference can escape.
unsafe { self.0.unlock() };
}
}
impl Default for IntrusiveTaskLock { impl Default for IntrusiveTaskLock {
fn default() -> Self { fn default() -> Self {
Self::new() Self::new()
...@@ -99,10 +90,82 @@ impl fmt::Debug for IntrusiveTaskLock { ...@@ -99,10 +90,82 @@ impl fmt::Debug for IntrusiveTaskLock {
} }
} }
impl ShrinkToFit for IntrusiveTaskLock { /// Opaque interior-mutable task slot used when a resident map guard provides lifetime but not
fn shrink_to_fit(&mut self) {} /// payload exclusion.
///
/// # Unsafe access protocol
///
/// - An unlocked slot never exposes a `TaskStorage` reference.
/// - `lock` acquires the embedded task lock through the raw slot pointer before references exist.
/// - Only a lock-owning storage guard may call `get` or materialize a mutable reference from
/// `as_ptr`.
/// - A structural map write guard may call `get_mut_exclusive` because it excludes all map readers.
/// - The task lock is released before the map guard that keeps this allocation alive.
#[repr(transparent)]
pub(crate) struct TaskSlot(UnsafeCell<TaskStorage>);
impl TaskSlot {
pub(crate) fn new() -> Self {
Self(UnsafeCell::new(TaskStorage::new()))
}
pub(crate) fn as_ptr(&self) -> *mut TaskStorage {
self.0.get()
}
pub(crate) fn lock(&self) {
// SAFETY: Project only the embedded lock from the opaque cell. No TaskStorage reference is
// materialized before exclusion is acquired.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).lock() };
}
/// # Safety
/// The caller must own this slot's task lock and perform no protected access after unlocking.
pub(crate) unsafe fn unlock(&self) {
// SAFETY: Forwarded from the caller; projection does not materialize TaskStorage.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).unlock() };
}
/// # Safety
/// The caller must own this slot's task lock.
pub(crate) unsafe fn get(&self) -> &TaskStorage {
// SAFETY: Forwarded from the caller.
unsafe { &*self.as_ptr() }
}
pub(crate) fn get_mut_exclusive(&mut self) -> &mut TaskStorage {
self.0.get_mut()
}
pub(crate) fn into_inner(self) -> TaskStorage {
self.0.into_inner()
}
#[cfg(test)]
pub(crate) fn is_locked(&self) -> bool {
// SAFETY: Reading the raw lock state does not expose the task payload.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).is_locked() }
}
pub(crate) fn with_lock<R>(&self, f: impl FnOnce(&TaskStorage) -> R) -> R {
self.lock();
struct Unlock<'a>(&'a TaskSlot);
impl Drop for Unlock<'_> {
fn drop(&mut self) {
// SAFETY: `with_lock` acquired this lock and the closure has returned/unwound.
unsafe { self.0.unlock() };
}
}
let _unlock = Unlock(self);
// SAFETY: The lock-owning lexical guard remains alive through the closure.
f(unsafe { self.get() })
}
} }
// SAFETY: all shared payload access is governed by the protocol above. Structural mutation uses
// `&mut TaskSlot`, while ordinary mutation owns the embedded task lock.
unsafe impl Sync for TaskSlot {}
/// The complete task storage schema. /// The complete task storage schema.
/// ///
/// This struct defines all storage fields for a task. The `#[task_storage]` macro /// This struct defines all storage fields for a task. The `#[task_storage]` macro
...@@ -606,33 +669,6 @@ pub enum KeyEvictability { ...@@ -606,33 +669,6 @@ pub enum KeyEvictability {
} }
impl TaskStorage { impl TaskStorage {
/// Lock this task's storage while a resident-map guard keeps its allocation alive.
pub(crate) fn lock(&self) {
self.lock.lock();
}
/// Unlock this task's storage.
///
/// # Safety
///
/// The current thread must own the task lock, hold the resident-map guard that keeps this
/// storage alive, and perform no protected access after this call.
pub(crate) unsafe fn unlock(&self) {
// SAFETY: Forwarded from the caller.
unsafe { self.lock.unlock() };
}
#[cfg(test)]
pub(crate) fn is_locked(&self) -> bool {
self.lock.is_locked()
}
pub(crate) fn with_lock<R>(&self, f: impl FnOnce(&Self) -> R) -> R {
self.lock();
let _guard = IntrusiveTaskLockGuard(&self.lock);
f(self)
}
/// Determine the evictability level of this task based on its flags. /// Determine the evictability level of this task based on its flags.
/// ///
/// This checks only the flags on the TaskStorage itself. The caller /// This checks only the flags on the TaskStorage itself. The caller
...@@ -1926,14 +1962,15 @@ mod tests { ...@@ -1926,14 +1962,15 @@ mod tests {
// are not meaningful under it. // are not meaningful under it.
#[test] #[test]
fn snapshot_clone_has_an_independent_unlocked_task_lock() { fn snapshot_clone_has_an_independent_unlocked_task_lock() {
let storage = TaskStorage::new(); let slot = TaskSlot::new();
storage.lock(); slot.lock();
assert!(storage.is_locked()); assert!(slot.is_locked());
let snapshot = storage.clone_snapshot(); // SAFETY: This test holds the slot lock until after cloning.
assert!(storage.is_locked()); let snapshot = unsafe { slot.get() }.clone_snapshot();
assert!(!snapshot.is_locked()); assert!(slot.is_locked());
assert!(!snapshot.lock.is_locked());
// SAFETY: This test acquired the source lock above and accesses it no further. // SAFETY: This test acquired the source lock above and accesses it no further.
unsafe { storage.unlock() }; unsafe { slot.unlock() };
} }
#[test] #[test]
...@@ -1944,6 +1981,11 @@ mod tests { ...@@ -1944,6 +1981,11 @@ mod tests {
128, 128,
"TaskStorage size changed! Update this test." "TaskStorage size changed! Update this test."
); );
assert_eq!(
size_of::<TaskSlot>(),
size_of::<TaskStorage>(),
"the transparent resident slot must add no per-task size"
);
// `LazyField` is 40 B = 32 B largest payload + 8 B discriminant. // `LazyField` is 40 B = 32 B largest payload + 8 B discriminant.
assert_eq!( assert_eq!(
size_of::<LazyField>(), size_of::<LazyField>(),
......
...@@ -25,12 +25,12 @@ pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: boo ...@@ -25,12 +25,12 @@ pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: boo
// for some constant `k` the probability of at least one collision for large `N` can be // for some constant `k` the probability of at least one collision for large `N` can be
// approximated as: P = 1 - exp(-N^2 / (2*S)) = 1 - exp(-1/(2*k)) // approximated as: P = 1 - exp(-N^2 / (2*S)) = 1 - exp(-1/(2*k))
// //
// For `k = 16` this results in a collision probability of about 3%. // For `k = 2` this results in a collision probability of about 22%. Fine-grained task locks
// For `k = 1` this results in a collision probability of about 39%. // keep a shard collision from serializing independent task mutations, so the lower shard count
// saves memory without recreating the old shard-write-lock bottleneck.
// //
// We clamp the number of shards to 1 << 16 to avoid excessive memory usage in case of a very // We clamp the number of shards to 1 << 16 to avoid excessive memory usage in case of a very
// high number of worker threads. This case is hit with more than 64 worker threads for `k = // high number of worker threads. With `k = 2`, this case is hit above roughly 181 workers.
// 16` and more than 256 worker threads for `k = 1`.
if small_preallocation { if small_preallocation {
// We also clamp the maximum number of workers to 256 so all following multiplications can't // We also clamp the maximum number of workers to 256 so all following multiplications can't
...@@ -38,9 +38,8 @@ pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: boo ...@@ -38,9 +38,8 @@ pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: boo
let num_workers = num_workers.min(256); let num_workers = num_workers.min(256);
(num_workers * num_workers).next_power_of_two() (num_workers * num_workers).next_power_of_two()
} else { } else {
// We also clamp the maximum number of workers to 64 so all following multiplications can't // Clamp the worker count so the shard total stays at or below 1 << 16.
// overflow. let num_workers = num_workers.min(181);
let num_workers = num_workers.min(64); (num_workers * num_workers * 2).next_power_of_two()
(num_workers * num_workers * 16).next_power_of_two()
} }
} }
...@@ -1407,7 +1407,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1407,7 +1407,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#flags_field #flags_field
#lazy_field #lazy_field
#[doc = "Per-task lock used with a shared resident-map guard."] #[doc = "Per-task lock used with a shared resident-map guard."]
pub(crate) lock: IntrusiveTaskLock, lock: IntrusiveTaskLock,
} }
#[automatically_derived] #[automatically_derived]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment