Unverified Commit 3ecbba4e authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Use Papaya for resident task storage

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 48a01714
......@@ -4741,6 +4741,16 @@ version = "4.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e"
[[package]]
name = "papaya"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da2442474a9404698c42509b8967f437249dbc7b50493e83020333d3943ec0ae"
dependencies = [
"equivalent",
"seize",
]
[[package]]
name = "papergrid"
version = "0.7.1"
......@@ -6362,6 +6372,22 @@ dependencies = [
"libc",
]
[[package]]
name = "seize"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521"
dependencies = [
"libc",
"windows-sys 0.61.2",
]
[[package]]
name = "self_cell"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba"
[[package]]
name = "semver"
version = "1.0.23"
......@@ -9209,6 +9235,7 @@ dependencies = [
"bincode",
"bitfield",
"codspeed-criterion-compat",
"concurrent-queue",
"crossbeam-utils",
"dashmap 7.0.0-rc2",
"fs-err",
......@@ -9218,6 +9245,7 @@ dependencies = [
"indoc",
"jiff",
"lz4_flex",
"papaya",
"parking_lot",
"rand 0.10.1",
"regex",
......
......@@ -313,6 +313,7 @@ nohash-hasher = "0.2.0"
notify = "9.0.0-rc.5"
owo-colors = "4.2.2"
parcel_selectors = "0.28.2"
papaya = "0.2.5"
parking_lot = "0.12.1"
pathdiff = "0.2.1"
phf = { version = "0.11", features = ["macros"] }
......
......@@ -41,6 +41,7 @@ anyhow = { workspace = true }
bincode = { workspace = true }
auto-hash-map = { workspace = true }
bitfield = { workspace = true }
concurrent-queue = { workspace = true }
crossbeam-utils = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]}
fs-err = { workspace = true }
......@@ -48,6 +49,7 @@ hashbrown = { workspace = true }
indexmap = { workspace = true }
jiff = "0.2.10"
lz4_flex = { workspace = true, optional = true }
papaya = { workspace = true }
parking_lot = { workspace = true }
rand = { workspace = true }
ringmap = { workspace = true, features = ["serde"] }
......
......@@ -5,6 +5,8 @@ use criterion::{Criterion, criterion_group, criterion_main};
pub(crate) mod gc;
pub(crate) mod overhead;
pub(crate) mod papaya_dirty;
pub(crate) mod papaya_map;
pub(crate) mod scope_stress;
pub(crate) mod storage_locking;
pub(crate) mod stress;
......@@ -12,6 +14,6 @@ pub(crate) mod stress;
criterion_group!(
name = turbo_tasks_backend_stress;
config = Criterion::default();
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc, storage_locking::storage_locking
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc, storage_locking::storage_locking, papaya_dirty::papaya_dirty, papaya_map::papaya_map
);
criterion_main!(turbo_tasks_backend_stress);
use std::{
hint::black_box,
sync::atomic::{AtomicBool, AtomicU64, Ordering},
};
use concurrent_queue::ConcurrentQueue;
use criterion::{BatchSize, BenchmarkId, Criterion, Throughput};
use papaya::HashMap;
use rustc_hash::FxBuildHasher;
const TASKS: u32 = 100_000;
fn map_with_dirty(dirty: u32) -> HashMap<u32, AtomicBool, FxBuildHasher> {
let map = HashMap::with_capacity_and_hasher(TASKS as usize, FxBuildHasher);
let guard = map.guard();
for task_id in 0..TASKS {
map.insert(task_id, AtomicBool::new(task_id < dirty), &guard);
}
drop(guard);
map
}
pub fn papaya_dirty(criterion: &mut Criterion) {
let mut mark = criterion.benchmark_group("papaya_dirty_mark_all");
mark.throughput(Throughput::Elements(TASKS as u64));
mark.bench_function("atomic_count", |b| {
b.iter_batched(
|| AtomicU64::new(0),
|count| {
for _ in 0..TASKS {
black_box(count.fetch_add(1, Ordering::Relaxed));
}
},
BatchSize::LargeInput,
)
});
mark.bench_function("concurrent_queue", |b| {
b.iter_batched(
ConcurrentQueue::unbounded,
|queue| {
for task_id in 0..TASKS {
queue.push(black_box(task_id)).unwrap();
}
},
BatchSize::LargeInput,
)
});
mark.finish();
for dirty in [1, 10, 100, 10_000, TASKS] {
let map = map_with_dirty(dirty);
let ids: Vec<_> = (0..dirty).collect();
let mut process = criterion.benchmark_group("papaya_dirty_process");
process.throughput(Throughput::Elements(dirty as u64));
process.bench_with_input(BenchmarkId::new("full_scan", dirty), &dirty, |b, _| {
b.iter(|| {
let guard = map.guard();
let count = map
.iter(&guard)
.filter(|(_, dirty)| dirty.load(Ordering::Relaxed))
.count();
black_box(count)
})
});
process.bench_with_input(BenchmarkId::new("queued_ids", dirty), &dirty, |b, _| {
b.iter(|| {
let guard = map.guard();
let count = ids
.iter()
.filter(|task_id| {
map.get(*task_id, &guard)
.is_some_and(|dirty| dirty.load(Ordering::Relaxed))
})
.count();
black_box(count)
})
});
process.finish();
}
}
use std::{hint::black_box, thread};
use criterion::{BatchSize, Criterion, Throughput};
use crossbeam_utils::CachePadded;
use dashmap::DashMap;
use papaya::HashMap;
use parking_lot::Mutex;
use rustc_hash::FxBuildHasher;
const TASKS: u32 = 4096;
const OPS: usize = 1000;
type Payload = CachePadded<Mutex<u64>>;
type Dash = DashMap<u32, Box<Payload>, FxBuildHasher>;
type Papaya = HashMap<u32, Payload, FxBuildHasher>;
fn dash() -> Dash {
let map = DashMap::with_capacity_and_hasher_and_shard_amount(TASKS as usize, FxBuildHasher, 64);
for id in 1..=TASKS {
map.insert(id, Box::new(CachePadded::new(Mutex::new(id as u64))));
}
map
}
fn papaya() -> Papaya {
let map = HashMap::with_capacity_and_hasher(TASKS as usize, FxBuildHasher);
let guard = map.guard();
for id in 1..=TASKS {
map.insert(id, CachePadded::new(Mutex::new(id as u64)), &guard);
}
drop(guard);
map
}
pub fn papaya_map(criterion: &mut Criterion) {
let before = turbo_tasks_malloc::TurboMalloc::allocation_counters();
let dash = dash();
let after_dash = turbo_tasks_malloc::TurboMalloc::allocation_counters();
eprintln!(
"dashmap resident allocation: {} bytes in {} allocations",
(after_dash.allocations - before.allocations)
.saturating_sub(after_dash.deallocations - before.deallocations),
(after_dash.allocation_count - before.allocation_count)
.saturating_sub(after_dash.deallocation_count - before.deallocation_count),
);
let before = after_dash;
let papaya = papaya();
let after_papaya = turbo_tasks_malloc::TurboMalloc::allocation_counters();
eprintln!(
"papaya resident allocation: {} bytes in {} allocations",
(after_papaya.allocations - before.allocations)
.saturating_sub(after_papaya.deallocations - before.deallocations),
(after_papaya.allocation_count - before.allocation_count)
.saturating_sub(after_papaya.deallocation_count - before.deallocation_count),
);
let mut hit = criterion.benchmark_group("papaya_map_hit");
hit.throughput(Throughput::Elements(1));
hit.bench_function("dashmap_read_intrusive", |b| {
b.iter(|| {
let task = dash.get(black_box(&1)).unwrap();
*task.lock() = black_box(1);
})
});
hit.bench_function("papaya_guard_intrusive", |b| {
b.iter(|| {
let guard = papaya.guard();
let task = papaya.get(black_box(&1), &guard).unwrap();
*task.lock() = black_box(1);
})
});
hit.finish();
let mut insert = criterion.benchmark_group("papaya_map_insert_remove");
insert.throughput(Throughput::Elements(1));
insert.bench_function("dashmap", |b| {
let map = DashMap::with_hasher(FxBuildHasher);
let mut key = 1u32;
b.iter(|| {
let current = key;
key = key.wrapping_add(1);
map.entry(current)
.or_insert_with(|| Box::new(CachePadded::new(Mutex::new(0))));
black_box(map.remove(&current));
})
});
insert.bench_function("papaya", |b| {
let map = HashMap::with_hasher(FxBuildHasher);
let mut key = 1u32;
b.iter(|| {
let current = key;
key = key.wrapping_add(1);
let guard = map.guard();
map.get_or_insert_with(current, || CachePadded::new(Mutex::new(0)), &guard);
black_box(map.remove(&current, &guard));
})
});
insert.finish();
let keys = [1, 65, 129, 193];
let mut parallel = criterion.benchmark_group("papaya_map_parallel_independent");
parallel.throughput(Throughput::Elements((keys.len() * OPS) as u64));
parallel.bench_function("dashmap_read_intrusive", |b| {
b.iter(|| {
thread::scope(|scope| {
for &key in &keys {
let map = &dash;
scope.spawn(move || {
for _ in 0..OPS {
let task = map.get(&key).unwrap();
*task.lock() = black_box(key as u64);
}
});
}
});
})
});
parallel.bench_function("papaya_guard_intrusive", |b| {
b.iter(|| {
thread::scope(|scope| {
for &key in &keys {
let map = &papaya;
scope.spawn(move || {
for _ in 0..OPS {
let guard = map.guard();
let task = map.get(&key, &guard).unwrap();
*task.lock() = black_box(key as u64);
}
});
}
});
})
});
parallel.finish();
let mut iteration = criterion.benchmark_group("papaya_map_iteration");
iteration.throughput(Throughput::Elements(TASKS as u64));
iteration.bench_function("dashmap_shards", |b| {
b.iter(|| {
let mut sum = 0u64;
for item in dash.iter() {
sum = sum.wrapping_add(*item.lock());
}
black_box(sum)
})
});
iteration.bench_function("papaya_guard", |b| {
b.iter(|| {
let guard = papaya.guard();
let mut sum = 0u64;
for (_, task) in papaya.iter(&guard) {
sum = sum.wrapping_add(*task.lock());
}
black_box(sum)
})
});
iteration.finish();
let mut snapshots = criterion.benchmark_group("papaya_snapshot_map_roundtrip");
snapshots.throughput(Throughput::Elements(TASKS as u64));
snapshots.bench_function("dashmap", |b| {
b.iter_batched(
|| DashMap::with_hasher(FxBuildHasher),
|map| {
for id in 1..=TASKS {
map.insert(id, id as u64);
}
for id in 1..=TASKS {
black_box(map.remove(&id));
}
},
BatchSize::LargeInput,
)
});
snapshots.bench_function("papaya", |b| {
b.iter_batched(
|| HashMap::with_hasher(FxBuildHasher),
|map| {
let guard = map.guard();
for id in 1..=TASKS {
map.insert(id, id as u64, &guard);
}
for id in 1..=TASKS {
black_box(map.remove(&id, &guard));
}
},
BatchSize::LargeInput,
)
});
snapshots.finish();
}
......@@ -44,13 +44,9 @@ use crate::{
/// How long a GC root may go un-anchored before it is collected.
/// Default to 3 days so that a root that is at least occasionally used can survive a weekend.
///
/// Aging out roots solves the problem of missing `gc_unpin` calls. We can miss them for structural
/// reasons, bugs or just shutdown races (drops from native threads race with turbopack shutdown).
/// So using a TTL to track roots that haven't shown up in new sessions we can solve this leak.
///
/// The TTL counter is serving as a check for both new sessions and time. To be aged out you get
/// one session to start the clock and then eventually the timer expires. This is intentionally
/// course.
/// Aging out roots solves the problem of missing `gc_unpin` calls. We can miss them for structural
/// reasons, bugs or shutdown races (drops from native threads race with turbopack shutdown).
/// The TTL counter checks both new sessions and elapsed time.
pub(crate) const DEFAULT_GC_ROOT_TTL: Duration = Duration::from_secs(3 * 24 * 60 * 60);
/// How long a GC root has gone without being observed live, as stored in the persisted roots map.
......@@ -58,15 +54,13 @@ pub(crate) const DEFAULT_GC_ROOT_TTL: Duration = Duration::from_secs(3 * 24 * 60
pub enum TtlCounter {
/// Observed live (a durable, anchored root) in the most recent session.
MostRecent,
/// System time millis at which a session's **first** GC pass first found this root not live.
/// System time millis at which a session's first GC pass found this root not live.
FirstStale(u64),
}
/// One unit of GC work.
enum GcJob {
/// Scan one shard of the resident map (by index) and enqueue its candidates as
/// [`GcJob::Collect`].
ScanShard(usize),
ScanBatch(Vec<TaskId>),
/// Collect a single task.
Collect(TaskId),
}
......@@ -240,35 +234,36 @@ impl TurboTasksBackend {
None
};
// Batch ID scans to avoid one shared-queue operation for each resident task, while the
// aged-out roots still enter the same collection scope as direct jobs.
const GC_SCAN_BATCH_SIZE: usize = 256;
let task_ids = self.storage.gc_task_ids();
let batches = task_ids
.chunks(GC_SCAN_BATCH_SIZE)
.map(|ids| GcJob::ScanBatch(ids.to_vec()))
.collect::<Vec<_>>();
let (mut stats, mut result): (GcStats, GcPassResult) = scope_unbounded_with(
// Start by scanning all shards and collecting the aged out roots from prior sessions.
(0..self.storage.shard_count())
.map(GcJob::ScanShard)
.chain(aged_out.into_iter().map(GcJob::Collect)),
batches.into_iter().chain(aged_out.into_iter().map(GcJob::Collect)),
Default::default,
|spawner, job, (stats, result): &mut (GcStats, GcPassResult)| {
// Abort the gc loop if we are interrupted
if let Some(budget) = &budget
&& budget.should_stop()
{
return ControlFlow::Break(());
}
let collector = |task_id| spawner.spawn(GcJob::Collect(task_id));
let task_id = match job {
GcJob::ScanShard(index) => {
let collector = |task_id| spawner.spawn(GcJob::Collect(task_id));
self.storage.gc_scan_shard(index, collector);
GcJob::ScanBatch(task_ids) => {
self.storage.gc_scan_batch(&task_ids, collector);
return ControlFlow::Continue(());
}
GcJob::Collect(task_id) => task_id,
};
let collector = |child_id| spawner.spawn(GcJob::Collect(child_id));
// Recheck under the task guard: the resident scan saw an earlier state and
// another thread may have changed graph edges before the collector got here.
let mut ctx = ExecuteContextImpl::new_for_gc(self, turbo_tasks, phase, &collector);
// `All` restores Data so `capture_all_edges` below can read the
// Data-category dependency sets. The recheck itself only needs Meta.
// `All` restores Data so capture_all_edges can read dependency sets.
let mut task = ctx.task(task_id, TaskDataCategory::All);
// Recheck under the guard: the shard scan saw this task without holding it, and a
// racing teardown can add uppers that temporarily remove collectibility. Such a
// task is re-enqueued by a later pass.
if !task.is_gc_collectible() {
return ControlFlow::Continue(());
}
......
......@@ -6,6 +6,7 @@ mod operation;
mod snapshot_coordinator;
mod storage;
pub mod storage_schema;
mod task_map;
// Only the `verify_aggregation_graph` feature still uses atomics here; `stopping` is an
// `RwLock<bool>` so that checking it and acting on it cannot be split (see the field's docs).
......@@ -84,7 +85,6 @@ use crate::{
kv_backing_storage::TurboBackingStorage,
utils::{
dash_map_entry::{get_in_shard, get_shard, with_entry_in_shard},
shard_amount::compute_shard_amount,
stopwatch::Stopwatch,
},
};
......@@ -307,8 +307,6 @@ impl TurboTasksBackend {
}
pub fn new(mut options: BackendOptions, backing_storage: TurboBackingStorage) -> Self {
let resident_shard_amount =
compute_shard_amount(options.num_workers, options.small_preallocation);
if !options.dependency_tracking {
options.active_tracking = false;
}
......@@ -344,7 +342,7 @@ impl TurboTasksBackend {
TaskId::try_from(TRANSIENT_TASK_BIT).unwrap(),
TaskId::MAX,
),
storage: Storage::new(resident_shard_amount, small_preallocation),
storage: Storage::new(small_preallocation),
snapshot_coord: SnapshotCoordinator::new(),
snapshot_in_progress: Mutex::new(()),
stopping: RwLock::new(false),
......
......@@ -2163,7 +2163,7 @@ mod filter_transient_tracking_tests {
#[test]
fn autoset_add_remove_only_tracks_persistent_keys() {
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
let task_id = persistent_task(1);
......@@ -2214,7 +2214,7 @@ mod filter_transient_tracking_tests {
#[test]
fn autoset_extend_tracks_iff_any_persistent() {
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
// Extend with only transient children: no meta modification.
......@@ -2243,7 +2243,7 @@ mod filter_transient_tracking_tests {
#[test]
fn countermap_update_count_only_tracks_persistent_keys() {
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
let task_id = persistent_task(1);
......@@ -2267,7 +2267,7 @@ mod filter_transient_tracking_tests {
#[test]
fn countermap_update_counts_batch_tracks_iff_any_persistent() {
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
// followers: CounterMap<TaskId>, filter_transient, has update_counts.
......@@ -2294,7 +2294,7 @@ mod filter_transient_tracking_tests {
#[test]
fn direct_option_set_take_tracks_by_value_transience() {
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
// `output` is a direct Option<OutputValue> meta field with filter_transient.
......@@ -2443,7 +2443,7 @@ mod cell_data_tracking_tests {
// (HashOnly persists via the separate `cell_data_hash` field) — but both
// are still stored in memory. Tracking is monotonic: a later persistable
// write flips the flag even after skipped writes left it clean.
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
let mut g = guard_for(&storage, &counter, persistent_task(1));
......@@ -2467,7 +2467,7 @@ mod cell_data_tracking_tests {
#[test]
fn remove_tracks_only_for_persistable() {
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
let skip = cell_of::<SkipCheapV>(0);
let persistable = cell_of::<PersistableV>(0);
......@@ -2508,7 +2508,7 @@ mod cell_data_tracking_tests {
// `drop_partial` (which keys on Evictability, not the modified flag), so
// a task that only wrote such a cell is both evictable-clean AND keeps the
// value. This is the core safety property of not tracking Skip writes.
let storage = Storage::new(2, true);
let storage = Storage::new(true);
let counter = TaskLockCounter::new();
let task_id = persistent_task(1);
let cell = cell_of::<SkipNeverV>(0);
......
......@@ -97,16 +97,20 @@ impl fmt::Debug for IntrusiveTaskLock {
///
/// - An unlocked slot never exposes a `TaskStorage` reference.
/// - `lock` acquires the embedded task lock through the raw slot pointer before references exist.
/// - Only a lock-owning storage guard may call `get` or materialize a mutable reference from
/// `as_ptr`.
/// - A structural map write guard may call `get_mut_exclusive` because it excludes all map readers.
/// - After locking, a reader checks `present`; a stale Papaya reader retries when removal cleared
/// it.
/// - Only a lock-owning storage guard that observed `present` may call `get` or materialize a
/// mutable reference from `as_ptr`.
/// - Structural removal detaches the exact entry and clears `present` before unlocking.
/// - The task lock is released before the map guard that keeps this allocation alive.
#[repr(transparent)]
pub(crate) struct TaskSlot(UnsafeCell<TaskStorage>);
impl TaskSlot {
pub(crate) fn new() -> Self {
Self(UnsafeCell::new(TaskStorage::new()))
let mut task = TaskStorage::new();
task.present = true;
Self(UnsafeCell::new(task))
}
pub(crate) fn as_ptr(&self) -> *mut TaskStorage {
......@@ -126,20 +130,29 @@ impl TaskSlot {
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).unlock() };
}
/// Whether this slot is still published as the resident value for its TaskId.
///
/// # Safety
/// The caller must own this slot's task lock, or hold an exclusive shard write guard that
/// prevents every concurrent access to this slot.
pub(crate) unsafe fn get(&self) -> &TaskStorage {
// SAFETY: Forwarded from the caller.
unsafe { &*self.as_ptr() }
/// The caller must own this slot's task lock.
pub(crate) unsafe fn is_present(&self) -> bool {
// SAFETY: Forwarded from the caller; projection does not materialize TaskStorage.
unsafe { std::ptr::addr_of!((*self.as_ptr()).present).read() }
}
pub(crate) fn get_mut_exclusive(&mut self) -> &mut TaskStorage {
self.0.get_mut()
/// Marks a detached resident-map value so stale protected readers retry their lookup.
///
/// # Safety
/// The caller must own this slot's task lock and must have detached this exact map value.
pub(crate) unsafe fn mark_removed(&self) {
// SAFETY: Forwarded from the caller; projection does not materialize TaskStorage.
unsafe { std::ptr::addr_of_mut!((*self.as_ptr()).present).write(false) };
}
pub(crate) fn into_inner(self) -> TaskStorage {
self.0.into_inner()
/// # Safety
/// The caller must own this slot's task lock and have observed `is_present()`.
pub(crate) unsafe fn get(&self) -> &TaskStorage {
// SAFETY: Forwarded from the caller.
unsafe { &*self.as_ptr() }
}
#[cfg(test)]
......@@ -147,24 +160,10 @@ impl TaskSlot {
// SAFETY: Reading the raw lock state does not expose the task payload.
unsafe { (&*std::ptr::addr_of!((*self.as_ptr()).lock)).is_locked() }
}
pub(crate) fn with_lock<R>(&self, f: impl FnOnce(&TaskStorage) -> R) -> R {
self.lock();
struct Unlock<'a>(&'a TaskSlot);
impl Drop for Unlock<'_> {
fn drop(&mut self) {
// SAFETY: `with_lock` acquired this lock and the closure has returned/unwound.
unsafe { self.0.unlock() };
}
}
let _unlock = Unlock(self);
// SAFETY: The lock-owning lexical guard remains alive through the closure.
f(unsafe { self.get() })
}
}
// SAFETY: all shared payload access is governed by the protocol above. Structural mutation uses
// `&mut TaskSlot`, while ordinary mutation owns the embedded task lock.
// SAFETY: all shared payload access is governed by the protocol above and owns the embedded task
// lock.
unsafe impl Sync for TaskSlot {}
/// The complete task storage schema.
......@@ -1967,9 +1966,14 @@ mod tests {
slot.lock();
assert!(slot.is_locked());
// SAFETY: This test holds the slot lock until after cloning.
assert!(unsafe { slot.is_present() });
let snapshot = unsafe { slot.get() }.clone_snapshot();
assert!(slot.is_locked());
assert!(!snapshot.lock.is_locked());
assert!(
!snapshot.present,
"snapshot copies are not resident map slots"
);
// SAFETY: This test acquired the source lock above and accesses it no further.
unsafe { slot.unlock() };
}
......
#[cfg(test)]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{
marker::PhantomData,
ops::{Deref, DerefMut},
ptr,
};
use papaya::{HashMap, LocalGuard};
use rustc_hash::FxBuildHasher;
use turbo_tasks::TaskId;
use crate::backend::storage_schema::{TaskSlot, TaskStorage};
/// Sparse resident task map backed by Papaya.
///
/// Papaya pins keep entries alive across concurrent removal. `TaskSlot`'s intrusive lock provides
/// payload exclusion. A lookup that loaded a value before removal locks it and observes its
/// lock-protected `present` byte; removal clears that byte before unlock, so the stale reader
/// retries without a second map probe.
pub(crate) struct TaskMap {
map: HashMap<TaskId, TaskSlot, FxBuildHasher>,
#[cfg(test)]
task_id_scans: AtomicUsize,
}
impl TaskMap {
pub(crate) fn new(capacity: usize) -> Self {
// DashMap's million-entry preallocation is split across many shards. Applying it to one
// contiguous Papaya table hurts small working sets; start modestly and let Papaya resize.
let capacity = capacity.min(16 * 1024);
Self {
map: HashMap::with_capacity_and_hasher(capacity, FxBuildHasher),
#[cfg(test)]
task_id_scans: AtomicUsize::new(0),
}
}
pub(crate) fn len(&self) -> usize {
self.map.len()
}
pub(crate) fn get(&self, task_id: TaskId) -> Option<TaskMapGuard<'_>> {
self.get_inner(task_id, false)
}
pub(crate) fn get_or_insert(&self, task_id: TaskId) -> TaskMapGuard<'_> {
self.get(task_id).unwrap_or_else(|| {
self.get_inner(task_id, true)
.expect("get-or-insert must return the inserted task")
})
}
fn get_inner(&self, task_id: TaskId, insert: bool) -> Option<TaskMapGuard<'_>> {
loop {
let guard = self.map.guard();
let slot = if insert {
self.map.get_or_insert_with(task_id, TaskSlot::new, &guard)
} else {
self.map.get(&task_id, &guard)?
} as *const TaskSlot;
// The Papaya guard keeps this slot allocated while its task lock is acquired.
unsafe { &*slot }.lock();
// Removal may have detached this slot while we waited. It clears `present` under this
// same lock before releasing it, so a stale reader can retry without another hash
// probe.
if unsafe { (&*slot).is_present() } {
return Some(TaskMapGuard {
map: self,
guard,
task_id,
slot,
_not_send: PhantomData,
});
}
// SAFETY: We acquired this exact slot's task lock above and expose no task reference.
unsafe { (&*slot).unlock() };
}
}
pub(crate) fn task_ids(&self) -> Vec<TaskId> {
#[cfg(test)]
self.task_id_scans.fetch_add(1, Ordering::Relaxed);
let guard = self.map.guard();
self.map.iter(&guard).map(|(key, _)| *key).collect()
}
/// Process a batch of still-present tasks with one Papaya protection guard. Each callback owns
/// only the current task's intrusive lock; no task reference escapes the call.
pub(crate) fn for_each_locked(
&self,
task_ids: &[TaskId],
mut process: impl FnMut(TaskId, &TaskStorage),
) {
struct Unlock<'a>(&'a TaskSlot);
impl Drop for Unlock<'_> {
fn drop(&mut self) {
// SAFETY: The loop acquired this exact slot lock before constructing the guard.
unsafe { self.0.unlock() };
}
}
let guard = self.map.guard();
for &task_id in task_ids {
let Some(slot) = self.map.get(&task_id, &guard) else {
continue;
};
slot.lock();
let _unlock = Unlock(slot);
// SAFETY: `_unlock` owns the slot lock. Removed entries have `present` cleared before
// unlock, so stale protected values are skipped.
if unsafe { slot.is_present() } {
process(task_id, unsafe { slot.get() });
}
}
}
#[cfg(test)]
pub(crate) fn task_id_scan_count(&self) -> usize {
self.task_id_scans.load(Ordering::Relaxed)
}
#[cfg(test)]
pub(crate) fn contains(&self, task_id: TaskId) -> bool {
let guard = self.map.guard();
self.map.get(&task_id, &guard).is_some()
}
#[cfg(test)]
pub(crate) fn remove(&self, task_id: TaskId) -> bool {
self.get(task_id)
.is_some_and(|task| self.remove_locked(&task))
}
pub(crate) fn remove_locked(&self, task: &TaskMapGuard<'_>) -> bool {
debug_assert!(ptr::eq(self, task.map));
let removed = self
.map
.remove_if(
&task.task_id,
|_, current| ptr::eq(current, task.slot),
&task.guard,
)
.is_ok_and(|removed| removed.is_some());
if removed {
// SAFETY: `task` owns this exact slot's lock, and remove_if detached this exact value.
unsafe { (&*task.slot).mark_removed() };
}
removed
}
pub(crate) fn clear(&self) {
for task_id in self.task_ids() {
if let Some(task) = self.get(task_id) {
self.remove_locked(&task);
}
}
}
}
/// Owns both Papaya's lifetime protection and one task's intrusive lock.
pub(crate) struct TaskMapGuard<'a> {
map: &'a TaskMap,
// Declared before `slot` only for clarity; `Drop` releases the task lock explicitly before
// Papaya protection is dropped.
guard: LocalGuard<'a>,
task_id: TaskId,
slot: *const TaskSlot,
// Task guards must not cross `.await`, even if Papaya changes LocalGuard's auto traits.
_not_send: PhantomData<*const ()>,
}
impl TaskMapGuard<'_> {
pub(crate) fn key(&self) -> &TaskId {
&self.task_id
}
}
impl Deref for TaskMapGuard<'_> {
type Target = TaskStorage;
fn deref(&self) -> &Self::Target {
// SAFETY: TaskMapGuard owns this slot's task lock and observed it present.
unsafe { (&*self.slot).get() }
}
}
impl DerefMut for TaskMapGuard<'_> {
fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY: TaskMapGuard exclusively owns this slot's task lock and observed it present.
unsafe { &mut *(&*self.slot).as_ptr() }
}
}
impl Drop for TaskMapGuard<'_> {
fn drop(&mut self) {
// SAFETY: This guard acquired this slot's task lock and performs no payload access after
// unlocking. `guard` remains alive until after this Drop body, so the slot cannot be freed.
unsafe { (&*self.slot).unlock() };
}
}
#[cfg(test)]
mod tests {
use turbo_tasks::TaskId;
use super::{TaskMap, TaskSlot};
fn task_id(id: u32) -> TaskId {
TaskId::new(id).unwrap()
}
#[test]
fn removed_guard_stays_alive_but_cannot_replace_reinserted_task() {
let map = TaskMap::new(16);
let id = task_id(1);
let old = map.get_or_insert(id);
let stale_guard = map.map.guard();
let stale_slot = map.map.get(&id, &stale_guard).unwrap() as *const TaskSlot;
assert_eq!(stale_slot, old.slot);
assert!(map.remove_locked(&old));
let replacement = map.get_or_insert(id);
assert_ne!(stale_slot, replacement.slot);
assert_eq!(*old.key(), id);
assert_eq!(*replacement.key(), id);
assert!(!map.remove_locked(&old));
drop(old);
// This simulates a reader that protected the old entry before removal but reached its task
// lock only after the replacement was published.
unsafe { &*stale_slot }.lock();
// SAFETY: This test owns the stale slot's lock, and stale_guard keeps it allocated.
assert!(!unsafe { (&*stale_slot).is_present() });
// SAFETY: This test acquired the stale slot lock above.
unsafe { (&*stale_slot).unlock() };
drop(stale_guard);
assert!(map.remove_locked(&replacement));
assert_eq!(map.len(), 0);
}
}
......@@ -55,43 +55,3 @@ pub fn with_entry_in_shard<K: Eq + Hash, V, S: BuildHasher + Clone, Q: ?Sized, R
);
then(entry, query)
}
/// Outcome of [`try_lock_and_remove`].
pub enum TryLockAndRemove {
/// The shard lock was acquired and a matching entry was removed.
Removed,
/// The shard lock was acquired but no matching entry was present.
NotFound,
/// The shard lock was contended; the caller should retry later after releasing
/// any other locks they are holding.
WouldBlock,
}
/// Remove `key` from `map` without blocking on shard contention.
///
/// Intended for call sites that already hold another lock and want to avoid a
/// cyclic wait. On contention (`WouldBlock`), the caller is expected to defer the
/// removal and retry after dropping the other lock.
pub fn try_lock_and_remove<
K: Eq + Hash + AsRef<Q>,
V,
Q: Eq + Hash + ?Sized,
S: BuildHasher + Clone,
>(
map: &DashMap<K, V, S>,
key: &Q,
) -> TryLockAndRemove {
let hasher = map.hasher();
let hash = hasher.hash_one(key);
let shard_idx = map.determine_shard(hash as usize);
let Some(mut shard) = map.shards()[shard_idx].try_write() else {
return TryLockAndRemove::WouldBlock;
};
match shard.find_entry(hash, |(k, _v)| k.as_ref() == key) {
Ok(entry) => {
entry.remove();
TryLockAndRemove::Removed
}
Err(_) => TryLockAndRemove::NotFound,
}
}
use std::{
hash::{BuildHasher, Hash},
ops::Deref,
ptr::NonNull,
sync::Arc,
};
use dashmap::{DashMap, RawRwLock};
use hashbrown::HashTable;
use parking_lot::lock_api::{RwLockReadGuard, RwLockWriteGuard};
type ReadTableGuard<'a, K, V> = RwLockReadGuard<'a, RawRwLock, HashTable<(K, V)>>;
/// A read reference to one DashMap entry.
///
/// The shared variant lets two disjoint entries in the same shard share one read guard. This
/// avoids recursively acquiring a writer-preferring shard lock while retaining stable entry
/// addresses for each task's intrusive lock.
pub enum Ref<'a, K, V> {
Base(dashmap::mapref::one::Ref<'a, K, V>),
Simple {
_guard: ReadTableGuard<'a, K, V>,
entry: NonNull<(K, V)>,
},
Shared {
_guard: Arc<ReadTableGuard<'a, K, V>>,
entry: NonNull<(K, V)>,
},
}
// SAFETY: Each entry pointer remains valid under its shard read guard. Shared references never
// mutate the HashTable entry; payload mutation is independently synchronized by its intrusive
// lock. `K: Sync + V: Sync` makes shared references safe across threads.
unsafe impl<K: Eq + Hash + Sync, V: Sync> Sync for Ref<'_, K, V> {}
impl<K: Eq + Hash, V> Ref<'_, K, V> {
pub fn key(&self) -> &K {
self.pair().0
}
pub fn value(&self) -> &V {
self.pair().1
}
fn pair(&self) -> (&K, &V) {
match self {
Self::Base(reference) => reference.pair(),
Self::Simple { entry, .. } | Self::Shared { entry, .. } => {
// SAFETY: The entry remains valid while the corresponding shard read guard lives.
let entry = unsafe { entry.as_ref() };
(&entry.0, &entry.1)
}
}
}
}
impl<K: Eq + Hash, V> Deref for Ref<'_, K, V> {
type Target = V;
fn deref(&self) -> &Self::Target {
self.value()
}
}
impl<'a, K, V> From<dashmap::mapref::one::Ref<'a, K, V>> for Ref<'a, K, V>
where
K: Hash + Eq,
{
fn from(reference: dashmap::mapref::one::Ref<'a, K, V>) -> Self {
Self::Base(reference)
}
}
/// Get two disjoint read references, inserting missing values under shard write locks.
///
/// The caller must order calls to this function consistently when composing it with other locks.
pub fn get_disjoint<K, V>(
map: &DashMap<K, V, impl BuildHasher + Clone>,
key1: K,
key2: K,
insert_with: impl Fn() -> V,
) -> (Ref<'_, K, V>, Ref<'_, K, V>)
where
K: Hash + Eq + Clone,
{
assert!(
key1 != key2,
"`get_disjoint` was called with equal keys, which cannot produce disjoint task guards"
);
let hasher = map.hasher();
let hash_entry = |entry: &(K, _)| hasher.hash_one(&entry.0);
let hash1 = hasher.hash_one(&key1);
let hash2 = hasher.hash_one(&key2);
let shard1 = map.determine_shard(hash1 as usize);
let shard2 = map.determine_shard(hash2 as usize);
let shards = map.shards();
let find1 = |entry: &(K, _)| key1.eq(&entry.0);
let find2 = |entry: &(K, _)| key2.eq(&entry.0);
if shard1 == shard2 {
let guard = shards[shard1].read();
let entry1 = guard.find(hash1, find1).map(NonNull::from);
let entry2 = guard.find(hash2, find2).map(NonNull::from);
if let (Some(entry1), Some(entry2)) = (entry1, entry2) {
return shared_pair(guard, entry1, entry2);
}
drop(guard);
let mut guard = shards[shard1].write();
if guard.find(hash1, find1).is_none() {
guard.insert_unique(hash1, (key1.clone(), insert_with()), hash_entry);
}
if guard.find(hash2, find2).is_none() {
guard.insert_unique(hash2, (key2.clone(), insert_with()), hash_entry);
}
let entry1 = NonNull::from(guard.find(hash1, find1).expect("first entry was inserted"));
let entry2 = NonNull::from(guard.find(hash2, find2).expect("second entry was inserted"));
return shared_pair(RwLockWriteGuard::downgrade(guard), entry1, entry2);
}
let (first_shard, second_shard, first_is_key1) = if shard1 < shard2 {
(shard1, shard2, true)
} else {
(shard2, shard1, false)
};
let first_guard = shards[first_shard].read();
let second_guard = shards[second_shard].read();
let (guard1, guard2) = if first_is_key1 {
(first_guard, second_guard)
} else {
(second_guard, first_guard)
};
let entry1 = guard1.find(hash1, find1).map(NonNull::from);
let entry2 = guard2.find(hash2, find2).map(NonNull::from);
if let (Some(entry1), Some(entry2)) = (entry1, entry2) {
return (
Ref::Simple {
_guard: guard1,
entry: entry1,
},
Ref::Simple {
_guard: guard2,
entry: entry2,
},
);
}
drop(guard2);
drop(guard1);
let (mut guard1, mut guard2) = loop {
{
let guard1 = shards[shard1].write();
if let Some(guard2) = shards[shard2].try_write() {
break (guard1, guard2);
}
}
{
let guard2 = shards[shard2].write();
if let Some(guard1) = shards[shard1].try_write() {
break (guard1, guard2);
}
}
};
if guard1.find(hash1, find1).is_none() {
guard1.insert_unique(hash1, (key1.clone(), insert_with()), hash_entry);
}
if guard2.find(hash2, find2).is_none() {
guard2.insert_unique(hash2, (key2.clone(), insert_with()), hash_entry);
}
let entry1 = NonNull::from(guard1.find(hash1, find1).expect("first entry was inserted"));
let entry2 = NonNull::from(
guard2
.find(hash2, find2)
.expect("second entry was inserted"),
);
(
Ref::Simple {
_guard: RwLockWriteGuard::downgrade(guard1),
entry: entry1,
},
Ref::Simple {
_guard: RwLockWriteGuard::downgrade(guard2),
entry: entry2,
},
)
}
fn shared_pair<'a, K, V>(
guard: ReadTableGuard<'a, K, V>,
entry1: NonNull<(K, V)>,
entry2: NonNull<(K, V)>,
) -> (Ref<'a, K, V>, Ref<'a, K, V>) {
let guard = Arc::new(guard);
(
Ref::Shared {
_guard: guard.clone(),
entry: entry1,
},
Ref::Shared {
_guard: guard,
entry: entry2,
},
)
}
#[cfg(test)]
mod tests {
use std::thread::scope;
use rand::prelude::SliceRandom;
use turbo_tasks::FxDashMap;
use super::get_disjoint;
// The old writer-guard stress test is still useful as a check for concurrent insertions and
// same-shard pairs, but the shared-reference API no longer mutates the values it returns.
#[cfg(not(miri))]
#[test]
#[cfg_attr(
target_family = "wasm",
ignore = "stalls indefinitely under the Worker-based WASI host; tracked separately"
)]
fn stress_deadlock() {
const N: usize = 100_000;
const THREADS: usize = 20;
let map = FxDashMap::with_hasher_and_shard_amount(Default::default(), 4);
let indices = (0..THREADS)
.map(|_| {
let mut values = (0..N).collect::<Vec<_>>();
values.shuffle(&mut rand::rng());
values
})
.collect::<Vec<_>>();
scope(|scope| {
for indices in indices {
let map = &map;
scope.spawn(move || {
for i in indices {
let (a, b) = get_disjoint(map, i, i + 1, || 0);
assert_eq!(*a, 0);
assert_eq!(*b, 0);
}
});
}
});
assert_eq!(map.len(), N + 1);
}
}
pub mod dash_map_drop_contents;
pub mod dash_map_entry;
pub mod dash_map_multi;
pub mod markdown_table;
pub mod ptr_eq_arc;
pub mod shard_amount;
pub mod stopwatch;
pub mod swap_retain;
#[cfg(test)]
......
use turbo_tasks::parallel::available_parallelism;
/// Compute a good number of shards to use for sharded data structures.
/// The number of shards is computed based on the number of worker threads
/// and whether a small preallocation is requested.
/// A small preallocation is useful for tests where performance is not
/// critical and we want to reduce memory usage and startup time.
/// The number of shards is chosen to minimize the probability of shard
/// collisions (which can lead to false sharing) while keeping memory
/// usage reasonable.
/// The returned number is always a power of two as this is often required
/// by sharded data structures. The maximum shard amount is capped at 1 << 16 (65536).
pub fn compute_shard_amount(num_workers: Option<usize>, small_preallocation: bool) -> usize {
let num_workers = num_workers
.unwrap_or_else(|| available_parallelism().map_or(4, |v| v.get()))
.max(4);
// Once can compute the probability of a shard collision (which leads to false sharing) using
// the birthday paradox formula. It's notable that the probability of collisions increases
// with more worker threads. To mitigate this effect, the number of shards need to grow
// quadratically with the number of worker threads. This way the probability of at least one
// collision remains constant.
//
// Lets call the worker thread count `N` and the number of shards `S`. When using `S = k * N^2`
// for some constant `k` the probability of at least one collision for large `N` can be
// approximated as: P = 1 - exp(-N^2 / (2*S)) = 1 - exp(-1/(2*k))
//
// For `k = 2` this results in a collision probability of about 22%. Fine-grained task locks
// keep a shard collision from serializing independent task mutations, so the lower shard count
// saves memory without recreating the old shard-write-lock bottleneck.
//
// We clamp the number of shards to 1 << 16 to avoid excessive memory usage in case of a very
// high number of worker threads. With `k = 2`, this case is hit above roughly 181 workers.
if small_preallocation {
// We also clamp the maximum number of workers to 256 so all following multiplications can't
// overflow.
let num_workers = num_workers.min(256);
(num_workers * num_workers).next_power_of_two()
} else {
// Clamp the worker count so the shard total stays at or below 1 << 16.
let num_workers = num_workers.min(181);
(num_workers * num_workers * 2).next_power_of_two()
}
}
......@@ -1408,6 +1408,8 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#lazy_field
#[doc = "Per-task lock used with a shared resident-map guard."]
lock: IntrusiveTaskLock,
#[doc = "Whether this allocation is still the resident-map value for its TaskId."]
present: bool,
}
#[automatically_derived]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment