Unverified Commit 0221ecdc authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Use reclaimable dense task storage

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 48a01714
...@@ -724,6 +724,12 @@ dependencies = [ ...@@ -724,6 +724,12 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "boxcar"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e"
[[package]] [[package]]
name = "brotli-decompressor" name = "brotli-decompressor"
version = "2.3.4" version = "2.3.4"
...@@ -3483,6 +3489,16 @@ dependencies = [ ...@@ -3483,6 +3489,16 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "kovan"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "069314825b47e60f2882bf6fc90ab66e3eb37ee5e4407de6357a124842fc3b26"
dependencies = [
"once_cell",
"portable-atomic",
]
[[package]] [[package]]
name = "kqueue" name = "kqueue"
version = "1.2.1" version = "1.2.1"
...@@ -5129,9 +5145,9 @@ dependencies = [ ...@@ -5129,9 +5145,9 @@ dependencies = [
[[package]] [[package]]
name = "portable-atomic" name = "portable-atomic"
version = "1.11.0" version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
[[package]] [[package]]
name = "portable-atomic-util" name = "portable-atomic-util"
...@@ -9208,6 +9224,7 @@ dependencies = [ ...@@ -9208,6 +9224,7 @@ dependencies = [
"auto-hash-map", "auto-hash-map",
"bincode", "bincode",
"bitfield", "bitfield",
"boxcar",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"crossbeam-utils", "crossbeam-utils",
"dashmap 7.0.0-rc2", "dashmap 7.0.0-rc2",
...@@ -9217,6 +9234,7 @@ dependencies = [ ...@@ -9217,6 +9234,7 @@ dependencies = [
"indexmap 2.14.0", "indexmap 2.14.0",
"indoc", "indoc",
"jiff", "jiff",
"kovan",
"lz4_flex", "lz4_flex",
"parking_lot", "parking_lot",
"rand 0.10.1", "rand 0.10.1",
......
...@@ -254,6 +254,7 @@ async-trait = "0.1.64" ...@@ -254,6 +254,7 @@ async-trait = "0.1.64"
base64 = "0.22.1" base64 = "0.22.1"
bincode = { version = "2.0.1", features = ["serde"] } bincode = { version = "2.0.1", features = ["serde"] }
bitfield = "0.19.4" bitfield = "0.19.4"
boxcar = "0.2.14"
bumpalo = { version = "3.19.0", features = ["boxed", "collections", "allocator-api2"] } bumpalo = { version = "3.19.0", features = ["boxed", "collections", "allocator-api2"] }
byteorder = "1.5.0" byteorder = "1.5.0"
bytes = "1.11.1" bytes = "1.11.1"
...@@ -285,6 +286,7 @@ image = { version = "0.25.8", default-features = false } ...@@ -285,6 +286,7 @@ image = { version = "0.25.8", default-features = false }
indexmap = "2.14.0" indexmap = "2.14.0"
indoc = "2.0.0" indoc = "2.0.0"
itertools = "0.10.5" itertools = "0.10.5"
kovan = "0.1.20"
lightningcss = { version = "1.0.0-alpha.72", features = [ lightningcss = { version = "1.0.0-alpha.72", features = [
"serde", "serde",
"visitor", "visitor",
......
...@@ -41,12 +41,14 @@ anyhow = { workspace = true } ...@@ -41,12 +41,14 @@ anyhow = { workspace = true }
bincode = { workspace = true } bincode = { workspace = true }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
bitfield = { workspace = true } bitfield = { workspace = true }
boxcar = { workspace = true }
crossbeam-utils = { workspace = true } crossbeam-utils = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]} dashmap = { workspace = true, features = ["raw-api"]}
fs-err = { workspace = true } fs-err = { workspace = true }
hashbrown = { workspace = true } hashbrown = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
kovan = { workspace = true }
lz4_flex = { workspace = true, optional = true } lz4_flex = { workspace = true, optional = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
rand = { workspace = true } rand = { workspace = true }
......
...@@ -5,6 +5,7 @@ use criterion::{Criterion, criterion_group, criterion_main}; ...@@ -5,6 +5,7 @@ use criterion::{Criterion, criterion_group, criterion_main};
pub(crate) mod gc; pub(crate) mod gc;
pub(crate) mod overhead; pub(crate) mod overhead;
pub(crate) mod resident_storage;
pub(crate) mod scope_stress; pub(crate) mod scope_stress;
pub(crate) mod storage_locking; pub(crate) mod storage_locking;
pub(crate) mod stress; pub(crate) mod stress;
...@@ -12,6 +13,6 @@ pub(crate) mod stress; ...@@ -12,6 +13,6 @@ pub(crate) mod stress;
criterion_group!( criterion_group!(
name = turbo_tasks_backend_stress; name = turbo_tasks_backend_stress;
config = Criterion::default(); config = Criterion::default();
targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc, storage_locking::storage_locking targets = stress::fibonacci, scope_stress::scope_stress, overhead::overhead, gc::gc, storage_locking::storage_locking, resident_storage::resident_storage
); );
criterion_main!(turbo_tasks_backend_stress); criterion_main!(turbo_tasks_backend_stress);
...@@ -24,9 +24,15 @@ type InnerMap<K, V, const I: usize> = AutoMap<K, V, BuildHasherDefault<FxHasher> ...@@ -24,9 +24,15 @@ type InnerMap<K, V, const I: usize> = AutoMap<K, V, BuildHasherDefault<FxHasher>
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct CounterMap<K, V, const I: usize>(InnerMap<K, V, I>); pub struct CounterMap<K, V, const I: usize>(InnerMap<K, V, I>);
impl<K, V, const I: usize> CounterMap<K, V, I> {
pub const fn new() -> Self {
Self(InnerMap::with_hasher())
}
}
impl<K, V, const I: usize> Default for CounterMap<K, V, I> { impl<K, V, const I: usize> Default for CounterMap<K, V, I> {
fn default() -> Self { fn default() -> Self {
Self(InnerMap::default()) Self::new()
} }
} }
...@@ -84,11 +90,6 @@ impl CounterValue for i32 { ...@@ -84,11 +90,6 @@ impl CounterValue for i32 {
} }
impl<K, V, const I: usize> CounterMap<K, V, I> { impl<K, V, const I: usize> CounterMap<K, V, I> {
#[cfg(test)]
pub fn new() -> Self {
Self(AutoMap::default())
}
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.0.is_empty() self.0.is_empty()
} }
......
mod cell_data; mod cell_data;
mod counter_map; mod counter_map;
mod dense_task_map;
mod eviction; mod eviction;
mod gc; mod gc;
mod operation; mod operation;
......
use std::{
hash::{BuildHasher, Hash},
ops::Deref,
ptr::NonNull,
sync::Arc,
};
use dashmap::{DashMap, RawRwLock};
use hashbrown::HashTable;
use parking_lot::lock_api::{RwLockReadGuard, RwLockWriteGuard};
type ReadTableGuard<'a, K, V> = RwLockReadGuard<'a, RawRwLock, HashTable<(K, V)>>;
/// A read reference to one DashMap entry.
///
/// The shared variant lets two disjoint entries in the same shard share one read guard. This
/// avoids recursively acquiring a writer-preferring shard lock while retaining stable entry
/// addresses for each task's intrusive lock.
pub enum Ref<'a, K, V> {
Base(dashmap::mapref::one::Ref<'a, K, V>),
Simple {
_guard: ReadTableGuard<'a, K, V>,
entry: NonNull<(K, V)>,
},
Shared {
_guard: Arc<ReadTableGuard<'a, K, V>>,
entry: NonNull<(K, V)>,
},
}
// SAFETY: Each entry pointer remains valid under its shard read guard. Shared references never
// mutate the HashTable entry; payload mutation is independently synchronized by its intrusive
// lock. `K: Sync + V: Sync` makes shared references safe across threads.
unsafe impl<K: Eq + Hash + Sync, V: Sync> Sync for Ref<'_, K, V> {}
impl<K: Eq + Hash, V> Ref<'_, K, V> {
pub fn key(&self) -> &K {
self.pair().0
}
pub fn value(&self) -> &V {
self.pair().1
}
fn pair(&self) -> (&K, &V) {
match self {
Self::Base(reference) => reference.pair(),
Self::Simple { entry, .. } | Self::Shared { entry, .. } => {
// SAFETY: The entry remains valid while the corresponding shard read guard lives.
let entry = unsafe { entry.as_ref() };
(&entry.0, &entry.1)
}
}
}
}
impl<K: Eq + Hash, V> Deref for Ref<'_, K, V> {
type Target = V;
fn deref(&self) -> &Self::Target {
self.value()
}
}
impl<'a, K, V> From<dashmap::mapref::one::Ref<'a, K, V>> for Ref<'a, K, V>
where
K: Hash + Eq,
{
fn from(reference: dashmap::mapref::one::Ref<'a, K, V>) -> Self {
Self::Base(reference)
}
}
/// Get two disjoint read references, inserting missing values under shard write locks.
///
/// The caller must order calls to this function consistently when composing it with other locks.
pub fn get_disjoint<K, V>(
map: &DashMap<K, V, impl BuildHasher + Clone>,
key1: K,
key2: K,
insert_with: impl Fn() -> V,
) -> (Ref<'_, K, V>, Ref<'_, K, V>)
where
K: Hash + Eq + Clone,
{
assert!(
key1 != key2,
"`get_disjoint` was called with equal keys, which cannot produce disjoint task guards"
);
let hasher = map.hasher();
let hash_entry = |entry: &(K, _)| hasher.hash_one(&entry.0);
let hash1 = hasher.hash_one(&key1);
let hash2 = hasher.hash_one(&key2);
let shard1 = map.determine_shard(hash1 as usize);
let shard2 = map.determine_shard(hash2 as usize);
let shards = map.shards();
let find1 = |entry: &(K, _)| key1.eq(&entry.0);
let find2 = |entry: &(K, _)| key2.eq(&entry.0);
if shard1 == shard2 {
let guard = shards[shard1].read();
let entry1 = guard.find(hash1, find1).map(NonNull::from);
let entry2 = guard.find(hash2, find2).map(NonNull::from);
if let (Some(entry1), Some(entry2)) = (entry1, entry2) {
return shared_pair(guard, entry1, entry2);
}
drop(guard);
let mut guard = shards[shard1].write();
if guard.find(hash1, find1).is_none() {
guard.insert_unique(hash1, (key1.clone(), insert_with()), hash_entry);
}
if guard.find(hash2, find2).is_none() {
guard.insert_unique(hash2, (key2.clone(), insert_with()), hash_entry);
}
let entry1 = NonNull::from(guard.find(hash1, find1).expect("first entry was inserted"));
let entry2 = NonNull::from(guard.find(hash2, find2).expect("second entry was inserted"));
return shared_pair(RwLockWriteGuard::downgrade(guard), entry1, entry2);
}
let (first_shard, second_shard, first_is_key1) = if shard1 < shard2 {
(shard1, shard2, true)
} else {
(shard2, shard1, false)
};
let first_guard = shards[first_shard].read();
let second_guard = shards[second_shard].read();
let (guard1, guard2) = if first_is_key1 {
(first_guard, second_guard)
} else {
(second_guard, first_guard)
};
let entry1 = guard1.find(hash1, find1).map(NonNull::from);
let entry2 = guard2.find(hash2, find2).map(NonNull::from);
if let (Some(entry1), Some(entry2)) = (entry1, entry2) {
return (
Ref::Simple {
_guard: guard1,
entry: entry1,
},
Ref::Simple {
_guard: guard2,
entry: entry2,
},
);
}
drop(guard2);
drop(guard1);
let (mut guard1, mut guard2) = loop {
{
let guard1 = shards[shard1].write();
if let Some(guard2) = shards[shard2].try_write() {
break (guard1, guard2);
}
}
{
let guard2 = shards[shard2].write();
if let Some(guard1) = shards[shard1].try_write() {
break (guard1, guard2);
}
}
};
if guard1.find(hash1, find1).is_none() {
guard1.insert_unique(hash1, (key1.clone(), insert_with()), hash_entry);
}
if guard2.find(hash2, find2).is_none() {
guard2.insert_unique(hash2, (key2.clone(), insert_with()), hash_entry);
}
let entry1 = NonNull::from(guard1.find(hash1, find1).expect("first entry was inserted"));
let entry2 = NonNull::from(
guard2
.find(hash2, find2)
.expect("second entry was inserted"),
);
(
Ref::Simple {
_guard: RwLockWriteGuard::downgrade(guard1),
entry: entry1,
},
Ref::Simple {
_guard: RwLockWriteGuard::downgrade(guard2),
entry: entry2,
},
)
}
fn shared_pair<'a, K, V>(
guard: ReadTableGuard<'a, K, V>,
entry1: NonNull<(K, V)>,
entry2: NonNull<(K, V)>,
) -> (Ref<'a, K, V>, Ref<'a, K, V>) {
let guard = Arc::new(guard);
(
Ref::Shared {
_guard: guard.clone(),
entry: entry1,
},
Ref::Shared {
_guard: guard,
entry: entry2,
},
)
}
#[cfg(test)]
mod tests {
use std::thread::scope;
use rand::prelude::SliceRandom;
use turbo_tasks::FxDashMap;
use super::get_disjoint;
// The old writer-guard stress test is still useful as a check for concurrent insertions and
// same-shard pairs, but the shared-reference API no longer mutates the values it returns.
#[cfg(not(miri))]
#[test]
#[cfg_attr(
target_family = "wasm",
ignore = "stalls indefinitely under the Worker-based WASI host; tracked separately"
)]
fn stress_deadlock() {
const N: usize = 100_000;
const THREADS: usize = 20;
let map = FxDashMap::with_hasher_and_shard_amount(Default::default(), 4);
let indices = (0..THREADS)
.map(|_| {
let mut values = (0..N).collect::<Vec<_>>();
values.shuffle(&mut rand::rng());
values
})
.collect::<Vec<_>>();
scope(|scope| {
for indices in indices {
let map = &map;
scope.spawn(move || {
for i in indices {
let (a, b) = get_disjoint(map, i, i + 1, || 0);
assert_eq!(*a, 0);
assert_eq!(*b, 0);
}
});
}
});
assert_eq!(map.len(), N + 1);
}
}
pub mod dash_map_drop_contents; pub mod dash_map_drop_contents;
pub mod dash_map_entry; pub mod dash_map_entry;
pub mod dash_map_multi;
pub mod markdown_table; pub mod markdown_table;
pub mod ptr_eq_arc; pub mod ptr_eq_arc;
pub mod shard_amount; pub mod shard_amount;
......
...@@ -1102,6 +1102,11 @@ fn generate_task_flags_bitfield(grouped_fields: &GroupedFields) -> TokenStream { ...@@ -1102,6 +1102,11 @@ fn generate_task_flags_bitfield(grouped_fields: &GroupedFields) -> TokenStream {
#[doc = "Mask for all persisted flags (meta + data)"] #[doc = "Mask for all persisted flags (meta + data)"]
pub const PERSISTED_MASK: u16 = #persisted_mask; pub const PERSISTED_MASK: u16 = #persisted_mask;
#[doc = "Construct empty task flags in a const context"]
pub const fn empty() -> Self {
Self(0)
}
#[doc = "Get the raw bits value"] #[doc = "Get the raw bits value"]
pub fn bits(&self) -> u16 { pub fn bits(&self) -> u16 {
self.0 self.0
...@@ -1359,12 +1364,14 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1359,12 +1364,14 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
// Collect all field definitions from both categories // Collect all field definitions from both categories
let mut field_defs = Vec::new(); let mut field_defs = Vec::new();
let mut field_names = Vec::new();
// Add inline fields directly on TaskStorage (private - use accessor methods) // Add inline fields directly on TaskStorage (private - use accessor methods)
// Note: No bincode attributes since we don't derive Encode/Decode (manual serialization) // Note: No bincode attributes since we don't derive Encode/Decode (manual serialization)
for field in grouped_fields.all_inline() { for field in grouped_fields.all_inline() {
let field_name = &field.field_name; let field_name = &field.field_name;
let field_type = &field.field_type; let field_type = &field.field_type;
field_names.push(field_name.clone());
field_defs.push(quote! { field_defs.push(quote! {
#field_name: #field_type #field_name: #field_type
}); });
...@@ -1390,6 +1397,19 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1390,6 +1397,19 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
quote! {} quote! {}
}; };
let swap_flags = has_flags.then(|| {
quote! { std::mem::swap(&mut self.flags, &mut detached.flags); }
});
let swap_lazy = has_lazy.then(|| {
quote! { std::mem::swap(&mut self.lazy, &mut detached.lazy); }
});
let reset_flags = has_flags.then(|| {
quote! { self.flags = Default::default(); }
});
let reset_lazy = has_lazy.then(|| {
quote! { self.lazy = Default::default(); }
});
// Note: Helper methods like find_lazy, find_lazy_mut, get_or_create_lazy, and // Note: Helper methods like find_lazy, find_lazy_mut, get_or_create_lazy, and
// remove_if_empty are defined in storage_schema.rs rather than generated here. // remove_if_empty are defined in storage_schema.rs rather than generated here.
// This provides better IDE support (autocomplete, go-to-definition, etc.). // This provides better IDE support (autocomplete, go-to-definition, etc.).
...@@ -1406,15 +1426,40 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1406,15 +1426,40 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#(#field_defs,)* #(#field_defs,)*
#flags_field #flags_field
#lazy_field #lazy_field
#[doc = "Per-task lock used with a shared resident-map guard."] #[doc = "Intrusive lock protecting this stable task slot"]
lock: IntrusiveTaskLock, pub(crate) lock: IntrusiveTaskLock,
#[doc = "Authoritative slot presence, accessed only while `lock` is held"]
pub(crate) occupied: bool,
} }
#[automatically_derived] #[automatically_derived]
impl TaskStorage { impl TaskStorage {
#[doc = "Constructs fresh task payload with unlocked, vacant slot metadata."]
pub fn new() -> Self { pub fn new() -> Self {
Self::default() Self::default()
} }
#[doc = "Moves payload into a fresh unlocked value and leaves this locked slot vacant."]
#[doc = "The caller must hold this task's intrusive lock."]
pub(crate) fn take_and_vacate(&mut self) -> Self {
debug_assert!(self.occupied);
let mut detached = Self::default();
#(std::mem::swap(&mut self.#field_names, &mut detached.#field_names);)*
#swap_flags
#swap_lazy
self.occupied = false;
detached
}
#[doc = "Drops and resets payload in place while preserving the held mutex."]
#[doc = "The caller must hold this task's intrusive lock."]
pub(crate) fn vacate_in_place(&mut self) {
debug_assert!(self.occupied);
#(self.#field_names = Default::default();)*
#reset_flags
#reset_lazy
self.occupied = false;
}
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment