Unverified Commit 1370f484 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Use sparse pointer pages for task storage

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 7f97a3e7
...@@ -3489,16 +3489,6 @@ dependencies = [ ...@@ -3489,16 +3489,6 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "kovan"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "069314825b47e60f2882bf6fc90ab66e3eb37ee5e4407de6357a124842fc3b26"
dependencies = [
"once_cell",
"portable-atomic",
]
[[package]] [[package]]
name = "kqueue" name = "kqueue"
version = "1.2.1" version = "1.2.1"
...@@ -9234,7 +9224,6 @@ dependencies = [ ...@@ -9234,7 +9224,6 @@ dependencies = [
"indexmap 2.14.0", "indexmap 2.14.0",
"indoc", "indoc",
"jiff", "jiff",
"kovan",
"lz4_flex", "lz4_flex",
"parking_lot", "parking_lot",
"rand 0.10.1", "rand 0.10.1",
...@@ -9246,6 +9235,7 @@ dependencies = [ ...@@ -9246,6 +9235,7 @@ dependencies = [
"serde_json", "serde_json",
"serde_path_to_error", "serde_path_to_error",
"smallvec", "smallvec",
"static_assertions",
"tempfile", "tempfile",
"thiserror 1.0.69", "thiserror 1.0.69",
"thread_local", "thread_local",
......
...@@ -286,7 +286,6 @@ image = { version = "0.25.8", default-features = false } ...@@ -286,7 +286,6 @@ image = { version = "0.25.8", default-features = false }
indexmap = "2.14.0" indexmap = "2.14.0"
indoc = "2.0.0" indoc = "2.0.0"
itertools = "0.10.5" itertools = "0.10.5"
kovan = "0.1.20"
lightningcss = { version = "1.0.0-alpha.72", features = [ lightningcss = { version = "1.0.0-alpha.72", features = [
"serde", "serde",
"visitor", "visitor",
...@@ -348,6 +347,7 @@ smallvec = { version = "1.15.1", features = [ ...@@ -348,6 +347,7 @@ smallvec = { version = "1.15.1", features = [
"const_new", "const_new",
] } ] }
shrink-to-fit = "0.2.10" shrink-to-fit = "0.2.10"
static_assertions = "1.1.0"
strsim = "0.11.1" strsim = "0.11.1"
swc_sourcemap = "10.0.2" swc_sourcemap = "10.0.2"
syn = "2.0.100" syn = "2.0.100"
......
...@@ -48,7 +48,6 @@ fs-err = { workspace = true } ...@@ -48,7 +48,6 @@ fs-err = { workspace = true }
hashbrown = { workspace = true } hashbrown = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
kovan = { workspace = true }
lz4_flex = { workspace = true, optional = true } lz4_flex = { workspace = true, optional = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
rand = { workspace = true } rand = { workspace = true }
...@@ -83,6 +82,7 @@ triomphe = { workspace = true } ...@@ -83,6 +82,7 @@ triomphe = { workspace = true }
thiserror = { workspace = true } thiserror = { workspace = true }
turbo-tasks-malloc = { workspace = true , features = ["custom_allocator"]} turbo-tasks-malloc = { workspace = true , features = ["custom_allocator"]}
rstest = { workspace = true } rstest = { workspace = true }
static_assertions = { workspace = true }
turbo-tasks-testing = { workspace = true } turbo-tasks-testing = { workspace = true }
# The read-outcome tests assert on the counters, so they need them compiled in. Only for tests — # The read-outcome tests assert on the counters, so they need them compiled in. Only for tests —
# normal builds of this crate get the default (disabled) turbo-tasks. # normal builds of this crate get the default (disabled) turbo-tasks.
......
...@@ -64,9 +64,8 @@ pub enum TtlCounter { ...@@ -64,9 +64,8 @@ pub enum TtlCounter {
/// One unit of GC work. /// One unit of GC work.
enum GcJob { enum GcJob {
/// Scan one shard of the resident map (by index) and enqueue its candidates as /// Scan one live pointer page in either TaskId space.
/// [`GcJob::Collect`]. ScanShard((bool, usize)),
ScanShard(usize),
/// Collect a single task. /// Collect a single task.
Collect(TaskId), Collect(TaskId),
} }
...@@ -240,9 +239,12 @@ impl TurboTasksBackend { ...@@ -240,9 +239,12 @@ impl TurboTasksBackend {
None None
}; };
let access = phase.access_token();
let (mut stats, mut result): (GcStats, GcPassResult) = scope_unbounded_with( let (mut stats, mut result): (GcStats, GcPassResult) = scope_unbounded_with(
// Start by scanning all shards and collecting the aged out roots from prior sessions. // Scan live pages from both ID spaces, as well as aged-out durable roots.
(0..self.storage.shard_count()) self.storage
.shard_indices(access)
.into_iter()
.map(GcJob::ScanShard) .map(GcJob::ScanShard)
.chain(aged_out.into_iter().map(GcJob::Collect)), .chain(aged_out.into_iter().map(GcJob::Collect)),
Default::default, Default::default,
...@@ -256,7 +258,7 @@ impl TurboTasksBackend { ...@@ -256,7 +258,7 @@ impl TurboTasksBackend {
let task_id = match job { let task_id = match job {
GcJob::ScanShard(index) => { GcJob::ScanShard(index) => {
let collector = |task_id| spawner.spawn(GcJob::Collect(task_id)); let collector = |task_id| spawner.spawn(GcJob::Collect(task_id));
self.storage.gc_scan_shard(index, collector); self.storage.gc_scan_shard(access, index, collector);
return ControlFlow::Continue(()); return ControlFlow::Continue(());
} }
GcJob::Collect(task_id) => task_id, GcJob::Collect(task_id) => task_id,
...@@ -347,7 +349,7 @@ impl TurboTasksBackend { ...@@ -347,7 +349,7 @@ impl TurboTasksBackend {
// We don't do this in the GC pass because a task detected as a root 'early' might become a // We don't do this in the GC pass because a task detected as a root 'early' might become a
// non-root later due to other operations (e.g. it might get promoted to a live aggregation // non-root later due to other operations (e.g. it might get promoted to a live aggregation
// root). // root).
for id in self.storage.gc_scan_roots() { for id in self.storage.gc_scan_roots(access) {
roots.insert(id, TtlCounter::MostRecent); roots.insert(id, TtlCounter::MostRecent);
} }
......
mod cell_data; mod cell_data;
mod counter_map; mod counter_map;
mod dense_task_map;
mod eviction; mod eviction;
mod gc; mod gc;
mod operation; mod operation;
mod snapshot_coordinator; mod snapshot_coordinator;
mod storage; mod storage;
pub mod storage_schema; pub mod storage_schema;
mod task_page_map;
// Only the `verify_aggregation_graph` feature still uses atomics here; `stopping` is an // Only the `verify_aggregation_graph` feature still uses atomics here; `stopping` is an
// `RwLock<bool>` so that checking it and acting on it cannot be split (see the field's docs). // `RwLock<bool>` so that checking it and acting on it cannot be split (see the field's docs).
...@@ -374,9 +374,6 @@ impl TurboTasksBackend { ...@@ -374,9 +374,6 @@ impl TurboTasksBackend {
/// shutdown has begun, and blocks shutdown for as long as the returned context is alive. /// shutdown has begun, and blocks shutdown for as long as the returned context is alive.
/// ///
/// Use this for entry points reachable from threads that `stop_and_wait` does **not** drain. /// Use this for entry points reachable from threads that `stop_and_wait` does **not** drain.
///
/// Returns `None` once [`TurboTasksBackend::stopping`] has run, in which case the caller must
/// do nothing: storage teardown is imminent or already underway.
fn try_execute_context<'a>( fn try_execute_context<'a>(
&'a self, &'a self,
turbo_tasks: &'a TurboTasks<TurboTasksBackend>, turbo_tasks: &'a TurboTasks<TurboTasksBackend>,
...@@ -392,11 +389,8 @@ impl TurboTasksBackend { ...@@ -392,11 +389,8 @@ impl TurboTasksBackend {
)) ))
} }
pub(crate) fn start_operation(&self) -> Option<OperationGuard<'_, AnyOperation>> { pub(crate) fn start_operation(&self) -> OperationGuard<'_, AnyOperation> {
if !self.should_persist() { self.snapshot_coord.begin_operation()
return None;
}
Some(self.snapshot_coord.begin_operation())
} }
fn should_persist(&self) -> bool { fn should_persist(&self) -> bool {
...@@ -429,7 +423,7 @@ impl TurboTasksBackend { ...@@ -429,7 +423,7 @@ impl TurboTasksBackend {
return TestSnapshotOutcome::default(); return TestSnapshotOutcome::default();
} }
}; };
let eviction_counts = self.storage.evict_after_snapshot(None); let eviction_counts = self.storage.evict_after_snapshot(&self.snapshot_coord, None);
TestSnapshotOutcome { TestSnapshotOutcome {
had_new_data, had_new_data,
eviction_counts, eviction_counts,
...@@ -440,23 +434,30 @@ impl TurboTasksBackend { ...@@ -440,23 +434,30 @@ impl TurboTasksBackend {
/// The number oftasks resident in the map. /// The number oftasks resident in the map.
#[doc(hidden)] #[doc(hidden)]
pub fn resident_task_count_for_testing(&self) -> usize { pub fn resident_task_count_for_testing(&self) -> usize {
self.storage.resident_task_count_for_testing() let operation = self.start_operation();
self.storage
.resident_task_count_for_testing(operation.access_token())
}
} }
/// The persistent `parent_count` of a resident task (0 if absent or not resident). Test-only /// The persistent `parent_count` of a resident task (0 if absent or not resident). Test-only
/// hook for verifying incremental refcount maintenance. /// hook for verifying incremental refcount maintenance.
#[doc(hidden)] #[doc(hidden)]
pub fn parent_count_for_testing(&self, task: TaskId) -> u32 { pub fn parent_count_for_testing(&self, task: TaskId) -> u32 {
let operation = self.start_operation();
self.storage self.storage
.with_task(task, |t| t.gc_parent_count()) .with_task(operation.access_token(), task, |t| t.gc_parent_count())
.unwrap_or(0) .unwrap_or(0)
} }
/// The transient `transient_ref_count` of a resident task (0 if absent or not resident). /// The transient `transient_ref_count` of a resident task (0 if absent or not resident).
#[doc(hidden)] #[doc(hidden)]
pub fn transient_ref_count_for_testing(&self, task: TaskId) -> u32 { pub fn transient_ref_count_for_testing(&self, task: TaskId) -> u32 {
let operation = self.start_operation();
self.storage self.storage
.with_task(task, |t| t.gc_transient_ref_count()) .with_task(operation.access_token(), task, |t| {
t.gc_transient_ref_count()
})
.unwrap_or(0) .unwrap_or(0)
} }
...@@ -1165,7 +1166,7 @@ impl TurboTasksBackend { ...@@ -1165,7 +1166,7 @@ impl TurboTasksBackend {
// since epoch). Instant is monotonic but has no defined epoch, so it // since epoch). Instant is monotonic but has no defined epoch, so it
// can't be used for cross-process trace correlation. // can't be used for cross-process trace correlation.
let wall_start = SystemTime::now(); let wall_start = SystemTime::now();
let mut snapshot_phase = self.snapshot_coord.begin_snapshot(); let mut snapshot_phase = self.snapshot_coord.begin_exclusion();
let (gc_elapsed, gc_roots_to_persist, gc_outcome) = if self.gc_enabled { let (gc_elapsed, gc_roots_to_persist, gc_outcome) = if self.gc_enabled {
let gc_span = tracing::info_span!( let gc_span = tracing::info_span!(
"gc", "gc",
...@@ -1192,12 +1193,15 @@ impl TurboTasksBackend { ...@@ -1192,12 +1193,15 @@ impl TurboTasksBackend {
debug_assert!(self.should_persist()); debug_assert!(self.should_persist());
// Checking after start_snapshot ensures no concurrent increments can race. // Checking after start_snapshot ensures no concurrent increments can race.
let (snapshot_guard, has_modifications) = self.storage.start_snapshot(); let (snapshot_guard, has_modifications) = self
.storage
.start_snapshot(&self.snapshot_coord, snapshot_phase.access_token());
let suspended_operations = snapshot_phase.take_suspended_operations(); let suspended_operations = snapshot_phase.take_suspended_operations();
let snapshot_time = Instant::now(); let snapshot_time = Instant::now();
drop(snapshot_phase); drop(snapshot_phase);
snapshot_guard.begin_concurrent_phase();
if !has_modifications && gc_roots_to_persist.is_none() { if !has_modifications && gc_roots_to_persist.is_none() {
// No tasks modified since the last snapshot — drop the guard (which // No tasks modified since the last snapshot — drop the guard (which
...@@ -1681,7 +1685,9 @@ impl TurboTasksBackend { ...@@ -1681,7 +1685,9 @@ impl TurboTasksBackend {
{ {
eprintln!("Persisting failed during shutdown: {err:?}"); eprintln!("Persisting failed during shutdown: {err:?}");
} }
self.storage.drop_contents(); let clear_phase = self.snapshot_coord.begin_exclusion();
self.storage.drop_contents(clear_phase.access_token());
drop(clear_phase);
if let Err(err) = self.backing_storage.shutdown() { if let Err(err) = self.backing_storage.shutdown() {
println!("Shutting down failed: {err}"); println!("Shutting down failed: {err}");
} }
...@@ -1836,8 +1842,11 @@ impl TurboTasksBackend { ...@@ -1836,8 +1842,11 @@ impl TurboTasksBackend {
// Initialize storage BEFORE making task_id visible in the cache. // Initialize storage BEFORE making task_id visible in the cache.
// This ensures any thread that reads task_id from the cache sees // This ensures any thread that reads task_id from the cache sees
// the storage entry already initialized (restored flags set). // the storage entry already initialized (restored flags set).
self.storage self.storage.initialize_new_task(
.initialize_new_task(task_id, Some(task_type.clone())); ctx.storage_access_token(),
task_id,
Some(task_type.clone()),
);
entry.insert((task_type, task_id)); entry.insert((task_type, task_id));
(task_id, true) (task_id, true)
} }
...@@ -1939,7 +1948,8 @@ impl TurboTasksBackend { ...@@ -1939,7 +1948,8 @@ impl TurboTasksBackend {
} }
fn debug_get_task_description(&self, task_id: TaskId) -> String { fn debug_get_task_description(&self, task_id: TaskId) -> String {
let task = self.storage.access_mut(task_id); let operation = self.start_operation();
let task = self.storage.access_mut(operation.access_token(), task_id);
if let Some(value) = task.get_persistent_task_type() { if let Some(value) = task.get_persistent_task_type() {
format!("{task_id:?} {}", value) format!("{task_id:?} {}", value)
} else if let Some(value) = task.get_transient_task_type() { } else if let Some(value) = task.get_transient_task_type() {
...@@ -1968,7 +1978,8 @@ impl TurboTasksBackend { ...@@ -1968,7 +1978,8 @@ impl TurboTasksBackend {
} }
fn debug_get_cached_task_type(&self, task_id: TaskId) -> Option<CachedTaskTypeArc> { fn debug_get_cached_task_type(&self, task_id: TaskId) -> Option<CachedTaskTypeArc> {
let task = self.storage.access_mut(task_id); let operation = self.start_operation();
let task = self.storage.access_mut(operation.access_token(), task_id);
task.get_persistent_task_type().cloned() task.get_persistent_task_type().cloned()
} }
...@@ -3159,7 +3170,10 @@ impl TurboTasksBackend { ...@@ -3159,7 +3170,10 @@ impl TurboTasksBackend {
// when enabled we should expect it to reclaim substantial // when enabled we should expect it to reclaim substantial
// memory so racing with execution is as likely to save time as // memory so racing with execution is as likely to save time as
// cost it. // cost it.
self.storage.evict_after_snapshot(background_span.id()); self.storage.evict_after_snapshot(
&self.snapshot_coord,
background_span.id(),
);
true true
} else { } else {
false false
...@@ -3416,7 +3430,8 @@ impl TurboTasksBackend { ...@@ -3416,7 +3430,8 @@ impl TurboTasksBackend {
fn create_transient_task(&self, task_type: TransientTaskType) -> TaskId { fn create_transient_task(&self, task_type: TransientTaskType) -> TaskId {
let task_id = self.transient_task_id_factory.get(); let task_id = self.transient_task_id_factory.get();
{ {
let mut task = self.storage.access_mut(task_id); let operation = self.start_operation();
let mut task = self.storage.access_mut(operation.access_token(), task_id);
task.init_transient_task(task_id, task_type, self.should_track_activeness()); task.init_transient_task(task_id, task_type, self.should_track_activeness());
} }
#[cfg(feature = "verify_aggregation_graph")] #[cfg(feature = "verify_aggregation_graph")]
......
...@@ -25,7 +25,10 @@ use parking_lot::{Condvar, Mutex}; ...@@ -25,7 +25,10 @@ use parking_lot::{Condvar, Mutex};
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use tracing::info_span; use tracing::info_span;
use crate::{backend::AnyOperation, utils::ptr_eq_arc::PtrEqArc}; use crate::{
backend::{AnyOperation, task_page_map::StorageAccessToken},
utils::ptr_eq_arc::PtrEqArc,
};
/// High bit: set while a snapshot is requested or in flight. /// High bit: set while a snapshot is requested or in flight.
/// Low bits: count of operations currently executing (not suspended). /// Low bits: count of operations currently executing (not suspended).
...@@ -200,7 +203,11 @@ pub struct OperationGuard<'a, O> { ...@@ -200,7 +203,11 @@ pub struct OperationGuard<'a, O> {
coord: &'a SnapshotCoordinator<O>, coord: &'a SnapshotCoordinator<O>,
} }
impl<O> OperationGuard<'_, O> { impl<'a, O> OperationGuard<'a, O> {
pub(crate) fn access_token(&self) -> StorageAccessToken<'a> {
StorageAccessToken::operation()
}
/// Suspend this operation if a snapshot is requested. Otherwise a no-op. The closure is called /// Suspend this operation if a snapshot is requested. Otherwise a no-op. The closure is called
/// only when actually suspending and must produce a handle to this operation so the /// only when actually suspending and must produce a handle to this operation so the
/// snapshotter can persist it for replay on the next startup. /// snapshotter can persist it for replay on the next startup.
...@@ -287,7 +294,11 @@ pub struct SnapshotPhase<'a, O> { ...@@ -287,7 +294,11 @@ pub struct SnapshotPhase<'a, O> {
suspended_operations: Vec<Arc<O>>, suspended_operations: Vec<Arc<O>>,
} }
impl<O> SnapshotPhase<'_, O> { impl<'a, O> SnapshotPhase<'a, O> {
pub(crate) fn access_token(&self) -> StorageAccessToken<'a> {
StorageAccessToken::exclusive()
}
/// Operations that were suspended at the moment the snapshot started. /// Operations that were suspended at the moment the snapshot started.
/// The snapshotter must persist these so they can be replayed on the /// The snapshotter must persist these so they can be replayed on the
/// next startup. /// next startup.
......
...@@ -33,7 +33,7 @@ use turbo_tasks::{ ...@@ -33,7 +33,7 @@ use turbo_tasks::{
}; };
use crate::{ use crate::{
backend::{cell_data::CellData, counter_map::CounterMap, dense_task_map::TaskSlotValue}, backend::{cell_data::CellData, counter_map::CounterMap, task_page_map::TaskSlotValue},
data::{ data::{
ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, ActivenessState, AggregationNumber, CellRef, CollectibleRef, CollectiblesRef, Dirtyness,
InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask, InProgressCellState, InProgressState, LeafDistance, OutputValue, RootType, TransientTask,
...@@ -48,7 +48,7 @@ type AutoSet<K, const I: usize> = auto_hash_map::AutoSet<K, BuildHasherDefault<F ...@@ -48,7 +48,7 @@ type AutoSet<K, const I: usize> = auto_hash_map::AutoSet<K, BuildHasherDefault<F
/// See [`AutoSet`] for the meaning of `I`. /// See [`AutoSet`] for the meaning of `I`.
type AutoMap<K, V, const I: usize> = auto_hash_map::AutoMap<K, V, BuildHasherDefault<FxHasher>, I>; type AutoMap<K, V, const I: usize> = auto_hash_map::AutoMap<K, V, BuildHasherDefault<FxHasher>, I>;
/// Intrusive parking_lot lock embedded in each always-initialized task slot. /// Intrusive parking_lot lock embedded in each stable task allocation.
/// ///
/// The newtype supplies the generated `Debug`, `Default`, and `ShrinkToFit` requirements while /// The newtype supplies the generated `Debug`, `Default`, and `ShrinkToFit` requirements while
/// retaining parking_lot's standard RAII guard and one-byte mutex layout. /// retaining parking_lot's standard RAII guard and one-byte mutex layout.
...@@ -603,10 +603,13 @@ pub enum KeyEvictability { ...@@ -603,10 +603,13 @@ pub enum KeyEvictability {
Unevictable, Unevictable,
} }
// SAFETY: `TaskStorage::empty_slot` always initializes the mutex. Dense chunks pin every slot // SAFETY: `TaskStorage::new` initializes the mutex. Resident values remain at stable boxed
// after publication, and `take_and_vacate` moves only payload fields while preserving that mutex. // addresses from release publication until an exclusive storage epoch detaches them.
unsafe impl TaskSlotValue for TaskStorage { unsafe impl TaskSlotValue for TaskStorage {
const EMPTY: Self = Self::empty_slot(); fn new() -> Self {
TaskStorage::new()
}
unsafe fn lock_raw(value: *const Self) { unsafe fn lock_raw(value: *const Self) {
// SAFETY: TaskSlotValue guarantees a live, stably-addressed TaskStorage. // SAFETY: TaskSlotValue guarantees a live, stably-addressed TaskStorage.
unsafe { IntrusiveTaskLock::lock_raw(std::ptr::addr_of!((*value).lock)) }; unsafe { IntrusiveTaskLock::lock_raw(std::ptr::addr_of!((*value).lock)) };
...@@ -616,36 +619,11 @@ unsafe impl TaskSlotValue for TaskStorage { ...@@ -616,36 +619,11 @@ unsafe impl TaskSlotValue for TaskStorage {
// SAFETY: TaskSlotGuard owns the lock acquired through the same raw value pointer. // SAFETY: TaskSlotGuard owns the lock acquired through the same raw value pointer.
unsafe { IntrusiveTaskLock::unlock_raw(std::ptr::addr_of!((*value).lock)) }; unsafe { IntrusiveTaskLock::unlock_raw(std::ptr::addr_of!((*value).lock)) };
} }
fn is_occupied(&self) -> bool {
self.is_occupied()
}
fn occupy(&mut self) {
self.occupy();
}
fn take_and_vacate(&mut self) -> Self {
self.take_and_vacate()
}
fn vacate_in_place(&mut self) {
self.vacate_in_place();
}
} }
impl TaskStorage { impl TaskStorage {
pub(crate) fn is_occupied(&self) -> bool { /// Canonical const representation of a fresh boxed task payload.
self.occupied pub const fn empty_task() -> Self {
}
pub(crate) fn occupy(&mut self) {
debug_assert!(!self.occupied);
self.occupied = true;
}
/// Canonical const representation of a vacant, always-initialized dense task slot.
pub const fn empty_slot() -> Self {
Self { Self {
leaf_distance: LeafDistance { leaf_distance: LeafDistance {
distance: 0, distance: 0,
...@@ -665,7 +643,6 @@ impl TaskStorage { ...@@ -665,7 +643,6 @@ impl TaskStorage {
flags: TaskFlags::empty(), flags: TaskFlags::empty(),
lazy: TinyVec::new(), lazy: TinyVec::new(),
lock: IntrusiveTaskLock::new(), lock: IntrusiveTaskLock::new(),
occupied: false,
} }
} }
...@@ -1224,13 +1201,16 @@ impl<K: IsTransient + Hash + Eq, V: IsTransient, const I: usize> DropPartial for ...@@ -1224,13 +1201,16 @@ impl<K: IsTransient + Hash + Eq, V: IsTransient, const I: usize> DropPartial for
mod tests { mod tests {
// Only used by the 64-bit schema-size assertions. // Only used by the 64-bit schema-size assertions.
#[cfg(target_pointer_width = "64")] #[cfg(target_pointer_width = "64")]
use std::{mem::size_of, sync::atomic::AtomicU64}; use std::{
mem::size_of,
sync::atomic::{AtomicPtr, AtomicU64},
};
use turbo_tasks::{CellId, TaskId}; use turbo_tasks::{CellId, TaskId};
use super::*; use super::*;
use crate::{ use crate::{
backend::dense_task_map::{BITMAP_WORDS, CHUNK_SIZE, TaskChunk, TaskSlot}, backend::task_page_map::{BITMAP_WORDS, PAGE_SIZE, TaskMap},
data::{AggregationNumber, CellRef, Dirtyness, OutputValue}, data::{AggregationNumber, CellRef, Dirtyness, OutputValue},
}; };
...@@ -1974,11 +1954,7 @@ mod tests { ...@@ -1974,11 +1954,7 @@ mod tests {
#[test] #[test]
fn const_new_slots_have_independent_unlocked_mutexes() { fn const_new_slots_have_independent_unlocked_mutexes() {
let empty_task = const { TaskStorage::new() };
let slots = [const { TaskStorage::new() }; 2]; let slots = [const { TaskStorage::new() }; 2];
assert!(!empty_task.is_occupied());
assert!(!slots[0].is_occupied());
assert!(!slots[1].is_occupied());
assert!(!slots[0].lock.is_locked()); assert!(!slots[0].lock.is_locked());
assert!(!slots[1].lock.is_locked()); assert!(!slots[1].lock.is_locked());
let guard = slots[0].lock.lock(); let guard = slots[0].lock.lock();
...@@ -1987,41 +1963,10 @@ mod tests { ...@@ -1987,41 +1963,10 @@ mod tests {
drop(guard); drop(guard);
} }
#[test]
fn vacate_preserves_lock_and_resets_payload() {
let mut task = TaskStorage::empty_slot();
let lock_address = std::ptr::addr_of!(task.lock);
let detached = with_task_locked(&mut task, |task| {
task.occupy();
task.set_output(OutputValue::Output(TaskId::new(1).unwrap()));
task.take_and_vacate()
});
assert_eq!(std::ptr::addr_of!(task.lock), lock_address);
assert!(!task.lock.is_locked());
assert!(!task.is_occupied());
assert_eq!(task.get_output(), None);
assert_eq!(
detached.get_output(),
Some(&OutputValue::Output(TaskId::new(1).unwrap()))
);
assert!(!detached.lock.is_locked());
assert!(!detached.is_occupied());
with_task_locked(&mut task, |task| {
task.occupy();
task.set_output(OutputValue::Output(TaskId::new(2).unwrap()));
assert!(task.is_occupied());
task.vacate_in_place();
assert!(task.lock.is_locked());
assert!(!task.is_occupied());
assert_eq!(task.get_output(), None);
});
}
#[test] #[test]
fn snapshot_clone_has_fresh_synchronization_state() { fn snapshot_clone_has_fresh_synchronization_state() {
let mut task = TaskStorage::empty_slot(); let mut task = TaskStorage::empty_task();
let snapshot = with_task_locked(&mut task, |task| { let snapshot = with_task_locked(&mut task, |task| {
task.occupy();
task.set_output(OutputValue::Output(TaskId::new(1).unwrap())); task.set_output(OutputValue::Output(TaskId::new(1).unwrap()));
let snapshot = task.clone_snapshot(); let snapshot = task.clone_snapshot();
assert!(task.lock.is_locked()); assert!(task.lock.is_locked());
...@@ -2029,7 +1974,6 @@ mod tests { ...@@ -2029,7 +1974,6 @@ mod tests {
snapshot snapshot
}); });
assert!(!snapshot.lock.is_locked()); assert!(!snapshot.lock.is_locked());
assert!(!snapshot.is_occupied());
} }
// `hanging_detection` grows LazyField via Event descriptions; diagnostic-only layouts are not // `hanging_detection` grows LazyField via Event descriptions; diagnostic-only layouts are not
...@@ -2047,25 +1991,25 @@ mod tests { ...@@ -2047,25 +1991,25 @@ mod tests {
128, 128,
"TaskStorage's niche should make its Option free" "TaskStorage's niche should make its Option free"
); );
assert_eq!(
size_of::<TaskSlot<TaskStorage>>(),
128,
"intrusive task slot must not add padding"
);
assert_eq!( assert_eq!(
size_of::<[AtomicU64; BITMAP_WORDS]>(), size_of::<[AtomicU64; BITMAP_WORDS]>(),
CHUNK_SIZE / 8, PAGE_SIZE / 8,
"chunk occupancy bitmap size changed" "chunk occupancy bitmap size changed"
); );
assert_eq!( assert_eq!(
size_of::<TaskChunk<TaskStorage>>(), TaskMap::<TaskStorage>::directory_entry_size(),
16 + CHUNK_SIZE / 8, size_of::<AtomicPtr<()>>(),
"chunk counters and bitmap size changed" "permanent page-directory entry must remain one pointer"
);
assert_eq!(
size_of::<[AtomicPtr<TaskStorage>; PAGE_SIZE]>(),
size_of::<AtomicPtr<TaskStorage>>() * PAGE_SIZE,
"pointer-page slot array size changed"
); );
assert_eq!( assert_eq!(
size_of::<[TaskSlot<TaskStorage>; CHUNK_SIZE]>(), TaskMap::<TaskStorage>::pointer_page_size(),
128 * CHUNK_SIZE, 24 + size_of::<AtomicPtr<TaskStorage>>() * PAGE_SIZE,
"chunk slot allocation size changed" "pointer-page counters, bitmap, or slots changed"
); );
// `LazyField` is 40 B = 32 B largest payload + 8 B discriminant. // `LazyField` is 40 B = 32 B largest payload + 8 B discriminant.
assert_eq!( assert_eq!(
......
...@@ -1364,14 +1364,12 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1364,14 +1364,12 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
// Collect all field definitions from both categories // Collect all field definitions from both categories
let mut field_defs = Vec::new(); let mut field_defs = Vec::new();
let mut field_names = Vec::new();
// Add inline fields directly on TaskStorage (private - use accessor methods) // Add inline fields directly on TaskStorage (private - use accessor methods)
// Note: No bincode attributes since we don't derive Encode/Decode (manual serialization) // Note: No bincode attributes since we don't derive Encode/Decode (manual serialization)
for field in grouped_fields.all_inline() { for field in grouped_fields.all_inline() {
let field_name = &field.field_name; let field_name = &field.field_name;
let field_type = &field.field_type; let field_type = &field.field_type;
field_names.push(field_name.clone());
field_defs.push(quote! { field_defs.push(quote! {
#field_name: #field_type #field_name: #field_type
}); });
...@@ -1397,19 +1395,6 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1397,19 +1395,6 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
quote! {} quote! {}
}; };
let swap_flags = has_flags.then(|| {
quote! { std::mem::swap(&mut self.flags, &mut detached.flags); }
});
let swap_lazy = has_lazy.then(|| {
quote! { std::mem::swap(&mut self.lazy, &mut detached.lazy); }
});
let reset_flags = has_flags.then(|| {
quote! { self.flags = Default::default(); }
});
let reset_lazy = has_lazy.then(|| {
quote! { self.lazy = Default::default(); }
});
// Note: Helper methods like find_lazy, find_lazy_mut, get_or_create_lazy, and // Note: Helper methods like find_lazy, find_lazy_mut, get_or_create_lazy, and
// remove_if_empty are defined in storage_schema.rs rather than generated here. // remove_if_empty are defined in storage_schema.rs rather than generated here.
// This provides better IDE support (autocomplete, go-to-definition, etc.). // This provides better IDE support (autocomplete, go-to-definition, etc.).
...@@ -1426,39 +1411,15 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream ...@@ -1426,39 +1411,15 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#(#field_defs,)* #(#field_defs,)*
#flags_field #flags_field
#lazy_field #lazy_field
#[doc = "Intrusive lock protecting this stable task slot"] #[doc = "Intrusive lock protecting this stable task allocation"]
pub(crate) lock: IntrusiveTaskLock, pub(crate) lock: IntrusiveTaskLock,
#[doc = "Authoritative slot presence, accessed only while `lock` is held"]
pub(crate) occupied: bool,
} }
#[automatically_derived] #[automatically_derived]
impl TaskStorage { impl TaskStorage {
#[doc = "Constructs fresh task payload with unlocked, vacant slot metadata."] #[doc = "Constructs a fresh task payload with an unlocked intrusive mutex."]
pub const fn new() -> Self { pub const fn new() -> Self {
Self::empty_slot() Self::empty_task()
}
#[doc = "Moves payload into a fresh unlocked value and leaves this locked slot vacant."]
#[doc = "The caller must hold this task's intrusive lock."]
pub(crate) fn take_and_vacate(&mut self) -> Self {
debug_assert!(self.occupied);
let mut detached = Self::default();
#(std::mem::swap(&mut self.#field_names, &mut detached.#field_names);)*
#swap_flags
#swap_lazy
self.occupied = false;
detached
}
#[doc = "Drops and resets payload in place while preserving the held mutex."]
#[doc = "The caller must hold this task's intrusive lock."]
pub(crate) fn vacate_in_place(&mut self) {
debug_assert!(self.occupied);
#(self.#field_names = Default::default();)*
#reset_flags
#reset_lazy
self.occupied = false;
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment