Unverified Commit d23fb5dc authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Allocate compact state identities and persist changed rows

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 3de15d2b
...@@ -19,10 +19,7 @@ ...@@ -19,10 +19,7 @@
use std::{ use std::{
fmt::Display, fmt::Display,
ops::ControlFlow, ops::ControlFlow,
sync::{ sync::atomic::{AtomicBool, Ordering},
Arc,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant, SystemTime, UNIX_EPOCH}, time::{Duration, Instant, SystemTime, UNIX_EPOCH},
}; };
...@@ -302,11 +299,10 @@ impl TurboTasksBackend { ...@@ -302,11 +299,10 @@ impl TurboTasksBackend {
{ {
let mut states = self.states.lock(); let mut states = self.states.lock();
states.retain(|key, state| { states.retain(|key, state| {
if key.owner == StateOwner::Task(task_id) { if state.lookup.owner == StateOwner::Task(task_id) {
result result
.deleted_state_dependents .deleted_state_dependents
.push((key.clone(), state.dependents.iter().copied().collect())); .push((*key, state.dependents.iter().copied().collect()));
self.state_dirty.store(true, Ordering::Release);
false false
} else { } else {
true true
...@@ -358,13 +354,11 @@ impl TurboTasksBackend { ...@@ -358,13 +354,11 @@ impl TurboTasksBackend {
let pins = self.named_state_pins.lock(); let pins = self.named_state_pins.lock();
let mut states = self.states.lock(); let mut states = self.states.lock();
states.retain(|key, state| { states.retain(|key, state| {
let StateOwner::Named(name) = &key.owner else { let StateOwner::Named(name) = &state.lookup.owner else {
return true; return true;
}; };
if pins.contains_key(name) { if pins.contains_key(name) {
if state.unrooted_since.take().is_some() { state.unrooted_since = None;
self.state_dirty.store(true, Ordering::Release);
}
return true; return true;
} }
if let Some(since) = state.unrooted_since if let Some(since) = state.unrooted_since
...@@ -372,13 +366,11 @@ impl TurboTasksBackend { ...@@ -372,13 +366,11 @@ impl TurboTasksBackend {
{ {
result result
.deleted_state_dependents .deleted_state_dependents
.push((key.clone(), state.dependents.iter().copied().collect())); .push((*key, state.dependents.iter().copied().collect()));
self.state_dirty.store(true, Ordering::Release);
return false; return false;
} }
if state.unrooted_since.is_none() { if state.unrooted_since.is_none() {
state.unrooted_since = Some(now); state.unrooted_since = Some(now);
self.state_dirty.store(true, Ordering::Release);
} }
true true
}); });
...@@ -392,7 +384,7 @@ impl TurboTasksBackend { ...@@ -392,7 +384,7 @@ impl TurboTasksBackend {
let noop_collector = |_task_id| {}; let noop_collector = |_task_id| {};
let mut ctx = ExecuteContext::new_for_gc(self, turbo_tasks, phase, &noop_collector); let mut ctx = ExecuteContext::new_for_gc(self, turbo_tasks, phase, &noop_collector);
for (key, readers) in removed_states { for (key, readers) in removed_states {
let shared_key = Arc::new(key); let shared_key = key;
for reader in readers { for reader in readers {
if let Some(mut task) = ctx.try_task(reader, TaskDataCategory::Data) if let Some(mut task) = ctx.try_task(reader, TaskDataCategory::Data)
&& task.remove_state_dependencies(&shared_key) && task.remove_state_dependencies(&shared_key)
......
...@@ -54,7 +54,7 @@ pub fn capture_all_edges(task: &impl TaskStorageAccessors) -> Vec<OutdatedEdge> ...@@ -54,7 +54,7 @@ pub fn capture_all_edges(task: &impl TaskStorageAccessors) -> Vec<OutdatedEdge>
); );
old_edges.extend( old_edges.extend(
task.iter_state_dependencies() task.iter_state_dependencies()
.map(|key| OutdatedEdge::StateDependency((*key).clone())), .map(OutdatedEdge::StateDependency),
); );
old_edges.extend( old_edges.extend(
task.iter_collectibles_dependencies() task.iter_collectibles_dependencies()
......
...@@ -113,7 +113,10 @@ impl StateMutationScope { ...@@ -113,7 +113,10 @@ impl StateMutationScope {
pub(crate) fn enter() -> Self { pub(crate) fn enter() -> Self {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
IN_STATE_MUTATION.with(|flag| { IN_STATE_MUTATION.with(|flag| {
assert!(!flag.replace(true), "state mutation must not call back into turbo-tasks"); assert!(
!flag.replace(true),
"state mutation must not call back into turbo-tasks"
);
}); });
Self Self
} }
...@@ -121,7 +124,10 @@ impl StateMutationScope { ...@@ -121,7 +124,10 @@ impl StateMutationScope {
fn assert_not_inside() { fn assert_not_inside() {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
IN_STATE_MUTATION.with(|flag| { IN_STATE_MUTATION.with(|flag| {
assert!(!flag.get(), "state mutation must not call back into turbo-tasks"); assert!(
!flag.get(),
"state mutation must not call back into turbo-tasks"
);
}); });
} }
} }
...@@ -938,36 +944,38 @@ impl<'e> ExecuteContext<'e> { ...@@ -938,36 +944,38 @@ impl<'e> ExecuteContext<'e> {
self.open_task(task_id, category, TaskAccess::AllowMissing) self.open_task(task_id, category, TaskAccess::AllowMissing)
} }
/// Opens a task, materializing an in-memory storage entry for it if one is not resident yet /// Remove an outdated forward state edge and its reverse reader entry.
/// (inserting a blank, then restoring `category` from disk if present). Use only where the
/// task's storage may not be resident: the first connect of a freshly-minted child (threads can
/// race to first-touch it).
///
/// This creates *storage for* an already-minted `TaskId`; it does not mint one. Compare
/// `TurboTasksBackend::get_or_create_task`, which takes a function and arguments and returns a
/// `TaskId` (reusing one if the task is already cached).
pub fn remove_state_dependency(&mut self, reader: TaskId, key: &turbo_tasks::StateKey) { pub fn remove_state_dependency(&mut self, reader: TaskId, key: &turbo_tasks::StateKey) {
// State before reader, as for registration. The caller has released its // State before reader, as for registration. The caller has released its
// task guard, so a re-read either revives the edge or observes removal. // task guard, so a re-read either revives the edge or observes removal.
let backend = self.backend; let backend = self.backend;
let mut states = backend.states.lock(); let mut states = backend.states.lock();
let Some(mut task) = self.try_task(reader, TaskDataCategory::Data) else { let Some(mut task) = self.try_task(reader, TaskDataCategory::Data) else {
if let Some(state) = states.get_mut(key) { if let Some(state) = states.entries.get_mut(key)
state.dependents.remove(&reader); && state.dependents.remove(&reader)
backend.state_dirty.store(true, std::sync::atomic::Ordering::Release); {
states.dirty.insert(*key);
} }
return; return;
}; };
let shared_key = Arc::new(key.clone()); if task.remove_outdated_state_dependencies(key) {
if task.remove_outdated_state_dependencies(&shared_key) { task.remove_state_dependencies(key);
task.remove_state_dependencies(&shared_key); if let Some(state) = states.entries.get_mut(key)
if let Some(state) = states.get_mut(key) { && state.dependents.remove(&reader)
state.dependents.remove(&reader); {
backend.state_dirty.store(true, std::sync::atomic::Ordering::Release); states.dirty.insert(*key);
} }
} }
} }
/// Opens a task, materializing an in-memory storage entry for it if one is not resident yet
/// (inserting a blank, then restoring `category` from disk if present). Use only where the
/// task's storage may not be resident: the first connect of a freshly-minted child (threads can
/// race to first-touch it).
///
/// This creates *storage for* an already-minted `TaskId`; it does not mint one. Compare
/// `TurboTasksBackend::get_or_create_task`, which takes a function and arguments and returns a
/// `TaskId` (reusing one if the task is already cached).
pub fn task_or_create(&mut self, task_id: TaskId, category: TaskDataCategory) -> TaskGuard<'e> { pub fn task_or_create(&mut self, task_id: TaskId, category: TaskDataCategory) -> TaskGuard<'e> {
self.open_task(task_id, category, TaskAccess::MaybeCreate) self.open_task(task_id, category, TaskAccess::MaybeCreate)
.expect("a MaybeCreate open always yields a task") .expect("a MaybeCreate open always yields a task")
......
use anyhow::Result;
use turbo_tasks::{StateOwnerRoot, StateSlot, TurboTasks, TurboTasksState, Vc};
use crate::{
BackendOptions,
backend::{TaskDataCategory, TurboTasksBackend, storage_schema::TaskStorageAccessors},
};
#[turbo_tasks::state]
static COUNTER: StateSlot<u32> = StateSlot::new();
#[turbo_tasks::function(operation, root)]
fn reader(state: TurboTasksState<u32>) -> Vc<u32> {
Vc::cell(state.get())
}
/// Inspect the captured batch itself, not just restored values: unchanged rows
/// must be absent, and transient identities/dependents cannot cross a restart.
#[tokio::test(flavor = "multi_thread")]
async fn incremental_capture_and_explicit_reader_mutability() -> Result<()> {
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions::default(),
crate::noop_backing_storage(),
));
let test_tt = tt.clone();
turbo_tasks::run_once(tt.clone(), async move {
let root = StateOwnerRoot::named("state-batch-test".into());
let a = COUNTER.for_named_owner(&root, 1);
let b = COUNTER.for_named_owner(&StateOwnerRoot::named("other".into()), 2);
let transient = COUNTER.for_current_task(3);
let task = reader(a);
assert_eq!(*task.read_strongly_consistent().await?, 1);
{
let mut ctx = test_tt.backend().execute_context(&test_tt);
let task = ctx.task(task.task_id(), TaskDataCategory::Data);
assert!(!task.invalidator());
assert!(!task.immutable());
assert!(!task.is_state_dependencies_empty());
}
// This transient root becomes a reverse dependent but is excluded from
// the durable record, as is the transient state itself.
a.get();
let first = test_tt.backend().states.lock().snapshot()?.unwrap();
assert_eq!(first.updates.len(), 2);
assert!(first.updates.iter().all(|(key, _)| key != transient.key()));
for (_, bytes) in first.updates {
let record: crate::backend::PersistedState =
turbo_bincode::turbo_bincode_decode(&bytes.unwrap())?;
assert!(record.dependents.iter().all(|id| !id.is_transient()));
}
assert!(test_tt.backend().states.lock().snapshot()?.is_none());
a.set(10);
let update = test_tt.backend().states.lock().snapshot()?.unwrap();
assert_eq!(update.updates.len(), 1);
assert_eq!(update.updates[0].0, *a.key());
// Re-reading the same edge and no-op writes produce no state row work.
a.set(10);
a.get();
assert!(test_tt.backend().states.lock().snapshot()?.is_none());
assert_eq!(b.get_untracked(), 2);
anyhow::Ok(())
})
.await?;
tt.stop_and_wait().await;
Ok(())
}
...@@ -25,8 +25,7 @@ use std::{ ...@@ -25,8 +25,7 @@ use std::{
use parking_lot::Mutex; use parking_lot::Mutex;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use turbo_tasks::{ use turbo_tasks::{
CellId, SharedReference, StateKey, StateOwner, TaskExecutionReason, TaskId, TraitTypeId, CellId, SharedReference, StateKey, TaskExecutionReason, TaskId, TraitTypeId, ValueTypeId,
ValueTypeId,
backend::{CachedTaskTypeArc, CellHash, TransientTaskType}, backend::{CachedTaskTypeArc, CellHash, TransientTaskType},
event::Event, event::Event,
task_storage, task_storage,
...@@ -220,9 +219,10 @@ struct TaskStorageSchema { ...@@ -220,9 +219,10 @@ struct TaskStorageSchema {
#[field(storage = "flag", category = "transient")] #[field(storage = "flag", category = "transient")]
prefetched: bool, prefetched: bool,
/// Whether this task has allocated a State (has interior mutability). /// Whether this task has allocated legacy cell-owned `State` (interior mutability).
/// Only set when `verify_determinism`` feature is enabled. /// Only set when the `verify_determinism` feature is enabled.
/// Used to skip determinism checks for stateful tasks. /// Legacy callers still need the determinism exemption; backend-owned state
/// readers instead use explicit state dependencies for mutability.
#[field(storage = "flag", category = "transient")] #[field(storage = "flag", category = "transient")]
stateful: bool, stateful: bool,
...@@ -293,7 +293,7 @@ struct TaskStorageSchema { ...@@ -293,7 +293,7 @@ struct TaskStorageSchema {
shrink_on_completion, shrink_on_completion,
drop_on_completion_if_immutable drop_on_completion_if_immutable
)] )]
state_dependencies: AutoSet<Arc<StateKey>, 2>, state_dependencies: AutoSet<StateKey, 2>,
/// Collectibles this task depends on. /// Collectibles this task depends on.
#[field( #[field(
...@@ -319,7 +319,7 @@ struct TaskStorageSchema { ...@@ -319,7 +319,7 @@ struct TaskStorageSchema {
/// State dependencies not yet re-read in this execution. /// State dependencies not yet re-read in this execution.
#[field(storage = "auto_set", category = "transient", shrink_on_completion)] #[field(storage = "auto_set", category = "transient", shrink_on_completion)]
outdated_state_dependencies: AutoSet<Arc<StateKey>, 2>, outdated_state_dependencies: AutoSet<StateKey, 2>,
/// Outdated collectibles dependencies to be cleaned up (transient). /// Outdated collectibles dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", drop_on_completion)] #[field(storage = "auto_set", category = "transient", drop_on_completion)]
...@@ -985,9 +985,9 @@ trait IsTransient { ...@@ -985,9 +985,9 @@ trait IsTransient {
fn is_transient(&self) -> bool; fn is_transient(&self) -> bool;
} }
impl IsTransient for Arc<StateKey> { impl IsTransient for StateKey {
fn is_transient(&self) -> bool { fn is_transient(&self) -> bool {
matches!(&self.owner, StateOwner::Task(id) if id.is_transient()) self.id.is_transient()
} }
} }
......
...@@ -6,6 +6,8 @@ pub enum KeySpace { ...@@ -6,6 +6,8 @@ pub enum KeySpace {
TaskMeta = 1, TaskMeta = 1,
TaskData = 2, TaskData = 2,
TaskCache = 3, TaskCache = 3,
StateData = 4,
StateIndex = 5,
} }
impl KeySpace { impl KeySpace {
/// Constructs a [`KeySpace`] from its numeric index (i.e., the `usize` discriminant). /// Constructs a [`KeySpace`] from its numeric index (i.e., the `usize` discriminant).
...@@ -19,6 +21,8 @@ impl KeySpace { ...@@ -19,6 +21,8 @@ impl KeySpace {
1 => KeySpace::TaskMeta, 1 => KeySpace::TaskMeta,
2 => KeySpace::TaskData, 2 => KeySpace::TaskData,
3 => KeySpace::TaskCache, 3 => KeySpace::TaskCache,
4 => KeySpace::StateData,
5 => KeySpace::StateIndex,
_ => panic!("KeySpace index out of range"), _ => panic!("KeySpace index out of range"),
} }
} }
...@@ -29,6 +33,8 @@ impl KeySpace { ...@@ -29,6 +33,8 @@ impl KeySpace {
KeySpace::TaskMeta => "TaskMeta", KeySpace::TaskMeta => "TaskMeta",
KeySpace::TaskData => "TaskData", KeySpace::TaskData => "TaskData",
KeySpace::TaskCache => "TaskCache", KeySpace::TaskCache => "TaskCache",
KeySpace::StateData => "StateData",
KeySpace::StateIndex => "StateIndex",
} }
} }
...@@ -44,15 +50,16 @@ impl KeySpace { ...@@ -44,15 +50,16 @@ impl KeySpace {
_ => 2, _ => 2,
}), }),
}, },
KeySpace::TaskData => FamilyConfig { KeySpace::TaskData | KeySpace::StateData => FamilyConfig {
name: self.name(), name: self.name(),
kind: FamilyKind::SingleValue, kind: FamilyKind::SingleValue,
compression: Compression::Zstd3, compression: Compression::Zstd3,
initial_shard_bits: ShardBits::new(3), initial_shard_bits: ShardBits::new(3),
}, },
KeySpace::TaskCache => FamilyConfig { KeySpace::TaskCache | KeySpace::StateIndex => FamilyConfig {
name: self.name(), name: self.name(),
// TaskCache uses hash-based lookups with potential collisions. // Hash candidates and live-state IDs are individual mappings,
// with incremental insertion/deletion instead of whole buckets.
kind: FamilyKind::MultiValue, kind: FamilyKind::MultiValue,
compression: Compression::Lz4, compression: Compression::Lz4,
initial_shard_bits: ShardBits::new(0), initial_shard_bits: ShardBits::new(0),
......
...@@ -26,7 +26,7 @@ mod parallel_scheduler; ...@@ -26,7 +26,7 @@ mod parallel_scheduler;
pub(crate) use parallel_scheduler::TurboTasksParallelScheduler; pub(crate) use parallel_scheduler::TurboTasksParallelScheduler;
/// Number of key families, see [`KeySpace`] enum for their numbers. /// Number of key families, see [`KeySpace`] enum for their numbers.
pub const FAMILIES: usize = 4; pub const FAMILIES: usize = 6;
const COMPACTION_MESSAGE: &str = "Finished filesystem cache database compaction"; const COMPACTION_MESSAGE: &str = "Finished filesystem cache database compaction";
...@@ -233,7 +233,7 @@ impl<'a> TurboWriteBatch<'a> { ...@@ -233,7 +233,7 @@ impl<'a> TurboWriteBatch<'a> {
} }
/// Writes a tombstone for a single `key` -> `value` mapping, leaving other values under `key` /// Writes a tombstone for a single `key` -> `value` mapping, leaving other values under `key`
/// intact. Only valid for `MultiValue` families (`TaskCache`). /// intact. Only valid for `MultiValue` families (`TaskCache`, `StateIndex`).
pub fn delete_value( pub fn delete_value(
&self, &self,
key_space: KeySpace, key_space: KeySpace,
......
...@@ -10,7 +10,7 @@ use smallvec::SmallVec; ...@@ -10,7 +10,7 @@ use smallvec::SmallVec;
use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode}; use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode};
use turbo_persistence::CommitStats; use turbo_persistence::CommitStats;
use turbo_tasks::{ use turbo_tasks::{
DynTaskInputs, RawVc, TaskId, DynTaskInputs, RawVc, StateId, StateKey, TaskId,
macro_helpers::NativeFunction, macro_helpers::NativeFunction,
panic_hooks::{PanicHookGuard, register_panic_hook}, panic_hooks::{PanicHookGuard, register_panic_hook},
parallel, parallel,
...@@ -18,7 +18,10 @@ use turbo_tasks::{ ...@@ -18,7 +18,10 @@ use turbo_tasks::{
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{SpecificTaskDataCategory, StateSnapshot, TtlCounter, storage_schema::TaskStorage}, backend::{
PersistedState, SpecificTaskDataCategory, StateSnapshot, TtlCounter,
storage_schema::TaskStorage,
},
backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash_from_components}, backing_storage::{SnapshotItem, SnapshotMeta, compute_task_type_hash_from_components},
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
...@@ -36,7 +39,7 @@ use crate::{ ...@@ -36,7 +39,7 @@ use crate::{
enum InfraKey { enum InfraKey {
NextFreeTaskId = 0, NextFreeTaskId = 0,
GcRoots = 1, GcRoots = 1,
States = 2, NextFreeStateId = 2,
} }
impl InfraKey { impl InfraKey {
...@@ -250,22 +253,43 @@ impl TurboBackingStorage { ...@@ -250,22 +253,43 @@ impl TurboBackingStorage {
get(&self.inner.database).context("Unable to read GC roots from database") get(&self.inner.database).context("Unable to read GC roots from database")
} }
pub(crate) fn load_states(&self) -> Result<StateSnapshot> { /// Enumerate live IDs through 256 fixed MultiValue buckets. Each membership
let Some(bytes) = self /// is independently written/deleted; no bucket or state-value blob is rewritten.
pub(crate) fn load_states(&self) -> Result<(u32, Vec<(StateKey, PersistedState)>)> {
let db = &self.inner.database;
let next_id = self
.inner .inner
.database .get_infra_u32(InfraKey::NextFreeStateId)?
.get(KeySpace::Infra, InfraKey::States.key().as_ref())? .unwrap_or(1);
else { anyhow::ensure!(next_id > 0 && next_id <= 1 << 31, "invalid next state ID");
return Ok(Vec::new()); let mut states = Vec::new();
}; for bucket in 0..=u8::MAX {
turbo_bincode_decode(bytes.borrow()).context("Unable to restore backend-owned states") for bytes in db.get_multiple(KeySpace::StateIndex, &[bucket])? {
let raw_id = as_u32(bytes)?;
let id = StateId::new(raw_id).context("invalid state ID")?;
anyhow::ensure!(
!id.is_transient() && raw_id < next_id && raw_id as u8 == bucket,
"invalid live state index"
);
let bytes = db
.get(KeySpace::StateData, IntKey::new(raw_id).as_ref())?
.context("live state record missing")?;
let record: PersistedState = turbo_bincode_decode(bytes.borrow())?;
let key = StateKey {
slot: record.lookup.slot,
id,
};
states.push((key, record));
}
}
Ok((next_id, states))
} }
pub(crate) fn save_snapshot<I>( pub(crate) fn save_snapshot<I>(
&self, &self,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
snapshots: Vec<I>, snapshots: Vec<I>,
state_snapshot: Option<Vec<u8>>, state_snapshot: Option<StateSnapshot>,
) -> Result<SnapshotMeta> ) -> Result<SnapshotMeta>
where where
I: IntoIterator<Item = SnapshotItem> + Send + Sync, I: IntoIterator<Item = SnapshotItem> + Send + Sync,
...@@ -494,7 +518,7 @@ fn save_infra( ...@@ -494,7 +518,7 @@ fn save_infra(
batch: &TurboWriteBatch<'_>, batch: &TurboWriteBatch<'_>,
next_task_id: u32, next_task_id: u32,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
state_snapshot: Option<Vec<u8>>, state_snapshot: Option<StateSnapshot>,
) -> Result<(), anyhow::Error> { ) -> Result<(), anyhow::Error> {
batch batch
.put( .put(
...@@ -515,13 +539,43 @@ fn save_infra( ...@@ -515,13 +539,43 @@ fn save_infra(
.context("Unable to write GC roots")?; .context("Unable to write GC roots")?;
} }
if let Some(states) = state_snapshot { if let Some(states) = state_snapshot {
batch for (key, record) in states.updates {
.put( let id = IntKey::new(key.id.to_primitive());
KeySpace::Infra, let bucket = ByteKey::new(key.id.to_primitive() as u8);
WriteBuffer::Borrowed(InfraKey::States.key().as_ref()), if let Some(record) = record {
WriteBuffer::Vec(states), // MultiValue preserves duplicate mappings. Only newly persisted
) // IDs acquire membership; ordinary updates touch just the row.
.context("Unable to write backend-owned states")?; if batch.get(KeySpace::StateData, id.as_ref())?.is_none() {
batch.put(
KeySpace::StateIndex,
WriteBuffer::Borrowed(bucket.as_ref()),
WriteBuffer::Borrowed(id.as_ref()),
)?;
}
batch.put(
KeySpace::StateData,
WriteBuffer::Borrowed(id.as_ref()),
WriteBuffer::Vec(record),
)?;
} else {
batch.delete(KeySpace::StateData, WriteBuffer::Borrowed(id.as_ref()))?;
batch.delete_value(
KeySpace::StateIndex,
WriteBuffer::Borrowed(bucket.as_ref()),
WriteBuffer::Borrowed(id.as_ref()),
)?;
}
}
batch.put(
KeySpace::Infra,
WriteBuffer::Borrowed(InfraKey::NextFreeStateId.key().as_ref()),
WriteBuffer::Borrowed(&states.next_id.to_le_bytes()),
)?;
// All state writes are complete, and share this task snapshot's commit.
unsafe {
batch.flush(KeySpace::StateData)?;
batch.flush(KeySpace::StateIndex)?;
}
} }
// Safety: save_infra is called after all concurrent writes to Infra are done. // Safety: save_infra is called after all concurrent writes to Infra are done.
unsafe { batch.flush(KeySpace::Infra)? }; unsafe { batch.flush(KeySpace::Infra)? };
...@@ -614,6 +668,107 @@ mod tests { ...@@ -614,6 +668,107 @@ mod tests {
Ok(()) Ok(())
} }
#[tokio::test(flavor = "multi_thread")]
async fn state_rows_and_live_index_update_incrementally_and_reopen() -> Result<()> {
use turbo_tasks::{StateFactoryId, StateLookupKey, StateOwner};
let dir = test_temp_dir()?;
let storage = TurboBackingStorage::new_in_memory(TurboKeyValueDatabase::new(
dir.path().to_path_buf(),
TEST_STORAGE_OPTIONS,
)?);
let a = StateKey {
slot: StateFactoryId::MIN,
id: StateId::new(1).unwrap(),
};
// Same index bucket, so deleting one must retain the other mapping.
let b = StateKey {
slot: a.slot,
id: StateId::new(257).unwrap(),
};
let encode = |name: &str, value: &str| -> Result<Vec<u8>> {
Ok(turbo_bincode_encode(&PersistedState {
lookup: StateLookupKey {
owner: StateOwner::Named(name.into()),
slot: a.slot,
},
value: turbo_bincode_encode(&turbo_rcstr::RcStr::from(value))?.into_vec(),
dependents: vec![TaskId::MIN],
unrooted_since: Some(42),
})?
.into_vec())
};
let original_b = encode("b", "unchanged")?;
storage.save_snapshot::<Vec<SnapshotItem>>(
None,
vec![],
Some(StateSnapshot {
next_id: 258,
updates: vec![
(a, Some(encode("a", "first")?)),
(b, Some(original_b.clone())),
],
}),
)?;
assert_eq!(storage.load_states()?.1.len(), 2);
storage.save_snapshot::<Vec<SnapshotItem>>(
None,
vec![],
Some(StateSnapshot {
next_id: 258,
updates: vec![(a, Some(encode("a", "updated")?))],
}),
)?;
assert_eq!(
storage.load_states()?.1.len(),
2,
"updates duplicated live-ID membership"
);
assert_eq!(
storage
.inner
.database
.get(KeySpace::StateData, &257u32.to_le_bytes())?
.unwrap()
.as_ref(),
&original_b
);
storage.save_snapshot::<Vec<SnapshotItem>>(
None,
vec![],
Some(StateSnapshot {
next_id: 258,
updates: vec![(a, None)],
}),
)?;
assert!(
storage
.inner
.database
.get(KeySpace::StateData, &1u32.to_le_bytes())?
.is_none()
);
let (next, records) = storage.load_states()?;
assert_eq!(next, 258);
assert_eq!(records.len(), 1);
assert_eq!(records[0].0, b);
storage.shutdown()?;
drop(storage);
let reopened = TurboBackingStorage::new_in_memory(TurboKeyValueDatabase::new(
dir.path().to_path_buf(),
TEST_STORAGE_OPTIONS,
)?);
let (next, records) = reopened.load_states()?;
assert_eq!(next, 258);
assert_eq!(records.len(), 1);
assert_eq!(records[0].0, b);
assert_eq!(records[0].1.unrooted_since, Some(42));
assert_eq!(records[0].1.dependents, vec![TaskId::MIN]);
let value: turbo_rcstr::RcStr = turbo_bincode_decode(&records[0].1.value)?;
assert_eq!(value, "unchanged");
reopened.shutdown()?;
Ok(())
}
/// Tests that multiple distinct keys written in a single batch with flush can be read back. /// Tests that multiple distinct keys written in a single batch with flush can be read back.
/// This mirrors the actual save_snapshot pattern: write many TaskCache entries, flush, commit. /// This mirrors the actual save_snapshot pattern: write many TaskCache entries, flush, commit.
// This test is too slow to run under Miri. // This test is too slow to run under Miri.
......
...@@ -22,6 +22,34 @@ There are a few design patterns that are commonly used with Turbo Tasks: ...@@ -22,6 +22,34 @@ There are a few design patterns that are commonly used with Turbo Tasks:
[cell id equality]: crate::ResolvedVc#equality--hashing [cell id equality]: crate::ResolvedVc#equality--hashing
[`Vc`]: crate::Vc [`Vc`]: crate::Vc
## Backend-owned state
`#[turbo_tasks::state] static SLOT: StateSlot<T> = StateSlot::new()` declares a
registered codec and a distinct factory identity, even when another slot has the
same Rust value type. Creating a slot for the current task, or for an explicitly
pinned `StateOwnerRoot::named(...)`, resolves the backend's `(owner, slot)` lookup
without resetting an existing value.
`TurboTasksState<T>` is a **Copy** reference containing only a factory ID and an
allocated instance ID. Owner names and mutable values stay in the backend, not in
task-input keys. Collected IDs are never recycled; an old handle returns a missing
state error even if the same logical owner/slot is created again. Named owners are
persistent; transient task owners produce transient references.
Reads are synchronous snapshots. `get` tracks a dependency and `get_untracked`
does not. Changed writes dirty current readers before publishing the value;
no-op writes do not invalidate them. State locks precede task locks. A contended
getter releases its operation guard before waiting, then retries acquisition.
Initializers/serialization under the state lock must not re-enter turbo-tasks.
Values containing operations still require their usual call-graph connections.
Persistence writes only dirty state rows and collected-ID tombstones, atomically
with affected task data and allocator progress. A fixed-bucket multi-value index
tracks live IDs through individual insertions/deletions, without rewriting an
entire value store or scanning holes in the allocated ID range on restart. Task
GC collects task-owned states; named roots pin their namespace, and normal GC
aging applies after the last root drops.
## Functions and Tasks ## Functions and Tasks
<figure style="display: flex; flex-direction: column; justify-content: center;"> <figure style="display: flex; flex-direction: column; justify-content: center;">
......
...@@ -32,7 +32,7 @@ use crate::{ ...@@ -32,7 +32,7 @@ use crate::{
CellId, RawVc, ReadCellOptions, ReadOutcome, ReadOutputOptions, ReadRef, SharedReference, CellId, RawVc, ReadCellOptions, ReadOutcome, ReadOutputOptions, ReadRef, SharedReference,
TaskId, TaskIdSet, TaskPriority, TraitRef, TraitTypeId, TurboTasksCallApi, TurboTasksPanic, TaskId, TaskIdSet, TaskPriority, TraitRef, TraitTypeId, TurboTasksCallApi, TurboTasksPanic,
ValueTypeId, ValueTypePersistence, VcValueTrait, VcValueType, ValueTypeId, ValueTypePersistence, VcValueTrait, VcValueType,
backend_state::StateKey, backend_state::{StateKey, StateLookupKey},
dyn_task_inputs::{DynTaskInputs, DynTaskInputsStorage}, dyn_task_inputs::{DynTaskInputs, DynTaskInputsStorage},
macro_helpers::NativeFunction, macro_helpers::NativeFunction,
manager::{TaskPersistence, TurboTasks}, manager::{TaskPersistence, TurboTasks},
...@@ -614,10 +614,10 @@ pub trait Backend: Sized + Sync + Send { ...@@ -614,10 +614,10 @@ pub trait Backend: Sized + Sync + Send {
/// under the state-store lock and must not call back into turbo-tasks. /// under the state-store lock and must not call back into turbo-tasks.
fn create_state( fn create_state(
&self, &self,
key: &StateKey, key: &StateLookupKey,
initial: &mut dyn FnMut() -> Vec<u8>, initial: &mut dyn FnMut() -> Vec<u8>,
turbo_tasks: &TurboTasks<Self>, turbo_tasks: &TurboTasks<Self>,
); ) -> StateKey;
/// Return a copy of the canonical state value, atomically registering a /// Return a copy of the canonical state value, atomically registering a
/// dependency if `reader` is supplied. /// dependency if `reader` is supplied.
......
...@@ -17,7 +17,7 @@ use turbo_rcstr::RcStr; ...@@ -17,7 +17,7 @@ use turbo_rcstr::RcStr;
use crate::{ use crate::{
NonLocalValue, OperationValue, TaskId, TaskInput, TurboTasksApi, NonLocalValue, OperationValue, TaskId, TaskInput, TurboTasksApi,
id::StateFactoryId, id::{StateFactoryId, StateId},
manager::{current_task, with_turbo_tasks}, manager::{current_task, with_turbo_tasks},
registry::{self, RegistryType, impl_ptr_identity}, registry::{self, RegistryType, impl_ptr_identity},
}; };
...@@ -69,14 +69,21 @@ impl Drop for StateOwnerRoot { ...@@ -69,14 +69,21 @@ impl Drop for StateOwnerRoot {
} }
} }
/// Stable identity used in inputs and persistent storage. The mutable value is /// Logical lookup key retained by the backend, not stored in handles.
/// deliberately not included in the identity.
#[derive(Clone, Debug, PartialEq, Eq, Hash, Encode, Decode)] #[derive(Clone, Debug, PartialEq, Eq, Hash, Encode, Decode)]
pub struct StateKey { pub struct StateLookupKey {
pub owner: StateOwner, pub owner: StateOwner,
pub slot: StateFactoryId, pub slot: StateFactoryId,
} }
/// Allocated identity used in inputs and dependencies. Neither the owner's
/// name nor the mutable value is copied with this reference.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encode, Decode)]
pub struct StateKey {
pub slot: StateFactoryId,
pub id: StateId,
}
/// Erased codec for values of a registered state slot. /// Erased codec for values of a registered state slot.
#[doc(hidden)] #[doc(hidden)]
#[derive(Debug)] #[derive(Debug)]
...@@ -176,12 +183,12 @@ impl<T> StateSlot<T> { ...@@ -176,12 +183,12 @@ impl<T> StateSlot<T> {
TypeId::of::<T>(), TypeId::of::<T>(),
"state slot value type mismatch" "state slot value type mismatch"
); );
let key = StateKey { let lookup = StateLookupKey {
owner, owner,
slot: registry::get_state_factory_id(factory), slot: registry::get_state_factory_id(factory),
}; };
with_turbo_tasks(|tt| { let key = with_turbo_tasks(|tt| {
tt.create_state(&key, &mut || { tt.create_state(&lookup, &mut || {
(factory.encode)(&initial).expect("state value could not be encoded") (factory.encode)(&initial).expect("state value could not be encoded")
}) })
}); });
...@@ -204,12 +211,11 @@ pub struct TurboTasksState<T> { ...@@ -204,12 +211,11 @@ pub struct TurboTasksState<T> {
_marker: PhantomData<fn() -> T>, _marker: PhantomData<fn() -> T>,
} }
impl<T> Copy for TurboTasksState<T> {}
impl<T> Clone for TurboTasksState<T> { impl<T> Clone for TurboTasksState<T> {
fn clone(&self) -> Self { fn clone(&self) -> Self {
Self { *self
key: self.key.clone(),
_marker: PhantomData,
}
} }
} }
...@@ -319,7 +325,7 @@ impl<T> Hash for TurboTasksState<T> { ...@@ -319,7 +325,7 @@ impl<T> Hash for TurboTasksState<T> {
impl<T: Send + Sync + 'static> TaskInput for TurboTasksState<T> { impl<T: Send + Sync + 'static> TaskInput for TurboTasksState<T> {
fn is_transient(&self) -> bool { fn is_transient(&self) -> bool {
matches!(self.key.owner, StateOwner::Task(task) if task.is_transient()) self.key.id.is_transient()
} }
} }
...@@ -357,10 +363,10 @@ mod tests { ...@@ -357,10 +363,10 @@ mod tests {
#[test] #[test]
fn a_deserialized_handle_cannot_read_a_slot_of_another_type() { fn a_deserialized_handle_cannot_read_a_slot_of_another_type() {
use crate::backend_state::{StateKey, StateOwner, TurboTasksState}; use crate::backend_state::{StateKey, TurboTasksState};
let handle = TurboTasksState::<i32>::new(StateKey { let handle = TurboTasksState::<i32>::new(StateKey {
owner: StateOwner::Named("type-check".into()), id: crate::StateId::MIN,
slot: get_state_factory_id(FIRST.factory()), slot: get_state_factory_id(FIRST.factory()),
}); });
let encoded = turbo_bincode::turbo_bincode_encode(&handle).unwrap(); let encoded = turbo_bincode::turbo_bincode_encode(&handle).unwrap();
......
...@@ -150,6 +150,14 @@ define_id!( ...@@ -150,6 +150,14 @@ define_id!(
define_id!(FunctionId: u16); define_id!(FunctionId: u16);
define_id!(TraitTypeId: u16); define_id!(TraitTypeId: u16);
define_id!(StateFactoryId: u16); define_id!(StateFactoryId: u16);
define_id!(StateId: u32, derive(Debug, Encode, Decode));
impl StateId {
/// Transient state lives in a disjoint ID range and is never persisted.
pub fn is_transient(self) -> bool {
self.to_primitive() & (1 << 31) != 0
}
}
define_id!( define_id!(
LocalTaskId: u32, LocalTaskId: u32,
derive(Debug, Serialize, Deserialize, Encode, Decode), derive(Debug, Serialize, Deserialize, Encode, Decode),
......
...@@ -80,7 +80,9 @@ pub use crate::manager::InlineExecutionStats; ...@@ -80,7 +80,9 @@ pub use crate::manager::InlineExecutionStats;
#[cfg(feature = "task_dirty_cause")] #[cfg(feature = "task_dirty_cause")]
pub use crate::task_dirty_cause::TaskDirtyCause; pub use crate::task_dirty_cause::TaskDirtyCause;
pub use crate::{ pub use crate::{
backend_state::{StateKey, StateOwner, StateOwnerRoot, StateSlot, TurboTasksState}, backend_state::{
StateKey, StateLookupKey, StateOwner, StateOwnerRoot, StateSlot, TurboTasksState,
},
capture_future::TurboTasksPanic, capture_future::TurboTasksPanic,
collectibles::CollectiblesSource, collectibles::CollectiblesSource,
completion::{Completion, Completions}, completion::{Completion, Completions},
...@@ -95,7 +97,7 @@ pub use crate::{ ...@@ -95,7 +97,7 @@ pub use crate::{
}, },
error::PrettyPrintError, error::PrettyPrintError,
id::{ id::{
ExecutionId, FunctionId, LocalTaskId, StateFactoryId, TRANSIENT_TASK_BIT, TaskId, ExecutionId, FunctionId, LocalTaskId, StateFactoryId, StateId, TRANSIENT_TASK_BIT, TaskId,
TraitTypeId, ValueTypeId, TraitTypeId, ValueTypeId,
}, },
invalidation::{ invalidation::{
......
...@@ -134,7 +134,11 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { ...@@ -134,7 +134,11 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send {
fn unpin_named_state_owner(&self, name: &RcStr); fn unpin_named_state_owner(&self, name: &RcStr);
/// Create a canonical state value if the slot is vacant. /// Create a canonical state value if the slot is vacant.
fn create_state(&self, key: &crate::StateKey, initial: &mut dyn FnMut() -> Vec<u8>); fn create_state(
&self,
key: &crate::StateLookupKey,
initial: &mut dyn FnMut() -> Vec<u8>,
) -> crate::StateKey;
/// Read the state value synchronously, optionally tracking the current task. /// Read the state value synchronously, optionally tracking the current task.
fn read_state(&self, key: &crate::StateKey, tracked: bool) -> Result<Vec<u8>>; fn read_state(&self, key: &crate::StateKey, tracked: bool) -> Result<Vec<u8>>;
...@@ -1823,8 +1827,12 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> { ...@@ -1823,8 +1827,12 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> {
self.backend.unpin_named_state_owner(name, self); self.backend.unpin_named_state_owner(name, self);
} }
fn create_state(&self, key: &crate::StateKey, initial: &mut dyn FnMut() -> Vec<u8>) { fn create_state(
self.backend.create_state(key, initial, self); &self,
key: &crate::StateLookupKey,
initial: &mut dyn FnMut() -> Vec<u8>,
) -> crate::StateKey {
self.backend.create_state(key, initial, self)
} }
fn read_state(&self, key: &crate::StateKey, tracked: bool) -> Result<Vec<u8>> { fn read_state(&self, key: &crate::StateKey, tracked: bool) -> Result<Vec<u8>> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment