Unverified Commit 17a84287 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

fix rebase error

parent 3b5d5b82
...@@ -391,6 +391,8 @@ impl TurboTasksBackend { ...@@ -391,6 +391,8 @@ impl TurboTasksBackend {
&& let Err(err) = self.backing_storage.save_snapshot( && let Err(err) = self.backing_storage.save_snapshot(
Vec::new(), Vec::new(),
Some(roots), Some(roots),
// This hook runs no eviction, so it frees no ids to persist.
None,
Vec::<Vec<SnapshotItem>>::new(), Vec::<Vec<SnapshotItem>>::new(),
) )
{ {
......
...@@ -38,6 +38,7 @@ ...@@ -38,6 +38,7 @@
use std::collections::VecDeque; use std::collections::VecDeque;
use parking_lot::Mutex; use parking_lot::Mutex;
use rustc_hash::FxHashSet;
use turbo_tasks::TaskId; use turbo_tasks::TaskId;
/// How many snapshot cycles a freed id waits before it may be handed out again. /// How many snapshot cycles a freed id waits before it may be handed out again.
...@@ -61,6 +62,10 @@ pub(crate) struct DeferredIdReuse { ...@@ -61,6 +62,10 @@ pub(crate) struct DeferredIdReuse {
/// Cycles an id must wait. `0` releases immediately, which is what the aliasing-pressure tests /// Cycles an id must wait. `0` releases immediately, which is what the aliasing-pressure tests
/// want. /// want.
delay_cycles: u32, delay_cycles: u32,
/// Ids already handed back to the factory this session. Retained only so that
/// [`Self::persistable`] can offer them to the next session too — an id reused in *this*
/// session is still free from the perspective of a fresh one, which rebuilds its own graph.
released: Mutex<FxHashSet<TaskId>>,
} }
impl DeferredIdReuse { impl DeferredIdReuse {
...@@ -69,6 +74,7 @@ impl DeferredIdReuse { ...@@ -69,6 +74,7 @@ impl DeferredIdReuse {
queue: Mutex::new(VecDeque::new()), queue: Mutex::new(VecDeque::new()),
cycle: Mutex::new(0), cycle: Mutex::new(0),
delay_cycles, delay_cycles,
released: Mutex::new(FxHashSet::default()),
} }
} }
...@@ -101,6 +107,7 @@ impl DeferredIdReuse { ...@@ -101,6 +107,7 @@ impl DeferredIdReuse {
} }
queue.pop_front(); queue.pop_front();
released.push(id); released.push(id);
self.released.lock().insert(id);
} }
released released
} }
...@@ -109,6 +116,21 @@ impl DeferredIdReuse { ...@@ -109,6 +116,21 @@ impl DeferredIdReuse {
pub(crate) fn pending(&self) -> usize { pub(crate) fn pending(&self) -> usize {
self.queue.lock().len() self.queue.lock().len()
} }
/// Every id this session has freed and not since seen resurrected, for persistence.
///
/// No retraction path is needed for resurrection: `resurrect_deleted` can only revive a task
/// while it is still resident, and eviction is what puts an id in this queue, so a resurrected
/// task never reaches [`Self::defer`] in the first place. Includes ids still
/// inside their in-memory window: the window exists to protect *this* session's stale
/// references, and a fresh session starts with no resident state, so nothing can hold one.
pub(crate) fn persistable(&self) -> Vec<TaskId> {
let queue = self.queue.lock();
let released = self.released.lock();
let mut ids: Vec<TaskId> = queue.iter().map(|&(id, _)| id).collect();
ids.extend(released.iter().copied());
ids
}
} }
#[cfg(test)] #[cfg(test)]
...@@ -147,6 +169,19 @@ mod tests { ...@@ -147,6 +169,19 @@ mod tests {
assert_eq!(q.advance_cycle(), vec![id(2)]); assert_eq!(q.advance_cycle(), vec![id(2)]);
} }
#[test]
fn persistable_covers_both_waiting_and_released_ids() {
// The next session has no resident state, so an id that is still inside this session's
// window is nonetheless free from its point of view — both halves must be offered.
let q = DeferredIdReuse::new(1);
q.defer([id(1)]);
assert_eq!(q.advance_cycle(), vec![id(1)]);
q.defer([id(2)]);
let mut persistable = q.persistable();
persistable.sort();
assert_eq!(persistable, vec![id(1), id(2)]);
}
#[test] #[test]
fn a_quiet_cycle_releases_nothing() { fn a_quiet_cycle_releases_nothing() {
let q = DeferredIdReuse::new(1); let q = DeferredIdReuse::new(1);
......
...@@ -345,15 +345,47 @@ impl TurboTasksBackend { ...@@ -345,15 +345,47 @@ impl TurboTasksBackend {
gc_enabled = false; gc_enabled = false;
} }
// Ids the previous session freed. Validated before use: a corrupt or stale entry would
// otherwise hand out an id that is still live, which reuse must never do. Anything
// suspicious drops the whole set rather than part of it — the cost is lost density, and
// the alternative is trusting the rest of a record we already know is wrong.
let free_task_ids = match backing_storage.free_task_ids() {
Ok(ids) => {
let valid = ids
.iter()
.all(|id| !id.is_transient() && **id < *next_task_id);
if valid {
ids
} else {
eprintln!(
"warning: persisted free task id set contains ids outside the allocated range; ignoring it"
);
Vec::new()
}
}
Err(err) => {
eprintln!("failed to read free task ids, treating as empty: {err:?}");
Vec::new()
}
};
Self { Self {
options, options,
gc_enabled, gc_enabled,
start_time: Instant::now(), start_time: Instant::now(),
deferred_id_reuse: DeferredIdReuse::new(id_reuse_delay_cycles), deferred_id_reuse: DeferredIdReuse::new(id_reuse_delay_cycles),
persisted_task_id_factory: IdFactoryWithReuse::new( persisted_task_id_factory: {
next_task_id, let factory = IdFactoryWithReuse::new(
TaskId::try_from(TRANSIENT_TASK_BIT - 1).unwrap(), next_task_id,
), TaskId::try_from(TRANSIENT_TASK_BIT - 1).unwrap(),
);
// SAFETY: each id was freed by a previous session's GC, its tombstone was
// committed, and it was validated above to be a persistent id below the
// allocation watermark. A fresh session has no resident state, so nothing in this
// process can be holding a reference to one.
unsafe { factory.seed_free_ids(free_task_ids) };
factory
},
transient_task_id_factory: IdFactoryWithReuse::new( transient_task_id_factory: IdFactoryWithReuse::new(
TaskId::try_from(TRANSIENT_TASK_BIT).unwrap(), TaskId::try_from(TRANSIENT_TASK_BIT).unwrap(),
TaskId::MAX, TaskId::MAX,
...@@ -1259,7 +1291,12 @@ impl TurboTasksBackend { ...@@ -1259,7 +1291,12 @@ impl TurboTasksBackend {
let snapshot_time = Instant::now(); let snapshot_time = Instant::now();
drop(snapshot_phase); drop(snapshot_phase);
if !has_modifications && gc_roots_to_persist.is_none() { // Ids this session freed still need to reach disk even when nothing else changed —
// otherwise a session whose last cycle had no modifications silently drops them and the
// next session re-mints instead of reusing.
let has_free_ids_to_persist =
self.gc_enabled && !self.deferred_id_reuse.persistable().is_empty();
if !has_modifications && gc_roots_to_persist.is_none() && !has_free_ids_to_persist {
// No tasks modified since the last snapshot — drop the guard (which // No tasks modified since the last snapshot — drop the guard (which
// calls end_snapshot) and skip the expensive O(N) scan. // calls end_snapshot) and skip the expensive O(N) scan.
drop(snapshot_guard); drop(snapshot_guard);
...@@ -1437,6 +1474,10 @@ impl TurboTasksBackend { ...@@ -1437,6 +1474,10 @@ impl TurboTasksBackend {
// For tasks accessed during snapshot mode, a frozen copy was made and its `modified` // For tasks accessed during snapshot mode, a frozen copy was made and its `modified`
// flags were copied from the live task at snapshot creation time, reflecting which // flags were copied from the live task at snapshot creation time, reflecting which
// categories were dirtied before the snapshot was taken. // categories were dirtied before the snapshot was taken.
// Only collected at shutdown; see the `Delete` branch below. `Arc` because `process` is
// borrowed by the parallel scan and must not keep the value borrowed past it.
let tombstoned_this_commit: Option<Arc<Mutex<Vec<TaskId>>>> =
(self.gc_enabled && reason.drain_entries()).then(|| Arc::new(Mutex::new(Vec::new())));
let process = |task_id: TaskId, inner: &TaskStorage, buffer: &mut TurboBincodeBuffer| { let process = |task_id: TaskId, inner: &TaskStorage, buffer: &mut TurboBincodeBuffer| {
let encode_category = |task_id: TaskId, let encode_category = |task_id: TaskId,
data: &TaskStorage, data: &TaskStorage,
...@@ -1482,6 +1523,12 @@ impl TurboTasksBackend { ...@@ -1482,6 +1523,12 @@ impl TurboTasksBackend {
.get_persistent_task_type() .get_persistent_task_type()
.expect("a GC-deleted task must have a task type"), .expect("a GC-deleted task must have a task type"),
); );
// Shutdown only: this is the last commit, so these ids can be offered to the
// next session directly instead of waiting for a follow-up commit that will
// never come. `process` runs in parallel across shards, hence the lock.
if let Some(tombstoned) = &tombstoned_this_commit {
tombstoned.lock().push(task_id);
}
return SnapshotItem::Delete { return SnapshotItem::Delete {
task_id, task_id,
task_type_hash, task_type_hash,
...@@ -1553,7 +1600,7 @@ impl TurboTasksBackend { ...@@ -1553,7 +1600,7 @@ impl TurboTasksBackend {
let snapshot_duration = start.elapsed(); let snapshot_duration = start.elapsed();
let task_count = task_snapshots.len(); let task_count = task_snapshots.len();
if task_snapshots.is_empty() && gc_roots_to_persist.is_none() { if task_snapshots.is_empty() && gc_roots_to_persist.is_none() && !has_free_ids_to_persist {
// This should be impossible — if we got here, modified_count was nonzero or gc_roots // This should be impossible — if we got here, modified_count was nonzero or gc_roots
// was present, and every modification that increments the count also failed // was present, and every modification that increments the count also failed
// during encoding. // during encoding.
...@@ -1572,9 +1619,26 @@ impl TurboTasksBackend { ...@@ -1572,9 +1619,26 @@ impl TurboTasksBackend {
// Tasks were already consumed by take_snapshot, so a future snapshot // Tasks were already consumed by take_snapshot, so a future snapshot
// would not re-persist them — returning an error signals to the caller // would not re-persist them — returning an error signals to the caller
// that further persist attempts would corrupt the task graph in storage. // that further persist attempts would corrupt the task graph in storage.
// Recomputed each commit rather than accumulated: an id that was freed and has since been
// resurrected simply drops out of the set. Ids freed by *this* cycle's eviction are not
// here yet — eviction runs after this returns — so they ride the next commit, which is
// also the commit after their tombstone is durable.
//
// At shutdown there is no next commit, so fold in the tasks this snapshot is tombstoning
// right now. Nothing can resurrect them: this is the process's last write.
let free_task_ids = if self.gc_enabled {
let mut ids = self.deferred_id_reuse.persistable();
if let Some(tombstoned) = &tombstoned_this_commit {
ids.extend(tombstoned.lock().iter().copied());
}
Some(ids)
} else {
None
};
let snapshot_meta = self.backing_storage.save_snapshot( let snapshot_meta = self.backing_storage.save_snapshot(
suspended_operations, suspended_operations,
gc_roots_to_persist, gc_roots_to_persist,
free_task_ids,
task_snapshots, task_snapshots,
)?; )?;
span.record("snapshot_meta", display(snapshot_meta)); span.record("snapshot_meta", display(snapshot_meta));
......
...@@ -37,6 +37,8 @@ enum InfraKey { ...@@ -37,6 +37,8 @@ enum InfraKey {
Operations = 0, Operations = 0,
NextFreeTaskId = 1, NextFreeTaskId = 1,
GcRoots = 2, GcRoots = 2,
/// Task ids that GC freed and eviction erased, available for reuse by the next session.
FreeTaskIds = 3,
} }
impl InfraKey { impl InfraKey {
...@@ -263,10 +265,27 @@ impl TurboBackingStorage { ...@@ -263,10 +265,27 @@ impl TurboBackingStorage {
get(&self.inner.database).context("Unable to read GC roots from database") get(&self.inner.database).context("Unable to read GC roots from database")
} }
/// Reads the persisted free task id set (see [`InfraKey::FreeTaskIds`]). Empty on a fresh
/// database.
///
/// The caller must validate the contents before trusting them; a corrupt or stale entry could
/// otherwise hand out an id that is still in use.
pub(crate) fn free_task_ids(&self) -> Result<Vec<TaskId>> {
fn get(database: &TurboKeyValueDatabase) -> Result<Vec<TaskId>> {
let Some(ids) = database.get(KeySpace::Infra, InfraKey::FreeTaskIds.key().as_ref())?
else {
return Ok(Vec::new());
};
Ok(turbo_bincode_decode(ids.borrow())?)
}
get(&self.inner.database).context("Unable to read free task ids from database")
}
pub(crate) fn save_snapshot<I>( pub(crate) fn save_snapshot<I>(
&self, &self,
operations: Vec<Arc<AnyOperation>>, operations: Vec<Arc<AnyOperation>>,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
free_task_ids: Option<Vec<TaskId>>,
snapshots: Vec<I>, snapshots: Vec<I>,
) -> Result<SnapshotMeta> ) -> Result<SnapshotMeta>
where where
...@@ -367,7 +386,7 @@ impl TurboBackingStorage { ...@@ -367,7 +386,7 @@ impl TurboBackingStorage {
let mut next_task_id = get_next_free_task_id(&batch)?; let mut next_task_id = get_next_free_task_id(&batch)?;
next_task_id = next_task_id.max(snapshot_meta.max_next_task_id + 1); next_task_id = next_task_id.max(snapshot_meta.max_next_task_id + 1);
save_infra(&batch, next_task_id, operations, roots)?; save_infra(&batch, next_task_id, operations, roots, free_task_ids)?;
{ {
let _span = tracing::trace_span!("commit").entered(); let _span = tracing::trace_span!("commit").entered();
// Byte totals are the physical on-disk bytes (post-compression, including .sst / // Byte totals are the physical on-disk bytes (post-compression, including .sst /
...@@ -496,6 +515,7 @@ fn save_infra( ...@@ -496,6 +515,7 @@ fn save_infra(
next_task_id: u32, next_task_id: u32,
operations: Vec<Arc<AnyOperation>>, operations: Vec<Arc<AnyOperation>>,
roots: Option<Vec<(TaskId, TtlCounter)>>, roots: Option<Vec<(TaskId, TtlCounter)>>,
free_task_ids: Option<Vec<TaskId>>,
) -> Result<(), anyhow::Error> { ) -> Result<(), anyhow::Error> {
batch batch
.put( .put(
...@@ -528,6 +548,20 @@ fn save_infra( ...@@ -528,6 +548,20 @@ fn save_infra(
) )
.context("Unable to write GC roots")?; .context("Unable to write GC roots")?;
} }
if let Some(free_task_ids) = free_task_ids {
let _span =
tracing::trace_span!("update free task ids", free_task_ids = free_task_ids.len())
.entered();
let free_task_ids =
turbo_bincode_encode(&free_task_ids).context("Unable to serialize free task ids")?;
batch
.put(
KeySpace::Infra,
WriteBuffer::Borrowed(InfraKey::FreeTaskIds.key().as_ref()),
WriteBuffer::SmallVec(free_task_ids),
)
.context("Unable to write free task ids")?;
}
// Safety: save_infra is called after all concurrent writes to Infra are done. // Safety: save_infra is called after all concurrent writes to Infra are done.
unsafe { batch.flush(KeySpace::Infra)? }; unsafe { batch.flush(KeySpace::Infra)? };
Ok(()) Ok(())
...@@ -730,6 +764,7 @@ mod tests { ...@@ -730,6 +764,7 @@ mod tests {
storage.save_snapshot( storage.save_snapshot(
Vec::new(), Vec::new(),
None, None,
None,
vec![vec![SnapshotItem::Delete { vec![vec![SnapshotItem::Delete {
task_id: deleted_id, task_id: deleted_id,
task_type_hash: collision_hash.to_le_bytes(), task_type_hash: collision_hash.to_le_bytes(),
......
...@@ -20,7 +20,7 @@ use turbo_tasks_backend::{BackendOptions, EvictionMode}; ...@@ -20,7 +20,7 @@ use turbo_tasks_backend::{BackendOptions, EvictionMode};
use crate::{ use crate::{
gc_fixture::{Selector, create_selector}, gc_fixture::{Selector, create_selector},
util::create_tt_with_options, util::{create_persistence_dir, create_tt_with_options, reopen_tt_with_id_reuse},
}; };
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -58,7 +58,7 @@ async fn select_generation(selector: ResolvedVc<Selector>) -> Result<Vc<u32>> { ...@@ -58,7 +58,7 @@ async fn select_generation(selector: ResolvedVc<Selector>) -> Result<Vc<u32>> {
} }
/// Options with GC on and the reuse delay pinned, so a test controls the window explicitly rather /// Options with GC on and the reuse delay pinned, so a test controls the window explicitly rather
/// than depending on the default or the env var. /// than depending on the default.
fn options(delay_cycles: u32) -> BackendOptions { fn options(delay_cycles: u32) -> BackendOptions {
BackendOptions { BackendOptions {
num_workers: Some(2), num_workers: Some(2),
...@@ -183,3 +183,65 @@ async fn freed_ids_wait_out_the_window() { ...@@ -183,3 +183,65 @@ async fn freed_ids_wait_out_the_window() {
tt.stop_and_wait().await; tt.stop_and_wait().await;
} }
/// Ids freed in one session are offered to the next one: the id space does not grow just because
/// the process restarted. This is the cross-session half of reuse — the in-memory free list dies
/// with the process, so without the persisted set a restart would always mint fresh ids.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn freed_ids_survive_a_restart() {
let dir = create_persistence_dir("gc_id_reuse_cross_session");
// Session 1: build a subtree, orphan it, collect it, and shut down.
let tt = reopen_tt_with_id_reuse(&dir, 0);
let tt2 = tt.clone();
let result = turbo_tasks::run_once(tt.clone(), async move {
unmark_top_level_task_may_leak_eventually_consistent_state();
let selector_op = create_selector(false);
let selector_vc = selector_op.resolve().strongly_consistent().await?;
let selector = selector_op.read_strongly_consistent().await?;
let output = select_generation(selector_vc);
output.read_strongly_consistent().await?;
selector.set(true);
output.read_strongly_consistent().await?;
anyhow::Ok(())
})
.await;
result.unwrap();
let collected = tt2.backend().gc_for_testing(&tt2);
assert!(collected > 0, "the orphaned branch should be collected");
tt2.backend().snapshot_and_evict_for_testing(&tt2);
let (_, watermark_session1) = tt2.backend().id_reuse_state_for_testing();
tt.stop_and_wait().await;
// Session 2: the freed ids should have been seeded into the factory's free list, so the
// allocation watermark starts where session 1 left it and new tasks draw from the free list.
let tt = reopen_tt_with_id_reuse(&dir, 0);
let tt2 = tt.clone();
let (_, watermark_start) = tt2.backend().id_reuse_state_for_testing();
assert!(
watermark_start <= watermark_session1,
"a restart must not advance the id watermark ({watermark_session1} -> {watermark_start})"
);
let needed = collected as u32;
let result = turbo_tasks::run_once(tt.clone(), async move {
unmark_top_level_task_may_leak_eventually_consistent_state();
allocate_fresh(500, needed)
.read_strongly_consistent()
.await?;
anyhow::Ok(())
})
.await;
result.unwrap();
let (_, watermark_after) = tt2.backend().id_reuse_state_for_testing();
let minted = watermark_after - watermark_start;
assert!(
minted <= 1,
"allocating {needed} tasks in a fresh session should draw on the ids the previous session \
freed; the factory minted {minted} instead ({watermark_start} -> {watermark_after})"
);
tt.stop_and_wait().await;
}
...@@ -96,6 +96,39 @@ pub fn reopen_tt_with_gc_ttl( ...@@ -96,6 +96,39 @@ pub fn reopen_tt_with_gc_ttl(
open_tt_at_with_gc(dir.path(), 2, Some(true), Some(gc_root_ttl), None) open_tt_at_with_gc(dir.path(), 2, Some(true), Some(gc_root_ttl), None)
} }
/// Reopens a backend on an existing persistence directory with GC on and the id-reuse window
/// pinned, for tests that assert on ids surviving a restart.
pub fn reopen_tt_with_id_reuse(
dir: &tempfile::TempDir,
id_reuse_delay_cycles: u32,
) -> Arc<TurboTasks<TurboTasksBackend>> {
TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
num_workers: Some(2),
small_preallocation: true,
storage_mode: Some(turbo_tasks_backend::StorageMode::ReadWriteOnShutdown),
eviction_mode: EvictionMode::Full,
gc: Some(true),
id_reuse_delay_cycles: Some(id_reuse_delay_cycles),
..Default::default()
},
turbo_tasks_backend::turbo_backing_storage(
dir.path(),
&GitVersionInfo {
describe: "test-unversioned",
dirty: false,
},
BackingStorageOptions {
is_short_session: true,
skip_compaction: true,
..Default::default()
},
)
.unwrap()
.0,
))
}
/// A fresh persistent backend in its own temp directory, with `num_workers` workers. /// A fresh persistent backend in its own temp directory, with `num_workers` workers.
pub fn create_tt_with_workers( pub fn create_tt_with_workers(
name: &str, name: &str,
......
...@@ -164,6 +164,18 @@ where ...@@ -164,6 +164,18 @@ where
self.free_ids.pop().unwrap_or_else(|_| self.factory.get()) self.free_ids.pop().unwrap_or_else(|_| self.factory.get())
} }
/// Seed the free list with ids that are known to be unused.
///
/// # Safety
///
/// Same contract as [`IdFactoryWithReuse::reuse`], for every id: each must be a valid id in
/// this factory's range that nothing refers to any more.
pub unsafe fn seed_free_ids(&self, ids: impl IntoIterator<Item = T>) {
for id in ids {
let _ = self.free_ids.push(id);
}
}
/// The next id this factory would mint from its counter, ignoring the free list. /// The next id this factory would mint from its counter, ignoring the free list.
/// ///
/// A watermark, not an allocation: it is the high-water mark of the id space this factory has /// A watermark, not an allocation: it is the high-water mark of the id space this factory has
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment