Unverified Commit daa0e7e3 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Preserve current GC and restore semantics with pointer pages

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 1370f484
use std::{
hint::black_box,
mem::size_of,
num::NonZeroU64,
sync::{
Arc, OnceLock,
atomic::{AtomicU8, AtomicUsize, Ordering},
atomic::{AtomicPtr, AtomicU8, AtomicUsize, Ordering},
},
thread,
time::Instant,
......@@ -296,7 +297,7 @@ fn report_reclamation(count: u32, stride: u32) {
let before = TurboMalloc::allocation_counters();
let dense = dense_map(count, stride);
let live = TurboMalloc::allocation_counters();
let loaded_before = dense.loaded_page_count();
let loaded_before = dense.pages(operation()).len();
for i in 1..=count {
assert!(dense_remove_discard(&dense, task_id(i * stride)));
}
......@@ -314,9 +315,9 @@ fn report_reclamation(count: u32, stride: u32) {
after_counter_delta={after_delta}",
count * stride,
task_page_map::PAGE_SIZE,
TaskMap::<Payload>::directory_entry_size(),
dense.directory_entry_count(),
dense.loaded_page_count(),
size_of::<AtomicPtr<()>>(),
((count as usize * stride as usize) >> PAGE_SHIFT) + 1,
dense.pages(operation()).len(),
);
black_box(dense);
}
......
......@@ -37,6 +37,7 @@ use crate::{
snapshot_coordinator::SnapshotPhase,
storage::{SpecificTaskDataCategory, TaskDataCategory},
storage_schema::TaskStorageAccessors,
task_page_map::StorageAccessToken,
},
backing_storage::SnapshotItem,
};
......@@ -224,7 +225,7 @@ impl TurboTasksBackend {
.collect::<FxHashMap<TaskId, TtlCounter>>();
let roots_before = roots.clone();
let aged_out = self.gc_roots_refresh_and_age_out(&mut roots, now);
let aged_out = self.gc_roots_refresh_and_age_out(&mut roots, now, phase.access_token());
let aged_out_count = aged_out.len();
// TODO(perf): recycle the task ids of collected tasks.
......@@ -373,12 +374,13 @@ impl TurboTasksBackend {
&self,
map: &mut FxHashMap<TaskId, TtlCounter>,
now: u64,
access: StorageAccessToken<'_>,
) -> Vec<TaskId> {
let ttl_ms = self.gc_root_ttl.as_millis() as u64;
let mut aged_out = Vec::new();
map.retain(|id, counter| {
if self.storage.with_task(*id, |_| ()).is_some() {
if self.storage.with_task(access, *id, |_| ()).is_some() {
// Resident: `gc_scan_roots` decides. Drop it either way.
return false;
}
......
......@@ -423,7 +423,9 @@ impl TurboTasksBackend {
return TestSnapshotOutcome::default();
}
};
let eviction_counts = self.storage.evict_after_snapshot(&self.snapshot_coord, None);
let eviction_counts = self
.storage
.evict_after_snapshot(&self.snapshot_coord, None);
TestSnapshotOutcome {
had_new_data,
eviction_counts,
......@@ -438,7 +440,6 @@ impl TurboTasksBackend {
self.storage
.resident_task_count_for_testing(operation.access_token())
}
}
/// The persistent `parent_count` of a resident task (0 if absent or not resident). Test-only
/// hook for verifying incremental refcount maintenance.
......
......@@ -195,6 +195,11 @@ impl<O> SnapshotCoordinator<O> {
suspended_operations,
}
}
/// Exclude operations for a snapshot, eviction-removal, GC, or shutdown phase.
pub fn begin_exclusion(&self) -> SnapshotPhase<'_, O> {
self.begin_snapshot()
}
}
/// Guard returned by [`SnapshotCoordinator::begin_operation`]. Decrements the
......
......@@ -752,10 +752,6 @@ pub struct TaskEntryGuard<'a> {
}
impl<'a> TaskEntryGuard<'a> {
pub fn discard(self) {
self.inner.inner.vacate();
}
pub fn into_write_guard(self) -> StorageWriteGuard<'a> {
self.inner
}
......@@ -1295,13 +1291,12 @@ mod tests {
fn new_task_is_pinned_during_construction() {
let storage = Storage::new(2, true);
let task_id = non_transient_task(1);
storage.initialize_new_task(task_id, None);
let task = storage.access_mut(task_id);
storage.initialize_new_task(access(), task_id, None);
let task = storage.access_mut(access(), task_id);
assert_eq!(task.gc_transient_ref_count(), 1);
assert!(!task.gc_collectible());
}
#[test]
fn unrelated_task_locks_do_not_block_each_other() {
let storage = Arc::new(Storage::new(2, true));
......
......@@ -100,7 +100,6 @@ impl fmt::Debug for IntrusiveTaskLock {
}
}
/// The complete task storage schema.
///
/// This struct defines all storage fields for a task. The `#[task_storage]` macro
......@@ -1951,7 +1950,6 @@ mod tests {
result
}
#[test]
fn const_new_slots_have_independent_unlocked_mutexes() {
let slots = [const { TaskStorage::new() }; 2];
......
......@@ -324,6 +324,7 @@ impl<T: TaskSlotValue + Send + 'static> PagedVec<T> {
.count()
}
#[cfg(test)]
fn live_page_count(&self) -> usize {
self.entries()
.filter(|(_, entry)| !entry.page.load(Ordering::Acquire).is_null())
......@@ -365,7 +366,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
}
}
fn namespace_and_index(&self, key: TaskId) -> (&PagedVec<T>, usize) {
fn id_space(&self, key: TaskId) -> (&PagedVec<T>, usize) {
let raw = *key as usize;
if key.is_transient() {
(&self.transient, raw & !(TRANSIENT_TASK_BIT as usize))
......@@ -379,7 +380,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
key: TaskId,
access: StorageAccessToken<'a>,
) -> Option<TaskMapGuard<'a, T>> {
let (namespace, index) = self.namespace_and_index(key);
let (namespace, index) = self.id_space(key);
let entry = namespace.entry(index)?;
let page = entry.load()?;
let offset = index & PAGE_MASK;
......@@ -401,7 +402,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
if let Some(task) = self.get(key, access) {
return task;
}
let (namespace, index) = self.namespace_and_index(key);
let (namespace, index) = self.id_space(key);
let entry = namespace.get_or_create_entry(index);
let page = entry.get_or_insert();
let offset = index & PAGE_MASK;
......@@ -467,22 +468,22 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
self.persistent.reclaim_empty_pages(access) + self.transient.reclaim_empty_pages(access)
}
#[allow(dead_code)]
#[cfg(test)]
pub(crate) fn loaded_page_count(&self) -> usize {
self.persistent.live_page_count() + self.transient.live_page_count()
}
#[allow(dead_code)]
#[cfg(test)]
pub(crate) fn directory_entry_count(&self) -> usize {
self.persistent.entries.count() + self.transient.entries.count()
}
#[allow(dead_code)]
#[cfg(test)]
pub(crate) fn directory_entry_size() -> usize {
std::mem::size_of::<PageDirectoryEntry<T>>()
}
#[allow(dead_code)]
#[cfg(test)]
pub(crate) fn pointer_page_size() -> usize {
std::mem::size_of::<TaskPointerPage<T>>()
}
......@@ -506,13 +507,6 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
pages
}
pub(crate) fn persistent_pages<'a>(
&'a self,
access: StorageAccessToken<'a>,
) -> Vec<TaskPageRef<'a, T>> {
self.persistent.live_pages(access, false)
}
pub(crate) fn page_indices(&self, _access: StorageAccessToken<'_>) -> Vec<(bool, usize)> {
self.persistent
.entries()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment