Unverified Commit daa0e7e3 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Preserve current GC and restore semantics with pointer pages

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 1370f484
use std::{ use std::{
hint::black_box, hint::black_box,
mem::size_of,
num::NonZeroU64, num::NonZeroU64,
sync::{ sync::{
Arc, OnceLock, Arc, OnceLock,
atomic::{AtomicU8, AtomicUsize, Ordering}, atomic::{AtomicPtr, AtomicU8, AtomicUsize, Ordering},
}, },
thread, thread,
time::Instant, time::Instant,
...@@ -296,7 +297,7 @@ fn report_reclamation(count: u32, stride: u32) { ...@@ -296,7 +297,7 @@ fn report_reclamation(count: u32, stride: u32) {
let before = TurboMalloc::allocation_counters(); let before = TurboMalloc::allocation_counters();
let dense = dense_map(count, stride); let dense = dense_map(count, stride);
let live = TurboMalloc::allocation_counters(); let live = TurboMalloc::allocation_counters();
let loaded_before = dense.loaded_page_count(); let loaded_before = dense.pages(operation()).len();
for i in 1..=count { for i in 1..=count {
assert!(dense_remove_discard(&dense, task_id(i * stride))); assert!(dense_remove_discard(&dense, task_id(i * stride)));
} }
...@@ -314,9 +315,9 @@ fn report_reclamation(count: u32, stride: u32) { ...@@ -314,9 +315,9 @@ fn report_reclamation(count: u32, stride: u32) {
after_counter_delta={after_delta}", after_counter_delta={after_delta}",
count * stride, count * stride,
task_page_map::PAGE_SIZE, task_page_map::PAGE_SIZE,
TaskMap::<Payload>::directory_entry_size(), size_of::<AtomicPtr<()>>(),
dense.directory_entry_count(), ((count as usize * stride as usize) >> PAGE_SHIFT) + 1,
dense.loaded_page_count(), dense.pages(operation()).len(),
); );
black_box(dense); black_box(dense);
} }
......
...@@ -37,6 +37,7 @@ use crate::{ ...@@ -37,6 +37,7 @@ use crate::{
snapshot_coordinator::SnapshotPhase, snapshot_coordinator::SnapshotPhase,
storage::{SpecificTaskDataCategory, TaskDataCategory}, storage::{SpecificTaskDataCategory, TaskDataCategory},
storage_schema::TaskStorageAccessors, storage_schema::TaskStorageAccessors,
task_page_map::StorageAccessToken,
}, },
backing_storage::SnapshotItem, backing_storage::SnapshotItem,
}; };
...@@ -224,7 +225,7 @@ impl TurboTasksBackend { ...@@ -224,7 +225,7 @@ impl TurboTasksBackend {
.collect::<FxHashMap<TaskId, TtlCounter>>(); .collect::<FxHashMap<TaskId, TtlCounter>>();
let roots_before = roots.clone(); let roots_before = roots.clone();
let aged_out = self.gc_roots_refresh_and_age_out(&mut roots, now); let aged_out = self.gc_roots_refresh_and_age_out(&mut roots, now, phase.access_token());
let aged_out_count = aged_out.len(); let aged_out_count = aged_out.len();
// TODO(perf): recycle the task ids of collected tasks. // TODO(perf): recycle the task ids of collected tasks.
...@@ -373,12 +374,13 @@ impl TurboTasksBackend { ...@@ -373,12 +374,13 @@ impl TurboTasksBackend {
&self, &self,
map: &mut FxHashMap<TaskId, TtlCounter>, map: &mut FxHashMap<TaskId, TtlCounter>,
now: u64, now: u64,
access: StorageAccessToken<'_>,
) -> Vec<TaskId> { ) -> Vec<TaskId> {
let ttl_ms = self.gc_root_ttl.as_millis() as u64; let ttl_ms = self.gc_root_ttl.as_millis() as u64;
let mut aged_out = Vec::new(); let mut aged_out = Vec::new();
map.retain(|id, counter| { map.retain(|id, counter| {
if self.storage.with_task(*id, |_| ()).is_some() { if self.storage.with_task(access, *id, |_| ()).is_some() {
// Resident: `gc_scan_roots` decides. Drop it either way. // Resident: `gc_scan_roots` decides. Drop it either way.
return false; return false;
} }
......
...@@ -423,7 +423,9 @@ impl TurboTasksBackend { ...@@ -423,7 +423,9 @@ impl TurboTasksBackend {
return TestSnapshotOutcome::default(); return TestSnapshotOutcome::default();
} }
}; };
let eviction_counts = self.storage.evict_after_snapshot(&self.snapshot_coord, None); let eviction_counts = self
.storage
.evict_after_snapshot(&self.snapshot_coord, None);
TestSnapshotOutcome { TestSnapshotOutcome {
had_new_data, had_new_data,
eviction_counts, eviction_counts,
...@@ -438,7 +440,6 @@ impl TurboTasksBackend { ...@@ -438,7 +440,6 @@ impl TurboTasksBackend {
self.storage self.storage
.resident_task_count_for_testing(operation.access_token()) .resident_task_count_for_testing(operation.access_token())
} }
}
/// The persistent `parent_count` of a resident task (0 if absent or not resident). Test-only /// The persistent `parent_count` of a resident task (0 if absent or not resident). Test-only
/// hook for verifying incremental refcount maintenance. /// hook for verifying incremental refcount maintenance.
......
...@@ -195,6 +195,11 @@ impl<O> SnapshotCoordinator<O> { ...@@ -195,6 +195,11 @@ impl<O> SnapshotCoordinator<O> {
suspended_operations, suspended_operations,
} }
} }
/// Exclude operations for a snapshot, eviction-removal, GC, or shutdown phase.
pub fn begin_exclusion(&self) -> SnapshotPhase<'_, O> {
self.begin_snapshot()
}
} }
/// Guard returned by [`SnapshotCoordinator::begin_operation`]. Decrements the /// Guard returned by [`SnapshotCoordinator::begin_operation`]. Decrements the
......
...@@ -752,10 +752,6 @@ pub struct TaskEntryGuard<'a> { ...@@ -752,10 +752,6 @@ pub struct TaskEntryGuard<'a> {
} }
impl<'a> TaskEntryGuard<'a> { impl<'a> TaskEntryGuard<'a> {
pub fn discard(self) {
self.inner.inner.vacate();
}
pub fn into_write_guard(self) -> StorageWriteGuard<'a> { pub fn into_write_guard(self) -> StorageWriteGuard<'a> {
self.inner self.inner
} }
...@@ -1295,13 +1291,12 @@ mod tests { ...@@ -1295,13 +1291,12 @@ mod tests {
fn new_task_is_pinned_during_construction() { fn new_task_is_pinned_during_construction() {
let storage = Storage::new(2, true); let storage = Storage::new(2, true);
let task_id = non_transient_task(1); let task_id = non_transient_task(1);
storage.initialize_new_task(task_id, None); storage.initialize_new_task(access(), task_id, None);
let task = storage.access_mut(task_id); let task = storage.access_mut(access(), task_id);
assert_eq!(task.gc_transient_ref_count(), 1); assert_eq!(task.gc_transient_ref_count(), 1);
assert!(!task.gc_collectible()); assert!(!task.gc_collectible());
} }
#[test] #[test]
fn unrelated_task_locks_do_not_block_each_other() { fn unrelated_task_locks_do_not_block_each_other() {
let storage = Arc::new(Storage::new(2, true)); let storage = Arc::new(Storage::new(2, true));
......
...@@ -100,7 +100,6 @@ impl fmt::Debug for IntrusiveTaskLock { ...@@ -100,7 +100,6 @@ impl fmt::Debug for IntrusiveTaskLock {
} }
} }
/// The complete task storage schema. /// The complete task storage schema.
/// ///
/// This struct defines all storage fields for a task. The `#[task_storage]` macro /// This struct defines all storage fields for a task. The `#[task_storage]` macro
...@@ -1951,7 +1950,6 @@ mod tests { ...@@ -1951,7 +1950,6 @@ mod tests {
result result
} }
#[test] #[test]
fn const_new_slots_have_independent_unlocked_mutexes() { fn const_new_slots_have_independent_unlocked_mutexes() {
let slots = [const { TaskStorage::new() }; 2]; let slots = [const { TaskStorage::new() }; 2];
......
...@@ -324,6 +324,7 @@ impl<T: TaskSlotValue + Send + 'static> PagedVec<T> { ...@@ -324,6 +324,7 @@ impl<T: TaskSlotValue + Send + 'static> PagedVec<T> {
.count() .count()
} }
#[cfg(test)]
fn live_page_count(&self) -> usize { fn live_page_count(&self) -> usize {
self.entries() self.entries()
.filter(|(_, entry)| !entry.page.load(Ordering::Acquire).is_null()) .filter(|(_, entry)| !entry.page.load(Ordering::Acquire).is_null())
...@@ -365,7 +366,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> { ...@@ -365,7 +366,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
} }
} }
fn namespace_and_index(&self, key: TaskId) -> (&PagedVec<T>, usize) { fn id_space(&self, key: TaskId) -> (&PagedVec<T>, usize) {
let raw = *key as usize; let raw = *key as usize;
if key.is_transient() { if key.is_transient() {
(&self.transient, raw & !(TRANSIENT_TASK_BIT as usize)) (&self.transient, raw & !(TRANSIENT_TASK_BIT as usize))
...@@ -379,7 +380,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> { ...@@ -379,7 +380,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
key: TaskId, key: TaskId,
access: StorageAccessToken<'a>, access: StorageAccessToken<'a>,
) -> Option<TaskMapGuard<'a, T>> { ) -> Option<TaskMapGuard<'a, T>> {
let (namespace, index) = self.namespace_and_index(key); let (namespace, index) = self.id_space(key);
let entry = namespace.entry(index)?; let entry = namespace.entry(index)?;
let page = entry.load()?; let page = entry.load()?;
let offset = index & PAGE_MASK; let offset = index & PAGE_MASK;
...@@ -401,7 +402,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> { ...@@ -401,7 +402,7 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
if let Some(task) = self.get(key, access) { if let Some(task) = self.get(key, access) {
return task; return task;
} }
let (namespace, index) = self.namespace_and_index(key); let (namespace, index) = self.id_space(key);
let entry = namespace.get_or_create_entry(index); let entry = namespace.get_or_create_entry(index);
let page = entry.get_or_insert(); let page = entry.get_or_insert();
let offset = index & PAGE_MASK; let offset = index & PAGE_MASK;
...@@ -467,22 +468,22 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> { ...@@ -467,22 +468,22 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
self.persistent.reclaim_empty_pages(access) + self.transient.reclaim_empty_pages(access) self.persistent.reclaim_empty_pages(access) + self.transient.reclaim_empty_pages(access)
} }
#[allow(dead_code)] #[cfg(test)]
pub(crate) fn loaded_page_count(&self) -> usize { pub(crate) fn loaded_page_count(&self) -> usize {
self.persistent.live_page_count() + self.transient.live_page_count() self.persistent.live_page_count() + self.transient.live_page_count()
} }
#[allow(dead_code)] #[cfg(test)]
pub(crate) fn directory_entry_count(&self) -> usize { pub(crate) fn directory_entry_count(&self) -> usize {
self.persistent.entries.count() + self.transient.entries.count() self.persistent.entries.count() + self.transient.entries.count()
} }
#[allow(dead_code)] #[cfg(test)]
pub(crate) fn directory_entry_size() -> usize { pub(crate) fn directory_entry_size() -> usize {
std::mem::size_of::<PageDirectoryEntry<T>>() std::mem::size_of::<PageDirectoryEntry<T>>()
} }
#[allow(dead_code)] #[cfg(test)]
pub(crate) fn pointer_page_size() -> usize { pub(crate) fn pointer_page_size() -> usize {
std::mem::size_of::<TaskPointerPage<T>>() std::mem::size_of::<TaskPointerPage<T>>()
} }
...@@ -506,13 +507,6 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> { ...@@ -506,13 +507,6 @@ impl<T: TaskSlotValue + Send + 'static> TaskMap<T> {
pages pages
} }
pub(crate) fn persistent_pages<'a>(
&'a self,
access: StorageAccessToken<'a>,
) -> Vec<TaskPageRef<'a, T>> {
self.persistent.live_pages(access, false)
}
pub(crate) fn page_indices(&self, _access: StorageAccessToken<'_>) -> Vec<(bool, usize)> { pub(crate) fn page_indices(&self, _access: StorageAccessToken<'_>) -> Vec<(bool, usize)> {
self.persistent self.persistent
.entries() .entries()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment