Unverified Commit 9e97759a authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: refactor data storage to avoid reverse task cache (#88492)

### What?

* task type is stored as CachedDataItem
* reverse task cache is removed
parent 4fe058fc
......@@ -5,6 +5,7 @@ use std::{
mem::take,
num::NonZeroU32,
ops::{ControlFlow, Deref},
sync::Arc,
thread::yield_now,
time::{Duration, Instant},
};
......@@ -21,8 +22,10 @@ use tracing::span::Span;
feature = "trace_aggregation_update",
feature = "trace_find_and_schedule"
))]
use tracing::{span::Span, trace_span};
use turbo_tasks::{FxIndexMap, TaskExecutionReason, TaskId};
use tracing::trace_span;
use turbo_tasks::{
FxIndexMap, TaskExecutionReason, TaskId, backend::CachedTaskType, event::EventDescription,
};
#[cfg(feature = "trace_task_dirty")]
use crate::backend::operation::invalidate::TaskDirtyCause;
......@@ -264,6 +267,8 @@ pub enum AggregationUpdateJob {
// upon attempted serialization) similar to #[serde(skip)] on variants
#[bincode(skip, default = "unreachable_decode")]
task: TaskId,
/// Set the task type if not already set
task_type: Option<Arc<CachedTaskType>>,
},
/// Increases the active counters of the tasks
IncreaseActiveCounts {
......@@ -1153,12 +1158,12 @@ impl AggregationUpdateQueue {
}
}
}
AggregationUpdateJob::IncreaseActiveCount { task } => {
self.increase_active_count(ctx, task);
AggregationUpdateJob::IncreaseActiveCount { task, task_type } => {
self.increase_active_count(ctx, task, task_type);
}
AggregationUpdateJob::IncreaseActiveCounts { mut task_ids } => {
if let Some(task_id) = task_ids.pop() {
self.increase_active_count(ctx, task_id);
self.increase_active_count(ctx, task_id, None);
if !task_ids.is_empty() {
self.jobs.push_front(AggregationUpdateJobItem::new(
AggregationUpdateJob::IncreaseActiveCounts { task_ids },
......@@ -1371,6 +1376,7 @@ impl AggregationUpdateQueue {
if has_active_count {
self.push(AggregationUpdateJob::IncreaseActiveCount {
task: task_id,
task_type: None,
});
}
}
......@@ -1476,11 +1482,12 @@ impl AggregationUpdateQueue {
activeness_state.set_active_until_clean();
}
}
if let Some((reason, parent_priority)) = should_schedule
&& task.add_scheduled(reason, || ctx.get_task_desc_fn(task_id))
{
drop(task);
ctx.schedule(task_id, parent_priority);
if let Some((reason, parent_priority)) = should_schedule {
let description = EventDescription::new(|| task.get_task_desc_fn());
if task.add_scheduled(reason, description) {
drop(task);
ctx.schedule(task_id, parent_priority);
}
}
}
......@@ -1640,10 +1647,15 @@ impl AggregationUpdateQueue {
"inner_of_uppers_lost_follower is not able to remove follower \
{lost_follower_id} ({}) from {} as they don't exist as upper or follower \
edges",
ctx.get_task_description(lost_follower_id),
ctx.task(lost_follower_id, TaskDataCategory::Data)
.get_task_description(),
upper_ids
.iter()
.map(|id| format!("{} ({})", id, ctx.get_task_description(*id)))
.map(|id| format!(
"{} ({})",
id,
ctx.task(*id, TaskDataCategory::Data).get_task_description()
))
.collect::<Vec<_>>()
.join(", ")
);
......@@ -1789,10 +1801,15 @@ impl AggregationUpdateQueue {
{upper_id} ({}) as they don't exist as upper or follower edges",
lost_follower_ids
.iter()
.map(|id| format!("{} ({})", id, ctx.get_task_description(*id)))
.map(|id| format!(
"{} ({})",
id,
ctx.task(*id, TaskDataCategory::Data).get_task_description()
))
.collect::<Vec<_>>()
.join(", "),
ctx.get_task_description(upper_id),
ctx.task(upper_id, TaskDataCategory::Data)
.get_task_description()
)
}
self.push(AggregationUpdateJob::InnerOfUpperLostFollowers {
......@@ -1987,7 +2004,7 @@ impl AggregationUpdateQueue {
fn inner_of_upper_has_new_followers<T: TaskIdWithOptionalCount, const N: usize>(
&mut self,
ctx: &mut impl ExecuteContext,
ctx: &mut impl ExecuteContext<'_>,
new_follower_ids: SmallVec<[T; N]>,
upper_id: TaskId,
) {
......@@ -2252,6 +2269,7 @@ impl AggregationUpdateQueue {
if has_active_count {
self.push(AggregationUpdateJob::IncreaseActiveCount {
task: new_follower_id,
task_type: None,
});
}
// notify uppers about new follower
......@@ -2337,7 +2355,7 @@ impl AggregationUpdateQueue {
/// Decreases the active count of a task.
///
/// Only used when activeness is tracked.
fn decrease_active_count(&mut self, ctx: &mut impl ExecuteContext, task_id: TaskId) {
fn decrease_active_count(&mut self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) {
#[cfg(feature = "trace_aggregation_update")]
let _span = trace_span!("decrease active count").entered();
......@@ -2381,15 +2399,29 @@ impl AggregationUpdateQueue {
/// Increases the active count of a task.
///
/// Only used when activeness is tracked.
fn increase_active_count(&mut self, ctx: &mut impl ExecuteContext, task_id: TaskId) {
fn increase_active_count(
&mut self,
ctx: &mut impl ExecuteContext,
task_id: TaskId,
task_type: Option<Arc<CachedTaskType>>,
) {
#[cfg(feature = "trace_aggregation_update")]
let _span = trace_span!("increase active count").entered();
let mut task = ctx.task(
task_id,
// For performance reasons this should stay `Meta` and not `All`
TaskDataCategory::Meta,
if task_type.is_some() {
TaskDataCategory::All
} else {
// For performance reasons this should stay `Meta` and not `All`
TaskDataCategory::Meta
},
);
if let Some(task_type) = task_type
&& !task.has_persistent_task_type()
{
let _ = task.set_persistent_task_type(task_type);
}
let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id));
let is_new = state.is_empty();
let is_positive_now = state.increment_active_counter();
......@@ -2423,7 +2455,7 @@ impl AggregationUpdateQueue {
fn update_aggregation_number(
&mut self,
ctx: &mut impl ExecuteContext,
ctx: &mut impl ExecuteContext<'_>,
task_id: TaskId,
base_effective_distance: Option<std::num::NonZero<u32>>,
base_aggregation_number: u32,
......
use std::sync::Arc;
use bincode::{Decode, Encode};
use turbo_tasks::{TaskExecutionReason, TaskId};
use turbo_tasks::{TaskExecutionReason, TaskId, backend::CachedTaskType, event::EventDescription};
use crate::{
backend::{
......@@ -11,6 +13,7 @@ use crate::{
storage_schema::TaskStorageAccessors,
},
data::{InProgressState, InProgressStateInner},
utils::arc_or_owned::ArcOrOwned,
};
#[derive(Encode, Decode, Clone, Default)]
......@@ -27,24 +30,36 @@ impl ConnectChildOperation {
pub fn run(
parent_task_id: Option<TaskId>,
child_task_id: TaskId,
child_task_type: Option<ArcOrOwned<CachedTaskType>>,
mut ctx: impl ExecuteContext<'_>,
) {
if let Some(parent_task_id) = parent_task_id {
let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::All);
let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::Meta);
let Some(InProgressState::InProgress(box InProgressStateInner {
new_children, ..
})) = parent_task.get_in_progress_mut()
})) = parent_task.get_in_progress()
else {
panic!("Task is not in progress while calling another task: {parent_task:?}");
};
// Quick skip if the child was already connected before
if !new_children.insert(child_task_id) {
// We can't call insert here as this would skip the mandatory task type update below
// Instead we only add it after updating the child task type
if new_children.contains(&child_task_id) {
return;
}
if parent_task.children_contains(&child_task_id) {
// It is already connected, we can skip the rest
// but we still need to update the new_children set
let Some(InProgressState::InProgress(box InProgressStateInner {
new_children,
..
})) = parent_task.get_in_progress_mut()
else {
unreachable!();
};
new_children.insert(child_task_id);
return;
}
}
......@@ -63,14 +78,21 @@ impl ConnectChildOperation {
if ctx.should_track_activeness() && parent_task_id.is_some() {
queue.push(AggregationUpdateJob::IncreaseActiveCount {
task: child_task_id,
task_type: child_task_type.map(Arc::from),
});
} else {
let mut child_task = ctx.task(child_task_id, TaskDataCategory::All);
if let Some(child_task_type) = child_task_type
&& !child_task.has_persistent_task_type()
{
child_task.set_persistent_task_type(child_task_type.into());
}
if !child_task.has_output()
&& child_task.add_scheduled(TaskExecutionReason::Connect, || {
ctx.get_task_desc_fn(child_task_id)
})
&& child_task.add_scheduled(
TaskExecutionReason::Connect,
EventDescription::new(|| child_task.get_task_desc_fn()),
)
{
ctx.schedule_task(child_task, ctx.get_current_task_priority());
}
......@@ -80,6 +102,30 @@ impl ConnectChildOperation {
aggregation_update: queue,
}
.execute(&mut ctx);
if let Some(parent_task_id) = parent_task_id {
let mut parent_task = ctx.task(parent_task_id, TaskDataCategory::Meta);
let Some(InProgressState::InProgress(box InProgressStateInner {
new_children, ..
})) = parent_task.get_in_progress_mut()
else {
panic!("Task is not in progress while calling another task: {parent_task:?}");
};
// Really add the child to the new children set
if !new_children.insert(child_task_id) {
drop(parent_task);
// There was a concurrent connect child operation,
// so we need to undo the active count update.
AggregationUpdateQueue::run(
AggregationUpdateJob::DecreaseActiveCount {
task: child_task_id,
},
&mut ctx,
);
}
}
}
}
......
use bincode::{Decode, Encode};
use smallvec::SmallVec;
use turbo_tasks::{TaskExecutionReason, TaskId};
use turbo_tasks::{TaskExecutionReason, TaskId, event::EventDescription};
use crate::{
backend::{
......@@ -107,25 +107,28 @@ pub enum TaskDirtyCause {
}
#[cfg(feature = "trace_task_dirty")]
struct TaskDirtyCauseInContext<'l, 'e, E: ExecuteContext<'e>> {
struct TaskDirtyCauseInContext<'l> {
cause: &'l TaskDirtyCause,
ctx: &'l E,
_phantom: std::marker::PhantomData<&'e ()>,
task_description: String,
}
#[cfg(feature = "trace_task_dirty")]
impl<'l, 'e, E: ExecuteContext<'e>> TaskDirtyCauseInContext<'l, 'e, E> {
fn new(cause: &'l TaskDirtyCause, ctx: &'l E) -> Self {
impl<'l> TaskDirtyCauseInContext<'l> {
fn new(cause: &'l TaskDirtyCause, ctx: &'l mut impl ExecuteContext<'_>) -> Self {
Self {
cause,
ctx,
_phantom: Default::default(),
task_description: match cause {
TaskDirtyCause::OutputChange { task_id } => ctx
.task(*task_id, TaskDataCategory::Data)
.get_task_description(),
_ => String::new(),
},
}
}
}
#[cfg(feature = "trace_task_dirty")]
impl<'e, E: ExecuteContext<'e>> std::fmt::Display for TaskDirtyCauseInContext<'_, 'e, E> {
impl std::fmt::Display for TaskDirtyCauseInContext<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.cause {
TaskDirtyCause::InitialDirty => write!(f, "initial dirty"),
......@@ -158,12 +161,8 @@ impl<'e, E: ExecuteContext<'e>> std::fmt::Display for TaskDirtyCauseInContext<'_
turbo_tasks::registry::get_value_type(*value_type).name
)
}
TaskDirtyCause::OutputChange { task_id } => {
write!(
f,
"task {} output changed",
self.ctx.get_task_description(*task_id)
)
TaskDirtyCause::OutputChange { .. } => {
write!(f, "task {} output changed", self.task_description)
}
TaskDirtyCause::CollectiblesChange { collectible_type } => {
write!(
......@@ -184,12 +183,7 @@ pub fn make_task_dirty(
queue: &mut AggregationUpdateQueue,
ctx: &mut impl ExecuteContext<'_>,
) {
if ctx.is_once_task(task_id) {
return;
}
let task = ctx.task(task_id, TaskDataCategory::Meta);
let task = ctx.task(task_id, TaskDataCategory::All);
make_task_dirty_internal(
task,
task_id,
......@@ -224,10 +218,12 @@ pub fn make_task_dirty_internal(
panic!(
"Task {} is immutable, but was made dirty. This should not happen and is a \
bug.{extra_info}",
ctx.get_task_description(task_id),
task.get_task_description(),
);
}
#[cfg(feature = "trace_task_dirty")]
let task_name = task.get_task_name();
if make_stale
&& let Some(InProgressState::InProgress(box InProgressStateInner { stale, .. })) =
task.get_in_progress_mut()
......@@ -237,7 +233,7 @@ pub fn make_task_dirty_internal(
let _span = tracing::trace_span!(
"make task stale",
task_id = display(task_id),
name = ctx.get_task_description(task_id),
name = task_name,
cause = %TaskDirtyCauseInContext::new(&cause, ctx)
)
.entered();
......@@ -250,7 +246,7 @@ pub fn make_task_dirty_internal(
let _span = tracing::trace_span!(
"task already dirty",
task_id = display(task_id),
name = ctx.get_task_description(task_id),
name = task_name,
cause = %TaskDirtyCauseInContext::new(&cause, ctx)
)
.entered();
......@@ -276,7 +272,7 @@ pub fn make_task_dirty_internal(
#[cfg(feature = "trace_task_dirty")]
let _span = tracing::trace_span!(
"session-dependent task already dirty",
name = ctx.get_task_description(task_id),
name = task_name,
cause = %TaskDirtyCauseInContext::new(&cause, ctx)
)
.entered();
......@@ -308,7 +304,7 @@ pub fn make_task_dirty_internal(
let _span = tracing::trace_span!(
"make task dirty",
task_id = display(task_id),
name = ctx.get_task_description(task_id),
name = task_name,
cause = %TaskDirtyCauseInContext::new(&cause, ctx)
)
.entered();
......@@ -334,13 +330,12 @@ pub fn make_task_dirty_internal(
let should_schedule = !ctx.should_track_activeness() || task.has_activeness();
if should_schedule
&& task.add_scheduled(TaskExecutionReason::Invalidated, || {
ctx.get_task_desc_fn(task_id)
})
{
drop(task);
let task = ctx.task(task_id, TaskDataCategory::All);
ctx.schedule_task(task, parent_priority);
if should_schedule {
let description = EventDescription::new(|| task.get_task_desc_fn());
if task.add_scheduled(TaskExecutionReason::Invalidated, description) {
drop(task);
let task = ctx.task(task_id, TaskDataCategory::All);
ctx.schedule_task(task, parent_priority);
}
}
}
......@@ -7,32 +7,30 @@ mod leaf_distance_update;
mod prepare_new_children;
mod update_cell;
mod update_collectible;
use std::{
fmt::{Debug, Formatter},
fmt::{Debug, Display, Formatter},
mem::transmute,
sync::atomic::Ordering,
sync::{Arc, atomic::Ordering},
};
use bincode::{Decode, Encode};
use turbo_tasks::{
CellId, FxIndexMap, TaskExecutionReason, TaskId, TaskPriority, TurboTasksBackendApi,
TypedSharedReference,
TypedSharedReference, backend::CachedTaskType,
};
use crate::{
backend::{
OperationGuard, TaskDataCategory, TransientTask, TurboTasksBackend, TurboTasksBackendInner,
EventDescription, OperationGuard, TaskDataCategory, TurboTasksBackend,
TurboTasksBackendInner,
storage::{SpecificTaskDataCategory, StorageWriteGuard},
storage_schema::{TaskStorage, TaskStorageAccessors},
},
backing_storage::{BackingStorage, BackingStorageSealed},
data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState},
data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState, TransientTask},
};
pub trait Operation:
Encode + Decode<()> + Default + TryFrom<AnyOperation, Error = ()> + Into<AnyOperation>
{
pub trait Operation: Encode + Decode<()> + Default + TryFrom<AnyOperation, Error = ()> {
fn execute(self, ctx: &mut impl ExecuteContext<'_>);
}
......@@ -70,7 +68,16 @@ pub trait ExecuteContext<'e>: Sized {
func,
)
}
fn is_once_task(&self, task_id: TaskId) -> bool;
fn for_each_task_all(
&mut self,
task_ids: impl IntoIterator<Item = TaskId>,
func: impl FnMut(Self::TaskGuardImpl, &mut Self),
) {
self.for_each_task(
task_ids.into_iter().map(|id| (id, TaskDataCategory::All)),
func,
)
}
fn task_pair(
&mut self,
task_id1: TaskId,
......@@ -84,8 +91,6 @@ pub trait ExecuteContext<'e>: Sized {
where
T: Clone + Into<AnyOperation>;
fn suspending_requested(&self) -> bool;
fn get_task_desc_fn(&self, task_id: TaskId) -> impl Fn() -> String + Send + Sync + 'static;
fn get_task_description(&self, task_id: TaskId) -> String;
fn should_track_dependencies(&self) -> bool;
fn should_track_activeness(&self) -> bool;
}
......@@ -178,10 +183,9 @@ where
match result {
Ok(()) => storage,
Err(e) => {
let task_name = self.backend.get_task_description(task_id);
panic!(
"Failed to restore task data (corrupted database or bug): {:?}",
e.context(format!("{category:?} for {task_name} ({task_id}))"))
e.context(format!("{category:?} for {task_id})"))
)
}
}
......@@ -373,12 +377,14 @@ where
);
}
let mut task_type = None;
let mut task = self.backend.storage.access_mut(task_id);
if let Some(storage) = storage_for_data
&& !task.flags.is_restored(TaskDataCategory::Data)
{
task.restore_from(storage, TaskDataCategory::Data);
task.flags.set_restored(TaskDataCategory::Data);
task_type = task.get_persistent_task_type().cloned()
}
if let Some(storage) = storage_for_meta
&& !task.flags.is_restored(TaskDataCategory::Meta)
......@@ -389,6 +395,10 @@ where
prepared_task_callback(self, task_id, category, task);
#[cfg(debug_assertions)]
self.active_task_locks.fetch_sub(1, Ordering::AcqRel);
if let Some(task_type) = task_type {
// Insert into the task cache to avoid future lookups
self.backend.task_cache.entry(task_type).or_insert(task_id);
}
}
}
}
......@@ -460,7 +470,7 @@ where
TaskGuardImpl {
task,
task_id,
backend: self.backend,
_backend: self.backend,
#[cfg(debug_assertions)]
category,
#[cfg(debug_assertions)]
......@@ -490,7 +500,7 @@ where
let guard: TaskGuardImpl<'_, B> = TaskGuardImpl {
task,
task_id,
backend,
_backend: backend,
#[cfg(debug_assertions)]
category: _category,
#[cfg(debug_assertions)]
......@@ -500,17 +510,6 @@ where
});
}
fn is_once_task(&self, task_id: TaskId) -> bool {
if !task_id.is_transient() {
return false;
}
if let Some(ty) = self.backend.transient_tasks.get(&task_id) {
matches!(**ty, TransientTask::Once(_))
} else {
false
}
}
fn task_pair(
&mut self,
task_id1: TaskId,
......@@ -586,7 +585,7 @@ where
TaskGuardImpl {
task: task1,
task_id: task_id1,
backend: self.backend,
_backend: self.backend,
#[cfg(debug_assertions)]
category,
#[cfg(debug_assertions)]
......@@ -595,7 +594,7 @@ where
TaskGuardImpl {
task: task2,
task_id: task_id2,
backend: self.backend,
_backend: self.backend,
#[cfg(debug_assertions)]
category,
#[cfg(debug_assertions)]
......@@ -636,14 +635,6 @@ where
self.backend.suspending_requested()
}
fn get_task_desc_fn(&self, task_id: TaskId) -> impl Fn() -> String + Send + Sync + 'static {
self.backend.get_task_desc_fn(task_id)
}
fn get_task_description(&self, task_id: TaskId) -> String {
self.backend.get_task_description(task_id)
}
fn should_track_dependencies(&self) -> bool {
self.backend.should_track_dependencies()
}
......@@ -671,6 +662,43 @@ impl<'e, B: BackingStorage> ChildExecuteContext<'e> for ChildExecuteContextImpl<
}
}
pub enum TaskTypeRef<'l> {
Cached(&'l Arc<CachedTaskType>),
Transient(&'l Arc<TransientTask>),
}
impl TaskTypeRef<'_> {
pub fn to_owned(&self) -> TaskType {
match self {
TaskTypeRef::Cached(ty) => TaskType::Cached(Arc::clone(ty)),
TaskTypeRef::Transient(ty) => TaskType::Transient(Arc::clone(ty)),
}
}
}
impl Display for TaskTypeRef<'_> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
TaskTypeRef::Cached(ty) => write!(f, "{}", ty),
TaskTypeRef::Transient(ty) => write!(f, "{}", ty),
}
}
}
pub enum TaskType {
Cached(Arc<CachedTaskType>),
Transient(Arc<TransientTask>),
}
impl Display for TaskType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
TaskType::Cached(ty) => write!(f, "{}", ty),
TaskType::Transient(ty) => write!(f, "{}", ty),
}
}
}
pub trait TaskGuard: Debug + TaskStorageAccessors {
fn id(&self) -> TaskId;
......@@ -848,14 +876,11 @@ pub trait TaskGuard: Debug + TaskStorageAccessors {
/// Add a scheduled task item. Returns true if the task was successfully added (wasn't already
/// present).
#[must_use]
fn add_scheduled<InnerFn>(
fn add_scheduled(
&mut self,
reason: TaskExecutionReason,
description: impl FnOnce() -> InnerFn,
) -> bool
where
InnerFn: Fn() -> String + Sync + Send + 'static,
{
description: EventDescription,
) -> bool {
if self.has_in_progress() {
false
} else {
......@@ -864,8 +889,6 @@ pub trait TaskGuard: Debug + TaskStorageAccessors {
}
}
// ============ Collectible APIs ============
/// Insert an outdated collectible with count. Returns true if it was newly inserted.
#[must_use]
fn insert_outdated_collectible(&mut self, collectible: CollectibleRef, value: i32) -> bool {
......@@ -877,12 +900,35 @@ pub trait TaskGuard: Debug + TaskStorageAccessors {
self.add_outdated_collectibles(collectible, value);
true
}
fn get_task_type(&self) -> TaskTypeRef<'_> {
if let Some(task_type) = self.get_persistent_task_type() {
TaskTypeRef::Cached(task_type)
} else if let Some(task_type) = self.get_transient_task_type() {
TaskTypeRef::Transient(task_type)
} else {
panic!("Every task must have a task type {self:?}");
}
}
fn get_task_desc_fn(&self) -> impl Fn() -> String + Send + Sync + 'static {
let task_type = self.get_task_type().to_owned();
let task_id = self.id();
move || format!("{task_id:?} {task_type}")
}
fn get_task_description(&self) -> String {
let task_type = self.get_task_type().to_owned();
let task_id = self.id();
format!("{task_id:?} {task_type}")
}
fn get_task_name(&self) -> String {
let task_type = self.get_task_type().to_owned();
format!("{task_type}")
}
}
pub struct TaskGuardImpl<'a, B: BackingStorage> {
task_id: TaskId,
task: StorageWriteGuard<'a>,
backend: &'a TurboTasksBackendInner<B>,
_backend: &'a TurboTasksBackendInner<B>,
#[cfg(debug_assertions)]
category: TaskDataCategory,
#[cfg(debug_assertions)]
......@@ -933,9 +979,6 @@ impl<B: BackingStorage> Debug for TaskGuardImpl<'_, B> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut d = f.debug_struct("TaskGuard");
d.field("task_id", &self.task_id);
if let Some(task_type) = self.backend.task_cache.lookup_reverse(&self.task_id) {
d.field("task_type", &task_type);
};
d.field("storage", &*self.task);
d.finish()
}
......
......@@ -107,21 +107,17 @@ impl UpdateCellOperation {
Lazy::new(|| updated_key_hashes.into_iter().collect::<FxHashSet<u64>>())
});
let tasks_with_keys = task
.iter_cell_dependents()
.filter(|&(dependent_cell, key, _)| {
dependent_cell == cell
&& key.is_none_or(|key_hash| {
updated_key_hashes_set
.as_ref()
.is_none_or(|set| set.contains(&key_hash))
})
})
.map(|(_, key, task)| (task, key))
.filter(|&(dependent_task_id, _)| {
// once tasks are never invalidated
!ctx.is_once_task(dependent_task_id)
});
let tasks_with_keys =
task.iter_cell_dependents()
.filter_map(|(dependent_cell, key, task)| {
(dependent_cell == cell
&& key.is_none_or(|key_hash| {
updated_key_hashes_set
.as_ref()
.is_none_or(|set| set.contains(&key_hash))
}))
.then_some((task, key))
});
let mut dependent_tasks: FxIndexMap<TaskId, SmallVec<[Option<u64>; 2]>> =
FxIndexMap::default();
for (task, key) in tasks_with_keys {
......
......@@ -77,16 +77,6 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
unsafe fn reverse_lookup_task_cache(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
) -> Result<Option<Arc<CachedTaskType>>>;
/// Lookup and decode fields directly into TaskStorage.
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
unsafe fn lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
......@@ -191,23 +181,6 @@ where
}
}
unsafe fn reverse_lookup_task_cache(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
) -> Result<Option<Arc<CachedTaskType>>> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
unsafe { this.reverse_lookup_task_cache(tx, task_id) }
}
Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
unsafe { this.reverse_lookup_task_cache(tx, task_id) }
}
}
}
unsafe fn lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
......
use std::{
fmt::{self, Debug, Display},
pin::Pin,
};
use anyhow::Result;
use bincode::{Decode, Encode};
use parking_lot::Mutex;
use rustc_hash::FxHashSet;
use turbo_tasks::{
CellId, TaskExecutionReason, TaskId, TaskPriority, TraitTypeId,
backend::TurboTasksExecutionError,
event::{Event, EventListener},
CellId, RawVc, TaskExecutionReason, TaskId, TaskPriority, TraitTypeId,
backend::{TransientTaskRoot, TurboTasksExecutionError},
event::{Event, EventDescription, EventListener},
};
// this traits are needed for the transient variants of `CachedDataItem`
......@@ -23,6 +30,8 @@ macro_rules! transient_traits {
panic!(concat!(stringify!($name), " cannot be compared"));
}
}
impl Eq for $name {}
};
}
......@@ -157,7 +166,47 @@ impl ActivenessState {
transient_traits!(ActivenessState);
impl Eq for ActivenessState {}
type TransientTaskOnce =
Mutex<Option<Pin<Box<dyn Future<Output = Result<RawVc>> + Send + 'static>>>>;
pub enum TransientTask {
/// A root task that will track dependencies and re-execute when
/// dependencies change. Task will eventually settle to the correct
/// execution.
///
/// Always active. Automatically scheduled.
Root(TransientTaskRoot),
// TODO implement these strongly consistency
/// A single root task execution. It won't track dependencies.
/// Task will definitely include all invalidations that happened before the
/// start of the task. It may or may not include invalidations that
/// happened after that. It may see these invalidations partially
/// applied.
///
/// Active until done. Automatically scheduled.
Once(TransientTaskOnce),
}
impl Debug for TransientTask {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TransientTask::Root(_) => f.write_str("TransientTask::Root"),
TransientTask::Once(_) => f.write_str("TransientTask::Once"),
}
}
}
impl Display for TransientTask {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TransientTask::Root(_) => f.write_str("Root Task"),
TransientTask::Once(_) => f.write_str("Once Task"),
}
}
}
transient_traits!(TransientTask);
#[derive(Debug, Clone, Copy, Encode, Decode, PartialEq, Eq)]
pub enum Dirtyness {
......@@ -171,6 +220,15 @@ pub enum RootType {
OnceTask,
}
impl Display for RootType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
RootType::RootTask => f.write_str("Root Task"),
RootType::OnceTask => f.write_str("Once Task"),
}
}
}
#[derive(Debug)]
pub struct InProgressStateInner {
pub stale: bool,
......@@ -203,8 +261,6 @@ pub enum InProgressState {
transient_traits!(InProgressState);
impl Eq for InProgressState {}
#[derive(Debug)]
pub struct InProgressCellState {
pub event: Event,
......@@ -212,8 +268,6 @@ pub struct InProgressCellState {
transient_traits!(InProgressCellState);
impl Eq for InProgressCellState {}
impl InProgressCellState {
pub fn new(task_id: TaskId, cell: CellId) -> Self {
InProgressCellState {
......@@ -256,33 +310,17 @@ pub struct LeafDistance {
impl InProgressState {
/// Create a new scheduled state with a done event.
pub fn new_scheduled<InnerFnDescription>(
reason: TaskExecutionReason,
description: impl FnOnce() -> InnerFnDescription,
) -> Self
where
InnerFnDescription: Fn() -> String + Sync + Send + 'static,
{
let done_event = Event::new(move || {
let inner = description();
move || format!("{} done_event", inner())
});
pub fn new_scheduled(reason: TaskExecutionReason, description: EventDescription) -> Self {
let done_event = Event::new(move || move || format!("{description} done_event"));
InProgressState::Scheduled { done_event, reason }
}
pub fn new_scheduled_with_listener<InnerFnDescription, InnerFnNote>(
pub fn new_scheduled_with_listener(
reason: TaskExecutionReason,
description: impl FnOnce() -> InnerFnDescription,
note: impl FnOnce() -> InnerFnNote,
) -> (Self, EventListener)
where
InnerFnDescription: Fn() -> String + Sync + Send + 'static,
InnerFnNote: Fn() -> String + Sync + Send + 'static,
{
let done_event = Event::new(move || {
let inner = description();
move || format!("{} done_event", inner())
});
description: EventDescription,
note: EventDescription,
) -> (Self, EventListener) {
let done_event = Event::new(move || move || format!("{description} done_event"));
let listener = done_event.listen_with_note(note);
(InProgressState::Scheduled { done_event, reason }, listener)
}
......
......@@ -4,8 +4,7 @@ pub struct ByKeySpace<T> {
infra: T,
task_meta: T,
task_data: T,
forward_task_cache: T,
reverse_task_cache: T,
task_cache: T,
}
impl<T> ByKeySpace<T> {
......@@ -14,8 +13,7 @@ impl<T> ByKeySpace<T> {
infra: factory(KeySpace::Infra),
task_meta: factory(KeySpace::TaskMeta),
task_data: factory(KeySpace::TaskData),
forward_task_cache: factory(KeySpace::ForwardTaskCache),
reverse_task_cache: factory(KeySpace::ReverseTaskCache),
task_cache: factory(KeySpace::TaskCache),
}
}
......@@ -24,8 +22,7 @@ impl<T> ByKeySpace<T> {
KeySpace::Infra => &self.infra,
KeySpace::TaskMeta => &self.task_meta,
KeySpace::TaskData => &self.task_data,
KeySpace::ForwardTaskCache => &self.forward_task_cache,
KeySpace::ReverseTaskCache => &self.reverse_task_cache,
KeySpace::TaskCache => &self.task_cache,
}
}
......@@ -34,8 +31,7 @@ impl<T> ByKeySpace<T> {
KeySpace::Infra => &mut self.infra,
KeySpace::TaskMeta => &mut self.task_meta,
KeySpace::TaskData => &mut self.task_data,
KeySpace::ForwardTaskCache => &mut self.forward_task_cache,
KeySpace::ReverseTaskCache => &mut self.reverse_task_cache,
KeySpace::TaskCache => &mut self.task_cache,
}
}
......@@ -44,8 +40,7 @@ impl<T> ByKeySpace<T> {
(KeySpace::Infra, &self.infra),
(KeySpace::TaskMeta, &self.task_meta),
(KeySpace::TaskData, &self.task_data),
(KeySpace::ForwardTaskCache, &self.forward_task_cache),
(KeySpace::ReverseTaskCache, &self.reverse_task_cache),
(KeySpace::TaskCache, &self.task_cache),
]
.into_iter()
}
......
......@@ -9,8 +9,7 @@ pub enum KeySpace {
Infra = 0,
TaskMeta = 1,
TaskData = 2,
ForwardTaskCache = 3,
ReverseTaskCache = 4,
TaskCache = 3,
}
pub trait KeyValueDatabase {
......
......@@ -18,8 +18,7 @@ pub struct LmbdKeyValueDatabase {
infra_db: Database,
data_db: Database,
meta_db: Database,
forward_task_cache_db: Database,
reverse_task_cache_db: Database,
task_cache_db: Database,
}
impl LmbdKeyValueDatabase {
......@@ -38,23 +37,19 @@ impl LmbdKeyValueDatabase {
| EnvironmentFlags::NO_TLS,
)
.set_max_readers((available_parallelism().map_or(16, |v| v.get()) * 8) as u32)
.set_max_dbs(5)
.set_max_dbs(4)
.set_map_size(MAP_SIZE)
.open(path)?;
let infra_db = env.create_db(Some("infra"), DatabaseFlags::INTEGER_KEY)?;
let data_db = env.create_db(Some("data"), DatabaseFlags::INTEGER_KEY)?;
let meta_db = env.create_db(Some("meta"), DatabaseFlags::INTEGER_KEY)?;
let forward_task_cache_db =
env.create_db(Some("forward_task_cache"), DatabaseFlags::empty())?;
let reverse_task_cache_db =
env.create_db(Some("reverse_task_cache"), DatabaseFlags::INTEGER_KEY)?;
let task_cache_db = env.create_db(Some("task_cache"), DatabaseFlags::empty())?;
Ok(LmbdKeyValueDatabase {
env,
infra_db,
data_db,
meta_db,
forward_task_cache_db,
reverse_task_cache_db,
task_cache_db,
})
}
......@@ -63,8 +58,7 @@ impl LmbdKeyValueDatabase {
KeySpace::Infra => self.infra_db,
KeySpace::TaskMeta => self.meta_db,
KeySpace::TaskData => self.data_db,
KeySpace::ForwardTaskCache => self.forward_task_cache_db,
KeySpace::ReverseTaskCache => self.reverse_task_cache_db,
KeySpace::TaskCache => self.task_cache_db,
}
}
}
......@@ -91,8 +85,7 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
KeySpace::Infra => self.infra_db,
KeySpace::TaskMeta => self.meta_db,
KeySpace::TaskData => self.data_db,
KeySpace::ForwardTaskCache => self.forward_task_cache_db,
KeySpace::ReverseTaskCache => self.reverse_task_cache_db,
KeySpace::TaskCache => self.task_cache_db,
};
let value = match extended_key::get(transaction, db, key) {
......
......@@ -87,8 +87,7 @@ impl<T: KeyValueDatabase> StartupCacheLayer<T> {
KeySpace::Infra => 8,
KeySpace::TaskMeta => 1024 * 1024,
KeySpace::TaskData => 1024 * 1024,
KeySpace::ForwardTaskCache => 1024 * 1024,
KeySpace::ReverseTaskCache => 1024 * 1024,
KeySpace::TaskCache => 1024 * 1024,
},
Default::default(),
)
......@@ -322,8 +321,7 @@ fn write_key_value_pair(
KeySpace::Infra => 0,
KeySpace::TaskMeta => 1,
KeySpace::TaskData => 2,
KeySpace::ForwardTaskCache => 3,
KeySpace::ReverseTaskCache => 4,
KeySpace::TaskCache => 3,
})?;
let key_len = key.len();
size_buffer.copy_from_slice(&(key_len as u32).to_be_bytes());
......@@ -344,8 +342,7 @@ fn read_key_value_pair<'l>(
0 => KeySpace::Infra,
1 => KeySpace::TaskMeta,
2 => KeySpace::TaskData,
3 => KeySpace::ForwardTaskCache,
4 => KeySpace::ReverseTaskCache,
3 => KeySpace::TaskCache,
_ => return Err(anyhow::anyhow!("Invalid key space")),
};
*pos += 1;
......
......@@ -22,7 +22,7 @@ use crate::database::{
mod parallel_scheduler;
/// Number of key families, see KeySpace enum for their numbers.
const FAMILIES: usize = 5;
const FAMILIES: usize = 4;
const MB: u64 = 1024 * 1024;
const COMPACT_CONFIG: CompactConfig = CompactConfig {
......
......@@ -328,7 +328,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
batch
.put(
KeySpace::ForwardTaskCache,
KeySpace::TaskCache,
WriteBuffer::Borrowed(&task_type_bytes),
WriteBuffer::Borrowed(&task_id.to_le_bytes()),
)
......@@ -337,17 +337,6 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
"Unable to write task cache {task_type:?} => {task_id}"
)
})?;
batch
.put(
KeySpace::ReverseTaskCache,
WriteBuffer::Borrowed(IntKey::new(task_id).as_ref()),
WriteBuffer::Borrowed(&task_type_bytes),
)
.with_context(|| {
format!(
"Unable to write task cache {task_id} => {task_type:?}"
)
})?;
max_task_id = max_task_id.max(task_id);
}
......@@ -415,22 +404,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
batch
.put(
KeySpace::ForwardTaskCache,
KeySpace::TaskCache,
WriteBuffer::Borrowed(&task_type_bytes),
WriteBuffer::Borrowed(&task_id.to_le_bytes()),
)
.with_context(|| {
format!("Unable to write task cache {task_type:?} => {task_id}")
})?;
batch
.put(
KeySpace::ReverseTaskCache,
WriteBuffer::Borrowed(IntKey::new(task_id).as_ref()),
WriteBuffer::Borrowed(&task_type_bytes),
)
.with_context(|| {
format!("Unable to write task cache {task_id} => {task_type:?}")
})?;
next_task_id = next_task_id.max(task_id + 1);
}
}
......@@ -467,8 +447,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
) -> Result<Option<TaskId>> {
let mut task_type_bytes = TurboBincodeBuffer::new();
encode_task_type(task_type, &mut task_type_bytes, None)?;
let Some(bytes) = database.get(tx, KeySpace::ForwardTaskCache, &task_type_bytes)?
else {
let Some(bytes) = database.get(tx, KeySpace::TaskCache, &task_type_bytes)? else {
return Ok(None);
};
let bytes = bytes.borrow().try_into()?;
......@@ -485,32 +464,6 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
.with_context(|| format!("Looking up task id for {task_type:?} from database failed"))
}
unsafe fn reverse_lookup_task_cache(
&self,
tx: Option<&T::ReadTransaction<'_>>,
task_id: TaskId,
) -> Result<Option<Arc<CachedTaskType>>> {
let inner = &*self.inner;
fn lookup<D: KeyValueDatabase>(
database: &D,
tx: &D::ReadTransaction<'_>,
task_id: TaskId,
) -> Result<Option<Arc<CachedTaskType>>> {
let Some(bytes) = database.get(
tx,
KeySpace::ReverseTaskCache,
IntKey::new(*task_id).as_ref(),
)?
else {
return Ok(None);
};
Ok(Some(turbo_bincode_decode(bytes.borrow())?))
}
inner
.with_tx(tx, |tx| lookup(&inner.database, tx, task_id))
.with_context(|| format!("Looking up task type for {task_id} from database failed"))
}
unsafe fn lookup_data(
&self,
tx: Option<&T::ReadTransaction<'_>>,
......
use std::{fmt::Debug, ops::Deref, sync::Arc};
use bincode::{
Decode, Encode,
de::Decoder,
error::{DecodeError, EncodeError},
impl_borrow_decode_with_context,
};
#[derive(Clone)]
pub enum ArcOrOwned<T> {
Arc(Arc<T>),
Owned(T),
}
impl<T> ArcOrOwned<T> {}
impl<T> Deref for ArcOrOwned<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::Arc(arc) => arc.deref(),
Self::Owned(value) => value,
}
}
}
impl<T> AsRef<T> for ArcOrOwned<T> {
fn as_ref(&self) -> &T {
self.deref()
}
}
impl<T> From<T> for ArcOrOwned<T> {
fn from(value: T) -> Self {
Self::Owned(value)
}
}
impl<T> From<Arc<T>> for ArcOrOwned<T> {
fn from(arc: Arc<T>) -> Self {
Self::Arc(arc)
}
}
impl<T> From<ArcOrOwned<T>> for Arc<T> {
fn from(value: ArcOrOwned<T>) -> Self {
match value {
ArcOrOwned::Arc(arc) => arc,
ArcOrOwned::Owned(value) => Arc::new(value),
}
}
}
impl<T> Encode for ArcOrOwned<T>
where
T: Encode,
{
fn encode<E: bincode::enc::Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
match self {
ArcOrOwned::Arc(arc) => arc.encode(encoder),
ArcOrOwned::Owned(value) => value.encode(encoder),
}
}
}
impl<T, Context> Decode<Context> for ArcOrOwned<T>
where
T: Decode<Context>,
{
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
let value = T::decode(decoder)?;
Ok(ArcOrOwned::Owned(value))
}
}
impl_borrow_decode_with_context!(ArcOrOwned<T>, Context, Context, T: Decode<Context>);
impl<T> Debug for ArcOrOwned<T>
where
T: Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.as_ref().fmt(f)
}
}
use std::{borrow::Borrow, hash::Hash};
use dashmap::mapref::entry::Entry;
use turbo_tasks::FxDashMap;
use crate::utils::dash_map_drop_contents::drop_contents;
/// A bidirectional [`FxDashMap`] that allows lookup by key or value.
///
/// As keys and values are stored twice, they should be small types, such as
/// [`Arc`][`std::sync::Arc`].
pub struct BiMap<K, V> {
forward: FxDashMap<K, V>,
reverse: FxDashMap<V, K>,
}
impl<K, V> BiMap<K, V>
where
K: Eq + Hash + Clone,
V: Eq + Hash + Clone,
{
pub fn new() -> Self {
Self {
forward: FxDashMap::default(),
reverse: FxDashMap::default(),
}
}
pub fn lookup_forward<Q>(&self, key: &Q) -> Option<V>
where
K: Borrow<Q>,
Q: Hash + Eq,
{
self.forward.get(key).map(|v| v.value().clone())
}
pub fn lookup_reverse<Q>(&self, key: &Q) -> Option<K>
where
V: Borrow<Q>,
Q: Hash + Eq,
{
self.reverse.get(key).map(|v| v.value().clone())
}
pub fn try_insert(&self, key: K, value: V) -> Result<(), V> {
match self.forward.entry(key) {
Entry::Occupied(e) => Err(e.get().clone()),
Entry::Vacant(e) => {
let e = e.insert_entry(value.clone());
let key = e.key().clone();
self.reverse.insert(value, key);
drop(e);
Ok(())
}
}
}
}
impl<K, V> BiMap<K, V>
where
K: Eq + Hash + Send + Sync,
V: Eq + Hash + Send + Sync,
{
pub fn drop_contents(&self) {
drop_contents(&self.forward);
drop_contents(&self.reverse);
}
}
use std::{
hash::{BuildHasher, Hash},
ops::{Deref, DerefMut},
};
use dashmap::{DashMap, RwLockWriteGuard, SharedValue};
use hashbrown::raw::{Bucket, InsertSlot, RawTable};
pub fn raw_entry<'l, K: Eq + Hash + AsRef<Q>, V, Q: Eq + Hash, S: BuildHasher + Clone>(
map: &'l DashMap<K, V, S>,
key: &Q,
) -> RawEntry<'l, K, V> {
let hasher = map.hasher();
let hash = hasher.hash_one(key);
let shard = map.determine_shard(hash as usize);
let mut shard = map.shards()[shard].write();
let result = shard.find_or_find_insert_slot(
hash,
|(k, _v)| k.as_ref() == key,
|(k, _v)| hasher.hash_one(k),
);
match result {
Ok(bucket) => RawEntry::Occupied(OccupiedEntry { bucket, shard }),
Err(insert_slot) => RawEntry::Vacant(VacantEntry {
hash,
insert_slot,
shard,
}),
}
}
pub enum RawEntry<'l, K, V> {
Occupied(OccupiedEntry<'l, K, V>),
Vacant(VacantEntry<'l, K, V>),
}
impl<'l, K, V> RawEntry<'l, K, V> {
#[allow(dead_code)]
pub fn or_insert_with<F: FnOnce() -> (K, V)>(self, f: F) -> RefMut<'l, K, V> {
match self {
RawEntry::Occupied(occupied) => occupied.into_mut(),
RawEntry::Vacant(vacant) => {
let (key, value) = f();
vacant.insert(key, value)
}
}
}
}
pub struct OccupiedEntry<'l, K, V> {
bucket: Bucket<(K, SharedValue<V>)>,
#[allow(dead_code, reason = "kept to ensure the lock lives long enough")]
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
}
impl<'l, K, V> OccupiedEntry<'l, K, V> {
pub fn get(&self) -> &V {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_ref().1.get() }
}
#[allow(dead_code)]
pub fn get_mut(&mut self) -> &mut V {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_mut().1.get_mut() }
}
#[allow(dead_code)]
pub fn into_mut(self) -> RefMut<'l, K, V> {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
RefMut::from_bucket(self.bucket, self.shard)
}
}
pub struct VacantEntry<'l, K, V> {
hash: u64,
insert_slot: InsertSlot,
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
}
impl<'l, K, V> VacantEntry<'l, K, V> {
pub fn insert(mut self, key: K, value: V) -> RefMut<'l, K, V> {
let shared_value = SharedValue::new(value);
// Safety: The insert slot is valid and the map has not be modified since we obtained it (we
// hold the write lock).
unsafe {
let bucket =
self.shard
.insert_in_slot(self.hash, self.insert_slot, (key, shared_value));
RefMut::from_bucket(bucket, self.shard)
}
}
}
pub struct RefMut<'l, K, V> {
bucket: Bucket<(K, SharedValue<V>)>,
#[allow(dead_code, reason = "kept to ensure the lock lives long enough")]
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
}
impl<'l, K, V> RefMut<'l, K, V> {
fn from_bucket(
bucket: Bucket<(K, SharedValue<V>)>,
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
) -> Self {
Self { bucket, shard }
}
}
impl<'l, K, V> Deref for RefMut<'l, K, V> {
type Target = V;
fn deref(&self) -> &Self::Target {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_ref().1.get() }
}
}
impl<'l, K, V> DerefMut for RefMut<'l, K, V> {
fn deref_mut(&mut self) -> &mut Self::Target {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_mut().1.get_mut() }
}
}
pub mod bi_map;
pub mod arc_or_owned;
pub mod chunked_vec;
pub mod dash_map_drop_contents;
pub mod dash_map_multi;
pub mod dash_map_raw_entry;
pub mod ptr_eq_arc;
pub mod shard_amount;
pub mod sharded;
......
use proc_macro2::TokenStream;
use quote::quote;
use syn::{Fields, Ident, ItemStruct, Meta, Token, Type, punctuated::Punctuated, spanned::Spanned};
use syn::{
Fields, Ident, ItemStruct, Meta, Token, Type, Visibility, punctuated::Punctuated,
spanned::Spanned,
};
/// Derives the TaskStorage trait and generates optimized storage structures.
pub fn task_storage(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
......@@ -42,6 +45,7 @@ fn task_storage_impl(input: TokenStream) -> TokenStream {
/// along with pre-computed values like the PascalCase variant name.
#[derive(Debug, Clone)]
struct FieldInfo {
is_pub: bool,
/// The field's identifier (snake_case)
field_name: Ident,
/// The PascalCase variant name for use in LazyField enum
......@@ -342,6 +346,7 @@ fn expect_string_literal<'a>(expr: &'a syn::Expr, attr_name: &str) -> Option<&'a
fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo {
let field_name = field.ident.as_ref().unwrap().clone();
let field_type = field.ty.clone();
let is_pub = matches!(field.vis, Visibility::Public(_));
// Pre-compute the PascalCase variant name once
let variant_name = syn::Ident::new(&to_pascal_case(&field_name.to_string()), field_name.span());
......@@ -521,6 +526,7 @@ fn parse_field_storage_attributes(field: &syn::Field) -> FieldInfo {
};
FieldInfo {
is_pub,
field_name,
variant_name,
field_type,
......@@ -952,7 +958,7 @@ fn generate_typed_storage_struct(grouped_fields: &GroupedFields) -> TokenStream
#[doc = "Unified typed storage containing all task fields."]
#[doc = "This is designed to be embedded in the actual InnerStorage for incremental migration."]
#[automatically_derived]
#[derive(Debug, Default, PartialEq, turbo_tasks::ShrinkToFit)]
#[derive(Debug, Default, turbo_tasks::ShrinkToFit)]
#[shrink_to_fit(crate = "turbo_tasks::macro_helpers::shrink_to_fit")]
pub struct TaskStorage {
#(#field_defs,)*
......@@ -1013,6 +1019,11 @@ fn generate_field_accessors(field: &FieldInfo) -> TokenStream {
fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
let field_name = &field.field_name;
let field_type = &field.field_type;
let vis = if field.is_pub {
quote! {pub}
} else {
quote! {}
};
let get_name = field.get_ident();
let set_name = field.set_ident();
......@@ -1022,7 +1033,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
if field.is_inline() && field.use_default {
// Inline with default: field is T stored directly, uses Default::default() for "empty"
quote! {
fn #get_name(&self) -> Option<&#field_type> {
#vis fn #get_name(&self) -> Option<&#field_type> {
if self.#field_name != #field_type::default() {
Some(&self.#field_name)
} else {
......@@ -1030,7 +1041,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
}
}
fn #set_name(&mut self, value: #field_type) -> Option<#field_type> {
#vis fn #set_name(&mut self, value: #field_type) -> Option<#field_type> {
let old = std::mem::replace(&mut self.#field_name, value);
if old != #field_type::default() {
Some(old)
......@@ -1039,7 +1050,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
}
}
fn #take_name(&mut self) -> Option<#field_type> {
#vis fn #take_name(&mut self) -> Option<#field_type> {
let old = std::mem::take(&mut self.#field_name);
if old != #field_type::default() {
Some(old)
......@@ -1053,15 +1064,15 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
let inner_type = extract_option_inner_type(field_type);
quote! {
fn #get_name(&self) -> Option<&#inner_type> {
#vis fn #get_name(&self) -> Option<&#inner_type> {
self.#field_name.as_ref()
}
fn #set_name(&mut self, value: #inner_type) -> Option<#inner_type> {
#vis fn #set_name(&mut self, value: #inner_type) -> Option<#inner_type> {
self.#field_name.replace(value)
}
fn #take_name(&mut self) -> Option<#inner_type> {
#vis fn #take_name(&mut self) -> Option<#inner_type> {
self.#field_name.take()
}
}
......@@ -1073,16 +1084,16 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
let constructor = field.lazy_constructor(quote! { value });
quote! {
fn #get_name(&self) -> Option<&#field_type> {
#vis fn #get_name(&self) -> Option<&#field_type> {
self.find_lazy(#extractor)
}
#[doc = "Set the field value, returning the old value if present."]
fn #set_name(&mut self, value: #field_type) -> Option<#field_type> {
#vis fn #set_name(&mut self, value: #field_type) -> Option<#field_type> {
self.set_lazy(#matches_closure, #unwrap_owned, #constructor)
}
fn #take_name(&mut self) -> Option<#field_type> {
#vis fn #take_name(&mut self) -> Option<#field_type> {
self.take_lazy(#matches_closure, #unwrap_owned)
}
......@@ -1090,7 +1101,7 @@ fn generate_direct_field_accessors(field: &FieldInfo) -> TokenStream {
#[doc = ""]
#[doc = "Unlike `get_or_create_lazy` for collections, this does NOT allocate"]
#[doc = "if the field is absent - it returns None instead."]
fn #get_mut_name(&mut self) -> Option<&mut #field_type> {
#vis fn #get_mut_name(&mut self) -> Option<&mut #field_type> {
self.find_lazy_mut(#extractor)
}
}
......@@ -1106,19 +1117,24 @@ fn generate_collection_field_accessors(
let ref_name = field.ref_ident();
let mut_name = field.mut_ident();
let take_name = field.take_ident();
let vis = if field.is_pub {
quote! {pub}
} else {
quote! {}
};
if field.is_inline() {
// Inline: direct field access
quote! {
fn #ref_name(&self) -> &#field_type {
#vis fn #ref_name(&self) -> &#field_type {
&self.#field_name
}
fn #mut_name(&mut self) -> &mut #field_type {
#vis fn #mut_name(&mut self) -> &mut #field_type {
&mut self.#field_name
}
fn #take_name(&mut self) -> #field_type {
#vis fn #take_name(&mut self) -> #field_type {
std::mem::take(&mut self.#field_name)
}
}
......@@ -1130,11 +1146,11 @@ fn generate_collection_field_accessors(
let constructor = field.lazy_constructor(quote! { Default::default() });
quote! {
fn #ref_name(&self) -> Option<&#field_type> {
#vis fn #ref_name(&self) -> Option<&#field_type> {
self.find_lazy(#extractor)
}
fn #mut_name(&mut self) -> &mut #field_type {
#vis fn #mut_name(&mut self) -> &mut #field_type {
self.get_or_create_lazy(
#matches_closure,
#unwrap_closure,
......@@ -1142,7 +1158,7 @@ fn generate_collection_field_accessors(
)
}
fn #take_name(&mut self) -> Option<#field_type> {
#vis fn #take_name(&mut self) -> Option<#field_type> {
self.take_lazy(
#matches_closure,
#unwrap_closure,
......
......@@ -409,8 +409,6 @@ pub trait Backend: Sync + Send {
) {
}
fn get_task_description(&self, task: TaskId) -> String;
fn try_start_task_execution<'a>(
&'a self,
task: TaskId,
......
use std::{
fmt::{Debug, Formatter},
fmt::{Debug, Display, Formatter},
future::Future,
mem::replace,
pin::Pin,
......@@ -14,6 +14,58 @@ use std::{
#[cfg(feature = "hanging_detection")]
use tokio::time::{Timeout, timeout};
pub trait EventDescriptor {
#[cfg(feature = "hanging_detection")]
fn get_description(self) -> Arc<dyn Fn() -> String + Sync + Send>;
}
impl<T, InnerFn> EventDescriptor for T
where
T: FnOnce() -> InnerFn,
InnerFn: Fn() -> String + Sync + Send + 'static,
{
#[cfg(feature = "hanging_detection")]
fn get_description(self) -> Arc<dyn Fn() -> String + Sync + Send> {
Arc::new((self)())
}
}
#[derive(Clone)]
pub struct EventDescription {
#[cfg(feature = "hanging_detection")]
description: Arc<dyn Fn() -> String + Sync + Send>,
}
impl EventDescription {
#[inline(always)]
pub fn new<InnerFn>(#[allow(unused_variables)] description: impl FnOnce() -> InnerFn) -> Self
where
InnerFn: Fn() -> String + Sync + Send + 'static,
{
Self {
#[cfg(feature = "hanging_detection")]
description: Arc::new((description)()),
}
}
}
impl Display for EventDescription {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
#[cfg(not(feature = "hanging_detection"))]
return write!(f, "");
#[cfg(feature = "hanging_detection")]
return write!(f, "{}", (self.description)());
}
}
impl EventDescriptor for EventDescription {
#[cfg(feature = "hanging_detection")]
fn get_description(self) -> Arc<dyn Fn() -> String + Sync + Send> {
self.description
}
}
pub struct Event {
#[cfg(feature = "hanging_detection")]
description: Arc<dyn Fn() -> String + Sync + Send>,
......@@ -35,17 +87,14 @@ impl Event {
/// The outer closure allows avoiding extra lookups (e.g. task type info) that may be needed to
/// capture information needed for constructing (moving into) the inner closure.
#[inline(always)]
pub fn new<InnerFn>(_description: impl FnOnce() -> InnerFn) -> Self
where
InnerFn: Fn() -> String + Sync + Send + 'static,
{
pub fn new(#[allow(unused_variables)] description: impl EventDescriptor) -> Self {
#[cfg(not(feature = "hanging_detection"))]
return Self {
event: event_listener::Event::new(),
};
#[cfg(feature = "hanging_detection")]
return Self {
description: Arc::new((_description)()),
description: description.get_description(),
event: event_listener::Event::new(),
};
}
......@@ -80,10 +129,7 @@ impl Event {
///
/// The outer closure allow avoiding extra lookups (e.g. task type info) that may be needed to
/// capture information needed for constructing (moving into) the inner closure.
pub fn listen_with_note<InnerFn>(&self, _note: impl FnOnce() -> InnerFn) -> EventListener
where
InnerFn: Fn() -> String + Sync + Send + 'static,
{
pub fn listen_with_note(&self, _note: impl EventDescriptor) -> EventListener {
#[cfg(not(feature = "hanging_detection"))]
return EventListener {
listener: self.event.listen(),
......@@ -91,7 +137,7 @@ impl Event {
#[cfg(feature = "hanging_detection")]
return EventListener {
description: self.description.clone(),
note: Arc::new((_note)()),
note: _note.get_description(),
future: Some(Box::pin(timeout(
Duration::from_secs(30),
self.event.listen(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment