Unverified Commit 31a1b635 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbo-tasks: Reduce allocations on cache hits (#92756)

### What?

Reduce heap allocations when turbo-tasks functions get cache hits (~85% of calls).

### Why?

Every turbo-tasks function call (generated by `#[turbo_tasks::function]`) was boxing its arguments into `Box<dyn MagicAny>` before looking up the task cache. This allocation is wasted on cache hits, which are the overwhelmingly common case.

### How?

**Deferred boxing via `StackMagicAny` trait object:**

Introduce a `StackMagicAny` trait that abstracts over a stack-resident `Option<T>`:

- `as_ref(&self) -> &dyn MagicAny` — borrow the argument for hash/equality (cache lookup)
- `take_box(&mut self) -> Box<dyn MagicAny>` — move the value to the heap (zero clones)
- `as_any_mut(&mut self) -> &mut dyn Any` — downcast to concrete type without boxing

The data flow:

1. **Callsite** (macro-generated): creates `StackMagicAnySlot::new((args...))` on the stack, calls `dynamic_call(..., &mut arg)`
2. **`dynamic_call`**: checks resolution via `arg.as_ref()`, routes to `native_call` (resolved) or boxes via `arg.take_box()` for `LocalTaskSpec` (unresolved)
3. **Backend `get_or_create_task_inner`**: does a read-only `raw_get` lookup using `hash_from_components` + `eq_components` with the borrowed `&dyn MagicAny`. On cache hit (~85%), returns immediately — **zero heap allocation**. On cache miss, re-checks under write lock using the same borrowed reference, and only calls `arg.take_box()` in the vacant entry case (true cache miss).

Boxing is now deferred past all of these:
- **Memory cache hit** — the common case, no allocation at all
- **Backing storage hit** — found in persistent storage, no allocation needed
- **Lost race under write lock** — another thread inserted while we upgraded; we use their task_id, still no allocation
- **Trait method dispatch (no filtering)** — `filter_owned` is now `Option<FilterOwnedArgsFunctor>`; when `None` (the common case where all args are used), the original `&mut dyn StackMagicAny` passes straight through to `dynamic_call` without boxing

**Optimized `filter_owned` for traits:**

When trait methods *do* need argument filtering (unused `_`-prefixed parameters), the old path did `take_box()` → `downcast_args_owned()` → dereference → repack. This is an unnecessary heap round-trip. The new `downcast_stack_args_owned()` function uses `as_any_mut()` to downcast directly to `&mut StackMagicAnySlot<T>` and calls `take()` on the inner `Option`, skipping the intermediate `Box` entirely.

**Additional changes:**
- `Backend::get_or_create_*_task` now takes decomposed parameters (`native_fn`, `this`, `&mut dyn StackMagicAny`) instead of a pre-constructed `CachedTaskType`
- Persistent and transient task creation merged into a shared `get_or_create_task_inner(transient: bool)`
- `connect_child` uses eagerly-set `persistent_task_type` from `initialize_new_task`
- `OwnedMagicAny` adapter wraps already-boxed args (from async resolution tasks) to fit the `StackMagicAny` interface
- Both `dynamic_call` and `trait_call` take `&mut dyn StackMagicAny` (trait dispatch also benefits)
- `CachedTaskType::hash_encode` now delegates to `hash_encode_components` (deduplicated)
- Removed `try_native_call`, `native_call_if_consistent`, `try_get_or_create_*` — the deferred boxing approach subsumes these

### Binary size

Binary size is neutral (linux-x86_64, `--release`, stripped + gzipped: 30.9 MB on both canary and this branch).

### Overhead benchmark (turbo-tasks-backend, median, lower is better)

Measured on an isolated Firecracker microVM (linux-x86_64). Variance is nontrivial on this environment, but the direction is consistently positive across all turbo-tasks benchmarks.

| Benchmark | Canary | This PR | Delta |
|---|---|---|---|
| `turbo-cached-same-keys/1` | 512.6 ns | 490.2 ns | -4.4% |
| `turbo-cached-same-keys/10` | 493.4 ns | 483.8 ns | -2.0% |
| `turbo-cached-same-keys/100` | 502.1 ns | 484.9 ns | -3.4% |
| `turbo-cached-same-keys/1000` | 1.31 µs | 774.2 ns | -41.0% |
| `turbo-cached-different-keys/1` | 1.02 µs | 1.00 µs | -1.4% |
| `turbo-cached-different-keys/10` | 1.13 µs | 1.11 µs | -1.8% |
| `turbo-cached-different-keys/100` | 1.29 µs | 1.24 µs | -3.8% |
| `turbo-cached-different-keys/1000` | 2.37 µs | 2.32 µs | -2.5% |
| `turbo-uncached/1` | 23.43 µs | 20.47 µs | -12.6% |
| `turbo-uncached/10` | 32.23 µs | 29.71 µs | -7.8% |
| `turbo-uncached/100` | 126.01 µs | 124.95 µs | -0.8% |
| `turbo-uncached/1000` | 1.079 ms | 1.048 ms | -2.9% |
| `turbo-uncached-parallel/1` | 6.16 µs | 5.67 µs | -8.0% |
| `turbo-uncached-parallel/10` | 5.47 µs | 5.12 µs | -6.4% |
| `turbo-uncached-parallel/100` | 14.65 µs | 14.55 µs | -0.7% |
| `turbo-uncached-parallel/1000` | 132.09 µs | 129.13 µs | -2.2% |

<!-- NEXT_JS_LLM_PR -->
parent 73e89d9f
...@@ -9934,6 +9934,7 @@ dependencies = [ ...@@ -9934,6 +9934,7 @@ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"bitfield", "bitfield",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"crossbeam-utils",
"dashmap 6.1.0", "dashmap 6.1.0",
"either", "either",
"futures", "futures",
......
...@@ -260,6 +260,7 @@ crc32fast = "1.4.2" ...@@ -260,6 +260,7 @@ crc32fast = "1.4.2"
criterion = { package = "codspeed-criterion-compat", version = "4.3.0" } criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
ctor = "0.8" ctor = "0.8"
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5.8"
crossbeam-utils = "0.8"
dashmap = "6.1.0" dashmap = "6.1.0"
data-encoding = "2.3.3" data-encoding = "2.3.3"
dhat = { version = "0.3.2" } dhat = { version = "0.3.2" }
......
...@@ -38,6 +38,7 @@ anyhow = { workspace = true } ...@@ -38,6 +38,7 @@ anyhow = { workspace = true }
bincode = { workspace = true } bincode = { workspace = true }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
bitfield = { workspace = true } bitfield = { workspace = true }
crossbeam-utils = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]} dashmap = { workspace = true, features = ["raw-api"]}
either = { workspace = true } either = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] } hashbrown = { workspace = true, features = ["raw"] }
......
...@@ -5,7 +5,6 @@ use std::{ ...@@ -5,7 +5,6 @@ use std::{
mem::take, mem::take,
num::NonZeroU32, num::NonZeroU32,
ops::{ControlFlow, Deref}, ops::{ControlFlow, Deref},
sync::Arc,
thread::yield_now, thread::yield_now,
time::{Duration, Instant}, time::{Duration, Instant},
}; };
...@@ -23,10 +22,7 @@ use tracing::span::Span; ...@@ -23,10 +22,7 @@ use tracing::span::Span;
feature = "trace_find_and_schedule" feature = "trace_find_and_schedule"
))] ))]
use tracing::trace_span; use tracing::trace_span;
use turbo_tasks::{ use turbo_tasks::{FxIndexMap, TaskExecutionReason, TaskId, TaskPriority, event::EventDescription};
FxIndexMap, TaskExecutionReason, TaskId, TaskPriority, backend::CachedTaskType,
event::EventDescription,
};
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
use crate::backend::operation::invalidate::TaskDirtyCause; use crate::backend::operation::invalidate::TaskDirtyCause;
...@@ -272,8 +268,6 @@ pub enum AggregationUpdateJob { ...@@ -272,8 +268,6 @@ pub enum AggregationUpdateJob {
// upon attempted serialization) similar to #[serde(skip)] on variants // upon attempted serialization) similar to #[serde(skip)] on variants
#[bincode(skip, default = "unreachable_decode")] #[bincode(skip, default = "unreachable_decode")]
task: TaskId, task: TaskId,
/// Set the task type if not already set
task_type: Option<Arc<CachedTaskType>>,
}, },
/// Increases the active counters of the tasks /// Increases the active counters of the tasks
IncreaseActiveCounts { IncreaseActiveCounts {
...@@ -1168,12 +1162,12 @@ impl AggregationUpdateQueue { ...@@ -1168,12 +1162,12 @@ impl AggregationUpdateQueue {
} }
} }
} }
AggregationUpdateJob::IncreaseActiveCount { task, task_type } => { AggregationUpdateJob::IncreaseActiveCount { task } => {
self.increase_active_count(ctx, task, task_type); self.increase_active_count(ctx, task);
} }
AggregationUpdateJob::IncreaseActiveCounts { mut task_ids } => { AggregationUpdateJob::IncreaseActiveCounts { mut task_ids } => {
if let Some(task_id) = task_ids.pop() { if let Some(task_id) = task_ids.pop() {
self.increase_active_count(ctx, task_id, None); self.increase_active_count(ctx, task_id);
if !task_ids.is_empty() { if !task_ids.is_empty() {
self.jobs.push_front(AggregationUpdateJobItem::new( self.jobs.push_front(AggregationUpdateJobItem::new(
AggregationUpdateJob::IncreaseActiveCounts { task_ids }, AggregationUpdateJob::IncreaseActiveCounts { task_ids },
...@@ -1380,10 +1374,7 @@ impl AggregationUpdateQueue { ...@@ -1380,10 +1374,7 @@ impl AggregationUpdateQueue {
let has_active_count = let has_active_count =
upper.get_activeness().is_some_and(|a| a.active_counter > 0); upper.get_activeness().is_some_and(|a| a.active_counter > 0);
if has_active_count { if has_active_count {
self.push(AggregationUpdateJob::IncreaseActiveCount { self.push(AggregationUpdateJob::IncreaseActiveCount { task: task_id });
task: task_id,
task_type: None,
});
} }
} }
// notify uppers about new follower // notify uppers about new follower
...@@ -2475,7 +2466,6 @@ impl AggregationUpdateQueue { ...@@ -2475,7 +2466,6 @@ impl AggregationUpdateQueue {
if has_active_count { if has_active_count {
self.push(AggregationUpdateJob::IncreaseActiveCount { self.push(AggregationUpdateJob::IncreaseActiveCount {
task: new_follower_id, task: new_follower_id,
task_type: None,
}); });
} }
...@@ -2633,29 +2623,16 @@ impl AggregationUpdateQueue { ...@@ -2633,29 +2623,16 @@ impl AggregationUpdateQueue {
/// Increases the active count of a task. /// Increases the active count of a task.
/// ///
/// Only used when activeness is tracked. /// Only used when activeness is tracked.
fn increase_active_count( fn increase_active_count(&mut self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) {
&mut self,
ctx: &mut impl ExecuteContext<'_>,
task_id: TaskId,
task_type: Option<Arc<CachedTaskType>>,
) {
#[cfg(feature = "trace_aggregation_update")] #[cfg(feature = "trace_aggregation_update")]
let _span = trace_span!("increase active count").entered(); let _span = trace_span!("increase active count").entered();
let mut task = ctx.task( let mut task = ctx.task(
task_id, task_id,
if task_type.is_some() { // For performance reasons this should stay Meta and not All.
TaskDataCategory::All // persistent_task_type is now set eagerly in initialize_new_task.
} else { TaskDataCategory::Meta,
// For performance reasons this should stay `Meta` and not `All`
TaskDataCategory::Meta
},
); );
if let Some(task_type) = task_type
&& !task.has_persistent_task_type()
{
task.set_persistent_task_type(task_type);
}
let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id)); let state = task.get_activeness_mut_or_insert_with(|| ActivenessState::new(task_id));
let is_new = state.is_empty(); let is_new = state.is_empty();
let is_positive_now = state.increment_active_counter(); let is_positive_now = state.increment_active_counter();
......
use std::sync::Arc;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use turbo_tasks::{TaskExecutionReason, TaskId, backend::CachedTaskType, event::EventDescription}; use turbo_tasks::{TaskExecutionReason, TaskId, event::EventDescription};
use crate::{ use crate::{
backend::{ backend::{
...@@ -13,7 +11,6 @@ use crate::{ ...@@ -13,7 +11,6 @@ use crate::{
storage_schema::TaskStorageAccessors, storage_schema::TaskStorageAccessors,
}, },
data::{InProgressState, InProgressStateInner}, data::{InProgressState, InProgressStateInner},
utils::arc_or_owned::ArcOrOwned,
}; };
#[derive(Encode, Decode, Clone, Default)] #[derive(Encode, Decode, Clone, Default)]
...@@ -30,7 +27,6 @@ impl ConnectChildOperation { ...@@ -30,7 +27,6 @@ impl ConnectChildOperation {
pub fn run( pub fn run(
parent_task_id: Option<TaskId>, parent_task_id: Option<TaskId>,
child_task_id: TaskId, child_task_id: TaskId,
child_task_type: Option<ArcOrOwned<CachedTaskType>>,
mut ctx: impl ExecuteContext<'_>, mut ctx: impl ExecuteContext<'_>,
) { ) {
if let Some(parent_task_id) = parent_task_id { if let Some(parent_task_id) = parent_task_id {
...@@ -43,8 +39,7 @@ impl ConnectChildOperation { ...@@ -43,8 +39,7 @@ impl ConnectChildOperation {
}; };
// Quick skip if the child was already connected before // Quick skip if the child was already connected before
// We can't call insert here as this would skip the mandatory task type update below // We defer the insert until after the aggregation queue is processed.
// Instead we only add it after updating the child task type
if new_children.contains(&child_task_id) { if new_children.contains(&child_task_id) {
return; return;
} }
...@@ -78,15 +73,9 @@ impl ConnectChildOperation { ...@@ -78,15 +73,9 @@ impl ConnectChildOperation {
if ctx.should_track_activeness() && parent_task_id.is_some() { if ctx.should_track_activeness() && parent_task_id.is_some() {
queue.push(AggregationUpdateJob::IncreaseActiveCount { queue.push(AggregationUpdateJob::IncreaseActiveCount {
task: child_task_id, task: child_task_id,
task_type: child_task_type.map(Arc::from),
}); });
} else { } else {
let mut child_task = ctx.task(child_task_id, TaskDataCategory::All); let mut child_task = ctx.task(child_task_id, TaskDataCategory::Meta);
if let Some(child_task_type) = child_task_type
&& !child_task.has_persistent_task_type()
{
child_task.set_persistent_task_type(child_task_type.into());
}
if !child_task.has_output() if !child_task.has_output()
&& child_task.add_scheduled( && child_task.add_scheduled(
......
...@@ -18,8 +18,9 @@ use tracing::info_span; ...@@ -18,8 +18,9 @@ use tracing::info_span;
#[cfg(feature = "trace_prepare_tasks")] #[cfg(feature = "trace_prepare_tasks")]
use tracing::trace_span; use tracing::trace_span;
use turbo_tasks::{ use turbo_tasks::{
CellId, FxIndexMap, TaskExecutionReason, TaskId, TaskPriority, TurboTasksBackendApi, CellId, FxIndexMap, MagicAny, RawVc, TaskExecutionReason, TaskId, TaskPriority,
TurboTasksCallApi, TypedSharedReference, backend::CachedTaskType, TurboTasksBackendApi, TurboTasksCallApi, TypedSharedReference, backend::CachedTaskType,
macro_helpers::NativeFunction,
}; };
use self::aggregation_update::ComputeDirtyAndCleanUpdate; use self::aggregation_update::ComputeDirtyAndCleanUpdate;
...@@ -101,8 +102,16 @@ pub trait ExecuteContext<'e>: Sized { ...@@ -101,8 +102,16 @@ pub trait ExecuteContext<'e>: Sized {
/// ///
/// Uses hash-based lookup which may return multiple candidates due to hash collisions, /// Uses hash-based lookup which may return multiple candidates due to hash collisions,
/// then verifies each candidate by comparing the stored `persistent_task_type`. /// then verifies each candidate by comparing the stored `persistent_task_type`.
/// Returns `Some(task_id)` if a matching task is found, `None` otherwise. /// Returns `Some((task_id, task_type))` if a matching task is found, where `task_type` is
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId>; /// the existing `Arc<CachedTaskType>` from storage (avoiding a duplicate allocation).
///
/// Accepts exploded components so the caller does not need to box the argument before calling.
fn task_by_type(
&mut self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn MagicAny,
) -> Option<(TaskId, Arc<CachedTaskType>)>;
} }
pub trait ChildExecuteContext<'e>: Send + Sized { pub trait ChildExecuteContext<'e>: Send + Sized {
...@@ -969,7 +978,12 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> { ...@@ -969,7 +978,12 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> {
self.turbo_tasks.pin() self.turbo_tasks.pin()
} }
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId> { fn task_by_type(
&mut self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn MagicAny,
) -> Option<(TaskId, Arc<CachedTaskType>)> {
if !self.backend.should_restore() { if !self.backend.should_restore() {
return None; return None;
} }
...@@ -978,7 +992,7 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> { ...@@ -978,7 +992,7 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> {
let candidates = self let candidates = self
.backend .backend
.backing_storage .backing_storage
.lookup_task_candidates(task_type) .lookup_task_candidates(native_fn, this, arg)
.expect("Failed to lookup task ids"); .expect("Failed to lookup task ids");
// Verify each candidate by comparing the stored persistent_task_type. // Verify each candidate by comparing the stored persistent_task_type.
...@@ -986,9 +1000,9 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> { ...@@ -986,9 +1000,9 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> {
for candidate_id in candidates { for candidate_id in candidates {
let task = self.task(candidate_id, TaskDataCategory::Data); let task = self.task(candidate_id, TaskDataCategory::Data);
if let Some(stored_type) = task.get_persistent_task_type() if let Some(stored_type) = task.get_persistent_task_type()
&& stored_type.as_ref() == task_type && stored_type.eq_components(native_fn, this, arg)
{ {
return Some(candidate_id); return Some((candidate_id, stored_type.clone()));
} }
} }
None None
......
...@@ -10,7 +10,7 @@ use std::{ ...@@ -10,7 +10,7 @@ use std::{
use thread_local::ThreadLocal; use thread_local::ThreadLocal;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{FxDashMap, TaskId, event::Event, parallel}; use turbo_tasks::{FxDashMap, TaskId, backend::CachedTaskType, event::Event, parallel};
use crate::{ use crate::{
backend::storage_schema::TaskStorage, backend::storage_schema::TaskStorage,
...@@ -164,11 +164,18 @@ impl Storage { ...@@ -164,11 +164,18 @@ impl Storage {
} }
/// Mark a newly allocated task as restored (skip DB queries) and new (include in persistence /// Mark a newly allocated task as restored (skip DB queries) and new (include in persistence
/// snapshots). /// snapshots). Optionally sets the `persistent_task_type` eagerly so it's available for
pub fn initialize_new_task(&self, task_id: TaskId) { /// persistence snapshots without needing to propagate it through `connect_child`.
pub fn initialize_new_task(&self, task_id: TaskId, task_type: Option<Arc<CachedTaskType>>) {
let mut task = self.access_mut(task_id); let mut task = self.access_mut(task_id);
task.flags.set_restored(TaskDataCategory::All); task.flags.set_restored(TaskDataCategory::All);
task.flags.set_new_task(true); task.flags.set_new_task(true);
if let Some(task_type) = task_type {
task.set_persistent_task_type(task_type);
if !task_id.is_transient() {
task.track_modification(SpecificTaskDataCategory::Data, "persistent_task_type");
}
}
} }
/// Processes every modified item (resp. a snapshot of it) with the given function and returns /// Processes every modified item (resp. a snapshot of it) with the given function and returns
......
...@@ -4,7 +4,9 @@ use anyhow::Result; ...@@ -4,7 +4,9 @@ use anyhow::Result;
use either::Either; use either::Either;
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{TaskId, backend::CachedTaskType}; use turbo_tasks::{
MagicAny, RawVc, TaskId, backend::CachedTaskType, macro_helpers::NativeFunction,
};
use turbo_tasks_hash::Xxh3Hash64Hasher; use turbo_tasks_hash::Xxh3Hash64Hasher;
use crate::backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage}; use crate::backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage};
...@@ -43,6 +45,20 @@ pub fn compute_task_type_hash(task_type: &CachedTaskType) -> TaskTypeHash { ...@@ -43,6 +45,20 @@ pub fn compute_task_type_hash(task_type: &CachedTaskType) -> TaskTypeHash {
hash.to_le_bytes() hash.to_le_bytes()
} }
/// Computes a deterministic 64-bit hash from task type components for use as a TaskCache key.
///
/// Like [`compute_task_type_hash`], but works with borrowed components so the caller does not need
/// to construct (and box-allocate) a full [`CachedTaskType`] first.
pub fn compute_task_type_hash_from_components(
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn MagicAny,
) -> TaskTypeHash {
let mut hasher = Xxh3Hash64Hasher::new();
CachedTaskType::hash_encode_components(native_fn, this, arg, &mut hasher);
hasher.finish().to_le_bytes()
}
/// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by /// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by
/// [`turbo_backing_storage`] or [`noop_backing_storage`]. /// [`turbo_backing_storage`] or [`noop_backing_storage`].
/// ///
...@@ -83,7 +99,12 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -83,7 +99,12 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
/// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key. /// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key.
/// The caller must verify each returned TaskId by comparing the stored task type which will /// The caller must verify each returned TaskId by comparing the stored task type which will
/// require a second database read /// require a second database read
fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>>; fn lookup_task_candidates(
&self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn MagicAny,
) -> Result<SmallVec<[TaskId; 1]>>;
/// Looks up and decodes persisted data for a single task, updating the provided storage with /// Looks up and decodes persisted data for a single task, updating the provided storage with
/// data from the database in the given category. /// data from the database in the given category.
fn lookup_data( fn lookup_data(
...@@ -149,8 +170,13 @@ where ...@@ -149,8 +170,13 @@ where
)) ))
} }
fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>> { fn lookup_task_candidates(
either::for_both!(self, this => this.lookup_task_candidates(key)) &self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn MagicAny,
) -> Result<SmallVec<[TaskId; 1]>> {
either::for_both!(self, this_impl => this_impl.lookup_task_candidates(native_fn, this, arg))
} }
fn lookup_data( fn lookup_data(
......
...@@ -9,8 +9,8 @@ use anyhow::{Context, Result}; ...@@ -9,8 +9,8 @@ use anyhow::{Context, Result};
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode}; use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode};
use turbo_tasks::{ use turbo_tasks::{
TaskId, MagicAny, RawVc, TaskId,
backend::CachedTaskType, macro_helpers::NativeFunction,
panic_hooks::{PanicHookGuard, register_panic_hook}, panic_hooks::{PanicHookGuard, register_panic_hook},
parallel, parallel,
}; };
...@@ -18,7 +18,9 @@ use turbo_tasks::{ ...@@ -18,7 +18,9 @@ use turbo_tasks::{
use crate::{ use crate::{
GitVersionInfo, GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage}, backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backing_storage::{BackingStorage, BackingStorageSealed, SnapshotItem, compute_task_type_hash}, backing_storage::{
BackingStorage, BackingStorageSealed, SnapshotItem, compute_task_type_hash_from_components,
},
database::{ database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
db_versioning::handle_db_versioning, db_versioning::handle_db_versioning,
...@@ -296,19 +298,24 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -296,19 +298,24 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
} }
} }
fn lookup_task_candidates(&self, task_type: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>> { fn lookup_task_candidates(
&self,
native_fn: &'static NativeFunction,
this: Option<RawVc>,
arg: &dyn MagicAny,
) -> Result<SmallVec<[TaskId; 1]>> {
let inner = &*self.inner; let inner = &*self.inner;
if inner.database.is_empty() { if inner.database.is_empty() {
// Checking if the database is empty is a performance optimization // Checking if the database is empty is a performance optimization
// to avoid computing the hash. // to avoid computing the hash.
return Ok(SmallVec::new()); return Ok(SmallVec::new());
} }
let hash = compute_task_type_hash(task_type); let hash = compute_task_type_hash_from_components(native_fn, this, arg);
let buffers = inner let buffers = inner
.database .database
.get_multiple(KeySpace::TaskCache, &hash) .get_multiple(KeySpace::TaskCache, &hash)
.with_context(|| { .with_context(|| {
format!("Looking up task id for {task_type:?} from database failed") format!("Looking up task id for {native_fn:?}(this={this:?}) from database failed")
})?; })?;
let mut task_ids = SmallVec::with_capacity(buffers.len()); let mut task_ids = SmallVec::with_capacity(buffers.len());
......
use std::{fmt::Debug, ops::Deref, sync::Arc};
use bincode::{
Decode, Encode,
de::Decoder,
error::{DecodeError, EncodeError},
impl_borrow_decode_with_context,
};
#[derive(Clone)]
pub enum ArcOrOwned<T> {
Arc(Arc<T>),
Owned(T),
}
impl<T> ArcOrOwned<T> {}
impl<T> Deref for ArcOrOwned<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::Arc(arc) => arc.deref(),
Self::Owned(value) => value,
}
}
}
impl<T> AsRef<T> for ArcOrOwned<T> {
fn as_ref(&self) -> &T {
self.deref()
}
}
impl<T> From<T> for ArcOrOwned<T> {
fn from(value: T) -> Self {
Self::Owned(value)
}
}
impl<T> From<Arc<T>> for ArcOrOwned<T> {
fn from(arc: Arc<T>) -> Self {
Self::Arc(arc)
}
}
impl<T> From<ArcOrOwned<T>> for Arc<T> {
fn from(value: ArcOrOwned<T>) -> Self {
match value {
ArcOrOwned::Arc(arc) => arc,
ArcOrOwned::Owned(value) => Arc::new(value),
}
}
}
impl<T> Encode for ArcOrOwned<T>
where
T: Encode,
{
fn encode<E: bincode::enc::Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
match self {
ArcOrOwned::Arc(arc) => arc.encode(encoder),
ArcOrOwned::Owned(value) => value.encode(encoder),
}
}
}
impl<T, Context> Decode<Context> for ArcOrOwned<T>
where
T: Decode<Context>,
{
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
let value = T::decode(decoder)?;
Ok(ArcOrOwned::Owned(value))
}
}
impl_borrow_decode_with_context!(ArcOrOwned<T>, Context, Context, T: Decode<Context>);
impl<T> Debug for ArcOrOwned<T>
where
T: Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.as_ref().fmt(f)
}
}
use std::{ use std::hash::{BuildHasher, Hash};
hash::{BuildHasher, Hash},
ops::{Deref, DerefMut},
};
use dashmap::{DashMap, RwLockWriteGuard, SharedValue}; use crossbeam_utils::CachePadded;
use dashmap::{DashMap, RwLock, RwLockWriteGuard, SharedValue};
use hashbrown::raw::{Bucket, InsertSlot, RawTable}; use hashbrown::raw::{Bucket, InsertSlot, RawTable};
pub fn raw_entry<'l, K: Eq + Hash + AsRef<Q>, V, Q: Eq + Hash, S: BuildHasher + Clone>( /// The type of a single shard inside a [`DashMap`].
map: &'l DashMap<K, V, S>, ///
key: &Q, /// `dashmap::HashMap<K, V>` is a private alias for `RawTable<(K, SharedValue<V>)>`.
pub type Shard<K, V> = CachePadded<RwLock<RawTable<(K, SharedValue<V>)>>>;
/// Returns a reference to the shard that owns the given pre-computed hash,
/// without locking anything.
///
/// Pass the returned reference to [`raw_get_in_shard`] and
/// [`raw_entry_in_shard`] so that the shard is only located once even when a
/// read-lock miss is followed by a write-lock retry.
pub fn get_shard<K: Eq + Hash, V, S: BuildHasher + Clone>(
map: &DashMap<K, V, S>,
hash: u64,
) -> &Shard<K, V> {
let idx = map.determine_shard(hash as usize);
&map.shards()[idx]
}
/// Read-only heterogeneous lookup using a pre-located shard reference.
/// Returns `Some(value)` on hit, `None` on miss. Uses only a read lock.
pub fn raw_get_in_shard<K: Eq + Hash, V: Copy>(
shard: &Shard<K, V>,
hash: u64,
eq: impl Fn(&K) -> bool,
) -> Option<V> {
let guard = shard.read();
// Safety: We have a read lock on the shard.
guard
.find(hash, |(k, _v)| eq(k))
.map(|bucket| *unsafe { bucket.as_ref() }.1.get())
}
/// Write-lock entry lookup using a pre-located shard reference and
/// heterogeneous equality.
///
/// Takes a pre-located `shard` (from [`get_shard`]) and `hash` so the shard is
/// not located a second time on a read-miss / write-retry path.
pub fn raw_entry_in_shard<'l, K: Eq + Hash, V, S: BuildHasher + Clone>(
shard: &'l Shard<K, V>,
map_hasher: &S,
hash: u64,
eq: impl Fn(&K) -> bool,
) -> RawEntry<'l, K, V> { ) -> RawEntry<'l, K, V> {
let hasher = map.hasher(); let mut guard = shard.write();
let hash = hasher.hash_one(key); let result =
let shard = map.determine_shard(hash as usize); guard.find_or_find_insert_slot(hash, |(k, _v)| eq(k), |(k, _v)| map_hasher.hash_one(k));
let mut shard = map.shards()[shard].write();
let result = shard.find_or_find_insert_slot(
hash,
|(k, _v)| k.as_ref() == key,
|(k, _v)| hasher.hash_one(k),
);
match result { match result {
Ok(bucket) => RawEntry::Occupied(OccupiedEntry { bucket, shard }), Ok(bucket) => RawEntry::Occupied(OccupiedEntry {
bucket,
shard: guard,
}),
Err(insert_slot) => RawEntry::Vacant(VacantEntry { Err(insert_slot) => RawEntry::Vacant(VacantEntry {
hash, hash,
insert_slot, insert_slot,
shard, shard: guard,
}), }),
} }
} }
...@@ -34,19 +69,6 @@ pub enum RawEntry<'l, K, V> { ...@@ -34,19 +69,6 @@ pub enum RawEntry<'l, K, V> {
Vacant(VacantEntry<'l, K, V>), Vacant(VacantEntry<'l, K, V>),
} }
impl<'l, K, V> RawEntry<'l, K, V> {
#[allow(dead_code)]
pub fn or_insert_with<F: FnOnce() -> (K, V)>(self, f: F) -> RefMut<'l, K, V> {
match self {
RawEntry::Occupied(occupied) => occupied.into_mut(),
RawEntry::Vacant(vacant) => {
let (key, value) = f();
vacant.insert(key, value)
}
}
}
}
pub struct OccupiedEntry<'l, K, V> { pub struct OccupiedEntry<'l, K, V> {
bucket: Bucket<(K, SharedValue<V>)>, bucket: Bucket<(K, SharedValue<V>)>,
#[allow(dead_code, reason = "kept to ensure the lock lives long enough")] #[allow(dead_code, reason = "kept to ensure the lock lives long enough")]
...@@ -59,20 +81,6 @@ impl<'l, K, V> OccupiedEntry<'l, K, V> { ...@@ -59,20 +81,6 @@ impl<'l, K, V> OccupiedEntry<'l, K, V> {
// exist. // exist.
unsafe { self.bucket.as_ref().1.get() } unsafe { self.bucket.as_ref().1.get() }
} }
#[allow(dead_code)]
pub fn get_mut(&mut self) -> &mut V {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_mut().1.get_mut() }
}
#[allow(dead_code)]
pub fn into_mut(self) -> RefMut<'l, K, V> {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
RefMut::from_bucket(self.bucket, self.shard)
}
} }
pub struct VacantEntry<'l, K, V> { pub struct VacantEntry<'l, K, V> {
...@@ -82,48 +90,13 @@ pub struct VacantEntry<'l, K, V> { ...@@ -82,48 +90,13 @@ pub struct VacantEntry<'l, K, V> {
} }
impl<'l, K, V> VacantEntry<'l, K, V> { impl<'l, K, V> VacantEntry<'l, K, V> {
pub fn insert(mut self, key: K, value: V) -> RefMut<'l, K, V> { pub fn insert(mut self, key: K, value: V) {
let shared_value = SharedValue::new(value); let shared_value = SharedValue::new(value);
// Safety: The insert slot is valid and the map has not be modified since we obtained it (we // Safety: The insert slot is valid and the map has not been modified since we obtained it
// hold the write lock). // (we hold the write lock).
unsafe { unsafe {
let bucket = self.shard
self.shard .insert_in_slot(self.hash, self.insert_slot, (key, shared_value));
.insert_in_slot(self.hash, self.insert_slot, (key, shared_value));
RefMut::from_bucket(bucket, self.shard)
} }
} }
} }
pub struct RefMut<'l, K, V> {
bucket: Bucket<(K, SharedValue<V>)>,
#[allow(dead_code, reason = "kept to ensure the lock lives long enough")]
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
}
impl<'l, K, V> RefMut<'l, K, V> {
fn from_bucket(
bucket: Bucket<(K, SharedValue<V>)>,
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
) -> Self {
Self { bucket, shard }
}
}
impl<'l, K, V> Deref for RefMut<'l, K, V> {
type Target = V;
fn deref(&self) -> &Self::Target {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_ref().1.get() }
}
}
impl<'l, K, V> DerefMut for RefMut<'l, K, V> {
fn deref_mut(&mut self) -> &mut Self::Target {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_mut().1.get_mut() }
}
}
pub mod arc_or_owned;
pub mod dash_map_drop_contents; pub mod dash_map_drop_contents;
pub mod dash_map_multi; pub mod dash_map_multi;
pub mod dash_map_raw_entry; pub mod dash_map_raw_entry;
......
...@@ -452,11 +452,13 @@ impl TurboFn<'_> { ...@@ -452,11 +452,13 @@ impl TurboFn<'_> {
let exposed_input_types: Vec<_> = self.exposed_input_types().collect(); let exposed_input_types: Vec<_> = self.exposed_input_types().collect();
return Some(FilterTraitCallArgsTokens { return Some(FilterTraitCallArgsTokens {
filter_owned: quote! { filter_owned: quote! {
|magic_any| { |arg: &mut dyn turbo_tasks::StackMagicAny| {
let (#(#exposed_input_idents,)*) = let (#(#exposed_input_idents,)*) =
*turbo_tasks::macro_helpers turbo_tasks::macro_helpers
::downcast_args_owned::<(#(#exposed_input_types,)*)>(magic_any); ::downcast_stack_args_owned::<(#(#exposed_input_types,)*)>(arg);
::std::boxed::Box::new((#(#inline_input_idents,)*)) turbo_tasks::OwnedMagicAny::new(
::std::boxed::Box::new((#(#inline_input_idents,)*))
)
} }
}, },
filter_and_resolve: quote! { filter_and_resolve: quote! {
...@@ -484,13 +486,13 @@ impl TurboFn<'_> { ...@@ -484,13 +486,13 @@ impl TurboFn<'_> {
pub fn persistence(&self) -> impl ToTokens { pub fn persistence(&self) -> impl ToTokens {
quote! { quote! {
turbo_tasks::macro_helpers::get_persistence_from_inputs(&*inputs) turbo_tasks::macro_helpers::get_persistence_from_inputs(&inputs)
} }
} }
pub fn persistence_with_this(&self) -> impl ToTokens { pub fn persistence_with_this(&self) -> impl ToTokens {
quote! { quote! {
turbo_tasks::macro_helpers::get_persistence_from_inputs_and_this(this, &*inputs) turbo_tasks::macro_helpers::get_persistence_from_inputs_and_this(this, &inputs)
} }
} }
...@@ -553,16 +555,17 @@ impl TurboFn<'_> { ...@@ -553,16 +555,17 @@ impl TurboFn<'_> {
quote! { quote! {
{ {
#assertions #assertions
let inputs = std::boxed::Box::new((#(#inputs,)*)); let inputs = (#(#inputs,)*);
let this = #converted_this; let this = #converted_this;
let persistence = #persistence; let persistence = #persistence;
let mut arg = turbo_tasks::StackMagicAnySlot::new(inputs);
static TRAIT_METHOD: turbo_tasks::macro_helpers::Lazy<&'static turbo_tasks::TraitMethod> = static TRAIT_METHOD: turbo_tasks::macro_helpers::Lazy<&'static turbo_tasks::TraitMethod> =
turbo_tasks::macro_helpers::Lazy::new(|| #trait_type_ident.get(stringify!(#ident))); turbo_tasks::macro_helpers::Lazy::new(|| #trait_type_ident.get(stringify!(#ident)));
<#output as turbo_tasks::task::TaskOutput>::try_from_raw_vc( <#output as turbo_tasks::task::TaskOutput>::try_from_raw_vc(
turbo_tasks::trait_call( turbo_tasks::trait_call(
*TRAIT_METHOD, *TRAIT_METHOD,
this, this,
inputs as std::boxed::Box<dyn turbo_tasks::MagicAny>, &mut arg,
persistence, persistence,
) )
) )
...@@ -582,14 +585,15 @@ impl TurboFn<'_> { ...@@ -582,14 +585,15 @@ impl TurboFn<'_> {
quote! { quote! {
{ {
#assertions #assertions
let inputs = std::boxed::Box::new((#(#inputs,)*));
let this = #converted_this; let this = #converted_this;
let inputs = (#(#inputs,)*);
let persistence = #persistence; let persistence = #persistence;
let mut arg = turbo_tasks::StackMagicAnySlot::new(inputs);
<#output as turbo_tasks::task::TaskOutput>::try_from_raw_vc( <#output as turbo_tasks::task::TaskOutput>::try_from_raw_vc(
turbo_tasks::dynamic_call( turbo_tasks::dynamic_call(
&#native_function_ident, &#native_function_ident,
Some(this), Some(this),
inputs as std::boxed::Box<dyn turbo_tasks::MagicAny>, &mut arg,
persistence, persistence,
) )
) )
...@@ -600,13 +604,14 @@ impl TurboFn<'_> { ...@@ -600,13 +604,14 @@ impl TurboFn<'_> {
quote! { quote! {
{ {
#assertions #assertions
let inputs = std::boxed::Box::new((#(#inputs,)*)); let inputs = (#(#inputs,)*);
let persistence = #persistence; let persistence = #persistence;
let mut arg = turbo_tasks::StackMagicAnySlot::new(inputs);
<#output as turbo_tasks::task::TaskOutput>::try_from_raw_vc( <#output as turbo_tasks::task::TaskOutput>::try_from_raw_vc(
turbo_tasks::dynamic_call( turbo_tasks::dynamic_call(
&#native_function_ident, &#native_function_ident,
None, None,
inputs as std::boxed::Box<dyn turbo_tasks::MagicAny>, &mut arg,
persistence, persistence,
) )
) )
...@@ -1118,8 +1123,8 @@ impl NativeFn { ...@@ -1118,8 +1123,8 @@ impl NativeFn {
quote! { quote! {
turbo_tasks::macro_helpers::ArgMeta::with_filter_trait_call_from( turbo_tasks::macro_helpers::ArgMeta::with_filter_trait_call_from(
&#task_fn, &#task_fn,
#filter_owned, Some(#filter_owned),
#filter_and_resolve, Some(#filter_and_resolve),
) )
} }
} else { } else {
......
...@@ -18,7 +18,8 @@ use smallvec::SmallVec; ...@@ -18,7 +18,8 @@ use smallvec::SmallVec;
use tokio::sync::mpsc::Receiver; use tokio::sync::mpsc::Receiver;
use turbo_tasks::{ use turbo_tasks::{
CellId, ExecutionId, InvalidationReason, LocalTaskId, MagicAny, RawVc, ReadCellOptions, CellId, ExecutionId, InvalidationReason, LocalTaskId, MagicAny, RawVc, ReadCellOptions,
ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId, TurboTasksApi, TurboTasksCallApi, ReadOutputOptions, StackMagicAny, TaskId, TaskPersistence, TraitTypeId, TurboTasksApi,
TurboTasksCallApi,
backend::{CellContent, TaskCollectiblesMap, TypedCellContent, VerificationMode}, backend::{CellContent, TaskCollectiblesMap, TypedCellContent, VerificationMode},
event::{Event, EventListener}, event::{Event, EventListener},
message_queue::CompilationEvent, message_queue::CompilationEvent,
...@@ -96,26 +97,26 @@ impl TurboTasksCallApi for VcStorage { ...@@ -96,26 +97,26 @@ impl TurboTasksCallApi for VcStorage {
&self, &self,
func: &'static turbo_tasks::macro_helpers::NativeFunction, func: &'static turbo_tasks::macro_helpers::NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
_persistence: TaskPersistence, _persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
self.dynamic_call(func, this, arg) self.dynamic_call(func, this, arg.take_box())
} }
fn native_call( fn native_call(
&self, &self,
_func: &'static turbo_tasks::macro_helpers::NativeFunction, func: &'static turbo_tasks::macro_helpers::NativeFunction,
_this: Option<RawVc>, this: Option<RawVc>,
_arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
_persistence: TaskPersistence, _persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
unreachable!() self.dynamic_call(func, this, arg.take_box())
} }
fn trait_call( fn trait_call(
&self, &self,
_trait_type: &'static turbo_tasks::TraitMethod, _trait_type: &'static turbo_tasks::TraitMethod,
_this: RawVc, _this: RawVc,
_arg: Box<dyn MagicAny>, _arg: &mut dyn StackMagicAny,
_persistence: TaskPersistence, _persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
unreachable!() unreachable!()
......
This diff is collapsed.
...@@ -83,7 +83,7 @@ pub use crate::{ ...@@ -83,7 +83,7 @@ pub use crate::{
get_invalidator, get_invalidator,
}, },
join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt}, join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt},
magic_any::MagicAny, magic_any::{MagicAny, OwnedMagicAny, StackMagicAny, StackMagicAnySlot},
manager::{ manager::{
CurrentCellRef, ReadCellTracking, ReadConsistency, ReadTracking, TaskPersistence, CurrentCellRef, ReadCellTracking, ReadConsistency, ReadTracking, TaskPersistence,
TaskPriority, TurboTasks, TurboTasksApi, TurboTasksBackendApi, TurboTasksCallApi, Unused, TaskPriority, TurboTasks, TurboTasksApi, TurboTasksBackendApi, TurboTasksCallApi, Unused,
......
...@@ -22,6 +22,7 @@ pub use crate::{ ...@@ -22,6 +22,7 @@ pub use crate::{
manager::{find_cell_by_id, find_cell_by_type, spawn_detached_for_testing}, manager::{find_cell_by_id, find_cell_by_type, spawn_detached_for_testing},
native_function::{ native_function::{
ArgMeta, NativeFunction, VTABLE_DEFAULT, downcast_args_owned, downcast_args_ref, ArgMeta, NativeFunction, VTABLE_DEFAULT, downcast_args_owned, downcast_args_ref,
downcast_stack_args_owned,
}, },
registry::RegistryDef, registry::RegistryDef,
task::function::{into_task_fn, into_task_fn_with_this}, task::function::{into_task_fn, into_task_fn_with_this},
......
...@@ -38,3 +38,98 @@ pub fn any_as_encode<T: Any>(this: &dyn Any) -> &T { ...@@ -38,3 +38,98 @@ pub fn any_as_encode<T: Any>(this: &dyn Any) -> &T {
type_name::<T>() type_name::<T>()
); );
} }
/// A trait for task arguments that may reside on the stack.
///
/// This enables deferred boxing: on the cache-hit path (~85%), we only borrow
/// the argument via [`as_ref`](StackMagicAny::as_ref) for hash/equality lookups,
/// avoiding any heap allocation. On cache miss, [`take_box`](StackMagicAny::take_box)
/// moves the value into a `Box` with zero clones.
pub trait StackMagicAny {
/// Borrow the argument as a type-erased reference (for cache lookup).
fn as_ref(&self) -> &dyn MagicAny;
/// Move the argument out into a heap-allocated Box (panics if already taken).
fn take_box(&mut self) -> Box<dyn MagicAny>;
/// Downcast to `&mut dyn Any` for concrete type recovery without boxing.
fn as_any_mut(&mut self) -> &mut dyn Any;
}
/// Stack-resident slot wrapping a concrete typed value.
///
/// Created by macro-generated callsites. The value starts in `Some` on the
/// stack; [`take_box`](StackMagicAny::take_box) moves it to the heap on cache miss.
#[repr(transparent)]
pub struct StackMagicAnySlot<T> {
slot: Option<T>,
}
impl<T> StackMagicAnySlot<T> {
#[inline]
pub fn new(value: T) -> Self {
Self { slot: Some(value) }
}
#[inline]
pub fn take(&mut self) -> T {
self.slot
.take()
.expect("StackMagicAnySlot::take called after value was already taken")
}
}
impl<T: MagicAny> StackMagicAny for StackMagicAnySlot<T> {
#[inline]
fn as_ref(&self) -> &dyn MagicAny {
self.slot
.as_ref()
.expect("StackMagicAnySlot::as_ref called after take_box")
}
#[inline]
fn take_box(&mut self) -> Box<dyn MagicAny> {
Box::new(
self.slot
.take()
.expect("StackMagicAnySlot::take_box called twice"),
)
}
#[inline]
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
}
/// Adapter for an already-boxed value (e.g., from async resolution tasks).
pub struct OwnedMagicAny {
slot: Option<Box<dyn MagicAny>>,
}
impl OwnedMagicAny {
#[inline]
pub fn new(value: Box<dyn MagicAny>) -> Self {
Self { slot: Some(value) }
}
}
impl StackMagicAny for OwnedMagicAny {
#[inline]
fn as_ref(&self) -> &dyn MagicAny {
&**self
.slot
.as_ref()
.expect("OwnedMagicAny::as_ref called after take_box")
}
#[inline]
fn take_box(&mut self) -> Box<dyn MagicAny> {
self.slot
.take()
.expect("OwnedMagicAny::take_box called twice")
}
#[inline]
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
}
...@@ -29,8 +29,8 @@ use crate::{ ...@@ -29,8 +29,8 @@ use crate::{
ReadOutputOptions, ResolvedVc, SharedReference, TaskId, TraitMethod, ValueTypeId, Vc, VcRead, ReadOutputOptions, ResolvedVc, SharedReference, TaskId, TraitMethod, ValueTypeId, Vc, VcRead,
VcValueTrait, VcValueType, VcValueTrait, VcValueType,
backend::{ backend::{
Backend, CachedTaskType, CellContent, CellHash, TaskCollectiblesMap, TaskExecutionSpec, Backend, CellContent, CellHash, TaskCollectiblesMap, TaskExecutionSpec, TransientTaskType,
TransientTaskType, TurboTasksExecutionError, TypedCellContent, VerificationMode, TurboTasksExecutionError, TypedCellContent, VerificationMode,
}, },
capture_future::CaptureFuture, capture_future::CaptureFuture,
event::{Event, EventListener}, event::{Event, EventListener},
...@@ -38,7 +38,7 @@ use crate::{ ...@@ -38,7 +38,7 @@ use crate::{
id_factory::IdFactoryWithReuse, id_factory::IdFactoryWithReuse,
keyed::KeyedEq, keyed::KeyedEq,
macro_helpers::NativeFunction, macro_helpers::NativeFunction,
magic_any::MagicAny, magic_any::StackMagicAny,
message_queue::{CompilationEvent, CompilationEventQueue}, message_queue::{CompilationEvent, CompilationEventQueue},
priority_runner::{Executor, JoinHandle, PriorityRunner}, priority_runner::{Executor, JoinHandle, PriorityRunner},
raw_vc::{CellId, RawVc}, raw_vc::{CellId, RawVc},
...@@ -59,7 +59,7 @@ pub trait TurboTasksCallApi: Sync + Send { ...@@ -59,7 +59,7 @@ pub trait TurboTasksCallApi: Sync + Send {
&self, &self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc; ) -> RawVc;
/// Call a native function with arguments. /// Call a native function with arguments.
...@@ -68,7 +68,7 @@ pub trait TurboTasksCallApi: Sync + Send { ...@@ -68,7 +68,7 @@ pub trait TurboTasksCallApi: Sync + Send {
&self, &self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc; ) -> RawVc;
/// Calls a trait method with arguments. First input is the `self` object. /// Calls a trait method with arguments. First input is the `self` object.
...@@ -77,7 +77,7 @@ pub trait TurboTasksCallApi: Sync + Send { ...@@ -77,7 +77,7 @@ pub trait TurboTasksCallApi: Sync + Send {
&self, &self,
trait_method: &'static TraitMethod, trait_method: &'static TraitMethod,
this: RawVc, this: RawVc,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc; ) -> RawVc;
...@@ -791,38 +791,33 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -791,38 +791,33 @@ impl<B: Backend + 'static> TurboTasks<B> {
&self, &self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
let task_type = CachedTaskType { RawVc::TaskOutput(self.backend.get_or_create_task(
native_fn, native_fn,
this, this,
arg, arg,
}; current_task_if_available("turbo_function calls"),
RawVc::TaskOutput(match persistence { persistence,
TaskPersistence::Transient => self.backend.get_or_create_transient_task( self,
task_type, ))
current_task_if_available("turbo_function calls"),
self,
),
TaskPersistence::Persistent => self.backend.get_or_create_persistent_task(
task_type,
current_task_if_available("turbo_function calls"),
self,
),
})
} }
pub fn dynamic_call( pub fn dynamic_call(
&self, &self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
if this.is_none_or(|this| this.is_resolved()) && native_fn.arg_meta.is_resolved(&*arg) { if this.is_none_or(|this| this.is_resolved())
&& native_fn.arg_meta.is_resolved(arg.as_ref())
{
return self.native_call(native_fn, this, arg, persistence); return self.native_call(native_fn, this, arg, persistence);
} }
// Need async resolution — must move the arg to the heap now
let arg = arg.take_box();
let task_type = LocalTaskSpec { let task_type = LocalTaskSpec {
task_type: LocalTaskType::ResolveNative { native_fn }, task_type: LocalTaskType::ResolveNative { native_fn },
this, this,
...@@ -835,7 +830,7 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -835,7 +830,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
&self, &self,
trait_method: &'static TraitMethod, trait_method: &'static TraitMethod,
this: RawVc, this: RawVc,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
// avoid creating a wrapper task if self is already resolved // avoid creating a wrapper task if self is already resolved
...@@ -844,8 +839,12 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -844,8 +839,12 @@ impl<B: Backend + 'static> TurboTasks<B> {
if let RawVc::TaskCell(_, CellId { type_id, .. }) = this { if let RawVc::TaskCell(_, CellId { type_id, .. }) = this {
match registry::get_value_type(type_id).get_trait_method(trait_method) { match registry::get_value_type(type_id).get_trait_method(trait_method) {
Some(native_fn) => { Some(native_fn) => {
let arg = native_fn.arg_meta.filter_owned(arg); if let Some(filter) = native_fn.arg_meta.filter_owned {
return self.dynamic_call(native_fn, Some(this), arg, persistence); let mut arg = (filter)(arg);
return self.dynamic_call(native_fn, Some(this), &mut arg, persistence);
} else {
return self.dynamic_call(native_fn, Some(this), arg, persistence);
}
} }
None => { None => {
// We are destined to fail at this point, but we just retry resolution in the // We are destined to fail at this point, but we just retry resolution in the
...@@ -859,7 +858,7 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -859,7 +858,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
let task_type = LocalTaskSpec { let task_type = LocalTaskSpec {
task_type: LocalTaskType::ResolveTrait { trait_method }, task_type: LocalTaskType::ResolveTrait { trait_method },
this: Some(this), this: Some(this),
arg, arg: arg.take_box(),
}; };
self.schedule_local_task(task_type, persistence) self.schedule_local_task(task_type, persistence)
...@@ -1361,7 +1360,7 @@ impl<B: Backend + 'static> TurboTasksCallApi for TurboTasks<B> { ...@@ -1361,7 +1360,7 @@ impl<B: Backend + 'static> TurboTasksCallApi for TurboTasks<B> {
&self, &self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
self.dynamic_call(native_fn, this, arg, persistence) self.dynamic_call(native_fn, this, arg, persistence)
...@@ -1370,7 +1369,7 @@ impl<B: Backend + 'static> TurboTasksCallApi for TurboTasks<B> { ...@@ -1370,7 +1369,7 @@ impl<B: Backend + 'static> TurboTasksCallApi for TurboTasks<B> {
&self, &self,
native_fn: &'static NativeFunction, native_fn: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
self.native_call(native_fn, this, arg, persistence) self.native_call(native_fn, this, arg, persistence)
...@@ -1379,7 +1378,7 @@ impl<B: Backend + 'static> TurboTasksCallApi for TurboTasks<B> { ...@@ -1379,7 +1378,7 @@ impl<B: Backend + 'static> TurboTasksCallApi for TurboTasks<B> {
&self, &self,
trait_method: &'static TraitMethod, trait_method: &'static TraitMethod,
this: RawVc, this: RawVc,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
self.trait_call(trait_method, this, arg, persistence) self.trait_call(trait_method, this, arg, persistence)
...@@ -1838,7 +1837,7 @@ pub async fn run_once_with_reason<T: Send + 'static>( ...@@ -1838,7 +1837,7 @@ pub async fn run_once_with_reason<T: Send + 'static>(
pub fn dynamic_call( pub fn dynamic_call(
func: &'static NativeFunction, func: &'static NativeFunction,
this: Option<RawVc>, this: Option<RawVc>,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
with_turbo_tasks(|tt| tt.dynamic_call(func, this, arg, persistence)) with_turbo_tasks(|tt| tt.dynamic_call(func, this, arg, persistence))
...@@ -1848,7 +1847,7 @@ pub fn dynamic_call( ...@@ -1848,7 +1847,7 @@ pub fn dynamic_call(
pub fn trait_call( pub fn trait_call(
trait_method: &'static TraitMethod, trait_method: &'static TraitMethod,
this: RawVc, this: RawVc,
arg: Box<dyn MagicAny>, arg: &mut dyn StackMagicAny,
persistence: TaskPersistence, persistence: TaskPersistence,
) -> RawVc { ) -> RawVc {
with_turbo_tasks(|tt| tt.trait_call(trait_method, this, arg, persistence)) with_turbo_tasks(|tt| tt.trait_call(trait_method, this, arg, persistence))
......
...@@ -10,7 +10,7 @@ use turbo_tasks_hash::DeterministicHasher; ...@@ -10,7 +10,7 @@ use turbo_tasks_hash::DeterministicHasher;
use crate::{ use crate::{
RawVc, TaskExecutionReason, TaskInput, TaskPersistence, TaskPriority, RawVc, TaskExecutionReason, TaskInput, TaskPersistence, TaskPriority,
macro_helpers::into_task_fn, macro_helpers::into_task_fn,
magic_any::{MagicAny, any_as_encode}, magic_any::{MagicAny, OwnedMagicAny, StackMagicAny, StackMagicAnySlot, any_as_encode},
registry::{RegistryType, turbo_registry}, registry::{RegistryType, turbo_registry},
task::{TaskFn, TaskFnInputs, function::NativeTaskFuture}, task::{TaskFn, TaskFnInputs, function::NativeTaskFuture},
}; };
...@@ -21,7 +21,7 @@ type ResolveFunctor = for<'a> fn(&'a dyn MagicAny) -> ResolveFuture<'a>; ...@@ -21,7 +21,7 @@ type ResolveFunctor = for<'a> fn(&'a dyn MagicAny) -> ResolveFuture<'a>;
type IsResolvedFunctor = fn(&dyn MagicAny) -> bool; type IsResolvedFunctor = fn(&dyn MagicAny) -> bool;
#[doc(hidden)] #[doc(hidden)]
pub type FilterOwnedArgsFunctor = for<'a> fn(Box<dyn MagicAny>) -> Box<dyn MagicAny>; pub type FilterOwnedArgsFunctor = for<'a> fn(&'a mut dyn StackMagicAny) -> OwnedMagicAny;
#[doc(hidden)] #[doc(hidden)]
pub type FilterAndResolveFunctor = ResolveFunctor; pub type FilterAndResolveFunctor = ResolveFunctor;
...@@ -38,16 +38,18 @@ pub struct ArgMeta { ...@@ -38,16 +38,18 @@ pub struct ArgMeta {
pub hash_encode: AnyHashEncodeFn, pub hash_encode: AnyHashEncodeFn,
is_resolved: IsResolvedFunctor, is_resolved: IsResolvedFunctor,
resolve: ResolveFunctor, resolve: ResolveFunctor,
/// Used for trait methods, filters out unused arguments. /// Used for trait methods to filter out unused arguments. `None` when all arguments are used
filter_owned: FilterOwnedArgsFunctor, /// (no filtering needed).
pub(crate) filter_owned: Option<FilterOwnedArgsFunctor>,
/// Accepts a reference (instead of ownership) of arguments, and does the filtering and /// Accepts a reference (instead of ownership) of arguments, and does the filtering and
/// resolution in a single operation. /// resolution in a single operation. `None` when all arguments are used (no filtering needed),
/// in which case the caller should use [`resolve`](ArgMeta::resolve) directly.
// //
// When filtering a `&dyn MagicAny` while running a resolution task, we can't return a filtered // When filtering a `&dyn MagicAny` while running a resolution task, we can't return a filtered
// `&dyn MagicAny`, we'd be forced to return a `Box<dyn MagicAny>`. However, the next thing we // `&dyn MagicAny`, we'd be forced to return a `Box<dyn MagicAny>`. However, the next thing we
// do is resolution, which also accepts a `&dyn MagicAny` and returns a `Box<dyn MagicAny>`. // do is resolution, which also accepts a `&dyn MagicAny` and returns a `Box<dyn MagicAny>`.
// This functor combines the two operations to avoid extra cloning. // This functor combines the two operations to avoid extra cloning.
filter_and_resolve: FilterAndResolveFunctor, filter_and_resolve: Option<FilterAndResolveFunctor>,
} }
impl ArgMeta { impl ArgMeta {
...@@ -64,8 +66,8 @@ impl ArgMeta { ...@@ -64,8 +66,8 @@ impl ArgMeta {
#[doc(hidden)] #[doc(hidden)]
pub const fn with_filter_trait_call_from<T>( pub const fn with_filter_trait_call_from<T>(
_t: &T, _t: &T,
filter_owned: FilterOwnedArgsFunctor, filter_owned: Option<FilterOwnedArgsFunctor>,
filter_and_resolve: FilterAndResolveFunctor, filter_and_resolve: Option<FilterAndResolveFunctor>,
) -> Self ) -> Self
where where
T: TaskFnInputs, T: TaskFnInputs,
...@@ -77,15 +79,12 @@ impl ArgMeta { ...@@ -77,15 +79,12 @@ impl ArgMeta {
where where
T: TaskInput + Encode + Decode<()> + 'static, T: TaskInput + Encode + Decode<()> + 'static,
{ {
fn noop_filter_args(args: Box<dyn MagicAny>) -> Box<dyn MagicAny> { Self::with_filter_trait_call::<T>(None, None)
args
}
Self::with_filter_trait_call::<T>(noop_filter_args, resolve_functor_impl::<T>)
} }
pub const fn with_filter_trait_call<T>( pub const fn with_filter_trait_call<T>(
filter_owned: FilterOwnedArgsFunctor, filter_owned: Option<FilterOwnedArgsFunctor>,
filter_and_resolve: FilterAndResolveFunctor, filter_and_resolve: Option<FilterAndResolveFunctor>,
) -> Self ) -> Self
where where
T: TaskInput + Encode + Decode<()> + 'static, T: TaskInput + Encode + Decode<()> + 'static,
...@@ -121,14 +120,12 @@ impl ArgMeta { ...@@ -121,14 +120,12 @@ impl ArgMeta {
(self.resolve)(value).await (self.resolve)(value).await
} }
pub fn filter_owned(&self, args: Box<dyn MagicAny>) -> Box<dyn MagicAny> {
(self.filter_owned)(args)
}
/// This will return `(None, _)` even if the target is a method, if the method does not use
/// `self`.
pub async fn filter_and_resolve(&self, args: &dyn MagicAny) -> Result<Box<dyn MagicAny>> { pub async fn filter_and_resolve(&self, args: &dyn MagicAny) -> Result<Box<dyn MagicAny>> {
(self.filter_and_resolve)(args).await if let Some(filter_and_resolve) = self.filter_and_resolve {
(filter_and_resolve)(args).await
} else {
(self.resolve)(args).await
}
} }
} }
...@@ -179,6 +176,21 @@ pub fn downcast_args_ref<T: MagicAny>(args: &dyn MagicAny) -> &T { ...@@ -179,6 +176,21 @@ pub fn downcast_args_ref<T: MagicAny>(args: &dyn MagicAny) -> &T {
.unwrap() .unwrap()
} }
/// Downcast a `&mut dyn StackMagicAny` to a concrete [`StackMagicAnySlot<T>`] and take the value
/// out, avoiding the intermediate heap allocation that `take_box` + `downcast_args_owned` would
/// require.
pub fn downcast_stack_args_owned<T: MagicAny>(args: &mut dyn StackMagicAny) -> T {
args.as_any_mut()
.downcast_mut::<StackMagicAnySlot<T>>()
.unwrap_or_else(|| {
panic!(
"downcast_stack_args_owned::<{}> called with incorrect StackMagicAny type",
std::any::type_name::<T>(),
)
})
.take()
}
/// A native (rust) turbo-tasks function. It's used internally by /// A native (rust) turbo-tasks function. It's used internally by
/// `#[turbo_tasks::function]`. /// `#[turbo_tasks::function]`.
pub struct NativeFunction { pub struct NativeFunction {
......
...@@ -3,8 +3,9 @@ use std::{fmt, sync::Arc}; ...@@ -3,8 +3,9 @@ use std::{fmt, sync::Arc};
use anyhow::{Result, bail}; use anyhow::{Result, bail};
use crate::{ use crate::{
CellId, MagicAny, OutputContent, RawVc, TaskPersistence, TraitMethod, TurboTasksBackendApi, CellId, MagicAny, OutputContent, OwnedMagicAny, RawVc, TaskPersistence, TraitMethod,
ValueTypeId, backend::Backend, event::Event, macro_helpers::NativeFunction, registry, TurboTasksBackendApi, ValueTypeId, backend::Backend, event::Event,
macro_helpers::NativeFunction, registry,
}; };
/// A potentially in-flight local task stored in `CurrentGlobalTaskState::local_tasks`. /// A potentially in-flight local task stored in `CurrentGlobalTaskState::local_tasks`.
...@@ -60,7 +61,8 @@ impl LocalTaskType { ...@@ -60,7 +61,8 @@ impl LocalTaskType {
*this = this.resolve().await?; *this = this.resolve().await?;
} }
let arg = native_fn.arg_meta.resolve(arg).await?; let arg = native_fn.arg_meta.resolve(arg).await?;
Ok(turbo_tasks.native_call(native_fn, this, arg, persistence)) let mut arg = OwnedMagicAny::new(arg);
Ok(turbo_tasks.native_call(native_fn, this, &mut arg, persistence))
} }
/// Implementation of the LocalTaskType::ResolveTrait task. /// Implementation of the LocalTaskType::ResolveTrait task.
pub(crate) async fn run_resolve_trait<B: Backend + 'static>( pub(crate) async fn run_resolve_trait<B: Backend + 'static>(
...@@ -77,7 +79,8 @@ impl LocalTaskType { ...@@ -77,7 +79,8 @@ impl LocalTaskType {
let native_fn = Self::resolve_trait_method_from_value(trait_method, type_id)?; let native_fn = Self::resolve_trait_method_from_value(trait_method, type_id)?;
let arg = native_fn.arg_meta.filter_and_resolve(arg).await?; let arg = native_fn.arg_meta.filter_and_resolve(arg).await?;
Ok(turbo_tasks.native_call(native_fn, Some(this), arg, persistence)) let mut arg = OwnedMagicAny::new(arg);
Ok(turbo_tasks.native_call(native_fn, Some(this), &mut arg, persistence))
} }
fn resolve_trait_method_from_value( fn resolve_trait_method_from_value(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment