Unverified Commit b3a13b48 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbo-tasks: add batched keyed reads

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 4bfbf238
...@@ -40,18 +40,20 @@ impl EndpointTraceResult { ...@@ -40,18 +40,20 @@ impl EndpointTraceResult {
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn all_files(&self) -> Result<Vc<FileSystemPathVec>> { pub async fn all_files(&self) -> Result<Vc<FileSystemPathVec>> {
let module_data = self.module_data.await?; let module_data = self.module_data.await?;
let idents = module_data.idents.keyed().await?;
Ok(Vc::cell( Ok(Vc::cell(
self.includes self.includes
.iter() .iter()
.cloned() .cloned()
.chain( .map(Ok)
self.modules .chain(self.modules.iter().map(|m| {
.iter() Ok(idents
.map(async |m| Ok(module_data.idents.get(m).await?.unwrap().path.clone())) .get(m)
.try_join() .context("missing path for module")?
.await?, .path
) .clone())
.collect(), }))
.collect::<Result<_>>()?,
)) ))
} }
} }
...@@ -95,7 +97,7 @@ pub async fn trace_endpoint( ...@@ -95,7 +97,7 @@ pub async fn trace_endpoint(
let module_data = traced_module_data_for_graph(*module_graph, traced_entries, hash_salt) let module_data = traced_module_data_for_graph(*module_graph, traced_entries, hash_salt)
.to_resolved() .to_resolved()
.await?; .await?;
let module_paths = module_data.await?.idents; let module_paths = module_data.await?.idents.keyed().await?;
let modules = all_modules let modules = all_modules
.iter() .iter()
...@@ -103,7 +105,6 @@ pub async fn trace_endpoint( ...@@ -103,7 +105,6 @@ pub async fn trace_endpoint(
.map(async |module| { .map(async |module| {
let entry = module_paths let entry = module_paths
.get(&module) .get(&module)
.await?
.context("missing path for module")?; .context("missing path for module")?;
let referenced_chunk_path = &entry.path; let referenced_chunk_path = &entry.path;
...@@ -316,7 +317,9 @@ pub async fn traced_modules_for_entries( ...@@ -316,7 +317,9 @@ pub async fn traced_modules_for_entries(
) -> Result<Vc<Modules>> { ) -> Result<Vc<Modules>> {
let exclude_glob_and_module_idents = if let Some(exclude_glob) = exclude_glob { let exclude_glob_and_module_idents = if let Some(exclude_glob) = exclude_glob {
let exclude_glob = exclude_glob.await?; let exclude_glob = exclude_glob.await?;
let idents = traced_module_idents_for_graph(module_graph, traced_entries).await?; let idents = traced_module_idents_for_graph(module_graph, traced_entries)
.keyed()
.await?;
Some((exclude_glob, idents)) Some((exclude_glob, idents))
} else { } else {
None None
......
...@@ -128,6 +128,8 @@ impl Asset for NftJsonAsset { ...@@ -128,6 +128,8 @@ impl Asset for NftJsonAsset {
let traced_files = this.traced_files.await?; let traced_files = this.traced_files.await?;
let module_data = traced_files.module_data.await?; let module_data = traced_files.module_data.await?;
let module_idents = module_data.idents.keyed().await?;
let module_hashes = module_data.hashes.keyed().await?;
let result = all_assets let result = all_assets
.iter() .iter()
...@@ -155,16 +157,8 @@ impl Asset for NftJsonAsset { ...@@ -155,16 +157,8 @@ impl Asset for NftJsonAsset {
) )
} }
AssetOrModule::Module(v) => { AssetOrModule::Module(v) => {
let ident = module_data let ident = module_idents.get(&v).context("missing path for module")?;
.idents let hash = module_hashes.get(&v).context("missing hash for module")?;
.get(&v)
.await?
.context("missing path for module")?;
let hash = module_data
.hashes
.get(&v)
.await?
.context("missing hash for module")?;
let source = v.source().await?.context("NFT module has no content")?; let source = v.source().await?.context("NFT module has no content")?;
( (
ident.path.clone(), ident.path.clone(),
......
...@@ -16,4 +16,5 @@ These rules apply throughout this directory tree when developing with turbo_task ...@@ -16,4 +16,5 @@ These rules apply throughout this directory tree when developing with turbo_task
- Existing `.cell()` calls or wrapping tasks are not automatically good precedent. Verify their caller and cache-key assumptions before copying the pattern. - Existing `.cell()` calls or wrapping tasks are not automatically good precedent. Verify their caller and cache-key assumptions before copying the pattern.
- Put maps containing information for a large part of the build (for example, a map from modules to analysis results) into separate cells. Use `cell = "keyed"` on their value types, for example `#[turbo_tasks::value(transparent, cell = "keyed")]` for a map wrapper. - Put maps containing information for a large part of the build (for example, a map from modules to analysis results) into separate cells. Use `cell = "keyed"` on their value types, for example `#[turbo_tasks::value(transparent, cell = "keyed")]` for a map wrapper.
- Read keyed cells through keyed access, such as `map.get(&key).await?`, so invalidation depends only on the entries actually read. Awaiting the entire map and then looking up a key creates a dependency on the whole map instead. - Read keyed cells through keyed access, such as `map.get(&key).await?`, so invalidation depends only on the entries actually read. Awaiting the entire map and then looking up a key creates a dependency on the whole map instead.
- When a task reads many keys of the same keyed cell, use `let map = map.keyed().await?;` and then the synchronous `map.get(&key)` / `map.contains_key(&key)`. It reads the cell once and, when the `KeyedReadRef` is dropped within the task, narrows the dependency to the accessed keys in one batch.
- When changing task boundaries, check the intended invalidation behavior. An unrelated module change should not invalidate consumers of unchanged keyed entries; correct output alone does not demonstrate good cache granularity. - When changing task boundaries, check the intended invalidation behavior. An unrelated module change should not invalidate consumers of unchanged keyed entries; correct output alone does not demonstrate good cache granularity.
...@@ -36,8 +36,8 @@ use turbo_bincode::TurboBincodeBuffer; ...@@ -36,8 +36,8 @@ use turbo_bincode::TurboBincodeBuffer;
use turbo_tasks::{ use turbo_tasks::{
CellId, DynTaskInputsStorage, RawVc, RawVcUnpacked, ReadCellOptions, ReadCellTracking, CellId, DynTaskInputsStorage, RawVc, RawVcUnpacked, ReadCellOptions, ReadCellTracking,
ReadConsistency, ReadOutcome, ReadOutputOptions, ReadTracking, SharedReference, ReadConsistency, ReadOutcome, ReadOutputOptions, ReadTracking, SharedReference,
TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TaskPersistence, TaskPriority, TraitTypeId, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TaskPersistence, TaskPriority, TrackKey,
TurboTasks, TurboTasksCallApi, TurboTasksPanic, ValueTypeId, TraitTypeId, TurboTasks, TurboTasksCallApi, TurboTasksPanic, ValueTypeId,
backend::{ backend::{
Backend, CachedTaskType, CachedTaskTypeArc, CellContent, CellHash, TaskCollectiblesMap, Backend, CachedTaskType, CachedTaskTypeArc, CellContent, CellHash, TaskCollectiblesMap,
TaskExecutionSpec, TransientTaskType, TurboTaskContextError, TurboTaskLocalContextError, TaskExecutionSpec, TransientTaskType, TurboTaskContextError, TurboTaskLocalContextError,
...@@ -1052,15 +1052,21 @@ impl TurboTasksBackend { ...@@ -1052,15 +1052,21 @@ impl TurboTasksBackend {
reader: Option<TaskId>, reader: Option<TaskId>,
reader_task: Option<TaskGuard<'_>>, reader_task: Option<TaskGuard<'_>>,
cell: CellId, cell: CellId,
key: Option<u64>, key: TrackKey,
) { ) {
if let Some(mut reader_task) = reader_task { if let Some(mut reader_task) = reader_task {
let reader = reader.unwrap(); let reader = reader.unwrap();
let reverse = CellRef { task: reader, cell }; let reverse = CellRef { task: reader, cell };
if let Some(k) = key { match key {
let _ = task.add_cell_dependents_hashed((reverse, k)); TrackKey::None => {
} else { let _ = task.add_cell_dependents(reverse);
let _ = task.add_cell_dependents(reverse); }
TrackKey::Key(k) => {
let _ = task.add_cell_dependents_hashed((reverse, k));
}
TrackKey::Keyed(id) => {
let _ = task.add_cell_dependents_keyed((reverse, id));
}
} }
drop(task); drop(task);
...@@ -1073,12 +1079,22 @@ impl TurboTasksBackend { ...@@ -1073,12 +1079,22 @@ impl TurboTasksBackend {
task: task_id, task: task_id,
cell, cell,
}; };
if let Some(k) = key { match key {
if !reader_task.remove_outdated_cell_dependencies_hashed(&(target, k)) { TrackKey::None => {
let _ = reader_task.add_cell_dependencies_hashed((target, k)); if !reader_task.remove_outdated_cell_dependencies(&target) {
let _ = reader_task.add_cell_dependencies(target);
}
}
TrackKey::Key(k) => {
if !reader_task.remove_outdated_cell_dependencies_hashed(&(target, k)) {
let _ = reader_task.add_cell_dependencies_hashed((target, k));
}
}
TrackKey::Keyed(id) => {
if !reader_task.remove_outdated_cell_dependencies_keyed(&(target, id)) {
let _ = reader_task.add_cell_dependencies_keyed((target, id));
}
} }
} else if !reader_task.remove_outdated_cell_dependencies(&target) {
let _ = reader_task.add_cell_dependencies(target);
} }
drop(reader_task); drop(reader_task);
} }
...@@ -1189,6 +1205,52 @@ impl TurboTasksBackend { ...@@ -1189,6 +1205,52 @@ impl TurboTasksBackend {
Ok(ReadOutcome::Scheduled(listener)) Ok(ReadOutcome::Scheduled(listener))
} }
/// Replaces the [`TrackKey::Keyed`] dependency of `reader` on `cell` of `task_id` (created by a
/// `Vc::keyed()` read) with [`TrackKey::Key`] dependencies on `key_hashes`.
///
/// When the cell content is currently missing, the [`TrackKey::Keyed`] dependency is kept
/// (conservatively depending on the whole cell). That covers a concurrent `update_cell`
/// invalidation: it removes the cell content before it invalidates the dependents it collected
/// earlier, and it would skip a [`TrackKey::Keyed`] dependency that was removed in between,
/// without seeing the hashed dependencies that replaced it.
fn convert_keyed_cell_dependency(
&self,
task_id: TaskId,
cell: CellId,
reader: TaskId,
id: u32,
key_hashes: &mut dyn Iterator<Item = u64>,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
) {
if !self.should_track_dependencies() || reader == task_id {
return;
}
let mut ctx = self.execute_context(turbo_tasks);
let (mut task, mut reader_task) = ctx.task_pair(task_id, reader, TaskDataCategory::All);
let immutable = task.immutable() && !cfg!(feature = "verify_immutable");
if !immutable && !task.cell_data_contains(&cell) {
return;
}
let reverse = CellRef { task: reader, cell };
let target = CellRef {
task: task_id,
cell,
};
task.remove_cell_dependents_keyed(&(reverse, id));
reader_task.remove_cell_dependencies_keyed(&(target, id));
reader_task.remove_outdated_cell_dependencies_keyed(&(target, id));
// Immutable tasks never need dependency edges and can never be invalidated.
if immutable {
return;
}
for k in key_hashes {
let _ = task.add_cell_dependents_hashed((reverse, k));
if !reader_task.remove_outdated_cell_dependencies_hashed(&(target, k)) {
let _ = reader_task.add_cell_dependencies_hashed((target, k));
}
}
}
fn listen_to_cell( fn listen_to_cell(
&self, &self,
task: &mut TaskGuard<'_>, task: &mut TaskGuard<'_>,
...@@ -2148,6 +2210,8 @@ impl TurboTasksBackend { ...@@ -2148,6 +2210,8 @@ impl TurboTasksBackend {
task.set_outdated_cell_dependencies(cell_dependencies); task.set_outdated_cell_dependencies(cell_dependencies);
let cell_dependencies_hashed = task.iter_cell_dependencies_hashed().collect(); let cell_dependencies_hashed = task.iter_cell_dependencies_hashed().collect();
task.set_outdated_cell_dependencies_hashed(cell_dependencies_hashed); task.set_outdated_cell_dependencies_hashed(cell_dependencies_hashed);
let cell_dependencies_keyed = task.iter_cell_dependencies_keyed().collect();
task.set_outdated_cell_dependencies_keyed(cell_dependencies_keyed);
let outdated_output_dependencies = task.iter_output_dependencies().collect(); let outdated_output_dependencies = task.iter_output_dependencies().collect();
task.set_outdated_output_dependencies(outdated_output_dependencies); task.set_outdated_output_dependencies(outdated_output_dependencies);
...@@ -2484,6 +2548,7 @@ impl TurboTasksBackend { ...@@ -2484,6 +2548,7 @@ impl TurboTasksBackend {
task.iter_output_dependencies() task.iter_output_dependencies()
.chain(task.iter_cell_dependencies().map(|r| r.task)) .chain(task.iter_cell_dependencies().map(|r| r.task))
.chain(task.iter_cell_dependencies_hashed().map(|(r, _)| r.task)) .chain(task.iter_cell_dependencies_hashed().map(|(r, _)| r.task))
.chain(task.iter_cell_dependencies_keyed().map(|(r, _)| r.task))
.collect::<FxHashSet<_>>(), .collect::<FxHashSet<_>>(),
) )
} else { } else {
...@@ -2534,6 +2599,14 @@ impl TurboTasksBackend { ...@@ -2534,6 +2599,14 @@ impl TurboTasksBackend {
key, key,
}), }),
); );
old_edges.extend(
task.iter_outdated_cell_dependencies_keyed()
.map(|(cell, id)| OutdatedEdge::KeyedCellDependency {
dependent: task_id,
cell,
id,
}),
);
old_edges.extend(task.iter_outdated_output_dependencies().map(|output_task| { old_edges.extend(task.iter_outdated_output_dependencies().map(|output_task| {
OutdatedEdge::OutputDependency { OutdatedEdge::OutputDependency {
dependent: task_id, dependent: task_id,
...@@ -3948,6 +4021,18 @@ impl Backend for TurboTasksBackend { ...@@ -3948,6 +4021,18 @@ impl Backend for TurboTasksBackend {
self.try_read_task_cell(task_id, reader, cell, options, turbo_tasks) self.try_read_task_cell(task_id, reader, cell, options, turbo_tasks)
} }
fn convert_keyed_cell_dependency(
&self,
task_id: TaskId,
cell: CellId,
reader: TaskId,
id: u32,
key_hashes: &mut dyn Iterator<Item = u64>,
turbo_tasks: &TurboTasks<Self>,
) {
self.convert_keyed_cell_dependency(task_id, cell, reader, id, key_hashes, turbo_tasks)
}
fn try_read_own_task_cell( fn try_read_own_task_cell(
&self, &self,
task_id: TaskId, task_id: TaskId,
......
...@@ -34,6 +34,12 @@ pub enum OutdatedEdge { ...@@ -34,6 +34,12 @@ pub enum OutdatedEdge {
cell: CellRef, cell: CellRef,
key: u64, key: u64,
}, },
/// A `Vc::keyed()` read dependency (`TrackKey::Keyed(id)`).
KeyedCellDependency {
dependent: TaskId,
cell: CellRef,
id: u32,
},
/// `dependent` reads the output of `output_task`. /// `dependent` reads the output of `output_task`.
OutputDependency { OutputDependency {
dependent: TaskId, dependent: TaskId,
...@@ -67,6 +73,13 @@ pub fn capture_all_edges(task: &TaskGuard<'_>) -> Vec<OutdatedEdge> { ...@@ -67,6 +73,13 @@ pub fn capture_all_edges(task: &TaskGuard<'_>) -> Vec<OutdatedEdge> {
key, key,
} }
})); }));
old_edges.extend(task.iter_cell_dependencies_keyed().map(|(cell, id)| {
OutdatedEdge::KeyedCellDependency {
dependent: task_id,
cell,
id,
}
}));
old_edges.extend( old_edges.extend(
task.iter_collectibles_dependencies() task.iter_collectibles_dependencies()
.map(OutdatedEdge::CollectiblesDependency), .map(OutdatedEdge::CollectiblesDependency),
...@@ -93,6 +106,16 @@ pub fn capture_all_edges(task: &TaskGuard<'_>) -> Vec<OutdatedEdge> { ...@@ -93,6 +106,16 @@ pub fn capture_all_edges(task: &TaskGuard<'_>) -> Vec<OutdatedEdge> {
key, key,
} }
})); }));
old_edges.extend(task.iter_cell_dependents_keyed().map(|(entry, id)| {
OutdatedEdge::KeyedCellDependency {
dependent: entry.task,
cell: CellRef {
task: task_id,
cell: entry.cell,
},
id,
}
}));
old_edges.extend(task.iter_output_dependent().map(|dependent| { old_edges.extend(task.iter_output_dependent().map(|dependent| {
OutdatedEdge::OutputDependency { OutdatedEdge::OutputDependency {
dependent, dependent,
...@@ -296,6 +319,28 @@ fn cleanup_old_edges_inner( ...@@ -296,6 +319,28 @@ fn cleanup_old_edges_inner(
task.remove_outdated_cell_dependencies_hashed(&(cell, key)); task.remove_outdated_cell_dependencies_hashed(&(cell, key));
} }
} }
OutdatedEdge::KeyedCellDependency {
dependent,
cell,
id,
} => {
// Same as above but in the `_keyed` sets.
{
let mut task = ctx.task(cell.task, TaskDataCategory::Data);
task.remove_cell_dependents_keyed(&(
CellRef {
task: dependent,
cell: cell.cell,
},
id,
));
}
{
let mut task = ctx.task(dependent, TaskDataCategory::Data);
task.remove_cell_dependencies_keyed(&(cell, id));
task.remove_outdated_cell_dependencies_keyed(&(cell, id));
}
}
OutdatedEdge::OutputDependency { OutdatedEdge::OutputDependency {
dependent, dependent,
output_task, output_task,
......
...@@ -1779,6 +1779,10 @@ impl<'a> TaskGuard<'a> { ...@@ -1779,6 +1779,10 @@ impl<'a> TaskGuard<'a> {
self.iter_cell_dependencies_hashed() self.iter_cell_dependencies_hashed()
.map(|(r, _)| (r.task, TaskDataCategory::All)), .map(|(r, _)| (r.task, TaskDataCategory::All)),
) )
.chain(
self.iter_cell_dependencies_keyed()
.map(|(r, _)| (r.task, TaskDataCategory::All)),
)
.chain( .chain(
self.iter_collectibles_dependencies() self.iter_collectibles_dependencies()
.map(|target| (target.task, TaskDataCategory::All)), .map(|target| (target.task, TaskDataCategory::All)),
......
...@@ -289,6 +289,19 @@ struct TaskStorageSchema { ...@@ -289,6 +289,19 @@ struct TaskStorageSchema {
)] )]
cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>, cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Cells this task depends on via a `Vc::keyed()` read that hasn't been converted to hashed
/// dependencies (`TrackKey::Keyed`), identified by the `keyed()` call counter of the
/// execution. Invalidated like `cell_dependencies`. Rare and usually short-lived: the
/// `KeyedReadRef` converts it to `cell_dependencies_hashed` when dropped.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
shrink_on_completion,
drop_on_completion_if_immutable
)]
cell_dependencies_keyed: AutoSet<(CellRef, u32), 1>,
/// Collectibles this task depends on. /// Collectibles this task depends on.
#[field( #[field(
storage = "auto_set", storage = "auto_set",
...@@ -311,6 +324,10 @@ struct TaskStorageSchema { ...@@ -311,6 +324,10 @@ struct TaskStorageSchema {
#[field(storage = "auto_set", category = "transient", drop_on_completion)] #[field(storage = "auto_set", category = "transient", drop_on_completion)]
outdated_cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>, outdated_cell_dependencies_hashed: AutoSet<(CellRef, u64), 1>,
/// Outdated keyed-read cell dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", drop_on_completion)]
outdated_cell_dependencies_keyed: AutoSet<(CellRef, u32), 1>,
/// Outdated collectibles dependencies to be cleaned up (transient). /// Outdated collectibles dependencies to be cleaned up (transient).
#[field(storage = "auto_set", category = "transient", drop_on_completion)] #[field(storage = "auto_set", category = "transient", drop_on_completion)]
outdated_collectibles_dependencies: AutoSet<CollectiblesRef, 3>, outdated_collectibles_dependencies: AutoSet<CollectiblesRef, 3>,
...@@ -339,6 +356,16 @@ struct TaskStorageSchema { ...@@ -339,6 +356,16 @@ struct TaskStorageSchema {
)] )]
cell_dependents_hashed: AutoSet<(CellRef, u64), 1>, cell_dependents_hashed: AutoSet<(CellRef, u64), 1>,
/// Tasks that depend on this task's cells via a `Vc::keyed()` read. Reverse of
/// `cell_dependencies_keyed`. Invalidated like `cell_dependents`.
#[field(
storage = "auto_set",
category = "data",
filter_transient,
drop_on_completion_if_immutable
)]
cell_dependents_keyed: AutoSet<(CellRef, u32), 1>,
/// Tasks that depend on collectibles of a specific type from this task. /// Tasks that depend on collectibles of a specific type from this task.
/// Maps TraitTypeId -> Set<TaskId> /// Maps TraitTypeId -> Set<TaskId>
...@@ -983,6 +1010,11 @@ impl IsTransient for (CellRef, u64) { ...@@ -983,6 +1010,11 @@ impl IsTransient for (CellRef, u64) {
self.0.is_transient() self.0.is_transient()
} }
} }
impl IsTransient for (CellRef, u32) {
fn is_transient(&self) -> bool {
self.0.is_transient()
}
}
/// Defines a strategy for merging data from disk into this storage item. /// Defines a strategy for merging data from disk into this storage item.
/// ///
......
This diff is collapsed.
...@@ -667,6 +667,23 @@ pub trait Backend: Sized + Sync + Send { ...@@ -667,6 +667,23 @@ pub trait Backend: Sized + Sync + Send {
turbo_tasks: &TurboTasks<Self>, turbo_tasks: &TurboTasks<Self>,
) -> Result<ReadOutcome<TypedCellContent>>; ) -> Result<ReadOutcome<TypedCellContent>>;
/// Replaces the [`TrackKey::Keyed(id)`][crate::TrackKey::Keyed] dependency of `reader` on the
/// cell `index` of `task`, which was created by a [`Vc::keyed`][crate::Vc::keyed] read, with
/// [`TrackKey::Key`][crate::TrackKey::Key] dependencies on each of the `key_hashes`, in a
/// single operation.
///
/// Must only be called from within the same execution of `reader` that created the
/// dependency.
fn convert_keyed_cell_dependency(
&self,
task: TaskId,
index: CellId,
reader: TaskId,
id: u32,
key_hashes: &mut dyn Iterator<Item = u64>,
turbo_tasks: &TurboTasks<Self>,
);
/// INVALIDATION: Be careful with this, it will not track dependencies, so /// INVALIDATION: Be careful with this, it will not track dependencies, so
/// using it could break cache invalidation. /// using it could break cache invalidation.
fn try_read_own_task_cell( fn try_read_own_task_cell(
......
...@@ -101,11 +101,12 @@ pub use crate::{ ...@@ -101,11 +101,12 @@ pub use crate::{
join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt}, join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt},
manager::{ manager::{
CurrentCellRef, GcRoot, InputResolution, ReadCellTracking, ReadConsistency, ReadTracking, CurrentCellRef, GcRoot, InputResolution, ReadCellTracking, ReadConsistency, ReadTracking,
ScheduleKey, TaskPersistence, TaskPriority, TurboTasks, TurboTasksApi, TurboTasksCallApi, ScheduleKey, TaskPersistence, TaskPriority, TrackKey, TurboTasks, TurboTasksApi,
Unused, UpdateInfo, dynamic_call, emit, get_serialization_invalidator, mark_finished, TurboTasksCallApi, Unused, UpdateInfo, dynamic_call, emit, get_serialization_invalidator,
mark_stateful, mark_top_level_task, prevent_gc, run, run_once, run_once_with_reason, mark_finished, mark_stateful, mark_top_level_task, prevent_gc, run, run_once,
trait_call, try_turbo_tasks, turbo_tasks, turbo_tasks_scope, turbo_tasks_weak, run_once_with_reason, trait_call, try_turbo_tasks, turbo_tasks, turbo_tasks_scope,
unmark_top_level_task_may_leak_eventually_consistent_state, with_turbo_tasks, turbo_tasks_weak, unmark_top_level_task_may_leak_eventually_consistent_state,
with_turbo_tasks,
}, },
mapped_read_ref::MappedReadRef, mapped_read_ref::MappedReadRef,
output::OutputContent, output::OutputContent,
...@@ -123,12 +124,12 @@ pub use crate::{ ...@@ -123,12 +124,12 @@ pub use crate::{
value::{TransientInstance, TransientValue}, value::{TransientInstance, TransientValue},
value_type::{Evictability, TraitMethod, TraitType, ValueType, ValueTypePersistence}, value_type::{Evictability, TraitMethod, TraitType, ValueType, ValueTypePersistence},
vc::{ vc::{
CellId, Dynamic, NonLocalValue, OperationValue, OperationVc, OptionVcExt, OrdResolvedVc, CellId, Dynamic, KeyedReadRef, NonLocalValue, OperationValue, OperationVc, OptionVcExt,
RawVc, RawVcUnpacked, ReadRawVcFuture, ReadVcFuture, ResolveOperationVcFuture, OrdResolvedVc, RawVc, RawVcUnpacked, ReadKeyedRefVcFuture, ReadRawVcFuture, ReadVcFuture,
ResolveRawVcFuture, ResolveVcFuture, ResolvedVc, ToResolvedVcFuture, Upcast, UpcastStrict, ResolveOperationVcFuture, ResolveRawVcFuture, ResolveVcFuture, ResolvedVc,
ValueDefault, Vc, VcCast, VcCellCompareMode, VcCellHashedCompareMode, ToResolvedVcFuture, Upcast, UpcastStrict, ValueDefault, Vc, VcCast, VcCellCompareMode,
VcCellKeyedCompareMode, VcCellNewMode, VcDefaultRead, VcRead, VcTransparentRead, VcCellHashedCompareMode, VcCellKeyedCompareMode, VcCellNewMode, VcDefaultRead, VcRead,
VcValueTrait, VcValueTraitCast, VcValueType, VcValueTypeCast, VcTransparentRead, VcValueTrait, VcValueTraitCast, VcValueType, VcValueTypeCast,
}, },
}; };
......
...@@ -12,7 +12,7 @@ use std::{ ...@@ -12,7 +12,7 @@ use std::{
process::abort, process::abort,
sync::{ sync::{
Arc, Mutex, RwLock, Weak, Arc, Mutex, RwLock, Weak,
atomic::{AtomicBool, AtomicUsize, Ordering}, atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering},
}, },
task::{Context, Poll, Waker}, task::{Context, Poll, Waker},
time::{Duration, Instant}, time::{Duration, Instant},
...@@ -150,6 +150,18 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { ...@@ -150,6 +150,18 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send {
options: ReadCellOptions, options: ReadCellOptions,
) -> Result<ReadOutcome<TypedCellContent>>; ) -> Result<ReadOutcome<TypedCellContent>>;
/// Replaces the [`TrackKey::Keyed(id)`][TrackKey::Keyed] dependency of the current task
/// (`reader`) on the cell `index` of `task` with [`TrackKey::Key`] dependencies on each of the
/// `key_hashes`. See [`Backend::convert_keyed_cell_dependency`].
fn convert_keyed_cell_dependency(
&self,
task: TaskId,
index: CellId,
reader: TaskId,
id: u32,
key_hashes: &mut dyn Iterator<Item = u64>,
);
/// Reads a [`RawVcUnpacked::LocalOutput`][crate::RawVcUnpacked::LocalOutput]. If the task has /// Reads a [`RawVcUnpacked::LocalOutput`][crate::RawVcUnpacked::LocalOutput]. If the task has
/// completed, returns the [`RawVc`] the local task points to. /// completed, returns the [`RawVc`] the local task points to.
/// ///
...@@ -339,12 +351,30 @@ pub enum ReadConsistency { ...@@ -339,12 +351,30 @@ pub enum ReadConsistency {
Strong, Strong,
} }
/// Identifies which part of a cell a tracked cell read depends on.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Default)]
pub enum TrackKey {
/// The read depends on the whole cell content.
#[default]
None,
/// The read only depends on the sub-value identified by this key hash (see
/// [`KeyedEq`]). The dependency is only invalidated when the value of this key changes.
Key(u64),
/// The read is a [`Vc::keyed`](crate::Vc::keyed) read. For invalidation it behaves like
/// [`TrackKey::None`] (invalidated by any change of the cell), but it is a separate dependency
/// that is uniquely identified by this counter of `keyed()` calls within the current task
/// execution. This allows the [`KeyedReadRef`](crate::KeyedReadRef) to replace it with
/// [`TrackKey::Key`] dependencies of the accessed keys once it's dropped, without affecting
/// other (full) dependencies on the same cell.
Keyed(u32),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ReadCellTracking { pub enum ReadCellTracking {
/// Reads are tracked as dependencies of the current task. /// Reads are tracked as dependencies of the current task.
Tracked { Tracked {
/// The key used for the dependency /// The key used for the dependency
key: Option<u64>, key: TrackKey,
}, },
/// The read is only tracked when there is an error, otherwise it is untracked. /// The read is only tracked when there is an error, otherwise it is untracked.
/// ///
...@@ -367,26 +397,35 @@ impl ReadCellTracking { ...@@ -367,26 +397,35 @@ impl ReadCellTracking {
} }
} }
pub fn key(&self) -> Option<u64> { pub fn key(&self) -> TrackKey {
match self { match self {
ReadCellTracking::Tracked { key } => *key, ReadCellTracking::Tracked { key } => *key,
ReadCellTracking::TrackOnlyError => None, ReadCellTracking::TrackOnlyError => TrackKey::None,
ReadCellTracking::Untracked => None, ReadCellTracking::Untracked => TrackKey::None,
} }
} }
} }
impl Default for ReadCellTracking { impl Default for ReadCellTracking {
fn default() -> Self { fn default() -> Self {
ReadCellTracking::Tracked { key: None } ReadCellTracking::Tracked {
key: TrackKey::None,
}
} }
} }
impl Display for ReadCellTracking { impl Display for ReadCellTracking {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
ReadCellTracking::Tracked { key: None } => write!(f, "tracked"), ReadCellTracking::Tracked {
ReadCellTracking::Tracked { key: Some(key) } => write!(f, "tracked with key {key}"), key: TrackKey::None,
} => write!(f, "tracked"),
ReadCellTracking::Tracked {
key: TrackKey::Key(key),
} => write!(f, "tracked with key {key}"),
ReadCellTracking::Tracked {
key: TrackKey::Keyed(id),
} => write!(f, "tracked as keyed read {id}"),
ReadCellTracking::TrackOnlyError => write!(f, "track only error"), ReadCellTracking::TrackOnlyError => write!(f, "track only error"),
ReadCellTracking::Untracked => write!(f, "untracked"), ReadCellTracking::Untracked => write!(f, "untracked"),
} }
...@@ -824,6 +863,9 @@ struct CurrentTaskStateHandle { ...@@ -824,6 +863,9 @@ struct CurrentTaskStateHandle {
struct CurrentTaskStateInner { struct CurrentTaskStateInner {
current_task_id: Option<TaskId>, current_task_id: Option<TaskId>,
/// Counter of [`Vc::keyed`](crate::Vc::keyed) reads in this execution, used to give every
/// [`TrackKey::Keyed`] dependency a unique id.
keyed_read_counter: AtomicU32,
state: RwLock<CurrentTaskState>, state: RwLock<CurrentTaskState>,
} }
...@@ -832,6 +874,7 @@ impl CurrentTaskStateHandle { ...@@ -832,6 +874,7 @@ impl CurrentTaskStateHandle {
Self { Self {
inner: Arc::new(CurrentTaskStateInner { inner: Arc::new(CurrentTaskStateInner {
current_task_id: state.task_id, current_task_id: state.task_id,
keyed_read_counter: AtomicU32::new(0),
state: RwLock::new(state), state: RwLock::new(state),
}), }),
} }
...@@ -842,6 +885,52 @@ impl CurrentTaskStateHandle { ...@@ -842,6 +885,52 @@ impl CurrentTaskStateHandle {
} }
} }
/// Identifies the [`TrackKey::Keyed`] dependency of a [`Vc::keyed`](crate::Vc::keyed) read and the
/// task execution that created it.
pub(crate) struct KeyedReadDependency {
/// The execution state of the reading task. Only used for identity (a `Weak` keeps the
/// allocation alive, so the pointer can't be reused by another execution).
execution: Weak<CurrentTaskStateInner>,
pub(crate) reader: TaskId,
pub(crate) id: u32,
}
impl KeyedReadDependency {
/// Allocates a new [`TrackKey::Keyed`] dependency id for the current task execution. Returns
/// `None` when not called from within a task execution (e.g. a top-level `run`), where reads
/// are not tracked anyway, or when the ids of this execution are exhausted. In that case the
/// read is tracked as a normal (full) dependency.
pub(crate) fn new_for_current_task() -> Option<Self> {
CURRENT_TASK_STATE
.try_with(|ts| {
let reader = ts.current_task_id()?;
let id = ts
.inner
.keyed_read_counter
.try_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1))
.ok()?;
Some(Self {
execution: Arc::downgrade(&ts.inner),
reader,
id,
})
})
.ok()
.flatten()
}
/// Whether this is called within the task execution that created the dependency and that
/// execution is still running (it hasn't reported completion to the backend yet).
pub(crate) fn is_current_execution(&self) -> bool {
CURRENT_TASK_STATE
.try_with(|ts| {
std::ptr::eq(self.execution.as_ptr(), Arc::as_ptr(&ts.inner))
&& ts.read().unwrap().cell_counters.is_some()
})
.unwrap_or(false)
}
}
impl Deref for CurrentTaskStateHandle { impl Deref for CurrentTaskStateHandle {
type Target = RwLock<CurrentTaskState>; type Target = RwLock<CurrentTaskState>;
...@@ -1942,6 +2031,18 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> { ...@@ -1942,6 +2031,18 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> {
.try_read_task_cell(task, index, reader, options, self) .try_read_task_cell(task, index, reader, options, self)
} }
fn convert_keyed_cell_dependency(
&self,
task: TaskId,
index: CellId,
reader: TaskId,
id: u32,
key_hashes: &mut dyn Iterator<Item = u64>,
) {
self.backend
.convert_keyed_cell_dependency(task, index, reader, id, key_hashes, self)
}
fn try_read_own_task_cell( fn try_read_own_task_cell(
&self, &self,
current_task: TaskId, current_task: TaskId,
......
use std::{
fmt::{self, Debug},
hash::{BuildHasher, Hash},
};
use parking_lot::Mutex;
use rustc_hash::{FxBuildHasher, FxHashSet};
use crate::{
CellId, ReadRef, TaskId,
keyed::{KeyedAccess, KeyedEq},
manager::{KeyedReadDependency, with_turbo_tasks},
vc::{VcReadTarget, VcValueType},
};
/// The result of [`Vc::keyed`](crate::Vc::keyed): a read of a keyed cell (a
/// [`cell = "keyed"`](crate::value) map or set) that only allows synchronous keyed access via
/// [`KeyedReadRef::get`] and [`KeyedReadRef::contains_key`].
///
/// This is an efficient alternative to many [`Vc::get`](crate::Vc::get) /
/// [`Vc::contains_key`](crate::Vc::contains_key) reads of the same cell within a single task: the
/// cell is read (and the dependency registered) only once, and the accessed keys don't need an
/// `await` or any locking of the backend.
///
/// While it is alive, the reading task depends on the whole cell, so it is invalidated by any
/// change. The hashes of the accessed keys are recorded. When it is dropped within the task
/// execution that read it, that dependency is replaced (in a single batch operation) with
/// dependencies on only the accessed keys, so the task is only invalidated when one of the
/// accessed keys changes, like with [`Vc::get`](crate::Vc::get). When it is dropped outside of
/// that task execution (e.g. moved to another thread or task), the dependency on the whole cell
/// is kept.
///
/// Note: Like with [`Vc::contains_key`](crate::Vc::contains_key), a
/// [`contains_key`](KeyedReadRef::contains_key) access is also invalidated when the value of the
/// key changes, not only when its presence changes.
#[must_use]
pub struct KeyedReadRef<T>
where
T: VcValueType,
VcReadTarget<T>: KeyedEq,
{
read_ref: ReadRef<T>,
/// `None` if the read wasn't tracked (e.g. a read outside of a task execution).
dependency: Option<(TaskId, CellId, KeyedReadDependency)>,
accessed_key_hashes: Mutex<FxHashSet<u64>>,
}
impl<T> KeyedReadRef<T>
where
T: VcValueType,
VcReadTarget<T>: KeyedEq,
{
pub(crate) fn new(
read_ref: ReadRef<T>,
dependency: Option<(TaskId, CellId, KeyedReadDependency)>,
) -> Self {
Self {
read_ref,
dependency,
accessed_key_hashes: Mutex::new(FxHashSet::default()),
}
}
fn track<Q: Hash + ?Sized>(&self, key: &Q) {
if self.dependency.is_some() {
// Same hash as `Vc::get` uses.
self.accessed_key_hashes
.lock()
.insert(FxBuildHasher.hash_one(key));
}
}
/// Selects a keyed value. The task will only depend on the accessed keys.
pub fn get<Q>(&self, key: &Q) -> Option<&<VcReadTarget<T> as KeyedAccess<Q>>::Value>
where
Q: Hash + ?Sized,
VcReadTarget<T>: KeyedAccess<Q>,
{
self.track(key);
(*self.read_ref).get(key)
}
/// Checks if the value contains the given key. The task will only depend on the accessed keys.
///
/// Note: This is also invalidated when the value of the key changes, not only when the presence
/// of the key changes.
pub fn contains_key<Q>(&self, key: &Q) -> bool
where
Q: Hash + ?Sized,
VcReadTarget<T>: KeyedAccess<Q>,
{
self.track(key);
(*self.read_ref).contains_key(key)
}
}
impl<T> Drop for KeyedReadRef<T>
where
T: VcValueType,
VcReadTarget<T>: KeyedEq,
{
fn drop(&mut self) {
let Some((task, index, dependency)) = self.dependency.take() else {
return;
};
if !dependency.is_current_execution() {
// Keep the conservative dependency on the whole cell.
return;
}
let key_hashes = std::mem::take(self.accessed_key_hashes.get_mut());
with_turbo_tasks(|tt| {
tt.convert_keyed_cell_dependency(
task,
index,
dependency.reader,
dependency.id,
&mut key_hashes.into_iter(),
)
});
}
}
impl<T> Debug for KeyedReadRef<T>
where
T: VcValueType,
VcReadTarget<T>: KeyedEq,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("KeyedReadRef")
.field("accessed_keys", &self.accessed_key_hashes.lock().len())
.finish_non_exhaustive()
}
}
pub(crate) mod cast; pub(crate) mod cast;
mod cell_mode; mod cell_mode;
pub(crate) mod default; pub(crate) mod default;
mod keyed_read_ref;
mod local; mod local;
pub(crate) mod operation; pub(crate) mod operation;
mod ord; mod ord;
...@@ -34,11 +35,15 @@ pub use crate::vc::{ ...@@ -34,11 +35,15 @@ pub use crate::vc::{
VcCellNewMode, VcCellNewMode,
}, },
default::ValueDefault, default::ValueDefault,
keyed_read_ref::KeyedReadRef,
local::NonLocalValue, local::NonLocalValue,
operation::{OperationValue, OperationVc, ResolveOperationVcFuture}, operation::{OperationValue, OperationVc, ResolveOperationVcFuture},
ord::OrdResolvedVc, ord::OrdResolvedVc,
raw::{CellId, RawVc, RawVcUnpacked, ReadRawVcFuture, ResolveRawVcFuture}, raw::{CellId, RawVc, RawVcUnpacked, ReadRawVcFuture, ResolveRawVcFuture},
read::{ReadOwnedVcFuture, ReadVcFuture, VcDefaultRead, VcRead, VcTransparentRead}, read::{
ReadKeyedRefVcFuture, ReadOwnedVcFuture, ReadVcFuture, VcDefaultRead, VcRead,
VcTransparentRead,
},
resolved::ResolvedVc, resolved::ResolvedVc,
traits::{Dynamic, Upcast, UpcastStrict, VcValueTrait, VcValueType}, traits::{Dynamic, Upcast, UpcastStrict, VcValueTrait, VcValueType},
}; };
...@@ -569,6 +574,17 @@ where ...@@ -569,6 +574,17 @@ where
let future: ReadVcFuture<T> = self.node.into_read().into(); let future: ReadVcFuture<T> = self.node.into_read().into();
future.contains_key(key) future.contains_key(key)
} }
/// Read the value once for many synchronous keyed accesses via the returned
/// [`KeyedReadRef`] (`vc.keyed().await?.get(key)`). This is more efficient than many
/// [`Vc::get`] / [`Vc::contains_key`] reads of the same cell within a single task.
///
/// Once the [`KeyedReadRef`] is dropped (within the same task execution), the task only
/// depends on the accessed keys instead of the full value. See [`KeyedReadRef`] for details.
pub fn keyed(self) -> ReadKeyedRefVcFuture<T> {
let future: ReadVcFuture<T> = self.node.into_read().into();
future.keyed()
}
} }
impl<T> Unpin for Vc<T> where T: ?Sized {} impl<T> Unpin for Vc<T> where T: ?Sized {}
......
...@@ -26,7 +26,7 @@ use crate::{ ...@@ -26,7 +26,7 @@ use crate::{
id::{ExecutionId, LocalTaskId, TASK_ID_MAX}, id::{ExecutionId, LocalTaskId, TASK_ID_MAX},
manager::{ manager::{
ReadCellTracking, ReadTracking, SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK, ReadCellTracking, ReadTracking, SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK,
ScheduleKey, TurboTasksApi, execute_read_target_inline, read_local_output, ScheduleKey, TrackKey, TurboTasksApi, execute_read_target_inline, read_local_output,
with_turbo_tasks, with_turbo_tasks,
}, },
read_options::ReadOutcome, read_options::ReadOutcome,
...@@ -769,10 +769,49 @@ impl ReadRawVcFuture { ...@@ -769,10 +769,49 @@ impl ReadRawVcFuture {
/// Track the value as a dependency with an key. /// Track the value as a dependency with an key.
pub fn track_with_key(mut self, key: u64) -> Self { pub fn track_with_key(mut self, key: u64) -> Self {
self.read_cell_options.tracking = ReadCellTracking::Tracked { key: Some(key) }; self.read_cell_options.tracking = ReadCellTracking::Tracked {
key: TrackKey::Key(key),
};
self.map_resolve(|r| r.track_with_key()) self.map_resolve(|r| r.track_with_key())
} }
/// Track the value as a [`TrackKey::Keyed`] dependency of a [`Vc::keyed`](crate::Vc::keyed)
/// read. Like [`ReadRawVcFuture::track_with_key`], this also tracks the reads of task outputs
/// while resolving the [`RawVc`].
///
/// Must be called before the future is polled the first time or right at the start of the
/// first poll, while it's still resolving the [`RawVc`].
pub(crate) fn set_track_keyed(&mut self, id: u32) {
let ReadRawVcState::Resolving(resolve) = &mut self.state else {
unreachable!("set_track_keyed must be called before the cell is read");
};
resolve.read_output_options.tracking = ReadTracking::Tracked;
self.read_cell_options.tracking = ReadCellTracking::Tracked {
key: TrackKey::Keyed(id),
};
}
/// Track the remaining reads as normal full dependencies, e.g. when a [`TrackKey::Keyed`] read
/// continues in a different task execution than the one that allocated its id. Can be called
/// at any time before the future completes.
pub(crate) fn set_track_full(&mut self) {
if let ReadRawVcState::Resolving(resolve) = &mut self.state {
resolve.read_output_options.tracking = ReadTracking::Tracked;
}
self.read_cell_options.tracking = ReadCellTracking::Tracked {
key: TrackKey::None,
};
}
/// The task cell the [`RawVc`] was resolved to. Only available once the future has completed
/// successfully.
pub(crate) fn resolved_cell(&self) -> Option<(TaskId, CellId)> {
match self.state {
ReadRawVcState::Reading { task, index, .. } => Some((task, index)),
ReadRawVcState::Resolving(_) => None,
}
}
/// This will not track the value as dependency, but will still track the error as dependency, /// This will not track the value as dependency, but will still track the error as dependency,
/// if there is an error. /// if there is an error.
/// ///
......
...@@ -17,6 +17,8 @@ use crate::{ ...@@ -17,6 +17,8 @@ use crate::{
MappedReadRef, ReadRawVcFuture, ReadRef, VcCast, VcValueTrait, VcValueTraitCast, MappedReadRef, ReadRawVcFuture, ReadRef, VcCast, VcValueTrait, VcValueTraitCast,
VcValueTypeCast, VcValueTypeCast,
keyed::{KeyedAccess, KeyedEq}, keyed::{KeyedAccess, KeyedEq},
manager::KeyedReadDependency,
vc::keyed_read_ref::KeyedReadRef,
}; };
type VcReadTarget<T> = <<T as VcValueType>::Read as VcRead<T>>::Target; type VcReadTarget<T> = <<T as VcValueType>::Read as VcRead<T>>::Target;
...@@ -223,6 +225,16 @@ where ...@@ -223,6 +225,16 @@ where
self.raw = self.raw.track_with_key(FxBuildHasher.hash_one(key)); self.raw = self.raw.track_with_key(FxBuildHasher.hash_one(key));
ReadContainsKeyedVcFuture { future: self, key } ReadContainsKeyedVcFuture { future: self, key }
} }
/// Read the value once for many synchronous keyed accesses via the returned
/// [`KeyedReadRef`]. Once that is dropped, the task only depends on the accessed keys instead
/// of the full value. See [`KeyedReadRef`] for details.
pub fn keyed(self) -> ReadKeyedRefVcFuture<T> {
ReadKeyedRefVcFuture {
future: self,
dependency: KeyedDependencyState::NotStarted,
}
}
} }
impl<T> From<ReadRawVcFuture> for ReadVcFuture<T, VcValueTypeCast<T>> impl<T> From<ReadRawVcFuture> for ReadVcFuture<T, VcValueTypeCast<T>>
...@@ -332,6 +344,88 @@ where ...@@ -332,6 +344,88 @@ where
} }
} }
/// The [`KeyedReadDependency`] of a [`ReadKeyedRefVcFuture`]. It's allocated when the future is
/// first polled, as that's the task execution that reads the cell (not necessarily the one that
/// created the future).
enum KeyedDependencyState {
NotStarted,
/// `None` if the read isn't a [`TrackKey::Keyed`](crate::TrackKey::Keyed) read (no current
/// task, or the dependency was already handed to the [`KeyedReadRef`]).
Started(Option<KeyedReadDependency>),
}
pin_project! {
/// A future returned by [`Vc::keyed`](crate::Vc::keyed).
#[must_use]
pub struct ReadKeyedRefVcFuture<T>
where
T: VcValueType,
VcReadTarget<T>: KeyedEq,
{
#[pin]
future: ReadVcFuture<T, VcValueTypeCast<T>>,
dependency: KeyedDependencyState,
}
}
impl<T> Future for ReadKeyedRefVcFuture<T>
where
T: VcValueType,
VcReadTarget<T>: KeyedEq,
{
type Output = Result<KeyedReadRef<T>>;
fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
let mut this = self.project();
match this.dependency {
KeyedDependencyState::NotStarted => {
let dependency = KeyedReadDependency::new_for_current_task();
if let Some(dependency) = &dependency {
// Safety: Only a field is mutated, nothing is moved.
unsafe { this.future.as_mut().get_unchecked_mut() }
.raw
.set_track_keyed(dependency.id);
}
*this.dependency = KeyedDependencyState::Started(dependency);
}
KeyedDependencyState::Started(dependency) => {
if dependency
.as_ref()
.is_some_and(|dependency| !dependency.is_current_execution())
{
// The future moved to a different task execution while pending. The id
// belongs to the other execution and could collide with an id of this one, so
// fall back to a normal full dependency and never narrow it.
// Safety: Only a field is mutated, nothing is moved.
unsafe { this.future.as_mut().get_unchecked_mut() }
.raw
.set_track_full();
*dependency = None;
}
}
}
let result = std::task::ready!(this.future.as_mut().poll(cx));
Poll::Ready(result.map(|read_ref| {
let KeyedDependencyState::Started(dependency) = this.dependency else {
unreachable!("the dependency state is initialized above")
};
// Only narrow the dependency later when the read was completed in the execution that
// registered it. Otherwise the `KeyedReadRef` keeps the conservative dependency.
let dependency = dependency
.take()
.filter(|dependency| dependency.is_current_execution())
.map(|dependency| {
let Some((task, index)) = this.future.as_ref().get_ref().raw.resolved_cell()
else {
unreachable!("a successful read is always resolved to a cell")
};
(task, index, dependency)
});
KeyedReadRef::new(read_ref, dependency)
}))
}
}
pin_project! { pin_project! {
pub struct ReadContainsKeyedVcFuture<'l, T, Q> pub struct ReadContainsKeyedVcFuture<'l, T, Q>
where where
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment