Unverified Commit 2d4a2ef6 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: prefetch restoring of task dependencies (#82960)

### What?

When scheduling a task, we also prefetch all dependencies of the tasks so that they are restored from the persistent cache in parallel.
parent 004fea03
......@@ -3,6 +3,7 @@ mod operation;
mod storage;
use std::{
any::Any,
borrow::Cow,
fmt::{self, Write},
future::Future,
......@@ -23,7 +24,7 @@ use parking_lot::{Condvar, Mutex};
use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
use smallvec::{SmallVec, smallvec};
use tokio::time::{Duration, Instant};
use tracing::field::Empty;
use tracing::{field::Empty, info_span};
use turbo_tasks::{
CellId, FxDashMap, KeyValuePair, RawVc, ReadCellOptions, ReadConsistency, SessionId,
TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, TraitTypeId, TurboTasksBackendApi,
......@@ -38,7 +39,7 @@ use turbo_tasks::{
task_statistics::TaskStatisticsApi,
trace::TraceRawVcs,
turbo_tasks,
util::IdFactoryWithReuse,
util::{IdFactoryWithReuse, good_chunk_size, into_chunks},
};
pub use self::{operation::AnyOperation, storage::TaskDataCategory};
......@@ -71,6 +72,8 @@ use crate::{
const BACKEND_JOB_INITIAL_SNAPSHOT: BackendJobId = unsafe { BackendJobId::new_unchecked(1) };
const BACKEND_JOB_FOLLOW_UP_SNAPSHOT: BackendJobId = unsafe { BackendJobId::new_unchecked(2) };
const BACKEND_JOB_PREFETCH_TASK: BackendJobId = unsafe { BackendJobId::new_unchecked(3) };
const BACKEND_JOB_PREFETCH_CHUNK_TASK: BackendJobId = unsafe { BackendJobId::new_unchecked(4) };
const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1);
......@@ -744,7 +747,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
// It's not possible that the task is InProgress at this point. If it is InProgress {
// done: true } it must have Output and would early return.
task.add_new(item);
turbo_tasks.schedule(task_id);
ctx.schedule_task(task);
Ok(Err(listener))
}
......@@ -873,7 +876,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
|| self.get_task_desc_fn(task_id),
))
{
turbo_tasks.schedule(task_id);
ctx.schedule_task(task);
}
Ok(Err(listener))
......@@ -1207,7 +1210,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
if self.should_persist() {
// Schedule the snapshot job
turbo_tasks.schedule_backend_background_job(BACKEND_JOB_INITIAL_SNAPSHOT);
turbo_tasks.schedule_backend_background_job(BACKEND_JOB_INITIAL_SNAPSHOT, None);
}
}
......@@ -2133,6 +2136,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
fn run_backend_job<'a>(
self: &'a Arc<Self>,
id: BackendJobId,
data: Option<Box<dyn Any + Send>>,
turbo_tasks: &'a dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
Box::pin(async move {
......@@ -2203,10 +2207,45 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
Ordering::Relaxed,
);
turbo_tasks.schedule_backend_background_job(BACKEND_JOB_FOLLOW_UP_SNAPSHOT);
turbo_tasks
.schedule_backend_background_job(BACKEND_JOB_FOLLOW_UP_SNAPSHOT, None);
return;
}
}
} else if id == BACKEND_JOB_PREFETCH_TASK || id == BACKEND_JOB_PREFETCH_CHUNK_TASK {
const DATA_EXPECTATION: &str =
"Expected data to be a FxHashMap<TaskId, bool> for BACKEND_JOB_PREFETCH_TASK";
let data = Box::<dyn Any + Send>::downcast::<Vec<(TaskId, bool)>>(
data.expect(DATA_EXPECTATION),
)
.expect(DATA_EXPECTATION);
fn prefetch_task(ctx: &mut impl ExecuteContext<'_>, task: TaskId, with_data: bool) {
let category = if with_data {
TaskDataCategory::All
} else {
TaskDataCategory::Meta
};
// Prefetch the task
drop(ctx.task(task, category));
}
if id == BACKEND_JOB_PREFETCH_TASK && data.len() > 128 {
let chunk_size = good_chunk_size(data.len());
for chunk in into_chunks(*data, chunk_size) {
let data: Box<Vec<(TaskId, bool)>> = Box::new(chunk.collect::<Vec<_>>());
turbo_tasks.schedule_backend_foreground_job(
BACKEND_JOB_PREFETCH_CHUNK_TASK,
Some(data),
);
}
} else {
let _span = info_span!("prefetching").entered();
let mut ctx = self.execute_context(turbo_tasks);
for (task, with_data) in data.into_iter() {
prefetch_task(&mut ctx, task, with_data);
}
}
}
})
}
......@@ -2898,9 +2937,10 @@ impl<B: BackingStorage> Backend for TurboTasksBackend<B> {
fn run_backend_job<'a>(
&'a self,
id: BackendJobId,
data: Option<Box<dyn Any + Send>>,
turbo_tasks: &'a dyn TurboTasksBackendApi<Self>,
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
self.0.run_backend_job(id, turbo_tasks)
self.0.run_backend_job(id, data, turbo_tasks)
}
fn try_read_task_output(
......
......@@ -1218,24 +1218,26 @@ impl AggregationUpdateQueue {
} else {
None
};
if let Some(reason) = should_schedule {
let description = || ctx.get_task_desc_fn(task_id);
if task.add(CachedDataItem::new_scheduled(reason, description)) {
ctx.schedule(task_id);
}
}
// if it has `Activeness` we can skip visiting the nested nodes since
// this would already be scheduled by the `Activeness`
let is_active_until_clean = get!(task, Activeness).is_some_and(|a| a.active_until_clean);
if !is_active_until_clean {
let dirty_containers: Vec<_> = get_many!(task, AggregatedDirtyContainer { task } count if count.get(session_id) > 0 => task);
if !dirty_containers.is_empty() || dirty {
let mut dirty_containers = iter_many!(task, AggregatedDirtyContainer { task } count if count.get(session_id) > 0 => task).peekable();
let is_empty = dirty_containers.peek().is_none();
if !is_empty || dirty {
self.extend_find_and_schedule_dirty(dirty_containers);
let activeness_state =
get_mut_or_insert_with!(task, Activeness, || ActivenessState::new(task_id));
activeness_state.set_active_until_clean();
}
}
if let Some(reason) = should_schedule {
let description = || ctx.get_task_desc_fn(task_id);
if task.add(CachedDataItem::new_scheduled(reason, description)) {
drop(task);
self.extend_find_and_schedule_dirty(dirty_containers);
ctx.schedule(task_id);
}
}
}
......
......@@ -25,16 +25,14 @@ pub enum ConnectChildOperation {
impl ConnectChildOperation {
pub fn run(parent_task_id: TaskId, child_task_id: TaskId, mut ctx: impl ExecuteContext) {
if !ctx.should_track_children() {
let mut task = ctx.task(child_task_id, TaskDataCategory::All);
if !task.has_key(&CachedDataItemKey::Output {}) {
let should_schedule = task.add(CachedDataItem::new_scheduled(
let mut child_task = ctx.task(child_task_id, TaskDataCategory::All);
if !child_task.has_key(&CachedDataItemKey::Output {})
&& child_task.add(CachedDataItem::new_scheduled(
TaskExecutionReason::Connect,
|| ctx.get_task_desc_fn(child_task_id),
));
drop(task);
if should_schedule {
ctx.schedule(child_task_id);
}
))
{
ctx.schedule_task(child_task);
}
return;
}
......@@ -76,17 +74,15 @@ impl ConnectChildOperation {
task: child_task_id,
});
} else {
let mut task = ctx.task(child_task_id, TaskDataCategory::All);
let mut child_task = ctx.task(child_task_id, TaskDataCategory::All);
if !task.has_key(&CachedDataItemKey::Output {}) {
let should_schedule = task.add(CachedDataItem::new_scheduled(
if !child_task.has_key(&CachedDataItemKey::Output {})
&& child_task.add(CachedDataItem::new_scheduled(
TaskExecutionReason::Connect,
|| ctx.get_task_desc_fn(child_task_id),
));
drop(task);
if should_schedule {
ctx.schedule(child_task_id);
}
))
{
ctx.schedule_task(child_task);
}
}
......
......@@ -176,10 +176,10 @@ pub fn make_task_dirty(
return;
}
let mut task = ctx.task(task_id, TaskDataCategory::Meta);
let task = ctx.task(task_id, TaskDataCategory::Meta);
make_task_dirty_internal(
&mut task,
task,
task_id,
true,
#[cfg(feature = "trace_task_dirty")]
......@@ -190,12 +190,12 @@ pub fn make_task_dirty(
}
pub fn make_task_dirty_internal(
task: &mut impl TaskGuard,
mut task: impl TaskGuard,
task_id: TaskId,
make_stale: bool,
#[cfg(feature = "trace_task_dirty")] cause: TaskDirtyCause,
queue: &mut AggregationUpdateQueue,
ctx: &impl ExecuteContext,
ctx: &mut impl ExecuteContext,
) {
// There must be no way to invalidate immutable tasks. If there would be a way the task is not
// immutable.
......@@ -286,7 +286,7 @@ pub fn make_task_dirty_internal(
});
if !aggregated_update.is_zero() {
queue.extend(AggregationUpdateJob::data_update(
task,
&mut task,
AggregatedDataUpdate::new().dirty_container_update(task_id, aggregated_update),
));
}
......@@ -301,7 +301,9 @@ pub fn make_task_dirty_internal(
TaskExecutionReason::Invalidated,
description,
)) {
ctx.schedule(task_id);
drop(task);
let task = ctx.task(task_id, TaskDataCategory::All);
ctx.schedule_task(task);
}
}
}
......@@ -14,13 +14,15 @@ use std::{
sync::atomic::Ordering,
};
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use turbo_tasks::{KeyValuePair, SessionId, TaskId, TurboTasksBackendApi};
use crate::{
backend::{
OperationGuard, TaskDataCategory, TransientTask, TurboTasksBackend, TurboTasksBackendInner,
storage::{SpecificTaskDataCategory, StorageWriteGuard},
BACKEND_JOB_PREFETCH_TASK, OperationGuard, TaskDataCategory, TransientTask,
TurboTasksBackend, TurboTasksBackendInner,
storage::{SpecificTaskDataCategory, StorageWriteGuard, iter_many},
},
backing_storage::BackingStorage,
data::{
......@@ -56,7 +58,8 @@ pub trait ExecuteContext<'e>: Sized {
task_id2: TaskId,
category: TaskDataCategory,
) -> (impl TaskGuard + 'e, impl TaskGuard + 'e);
fn schedule(&self, task_id: TaskId);
fn schedule(&mut self, task_id: TaskId);
fn schedule_task(&self, task: impl TaskGuard + '_);
fn operation_suspend_point<T>(&mut self, op: &T)
where
T: Clone + Into<AnyOperation>;
......@@ -255,8 +258,19 @@ where
)
}
fn schedule(&self, task_id: TaskId) {
self.turbo_tasks.schedule(task_id);
fn schedule(&mut self, task_id: TaskId) {
let task = self.task(task_id, TaskDataCategory::All);
self.schedule_task(task);
}
fn schedule_task(&self, mut task: impl TaskGuard + '_) {
if let Some(tasks_to_prefetch) = task.prefetch() {
self.turbo_tasks.schedule_backend_foreground_job(
BACKEND_JOB_PREFETCH_TASK,
Some(Box::new(tasks_to_prefetch)),
);
}
self.turbo_tasks.schedule(task.id());
}
fn operation_suspend_point<T: Clone + Into<AnyOperation>>(&mut self, op: &T) {
......@@ -322,6 +336,7 @@ pub trait TaskGuard: Debug {
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l;
fn invalidate_serialization(&mut self);
fn prefetch(&mut self) -> Option<Vec<(TaskId, bool)>>;
fn is_immutable(&self) -> bool;
}
......@@ -511,6 +526,20 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
}
}
fn prefetch(&mut self) -> Option<Vec<(TaskId, bool)>> {
if !self.task.state().prefetched() {
self.task.state_mut().set_prefetched(true);
let map = iter_many!(self, OutputDependency { target } => (target, false))
.chain(iter_many!(self, CellDependency { target } => (target.task, true)))
.chain(iter_many!(self, CollectiblesDependency { target } => (target.task, true)))
.collect::<FxHashMap<_, _>>();
if map.len() > 16 {
return Some(map.into_iter().collect());
}
}
None
}
fn is_immutable(&self) -> bool {
self.task.contains_key(&CachedDataItemKey::Immutable {})
}
......
......@@ -116,16 +116,15 @@ impl UpdateOutputOperation {
}
make_task_dirty_internal(
&mut task,
task,
task_id,
false,
#[cfg(feature = "trace_task_dirty")]
TaskDirtyCause::InitialDirty,
&mut queue,
&ctx,
&mut ctx,
);
drop(task);
drop(old_content);
}
......@@ -173,10 +172,10 @@ impl Operation for UpdateOutputOperation {
ref mut queue,
} => {
if let Some(child_id) = children.pop() {
let mut child_task = ctx.task(child_id, TaskDataCategory::Meta);
let child_task = ctx.task(child_id, TaskDataCategory::Meta);
if !child_task.has_key(&CachedDataItemKey::Output {}) {
make_task_dirty_internal(
&mut child_task,
child_task,
child_id,
false,
#[cfg(feature = "trace_task_dirty")]
......
......@@ -101,6 +101,8 @@ bitfield! {
/// Item was modified after snapshot mode was entered. A snapshot was taken.
pub meta_snapshot, set_meta_snapshot: 4;
pub data_snapshot, set_data_snapshot: 5;
/// Prefetched dependencies
pub prefetched, set_prefetched: 6;
}
impl InnerStorageState {
......
use std::{
any::Any,
borrow::Cow,
error::Error,
fmt::{self, Debug, Display},
......@@ -585,6 +586,7 @@ pub trait Backend: Sync + Send {
fn run_backend_job<'a>(
&'a self,
id: BackendJobId,
data: Option<Box<dyn Any + Send>>,
turbo_tasks: &'a dyn TurboTasksBackendApi<Self>,
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
......
......@@ -250,8 +250,8 @@ pub trait TurboTasksBackendApi<B: Backend + 'static>: TurboTasksCallApi + Sync +
unsafe fn reuse_transient_task_id(&self, id: Unused<TaskId>);
fn schedule(&self, task: TaskId);
fn schedule_backend_background_job(&self, id: BackendJobId);
fn schedule_backend_foreground_job(&self, id: BackendJobId);
fn schedule_backend_background_job(&self, id: BackendJobId, data: Option<Box<dyn Any + Send>>);
fn schedule_backend_foreground_job(&self, id: BackendJobId, data: Option<Box<dyn Any + Send>>);
fn try_foreground_done(&self) -> Result<(), EventListener>;
fn wait_foreground_done_excluding_own<'a>(
......@@ -1478,16 +1478,16 @@ impl<B: Backend + 'static> TurboTasksBackendApi<B> for TurboTasks<B> {
}
#[track_caller]
fn schedule_backend_background_job(&self, id: BackendJobId) {
fn schedule_backend_background_job(&self, id: BackendJobId, data: Option<Box<dyn Any + Send>>) {
self.schedule_background_job(move |this| async move {
this.backend.run_backend_job(id, &*this).await;
this.backend.run_backend_job(id, data, &*this).await;
})
}
#[track_caller]
fn schedule_backend_foreground_job(&self, id: BackendJobId) {
fn schedule_backend_foreground_job(&self, id: BackendJobId, data: Option<Box<dyn Any + Send>>) {
self.schedule_foreground_job(move |this| async move {
this.backend.run_backend_job(id, &*this).await;
this.backend.run_backend_job(id, data, &*this).await;
})
}
......
......@@ -4,18 +4,10 @@
//! tokio. It also avoid having multiple thread pools.
//! see also https://pwy.io/posts/mimalloc-cigarette/
use std::{sync::LazyLock, thread::available_parallelism};
use crate::{scope::scope_and_block, util::into_chunks};
/// Calculates a good chunk size for parallel processing based on the number of available threads.
/// This is used to ensure that the workload is evenly distributed across the threads.
fn good_chunk_size(len: usize) -> usize {
static GOOD_CHUNK_COUNT: LazyLock<usize> =
LazyLock::new(|| available_parallelism().map_or(16, |c| c.get() * 4));
let min_chunk_count = *GOOD_CHUNK_COUNT;
len.div_ceil(min_chunk_count)
}
use crate::{
scope::scope_and_block,
util::{good_chunk_size, into_chunks},
};
pub fn for_each<'l, T, F>(items: &'l [T], f: F)
where
......
......@@ -7,8 +7,9 @@ use std::{
mem::ManuallyDrop,
ops::Deref,
pin::Pin,
sync::Arc,
sync::{Arc, LazyLock},
task::{Context, Poll},
thread::available_parallelism,
time::Duration,
};
......@@ -262,6 +263,15 @@ impl<F: Future, W: for<'a> Fn(Pin<&mut F>, &mut Context<'a>) -> Poll<F::Output>>
}
}
/// Calculates a good chunk size for parallel processing based on the number of available threads.
/// This is used to ensure that the workload is evenly distributed across the threads.
pub fn good_chunk_size(len: usize) -> usize {
static GOOD_CHUNK_COUNT: LazyLock<usize> =
LazyLock::new(|| available_parallelism().map_or(16, |c| c.get() * 4));
let min_chunk_count = *GOOD_CHUNK_COUNT;
len.div_ceil(min_chunk_count)
}
/// Similar to slice::chunks but for owned data. Chunks are Send and Sync to allow to use it for
/// parallelism.
pub fn into_chunks<T>(data: Vec<T>, chunk_size: usize) -> IntoChunks<T> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment