Commit 2ba55e05 authored by Tobias Koppers's avatar Tobias Koppers Committed by Tobias Koppers

Turbopack: compute leaf distance

parent a43b362e
......@@ -21,6 +21,7 @@ verify_aggregation_graph = []
verify_immutable = []
verify_determinism = []
trace_aggregation_update = []
trace_leaf_distance_update = []
trace_find_and_schedule = []
trace_task_completion = []
trace_task_dirty = []
......
......@@ -49,8 +49,8 @@ use crate::{
operation::{
AggregationUpdateJob, AggregationUpdateQueue, ChildExecuteContext,
CleanupOldEdgesOperation, ComputeDirtyAndCleanUpdate, ConnectChildOperation,
ExecuteContext, ExecuteContextImpl, Operation, OutdatedEdge, TaskGuard,
connect_children, get_aggregation_number, get_uppers, is_root_node,
ExecuteContext, ExecuteContextImpl, LeafDistanceUpdateQueue, Operation, OutdatedEdge,
TaskGuard, connect_children, get_aggregation_number, get_uppers, is_root_node,
make_task_dirty_internal, prepare_new_children,
},
storage::{
......@@ -731,10 +731,21 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
dependent_task = ?reader
)
.entered();
let _ = task.add(CachedDataItem::OutputDependent {
task: reader.unwrap(),
let mut queue = LeafDistanceUpdateQueue::new();
let reader = reader.unwrap();
if task.add(CachedDataItem::OutputDependent {
task: reader,
value: (),
});
}) {
// Ensure that dependent leaf distance is strictly monotonic increasing
let leaf_distance = get!(task, LeafDistance).copied().unwrap_or_default();
let reader_leaf_distance =
get!(reader_task, LeafDistance).copied().unwrap_or_default();
if reader_leaf_distance.min <= leaf_distance.min {
queue.push(reader, leaf_distance.min, leaf_distance.max);
}
}
drop(task);
// Note: We use `task_pair` earlier to lock the task and its reader at the same
......@@ -751,6 +762,9 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
value: (),
});
}
drop(reader_task);
queue.execute(&mut ctx);
}
return result;
......@@ -793,6 +807,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
self.assert_not_persistent_calling_transient(reader, task_id, Some(cell));
fn add_cell_dependency(
ctx: &mut impl ExecuteContext<'_>,
task_id: TaskId,
mut task: impl TaskGuard,
reader: Option<TaskId>,
......@@ -803,12 +818,22 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
if let Some(mut reader_task) = reader_task
&& (!task.is_immutable() || cfg!(feature = "verify_immutable"))
{
let _ = task.add(CachedDataItem::CellDependent {
let mut queue = LeafDistanceUpdateQueue::new();
let reader = reader.unwrap();
if task.add(CachedDataItem::CellDependent {
cell,
key,
task: reader.unwrap(),
task: reader,
value: (),
});
}) {
// Ensure that dependent leaf distance is strictly monotonic increasing
let leaf_distance = get!(task, LeafDistance).copied().unwrap_or_default();
let reader_leaf_distance =
get!(reader_task, LeafDistance).copied().unwrap_or_default();
if reader_leaf_distance.min <= leaf_distance.min {
queue.push(reader, leaf_distance.min, leaf_distance.max);
}
}
drop(task);
// Note: We use `task_pair` earlier to lock the task and its reader at the same
......@@ -830,6 +855,9 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
value: (),
});
}
drop(reader_task);
queue.execute(ctx);
}
}
......@@ -861,7 +889,15 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
};
if let Some(content) = content {
if tracking.should_track(false) {
add_cell_dependency(task_id, task, reader, reader_task, cell, tracking.key());
add_cell_dependency(
&mut ctx,
task_id,
task,
reader,
reader_task,
cell,
tracking.key(),
);
}
return Ok(Ok(TypedCellContent(
cell.type_id,
......@@ -888,7 +924,15 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
.copied();
let Some(max_id) = max_id else {
if tracking.should_track(true) {
add_cell_dependency(task_id, task, reader, reader_task, cell, tracking.key());
add_cell_dependency(
&mut ctx,
task_id,
task,
reader,
reader_task,
cell,
tracking.key(),
);
}
bail!(
"Cell {cell:?} no longer exists in task {} (no cell of this type exists)",
......@@ -897,7 +941,15 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
};
if cell.index >= max_id {
if tracking.should_track(true) {
add_cell_dependency(task_id, task, reader, reader_task, cell, tracking.key());
add_cell_dependency(
&mut ctx,
task_id,
task,
reader,
reader_task,
cell,
tracking.key(),
);
}
bail!(
"Cell {cell:?} no longer exists in task {} (index out of bounds)",
......
......@@ -883,8 +883,8 @@ pub struct AggregationUpdateQueue {
#[bincode(with = "encode_jobs")]
jobs: VecDeque<AggregationUpdateJobItem>,
#[bincode(with = "turbo_bincode::indexmap")]
number_updates: FxIndexMap<TaskId, AggregationNumberUpdate>,
done_number_updates: FxHashMap<TaskId, AggregationNumberUpdate>,
aggregation_number_updates: FxIndexMap<TaskId, AggregationNumberUpdate>,
done_aggregation_number_updates: FxHashMap<TaskId, AggregationNumberUpdate>,
#[bincode(with = "turbo_bincode::ringset")]
find_and_schedule: FxRingSet<FindAndScheduleJob>,
#[bincode(with = "turbo_bincode::ringset")]
......@@ -898,8 +898,8 @@ impl AggregationUpdateQueue {
pub fn new() -> Self {
Self {
jobs: VecDeque::with_capacity(0),
number_updates: FxIndexMap::default(),
done_number_updates: FxHashMap::default(),
aggregation_number_updates: FxIndexMap::default(),
done_aggregation_number_updates: FxHashMap::default(),
find_and_schedule: FxRingSet::default(),
balance_queue: FxRingSet::default(),
optimize_queue: FxRingSet::default(),
......@@ -910,14 +910,14 @@ impl AggregationUpdateQueue {
pub fn is_empty(&self) -> bool {
let Self {
jobs,
number_updates,
aggregation_number_updates,
find_and_schedule,
balance_queue,
optimize_queue,
done_number_updates: _,
done_aggregation_number_updates: _,
} = self;
jobs.is_empty()
&& number_updates.is_empty()
&& aggregation_number_updates.is_empty()
&& find_and_schedule.is_empty()
&& balance_queue.is_empty()
&& optimize_queue.is_empty()
......@@ -931,7 +931,7 @@ impl AggregationUpdateQueue {
base_aggregation_number,
distance,
} => {
match self.number_updates.entry(task_id) {
match self.aggregation_number_updates.entry(task_id) {
Entry::Occupied(mut entry) => {
let update = entry.get_mut();
update.base_aggregation_number =
......@@ -945,7 +945,7 @@ impl AggregationUpdateQueue {
}
}
Entry::Vacant(entry) => {
match self.done_number_updates.entry(task_id) {
match self.done_aggregation_number_updates.entry(task_id) {
HashMapEntry::Occupied(mut entry) => {
let update = entry.get_mut();
let change =
......@@ -1207,7 +1207,7 @@ impl AggregationUpdateQueue {
}
}
false
} else if !self.number_updates.is_empty() {
} else if !self.aggregation_number_updates.is_empty() {
let mut remaining = MAX_COUNT_BEFORE_YIELD;
while remaining > 0 {
if let Some((
......@@ -1218,11 +1218,11 @@ impl AggregationUpdateQueue {
#[cfg(feature = "trace_aggregation_update")]
span,
},
)) = self.number_updates.pop()
)) = self.aggregation_number_updates.pop()
{
#[cfg(feature = "trace_aggregation_update")]
let _guard = span.map(|s| s.entered());
self.done_number_updates.insert(
self.done_aggregation_number_updates.insert(
task_id,
AggregationNumberUpdate {
base_aggregation_number,
......
use std::{
cmp::Reverse,
collections::{BinaryHeap, hash_map::Entry},
};
use bincode::{Decode, Encode};
use rustc_hash::FxHashMap;
#[cfg(feature = "trace_leaf_distance_update")]
use tracing::{span::Span, trace_span};
use turbo_tasks::TaskId;
use crate::{
backend::{
TaskDataCategory,
operation::{ExecuteContext, Operation, TaskGuard},
storage::{get, iter_many},
},
data::CachedDataItem,
};
const MAX_COUNT_BEFORE_YIELD: usize = 1000;
const BASE_LEAF_DISTANCE_BUFFER: u32 = 128;
/// An leaf distance update job that is enqueued.
#[derive(Encode, Decode, Clone)]
struct LeafDistanceUpdate {
dependencies_min: u32,
dependencies_max: u32,
done: bool,
#[cfg(feature = "trace_leaf_distance_update")]
#[bincode(skip, default)]
span: Option<Span>,
}
impl LeafDistanceUpdate {
fn add(&mut self, dependency_min: u32, dependency_max: u32) {
self.dependencies_min = self.dependencies_min.max(dependency_min);
self.dependencies_max = self.dependencies_max.max(dependency_max);
}
}
/// A queue of leaf distance update jobs.
/// It will execute these jobs in order of their minimum dependency leaf distance.
/// This ensures that we never have to re-process a task.
#[derive(Default, Encode, Decode, Clone)]
pub struct LeafDistanceUpdateQueue {
queue: BinaryHeap<(Reverse<u32>, TaskId)>,
leaf_distance_updates: FxHashMap<TaskId, LeafDistanceUpdate>,
}
impl LeafDistanceUpdateQueue {
pub fn new() -> Self {
Self::default()
}
pub fn is_empty(&self) -> bool {
self.queue.is_empty()
}
pub fn push(&mut self, task_id: TaskId, dependency_min: u32, dependency_max: u32) {
match self.leaf_distance_updates.entry(task_id) {
Entry::Occupied(mut entry) => {
let update = entry.get_mut();
if update.done {
if update.dependencies_min < dependency_min {
update.done = false;
self.queue.push((Reverse(dependency_min), task_id));
}
}
update.add(dependency_min, dependency_max);
}
Entry::Vacant(entry) => {
entry.insert(LeafDistanceUpdate {
dependencies_min: dependency_min,
dependencies_max: dependency_max,
done: false,
});
self.queue.push((Reverse(dependency_min), task_id));
}
};
}
/// Executes a single step of the queue. Returns true, when the queue is empty.
pub fn process(&mut self, ctx: &mut impl ExecuteContext) -> bool {
let mut remaining = MAX_COUNT_BEFORE_YIELD;
while remaining > 0 {
if let Some((Reverse(queue_dependencies_min), task_id)) = self.queue.pop() {
let &mut LeafDistanceUpdate {
dependencies_min,
dependencies_max,
ref mut done,
#[cfg(feature = "trace_leaf_distance_update")]
span,
} = self.leaf_distance_updates.get_mut(&task_id).unwrap();
if queue_dependencies_min != dependencies_min {
// Stale entry in queue
// Re-enqueue to keep the ordering correct
self.queue.push((Reverse(dependencies_min), task_id));
continue;
}
#[cfg(feature = "trace_leaf_distance_update")]
let _guard = span.map(|s| s.entered());
*done = true;
self.update_leaf_distance(ctx, task_id, dependencies_min, dependencies_max);
remaining -= 1;
} else {
return true;
}
}
false
}
fn update_leaf_distance(
&mut self,
ctx: &mut impl ExecuteContext,
task_id: TaskId,
dependencies_min: u32,
dependencies_max: u32,
) {
#[cfg(feature = "trace_leaf_distance_update")]
let _span =
trace_span!("update leaf distance", dependencies_min, dependencies_max).entered();
let mut task = ctx.task(
task_id,
// For performance reasons this should stay `Data` and not `All`
TaskDataCategory::Data,
);
debug_assert!(dependencies_max < u32::MAX / 2);
let mut leaf_distance = get!(task, LeafDistance).copied().unwrap_or_default();
if leaf_distance.min > dependencies_min {
// It is strictly monotonic. No need to update.
return;
}
// It's not strictly monotonic, we need to update
if leaf_distance.max <= dependencies_min {
// We overshoot the buffer zone.
let old_value = leaf_distance.min;
leaf_distance.min = dependencies_max + 1;
let buffer_size = BASE_LEAF_DISTANCE_BUFFER
- BASE_LEAF_DISTANCE_BUFFER.saturating_mul(old_value) / leaf_distance.min;
leaf_distance.max = leaf_distance.min + buffer_size;
} else {
// We are within the buffer zone, keep the max as is
leaf_distance.min = dependencies_min + 1;
}
let dependents = iter_many!(task, OutputDependent { task } => task)
// TODO Technically this is also needed, but there are cycles in the CellDependent graph
// So we need to handle that properly first
// .chain(iter_many!(task, CellDependent { task, .. } => task))
;
for dependent_id in dependents {
self.push(dependent_id, leaf_distance.min, leaf_distance.max);
}
task.insert(CachedDataItem::LeafDistance {
value: leaf_distance,
});
}
}
impl Operation for LeafDistanceUpdateQueue {
fn execute(mut self, ctx: &mut impl ExecuteContext) {
if self.is_empty() {
return;
}
loop {
ctx.operation_suspend_point(&self);
if self.process(ctx) {
return;
}
}
}
}
......@@ -3,6 +3,7 @@ mod cleanup_old_edges;
mod connect_child;
mod connect_children;
mod invalidate;
mod leaf_distance_update;
mod prepare_new_children;
mod update_cell;
mod update_collectible;
......@@ -1043,6 +1044,7 @@ pub enum AnyOperation {
UpdateCell(update_cell::UpdateCellOperation),
CleanupOldEdges(cleanup_old_edges::CleanupOldEdgesOperation),
AggregationUpdate(aggregation_update::AggregationUpdateQueue),
LeafDistanceUpdate(leaf_distance_update::LeafDistanceUpdateQueue),
Nested(Vec<AnyOperation>),
}
......@@ -1054,6 +1056,7 @@ impl AnyOperation {
AnyOperation::UpdateCell(op) => op.execute(ctx),
AnyOperation::CleanupOldEdges(op) => op.execute(ctx),
AnyOperation::AggregationUpdate(op) => op.execute(ctx),
AnyOperation::LeafDistanceUpdate(op) => op.execute(ctx),
AnyOperation::Nested(ops) => {
for op in ops {
op.execute(ctx);
......@@ -1068,6 +1071,7 @@ impl_operation!(Invalidate invalidate::InvalidateOperation);
impl_operation!(UpdateCell update_cell::UpdateCellOperation);
impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation);
impl_operation!(AggregationUpdate aggregation_update::AggregationUpdateQueue);
impl_operation!(LeafDistanceUpdate leaf_distance_update::LeafDistanceUpdateQueue);
#[cfg(feature = "trace_task_dirty")]
pub use self::invalidate::TaskDirtyCause;
......
......@@ -12,7 +12,8 @@ use crate::{
backend::dynamic_storage::DynamicStorage,
data::{
AggregationNumber, CachedDataItem, CachedDataItemKey, CachedDataItemType,
CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut, OutputValue,
CachedDataItemValue, CachedDataItemValueRef, CachedDataItemValueRefMut, LeafDistance,
OutputValue,
},
data_storage::{AutoMapStorage, DefaultStorage, OptionStorage},
utils::{
......@@ -138,6 +139,7 @@ impl InnerStorageState {
}
pub struct InnerStorageSnapshot {
leaf_distance: DefaultStorage<LeafDistance>,
aggregation_number: DefaultStorage<AggregationNumber>,
output_dependent: AutoMapStorage<TaskId, ()>,
output: OptionStorage<OutputValue>,
......@@ -150,6 +152,7 @@ pub struct InnerStorageSnapshot {
impl From<&InnerStorage> for InnerStorageSnapshot {
fn from(inner: &InnerStorage) -> Self {
Self {
leaf_distance: inner.leaf_distance.clone(),
aggregation_number: inner.aggregation_number.clone(),
output_dependent: inner.output_dependent.clone(),
output: inner.output.clone(),
......@@ -168,6 +171,12 @@ impl InnerStorageSnapshot {
use crate::data_storage::Storage;
self.dynamic
.iter_all()
.chain(self.leaf_distance.iter().map(|(_, value)| {
(
CachedDataItemKey::LeafDistance {},
CachedDataItemValueRef::LeafDistance { value },
)
}))
.chain(self.aggregation_number.iter().map(|(_, value)| {
(
CachedDataItemKey::AggregationNumber {},
......@@ -197,6 +206,7 @@ impl InnerStorageSnapshot {
pub fn len(&self) -> usize {
use crate::data_storage::Storage;
self.dynamic.len()
+ self.leaf_distance.len()
+ self.aggregation_number.len()
+ self.output.len()
+ self.upper.len()
......@@ -206,6 +216,7 @@ impl InnerStorageSnapshot {
#[derive(Debug, Clone)]
pub struct InnerStorage {
leaf_distance: DefaultStorage<LeafDistance>,
aggregation_number: DefaultStorage<AggregationNumber>,
output_dependent: AutoMapStorage<TaskId, ()>,
output: OptionStorage<OutputValue>,
......@@ -217,6 +228,7 @@ pub struct InnerStorage {
impl InnerStorage {
fn new() -> Self {
Self {
leaf_distance: Default::default(),
aggregation_number: Default::default(),
output_dependent: Default::default(),
output: Default::default(),
......@@ -553,32 +565,36 @@ macro_rules! generate_inner_storage {
}
generate_inner_storage!(
LeafDistance => leaf_distance,
AggregationNumber => aggregation_number,
OutputDependent task => output_dependent,
Output => output,
Upper task => upper,
);
enum InnerStorageIter<A, B, C, D, E> {
AggregationNumber(A),
OutputDependent(B),
Output(C),
Upper(D),
Dynamic(E),
enum InnerStorageIter<A, B, C, D, E, F> {
LeafDistance(A),
AggregationNumber(B),
OutputDependent(C),
Output(D),
Upper(E),
Dynamic(F),
}
impl<T, A, B, C, D, E> Iterator for InnerStorageIter<A, B, C, D, E>
impl<T, A, B, C, D, E, F> Iterator for InnerStorageIter<A, B, C, D, E, F>
where
A: Iterator<Item = T>,
B: Iterator<Item = T>,
C: Iterator<Item = T>,
D: Iterator<Item = T>,
E: Iterator<Item = T>,
F: Iterator<Item = T>,
{
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
match self {
InnerStorageIter::LeafDistance(iter) => iter.next(),
InnerStorageIter::AggregationNumber(iter) => iter.next(),
InnerStorageIter::OutputDependent(iter) => iter.next(),
InnerStorageIter::Output(iter) => iter.next(),
......@@ -595,6 +611,12 @@ impl InnerStorage {
use crate::data_storage::Storage;
self.dynamic
.iter_all()
.chain(self.leaf_distance.iter().map(|(_, value)| {
(
CachedDataItemKey::LeafDistance {},
CachedDataItemValueRef::LeafDistance { value },
)
}))
.chain(self.aggregation_number.iter().map(|(_, value)| {
(
CachedDataItemKey::AggregationNumber {},
......@@ -624,6 +646,7 @@ impl InnerStorage {
pub fn len(&self) -> usize {
use crate::data_storage::Storage;
self.dynamic.len()
+ self.leaf_distance.len()
+ self.aggregation_number.len()
+ self.output.len()
+ self.upper.len()
......
......@@ -212,6 +212,24 @@ pub struct AggregationNumber {
pub effective: u32,
}
/// Monotonic increasing distance range to leaf nodes when following "dependencies" edges.
/// It is a range and ranges might overlap. There is a strictly monotonic increasing `min` value.
/// `max` value might not be monotonic. The `max` value is used as buffer zone to avoid too many
/// updates to dependent nodes when the leaf distance increases slightly. When the leaf distance is
/// increased it tries to keep the `max` value equal. When increasing there are three cases:
/// - `min` >= `min` of the dependency + 1: no change.
/// - `min` <= `max`: only `min` is increased to the smallest possible value.
/// - `min` > `max`: `min` is increased to the `max` value of the dependency + 1 and `max` is
/// increased to `min` + buffer zone.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Encode, Decode)]
pub struct LeafDistance {
/// This
pub min: u32,
/// A buffer zone value in which is usually safe to increase the leaf distance without causing
/// too many updates to dependent nodes.
pub max: u32,
}
#[derive(Debug, Clone, KeyValuePair, Encode, Decode)]
pub enum CachedDataItem {
// Output
......@@ -288,6 +306,11 @@ pub enum CachedDataItem {
value: (),
},
// Priority
LeafDistance {
value: LeafDistance,
},
// Aggregation Graph
AggregationNumber {
value: AggregationNumber,
......@@ -418,6 +441,7 @@ impl CachedDataItem {
CachedDataItem::AggregationNumber { .. } => true,
CachedDataItem::Follower { task, .. } => !task.is_transient(),
CachedDataItem::Upper { task, .. } => !task.is_transient(),
CachedDataItem::LeafDistance { .. } => true,
CachedDataItem::AggregatedDirtyContainer { task, .. } => !task.is_transient(),
CachedDataItem::AggregatedCurrentSessionCleanContainer { .. } => false,
CachedDataItem::AggregatedCollectible { collectible, .. } => {
......@@ -483,7 +507,8 @@ impl CachedDataItem {
| Self::CellDependency { .. }
| Self::CollectiblesDependency { .. }
| Self::OutputDependent { .. }
| Self::CellDependent { .. } => TaskDataCategory::Data,
| Self::CellDependent { .. }
| Self::LeafDistance { .. } => TaskDataCategory::Data,
Self::Collectible { .. }
| Self::Output { .. }
......@@ -548,6 +573,7 @@ impl CachedDataItemKey {
CachedDataItemKey::AggregationNumber { .. } => true,
CachedDataItemKey::Follower { task, .. } => !task.is_transient(),
CachedDataItemKey::Upper { task, .. } => !task.is_transient(),
CachedDataItemKey::LeafDistance { .. } => true,
CachedDataItemKey::AggregatedDirtyContainer { task, .. } => !task.is_transient(),
CachedDataItemKey::AggregatedCurrentSessionCleanContainer { .. } => false,
CachedDataItemKey::AggregatedCollectible { collectible, .. } => {
......@@ -581,7 +607,8 @@ impl CachedDataItemType {
| Self::CellDependency { .. }
| Self::CollectiblesDependency { .. }
| Self::OutputDependent { .. }
| Self::CellDependent { .. } => TaskDataCategory::Data,
| Self::CellDependent { .. }
| Self::LeafDistance { .. } => TaskDataCategory::Data,
Self::Collectible { .. }
| Self::Output { .. }
......@@ -628,6 +655,7 @@ impl CachedDataItemType {
| Self::AggregationNumber
| Self::Follower
| Self::Upper
| Self::LeafDistance
| Self::AggregatedDirtyContainer
| Self::AggregatedCollectible
| Self::AggregatedDirtyContainerCount
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment