Unverified Commit 4744bc78 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] fix hanging bug caused by incorrect active counter handling on stale tasks (#75732)

### What?

fix cases where action counter is messed up when tasks are invalidated during execution
parent 0b1767d3
use anyhow::Result; use anyhow::Result;
use rustc_hash::FxHashSet;
use tracing::Instrument; use tracing::Instrument;
use turbo_tasks::{ use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal}, graph::{AdjacencyMap, GraphTraversal},
...@@ -112,6 +113,8 @@ pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<Out ...@@ -112,6 +113,8 @@ pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<Out
.completed()? .completed()?
.into_inner() .into_inner()
.into_reverse_topological() .into_reverse_topological()
.collect::<FxHashSet<_>>()
.into_iter()
.collect(), .collect(),
)) ))
} }
......
...@@ -1171,8 +1171,25 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1171,8 +1171,25 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
stateful: bool, stateful: bool,
turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>, turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) -> bool { ) -> bool {
// Task completion is a 4 step process:
// 1. Remove old edges (dependencies, collectibles, children, cells) and update the
// aggregation number of the task and the new children.
// 2. Connect the new children to the task (and do the relevant aggregation updates).
// 3. Remove dirty flag (and propagate that to uppers) and remove the in-progress state.
// 4. Shrink the task memory to reduce footprint of the task.
// Due to persistance it is possible that the process is cancelled after any step. This is
// ok, since the dirty flag won't be removed until step 3 and step 4 is only affecting the
// in-memory representation.
// The task might be invalidated during this process, so we need to change the stale flag
// at the start of every step.
let _span = tracing::trace_span!("task execution completed").entered(); let _span = tracing::trace_span!("task execution completed").entered();
let mut ctx = self.execute_context(turbo_tasks); let mut ctx = self.execute_context(turbo_tasks);
//// STEP 1 ////
let mut task = ctx.task(task_id, TaskDataCategory::All); let mut task = ctx.task(task_id, TaskDataCategory::All);
let Some(in_progress) = get_mut!(task, InProgress) else { let Some(in_progress) = get_mut!(task, InProgress) else {
panic!("Task execution completed, but task is not in progress: {task:#?}"); panic!("Task execution completed, but task is not in progress: {task:#?}");
...@@ -1192,7 +1209,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1192,7 +1209,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
if stale { if stale {
let Some(InProgressState::InProgress(box InProgressStateInner { let Some(InProgressState::InProgress(box InProgressStateInner {
done_event, done_event,
new_children, mut new_children,
.. ..
})) = remove!(task, InProgress) })) = remove!(task, InProgress)
else { else {
...@@ -1201,10 +1218,15 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1201,10 +1218,15 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
task.add_new(CachedDataItem::InProgress { task.add_new(CachedDataItem::InProgress {
value: InProgressState::Scheduled { done_event }, value: InProgressState::Scheduled { done_event },
}); });
// Remove old children from new_children to leave only the children that had their
// active count increased
for task in iter_many!(task, Child { task } => task) {
new_children.remove(&task);
}
drop(task); drop(task);
// All `new_children` are currently hold active with an active count and we need to undo // We need to undo the active count increase for the children since we throw away the
// that. // new_children list now.
AggregationUpdateQueue::run( AggregationUpdateQueue::run(
AggregationUpdateJob::DecreaseActiveCounts { AggregationUpdateJob::DecreaseActiveCounts {
task_ids: new_children.into_iter().collect(), task_ids: new_children.into_iter().collect(),
...@@ -1263,10 +1285,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1263,10 +1285,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
} }
// Filter actual new children // Filter actual new children
for old_child in iter_many!(task, Child { task } => task) { if has_children {
if !has_children || !new_children.remove(&old_child) { old_edges.extend(
old_edges.push(OutdatedEdge::Child(old_child)); iter_many!(task, Child { task } => task)
} .filter(|task| !new_children.remove(task))
.map(OutdatedEdge::Child),
);
} else {
old_edges.extend(iter_many!(task, Child { task } => task).map(OutdatedEdge::Child));
} }
// Remove no longer existing cells and notify in progress cells // Remove no longer existing cells and notify in progress cells
...@@ -1341,6 +1367,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1341,6 +1367,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
// suspend in `CleanupOldEdgesOperation`), but that's ok as the task is still dirty and // suspend in `CleanupOldEdgesOperation`), but that's ok as the task is still dirty and
// would be executed again. // would be executed again.
//// STEP 2 ////
let mut task = ctx.task(task_id, TaskDataCategory::All); let mut task = ctx.task(task_id, TaskDataCategory::All);
let Some(in_progress) = get!(task, InProgress) else { let Some(in_progress) = get!(task, InProgress) else {
panic!("Task execution completed, but task is not in progress: {task:#?}"); panic!("Task execution completed, but task is not in progress: {task:#?}");
...@@ -1352,11 +1380,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1352,11 +1380,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
// If the task is stale, reschedule it // If the task is stale, reschedule it
if *stale { if *stale {
let Some(InProgressState::InProgress(box InProgressStateInner { let Some(InProgressState::InProgress(box InProgressStateInner { done_event, .. })) =
done_event, remove!(task, InProgress)
new_children,
..
})) = remove!(task, InProgress)
else { else {
unreachable!(); unreachable!();
}; };
...@@ -1365,7 +1390,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1365,7 +1390,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
}); });
// All `new_children` are currently hold active with an active count and we need to undo // All `new_children` are currently hold active with an active count and we need to undo
// that. // that. (We already filtered out the old children from that list)
AggregationUpdateQueue::run( AggregationUpdateQueue::run(
AggregationUpdateJob::DecreaseActiveCounts { AggregationUpdateJob::DecreaseActiveCounts {
task_ids: new_children.into_iter().collect(), task_ids: new_children.into_iter().collect(),
...@@ -1487,6 +1512,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1487,6 +1512,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
drop(removed_data); drop(removed_data);
//// STEP 4 ////
let mut task = ctx.task(task_id, TaskDataCategory::All); let mut task = ctx.task(task_id, TaskDataCategory::All);
task.shrink_to_fit(CachedDataItemType::CellData); task.shrink_to_fit(CachedDataItemType::CellData);
task.shrink_to_fit(CachedDataItemType::CellTypeMaxIndex); task.shrink_to_fit(CachedDataItemType::CellTypeMaxIndex);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment