Unverified Commit f6c2d491 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add task_id_details feature (#84970)

### What?

Add a bunch of feature flags that help debugging cases where it is loosing invalidations.
parent 9375bc3d
...@@ -24,6 +24,8 @@ trace_aggregation_update = [] ...@@ -24,6 +24,8 @@ trace_aggregation_update = []
trace_find_and_schedule = [] trace_find_and_schedule = []
trace_task_completion = [] trace_task_completion = []
trace_task_dirty = [] trace_task_dirty = []
trace_task_output_dependencies = []
trace_task_details = []
lmdb = ["dep:lmdb-rkv"] lmdb = ["dep:lmdb-rkv"]
[dependencies] [dependencies]
......
...@@ -24,7 +24,7 @@ use parking_lot::{Condvar, Mutex}; ...@@ -24,7 +24,7 @@ use parking_lot::{Condvar, Mutex};
use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
use smallvec::{SmallVec, smallvec}; use smallvec::{SmallVec, smallvec};
use tokio::time::{Duration, Instant}; use tokio::time::{Duration, Instant};
use tracing::{Span, field::Empty, info_span, trace_span}; use tracing::{Span, info_span, trace_span};
use turbo_tasks::{ use turbo_tasks::{
CellId, FxDashMap, FxIndexMap, KeyValuePair, RawVc, ReadCellOptions, ReadConsistency, CellId, FxDashMap, FxIndexMap, KeyValuePair, RawVc, ReadCellOptions, ReadConsistency,
ReadOutputOptions, ReadTracking, SessionId, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId, ReadOutputOptions, ReadTracking, SessionId, TRANSIENT_TASK_BIT, TaskExecutionReason, TaskId,
...@@ -722,21 +722,23 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -722,21 +722,23 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let result = match output { let result = match output {
OutputValue::Cell(cell) => Ok(Ok(RawVc::TaskCell(cell.task, cell.cell))), OutputValue::Cell(cell) => Ok(Ok(RawVc::TaskCell(cell.task, cell.cell))),
OutputValue::Output(task) => Ok(Ok(RawVc::TaskOutput(*task))), OutputValue::Output(task) => Ok(Ok(RawVc::TaskOutput(*task))),
OutputValue::Error(error) => { OutputValue::Error(error) => Err(error
let err: anyhow::Error = error.clone().into(); .with_task_context(ctx.get_task_description(task_id), Some(task_id))
Err(err.context(format!( .into()),
"Execution of {} failed",
ctx.get_task_description(task_id)
)))
}
}; };
if let Some(mut reader_task) = reader_task if let Some(mut reader_task) = reader_task
&& options.tracking.should_track(result.is_err()) && options.tracking.should_track(result.is_err())
&& (!task.is_immutable() || cfg!(feature = "verify_immutable")) && (!task.is_immutable() || cfg!(feature = "verify_immutable"))
{ {
let reader = reader.unwrap(); #[cfg(feature = "trace_task_output_dependencies")]
let _span = tracing::trace_span!(
"add output dependency",
task = %task_id,
dependent_task = ?reader
)
.entered();
let _ = task.add(CachedDataItem::OutputDependent { let _ = task.add(CachedDataItem::OutputDependent {
task: reader, task: reader.unwrap(),
value: (), value: (),
}); });
drop(task); drop(task);
...@@ -1763,7 +1765,22 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1763,7 +1765,22 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
// The task might be invalidated during this process, so we need to check the stale flag // The task might be invalidated during this process, so we need to check the stale flag
// at the start of every step. // at the start of every step.
let span = tracing::trace_span!("task execution completed", immutable = Empty).entered(); #[cfg(not(feature = "trace_task_details"))]
let _span = tracing::trace_span!("task execution completed").entered();
#[cfg(feature = "trace_task_details")]
let span = tracing::trace_span!(
"task execution completed",
task_id = display(task_id),
result = match result.as_ref() {
Ok(value) => display(either::Either::Left(value)),
Err(err) => display(either::Either::Right(err)),
},
immutable = tracing::field::Empty,
new_output = tracing::field::Empty,
output_dependents = tracing::field::Empty,
stale = tracing::field::Empty,
)
.entered();
let mut ctx = self.execute_context(turbo_tasks); let mut ctx = self.execute_context(turbo_tasks);
let Some(TaskExecutionCompletePrepareResult { let Some(TaskExecutionCompletePrepareResult {
...@@ -1776,6 +1793,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1776,6 +1793,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
output_dependent_tasks, output_dependent_tasks,
}) = self.task_execution_completed_prepare( }) = self.task_execution_completed_prepare(
&mut ctx, &mut ctx,
#[cfg(feature = "trace_task_details")]
&span, &span,
task_id, task_id,
result, result,
...@@ -1785,9 +1803,16 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1785,9 +1803,16 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
) )
else { else {
// Task was stale and has been rescheduled // Task was stale and has been rescheduled
#[cfg(feature = "trace_task_details")]
span.record("stale", "true");
return true; return true;
}; };
#[cfg(feature = "trace_task_details")]
span.record("new_output", new_output.is_some());
#[cfg(feature = "trace_task_details")]
span.record("output_dependents", output_dependent_tasks.len());
// When restoring from filesystem cache the following might not be executed (since we can // When restoring from filesystem cache the following might not be executed (since we can
// suspend in `CleanupOldEdgesOperation`), but that's ok as the task is still dirty and // suspend in `CleanupOldEdgesOperation`), but that's ok as the task is still dirty and
// would be executed again. // would be executed again.
...@@ -1810,6 +1835,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1810,6 +1835,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
&& self.task_execution_completed_connect(&mut ctx, task_id, new_children) && self.task_execution_completed_connect(&mut ctx, task_id, new_children)
{ {
// Task was stale and has been rescheduled // Task was stale and has been rescheduled
#[cfg(feature = "trace_task_details")]
span.record("stale", "true");
return true; return true;
} }
...@@ -1823,6 +1850,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1823,6 +1850,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
is_now_immutable, is_now_immutable,
) { ) {
// Task was stale and has been rescheduled // Task was stale and has been rescheduled
#[cfg(feature = "trace_task_details")]
span.record("stale", "true");
return true; return true;
} }
...@@ -1836,7 +1865,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1836,7 +1865,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
fn task_execution_completed_prepare( fn task_execution_completed_prepare(
&self, &self,
ctx: &mut impl ExecuteContext<'_>, ctx: &mut impl ExecuteContext<'_>,
span: &Span, #[cfg(feature = "trace_task_details")] span: &Span,
task_id: TaskId, task_id: TaskId,
result: Result<RawVc, TurboTasksExecutionError>, result: Result<RawVc, TurboTasksExecutionError>,
cell_counters: &AutoMap<ValueTypeId, u32, BuildHasherDefault<FxHasher>, 8>, cell_counters: &AutoMap<ValueTypeId, u32, BuildHasherDefault<FxHasher>, 8>,
...@@ -2088,6 +2117,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2088,6 +2117,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
{ {
is_now_immutable = true; is_now_immutable = true;
} }
#[cfg(feature = "trace_task_details")]
span.record("immutable", is_immutable || is_now_immutable); span.record("immutable", is_immutable || is_now_immutable);
if !queue.is_empty() || !old_edges.is_empty() { if !queue.is_empty() || !old_edges.is_empty() {
...@@ -2120,19 +2150,33 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2120,19 +2150,33 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let mut queue = AggregationUpdateQueue::new(); let mut queue = AggregationUpdateQueue::new();
for dependent_task_id in output_dependent_tasks { for dependent_task_id in output_dependent_tasks {
#[cfg(feature = "trace_task_output_dependencies")]
let span = tracing::trace_span!(
"invalidate output dependency",
task = %task_id,
dependent_task = %dependent_task_id,
result = tracing::field::Empty,
)
.entered();
if ctx.is_once_task(dependent_task_id) { if ctx.is_once_task(dependent_task_id) {
// once tasks are never invalidated // once tasks are never invalidated
#[cfg(feature = "trace_task_output_dependencies")]
span.record("result", "once task");
continue; continue;
} }
let dependent = ctx.task(dependent_task_id, TaskDataCategory::All); let dependent = ctx.task(dependent_task_id, TaskDataCategory::All);
if dependent.has_key(&CachedDataItemKey::OutdatedOutputDependency { target: task_id }) { if dependent.has_key(&CachedDataItemKey::OutdatedOutputDependency { target: task_id }) {
// output dependency is outdated, so it hasn't read the output yet // output dependency is outdated, so it hasn't read the output yet
// and doesn't need to be invalidated // and doesn't need to be invalidated
#[cfg(feature = "trace_task_output_dependencies")]
span.record("result", "outdated dependency");
continue; continue;
} }
if !dependent.has_key(&CachedDataItemKey::OutputDependency { target: task_id }) { if !dependent.has_key(&CachedDataItemKey::OutputDependency { target: task_id }) {
// output dependency has been removed, so the task doesn't depend on the // output dependency has been removed, so the task doesn't depend on the
// output anymore and doesn't need to be invalidated // output anymore and doesn't need to be invalidated
#[cfg(feature = "trace_task_output_dependencies")]
span.record("result", "no backward dependency");
continue; continue;
} }
make_task_dirty_internal( make_task_dirty_internal(
...@@ -2144,6 +2188,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2144,6 +2188,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
&mut queue, &mut queue,
ctx, ctx,
); );
#[cfg(feature = "trace_task_output_dependencies")]
span.record("result", "marked dirty");
} }
queue.execute(ctx); queue.execute(ctx);
......
...@@ -182,6 +182,13 @@ impl Operation for CleanupOldEdgesOperation { ...@@ -182,6 +182,13 @@ impl Operation for CleanupOldEdgesOperation {
} }
} }
OutdatedEdge::OutputDependency(output_task_id) => { OutdatedEdge::OutputDependency(output_task_id) => {
#[cfg(feature = "trace_task_output_dependencies")]
let _span = tracing::trace_span!(
"remove output dependency",
task = %output_task_id,
dependent_task = %task_id
)
.entered();
{ {
let mut task = ctx.task(output_task_id, TaskDataCategory::Data); let mut task = ctx.task(output_task_id, TaskDataCategory::Data);
task.remove(&CachedDataItemKey::OutputDependent { task.remove(&CachedDataItemKey::OutputDependent {
......
...@@ -224,6 +224,7 @@ pub fn make_task_dirty_internal( ...@@ -224,6 +224,7 @@ pub fn make_task_dirty_internal(
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
let _span = tracing::trace_span!( let _span = tracing::trace_span!(
"make task stale", "make task stale",
task_id = display(task_id),
name = ctx.get_task_description(task_id), name = ctx.get_task_description(task_id),
cause = %TaskDirtyCauseInContext::new(&cause, ctx) cause = %TaskDirtyCauseInContext::new(&cause, ctx)
) )
...@@ -244,6 +245,7 @@ pub fn make_task_dirty_internal( ...@@ -244,6 +245,7 @@ pub fn make_task_dirty_internal(
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
let _span = tracing::trace_span!( let _span = tracing::trace_span!(
"task already dirty", "task already dirty",
task_id = display(task_id),
name = ctx.get_task_description(task_id), name = ctx.get_task_description(task_id),
cause = %TaskDirtyCauseInContext::new(&cause, ctx) cause = %TaskDirtyCauseInContext::new(&cause, ctx)
) )
...@@ -275,6 +277,7 @@ pub fn make_task_dirty_internal( ...@@ -275,6 +277,7 @@ pub fn make_task_dirty_internal(
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
let _span = tracing::trace_span!( let _span = tracing::trace_span!(
"make task dirty", "make task dirty",
task_id = display(task_id),
name = ctx.get_task_description(task_id), name = ctx.get_task_description(task_id),
cause = %TaskDirtyCauseInContext::new(&cause, ctx) cause = %TaskDirtyCauseInContext::new(&cause, ctx)
) )
......
...@@ -12,6 +12,7 @@ bench = false ...@@ -12,6 +12,7 @@ bench = false
default = [] default = []
tokio_tracing = ["tokio/tracing"] tokio_tracing = ["tokio/tracing"]
hanging_detection = [] hanging_detection = []
task_id_details = []
# TODO(bgw): This feature is here to unblock turning on local tasks by default. It's only turned on # TODO(bgw): This feature is here to unblock turning on local tasks by default. It's only turned on
# in unit tests. This will be removed very soon. # in unit tests. This will be removed very soon.
......
...@@ -427,6 +427,8 @@ pub struct TurboTasksError { ...@@ -427,6 +427,8 @@ pub struct TurboTasksError {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct TurboTaskContextError { pub struct TurboTaskContextError {
pub task: RcStr, pub task: RcStr,
#[cfg(feature = "task_id_details")]
pub task_id: Option<TaskId>,
pub source: Option<TurboTasksExecutionError>, pub source: Option<TurboTasksExecutionError>,
} }
...@@ -438,9 +440,11 @@ pub enum TurboTasksExecutionError { ...@@ -438,9 +440,11 @@ pub enum TurboTasksExecutionError {
} }
impl TurboTasksExecutionError { impl TurboTasksExecutionError {
pub fn with_task_context(&self, task: impl Display) -> Self { pub fn with_task_context(&self, task: impl Display, _task_id: Option<TaskId>) -> Self {
TurboTasksExecutionError::TaskContext(Arc::new(TurboTaskContextError { TurboTasksExecutionError::TaskContext(Arc::new(TurboTaskContextError {
task: RcStr::from(task.to_string()), task: RcStr::from(task.to_string()),
#[cfg(feature = "task_id_details")]
task_id: _task_id,
source: Some(self.clone()), source: Some(self.clone()),
})) }))
} }
...@@ -468,6 +472,14 @@ impl Display for TurboTasksExecutionError { ...@@ -468,6 +472,14 @@ impl Display for TurboTasksExecutionError {
write!(f, "{}", error.message) write!(f, "{}", error.message)
} }
TurboTasksExecutionError::TaskContext(context_error) => { TurboTasksExecutionError::TaskContext(context_error) => {
#[cfg(feature = "task_id_details")]
if let Some(task_id) = context_error.task_id {
return write!(
f,
"Execution of {} ({}) failed",
context_error.task, task_id
);
}
write!(f, "Execution of {} failed", context_error.task) write!(f, "Execution of {} failed", context_error.task)
} }
} }
......
...@@ -281,6 +281,15 @@ pub enum TaskPersistence { ...@@ -281,6 +281,15 @@ pub enum TaskPersistence {
Transient, Transient,
} }
impl Display for TaskPersistence {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TaskPersistence::Persistent => write!(f, "persistent"),
TaskPersistence::Transient => write!(f, "transient"),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Default)] #[derive(Clone, Copy, Debug, Eq, PartialEq, Default)]
pub enum ReadConsistency { pub enum ReadConsistency {
/// The default behavior for most APIs. Reads are faster, but may return stale values, which /// The default behavior for most APIs. Reads are faster, but may return stale values, which
...@@ -826,7 +835,7 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -826,7 +835,7 @@ impl<B: Backend + 'static> TurboTasks<B> {
let output = match result { let output = match result {
Ok(raw_vc) => OutputContent::Link(raw_vc), Ok(raw_vc) => OutputContent::Link(raw_vc),
Err(err) => OutputContent::Error( Err(err) => OutputContent::Error(
TurboTasksExecutionError::from(err).with_task_context(task_type), TurboTasksExecutionError::from(err).with_task_context(task_type, None),
), ),
}; };
......
...@@ -103,6 +103,22 @@ impl Debug for RawVc { ...@@ -103,6 +103,22 @@ impl Debug for RawVc {
} }
} }
impl Display for RawVc {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RawVc::TaskOutput(task_id) => write!(f, "output of task {}", **task_id),
RawVc::TaskCell(task_id, cell_id) => {
write!(f, "{} of task {}", cell_id, **task_id)
}
RawVc::LocalOutput(execution_id, local_task_id, task_persistence) => write!(
f,
"output of local task {} ({}, {})",
**local_task_id, **execution_id, task_persistence
),
}
}
}
impl RawVc { impl RawVc {
pub fn is_resolved(&self) -> bool { pub fn is_resolved(&self) -> bool {
match self { match self {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment