Unverified Commit 691ca603 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: fix cell not found bug (#78246)

### What?

* Some debug asserts were missing to validate the correct category on read access via `iter()` or `extract_if()`. Added these.
* incorrect accessed category led to empty list of dependencies after restoring from persistent caching, which led to weird bugs like Cell not found errors

So optimized the category assignment to make sure aggregation updates can operate on Meta only. Children, Collectibles and CollectiblesDependent were moved to Meta category since they are accessed during aggregation updates. `Children` are used as follower for non-aggregating nodes. `Collectibles` are read when computing the aggregated information and `CollectiblesDependent` are read when aggregated collectibles are modified.

* Added a unit test reproducting this bug and the next PR in the stack also added a e2e test reproducting it.
parent 47b0387d
...@@ -1212,7 +1212,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1212,7 +1212,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
}; };
{ {
let mut ctx = self.execute_context(turbo_tasks); let mut ctx = self.execute_context(turbo_tasks);
let mut task = ctx.task(task_id, TaskDataCategory::Data); let mut task = ctx.task(task_id, TaskDataCategory::All);
let in_progress = remove!(task, InProgress)?; let in_progress = remove!(task, InProgress)?;
let InProgressState::Scheduled { done_event } = in_progress else { let InProgressState::Scheduled { done_event } = in_progress else {
task.add_new(CachedDataItem::InProgress { value: in_progress }); task.add_new(CachedDataItem::InProgress { value: in_progress });
......
...@@ -422,14 +422,22 @@ impl<B: BackingStorage> TaskGuardImpl<'_, B> { ...@@ -422,14 +422,22 @@ impl<B: BackingStorage> TaskGuardImpl<'_, B> {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
debug_assert!( debug_assert!(
self.category == TaskDataCategory::Data self.category == TaskDataCategory::Data
|| self.category == TaskDataCategory::All || self.category == TaskDataCategory::All,
"To read data of {:?} the task need to be accessed with this category \
(It's accessed with {:?})",
category,
self.category
); );
} }
TaskDataCategory::Meta => { TaskDataCategory::Meta => {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
debug_assert!( debug_assert!(
self.category == TaskDataCategory::Meta self.category == TaskDataCategory::Meta
|| self.category == TaskDataCategory::All || self.category == TaskDataCategory::All,
"To read data of {:?} the task need to be accessed with this category \
(It's accessed with {:?})",
category,
self.category
); );
} }
} }
...@@ -631,6 +639,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -631,6 +639,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
&self, &self,
ty: CachedDataItemType, ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> { ) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.check_access(ty.category());
self.task.iter(ty) self.task.iter(ty)
} }
...@@ -646,6 +655,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -646,6 +655,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
where where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l, F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{ {
self.check_access(ty.category());
if !self.backend.should_persist() || self.task_id.is_transient() { if !self.backend.should_persist() || self.task_id.is_transient() {
return Either::Left(self.task.extract_if(ty, f)); return Either::Left(self.task.extract_if(ty, f));
} }
......
...@@ -47,7 +47,7 @@ impl UpdateOutputOperation { ...@@ -47,7 +47,7 @@ impl UpdateOutputOperation {
output: Result<Result<RawVc>, Option<Cow<'static, str>>>, output: Result<Result<RawVc>, Option<Cow<'static, str>>>,
mut ctx: impl ExecuteContext, mut ctx: impl ExecuteContext,
) { ) {
let mut task = ctx.task(task_id, TaskDataCategory::Meta); let mut task = ctx.task(task_id, TaskDataCategory::All);
let Some(InProgressState::InProgress(box InProgressStateInner { let Some(InProgressState::InProgress(box InProgressStateInner {
stale, stale,
new_children, new_children,
......
...@@ -557,26 +557,26 @@ impl CachedDataItem { ...@@ -557,26 +557,26 @@ impl CachedDataItem {
pub fn category(&self) -> TaskDataCategory { pub fn category(&self) -> TaskDataCategory {
match self { match self {
Self::Collectible { .. } Self::CellData { .. }
| Self::Child { .. }
| Self::CellData { .. }
| Self::CellTypeMaxIndex { .. } | Self::CellTypeMaxIndex { .. }
| Self::OutputDependency { .. } | Self::OutputDependency { .. }
| Self::CellDependency { .. } | Self::CellDependency { .. }
| Self::CollectiblesDependency { .. } | Self::CollectiblesDependency { .. }
| Self::OutputDependent { .. } | Self::OutputDependent { .. }
| Self::CellDependent { .. } | Self::CellDependent { .. } => TaskDataCategory::Data,
| Self::CollectiblesDependent { .. } => TaskDataCategory::Data,
Self::Output { .. } Self::Collectible { .. }
| Self::Output { .. }
| Self::AggregationNumber { .. } | Self::AggregationNumber { .. }
| Self::Dirty { .. } | Self::Dirty { .. }
| Self::Follower { .. } | Self::Follower { .. }
| Self::Child { .. }
| Self::Upper { .. } | Self::Upper { .. }
| Self::AggregatedDirtyContainer { .. } | Self::AggregatedDirtyContainer { .. }
| Self::AggregatedCollectible { .. } | Self::AggregatedCollectible { .. }
| Self::AggregatedDirtyContainerCount { .. } | Self::AggregatedDirtyContainerCount { .. }
| Self::Stateful { .. } => TaskDataCategory::Meta, | Self::Stateful { .. }
| Self::CollectiblesDependent { .. } => TaskDataCategory::Meta,
Self::OutdatedCollectible { .. } Self::OutdatedCollectible { .. }
| Self::OutdatedOutputDependency { .. } | Self::OutdatedOutputDependency { .. }
...@@ -639,26 +639,26 @@ impl CachedDataItemKey { ...@@ -639,26 +639,26 @@ impl CachedDataItemKey {
impl CachedDataItemType { impl CachedDataItemType {
pub fn category(&self) -> TaskDataCategory { pub fn category(&self) -> TaskDataCategory {
match self { match self {
Self::Collectible { .. } Self::CellData { .. }
| Self::Child { .. }
| Self::CellData { .. }
| Self::CellTypeMaxIndex { .. } | Self::CellTypeMaxIndex { .. }
| Self::OutputDependency { .. } | Self::OutputDependency { .. }
| Self::CellDependency { .. } | Self::CellDependency { .. }
| Self::CollectiblesDependency { .. } | Self::CollectiblesDependency { .. }
| Self::OutputDependent { .. } | Self::OutputDependent { .. }
| Self::CellDependent { .. } | Self::CellDependent { .. } => TaskDataCategory::Data,
| Self::CollectiblesDependent { .. } => TaskDataCategory::Data,
Self::Output { .. } Self::Collectible { .. }
| Self::Output { .. }
| Self::AggregationNumber { .. } | Self::AggregationNumber { .. }
| Self::Dirty { .. } | Self::Dirty { .. }
| Self::Follower { .. } | Self::Follower { .. }
| Self::Child { .. }
| Self::Upper { .. } | Self::Upper { .. }
| Self::AggregatedDirtyContainer { .. } | Self::AggregatedDirtyContainer { .. }
| Self::AggregatedCollectible { .. } | Self::AggregatedCollectible { .. }
| Self::AggregatedDirtyContainerCount { .. } | Self::AggregatedDirtyContainerCount { .. }
| Self::Stateful { .. } => TaskDataCategory::Meta, | Self::Stateful { .. }
| Self::CollectiblesDependent { .. } => TaskDataCategory::Meta,
Self::OutdatedCollectible { .. } Self::OutdatedCollectible { .. }
| Self::OutdatedOutputDependency { .. } | Self::OutdatedOutputDependency { .. }
......
...@@ -33,7 +33,7 @@ async fn dirty_in_progress() { ...@@ -33,7 +33,7 @@ async fn dirty_in_progress() {
output.await?; output.await?;
println!("update to {}", b); println!("update to {}", b);
input_val.state.set(b); input_val.state.set(b);
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(50)).await;
println!("update to {}", c); println!("update to {}", c);
input_val.state.set(c); input_val.state.set(c);
let read = output.strongly_consistent().await?; let read = output.strongly_consistent().await?;
......
...@@ -12,12 +12,12 @@ static REGISTRATION: Registration = register!(); ...@@ -12,12 +12,12 @@ static REGISTRATION: Registration = register!();
#[tokio::test] #[tokio::test]
async fn recompute() { async fn recompute() {
run(&REGISTRATION, || async { run(&REGISTRATION, || async {
let input = ChangingInput { let input = ChangingInput::new(1).resolve().await?;
state: State::new(1), let input2 = ChangingInput::new(2).resolve().await?;
} input.await?.state.set(1);
.cell(); input2.await?.state.set(1000);
let output = compute(input, 100); let output = compute(input, input2, 1);
let read = output.await?; let read = output.strongly_consistent().await?;
assert_eq!(read.value, 42); assert_eq!(read.value, 42);
assert_eq!(read.collectible, "1"); assert_eq!(read.collectible, "1");
...@@ -27,6 +27,13 @@ async fn recompute() { ...@@ -27,6 +27,13 @@ async fn recompute() {
assert_eq!(read.value, 42); assert_eq!(read.value, 42);
assert_eq!(read.collectible, i.to_string()); assert_eq!(read.collectible, i.to_string());
} }
for i in 0..100 {
input2.await?.state.set(i);
let read = output.strongly_consistent().await?;
assert_eq!(read.value, 42);
assert_eq!(read.collectible, "99");
}
anyhow::Ok(()) anyhow::Ok(())
}) })
.await .await
...@@ -38,6 +45,18 @@ struct ChangingInput { ...@@ -38,6 +45,18 @@ struct ChangingInput {
state: State<u32>, state: State<u32>,
} }
#[turbo_tasks::value_impl]
impl ChangingInput {
#[turbo_tasks::function]
fn new(key: u32) -> Vc<Self> {
let _ = key;
Self {
state: State::new(1),
}
.cell()
}
}
#[turbo_tasks::value] #[turbo_tasks::value]
struct Output { struct Output {
value: u32, value: u32,
...@@ -58,32 +77,39 @@ impl ValueToString for Collectible { ...@@ -58,32 +77,39 @@ impl ValueToString for Collectible {
} }
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
fn inner_compute(input: ResolvedVc<ChangingInput>) -> Vc<u32> { async fn inner_compute(
inner_compute2(*input, 1000) input: ResolvedVc<ChangingInput>,
input2: ResolvedVc<ChangingInput>,
) -> Result<Vc<u32>> {
println!("inner_compute()");
Ok(inner_compute2(*input, *input2.await?.state.get()))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn inner_compute2(input: Vc<ChangingInput>, innerness: u32) -> Result<Vc<u32>> { async fn inner_compute2(input: Vc<ChangingInput>, innerness: u32) -> Result<Vc<u32>> {
println!("inner_compute2({innerness})");
if innerness > 0 { if innerness > 0 {
return Ok(inner_compute2(input, innerness - 1)); return Ok(inner_compute2(input, innerness - 1));
} }
let collectible: ResolvedVc<Box<dyn ValueToString>> = ResolvedVc::upcast( let value = *input.await?.state.get();
Collectible { let collectible: ResolvedVc<Box<dyn ValueToString>> =
value: *input.await?.state.get(), ResolvedVc::upcast(Collectible { value }.resolved_cell());
}
.resolved_cell(),
);
emit(collectible); emit(collectible);
Ok(Vc::cell(42)) Ok(Vc::cell(42))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn compute(input: ResolvedVc<ChangingInput>, innerness: u32) -> Result<Vc<Output>> { async fn compute(
input: ResolvedVc<ChangingInput>,
input2: ResolvedVc<ChangingInput>,
innerness: u32,
) -> Result<Vc<Output>> {
println!("compute({innerness})");
if innerness > 0 { if innerness > 0 {
return Ok(compute(*input, innerness - 1)); return Ok(compute(*input, *input2, innerness - 1));
} }
let operation = inner_compute(input); let operation = inner_compute(input, input2);
let value = *operation.connect().await?; let value = *operation.connect().await?;
let collectibles = operation.peek_collectibles::<Box<dyn ValueToString>>(); let collectibles = operation.peek_collectibles::<Box<dyn ValueToString>>();
if collectibles.len() != 1 { if collectibles.len() != 1 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment