Unverified Commit 691ca603 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: fix cell not found bug (#78246)

### What?

* Some debug asserts were missing to validate the correct category on read access via `iter()` or `extract_if()`. Added these.
* incorrect accessed category led to empty list of dependencies after restoring from persistent caching, which led to weird bugs like Cell not found errors

So optimized the category assignment to make sure aggregation updates can operate on Meta only. Children, Collectibles and CollectiblesDependent were moved to Meta category since they are accessed during aggregation updates. `Children` are used as follower for non-aggregating nodes. `Collectibles` are read when computing the aggregated information and `CollectiblesDependent` are read when aggregated collectibles are modified.

* Added a unit test reproducting this bug and the next PR in the stack also added a e2e test reproducting it.
parent 47b0387d
......@@ -1212,7 +1212,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
};
{
let mut ctx = self.execute_context(turbo_tasks);
let mut task = ctx.task(task_id, TaskDataCategory::Data);
let mut task = ctx.task(task_id, TaskDataCategory::All);
let in_progress = remove!(task, InProgress)?;
let InProgressState::Scheduled { done_event } = in_progress else {
task.add_new(CachedDataItem::InProgress { value: in_progress });
......
......@@ -422,14 +422,22 @@ impl<B: BackingStorage> TaskGuardImpl<'_, B> {
#[cfg(debug_assertions)]
debug_assert!(
self.category == TaskDataCategory::Data
|| self.category == TaskDataCategory::All
|| self.category == TaskDataCategory::All,
"To read data of {:?} the task need to be accessed with this category \
(It's accessed with {:?})",
category,
self.category
);
}
TaskDataCategory::Meta => {
#[cfg(debug_assertions)]
debug_assert!(
self.category == TaskDataCategory::Meta
|| self.category == TaskDataCategory::All
|| self.category == TaskDataCategory::All,
"To read data of {:?} the task need to be accessed with this category \
(It's accessed with {:?})",
category,
self.category
);
}
}
......@@ -631,6 +639,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
&self,
ty: CachedDataItemType,
) -> impl Iterator<Item = (CachedDataItemKey, CachedDataItemValueRef<'_>)> {
self.check_access(ty.category());
self.task.iter(ty)
}
......@@ -646,6 +655,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
where
F: for<'a> FnMut(CachedDataItemKey, CachedDataItemValueRef<'a>) -> bool + 'l,
{
self.check_access(ty.category());
if !self.backend.should_persist() || self.task_id.is_transient() {
return Either::Left(self.task.extract_if(ty, f));
}
......
......@@ -47,7 +47,7 @@ impl UpdateOutputOperation {
output: Result<Result<RawVc>, Option<Cow<'static, str>>>,
mut ctx: impl ExecuteContext,
) {
let mut task = ctx.task(task_id, TaskDataCategory::Meta);
let mut task = ctx.task(task_id, TaskDataCategory::All);
let Some(InProgressState::InProgress(box InProgressStateInner {
stale,
new_children,
......
......@@ -557,26 +557,26 @@ impl CachedDataItem {
pub fn category(&self) -> TaskDataCategory {
match self {
Self::Collectible { .. }
| Self::Child { .. }
| Self::CellData { .. }
Self::CellData { .. }
| Self::CellTypeMaxIndex { .. }
| Self::OutputDependency { .. }
| Self::CellDependency { .. }
| Self::CollectiblesDependency { .. }
| Self::OutputDependent { .. }
| Self::CellDependent { .. }
| Self::CollectiblesDependent { .. } => TaskDataCategory::Data,
| Self::CellDependent { .. } => TaskDataCategory::Data,
Self::Output { .. }
Self::Collectible { .. }
| Self::Output { .. }
| Self::AggregationNumber { .. }
| Self::Dirty { .. }
| Self::Follower { .. }
| Self::Child { .. }
| Self::Upper { .. }
| Self::AggregatedDirtyContainer { .. }
| Self::AggregatedCollectible { .. }
| Self::AggregatedDirtyContainerCount { .. }
| Self::Stateful { .. } => TaskDataCategory::Meta,
| Self::Stateful { .. }
| Self::CollectiblesDependent { .. } => TaskDataCategory::Meta,
Self::OutdatedCollectible { .. }
| Self::OutdatedOutputDependency { .. }
......@@ -639,26 +639,26 @@ impl CachedDataItemKey {
impl CachedDataItemType {
pub fn category(&self) -> TaskDataCategory {
match self {
Self::Collectible { .. }
| Self::Child { .. }
| Self::CellData { .. }
Self::CellData { .. }
| Self::CellTypeMaxIndex { .. }
| Self::OutputDependency { .. }
| Self::CellDependency { .. }
| Self::CollectiblesDependency { .. }
| Self::OutputDependent { .. }
| Self::CellDependent { .. }
| Self::CollectiblesDependent { .. } => TaskDataCategory::Data,
| Self::CellDependent { .. } => TaskDataCategory::Data,
Self::Output { .. }
Self::Collectible { .. }
| Self::Output { .. }
| Self::AggregationNumber { .. }
| Self::Dirty { .. }
| Self::Follower { .. }
| Self::Child { .. }
| Self::Upper { .. }
| Self::AggregatedDirtyContainer { .. }
| Self::AggregatedCollectible { .. }
| Self::AggregatedDirtyContainerCount { .. }
| Self::Stateful { .. } => TaskDataCategory::Meta,
| Self::Stateful { .. }
| Self::CollectiblesDependent { .. } => TaskDataCategory::Meta,
Self::OutdatedCollectible { .. }
| Self::OutdatedOutputDependency { .. }
......
......@@ -33,7 +33,7 @@ async fn dirty_in_progress() {
output.await?;
println!("update to {}", b);
input_val.state.set(b);
tokio::time::sleep(Duration::from_millis(100)).await;
tokio::time::sleep(Duration::from_millis(50)).await;
println!("update to {}", c);
input_val.state.set(c);
let read = output.strongly_consistent().await?;
......
......@@ -12,12 +12,12 @@ static REGISTRATION: Registration = register!();
#[tokio::test]
async fn recompute() {
run(&REGISTRATION, || async {
let input = ChangingInput {
state: State::new(1),
}
.cell();
let output = compute(input, 100);
let read = output.await?;
let input = ChangingInput::new(1).resolve().await?;
let input2 = ChangingInput::new(2).resolve().await?;
input.await?.state.set(1);
input2.await?.state.set(1000);
let output = compute(input, input2, 1);
let read = output.strongly_consistent().await?;
assert_eq!(read.value, 42);
assert_eq!(read.collectible, "1");
......@@ -27,6 +27,13 @@ async fn recompute() {
assert_eq!(read.value, 42);
assert_eq!(read.collectible, i.to_string());
}
for i in 0..100 {
input2.await?.state.set(i);
let read = output.strongly_consistent().await?;
assert_eq!(read.value, 42);
assert_eq!(read.collectible, "99");
}
anyhow::Ok(())
})
.await
......@@ -38,6 +45,18 @@ struct ChangingInput {
state: State<u32>,
}
#[turbo_tasks::value_impl]
impl ChangingInput {
#[turbo_tasks::function]
fn new(key: u32) -> Vc<Self> {
let _ = key;
Self {
state: State::new(1),
}
.cell()
}
}
#[turbo_tasks::value]
struct Output {
value: u32,
......@@ -58,32 +77,39 @@ impl ValueToString for Collectible {
}
#[turbo_tasks::function(operation)]
fn inner_compute(input: ResolvedVc<ChangingInput>) -> Vc<u32> {
inner_compute2(*input, 1000)
async fn inner_compute(
input: ResolvedVc<ChangingInput>,
input2: ResolvedVc<ChangingInput>,
) -> Result<Vc<u32>> {
println!("inner_compute()");
Ok(inner_compute2(*input, *input2.await?.state.get()))
}
#[turbo_tasks::function]
async fn inner_compute2(input: Vc<ChangingInput>, innerness: u32) -> Result<Vc<u32>> {
println!("inner_compute2({innerness})");
if innerness > 0 {
return Ok(inner_compute2(input, innerness - 1));
}
let collectible: ResolvedVc<Box<dyn ValueToString>> = ResolvedVc::upcast(
Collectible {
value: *input.await?.state.get(),
}
.resolved_cell(),
);
let value = *input.await?.state.get();
let collectible: ResolvedVc<Box<dyn ValueToString>> =
ResolvedVc::upcast(Collectible { value }.resolved_cell());
emit(collectible);
Ok(Vc::cell(42))
}
#[turbo_tasks::function]
async fn compute(input: ResolvedVc<ChangingInput>, innerness: u32) -> Result<Vc<Output>> {
async fn compute(
input: ResolvedVc<ChangingInput>,
input2: ResolvedVc<ChangingInput>,
innerness: u32,
) -> Result<Vc<Output>> {
println!("compute({innerness})");
if innerness > 0 {
return Ok(compute(*input, innerness - 1));
return Ok(compute(*input, *input2, innerness - 1));
}
let operation = inner_compute(input);
let operation = inner_compute(input, input2);
let value = *operation.connect().await?;
let collectibles = operation.peek_collectibles::<Box<dyn ValueToString>>();
if collectibles.len() != 1 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment