Unverified Commit 61f08530 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Fix a crash under turbopackPersistentCaching (#82592)

Address a crash found at head when testing caching while switching back and forth from `dev` and `build`.  

```
thread '<unnamed>' panicked at turbopack/crates/turbo-persistence/src/db.rs:798:17:
attempt to subtract with overflow
stack backtrace:
   0: __rustc::rust_begin_unwind
   1: core::panicking::panic_fmt
   2: core::panicking::panic_const::panic_const_sub_overflow
   3: <core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}> as core::iter::traits::iterator::Iterator>::fold::<(), core::iter::traits::iterator::Iterator::for_each::call<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>, <alloc::vec::Vec<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>>>::extend_trusted<core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}>>::{closure#0}>::{closure#0}>
   4: <alloc::vec::Vec<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>, core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}>>>::from_iter
   5: <turbo_persistence::db::TurboPersistence>::compact
```

Scenario:
 * build turbopack and next
 * cd bench/module-cost
 * enable persistent caching
 * pnpm i
 * pnpm dev-turbopack
 * load localhost:3000 and click all the buttons
 * pnpm build-turbopack
 * pnpm dev-turbopack
 * load localhost:3000 and see the crash

## What?

The `get_merge_segments` function and its caller had a different idea about what `max_merge_segment_count` means.  Unify how this is accounted to avoid underflow.
parent 6881843a
...@@ -200,10 +200,13 @@ fn total_duplication_size(duplication: &IntervalMap<Option<DuplicationInfo>>) -> ...@@ -200,10 +200,13 @@ fn total_duplication_size(duplication: &IntervalMap<Option<DuplicationInfo>>) ->
type MergeSegments = Vec<SmallVec<[usize; 1]>>; type MergeSegments = Vec<SmallVec<[usize; 1]>>;
/// Computes the set of merge segments that should be run to compact the given compactables.
///
/// Returns both the mergeable segments and the actual number of merge segments that were created.
pub fn get_merge_segments<T: Compactable>( pub fn get_merge_segments<T: Compactable>(
compactables: &[T], compactables: &[T],
config: &CompactConfig, config: &CompactConfig,
) -> MergeSegments { ) -> (MergeSegments, usize) {
// Process all compactables in reverse order. // Process all compactables in reverse order.
// For each compactable, find the smallest set of compactables that overlaps with it and matches // For each compactable, find the smallest set of compactables that overlaps with it and matches
// the conditions. // the conditions.
...@@ -259,7 +262,7 @@ pub fn get_merge_segments<T: Compactable>( ...@@ -259,7 +262,7 @@ pub fn get_merge_segments<T: Compactable>(
continue 'outer; continue 'outer;
} }
// If we are limited by size or count, we might also crate a merge segment if it's // If we are limited by size or count, we might also create a merge segment if it's
// within the limits. // within the limits.
let valid_merge_job = current_set.len() >= config.min_merge_count let valid_merge_job = current_set.len() >= config.min_merge_count
&& duplication_size >= config.min_merge_duplication_bytes; && duplication_size >= config.min_merge_duplication_bytes;
...@@ -361,7 +364,7 @@ pub fn get_merge_segments<T: Compactable>( ...@@ -361,7 +364,7 @@ pub fn get_merge_segments<T: Compactable>(
true true
}); });
merge_segments (merge_segments, real_merge_segments)
} }
#[cfg(test)] #[cfg(test)]
...@@ -398,7 +401,7 @@ mod tests { ...@@ -398,7 +401,7 @@ mod tests {
.into_iter() .into_iter()
.map(|range| TestCompactable { range, size: 100 }) .map(|range| TestCompactable { range, size: 100 })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let jobs = get_merge_segments(&compactables, config); let (jobs, _) = get_merge_segments(&compactables, config);
jobs.into_iter() jobs.into_iter()
.map(|job| job.into_iter().collect()) .map(|job| job.into_iter().collect())
.collect() .collect()
...@@ -613,7 +616,7 @@ mod tests { ...@@ -613,7 +616,7 @@ mod tests {
optimal_merge_duplication_bytes: 500, optimal_merge_duplication_bytes: 500,
max_merge_segment_count: 4, max_merge_segment_count: 4,
}; };
let jobs = get_merge_segments(&containers, &config); let (jobs, _) = get_merge_segments(&containers, &config);
if !jobs.is_empty() { if !jobs.is_empty() {
println!("{jobs:?}"); println!("{jobs:?}");
......
...@@ -794,8 +794,9 @@ impl TurboPersistence { ...@@ -794,8 +794,9 @@ impl TurboPersistence {
if compact_config.max_merge_segment_count == 0 { if compact_config.max_merge_segment_count == 0 {
return Vec::new(); return Vec::new();
} }
let merge_jobs = get_merge_segments(ssts_with_ranges, &compact_config); let (merge_jobs, real_merge_job_size) =
compact_config.max_merge_segment_count -= merge_jobs.len(); get_merge_segments(ssts_with_ranges, &compact_config);
compact_config.max_merge_segment_count -= real_merge_job_size;
merge_jobs merge_jobs
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment