Unverified Commit 61f08530 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Fix a crash under turbopackPersistentCaching (#82592)

Address a crash found at head when testing caching while switching back and forth from `dev` and `build`.  

```
thread '<unnamed>' panicked at turbopack/crates/turbo-persistence/src/db.rs:798:17:
attempt to subtract with overflow
stack backtrace:
   0: __rustc::rust_begin_unwind
   1: core::panicking::panic_fmt
   2: core::panicking::panic_const::panic_const_sub_overflow
   3: <core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}> as core::iter::traits::iterator::Iterator>::fold::<(), core::iter::traits::iterator::Iterator::for_each::call<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>, <alloc::vec::Vec<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>>>::extend_trusted<core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}>>::{closure#0}>::{closure#0}>
   4: <alloc::vec::Vec<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>> as alloc::vec::spec_from_iter_nested::SpecFromIterNested<alloc::vec::Vec<smallvec::SmallVec<[usize; 1]>>, core::iter::adapters::map::Map<core::slice::iter::Iter<alloc::vec::Vec<<turbo_persistence::db::TurboPersistence>::compact_internal::SstWithRange>>, <turbo_persistence::db::TurboPersistence>::compact_internal::{closure#2}>>>::from_iter
   5: <turbo_persistence::db::TurboPersistence>::compact
```

Scenario:
 * build turbopack and next
 * cd bench/module-cost
 * enable persistent caching
 * pnpm i
 * pnpm dev-turbopack
 * load localhost:3000 and click all the buttons
 * pnpm build-turbopack
 * pnpm dev-turbopack
 * load localhost:3000 and see the crash

## What?

The `get_merge_segments` function and its caller had a different idea about what `max_merge_segment_count` means.  Unify how this is accounted to avoid underflow.
parent 6881843a
......@@ -200,10 +200,13 @@ fn total_duplication_size(duplication: &IntervalMap<Option<DuplicationInfo>>) ->
type MergeSegments = Vec<SmallVec<[usize; 1]>>;
/// Computes the set of merge segments that should be run to compact the given compactables.
///
/// Returns both the mergeable segments and the actual number of merge segments that were created.
pub fn get_merge_segments<T: Compactable>(
compactables: &[T],
config: &CompactConfig,
) -> MergeSegments {
) -> (MergeSegments, usize) {
// Process all compactables in reverse order.
// For each compactable, find the smallest set of compactables that overlaps with it and matches
// the conditions.
......@@ -259,7 +262,7 @@ pub fn get_merge_segments<T: Compactable>(
continue 'outer;
}
// If we are limited by size or count, we might also crate a merge segment if it's
// If we are limited by size or count, we might also create a merge segment if it's
// within the limits.
let valid_merge_job = current_set.len() >= config.min_merge_count
&& duplication_size >= config.min_merge_duplication_bytes;
......@@ -361,7 +364,7 @@ pub fn get_merge_segments<T: Compactable>(
true
});
merge_segments
(merge_segments, real_merge_segments)
}
#[cfg(test)]
......@@ -398,7 +401,7 @@ mod tests {
.into_iter()
.map(|range| TestCompactable { range, size: 100 })
.collect::<Vec<_>>();
let jobs = get_merge_segments(&compactables, config);
let (jobs, _) = get_merge_segments(&compactables, config);
jobs.into_iter()
.map(|job| job.into_iter().collect())
.collect()
......@@ -613,7 +616,7 @@ mod tests {
optimal_merge_duplication_bytes: 500,
max_merge_segment_count: 4,
};
let jobs = get_merge_segments(&containers, &config);
let (jobs, _) = get_merge_segments(&containers, &config);
if !jobs.is_empty() {
println!("{jobs:?}");
......
......@@ -794,8 +794,9 @@ impl TurboPersistence {
if compact_config.max_merge_segment_count == 0 {
return Vec::new();
}
let merge_jobs = get_merge_segments(ssts_with_ranges, &compact_config);
compact_config.max_merge_segment_count -= merge_jobs.len();
let (merge_jobs, real_merge_job_size) =
get_merge_segments(ssts_with_ranges, &compact_config);
compact_config.max_merge_segment_count -= real_merge_job_size;
merge_jobs
})
.collect::<Vec<_>>();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment