Unverified Commit d3303229 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: ensure max merge segments is respected accros families (#82223)

### What?

We try to limit the maximum number of merge jobs with the `max_merge_segment_count` option. But as we run `get_merge_segments` for every key family it only limits the merge jobs per family. This results in much more merge jobs then configured.

This change applied the limits across families. Lower number families will be compacted first, which is a new behavior, so maybe we want to reorder the key families to improve the order (see enum KeySpace)
parent b8eba4f7
......@@ -104,6 +104,7 @@ pub fn compute_metrics<T: Compactable>(
}
/// Configuration for the compaction algorithm.
#[derive(Clone)]
pub struct CompactConfig {
/// The minimum number of files to merge at once.
pub min_merge_count: usize,
......
......@@ -787,16 +787,28 @@ impl TurboPersistence {
keys_written: u64,
}
let mut compact_config = compact_config.clone();
let merge_jobs = sst_by_family
.iter()
.map(|ssts_with_ranges| {
if compact_config.max_merge_segment_count == 0 {
return Vec::new();
}
let merge_jobs = get_merge_segments(ssts_with_ranges, &compact_config);
compact_config.max_merge_segment_count -= merge_jobs.len();
merge_jobs
})
.collect::<Vec<_>>();
let result = sst_by_family
.into_par_iter()
.zip(merge_jobs.into_par_iter())
.with_min_len(1)
.enumerate()
.map(|(family, ssts_with_ranges)| {
.map(|(family, (ssts_with_ranges, merge_jobs))| {
let family = family as u32;
let _span = span.clone().entered();
let merge_jobs = get_merge_segments(&ssts_with_ranges, compact_config);
if merge_jobs.is_empty() {
return Ok(PartialResultPerFamily {
new_meta_file: None,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment