Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
05482087
Unverified
Commit
05482087
authored
Oct 16, 2026
by
Luke Sandberg
Committed by
GitHub
Oct 20, 2026
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
turbo-persistence: add tests for MergeIter
Also note why a tournament tree of losers was tried and not kept.
parent
4550be40
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
176 additions
and
0 deletions
+176
-0
merge_iter.rs
turbopack/crates/turbo-persistence/src/merge_iter.rs
+176
-0
No files found.
turbopack/crates/turbo-persistence/src/merge_iter.rs
View file @
05482087
...
...
@@ -53,6 +53,13 @@ impl<T: Iterator<Item = Result<LookupEntry>>> Ord for HeapNode<T> {
/// An iterator that merges multiple sorted iterators into a single sorted iterator. Internally it
/// uses a heap of iterators to iterate them in order.
///
/// A tournament tree of losers was tried instead (fewer comparisons per entry, and branchless with
/// `select_unpredictable`), but it wasn't worth it: about 1.5% faster on a full compaction of 32
/// equally sized, interleaved inputs of tiny entries, its best case, and no change on realistic
/// data. Matches nearly always compare two hashes, which `HeapNode` keeps inline, so they are cheap
/// either way, and in bottom merges, which are dominated by the bottom run, the heap's sift-down
/// usually stops at the root.
pub
struct
MergeIter
<
T
:
Iterator
<
Item
=
Result
<
LookupEntry
>>>
{
heap
:
BinaryHeap
<
HeapNode
<
T
>>
,
}
...
...
@@ -100,3 +107,172 @@ impl<T: Iterator<Item = Result<LookupEntry>>> Iterator for MergeIter<T> {
}
}
}
#[cfg(test)]
mod
tests
{
use
rand
::{
RngExt
,
SeedableRng
,
rngs
::
SmallRng
};
use
super
::
*
;
use
crate
::{
lookup_entry
::
IterValue
,
rc_bytes
::
RcBytes
};
/// An entry tagged with the input it came from and its position in that input.
fn
entry
(
hash
:
u64
,
key
:
&
[
u8
],
input
:
u32
,
position
:
u32
)
->
LookupEntry
{
LookupEntry
{
hash
,
key
:
RcBytes
::
from
(
Box
::
<
[
u8
]
>
::
from
(
key
)),
value
:
IterValue
::
Blob
{
sequence_number
:
input
<<
16
|
position
,
},
}
}
fn
tag
(
entry
:
&
LookupEntry
)
->
(
u64
,
Vec
<
u8
>
,
u32
,
u32
)
{
let
IterValue
::
Blob
{
sequence_number
}
=
entry
.value
else
{
panic!
(
"test entries are blobs"
);
};
(
entry
.hash
,
entry
.key
.to_vec
(),
sequence_number
>>
16
,
sequence_number
&
0xffff
,
)
}
type
Input
=
std
::
vec
::
IntoIter
<
Result
<
LookupEntry
>>
;
fn
input
(
index
:
u32
,
entries
:
&
[(
u64
,
&
[
u8
])])
->
Input
{
entries
.iter
()
.enumerate
()
.map
(|(
position
,
&
(
hash
,
key
))|
Ok
(
entry
(
hash
,
key
,
index
,
position
as
u32
)))
.collect
::
<
Vec
<
_
>>
()
.into_iter
()
}
fn
merge
(
inputs
:
Vec
<
Input
>
)
->
Vec
<
(
u64
,
Vec
<
u8
>
,
u32
,
u32
)
>
{
MergeIter
::
new
(
inputs
.into_iter
())
.unwrap
()
.map
(|
entry
|
tag
(
&
entry
.unwrap
()))
.collect
()
}
#[test]
fn
test_empty
()
{
assert
!
(
merge
(
vec!
[])
.is_empty
());
assert
!
(
merge
(
vec!
[
input
(
0
,
&
[]),
input
(
1
,
&
[])])
.is_empty
());
}
#[test]
fn
test_single_input
()
{
let
merged
=
merge
(
vec!
[
input
(
0
,
&
[(
1
,
b
"a"
),
(
2
,
b
"b"
),
(
3
,
b
"c"
)])]);
assert_eq!
(
merged
.iter
()
.map
(|
e
|
e
.
0
)
.collect
::
<
Vec
<
_
>>
(),
vec!
[
1
,
2
,
3
]
);
}
#[test]
fn
test_orders_by_hash_then_key
()
{
let
merged
=
merge
(
vec!
[
input
(
0
,
&
[(
1
,
b
"b"
),
(
3
,
b
"a"
)]),
input
(
1
,
&
[(
1
,
b
"a"
),
(
2
,
b
"z"
)]),
]);
assert_eq!
(
merged
.iter
()
.map
(|
e
|
(
e
.
0
,
e
.
1
.clone
()))
.collect
::
<
Vec
<
_
>>
(),
vec!
[
(
1
,
b
"a"
.to_vec
()),
(
1
,
b
"b"
.to_vec
()),
(
2
,
b
"z"
.to_vec
()),
(
3
,
b
"a"
.to_vec
())
]
);
}
#[test]
fn
test_equal_keys_come_from_later_inputs_first
()
{
// Input 2 holds two entries of the key (a MultiValue group), which stay together and in
// their order, before those of the older inputs.
let
merged
=
merge
(
vec!
[
input
(
0
,
&
[(
5
,
b
"k"
)]),
input
(
1
,
&
[(
5
,
b
"k"
)]),
input
(
2
,
&
[(
5
,
b
"k"
),
(
5
,
b
"k"
)]),
]);
assert_eq!
(
merged
.iter
()
.map
(|
e
|
(
e
.
2
,
e
.
3
))
.collect
::
<
Vec
<
_
>>
(),
vec!
[(
2
,
0
),
(
2
,
1
),
(
1
,
0
),
(
0
,
0
)]
);
}
#[test]
fn
test_error_drops_the_failing_input
()
{
let
failing
=
vec!
[
Ok
(
entry
(
1
,
b
"a"
,
0
,
0
)),
Err
(
anyhow
::
anyhow!
(
"broken"
)),
Ok
(
entry
(
9
,
b
"z"
,
0
,
2
)),
]
.into_iter
();
let
mut
merge
=
MergeIter
::
new
(
vec!
[
failing
,
input
(
1
,
&
[(
2
,
b
"b"
),
(
3
,
b
"c"
)])]
.into_iter
())
.unwrap
();
// Input 0 wins, but advancing it fails, which drops the input with the entry it was
// holding.
assert
!
(
merge
.next
()
.unwrap
()
.is_err
());
let
rest
=
merge
.map
(|
e
|
tag
(
&
e
.unwrap
())
.
0
)
.collect
::
<
Vec
<
_
>>
();
assert_eq!
(
rest
,
vec!
[
2
,
3
]);
}
/// Compares against sorting all entries by `(hash, key, newest input, position)`, for input
/// counts that are and aren't powers of two.
#[test]
fn
test_matches_sorting
()
{
let
mut
rng
=
SmallRng
::
seed_from_u64
(
7
);
for
input_count
in
1
..=
19u32
{
for
_
in
0
..
10
{
let
inputs
=
(
0
..
input_count
)
.map
(|
index
|
{
let
len
=
rng
.random_range
(
0
..
40
);
// Few distinct hashes and keys, so that equal hashes and equal keys are
// common.
let
mut
entries
=
(
0
..
len
)
.map
(|
_
|
(
rng
.random_range
(
0
..
16u64
),
[
rng
.random_range
(
b
'a'
..
b
'e'
)]))
.collect
::
<
Vec
<
_
>>
();
entries
.sort
();
(
index
,
entries
)
})
.collect
::
<
Vec
<
_
>>
();
let
mut
expected
=
inputs
.iter
()
.flat_map
(|(
index
,
entries
)|
{
entries
.iter
()
.enumerate
()
.map
(
move
|(
position
,
(
hash
,
key
))|
{
(
*
hash
,
key
.to_vec
(),
*
index
,
position
as
u32
)
})
})
.collect
::
<
Vec
<
_
>>
();
expected
.sort_by
(|
a
,
b
|
{
(
a
.
0
,
&
a
.
1
)
.cmp
(
&
(
b
.
0
,
&
b
.
1
))
.then
(
b
.
2
.cmp
(
&
a
.
2
))
.then
(
a
.
3
.cmp
(
&
b
.
3
))
});
let
merged
=
merge
(
inputs
.iter
()
.map
(|(
index
,
entries
)|
{
let
entries
=
entries
.iter
()
.map
(|(
hash
,
key
)|
(
*
hash
,
&
key
[
..
]))
.collect
::
<
Vec
<
_
>>
();
input
(
*
index
,
&
entries
)
})
.collect
(),
);
assert_eq!
(
merged
,
expected
,
"{input_count} inputs"
);
}
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment