Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
b05b6601
Unverified
Commit
b05b6601
authored
Sep 16, 2026
by
vercel-fleet-prod[bot]
Committed by
GitHub
Sep 16, 2026
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Switch LZ4 compression to lz4 crate
Co-authored-by:
Tobias Koppers
<
1365881+sokra@users.noreply.github.com
>
parent
c7094123
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
76 additions
and
38 deletions
+76
-38
Cargo.lock
Cargo.lock
+19
-9
Cargo.toml
Cargo.toml
+1
-1
setup-wasi-env.sh
scripts/setup-wasi-env.sh
+1
-1
Cargo.toml
turbopack/crates/turbo-persistence/Cargo.toml
+1
-1
sst_inspect.rs
turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs
+11
-4
compression.rs
turbopack/crates/turbo-persistence/src/compression.rs
+31
-9
static_sorted_file_builder.rs
...rates/turbo-persistence/src/static_sorted_file_builder.rs
+7
-5
write_batch.rs
turbopack/crates/turbo-persistence/src/write_batch.rs
+2
-1
Cargo.toml
turbopack/crates/turbo-tasks-backend/Cargo.toml
+2
-2
mod.rs
turbopack/crates/turbo-tasks-backend/src/backend/mod.rs
+1
-5
No files found.
Cargo.lock
View file @
b05b6601
...
@@ -4275,21 +4275,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
...
@@ -4275,21 +4275,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
[[package]]
name = "lz4
_flex
"
name = "lz4"
version = "
0.11.3
"
version = "
1.28.1
"
source = "registry+https://github.com/rust-lang/crates.io-index"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "
75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5
"
checksum = "
a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4
"
dependencies = [
dependencies = [
"
twox-hash
",
"
lz4-sys
",
]
]
[[package]]
[[package]]
name = "lz
zzz
"
name = "lz
4-sys
"
version = "
2.
0.0"
version = "
1.11.1+lz4-1.1
0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "
fb42ba2edf1f0491b2e2c85fd49cf2a7adad3cab92ea0e29bff82d6fdafcefda
"
checksum = "
6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6
"
dependencies = [
dependencies = [
"cc",
"cc",
"libc",
]
[[package]]
name = "lz4_flex"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
dependencies = [
"twox-hash",
]
]
[[package]]
[[package]]
...
@@ -9971,7 +9981,7 @@ dependencies = [
...
@@ -9971,7 +9981,7 @@ dependencies = [
"either",
"either",
"fs-err",
"fs-err",
"jiff",
"jiff",
"lz
zzz
",
"lz
4
",
"memmap2 0.9.5",
"memmap2 0.9.5",
"nohash-hasher",
"nohash-hasher",
"parking_lot",
"parking_lot",
...
@@ -10091,7 +10101,7 @@ dependencies = [
...
@@ -10091,7 +10101,7 @@ dependencies = [
"indexmap 2.14.0",
"indexmap 2.14.0",
"indoc",
"indoc",
"jiff",
"jiff",
"lz
zzz
",
"lz
4
",
"parking_lot",
"parking_lot",
"rand 0.10.1",
"rand 0.10.1",
"regex",
"regex",
...
...
Cargo.toml
View file @
b05b6601
...
@@ -291,7 +291,7 @@ lightningcss = { version = "1.0.0-alpha.72", features = [
...
@@ -291,7 +291,7 @@ lightningcss = { version = "1.0.0-alpha.72", features = [
"into_owned"
,
"into_owned"
,
"browserslist"
,
"browserslist"
,
]
}
]
}
lz
zzz
=
"2.0.0
"
lz
4
=
"1.28.1
"
markdown
=
"1.0.0-alpha.18"
markdown
=
"1.0.0-alpha.18"
memchr
=
"2.8.0"
memchr
=
"2.8.0"
mime
=
"0.3.16"
mime
=
"0.3.16"
...
...
scripts/setup-wasi-env.sh
View file @
b05b6601
...
@@ -32,7 +32,7 @@ setup_wasi_env() {
...
@@ -32,7 +32,7 @@ setup_wasi_env() {
rustup target add wasm32-wasip1-threads
||
return
1
rustup target add wasm32-wasip1-threads
||
return
1
# `lz
zzz` (LZ4,
via turbo-persistence) and `zstd-sys` have C build scripts, so a WASI clang and
# `lz
4-sys` (
via turbo-persistence) and `zstd-sys` have C build scripts, so a WASI clang and
# sysroot are required on top of the Rust target. The SDK build must match the host running the
# sysroot are required on top of the Rust target. The SDK build must match the host running the
# compiler; an x86_64 clang on arm64 fails with "Exec format error".
# compiler; an x86_64 clang on arm64 fails with "Exec format error".
local
wasi_sdk_version
=
33.0 wasi_sdk_arch wasi_sdk_sha256
local
wasi_sdk_version
=
33.0 wasi_sdk_arch wasi_sdk_sha256
...
...
turbopack/crates/turbo-persistence/Cargo.toml
View file @
b05b6601
...
@@ -22,7 +22,7 @@ dashmap = { workspace = true}
...
@@ -22,7 +22,7 @@ dashmap = { workspace = true}
either
=
{
workspace
=
true
}
either
=
{
workspace
=
true
}
fs-err
=
{
workspace
=
true
}
fs-err
=
{
workspace
=
true
}
jiff
=
{
version
=
"0.2.10"
,
features
=
["serde"]
}
jiff
=
{
version
=
"0.2.10"
,
features
=
["serde"]
}
lz
zzz
=
{
workspace
=
true
}
lz
4
=
{
workspace
=
true
}
memmap2
=
"0.9.5"
memmap2
=
"0.9.5"
nohash-hasher
=
{
workspace
=
true
}
nohash-hasher
=
{
workspace
=
true
}
parking_lot
=
{
workspace
=
true
}
parking_lot
=
{
workspace
=
true
}
...
...
turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs
View file @
b05b6601
...
@@ -17,7 +17,7 @@ use std::{
...
@@ -17,7 +17,7 @@ use std::{
use
anyhow
::{
Context
,
Result
,
bail
};
use
anyhow
::{
Context
,
Result
,
bail
};
use
byteorder
::{
BE
,
ReadBytesExt
};
use
byteorder
::{
BE
,
ReadBytesExt
};
use
fs_err
::{
self
as
fs
,
File
};
use
fs_err
::{
self
as
fs
,
File
};
use
lz
zzz
::
lz4
::
decompress
;
use
lz
4
::
block
::
decompress_to_buffer
;
use
memmap2
::
Mmap
;
use
memmap2
::
Mmap
;
use
turbo_persistence
::{
use
turbo_persistence
::{
BLOCK_HEADER_SIZE
,
Compression
,
MAX_INLINE_VALUE_SIZE
,
checksum_block
,
BLOCK_HEADER_SIZE
,
Compression
,
MAX_INLINE_VALUE_SIZE
,
checksum_block
,
...
@@ -348,9 +348,16 @@ fn read_block(
...
@@ -348,9 +348,16 @@ fn read_block(
let
data
=
if
was_compressed
{
let
data
=
if
was_compressed
{
let
mut
buffer
=
vec!
[
0u8
;
uncompressed_length
as
usize
];
let
mut
buffer
=
vec!
[
0u8
;
uncompressed_length
as
usize
];
let
bytes_written
=
match
compression
{
let
bytes_written
=
match
compression
{
Compression
::
Lz4
=>
{
Compression
::
Lz4
=>
decompress_to_buffer
(
decompress
(
compressed_data
,
&
mut
buffer
)
.context
(
"LZ4 decompression failed"
)
?
compressed_data
,
}
Some
(
uncompressed_length
.try_into
()
.context
(
"LZ4 uncompressed length exceeds i32::MAX"
)
?
,
),
&
mut
buffer
,
)
.context
(
"LZ4 decompression failed"
)
?
,
Compression
::
Zstd3
=>
zstd
::
bulk
::
decompress_to_buffer
(
compressed_data
,
&
mut
buffer
)
Compression
::
Zstd3
=>
zstd
::
bulk
::
decompress_to_buffer
(
compressed_data
,
&
mut
buffer
)
.context
(
"zstd decompression failed"
)
?
,
.context
(
"zstd decompression failed"
)
?
,
};
};
...
...
turbopack/crates/turbo-persistence/src/compression.rs
View file @
b05b6601
use
std
::{
cell
::
RefCell
,
mem
::
MaybeUninit
,
rc
::
Rc
,
sync
::
Arc
};
use
std
::{
cell
::
RefCell
,
mem
::
MaybeUninit
,
rc
::
Rc
,
sync
::
Arc
};
use
anyhow
::{
Context
,
Result
,
ensure
};
use
anyhow
::{
Context
,
Result
,
ensure
};
use
lz
zzz
::
lz4
::{
self
,
decompress
}
;
use
lz
4
::
block
as
lz4
;
/// Compression algorithm used for a family's SST blocks and blob values.
/// Compression algorithm used for a family's SST blocks and blob values.
#[derive(Clone,
Copy,
Debug,
Default,
PartialEq,
Eq)]
#[derive(Clone,
Copy,
Debug,
Default,
PartialEq,
Eq)]
...
@@ -35,7 +35,16 @@ fn decompress_block(
...
@@ -35,7 +35,16 @@ fn decompress_block(
zero-copy mmap path"
zero-copy mmap path"
);
);
let
bytes_written
=
match
compression
{
let
bytes_written
=
match
compression
{
Compression
::
Lz4
=>
decompress
(
block
,
dest
)
.map_err
(
anyhow
::
Error
::
from
),
Compression
::
Lz4
=>
lz4
::
decompress_to_buffer
(
block
,
Some
(
expected_len
.try_into
()
.context
(
"LZ4 uncompressed length exceeds i32::MAX"
)
?
,
),
dest
,
)
.map_err
(
anyhow
::
Error
::
from
),
Compression
::
Zstd3
=>
ZSTD_DECOMPRESSOR
.with_borrow_mut
(|
decompressor
|
{
Compression
::
Zstd3
=>
ZSTD_DECOMPRESSOR
.with_borrow_mut
(|
decompressor
|
{
decompressor
decompressor
.decompress_to_buffer
(
block
,
dest
)
.decompress_to_buffer
(
block
,
dest
)
...
@@ -116,27 +125,35 @@ impl Compressor {
...
@@ -116,27 +125,35 @@ impl Compressor {
Ok
(
Self
{
compression
,
zstd
})
Ok
(
Self
{
compression
,
zstd
})
}
}
/// Compresses `block` into the start of `buffer` and returns the compressed length.
///
/// The buffer may be longer than the returned length. Keeping the initialized LZ4 scratch
/// space avoids zero-filling the destination again on every block.
#[tracing
::
instrument(level
=
"trace"
,
skip_all)]
#[tracing
::
instrument(level
=
"trace"
,
skip_all)]
pub
(
crate
)
fn
compress_into_buffer
(
pub
(
crate
)
fn
compress_into_buffer
(
&
mut
self
,
&
mut
self
,
block
:
&
[
u8
],
block
:
&
[
u8
],
buffer
:
&
mut
Vec
<
u8
>
,
buffer
:
&
mut
Vec
<
u8
>
,
)
->
Result
<
()
>
{
)
->
Result
<
usize
>
{
match
self
.compression
{
match
self
.compression
{
Compression
::
Lz4
=>
{
Compression
::
Lz4
=>
{
lz4
::
compress_to_vec
(
block
,
buffer
,
lz4
::
ACC_LEVEL_DEFAULT
)
let
bound
=
lz4
::
compress_bound
(
block
.len
())
.context
(
"LZ4 input is too large"
)
?
;
.context
(
"LZ4 compression failed"
)
?
;
if
buffer
.len
()
<
bound
{
buffer
.resize
(
bound
,
0
);
}
lz4
::
compress_to_buffer
(
block
,
None
,
false
,
&
mut
buffer
[
..
bound
])
.context
(
"LZ4 compression failed"
)
}
}
Compression
::
Zstd3
=>
{
Compression
::
Zstd3
=>
{
buffer
.clear
();
buffer
.reserve
(
zstd
::
zstd_safe
::
compress_bound
(
block
.len
()));
buffer
.reserve
(
zstd
::
zstd_safe
::
compress_bound
(
block
.len
()));
self
.zstd
self
.zstd
.as_mut
()
.as_mut
()
.expect
(
"zstd compressor not initialized"
)
.expect
(
"zstd compressor not initialized"
)
.compress_to_buffer
(
block
,
buffer
)
.compress_to_buffer
(
block
,
buffer
)
.context
(
"zstd compression failed"
)
?
;
.context
(
"zstd compression failed"
)
}
}
}
}
Ok
(())
}
}
}
}
...
@@ -150,10 +167,15 @@ mod tests {
...
@@ -150,10 +167,15 @@ mod tests {
for
compression
in
[
Compression
::
Lz4
,
Compression
::
Zstd3
]
{
for
compression
in
[
Compression
::
Lz4
,
Compression
::
Zstd3
]
{
let
mut
compressor
=
Compressor
::
new
(
compression
)
.unwrap
();
let
mut
compressor
=
Compressor
::
new
(
compression
)
.unwrap
();
let
mut
compressed
=
Vec
::
new
();
let
mut
compressed
=
Vec
::
new
();
compressor
let
compressed_len
=
compressor
.compress_into_buffer
(
&
input
,
&
mut
compressed
)
.compress_into_buffer
(
&
input
,
&
mut
compressed
)
.unwrap
();
.unwrap
();
let
output
=
decompress_into_arc
(
compression
,
input
.len
()
as
u32
,
&
compressed
)
.unwrap
();
let
output
=
decompress_into_arc
(
compression
,
input
.len
()
as
u32
,
&
compressed
[
..
compressed_len
],
)
.unwrap
();
assert_eq!
(
&*
output
,
input
);
assert_eq!
(
&*
output
,
input
);
}
}
}
}
...
...
turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs
View file @
b05b6601
...
@@ -398,10 +398,13 @@ fn write_block_to_file(
...
@@ -398,10 +398,13 @@ fn write_block_to_file(
compressor
:
&
mut
Compressor
,
compressor
:
&
mut
Compressor
,
)
->
Result
<
u16
>
{
)
->
Result
<
u16
>
{
let
(
uncompressed_size
,
data_to_write
):
(
u32
,
&
[
u8
])
=
if
try_compress
{
let
(
uncompressed_size
,
data_to_write
):
(
u32
,
&
[
u8
])
=
if
try_compress
{
compressor
.compress_into_buffer
(
block
,
compress_buffer
)
?
;
let
compressed_len
=
compressor
.compress_into_buffer
(
block
,
compress_buffer
)
?
;
// Same threshold as LevelDB/RocksDB: require at least 12.5% savings.
// Same threshold as LevelDB/RocksDB: require at least 12.5% savings.
if
compress_buffer
.len
()
<
block
.len
()
-
(
block
.len
()
/
8
)
{
if
compressed_len
<
block
.len
()
-
(
block
.len
()
/
8
)
{
(
block
.len
()
.try_into
()
.unwrap
(),
compress_buffer
.as_slice
())
(
block
.len
()
.try_into
()
.unwrap
(),
&
compress_buffer
[
..
compressed_len
],
)
}
else
{
}
else
{
(
0
,
block
)
(
0
,
block
)
}
}
...
@@ -419,7 +422,6 @@ fn write_block_to_file(
...
@@ -419,7 +422,6 @@ fn write_block_to_file(
checksum
,
checksum
,
data_to_write
,
data_to_write
,
);
);
compress_buffer
.clear
();
result
result
}
}
...
@@ -1975,7 +1977,7 @@ mod tests {
...
@@ -1975,7 +1977,7 @@ mod tests {
body
.to_vec
()
body
.to_vec
()
}
else
{
}
else
{
let
mut
out
=
vec!
[
0u8
;
uncompressed_size
];
let
mut
out
=
vec!
[
0u8
;
uncompressed_size
];
lz
zzz
::
lz4
::
decompress
(
body
,
&
mut
out
)
?
;
lz
4
::
block
::
decompress_to_buffer
(
body
,
Some
(
uncompressed_size
.try_into
()
?
)
,
&
mut
out
)
?
;
out
out
})
})
}
}
...
...
turbopack/crates/turbo-persistence/src/write_batch.rs
View file @
b05b6601
...
@@ -488,9 +488,10 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
...
@@ -488,9 +488,10 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
let
seq
=
self
.current_sequence_number
.fetch_add
(
1
,
Ordering
::
SeqCst
)
+
1
;
let
seq
=
self
.current_sequence_number
.fetch_add
(
1
,
Ordering
::
SeqCst
)
+
1
;
let
mut
compressed
=
Vec
::
new
();
let
mut
compressed
=
Vec
::
new
();
let
compression
=
self
.family_configs
[
usize_from_u32
(
family
)]
.compression
;
let
compression
=
self
.family_configs
[
usize_from_u32
(
family
)]
.compression
;
Compressor
::
new
(
compression
)
?
let
compressed_len
=
Compressor
::
new
(
compression
)
?
.compress_into_buffer
(
value
,
&
mut
compressed
)
.compress_into_buffer
(
value
,
&
mut
compressed
)
.context
(
"Compression of value for blob file failed"
)
?
;
.context
(
"Compression of value for blob file failed"
)
?
;
compressed
.truncate
(
compressed_len
);
let
mut
buffer
=
Vec
::
with_capacity
(
8
+
compressed
.len
());
let
mut
buffer
=
Vec
::
with_capacity
(
8
+
compressed
.len
());
buffer
.write_u32
::
<
BE
>
(
value
.len
()
as
u32
)
?
;
buffer
.write_u32
::
<
BE
>
(
value
.len
()
as
u32
)
?
;
...
...
turbopack/crates/turbo-tasks-backend/Cargo.toml
View file @
b05b6601
...
@@ -14,7 +14,7 @@ workspace = true
...
@@ -14,7 +14,7 @@ workspace = true
[features]
[features]
default
=
[]
default
=
[]
print_cache_item_size_with_compressed
=
[
"print_cache_item_size"
,
"dep:lz
zzz
"
]
print_cache_item_size_with_compressed
=
[
"print_cache_item_size"
,
"dep:lz
4
"
]
print_cache_item_size
=
[]
print_cache_item_size
=
[]
no_fast_stale
=
[]
no_fast_stale
=
[]
aggregation_update_no_batch
=
[]
aggregation_update_no_batch
=
[]
...
@@ -47,7 +47,7 @@ fs-err = { workspace = true }
...
@@ -47,7 +47,7 @@ fs-err = { workspace = true }
hashbrown
=
{
workspace
=
true
}
hashbrown
=
{
workspace
=
true
}
indexmap
=
{
workspace
=
true
}
indexmap
=
{
workspace
=
true
}
jiff
=
"0.2.10"
jiff
=
"0.2.10"
lz
zzz
=
{
workspace
=
true
,
optional
=
true
}
lz
4
=
{
workspace
=
true
,
optional
=
true
}
parking_lot
=
{
workspace
=
true
}
parking_lot
=
{
workspace
=
true
}
rand
=
{
workspace
=
true
}
rand
=
{
workspace
=
true
}
ringmap
=
{
workspace
=
true
,
features
=
["serde"]
}
ringmap
=
{
workspace
=
true
,
features
=
["serde"]
}
...
...
turbopack/crates/turbo-tasks-backend/src/backend/mod.rs
View file @
b05b6601
...
@@ -1289,11 +1289,7 @@ impl TurboTasksBackend {
...
@@ -1289,11 +1289,7 @@ impl TurboTasksBackend {
impl
TaskCacheStats
{
impl
TaskCacheStats
{
#[cfg(feature
=
"print_cache_item_size_with_compressed"
)]
#[cfg(feature
=
"print_cache_item_size_with_compressed"
)]
fn
compressed_size
(
data
:
&
[
u8
])
->
Result
<
usize
>
{
fn
compressed_size
(
data
:
&
[
u8
])
->
Result
<
usize
>
{
Ok
(
lzzzz
::
lz4
::
Compressor
::
new
()
?
.next_to_vec
(
Ok
(
lz4
::
block
::
compress
(
data
,
None
,
false
)
?
.len
())
data
,
&
mut
Vec
::
new
(),
lzzzz
::
lz4
::
ACC_LEVEL_DEFAULT
,
)
?
)
}
}
fn
add_data
(
&
mut
self
,
data
:
&
[
u8
])
{
fn
add_data
(
&
mut
self
,
data
:
&
[
u8
])
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment