Unverified Commit b26b2b71 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Add a custom serde_self_describing Serializer/Deserializer (#86885)

Implements a minimal custom serde `Serializer` and `Deserializer` on top of bincode with a custom self-describing format. The API is exposed as functions that can be used in `#[bincode(with = "...")]` attributes.

This doesn't encode as efficiently as `pot`, but it lets us get rid of that dependency entirely. It also gives us a more efficient option than `serde_json` for certain values.
parent aaee236c
......@@ -9108,7 +9108,6 @@ dependencies = [
"mime",
"ringmap",
"serde",
"serde_json",
"smallvec",
"unty",
]
......@@ -9135,6 +9134,7 @@ name = "turbo-persistence"
version = "0.1.0"
dependencies = [
"anyhow",
"bincode 2.0.1",
"bitfield",
"byteorder",
"dashmap 6.1.0",
......@@ -9154,6 +9154,7 @@ dependencies = [
"tempfile",
"thread_local",
"tracing",
"turbo-bincode",
"twox-hash 2.1.0",
"zstd",
]
......@@ -9227,7 +9228,6 @@ dependencies = [
"once_cell",
"parking_lot",
"pin-project-lite",
"pot",
"rayon",
"regex",
"rustc-hash 2.1.1",
......
......@@ -92,7 +92,7 @@ pub struct NextConfig {
cache_handler: Option<RcStr>,
#[bincode(with_serde)]
cache_handlers: Option<FxIndexMap<RcStr, RcStr>>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
env: FxIndexMap<String, JsonValue>,
experimental: ExperimentalConfig,
images: ImageConfig,
......@@ -101,12 +101,12 @@ pub struct NextConfig {
react_production_profiling: Option<bool>,
react_strict_mode: Option<bool>,
transpile_packages: Option<Vec<RcStr>>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>,
dist_dir: RcStr,
dist_dir_root: RcStr,
deployment_id: Option<RcStr>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
sass_options: Option<serde_json::Value>,
trailing_slash: Option<bool>,
asset_prefix: Option<RcStr>,
......@@ -119,9 +119,9 @@ pub struct NextConfig {
output: Option<OutputType>,
turbopack: Option<TurbopackConfig>,
production_browser_source_maps: bool,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
output_file_tracing_includes: Option<serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
output_file_tracing_excludes: Option<serde_json::Value>,
// TODO: This option is not respected, it uses Turbopack's root instead.
output_file_tracing_root: Option<RcStr>,
......@@ -154,9 +154,9 @@ pub struct NextConfig {
http_agent_options: HttpAgentConfig,
on_demand_entries: OnDemandEntriesConfig,
powered_by_header: bool,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
public_runtime_config: FxIndexMap<String, serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
server_runtime_config: FxIndexMap<String, serde_json::Value>,
static_page_generation_timeout: f64,
target: Option<String>,
......@@ -686,9 +686,9 @@ pub enum RemotePatternProtocol {
)]
#[serde(rename_all = "camelCase")]
pub struct TurbopackConfig {
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
pub rules: Option<FxIndexMap<RcStr, RuleConfigCollection>>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
pub resolve_extensions: Option<Vec<RcStr>>,
pub debug_ids: Option<bool>,
......@@ -955,7 +955,7 @@ pub struct ExperimentalConfig {
/// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs)
mdx_rs: Option<MdxRsOptions>,
strict_next_head: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
external_middleware_rewrites_resolve: Option<bool>,
scroll_restoration: Option<bool>,
......@@ -964,7 +964,7 @@ pub struct ExperimentalConfig {
middleware_prefetch: Option<MiddlewarePrefetchType>,
/// optimizeCss can be boolean or critters' option object
/// Use Record<string, unknown> as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts))
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
optimize_css: Option<serde_json::Value>,
next_script_workers: Option<bool>,
web_vitals_attribution: Option<Vec<RcStr>>,
......@@ -982,7 +982,7 @@ pub struct ExperimentalConfig {
adjust_font_fallbacks_with_size_adjust: Option<bool>,
after: Option<bool>,
app_document_preloading: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
cache_life: Option<FxIndexMap<String, CacheLifeProfile>>,
case_sensitive_routes: Option<bool>,
cpus: Option<f64>,
......@@ -990,7 +990,7 @@ pub struct ExperimentalConfig {
disable_optimized_loading: Option<bool>,
disable_postcss_preset_env: Option<bool>,
esm_externals: Option<EsmExternals>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
extension_alias: Option<serde_json::Value>,
external_dir: Option<bool>,
/// If set to `false`, webpack won't fall back to polyfill Node.js modules
......@@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig {
instrumentation_hook: Option<bool>,
client_trace_metadata: Option<Vec<String>>,
large_page_data_bytes: Option<f64>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
logging: Option<serde_json::Value>,
memory_based_workers_count: Option<bool>,
/// Optimize React APIs for server builds.
......@@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig {
/// @internal Used by the Next.js internals only.
trust_host_header: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
url_imports: Option<serde_json::Value>,
/// This option is to enable running the Webpack build in a worker thread
/// (doesn't apply to Turbopack).
......@@ -1397,7 +1397,7 @@ pub struct ResolveExtensions(Option<Vec<RcStr>>);
#[turbo_tasks::value(transparent)]
pub struct SwcPlugins(
#[bincode(with = "turbo_bincode::serde_json")] Vec<(RcStr, serde_json::Value)>,
#[bincode(with = "turbo_bincode::serde_self_describing")] Vec<(RcStr, serde_json::Value)>,
);
#[turbo_tasks::value(transparent)]
......@@ -1415,7 +1415,7 @@ pub struct OptionServerActions(Option<ServerActions>);
#[turbo_tasks::value(transparent)]
pub struct OptionJsonValue(
#[bincode(with = "turbo_bincode::serde_json")] pub Option<serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_self_describing")] pub Option<serde_json::Value>,
);
fn turbopack_config_documentation_link() -> RcStr {
......@@ -1468,7 +1468,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue {
#[turbo_tasks::value(shared)]
struct InvalidLoaderRuleConditionIssue {
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
condition: ConfigConditionItem,
config_file_path: FileSystemPath,
}
......@@ -2218,7 +2218,7 @@ impl NextConfig {
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "camelCase")]
pub struct JsConfig {
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
compiler_options: Option<serde_json::Value>,
}
......
......@@ -17,6 +17,5 @@ indexmap = { workspace = true }
mime = { workspace = true }
ringmap = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
smallvec = { workspace = true }
unty = { workspace = true }
#[doc(hidden)]
pub mod macro_helpers;
pub mod serde_self_describing;
use std::{any::Any, ptr::copy_nonoverlapping};
......@@ -472,76 +473,6 @@ pub mod mime_option {
}
}
/// Encode/decode as a serialized string encoded using `serde_json`.
///
/// This encodes less efficiently than `#[bincode(with_serde)]` would, but avoids [bincode's known
/// compatibility issues][serde-issues]. Use this for infrequently-serialized types and when you're
/// unsure if the underlying type may trigger a serde compatibility issue.
///
/// In the future this could be replaced with a more efficient serde-compatible self-describing
/// format with a compact binary representation (e.g. pot or MessagePack), but `serde_json` is
/// convenient because it avoids introducing additional dependencies.
///
/// [serde-issues]: https://docs.rs/bincode/latest/bincode/serde/index.html#known-issues
pub mod serde_json {
use super::*;
pub fn encode<E: Encoder, T: serde::Serialize>(
value: &T,
encoder: &mut E,
) -> Result<(), EncodeError> {
let json_str =
::serde_json::to_string(value).map_err(|e| EncodeError::OtherString(e.to_string()))?;
Encode::encode(&json_str, encoder)
}
pub fn decode<Context, D: Decoder<Context = Context>, T: serde::de::DeserializeOwned>(
decoder: &mut D,
) -> Result<T, DecodeError> {
let json_str: String = Decode::decode(decoder)?;
::serde_json::from_str(&json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
}
pub fn borrow_decode<
'de,
Context,
D: BorrowDecoder<'de, Context = Context>,
T: serde::de::Deserialize<'de>,
>(
decoder: &mut D,
) -> Result<T, DecodeError> {
let json_str: &str = BorrowDecode::borrow_decode(decoder)?;
::serde_json::from_str(json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
}
#[cfg(test)]
mod tests {
use ::serde_json::{Value, json};
use bincode::{decode_from_slice, encode_to_vec};
use super::*;
#[test]
fn test_roundtrip() {
let cfg = bincode::config::standard();
#[derive(Encode, Decode)]
struct Wrapper(#[bincode(with = "crate::serde_json")] Value);
let value1 = Wrapper(json!({
"key1": [1, 2, 3],
"key2": [4, 5, 6]
}));
let value2: Wrapper = decode_from_slice(&encode_to_vec(&value1, cfg).unwrap(), cfg)
.unwrap()
.0;
assert_eq!(value1.0, value2.0);
}
}
}
pub mod either {
use ::either::Either;
......
This diff is collapsed.
//! Helpers for serializing serde-compatible types inside of a bincode [`Encode`] or [`Decode`]
//! implementation using a self-describing format. This works with [types that
//! `#[bincode(serde)]` does not support][bincode::serde#known-issues].
//!
//! These helper functions can be used in the [`Encode`] and [`Decode`] derive macros with the
//! `#[bincode(with = "turbo_bincode]` attribute.
//!
//! [`Encode`]: bincode::Encode
//! [`Decode`]: bincode::Decode
use bincode::{
de::{BorrowDecoder, Decoder},
enc::Encoder,
error::{DecodeError, EncodeError},
};
use serde::{Serialize, de::DeserializeOwned};
mod de;
mod ser;
/// Uses a u8 representation, which is slightly more efficient than bincode's default u32 varint
/// approach for enum discriminants:
/// https://docs.rs/bincode/latest/bincode/spec/index.html#discriminant-representation
#[derive(Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
enum TypeTag {
BoolTrue = 1,
BoolFalse,
U8,
U16,
U32,
U64,
I8,
I16,
I32,
I64,
F32,
F64,
Char,
String,
Bytes,
OptionNone,
OptionSome,
Unit,
UnitStruct,
UnitVariant,
NewtypeStruct,
NewtypeVariant,
SeqSized,
SeqUnsizedStart,
Tuple,
TupleStruct,
TupleVariant,
MapSized,
MapUnsizedStart,
Struct,
StructVariant,
CollectionEnd,
}
impl TryFrom<u8> for TypeTag {
type Error = DecodeError;
fn try_from(num: u8) -> Result<Self, DecodeError> {
let tag = match num {
1 => TypeTag::BoolTrue,
2 => TypeTag::BoolFalse,
3 => TypeTag::U8,
4 => TypeTag::U16,
5 => TypeTag::U32,
6 => TypeTag::U64,
7 => TypeTag::I8,
8 => TypeTag::I16,
9 => TypeTag::I32,
10 => TypeTag::I64,
11 => TypeTag::F32,
12 => TypeTag::F64,
13 => TypeTag::Char,
14 => TypeTag::String,
15 => TypeTag::Bytes,
16 => TypeTag::OptionNone,
17 => TypeTag::OptionSome,
18 => TypeTag::Unit,
19 => TypeTag::UnitStruct,
20 => TypeTag::UnitVariant,
21 => TypeTag::NewtypeStruct,
22 => TypeTag::NewtypeVariant,
23 => TypeTag::SeqSized,
24 => TypeTag::SeqUnsizedStart,
25 => TypeTag::Tuple,
26 => TypeTag::TupleStruct,
27 => TypeTag::TupleVariant,
28 => TypeTag::MapSized,
29 => TypeTag::MapUnsizedStart,
30 => TypeTag::Struct,
31 => TypeTag::StructVariant,
32 => TypeTag::CollectionEnd,
_ => {
return Err(DecodeError::OtherString(format!("invalid type tag: {num}")));
}
};
debug_assert_eq!(tag as u8, num);
Ok(tag)
}
}
pub fn encode<E: Encoder, T: Serialize>(value: &T, encoder: &mut E) -> Result<(), EncodeError> {
value
.serialize(&mut ser::BincodeSerializer::new(encoder))
.map_err(|e| e.0)
}
pub fn decode<Context, D: Decoder<Context = Context>, T: DeserializeOwned>(
decoder: &mut D,
) -> Result<T, DecodeError> {
T::deserialize(de::BincodeDeserializer::new(decoder)).map_err(|e| e.0)
}
pub fn borrow_decode<
'de,
Context,
D: BorrowDecoder<'de, Context = Context>,
T: serde::de::Deserialize<'de>,
>(
decoder: &mut D,
) -> Result<T, DecodeError> {
T::deserialize(de::BincodeDeserializer::new(decoder)).map_err(|e| e.0)
}
#[cfg(test)]
mod tests {
use std::{collections::HashMap, fmt::Debug};
use bincode::{Decode, Encode, decode_from_slice, encode_to_vec};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
fn round_trip<T: Serialize + DeserializeOwned + PartialEq + Debug>(value: T) -> T {
#[derive(Encode)]
#[bincode(encode_bounds = "T: Serialize")]
struct EncodeWrapper<'a, T>(#[bincode(with = "crate::serde_self_describing")] &'a T);
#[derive(Decode)]
#[bincode(
decode_bounds = "T: DeserializeOwned",
borrow_decode_bounds = "T: Deserialize<'__de>"
)]
struct DecodeWrapper<T>(#[bincode(with = "crate::serde_self_describing")] T);
let config = bincode::config::standard();
let encoded = encode_to_vec(EncodeWrapper(&value), config).unwrap();
let (DecodeWrapper(decoded), len) = decode_from_slice(&encoded, config).unwrap();
assert_eq!(value, decoded);
assert_eq!(len, encoded.len(), "the entire buffer must be decoded");
decoded
}
#[test]
fn test_primitives() {
round_trip(true);
round_trip(false);
round_trip(42u8);
round_trip(42u16);
round_trip(42u32);
round_trip(42u64);
round_trip(-42i8);
round_trip(-42i16);
round_trip(-42i32);
round_trip(-42i64);
round_trip(1.23f32);
round_trip(1.23f64);
round_trip('a');
}
#[test]
fn test_string() {
round_trip(String::new());
round_trip(String::from("hello world"));
}
#[test]
fn test_option() {
round_trip(Option::<i32>::None);
round_trip(Some(42));
round_trip(Some(String::from("hello")));
round_trip(Some(Some(42)));
}
#[test]
fn test_vec() {
round_trip(Vec::<i32>::new());
round_trip(vec![String::from("a"), String::from("b")]);
round_trip(vec![vec![1, 2], vec![3, 4]]);
round_trip(b"abc\0def".to_vec());
}
#[test]
fn test_tuple() {
round_trip(());
round_trip((vec![1, 2], "hello".to_string(), Some(42), false, true));
}
#[test]
fn test_hashmap() {
let mut map = HashMap::new();
map.insert("key1".to_string(), 1);
map.insert("key2".to_string(), 2);
round_trip(map);
let empty: HashMap<String, i32> = HashMap::new();
round_trip(empty);
}
#[test]
fn test_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Struct {
a: String,
b: Option<i32>,
#[serde(flatten)]
flattened: Flattened,
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Flattened {
d: i32,
}
round_trip(Struct {
a: "hello".to_string(),
b: None,
flattened: Flattened { d: 42 },
});
round_trip(Struct {
a: "hello".to_string(),
b: Some(42),
flattened: Flattened { d: 43 },
});
}
#[test]
fn test_unit_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct UnitStruct;
round_trip(UnitStruct);
}
#[test]
fn test_newtype_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct NewtypeStruct(i32);
round_trip(NewtypeStruct(42));
}
#[test]
fn test_tuple_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct TupleStruct(i32, String, bool);
round_trip(TupleStruct(42, "hello".to_string(), true));
}
#[test]
fn test_enum_unit_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Color {
Red,
Green,
Blue,
}
round_trip(Color::Red);
round_trip(Color::Green);
round_trip(Color::Blue);
}
#[test]
fn test_enum_newtype_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Value {
#[allow(unused)]
#[serde(skip)]
Empty(()),
Int(i32),
Text(String),
}
round_trip(Value::Int(42));
round_trip(Value::Text("hello".to_string()));
}
#[test]
fn test_enum_tuple_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Point {
TwoD(i32, i32),
ThreeD(i32, i32, i32),
}
round_trip(Point::TwoD(1, 2));
round_trip(Point::ThreeD(1, 2, 3));
}
#[test]
fn test_enum_struct_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Message {
Request { id: u32, method: String },
Response { id: u32, result: Option<String> },
}
round_trip(Message::Request {
id: 1,
method: "get".to_string(),
});
round_trip(Message::Response {
id: 1,
result: Some("ok".to_string()),
});
round_trip(Message::Response {
id: 2,
result: None,
});
}
}
......@@ -14,22 +14,24 @@ verbose_log = []
[dependencies]
anyhow = { workspace = true }
bincode = { workspace = true }
bitfield = { workspace = true }
byteorder = { workspace = true }
dashmap = { workspace = true}
either = { workspace = true }
pot = "3.0.0"
byteorder = { workspace = true }
jiff = "0.2.10"
lzzzz = "1.1.0"
memmap2 = "0.9.5"
nohash-hasher = { workspace = true }
parking_lot = { workspace = true }
pot = "3.0.0"
qfilter = { version = "0.2.4", features = ["serde"] }
quick_cache = { workspace = true }
rustc-hash = { workspace = true }
smallvec = { workspace = true }
thread_local = { workspace = true }
tracing = { workspace = true }
turbo-bincode= { workspace = true }
twox-hash = { workspace = true }
zstd = { version = "0.13.2", features = ["zdict_builder"] }
......
......@@ -9,12 +9,14 @@ use std::{
};
use anyhow::{Context, Result, bail};
use bincode::{Decode, Encode};
use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt};
use either::Either;
use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher;
use turbo_bincode::turbo_bincode_decode;
use crate::{
QueryKey,
......@@ -62,6 +64,14 @@ impl Display for MetaEntryFlags {
}
}
/// A wrapper around [`qfilter::Filter`] that implements [`Encode`] and [`Decode`].
#[derive(Encode, Decode)]
pub struct AmqfBincodeWrapper(
// this annotation can be replaced with `#[bincode(serde)]` once
// <https://github.com/arthurprs/qfilter/issues/13> is resolved
#[bincode(with = "turbo_bincode::serde_self_describing")] pub qfilter::Filter,
);
pub struct MetaEntry {
/// The metadata for the static sorted file.
sst_data: StaticSortedFileMetaData,
......@@ -113,13 +123,15 @@ impl MetaEntry {
pub fn deserialize_amqf(&self, meta: &MetaFile) -> Result<qfilter::Filter> {
let amqf = self.raw_amqf(meta.amqf_data());
pot::from_slice(amqf).with_context(|| {
format!(
"Failed to deserialize AMQF from {:08}.meta for {:08}.sst",
meta.sequence_number,
self.sequence_number()
)
})
Ok(turbo_bincode_decode::<AmqfBincodeWrapper>(amqf)
.with_context(|| {
format!(
"Failed to deserialize AMQF from {:08}.meta for {:08}.sst",
meta.sequence_number,
self.sequence_number()
)
})?
.0)
}
pub fn amqf(
......
......@@ -8,10 +8,11 @@ use std::{
use anyhow::{Context, Result};
use byteorder::{BE, ByteOrder, WriteBytesExt};
use turbo_bincode::{TurboBincodeBuffer, turbo_bincode_encode};
use crate::{
compression::compress_into_buffer,
meta_file::MetaEntryFlags,
meta_file::{AmqfBincodeWrapper, MetaEntryFlags},
static_sorted_file::{
BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL,
......@@ -141,7 +142,7 @@ pub fn write_static_stored_file<E: Entry>(
let meta = StaticSortedFileBuilderMeta {
min_hash,
max_hash,
amqf: Cow::Owned(amqf),
amqf: Cow::Owned(amqf.into_vec()),
key_compression_dictionary_length: key_dict.len().try_into().unwrap(),
block_count,
size: file.stream_position()?,
......@@ -395,7 +396,7 @@ fn write_key_blocks_and_compute_amqf(
key_compression_dictionary: &[u8],
writer: &mut BlockWriter<'_>,
buffer: &mut Vec<u8>,
) -> Result<Vec<u8>> {
) -> Result<TurboBincodeBuffer> {
let mut filter = qfilter::Filter::new(entries.len() as u64, AMQF_FALSE_POSITIVE_RATE)
// This won't fail as we limit the number of entries per SST file
.expect("Filter can't be constructed");
......@@ -502,7 +503,7 @@ fn write_key_blocks_and_compute_amqf(
writer.write_index_block(buffer, key_compression_dictionary)?;
buffer.clear();
Ok(pot::to_vec(&filter).expect("AMQF serialization failed"))
Ok(turbo_bincode_encode(&AmqfBincodeWrapper(filter)).expect("AMQF serialization failed"))
}
/// Builder for a single key block
......
......@@ -56,7 +56,6 @@ turbo-tasks-hash = { workspace = true }
turbo-tasks-macros = { workspace = true }
turbo-tasks-malloc = { workspace = true }
unsize = { workspace = true }
pot = "3.0.0"
[dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] }
......
......@@ -56,7 +56,7 @@ impl ManualEncodeWrapper for JsonValueEncodeWrapper<'_> {
impl Encode for JsonValueEncodeWrapper<'_> {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
turbo_bincode::serde_json::encode(self.0, encoder)
turbo_bincode::serde_self_describing::encode(self.0, encoder)
}
}
......@@ -72,6 +72,6 @@ impl ManualDecodeWrapper for JsonValueDecodeWrapper {
impl<Context> Decode<Context> for JsonValueDecodeWrapper {
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
Ok(Self(turbo_bincode::serde_json::decode(decoder)?))
Ok(Self(turbo_bincode::serde_self_describing::decode(decoder)?))
}
}
......@@ -18,7 +18,7 @@ use turbopack_core::{
pub struct EmittedAsset {
file: RcStr,
content: RcStr,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
source_map: Option<JsonValue>,
}
......
......@@ -98,7 +98,7 @@ struct WebpackLoadersProcessingResult {
pub struct WebpackLoaderItem {
pub loader: RcStr,
#[serde(default)]
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
pub options: serde_json::Map<String, serde_json::Value>,
}
......@@ -364,7 +364,7 @@ enum LogType {
pub struct LogInfo {
time: u64,
log_type: LogType,
#[bincode(with = "turbo_bincode::serde_json")]
#[bincode(with = "turbo_bincode::serde_self_describing")]
args: Vec<JsonValue>,
trace: Option<Vec<StackFrame<'static>>>,
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment