Unverified Commit b26b2b71 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Add a custom serde_self_describing Serializer/Deserializer (#86885)

Implements a minimal custom serde `Serializer` and `Deserializer` on top of bincode with a custom self-describing format. The API is exposed as functions that can be used in `#[bincode(with = "...")]` attributes.

This doesn't encode as efficiently as `pot`, but it lets us get rid of that dependency entirely. It also gives us a more efficient option than `serde_json` for certain values.
parent aaee236c
...@@ -9108,7 +9108,6 @@ dependencies = [ ...@@ -9108,7 +9108,6 @@ dependencies = [
"mime", "mime",
"ringmap", "ringmap",
"serde", "serde",
"serde_json",
"smallvec", "smallvec",
"unty", "unty",
] ]
...@@ -9135,6 +9134,7 @@ name = "turbo-persistence" ...@@ -9135,6 +9134,7 @@ name = "turbo-persistence"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bincode 2.0.1",
"bitfield", "bitfield",
"byteorder", "byteorder",
"dashmap 6.1.0", "dashmap 6.1.0",
...@@ -9154,6 +9154,7 @@ dependencies = [ ...@@ -9154,6 +9154,7 @@ dependencies = [
"tempfile", "tempfile",
"thread_local", "thread_local",
"tracing", "tracing",
"turbo-bincode",
"twox-hash 2.1.0", "twox-hash 2.1.0",
"zstd", "zstd",
] ]
...@@ -9227,7 +9228,6 @@ dependencies = [ ...@@ -9227,7 +9228,6 @@ dependencies = [
"once_cell", "once_cell",
"parking_lot", "parking_lot",
"pin-project-lite", "pin-project-lite",
"pot",
"rayon", "rayon",
"regex", "regex",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
......
...@@ -92,7 +92,7 @@ pub struct NextConfig { ...@@ -92,7 +92,7 @@ pub struct NextConfig {
cache_handler: Option<RcStr>, cache_handler: Option<RcStr>,
#[bincode(with_serde)] #[bincode(with_serde)]
cache_handlers: Option<FxIndexMap<RcStr, RcStr>>, cache_handlers: Option<FxIndexMap<RcStr, RcStr>>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
env: FxIndexMap<String, JsonValue>, env: FxIndexMap<String, JsonValue>,
experimental: ExperimentalConfig, experimental: ExperimentalConfig,
images: ImageConfig, images: ImageConfig,
...@@ -101,12 +101,12 @@ pub struct NextConfig { ...@@ -101,12 +101,12 @@ pub struct NextConfig {
react_production_profiling: Option<bool>, react_production_profiling: Option<bool>,
react_strict_mode: Option<bool>, react_strict_mode: Option<bool>,
transpile_packages: Option<Vec<RcStr>>, transpile_packages: Option<Vec<RcStr>>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>, modularize_imports: Option<FxIndexMap<String, ModularizeImportPackageConfig>>,
dist_dir: RcStr, dist_dir: RcStr,
dist_dir_root: RcStr, dist_dir_root: RcStr,
deployment_id: Option<RcStr>, deployment_id: Option<RcStr>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
sass_options: Option<serde_json::Value>, sass_options: Option<serde_json::Value>,
trailing_slash: Option<bool>, trailing_slash: Option<bool>,
asset_prefix: Option<RcStr>, asset_prefix: Option<RcStr>,
...@@ -119,9 +119,9 @@ pub struct NextConfig { ...@@ -119,9 +119,9 @@ pub struct NextConfig {
output: Option<OutputType>, output: Option<OutputType>,
turbopack: Option<TurbopackConfig>, turbopack: Option<TurbopackConfig>,
production_browser_source_maps: bool, production_browser_source_maps: bool,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
output_file_tracing_includes: Option<serde_json::Value>, output_file_tracing_includes: Option<serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
output_file_tracing_excludes: Option<serde_json::Value>, output_file_tracing_excludes: Option<serde_json::Value>,
// TODO: This option is not respected, it uses Turbopack's root instead. // TODO: This option is not respected, it uses Turbopack's root instead.
output_file_tracing_root: Option<RcStr>, output_file_tracing_root: Option<RcStr>,
...@@ -154,9 +154,9 @@ pub struct NextConfig { ...@@ -154,9 +154,9 @@ pub struct NextConfig {
http_agent_options: HttpAgentConfig, http_agent_options: HttpAgentConfig,
on_demand_entries: OnDemandEntriesConfig, on_demand_entries: OnDemandEntriesConfig,
powered_by_header: bool, powered_by_header: bool,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
public_runtime_config: FxIndexMap<String, serde_json::Value>, public_runtime_config: FxIndexMap<String, serde_json::Value>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
server_runtime_config: FxIndexMap<String, serde_json::Value>, server_runtime_config: FxIndexMap<String, serde_json::Value>,
static_page_generation_timeout: f64, static_page_generation_timeout: f64,
target: Option<String>, target: Option<String>,
...@@ -686,9 +686,9 @@ pub enum RemotePatternProtocol { ...@@ -686,9 +686,9 @@ pub enum RemotePatternProtocol {
)] )]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct TurbopackConfig { pub struct TurbopackConfig {
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
pub rules: Option<FxIndexMap<RcStr, RuleConfigCollection>>, pub rules: Option<FxIndexMap<RcStr, RuleConfigCollection>>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>, pub resolve_alias: Option<FxIndexMap<RcStr, JsonValue>>,
pub resolve_extensions: Option<Vec<RcStr>>, pub resolve_extensions: Option<Vec<RcStr>>,
pub debug_ids: Option<bool>, pub debug_ids: Option<bool>,
...@@ -955,7 +955,7 @@ pub struct ExperimentalConfig { ...@@ -955,7 +955,7 @@ pub struct ExperimentalConfig {
/// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs) /// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs)
mdx_rs: Option<MdxRsOptions>, mdx_rs: Option<MdxRsOptions>,
strict_next_head: Option<bool>, strict_next_head: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>, swc_plugins: Option<Vec<(RcStr, serde_json::Value)>>,
external_middleware_rewrites_resolve: Option<bool>, external_middleware_rewrites_resolve: Option<bool>,
scroll_restoration: Option<bool>, scroll_restoration: Option<bool>,
...@@ -964,7 +964,7 @@ pub struct ExperimentalConfig { ...@@ -964,7 +964,7 @@ pub struct ExperimentalConfig {
middleware_prefetch: Option<MiddlewarePrefetchType>, middleware_prefetch: Option<MiddlewarePrefetchType>,
/// optimizeCss can be boolean or critters' option object /// optimizeCss can be boolean or critters' option object
/// Use Record<string, unknown> as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts)) /// Use Record<string, unknown> as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts))
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
optimize_css: Option<serde_json::Value>, optimize_css: Option<serde_json::Value>,
next_script_workers: Option<bool>, next_script_workers: Option<bool>,
web_vitals_attribution: Option<Vec<RcStr>>, web_vitals_attribution: Option<Vec<RcStr>>,
...@@ -982,7 +982,7 @@ pub struct ExperimentalConfig { ...@@ -982,7 +982,7 @@ pub struct ExperimentalConfig {
adjust_font_fallbacks_with_size_adjust: Option<bool>, adjust_font_fallbacks_with_size_adjust: Option<bool>,
after: Option<bool>, after: Option<bool>,
app_document_preloading: Option<bool>, app_document_preloading: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
cache_life: Option<FxIndexMap<String, CacheLifeProfile>>, cache_life: Option<FxIndexMap<String, CacheLifeProfile>>,
case_sensitive_routes: Option<bool>, case_sensitive_routes: Option<bool>,
cpus: Option<f64>, cpus: Option<f64>,
...@@ -990,7 +990,7 @@ pub struct ExperimentalConfig { ...@@ -990,7 +990,7 @@ pub struct ExperimentalConfig {
disable_optimized_loading: Option<bool>, disable_optimized_loading: Option<bool>,
disable_postcss_preset_env: Option<bool>, disable_postcss_preset_env: Option<bool>,
esm_externals: Option<EsmExternals>, esm_externals: Option<EsmExternals>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
extension_alias: Option<serde_json::Value>, extension_alias: Option<serde_json::Value>,
external_dir: Option<bool>, external_dir: Option<bool>,
/// If set to `false`, webpack won't fall back to polyfill Node.js modules /// If set to `false`, webpack won't fall back to polyfill Node.js modules
...@@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig { ...@@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig {
instrumentation_hook: Option<bool>, instrumentation_hook: Option<bool>,
client_trace_metadata: Option<Vec<String>>, client_trace_metadata: Option<Vec<String>>,
large_page_data_bytes: Option<f64>, large_page_data_bytes: Option<f64>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
logging: Option<serde_json::Value>, logging: Option<serde_json::Value>,
memory_based_workers_count: Option<bool>, memory_based_workers_count: Option<bool>,
/// Optimize React APIs for server builds. /// Optimize React APIs for server builds.
...@@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig { ...@@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig {
/// @internal Used by the Next.js internals only. /// @internal Used by the Next.js internals only.
trust_host_header: Option<bool>, trust_host_header: Option<bool>,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
url_imports: Option<serde_json::Value>, url_imports: Option<serde_json::Value>,
/// This option is to enable running the Webpack build in a worker thread /// This option is to enable running the Webpack build in a worker thread
/// (doesn't apply to Turbopack). /// (doesn't apply to Turbopack).
...@@ -1397,7 +1397,7 @@ pub struct ResolveExtensions(Option<Vec<RcStr>>); ...@@ -1397,7 +1397,7 @@ pub struct ResolveExtensions(Option<Vec<RcStr>>);
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct SwcPlugins( pub struct SwcPlugins(
#[bincode(with = "turbo_bincode::serde_json")] Vec<(RcStr, serde_json::Value)>, #[bincode(with = "turbo_bincode::serde_self_describing")] Vec<(RcStr, serde_json::Value)>,
); );
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
...@@ -1415,7 +1415,7 @@ pub struct OptionServerActions(Option<ServerActions>); ...@@ -1415,7 +1415,7 @@ pub struct OptionServerActions(Option<ServerActions>);
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct OptionJsonValue( pub struct OptionJsonValue(
#[bincode(with = "turbo_bincode::serde_json")] pub Option<serde_json::Value>, #[bincode(with = "turbo_bincode::serde_self_describing")] pub Option<serde_json::Value>,
); );
fn turbopack_config_documentation_link() -> RcStr { fn turbopack_config_documentation_link() -> RcStr {
...@@ -1468,7 +1468,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue { ...@@ -1468,7 +1468,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue {
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
struct InvalidLoaderRuleConditionIssue { struct InvalidLoaderRuleConditionIssue {
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
condition: ConfigConditionItem, condition: ConfigConditionItem,
config_file_path: FileSystemPath, config_file_path: FileSystemPath,
} }
...@@ -2218,7 +2218,7 @@ impl NextConfig { ...@@ -2218,7 +2218,7 @@ impl NextConfig {
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct JsConfig { pub struct JsConfig {
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
compiler_options: Option<serde_json::Value>, compiler_options: Option<serde_json::Value>,
} }
......
...@@ -17,6 +17,5 @@ indexmap = { workspace = true } ...@@ -17,6 +17,5 @@ indexmap = { workspace = true }
mime = { workspace = true } mime = { workspace = true }
ringmap = { workspace = true } ringmap = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true }
unty = { workspace = true } unty = { workspace = true }
#[doc(hidden)] #[doc(hidden)]
pub mod macro_helpers; pub mod macro_helpers;
pub mod serde_self_describing;
use std::{any::Any, ptr::copy_nonoverlapping}; use std::{any::Any, ptr::copy_nonoverlapping};
...@@ -472,76 +473,6 @@ pub mod mime_option { ...@@ -472,76 +473,6 @@ pub mod mime_option {
} }
} }
/// Encode/decode as a serialized string encoded using `serde_json`.
///
/// This encodes less efficiently than `#[bincode(with_serde)]` would, but avoids [bincode's known
/// compatibility issues][serde-issues]. Use this for infrequently-serialized types and when you're
/// unsure if the underlying type may trigger a serde compatibility issue.
///
/// In the future this could be replaced with a more efficient serde-compatible self-describing
/// format with a compact binary representation (e.g. pot or MessagePack), but `serde_json` is
/// convenient because it avoids introducing additional dependencies.
///
/// [serde-issues]: https://docs.rs/bincode/latest/bincode/serde/index.html#known-issues
pub mod serde_json {
use super::*;
pub fn encode<E: Encoder, T: serde::Serialize>(
value: &T,
encoder: &mut E,
) -> Result<(), EncodeError> {
let json_str =
::serde_json::to_string(value).map_err(|e| EncodeError::OtherString(e.to_string()))?;
Encode::encode(&json_str, encoder)
}
pub fn decode<Context, D: Decoder<Context = Context>, T: serde::de::DeserializeOwned>(
decoder: &mut D,
) -> Result<T, DecodeError> {
let json_str: String = Decode::decode(decoder)?;
::serde_json::from_str(&json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
}
pub fn borrow_decode<
'de,
Context,
D: BorrowDecoder<'de, Context = Context>,
T: serde::de::Deserialize<'de>,
>(
decoder: &mut D,
) -> Result<T, DecodeError> {
let json_str: &str = BorrowDecode::borrow_decode(decoder)?;
::serde_json::from_str(json_str).map_err(|e| DecodeError::OtherString(e.to_string()))
}
#[cfg(test)]
mod tests {
use ::serde_json::{Value, json};
use bincode::{decode_from_slice, encode_to_vec};
use super::*;
#[test]
fn test_roundtrip() {
let cfg = bincode::config::standard();
#[derive(Encode, Decode)]
struct Wrapper(#[bincode(with = "crate::serde_json")] Value);
let value1 = Wrapper(json!({
"key1": [1, 2, 3],
"key2": [4, 5, 6]
}));
let value2: Wrapper = decode_from_slice(&encode_to_vec(&value1, cfg).unwrap(), cfg)
.unwrap()
.0;
assert_eq!(value1.0, value2.0);
}
}
}
pub mod either { pub mod either {
use ::either::Either; use ::either::Either;
......
This diff is collapsed.
//! Helpers for serializing serde-compatible types inside of a bincode [`Encode`] or [`Decode`]
//! implementation using a self-describing format. This works with [types that
//! `#[bincode(serde)]` does not support][bincode::serde#known-issues].
//!
//! These helper functions can be used in the [`Encode`] and [`Decode`] derive macros with the
//! `#[bincode(with = "turbo_bincode]` attribute.
//!
//! [`Encode`]: bincode::Encode
//! [`Decode`]: bincode::Decode
use bincode::{
de::{BorrowDecoder, Decoder},
enc::Encoder,
error::{DecodeError, EncodeError},
};
use serde::{Serialize, de::DeserializeOwned};
mod de;
mod ser;
/// Uses a u8 representation, which is slightly more efficient than bincode's default u32 varint
/// approach for enum discriminants:
/// https://docs.rs/bincode/latest/bincode/spec/index.html#discriminant-representation
#[derive(Copy, Clone, PartialEq, Eq)]
#[repr(u8)]
enum TypeTag {
BoolTrue = 1,
BoolFalse,
U8,
U16,
U32,
U64,
I8,
I16,
I32,
I64,
F32,
F64,
Char,
String,
Bytes,
OptionNone,
OptionSome,
Unit,
UnitStruct,
UnitVariant,
NewtypeStruct,
NewtypeVariant,
SeqSized,
SeqUnsizedStart,
Tuple,
TupleStruct,
TupleVariant,
MapSized,
MapUnsizedStart,
Struct,
StructVariant,
CollectionEnd,
}
impl TryFrom<u8> for TypeTag {
type Error = DecodeError;
fn try_from(num: u8) -> Result<Self, DecodeError> {
let tag = match num {
1 => TypeTag::BoolTrue,
2 => TypeTag::BoolFalse,
3 => TypeTag::U8,
4 => TypeTag::U16,
5 => TypeTag::U32,
6 => TypeTag::U64,
7 => TypeTag::I8,
8 => TypeTag::I16,
9 => TypeTag::I32,
10 => TypeTag::I64,
11 => TypeTag::F32,
12 => TypeTag::F64,
13 => TypeTag::Char,
14 => TypeTag::String,
15 => TypeTag::Bytes,
16 => TypeTag::OptionNone,
17 => TypeTag::OptionSome,
18 => TypeTag::Unit,
19 => TypeTag::UnitStruct,
20 => TypeTag::UnitVariant,
21 => TypeTag::NewtypeStruct,
22 => TypeTag::NewtypeVariant,
23 => TypeTag::SeqSized,
24 => TypeTag::SeqUnsizedStart,
25 => TypeTag::Tuple,
26 => TypeTag::TupleStruct,
27 => TypeTag::TupleVariant,
28 => TypeTag::MapSized,
29 => TypeTag::MapUnsizedStart,
30 => TypeTag::Struct,
31 => TypeTag::StructVariant,
32 => TypeTag::CollectionEnd,
_ => {
return Err(DecodeError::OtherString(format!("invalid type tag: {num}")));
}
};
debug_assert_eq!(tag as u8, num);
Ok(tag)
}
}
pub fn encode<E: Encoder, T: Serialize>(value: &T, encoder: &mut E) -> Result<(), EncodeError> {
value
.serialize(&mut ser::BincodeSerializer::new(encoder))
.map_err(|e| e.0)
}
pub fn decode<Context, D: Decoder<Context = Context>, T: DeserializeOwned>(
decoder: &mut D,
) -> Result<T, DecodeError> {
T::deserialize(de::BincodeDeserializer::new(decoder)).map_err(|e| e.0)
}
pub fn borrow_decode<
'de,
Context,
D: BorrowDecoder<'de, Context = Context>,
T: serde::de::Deserialize<'de>,
>(
decoder: &mut D,
) -> Result<T, DecodeError> {
T::deserialize(de::BincodeDeserializer::new(decoder)).map_err(|e| e.0)
}
#[cfg(test)]
mod tests {
use std::{collections::HashMap, fmt::Debug};
use bincode::{Decode, Encode, decode_from_slice, encode_to_vec};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
fn round_trip<T: Serialize + DeserializeOwned + PartialEq + Debug>(value: T) -> T {
#[derive(Encode)]
#[bincode(encode_bounds = "T: Serialize")]
struct EncodeWrapper<'a, T>(#[bincode(with = "crate::serde_self_describing")] &'a T);
#[derive(Decode)]
#[bincode(
decode_bounds = "T: DeserializeOwned",
borrow_decode_bounds = "T: Deserialize<'__de>"
)]
struct DecodeWrapper<T>(#[bincode(with = "crate::serde_self_describing")] T);
let config = bincode::config::standard();
let encoded = encode_to_vec(EncodeWrapper(&value), config).unwrap();
let (DecodeWrapper(decoded), len) = decode_from_slice(&encoded, config).unwrap();
assert_eq!(value, decoded);
assert_eq!(len, encoded.len(), "the entire buffer must be decoded");
decoded
}
#[test]
fn test_primitives() {
round_trip(true);
round_trip(false);
round_trip(42u8);
round_trip(42u16);
round_trip(42u32);
round_trip(42u64);
round_trip(-42i8);
round_trip(-42i16);
round_trip(-42i32);
round_trip(-42i64);
round_trip(1.23f32);
round_trip(1.23f64);
round_trip('a');
}
#[test]
fn test_string() {
round_trip(String::new());
round_trip(String::from("hello world"));
}
#[test]
fn test_option() {
round_trip(Option::<i32>::None);
round_trip(Some(42));
round_trip(Some(String::from("hello")));
round_trip(Some(Some(42)));
}
#[test]
fn test_vec() {
round_trip(Vec::<i32>::new());
round_trip(vec![String::from("a"), String::from("b")]);
round_trip(vec![vec![1, 2], vec![3, 4]]);
round_trip(b"abc\0def".to_vec());
}
#[test]
fn test_tuple() {
round_trip(());
round_trip((vec![1, 2], "hello".to_string(), Some(42), false, true));
}
#[test]
fn test_hashmap() {
let mut map = HashMap::new();
map.insert("key1".to_string(), 1);
map.insert("key2".to_string(), 2);
round_trip(map);
let empty: HashMap<String, i32> = HashMap::new();
round_trip(empty);
}
#[test]
fn test_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Struct {
a: String,
b: Option<i32>,
#[serde(flatten)]
flattened: Flattened,
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Flattened {
d: i32,
}
round_trip(Struct {
a: "hello".to_string(),
b: None,
flattened: Flattened { d: 42 },
});
round_trip(Struct {
a: "hello".to_string(),
b: Some(42),
flattened: Flattened { d: 43 },
});
}
#[test]
fn test_unit_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct UnitStruct;
round_trip(UnitStruct);
}
#[test]
fn test_newtype_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct NewtypeStruct(i32);
round_trip(NewtypeStruct(42));
}
#[test]
fn test_tuple_struct() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct TupleStruct(i32, String, bool);
round_trip(TupleStruct(42, "hello".to_string(), true));
}
#[test]
fn test_enum_unit_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Color {
Red,
Green,
Blue,
}
round_trip(Color::Red);
round_trip(Color::Green);
round_trip(Color::Blue);
}
#[test]
fn test_enum_newtype_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Value {
#[allow(unused)]
#[serde(skip)]
Empty(()),
Int(i32),
Text(String),
}
round_trip(Value::Int(42));
round_trip(Value::Text("hello".to_string()));
}
#[test]
fn test_enum_tuple_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Point {
TwoD(i32, i32),
ThreeD(i32, i32, i32),
}
round_trip(Point::TwoD(1, 2));
round_trip(Point::ThreeD(1, 2, 3));
}
#[test]
fn test_enum_struct_variants() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum Message {
Request { id: u32, method: String },
Response { id: u32, result: Option<String> },
}
round_trip(Message::Request {
id: 1,
method: "get".to_string(),
});
round_trip(Message::Response {
id: 1,
result: Some("ok".to_string()),
});
round_trip(Message::Response {
id: 2,
result: None,
});
}
}
...@@ -14,22 +14,24 @@ verbose_log = [] ...@@ -14,22 +14,24 @@ verbose_log = []
[dependencies] [dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
bincode = { workspace = true }
bitfield = { workspace = true } bitfield = { workspace = true }
byteorder = { workspace = true }
dashmap = { workspace = true} dashmap = { workspace = true}
either = { workspace = true } either = { workspace = true }
pot = "3.0.0"
byteorder = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
lzzzz = "1.1.0" lzzzz = "1.1.0"
memmap2 = "0.9.5" memmap2 = "0.9.5"
nohash-hasher = { workspace = true } nohash-hasher = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
pot = "3.0.0"
qfilter = { version = "0.2.4", features = ["serde"] } qfilter = { version = "0.2.4", features = ["serde"] }
quick_cache = { workspace = true } quick_cache = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true }
thread_local = { workspace = true } thread_local = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
turbo-bincode= { workspace = true }
twox-hash = { workspace = true } twox-hash = { workspace = true }
zstd = { version = "0.13.2", features = ["zdict_builder"] } zstd = { version = "0.13.2", features = ["zdict_builder"] }
......
...@@ -9,12 +9,14 @@ use std::{ ...@@ -9,12 +9,14 @@ use std::{
}; };
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use bincode::{Decode, Encode};
use bitfield::bitfield; use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use either::Either; use either::Either;
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use quick_cache::sync::GuardResult; use quick_cache::sync::GuardResult;
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use turbo_bincode::turbo_bincode_decode;
use crate::{ use crate::{
QueryKey, QueryKey,
...@@ -62,6 +64,14 @@ impl Display for MetaEntryFlags { ...@@ -62,6 +64,14 @@ impl Display for MetaEntryFlags {
} }
} }
/// A wrapper around [`qfilter::Filter`] that implements [`Encode`] and [`Decode`].
#[derive(Encode, Decode)]
pub struct AmqfBincodeWrapper(
// this annotation can be replaced with `#[bincode(serde)]` once
// <https://github.com/arthurprs/qfilter/issues/13> is resolved
#[bincode(with = "turbo_bincode::serde_self_describing")] pub qfilter::Filter,
);
pub struct MetaEntry { pub struct MetaEntry {
/// The metadata for the static sorted file. /// The metadata for the static sorted file.
sst_data: StaticSortedFileMetaData, sst_data: StaticSortedFileMetaData,
...@@ -113,13 +123,15 @@ impl MetaEntry { ...@@ -113,13 +123,15 @@ impl MetaEntry {
pub fn deserialize_amqf(&self, meta: &MetaFile) -> Result<qfilter::Filter> { pub fn deserialize_amqf(&self, meta: &MetaFile) -> Result<qfilter::Filter> {
let amqf = self.raw_amqf(meta.amqf_data()); let amqf = self.raw_amqf(meta.amqf_data());
pot::from_slice(amqf).with_context(|| { Ok(turbo_bincode_decode::<AmqfBincodeWrapper>(amqf)
.with_context(|| {
format!( format!(
"Failed to deserialize AMQF from {:08}.meta for {:08}.sst", "Failed to deserialize AMQF from {:08}.meta for {:08}.sst",
meta.sequence_number, meta.sequence_number,
self.sequence_number() self.sequence_number()
) )
}) })?
.0)
} }
pub fn amqf( pub fn amqf(
......
...@@ -8,10 +8,11 @@ use std::{ ...@@ -8,10 +8,11 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use byteorder::{BE, ByteOrder, WriteBytesExt}; use byteorder::{BE, ByteOrder, WriteBytesExt};
use turbo_bincode::{TurboBincodeBuffer, turbo_bincode_encode};
use crate::{ use crate::{
compression::compress_into_buffer, compression::compress_into_buffer,
meta_file::MetaEntryFlags, meta_file::{AmqfBincodeWrapper, MetaEntryFlags},
static_sorted_file::{ static_sorted_file::{
BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED, BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED,
KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL, KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL,
...@@ -141,7 +142,7 @@ pub fn write_static_stored_file<E: Entry>( ...@@ -141,7 +142,7 @@ pub fn write_static_stored_file<E: Entry>(
let meta = StaticSortedFileBuilderMeta { let meta = StaticSortedFileBuilderMeta {
min_hash, min_hash,
max_hash, max_hash,
amqf: Cow::Owned(amqf), amqf: Cow::Owned(amqf.into_vec()),
key_compression_dictionary_length: key_dict.len().try_into().unwrap(), key_compression_dictionary_length: key_dict.len().try_into().unwrap(),
block_count, block_count,
size: file.stream_position()?, size: file.stream_position()?,
...@@ -395,7 +396,7 @@ fn write_key_blocks_and_compute_amqf( ...@@ -395,7 +396,7 @@ fn write_key_blocks_and_compute_amqf(
key_compression_dictionary: &[u8], key_compression_dictionary: &[u8],
writer: &mut BlockWriter<'_>, writer: &mut BlockWriter<'_>,
buffer: &mut Vec<u8>, buffer: &mut Vec<u8>,
) -> Result<Vec<u8>> { ) -> Result<TurboBincodeBuffer> {
let mut filter = qfilter::Filter::new(entries.len() as u64, AMQF_FALSE_POSITIVE_RATE) let mut filter = qfilter::Filter::new(entries.len() as u64, AMQF_FALSE_POSITIVE_RATE)
// This won't fail as we limit the number of entries per SST file // This won't fail as we limit the number of entries per SST file
.expect("Filter can't be constructed"); .expect("Filter can't be constructed");
...@@ -502,7 +503,7 @@ fn write_key_blocks_and_compute_amqf( ...@@ -502,7 +503,7 @@ fn write_key_blocks_and_compute_amqf(
writer.write_index_block(buffer, key_compression_dictionary)?; writer.write_index_block(buffer, key_compression_dictionary)?;
buffer.clear(); buffer.clear();
Ok(pot::to_vec(&filter).expect("AMQF serialization failed")) Ok(turbo_bincode_encode(&AmqfBincodeWrapper(filter)).expect("AMQF serialization failed"))
} }
/// Builder for a single key block /// Builder for a single key block
......
...@@ -56,7 +56,6 @@ turbo-tasks-hash = { workspace = true } ...@@ -56,7 +56,6 @@ turbo-tasks-hash = { workspace = true }
turbo-tasks-macros = { workspace = true } turbo-tasks-macros = { workspace = true }
turbo-tasks-malloc = { workspace = true } turbo-tasks-malloc = { workspace = true }
unsize = { workspace = true } unsize = { workspace = true }
pot = "3.0.0"
[dev-dependencies] [dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] }
......
...@@ -56,7 +56,7 @@ impl ManualEncodeWrapper for JsonValueEncodeWrapper<'_> { ...@@ -56,7 +56,7 @@ impl ManualEncodeWrapper for JsonValueEncodeWrapper<'_> {
impl Encode for JsonValueEncodeWrapper<'_> { impl Encode for JsonValueEncodeWrapper<'_> {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> { fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
turbo_bincode::serde_json::encode(self.0, encoder) turbo_bincode::serde_self_describing::encode(self.0, encoder)
} }
} }
...@@ -72,6 +72,6 @@ impl ManualDecodeWrapper for JsonValueDecodeWrapper { ...@@ -72,6 +72,6 @@ impl ManualDecodeWrapper for JsonValueDecodeWrapper {
impl<Context> Decode<Context> for JsonValueDecodeWrapper { impl<Context> Decode<Context> for JsonValueDecodeWrapper {
fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> { fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
Ok(Self(turbo_bincode::serde_json::decode(decoder)?)) Ok(Self(turbo_bincode::serde_self_describing::decode(decoder)?))
} }
} }
...@@ -18,7 +18,7 @@ use turbopack_core::{ ...@@ -18,7 +18,7 @@ use turbopack_core::{
pub struct EmittedAsset { pub struct EmittedAsset {
file: RcStr, file: RcStr,
content: RcStr, content: RcStr,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
source_map: Option<JsonValue>, source_map: Option<JsonValue>,
} }
......
...@@ -98,7 +98,7 @@ struct WebpackLoadersProcessingResult { ...@@ -98,7 +98,7 @@ struct WebpackLoadersProcessingResult {
pub struct WebpackLoaderItem { pub struct WebpackLoaderItem {
pub loader: RcStr, pub loader: RcStr,
#[serde(default)] #[serde(default)]
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
pub options: serde_json::Map<String, serde_json::Value>, pub options: serde_json::Map<String, serde_json::Value>,
} }
...@@ -364,7 +364,7 @@ enum LogType { ...@@ -364,7 +364,7 @@ enum LogType {
pub struct LogInfo { pub struct LogInfo {
time: u64, time: u64,
log_type: LogType, log_type: LogType,
#[bincode(with = "turbo_bincode::serde_json")] #[bincode(with = "turbo_bincode::serde_self_describing")]
args: Vec<JsonValue>, args: Vec<JsonValue>,
trace: Option<Vec<StackFrame<'static>>>, trace: Option<Vec<StackFrame<'static>>>,
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment