Unverified Commit 4a4ff6c0 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] fix serialization problems/bugs (#69662)

### What?

* improve error
* fix CachedTaskType serialization
* fix next config serialization
* fix serialization of Rope
* fix serialization of AliasMap
* more efficent rope serialization
* add missing primitives
* Disable serialization for RuntimeVersions
parent 18f237c5
...@@ -5928,9 +5928,9 @@ dependencies = [ ...@@ -5928,9 +5928,9 @@ dependencies = [
[[package]] [[package]]
name = "serde_bytes" name = "serde_bytes"
version = "0.11.9" version = "0.11.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a"
dependencies = [ dependencies = [
"serde", "serde",
] ]
...@@ -8684,6 +8684,7 @@ dependencies = [ ...@@ -8684,6 +8684,7 @@ dependencies = [
"parking_lot", "parking_lot",
"rstest", "rstest",
"serde", "serde",
"serde_bytes",
"serde_json", "serde_json",
"serde_path_to_error", "serde_path_to_error",
"sha2", "sha2",
...@@ -8951,6 +8952,7 @@ dependencies = [ ...@@ -8951,6 +8952,7 @@ dependencies = [
"regex", "regex",
"rstest", "rstest",
"serde", "serde",
"serde_bytes",
"serde_json", "serde_json",
"sourcemap", "sourcemap",
"swc_core", "swc_core",
......
...@@ -187,6 +187,8 @@ rustc-hash = "1.1.0" ...@@ -187,6 +187,8 @@ rustc-hash = "1.1.0"
semver = "1.0.16" semver = "1.0.16"
serde = { version = "1.0.152", features = ["derive"] } serde = { version = "1.0.152", features = ["derive"] }
serde_json = "1.0.93" serde_json = "1.0.93"
serde_bytes = "0.11.15"
serde_path_to_error = "0.1.9"
serde_qs = "0.11.0" serde_qs = "0.11.0"
serde_with = "2.3.2" serde_with = "2.3.2"
serde_yaml = "0.9.17" serde_yaml = "0.9.17"
......
...@@ -314,10 +314,10 @@ pub struct ImageConfig { ...@@ -314,10 +314,10 @@ pub struct ImageConfig {
pub loader_file: Option<String>, pub loader_file: Option<String>,
pub domains: Vec<String>, pub domains: Vec<String>,
pub disable_static_images: bool, pub disable_static_images: bool,
#[serde(rename(deserialize = "minimumCacheTTL"))] #[serde(rename = "minimumCacheTTL")]
pub minimum_cache_ttl: u64, pub minimum_cache_ttl: u64,
pub formats: Vec<ImageFormat>, pub formats: Vec<ImageFormat>,
#[serde(rename(deserialize = "dangerouslyAllowSVG"))] #[serde(rename = "dangerouslyAllowSVG")]
pub dangerously_allow_svg: bool, pub dangerously_allow_svg: bool,
pub content_security_policy: String, pub content_security_policy: String,
pub remote_patterns: Vec<RemotePattern>, pub remote_patterns: Vec<RemotePattern>,
......
...@@ -40,8 +40,9 @@ mime = { workspace = true } ...@@ -40,8 +40,9 @@ mime = { workspace = true }
notify = { workspace = true } notify = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
serde = { workspace = true, features = ["rc"] } serde = { workspace = true, features = ["rc"] }
serde_bytes = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
serde_path_to_error = "0.1.9" serde_path_to_error = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
......
...@@ -14,6 +14,7 @@ use anyhow::{Context, Result}; ...@@ -14,6 +14,7 @@ use anyhow::{Context, Result};
use bytes::{Buf, Bytes}; use bytes::{Buf, Bytes};
use futures::Stream; use futures::Stream;
use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_bytes::ByteBuf;
use tokio::io::{AsyncRead, ReadBuf}; use tokio::io::{AsyncRead, ReadBuf};
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher}; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher};
use RopeElem::{Local, Shared}; use RopeElem::{Local, Shared};
...@@ -342,19 +343,34 @@ impl Serialize for Rope { ...@@ -342,19 +343,34 @@ impl Serialize for Rope {
/// possible owner of a individual "shared" data doesn't make sense). /// possible owner of a individual "shared" data doesn't make sense).
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::Error; use serde::ser::Error;
let s = self.to_str().map_err(Error::custom)?; let bytes = self.to_bytes().map_err(Error::custom)?;
serializer.serialize_str(&s) match bytes {
Cow::Borrowed(b) => serde_bytes::Bytes::new(b).serialize(serializer),
Cow::Owned(b) => ByteBuf::from(b).serialize(serializer),
}
} }
} }
impl<'de> Deserialize<'de> for Rope { impl<'de> Deserialize<'de> for Rope {
/// Deserializes strings into a contiguous, immutable Rope. /// Deserializes strings into a contiguous, immutable Rope.
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let bytes = <Vec<u8>>::deserialize(deserializer)?; let bytes = ByteBuf::deserialize(deserializer)?.into_vec();
Ok(Rope::from(bytes)) Ok(Rope::from(bytes))
} }
} }
pub mod ser_as_string {
use serde::{ser::Error, Serializer};
use super::Rope;
/// Serializes a Rope into a string.
pub fn serialize<S: Serializer>(rope: &Rope, serializer: S) -> Result<S::Ok, S::Error> {
let s = rope.to_str().map_err(Error::custom)?;
serializer.serialize_str(&s)
}
}
impl PartialEq for Rope { impl PartialEq for Rope {
// Ropes with similar contents are equals, regardless of their structure. // Ropes with similar contents are equals, regardless of their structure.
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
......
...@@ -164,23 +164,27 @@ mod ser { ...@@ -164,23 +164,27 @@ mod ser {
{ {
match self { match self {
CachedTaskType::Native { fn_type, this, arg } => { CachedTaskType::Native { fn_type, this, arg } => {
let mut s = serializer.serialize_seq(Some(3))?; let mut s = serializer.serialize_tuple(5)?;
s.serialize_element::<u8>(&0)?; s.serialize_element::<u8>(&0)?;
s.serialize_element(&FunctionAndArg::Borrowed { s.serialize_element(&FunctionAndArg::Borrowed {
fn_type: *fn_type, fn_type: *fn_type,
arg, arg: &**arg,
})?; })?;
s.serialize_element(this)?; s.serialize_element(this)?;
s.serialize_element(&())?;
s.serialize_element(&())?;
s.end() s.end()
} }
CachedTaskType::ResolveNative { fn_type, this, arg } => { CachedTaskType::ResolveNative { fn_type, this, arg } => {
let mut s = serializer.serialize_seq(Some(3))?; let mut s = serializer.serialize_tuple(5)?;
s.serialize_element::<u8>(&1)?; s.serialize_element::<u8>(&1)?;
s.serialize_element(&FunctionAndArg::Borrowed { s.serialize_element(&FunctionAndArg::Borrowed {
fn_type: *fn_type, fn_type: *fn_type,
arg: &**arg, arg: &**arg,
})?; })?;
s.serialize_element(this)?; s.serialize_element(this)?;
s.serialize_element(&())?;
s.serialize_element(&())?;
s.end() s.end()
} }
CachedTaskType::ResolveTrait { CachedTaskType::ResolveTrait {
...@@ -236,6 +240,12 @@ mod ser { ...@@ -236,6 +240,12 @@ mod ser {
let this = seq let this = seq
.next_element()? .next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(2, &self))?; .ok_or_else(|| serde::de::Error::invalid_length(2, &self))?;
let () = seq
.next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
let () = seq
.next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(4, &self))?;
Ok(CachedTaskType::Native { fn_type, this, arg }) Ok(CachedTaskType::Native { fn_type, this, arg })
} }
1 => { 1 => {
...@@ -248,18 +258,24 @@ mod ser { ...@@ -248,18 +258,24 @@ mod ser {
let this = seq let this = seq
.next_element()? .next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(2, &self))?; .ok_or_else(|| serde::de::Error::invalid_length(2, &self))?;
let () = seq
.next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
let () = seq
.next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(4, &self))?;
Ok(CachedTaskType::ResolveNative { fn_type, this, arg }) Ok(CachedTaskType::ResolveNative { fn_type, this, arg })
} }
2 => { 2 => {
let trait_type = seq let trait_type = seq
.next_element()? .next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(0, &self))?; .ok_or_else(|| serde::de::Error::invalid_length(1, &self))?;
let method_name = seq let method_name = seq
.next_element()? .next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(1, &self))?; .ok_or_else(|| serde::de::Error::invalid_length(2, &self))?;
let this = seq let this = seq
.next_element()? .next_element()?
.ok_or_else(|| serde::de::Error::invalid_length(2, &self))?; .ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
let Some(method) = let Some(method) =
registry::get_trait(trait_type).methods.get(&method_name) registry::get_trait(trait_type).methods.get(&method_name)
else { else {
...@@ -267,7 +283,7 @@ mod ser { ...@@ -267,7 +283,7 @@ mod ser {
}; };
let arg = seq let arg = seq
.next_element_seed(method.arg_deserializer)? .next_element_seed(method.arg_deserializer)?
.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?; .ok_or_else(|| serde::de::Error::invalid_length(4, &self))?;
Ok(CachedTaskType::ResolveTrait { Ok(CachedTaskType::ResolveTrait {
trait_type, trait_type,
method_name, method_name,
...@@ -279,7 +295,7 @@ mod ser { ...@@ -279,7 +295,7 @@ mod ser {
} }
} }
} }
deserializer.deserialize_seq(Visitor) deserializer.deserialize_tuple(5, Visitor)
} }
} }
} }
......
...@@ -108,7 +108,7 @@ impl Serialize for TypedSharedReference { ...@@ -108,7 +108,7 @@ impl Serialize for TypedSharedReference {
} else { } else {
Err(serde::ser::Error::custom(format!( Err(serde::ser::Error::custom(format!(
"{:?} is not serializable", "{:?} is not serializable",
arc registry::get_value_type_global_name(*ty)
))) )))
} }
} }
......
...@@ -58,7 +58,7 @@ macro_rules! ignore { ...@@ -58,7 +58,7 @@ macro_rules! ignore {
} }
} }
ignore!(i8, u8, i16, u16, i32, u32, i64, u64, f32, f64, char, bool, usize); ignore!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128, f32, f64, char, bool, isize, usize);
ignore!( ignore!(
AtomicI8, AtomicI8,
AtomicU8, AtomicU8,
......
...@@ -75,6 +75,7 @@ struct EcmascriptMergedChunkPartial { ...@@ -75,6 +75,7 @@ struct EcmascriptMergedChunkPartial {
#[derive(Serialize)] #[derive(Serialize)]
struct EcmascriptModuleEntry { struct EcmascriptModuleEntry {
#[serde(with = "turbo_tasks_fs::rope::ser_as_string")]
code: Rope, code: Rope,
url: String, url: String,
map: Option<String>, map: Option<String>,
......
...@@ -26,6 +26,7 @@ patricia_tree = "0.5.5" ...@@ -26,6 +26,7 @@ patricia_tree = "0.5.5"
ref-cast = "1.0.20" ref-cast = "1.0.20"
regex = { workspace = true } regex = { workspace = true }
serde = { workspace = true, features = ["rc"] } serde = { workspace = true, features = ["rc"] }
serde_bytes = { workspace = true }
serde_json = { workspace = true, features = ["preserve_order"] } serde_json = { workspace = true, features = ["preserve_order"] }
sourcemap = { workspace = true } sourcemap = { workspace = true }
swc_core = { workspace = true, features = ["ecma_preset_env", "common"] } swc_core = { workspace = true, features = ["ecma_preset_env", "common"] }
......
...@@ -299,7 +299,8 @@ pub struct BrowserEnvironment { ...@@ -299,7 +299,8 @@ pub struct BrowserEnvironment {
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct EdgeWorkerEnvironment {} pub struct EdgeWorkerEnvironment {}
#[turbo_tasks::value(transparent)] // TODO preset_env_base::Version implements Serialize/Deserialize incorrectly
#[turbo_tasks::value(transparent, serialization = "none")]
pub struct RuntimeVersions(#[turbo_tasks(trace_ignore)] pub Versions); pub struct RuntimeVersions(#[turbo_tasks(trace_ignore)] pub Versions);
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -10,6 +10,7 @@ use serde::{ ...@@ -10,6 +10,7 @@ use serde::{
ser::SerializeMap, ser::SerializeMap,
Deserialize, Deserializer, Serialize, Serializer, Deserialize, Deserializer, Serialize, Serializer,
}; };
use serde_bytes::{ByteBuf, Bytes};
use turbo_tasks::{ use turbo_tasks::{
debug::{internal::PassthroughDebug, ValueDebugFormat, ValueDebugFormatString}, debug::{internal::PassthroughDebug, ValueDebugFormat, ValueDebugFormatString},
trace::{TraceRawVcs, TraceRawVcsContext}, trace::{TraceRawVcs, TraceRawVcsContext},
...@@ -61,7 +62,8 @@ where ...@@ -61,7 +62,8 @@ where
S: Serializer, S: Serializer,
{ {
let mut map = serializer.serialize_map(Some(self.map.len()))?; let mut map = serializer.serialize_map(Some(self.map.len()))?;
for (key, value) in self.map.iter() { for (prefix, value) in self.map.iter() {
let key = ByteBuf::from(prefix);
map.serialize_entry(&key, value)?; map.serialize_entry(&key, value)?;
} }
map.end() map.end()
...@@ -87,8 +89,8 @@ where ...@@ -87,8 +89,8 @@ where
M: MapAccess<'de>, M: MapAccess<'de>,
{ {
let mut map = AliasMap::new(); let mut map = AliasMap::new();
while let Some((key, value)) = access.next_entry()? { while let Some((key, value)) = access.next_entry::<&Bytes, _>()? {
map.insert(key, value); map.map.insert(key, value);
} }
Ok(map) Ok(map)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment