Unverified Commit 2689a741 authored by Donny/강동윤's avatar Donny/강동윤 Committed by GitHub

perf(turbopack/rcstr): Add serialization-time optimization (#78645)

### What?

Interns `RcStr` as an index set upon serialization.

### Why?

It's required for future optimization.
parent 3d0e4721
...@@ -9163,9 +9163,11 @@ version = "0.1.0" ...@@ -9163,9 +9163,11 @@ version = "0.1.0"
dependencies = [ dependencies = [
"bytes-str", "bytes-str",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"indexmap 2.9.0",
"napi", "napi",
"new_debug_unreachable", "new_debug_unreachable",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"scoped-tls",
"serde", "serde",
"shrink-to-fit", "shrink-to-fit",
"triomphe 0.1.12", "triomphe 0.1.12",
...@@ -9257,6 +9259,7 @@ dependencies = [ ...@@ -9257,6 +9259,7 @@ dependencies = [
"rstest", "rstest",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"serde_bytes",
"serde_json", "serde_json",
"serde_path_to_error", "serde_path_to_error",
"smallvec", "smallvec",
......
...@@ -13,16 +13,15 @@ mod db; ...@@ -13,16 +13,15 @@ mod db;
mod key; mod key;
mod lookup_entry; mod lookup_entry;
mod merge_iter; mod merge_iter;
mod static_sorted_file;
mod static_sorted_file_builder;
mod write_batch;
mod meta_file; mod meta_file;
mod meta_file_builder; mod meta_file_builder;
mod sst_filter; mod sst_filter;
mod static_sorted_file;
mod static_sorted_file_builder;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
mod value_buf; mod value_buf;
mod write_batch;
pub use arc_slice::ArcSlice; pub use arc_slice::ArcSlice;
pub use db::{MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
......
...@@ -15,11 +15,15 @@ turbo-tasks-hash = { workspace = true } ...@@ -15,11 +15,15 @@ turbo-tasks-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
new_debug_unreachable = "1.0.6" new_debug_unreachable = "1.0.6"
shrink-to-fit = { workspace = true } shrink-to-fit = { workspace = true }
rustc-hash = { workspace = true }
bytes-str = { workspace = true } bytes-str = { workspace = true }
indexmap = { workspace = true }
rustc-hash = { workspace = true }
scoped-tls = "1.0.1"
[target.'cfg(not(target_family = "wasm"))'.dependencies] [target.'cfg(not(target_family = "wasm"))'.dependencies]
napi = { workspace = true, optional = true } napi = { workspace = true, optional = true }
scoped-tls = "1.0.1"
indexmap = { workspace = true }
[dev-dependencies] [dev-dependencies]
criterion = { workspace = true } criterion = { workspace = true }
......
...@@ -11,17 +11,18 @@ use std::{ ...@@ -11,17 +11,18 @@ use std::{
use bytes_str::BytesStr; use bytes_str::BytesStr;
use debug_unreachable::debug_unreachable; use debug_unreachable::debug_unreachable;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use shrink_to_fit::ShrinkToFit; use shrink_to_fit::ShrinkToFit;
use triomphe::Arc; use triomphe::Arc;
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher}; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher};
pub use crate::serde::{set_de_map, set_ser_map};
use crate::{ use crate::{
dynamic::{deref_from, new_atom}, dynamic::{deref_from, new_atom},
tagged_value::TaggedValue, tagged_value::TaggedValue,
}; };
mod dynamic; mod dynamic;
mod serde;
mod tagged_value; mod tagged_value;
/// An immutable reference counted [`String`], similar to [`Arc<String>`][std::sync::Arc]. /// An immutable reference counted [`String`], similar to [`Arc<String>`][std::sync::Arc].
...@@ -318,19 +319,6 @@ impl Hash for RcStr { ...@@ -318,19 +319,6 @@ impl Hash for RcStr {
} }
} }
impl Serialize for RcStr {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for RcStr {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let s = String::deserialize(deserializer)?;
Ok(RcStr::from(s))
}
}
impl Drop for RcStr { impl Drop for RcStr {
fn drop(&mut self) { fn drop(&mut self) {
if self.tag() == DYNAMIC_TAG { if self.tag() == DYNAMIC_TAG {
......
use std::{cell::RefCell, hash::BuildHasherDefault};
use indexmap::IndexSet;
use rustc_hash::FxHasher;
use scoped_tls::scoped_thread_local;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use crate::RcStr;
scoped_thread_local!(
/// Map of strings to their interned ids.
///
/// This is used to serialize strings to their interned ids.
static SER_MAP: RefCell<IndexSet<RcStr, BuildHasherDefault<FxHasher>>>
);
scoped_thread_local!(
/// Read-only map interned ids to their strings
static DE_MAP: Vec<RcStr>
);
pub fn set_ser_map<F, R>(f: F) -> (R, IndexSet<RcStr, BuildHasherDefault<FxHasher>>)
where
F: FnOnce() -> R,
{
let map = Default::default();
let r = { SER_MAP.set(&map, f) };
(r, map.into_inner())
}
pub fn set_de_map<F, R>(map: &Vec<RcStr>, f: F) -> R
where
F: FnOnce() -> R,
{
DE_MAP.set(map, f)
}
/// Intern a string for serialization.
///
/// This function exists to move the logic for accessing the SER_MAP out of the hot path of
/// `Serialize::serialize`.
#[inline(never)]
fn intern_for_serialize(str: &RcStr) -> Option<u32> {
if !SER_MAP.is_set() {
return None;
}
Some(SER_MAP.with(|ser| {
let mut borrow = ser.borrow_mut();
if let Some(id) = borrow.get_index_of(str) {
id as u32
} else {
let id = borrow.len();
borrow.insert(str.clone());
id as u32
}
}))
}
impl Serialize for RcStr {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
// If it's too short, it's not worth interning.
// If it's too long, it's unlikely to be repeated.
if self.len() >= 3 && self.len() < 512 {
let id = intern_for_serialize(self);
if let Some(id) = id {
return serializer.serialize_u32(id);
}
}
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for RcStr {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
use serde::de::Error;
#[derive(Deserialize)]
#[serde(untagged)]
enum Repr {
Id(u32),
String(String),
}
let repr = Repr::deserialize(deserializer)?;
match repr {
Repr::String(s) => Ok(RcStr::from(s)),
Repr::Id(id) => DE_MAP.with(|map| {
let s = map
.get(id as usize)
.ok_or_else(|| D::Error::custom(format!("failed to find id: {id}")))?;
Ok(s.clone())
}),
}
}
}
...@@ -42,9 +42,10 @@ rayon = { workspace = true } ...@@ -42,9 +42,10 @@ rayon = { workspace = true }
ringmap = { workspace = true, features = ["serde"] } ringmap = { workspace = true, features = ["serde"] }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_bytes = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
serde_path_to_error = { workspace = true } serde_path_to_error = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true, features = ["write"] }
tokio = { workspace = true } tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
thread_local = { workspace = true } thread_local = { workspace = true }
......
...@@ -62,6 +62,7 @@ use crate::{ ...@@ -62,6 +62,7 @@ use crate::{
CachedDataItemValueRef, CellRef, CollectibleRef, CollectiblesRef, DirtyState, CachedDataItemValueRef, CellRef, CollectibleRef, CollectiblesRef, DirtyState,
InProgressCellState, InProgressState, InProgressStateInner, OutputValue, RootType, InProgressCellState, InProgressState, InProgressStateInner, OutputValue, RootType,
}, },
interning_serde::RcStrToLocalId,
utils::{ utils::{
bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc, sharded::Sharded, swap_retain, bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc, sharded::Sharded, swap_retain,
}, },
...@@ -962,12 +963,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -962,12 +963,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let task_snapshots = snapshot let task_snapshots = snapshot
.into_iter() .into_iter()
.filter_map(|iter| { .filter_map(|iter| {
type SerializedData = (SmallVec<[u8; 16]>, RcStrToLocalId);
let mut iter = iter let mut iter = iter
.filter_map( .filter_map(
|(task_id, meta, data): ( |(task_id, meta, data): (
_, _,
Option<Result<SmallVec<_>>>, Option<Result<SerializedData>>,
Option<Result<SmallVec<_>>>, Option<Result<SerializedData>>,
)| { )| {
let meta = match meta { let meta = match meta {
Some(Ok(meta)) => { Some(Ok(meta)) => {
...@@ -1169,7 +1172,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1169,7 +1172,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
.flatten(); .flatten();
let task_id = { let task_id = {
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`. // Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
if let Some(task_id) = unsafe { if let Some((task_id, _rcstr_map)) = unsafe {
// We can ignore rcstr_map because those are all already stored in the database.
self.backing_storage self.backing_storage
.forward_lookup_task_cache(tx.as_ref(), &task_type) .forward_lookup_task_cache(tx.as_ref(), &task_type)
.expect("Failed to lookup task id") .expect("Failed to lookup task id")
......
...@@ -7,6 +7,7 @@ use turbo_tasks::{SessionId, TaskId, backend::CachedTaskType}; ...@@ -7,6 +7,7 @@ use turbo_tasks::{SessionId, TaskId, backend::CachedTaskType};
use crate::{ use crate::{
backend::{AnyOperation, TaskDataCategory}, backend::{AnyOperation, TaskDataCategory},
data::CachedDataItem, data::CachedDataItem,
interning_serde::RcStrToLocalId,
utils::chunked_vec::ChunkedVec, utils::chunked_vec::ChunkedVec,
}; };
...@@ -48,7 +49,10 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -48,7 +49,10 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
fn next_session_id(&self) -> Result<SessionId>; fn next_session_id(&self) -> Result<SessionId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>; fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
#[allow(clippy::ptr_arg)] #[allow(clippy::ptr_arg)]
fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>>; fn serialize(
task: TaskId,
data: &Vec<CachedDataItem>,
) -> Result<(SmallVec<[u8; 16]>, RcStrToLocalId)>;
fn save_snapshot<I>( fn save_snapshot<I>(
&self, &self,
session_id: SessionId, session_id: SessionId,
...@@ -60,8 +64,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -60,8 +64,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
I: Iterator< I: Iterator<
Item = ( Item = (
TaskId, TaskId,
Option<SmallVec<[u8; 16]>>, Option<(SmallVec<[u8; 16]>, RcStrToLocalId)>,
Option<SmallVec<[u8; 16]>>, Option<(SmallVec<[u8; 16]>, RcStrToLocalId)>,
), ),
> + Send > + Send
+ Sync; + Sync;
...@@ -73,7 +77,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync { ...@@ -73,7 +77,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
&self, &self,
tx: Option<&Self::ReadTransaction<'_>>, tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType, key: &CachedTaskType,
) -> Result<Option<TaskId>>; ) -> Result<Option<(TaskId, RcStrToLocalId)>>;
/// # Safety /// # Safety
/// ///
/// `tx` must be a transaction from this BackingStorage instance. /// `tx` must be a transaction from this BackingStorage instance.
......
...@@ -11,6 +11,8 @@ pub enum KeySpace { ...@@ -11,6 +11,8 @@ pub enum KeySpace {
TaskData = 2, TaskData = 2,
ForwardTaskCache = 3, ForwardTaskCache = 3,
ReverseTaskCache = 4, ReverseTaskCache = 4,
StringInternMap = 5,
ReverseStringInternMap = 6,
} }
pub trait KeyValueDatabase { pub trait KeyValueDatabase {
......
...@@ -112,7 +112,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -112,7 +112,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
} }
pub struct TurboWriteBatch<'a> { pub struct TurboWriteBatch<'a> {
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>, batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 7>,
db: &'a Arc<TurboPersistence>, db: &'a Arc<TurboPersistence>,
compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>, compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>,
initial_write: bool, initial_write: bool,
......
//! Exposed for usage in `turbo-tasks-backend`
use std::io::Write;
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::FxIndexSet;
pub fn to_vec<T>(config: &pot::Config, value: &T) -> anyhow::Result<(Vec<u8>, RcStrToLocalId)>
where
T: Serialize,
{
let mut vec = Vec::new();
let ser_map = to_writer(config, value, &mut vec)?;
Ok((vec, ser_map))
}
pub struct LocalIdToRcStr(Vec<RcStr>);
impl From<Vec<RcStr>> for LocalIdToRcStr {
fn from(value: Vec<RcStr>) -> Self {
Self(value)
}
}
#[derive(Default)]
pub struct RcStrToLocalId(FxIndexSet<RcStr>);
impl RcStrToLocalId {
pub fn iter(&self) -> impl Iterator<Item = &RcStr> {
self.0.iter()
}
}
#[derive(Default)]
pub struct LocalIdToGlobalId(Vec<u32>);
impl From<Vec<u32>> for LocalIdToGlobalId {
fn from(value: Vec<u32>) -> Self {
Self(value)
}
}
impl LocalIdToGlobalId {
pub fn len(&self) -> usize {
self.0.len()
}
pub fn iter(&self) -> impl '_ + Iterator<Item = u32> {
self.0.iter().copied()
}
pub fn write_to(&self, writer: &mut impl Write) -> anyhow::Result<()> {
let len = self.0.len() as u32;
for id in self.0.iter().rev() {
writer.write_all(&id.to_le_bytes())?;
}
writer.write_all(&len.to_le_bytes())?;
Ok(())
}
pub fn read_from_slice(mut bytes: &[u8]) -> anyhow::Result<(Self, &[u8])> {
let mut global_ids = Vec::new();
// Length is the last 4 bytes
let len = u32::from_le_bytes(bytes[bytes.len() - 4..].try_into().unwrap());
global_ids.reserve(len as usize);
bytes = &bytes[..bytes.len() - 4];
// Read the ids in reverse order
for _ in 0..len {
let id = u32::from_le_bytes(bytes[bytes.len() - 4..].try_into().unwrap());
global_ids.push(id);
bytes = &bytes[..bytes.len() - 4];
}
Ok((Self(global_ids), bytes))
}
}
pub fn to_writer<T, W>(config: &pot::Config, value: &T, writer: W) -> anyhow::Result<RcStrToLocalId>
where
T: Serialize,
W: Write,
{
let (result, ser_map) = turbo_rcstr::set_ser_map(|| config.serialize_into(value, writer));
result?;
Ok(RcStrToLocalId(ser_map))
}
pub fn from_slice<'de, T>(
config: &pot::Config,
bytes: &'de [u8],
de_map: &LocalIdToRcStr,
) -> anyhow::Result<T>
where
T: Deserialize<'de>,
{
turbo_rcstr::set_de_map(&de_map.0, || Ok(config.deserialize(bytes)?))
}
...@@ -2,12 +2,13 @@ ...@@ -2,12 +2,13 @@
#![feature(associated_type_defaults)] #![feature(associated_type_defaults)]
#![feature(iter_collect_into)] #![feature(iter_collect_into)]
#![feature(box_patterns)] #![feature(box_patterns)]
#![feature(once_cell_try)]
mod backend; mod backend;
mod backing_storage; mod backing_storage;
mod data; mod data;
mod data_storage; mod data_storage;
mod database; mod database;
mod interning_serde;
mod kv_backing_storage; mod kv_backing_storage;
mod utils; mod utils;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment