Unverified Commit 2689a741 authored by Donny/강동윤's avatar Donny/강동윤 Committed by GitHub

perf(turbopack/rcstr): Add serialization-time optimization (#78645)

### What?

Interns `RcStr` as an index set upon serialization.

### Why?

It's required for future optimization.
parent 3d0e4721
......@@ -9163,9 +9163,11 @@ version = "0.1.0"
dependencies = [
"bytes-str",
"codspeed-criterion-compat",
"indexmap 2.9.0",
"napi",
"new_debug_unreachable",
"rustc-hash 2.1.1",
"scoped-tls",
"serde",
"shrink-to-fit",
"triomphe 0.1.12",
......@@ -9257,6 +9259,7 @@ dependencies = [
"rstest",
"rustc-hash 2.1.1",
"serde",
"serde_bytes",
"serde_json",
"serde_path_to_error",
"smallvec",
......
......@@ -13,16 +13,15 @@ mod db;
mod key;
mod lookup_entry;
mod merge_iter;
mod static_sorted_file;
mod static_sorted_file_builder;
mod write_batch;
mod meta_file;
mod meta_file_builder;
mod sst_filter;
mod static_sorted_file;
mod static_sorted_file_builder;
#[cfg(test)]
mod tests;
mod value_buf;
mod write_batch;
pub use arc_slice::ArcSlice;
pub use db::{MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
......
......@@ -15,11 +15,15 @@ turbo-tasks-hash = { workspace = true }
serde = { workspace = true }
new_debug_unreachable = "1.0.6"
shrink-to-fit = { workspace = true }
rustc-hash = { workspace = true }
bytes-str = { workspace = true }
indexmap = { workspace = true }
rustc-hash = { workspace = true }
scoped-tls = "1.0.1"
[target.'cfg(not(target_family = "wasm"))'.dependencies]
napi = { workspace = true, optional = true }
scoped-tls = "1.0.1"
indexmap = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
......
......@@ -11,17 +11,18 @@ use std::{
use bytes_str::BytesStr;
use debug_unreachable::debug_unreachable;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use shrink_to_fit::ShrinkToFit;
use triomphe::Arc;
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher};
pub use crate::serde::{set_de_map, set_ser_map};
use crate::{
dynamic::{deref_from, new_atom},
tagged_value::TaggedValue,
};
mod dynamic;
mod serde;
mod tagged_value;
/// An immutable reference counted [`String`], similar to [`Arc<String>`][std::sync::Arc].
......@@ -318,19 +319,6 @@ impl Hash for RcStr {
}
}
impl Serialize for RcStr {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for RcStr {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let s = String::deserialize(deserializer)?;
Ok(RcStr::from(s))
}
}
impl Drop for RcStr {
fn drop(&mut self) {
if self.tag() == DYNAMIC_TAG {
......
use std::{cell::RefCell, hash::BuildHasherDefault};
use indexmap::IndexSet;
use rustc_hash::FxHasher;
use scoped_tls::scoped_thread_local;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use crate::RcStr;
scoped_thread_local!(
/// Map of strings to their interned ids.
///
/// This is used to serialize strings to their interned ids.
static SER_MAP: RefCell<IndexSet<RcStr, BuildHasherDefault<FxHasher>>>
);
scoped_thread_local!(
/// Read-only map interned ids to their strings
static DE_MAP: Vec<RcStr>
);
pub fn set_ser_map<F, R>(f: F) -> (R, IndexSet<RcStr, BuildHasherDefault<FxHasher>>)
where
F: FnOnce() -> R,
{
let map = Default::default();
let r = { SER_MAP.set(&map, f) };
(r, map.into_inner())
}
pub fn set_de_map<F, R>(map: &Vec<RcStr>, f: F) -> R
where
F: FnOnce() -> R,
{
DE_MAP.set(map, f)
}
/// Intern a string for serialization.
///
/// This function exists to move the logic for accessing the SER_MAP out of the hot path of
/// `Serialize::serialize`.
#[inline(never)]
fn intern_for_serialize(str: &RcStr) -> Option<u32> {
if !SER_MAP.is_set() {
return None;
}
Some(SER_MAP.with(|ser| {
let mut borrow = ser.borrow_mut();
if let Some(id) = borrow.get_index_of(str) {
id as u32
} else {
let id = borrow.len();
borrow.insert(str.clone());
id as u32
}
}))
}
impl Serialize for RcStr {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
// If it's too short, it's not worth interning.
// If it's too long, it's unlikely to be repeated.
if self.len() >= 3 && self.len() < 512 {
let id = intern_for_serialize(self);
if let Some(id) = id {
return serializer.serialize_u32(id);
}
}
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for RcStr {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
use serde::de::Error;
#[derive(Deserialize)]
#[serde(untagged)]
enum Repr {
Id(u32),
String(String),
}
let repr = Repr::deserialize(deserializer)?;
match repr {
Repr::String(s) => Ok(RcStr::from(s)),
Repr::Id(id) => DE_MAP.with(|map| {
let s = map
.get(id as usize)
.ok_or_else(|| D::Error::custom(format!("failed to find id: {id}")))?;
Ok(s.clone())
}),
}
}
}
......@@ -42,9 +42,10 @@ rayon = { workspace = true }
ringmap = { workspace = true, features = ["serde"] }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_bytes = { workspace = true }
serde_json = { workspace = true }
serde_path_to_error = { workspace = true }
smallvec = { workspace = true }
smallvec = { workspace = true, features = ["write"] }
tokio = { workspace = true }
tracing = { workspace = true }
thread_local = { workspace = true }
......
......@@ -62,6 +62,7 @@ use crate::{
CachedDataItemValueRef, CellRef, CollectibleRef, CollectiblesRef, DirtyState,
InProgressCellState, InProgressState, InProgressStateInner, OutputValue, RootType,
},
interning_serde::RcStrToLocalId,
utils::{
bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc, sharded::Sharded, swap_retain,
},
......@@ -962,12 +963,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let task_snapshots = snapshot
.into_iter()
.filter_map(|iter| {
type SerializedData = (SmallVec<[u8; 16]>, RcStrToLocalId);
let mut iter = iter
.filter_map(
|(task_id, meta, data): (
_,
Option<Result<SmallVec<_>>>,
Option<Result<SmallVec<_>>>,
Option<Result<SerializedData>>,
Option<Result<SerializedData>>,
)| {
let meta = match meta {
Some(Ok(meta)) => {
......@@ -1169,7 +1172,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
.flatten();
let task_id = {
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
if let Some(task_id) = unsafe {
if let Some((task_id, _rcstr_map)) = unsafe {
// We can ignore rcstr_map because those are all already stored in the database.
self.backing_storage
.forward_lookup_task_cache(tx.as_ref(), &task_type)
.expect("Failed to lookup task id")
......
......@@ -7,6 +7,7 @@ use turbo_tasks::{SessionId, TaskId, backend::CachedTaskType};
use crate::{
backend::{AnyOperation, TaskDataCategory},
data::CachedDataItem,
interning_serde::RcStrToLocalId,
utils::chunked_vec::ChunkedVec,
};
......@@ -48,7 +49,10 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
fn next_session_id(&self) -> Result<SessionId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
#[allow(clippy::ptr_arg)]
fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>>;
fn serialize(
task: TaskId,
data: &Vec<CachedDataItem>,
) -> Result<(SmallVec<[u8; 16]>, RcStrToLocalId)>;
fn save_snapshot<I>(
&self,
session_id: SessionId,
......@@ -60,8 +64,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
I: Iterator<
Item = (
TaskId,
Option<SmallVec<[u8; 16]>>,
Option<SmallVec<[u8; 16]>>,
Option<(SmallVec<[u8; 16]>, RcStrToLocalId)>,
Option<(SmallVec<[u8; 16]>, RcStrToLocalId)>,
),
> + Send
+ Sync;
......@@ -73,7 +77,7 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
&self,
tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType,
) -> Result<Option<TaskId>>;
) -> Result<Option<(TaskId, RcStrToLocalId)>>;
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
......
......@@ -11,6 +11,8 @@ pub enum KeySpace {
TaskData = 2,
ForwardTaskCache = 3,
ReverseTaskCache = 4,
StringInternMap = 5,
ReverseStringInternMap = 6,
}
pub trait KeyValueDatabase {
......
......@@ -112,7 +112,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
}
pub struct TurboWriteBatch<'a> {
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>,
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 7>,
db: &'a Arc<TurboPersistence>,
compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>,
initial_write: bool,
......
//! Exposed for usage in `turbo-tasks-backend`
use std::io::Write;
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::FxIndexSet;
pub fn to_vec<T>(config: &pot::Config, value: &T) -> anyhow::Result<(Vec<u8>, RcStrToLocalId)>
where
T: Serialize,
{
let mut vec = Vec::new();
let ser_map = to_writer(config, value, &mut vec)?;
Ok((vec, ser_map))
}
pub struct LocalIdToRcStr(Vec<RcStr>);
impl From<Vec<RcStr>> for LocalIdToRcStr {
fn from(value: Vec<RcStr>) -> Self {
Self(value)
}
}
#[derive(Default)]
pub struct RcStrToLocalId(FxIndexSet<RcStr>);
impl RcStrToLocalId {
pub fn iter(&self) -> impl Iterator<Item = &RcStr> {
self.0.iter()
}
}
#[derive(Default)]
pub struct LocalIdToGlobalId(Vec<u32>);
impl From<Vec<u32>> for LocalIdToGlobalId {
fn from(value: Vec<u32>) -> Self {
Self(value)
}
}
impl LocalIdToGlobalId {
pub fn len(&self) -> usize {
self.0.len()
}
pub fn iter(&self) -> impl '_ + Iterator<Item = u32> {
self.0.iter().copied()
}
pub fn write_to(&self, writer: &mut impl Write) -> anyhow::Result<()> {
let len = self.0.len() as u32;
for id in self.0.iter().rev() {
writer.write_all(&id.to_le_bytes())?;
}
writer.write_all(&len.to_le_bytes())?;
Ok(())
}
pub fn read_from_slice(mut bytes: &[u8]) -> anyhow::Result<(Self, &[u8])> {
let mut global_ids = Vec::new();
// Length is the last 4 bytes
let len = u32::from_le_bytes(bytes[bytes.len() - 4..].try_into().unwrap());
global_ids.reserve(len as usize);
bytes = &bytes[..bytes.len() - 4];
// Read the ids in reverse order
for _ in 0..len {
let id = u32::from_le_bytes(bytes[bytes.len() - 4..].try_into().unwrap());
global_ids.push(id);
bytes = &bytes[..bytes.len() - 4];
}
Ok((Self(global_ids), bytes))
}
}
pub fn to_writer<T, W>(config: &pot::Config, value: &T, writer: W) -> anyhow::Result<RcStrToLocalId>
where
T: Serialize,
W: Write,
{
let (result, ser_map) = turbo_rcstr::set_ser_map(|| config.serialize_into(value, writer));
result?;
Ok(RcStrToLocalId(ser_map))
}
pub fn from_slice<'de, T>(
config: &pot::Config,
bytes: &'de [u8],
de_map: &LocalIdToRcStr,
) -> anyhow::Result<T>
where
T: Deserialize<'de>,
{
turbo_rcstr::set_de_map(&de_map.0, || Ok(config.deserialize(bytes)?))
}
......@@ -2,12 +2,13 @@
#![feature(associated_type_defaults)]
#![feature(iter_collect_into)]
#![feature(box_patterns)]
#![feature(once_cell_try)]
mod backend;
mod backing_storage;
mod data;
mod data_storage;
mod database;
mod interning_serde;
mod kv_backing_storage;
mod utils;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment