Unverified Commit 5780d18a authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Deduplicate persisted filesystem paths

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 8a54c08e
......@@ -2,7 +2,7 @@
pub mod macro_helpers;
pub mod serde_self_describing;
use std::{any::Any, ptr::copy_nonoverlapping};
use std::{any::Any, mem::transmute, ptr::copy_nonoverlapping};
use ::smallvec::SmallVec;
use bincode::{
......@@ -29,14 +29,101 @@ pub type TurboBincodeDecoder<'a> =
pub type AnyEncodeFn = fn(&dyn Any, &mut TurboBincodeEncoder<'_>) -> Result<(), EncodeError>;
pub type AnyDecodeFn<T> = fn(&mut TurboBincodeDecoder<'_>) -> Result<T, DecodeError>;
/// A stable reference to a string stored outside of an individual bincode value.
///
/// `namespace` identifies the owner of the string table (for example, a filesystem), while `hash`
/// identifies the string by content within that namespace.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct InternedStringKey {
pub namespace: [u8; 10],
pub hash: [u8; 16],
}
/// Collects external strings referenced by one independently encoded bincode value.
///
/// The value is borrowed and type-erased so a higher-level owner can cheaply clone its concrete
/// shared-string type without this crate depending on that type or allocating per occurrence.
pub trait InternedStringCollector {
fn insert(
&mut self,
key: InternedStringKey,
value: &(dyn Any + Send + Sync),
) -> Result<(), EncodeError>;
}
/// Resolves a persistence string reference to its concrete, allocation-sharing value.
///
/// The erased return value keeps this low-level crate independent of the concrete shared-string
/// implementation. Callers must downcast it to the type registered by the resolver.
pub trait InternedStringResolver: Send + Sync {
fn resolve(
&self,
key: InternedStringKey,
receiver: &mut dyn FnMut(&(dyn Any + Send + Sync)),
) -> Result<(), DecodeError>;
}
pub fn new_turbo_bincode_encoder(buf: &mut TurboBincodeBuffer) -> TurboBincodeEncoder<'_> {
EncoderImpl::new(TurboBincodeWriter::new(buf), TURBO_BINCODE_CONFIG)
}
pub fn new_turbo_bincode_encoder_with_interned_strings<'a>(
buf: &'a mut TurboBincodeBuffer,
interned_strings: &'a mut dyn InternedStringCollector,
) -> TurboBincodeEncoder<'a> {
EncoderImpl::new(
TurboBincodeWriter::with_interned_strings(buf, interned_strings),
TURBO_BINCODE_CONFIG,
)
}
pub fn new_turbo_bincode_decoder(buffer: &[u8]) -> TurboBincodeDecoder<'_> {
DecoderImpl::new(TurboBincodeReader::new(buffer), TURBO_BINCODE_CONFIG, ())
}
pub fn new_turbo_bincode_decoder_with_interned_strings<'a>(
buffer: &'a [u8],
resolver: &'a dyn InternedStringResolver,
) -> TurboBincodeDecoder<'a> {
DecoderImpl::new(
TurboBincodeReader::with_interned_strings(buffer, resolver),
TURBO_BINCODE_CONFIG,
(),
)
}
/// Runs `f` when `encoder` is the concrete encoder used by turbo persistence.
///
/// Hash encoders and other bincode encoders return `None`, allowing callers to preserve a semantic
/// encoding that is independent of persistence-only string references.
pub fn with_turbo_bincode_encoder<E: Encoder, T>(
encoder: &mut E,
f: impl FnOnce(&mut TurboBincodeEncoder<'_>) -> T,
) -> Option<T> {
if unty::type_equal::<E, TurboBincodeEncoder>() {
// SAFETY: This is the same type-and-lifetime specialization used by the helper macros in
// `macro_helpers`. `type_equal` checks the concrete layout and the reference cannot escape
// this function.
let encoder = unsafe { transmute::<&mut E, &mut TurboBincodeEncoder<'_>>(encoder) };
Some(f(encoder))
} else {
None
}
}
pub fn with_turbo_bincode_decoder<Context, D: Decoder<Context = Context>, T>(
decoder: &mut D,
f: impl FnOnce(&mut TurboBincodeDecoder<'_>) -> T,
) -> Option<T> {
if unty::type_equal::<D, TurboBincodeDecoder>() {
// SAFETY: See `with_turbo_bincode_encoder`.
let decoder = unsafe { transmute::<&mut D, &mut TurboBincodeDecoder<'_>>(decoder) };
Some(f(decoder))
} else {
None
}
}
/// Encode the value into a new [`SmallVec`] using a [`TurboBincodeEncoder`].
///
/// Note: If you can re-use a buffer, you should. That will always be cheaper than creating a new
......@@ -73,11 +160,31 @@ pub fn turbo_bincode_decode<T: Decode<()>>(buf: &[u8]) -> Result<T, DecodeError>
pub struct TurboBincodeWriter<'a> {
pub buffer: &'a mut TurboBincodeBuffer,
interned_strings: Option<&'a mut dyn InternedStringCollector>,
}
impl<'a> TurboBincodeWriter<'a> {
pub fn new(buffer: &'a mut TurboBincodeBuffer) -> Self {
Self { buffer }
Self {
buffer,
interned_strings: None,
}
}
pub fn with_interned_strings(
buffer: &'a mut TurboBincodeBuffer,
interned_strings: &'a mut dyn InternedStringCollector,
) -> Self {
Self {
buffer,
interned_strings: Some(interned_strings),
}
}
pub fn interned_strings_mut(&mut self) -> Option<&mut dyn InternedStringCollector> {
self.interned_strings
.as_mut()
.map(|collector| &mut **collector as &mut dyn InternedStringCollector)
}
}
......@@ -92,11 +199,29 @@ impl Writer for TurboBincodeWriter<'_> {
/// avoid some redundant bounds checks, and `pub` access to the underlying `buffer`.
pub struct TurboBincodeReader<'a> {
pub buffer: &'a [u8],
interned_string_resolver: Option<&'a dyn InternedStringResolver>,
}
impl<'a> TurboBincodeReader<'a> {
pub fn new(buffer: &'a [u8]) -> Self {
Self { buffer }
Self {
buffer,
interned_string_resolver: None,
}
}
pub fn with_interned_strings(
buffer: &'a [u8],
resolver: &'a dyn InternedStringResolver,
) -> Self {
Self {
buffer,
interned_string_resolver: Some(resolver),
}
}
pub fn interned_string_resolver(&self) -> Option<&dyn InternedStringResolver> {
self.interned_string_resolver
}
}
......
......@@ -33,7 +33,11 @@ use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
use smallvec::{SmallVec, smallvec};
use tokio::time::{Duration, Instant};
use tracing::{Span, field::display, trace_span};
use turbo_bincode::{TurboBincodeBuffer, new_turbo_bincode_decoder, new_turbo_bincode_encoder};
use turbo_bincode::{
InternedStringKey, InternedStringResolver, TurboBincodeBuffer,
new_turbo_bincode_decoder_with_interned_strings,
new_turbo_bincode_encoder_with_interned_strings,
};
use turbo_tasks::{
CellId, DynTaskInputsStorage, RawVc, RawVcUnpacked, ReadCellOptions, ReadCellTracking,
ReadConsistency, ReadOutcome, ReadOutputOptions, ReadTracking, SharedReference,
......@@ -77,7 +81,7 @@ use crate::{
storage::Storage,
storage_schema::{TaskStorage, TaskStorageAccessors},
},
backing_storage::{SnapshotItem, compute_task_type_hash},
backing_storage::{EncodedTaskData, InternedStrings, SnapshotItem, compute_task_type_hash},
data::{
ActivenessState, CellRef, CollectibleRef, CollectiblesRef, Dirtyness, InProgressCellState,
InProgressState, InProgressStateInner, OutputValue, TransientTask,
......@@ -1378,7 +1382,7 @@ impl TurboTasksBackend {
data: &TaskStorage,
category: SpecificTaskDataCategory,
buffer: &mut TurboBincodeBuffer|
-> Option<TurboBincodeBuffer> {
-> Option<EncodedTaskData> {
match encode_task_data(task_id, data, category, buffer) {
Ok(encoded) => {
#[cfg(feature = "print_cache_item_size")]
......@@ -1386,8 +1390,8 @@ impl TurboTasksBackend {
let mut stats = task_cache_stats.lock();
let entry = stats.entry(TaskCacheStats::task_name(inner)).or_default();
match category {
SpecificTaskDataCategory::Meta => entry.add_meta(&encoded),
SpecificTaskDataCategory::Data => entry.add_data(&encoded),
SpecificTaskDataCategory::Meta => entry.add_meta(&encoded.bytes),
SpecificTaskDataCategory::Data => entry.add_data(&encoded.bytes),
}
}
Some(encoded)
......@@ -4163,16 +4167,38 @@ fn encode_task_data(
data: &TaskStorage,
category: SpecificTaskDataCategory,
scratch_buffer: &mut TurboBincodeBuffer,
) -> Result<TurboBincodeBuffer> {
) -> Result<EncodedTaskData> {
scratch_buffer.clear();
let mut encoder = new_turbo_bincode_encoder(scratch_buffer);
let mut interned_strings = InternedStrings::default();
let mut encoder =
new_turbo_bincode_encoder_with_interned_strings(scratch_buffer, &mut interned_strings);
data.encode(category, &mut encoder)?;
if cfg!(feature = "verify_serialization") {
struct CollectedResolver<'a>(&'a InternedStrings);
impl InternedStringResolver for CollectedResolver<'_> {
fn resolve(
&self,
key: InternedStringKey,
receiver: &mut dyn FnMut(&(dyn std::any::Any + Send + Sync)),
) -> Result<(), bincode::error::DecodeError> {
let value = self.0.get(&key).ok_or_else(|| {
bincode::error::DecodeError::OtherString(format!(
"missing collected interned string {key:?}"
))
})?;
receiver(value);
Ok(())
}
}
TaskStorage::new()
.decode(
category,
&mut new_turbo_bincode_decoder(&scratch_buffer[..]),
&mut new_turbo_bincode_decoder_with_interned_strings(
&scratch_buffer[..],
&CollectedResolver(&interned_strings),
),
)
.with_context(|| {
format!(
......@@ -4181,5 +4207,8 @@ fn encode_task_data(
)
})?;
}
Ok(SmallVec::from_slice(scratch_buffer))
Ok(EncodedTaskData {
bytes: SmallVec::from_slice(scratch_buffer),
interned_strings,
})
}
......@@ -1147,7 +1147,7 @@ mod tests {
use turbo_tasks::TaskId;
use super::{SpecificTaskDataCategory, Storage, TrackOutcome};
use crate::backing_storage::SnapshotItem;
use crate::backing_storage::{EncodedTaskData, SnapshotItem};
fn non_transient_task(id: u32) -> TaskId {
// TRANSIENT_TASK_BIT is 0x2000_0000; any id without that bit is non-transient.
......@@ -1175,7 +1175,10 @@ mod tests {
) -> SnapshotItem {
SnapshotItem::Put {
task_id,
meta: Some(TurboBincodeBuffer::default()),
meta: Some(EncodedTaskData {
bytes: TurboBincodeBuffer::default(),
interned_strings: Default::default(),
}),
data: None,
task_type_hash: None,
}
......
use std::cmp::max;
use std::{any::Any, cmp::max};
use turbo_bincode::TurboBincodeBuffer;
use bincode::error::EncodeError;
use smallvec::SmallVec;
use turbo_bincode::{InternedStringCollector, InternedStringKey, TurboBincodeBuffer};
use turbo_rcstr::RcStr;
use turbo_tasks::{
DynTaskInputs, RawVc, TaskId, backend::CachedTaskType, macro_helpers::NativeFunction,
};
......@@ -8,6 +11,51 @@ use turbo_tasks_hash::Xxh3Hash64Hasher;
pub type TaskTypeHash = [u8; 8];
#[derive(Default)]
pub struct InternedStrings(SmallVec<[(InternedStringKey, RcStr); 1]>);
impl InternedStrings {
pub fn get(&self, key: &InternedStringKey) -> Option<&RcStr> {
self.0
.iter()
.find_map(|(entry_key, value)| (entry_key == key).then_some(value))
}
pub fn into_entries(self) -> impl Iterator<Item = (InternedStringKey, RcStr)> {
self.0.into_iter()
}
}
impl InternedStringCollector for InternedStrings {
fn insert(
&mut self,
key: InternedStringKey,
value: &(dyn Any + Send + Sync),
) -> Result<(), EncodeError> {
let value = value.downcast_ref::<RcStr>().ok_or_else(|| {
EncodeError::OtherString(
"FileSystemPath collector received the wrong value type".into(),
)
})?;
if let Some(existing) = self.get(&key) {
if existing != value {
return Err(EncodeError::OtherString(format!(
"interned string identity collision for {key:?}"
)));
}
} else {
self.0.push((key, value.clone()));
}
Ok(())
}
}
/// One independently encoded task-data record and the external strings it references.
pub struct EncodedTaskData {
pub bytes: TurboBincodeBuffer,
pub interned_strings: InternedStrings,
}
/// A single item yielded by the snapshot iterator during persistence: either a put (persist a
/// modified task's meta/data + optionally register a new task's type) or a delete (tombstone a
/// GC-collected task's on-disk copy). Both ride the one iterator `save_snapshot` consumes, so
......@@ -16,9 +64,9 @@ pub enum SnapshotItem {
Put {
task_id: TaskId,
/// Serialized task meta data, if modified
meta: Option<TurboBincodeBuffer>,
meta: Option<EncodedTaskData>,
/// Serialized task data, if modified
data: Option<TurboBincodeBuffer>,
data: Option<EncodedTaskData>,
/// Task type for new tasks that need to be added to the task cache
task_type_hash: Option<TaskTypeHash>,
},
......
......@@ -6,6 +6,9 @@ pub enum KeySpace {
TaskMeta = 1,
TaskData = 2,
TaskCache = 3,
FileSystemPath = 4,
TaskMetaPathRefs = 5,
TaskDataPathRefs = 6,
}
impl KeySpace {
/// Constructs a [`KeySpace`] from its numeric index (i.e., the `usize` discriminant).
......@@ -19,6 +22,9 @@ impl KeySpace {
1 => KeySpace::TaskMeta,
2 => KeySpace::TaskData,
3 => KeySpace::TaskCache,
4 => KeySpace::FileSystemPath,
5 => KeySpace::TaskMetaPathRefs,
6 => KeySpace::TaskDataPathRefs,
_ => panic!("KeySpace index out of range"),
}
}
......@@ -29,6 +35,9 @@ impl KeySpace {
KeySpace::TaskMeta => "TaskMeta",
KeySpace::TaskData => "TaskData",
KeySpace::TaskCache => "TaskCache",
KeySpace::FileSystemPath => "FileSystemPath",
KeySpace::TaskMetaPathRefs => "TaskMetaPathRefs",
KeySpace::TaskDataPathRefs => "TaskDataPathRefs",
}
}
......@@ -51,6 +60,13 @@ impl KeySpace {
kind: FamilyKind::MultiValue,
compression: Compression::Lz4,
},
KeySpace::FileSystemPath | KeySpace::TaskMetaPathRefs | KeySpace::TaskDataPathRefs => {
FamilyConfig {
name: self.name(),
kind: FamilyKind::SingleValue,
compression: Compression::Lz4,
}
}
}
}
}
......@@ -26,7 +26,7 @@ mod parallel_scheduler;
pub(crate) use parallel_scheduler::TurboTasksParallelScheduler;
/// Number of key families, see [`KeySpace`] enum for their numbers.
pub const FAMILIES: usize = 4;
pub const FAMILIES: usize = 7;
const COMPACTION_MESSAGE: &str = "Finished filesystem cache database compaction";
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment