Unverified Commit 04a08435 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Remove LMDB backend and ReadTransaction abstractions from turbo-tasks-backend (#91284)

## Summary

Remove the unused LMDB database backend and the `ReadTransaction` abstraction layer from `turbo-tasks-backend`. This is a pure cleanup that deletes ~1,500 lines of code and eliminates all `unsafe` blocks related to transaction lifetime management.

**Motivation:**
- This was much more useful in the early days of the persistence layer where comparing against a 'known good' db was useful, but this has rotted and turbo-persistence has gained lots of tests and usage so that this is not needed.
- Carrying the code added a lot of complexity to APIs e.g.
    - The `ReadTransaction` abstraction existed solely to support LMDB's transaction-based read model
    - The only remaining backend (`TurboKeyValueDatabase` via `turbo-persistence`) already used `()` as its `ReadTransaction` type, making the entire abstraction dead weight
    - The transaction plumbing required `unsafe` transmute-based lifetime extension in `ExecuteContextImpl`, adding complexity and risk for no benefit
    - The WriteBatch::Serial branches complicated already subtle code paths
parent a65c3c86
......@@ -266,15 +266,6 @@ dependencies = [
"derive_arbitrary",
]
[[package]]
name = "arc-swap"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
dependencies = [
"rustversion",
]
[[package]]
name = "arg_enum_proc_macro"
version = "0.3.4"
......@@ -3936,29 +3927,6 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
[[package]]
name = "lmdb-rkv"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "447a296f7aca299cfbb50f4e4f3d49451549af655fb7215d7f8c0c3d64bad42b"
dependencies = [
"bitflags 1.3.2",
"byteorder",
"libc",
"lmdb-rkv-sys",
]
[[package]]
name = "lmdb-rkv-sys"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "lock_api"
version = "0.4.10"
......@@ -9572,11 +9540,9 @@ name = "turbo-tasks-backend"
version = "0.1.0"
dependencies = [
"anyhow",
"arc-swap",
"auto-hash-map",
"bincode 2.0.1",
"bitfield",
"byteorder",
"codspeed-criterion-compat",
"dashmap 6.1.0",
"either",
......@@ -9584,7 +9550,6 @@ dependencies = [
"hashbrown 0.14.5",
"indexmap 2.13.0",
"indoc",
"lmdb-rkv",
"lzzzz",
"once_cell",
"parking_lot",
......@@ -9598,7 +9563,6 @@ dependencies = [
"serde_path_to_error",
"smallvec",
"tempfile",
"thread_local",
"tokio",
"tracing",
"turbo-bincode",
......
......@@ -23,13 +23,13 @@ use turbo_tasks::{
message_queue::{CompilationEvent, Severity},
};
use turbo_tasks_backend::{
BackendOptions, DefaultBackingStorage, GitVersionInfo, NoopBackingStorage, StartupCacheState,
TurboTasksBackend, db_invalidation::invalidation_reasons, default_backing_storage,
noop_backing_storage,
BackendOptions, GitVersionInfo, NoopBackingStorage, StartupCacheState, TurboBackingStorage,
TurboTasksBackend, db_invalidation::invalidation_reasons, noop_backing_storage,
turbo_backing_storage,
};
pub type NextTurboTasks =
Arc<TurboTasks<TurboTasksBackend<Either<DefaultBackingStorage, NoopBackingStorage>>>>;
Arc<TurboTasks<TurboTasksBackend<Either<TurboBackingStorage, NoopBackingStorage>>>>;
/// A value often wrapped in [`napi::bindgen_prelude::External`] that retains the [TurboTasks]
/// instance used by Next.js, and [various napi helpers that are passed to us from
......@@ -222,7 +222,7 @@ pub fn create_turbo_tasks(
dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true",
};
let (backing_storage, cache_state) = default_backing_storage(
let (backing_storage, cache_state) = turbo_backing_storage(
&output_path.join("cache/turbopack"),
&version_info,
is_ci,
......
......@@ -30,20 +30,16 @@ trace_task_modification = []
trace_task_dirty = ["trace_aggregation_update_queue"]
trace_task_output_dependencies = []
trace_task_details = []
lmdb = ["dep:lmdb-rkv"]
[dependencies]
anyhow = { workspace = true }
bincode = { workspace = true }
arc-swap = { version = "1.8.2" }
auto-hash-map = { workspace = true }
bitfield = { workspace = true }
byteorder = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]}
either = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] }
indexmap = { workspace = true }
lmdb-rkv = { version = "0.14.0", optional = true }
lzzzz = { workspace = true, optional = true }
once_cell = { workspace = true }
parking_lot = { workspace = true }
......@@ -56,7 +52,6 @@ serde_path_to_error = { workspace = true }
smallvec = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
thread_local = { workspace = true }
turbo-bincode = { workspace = true }
turbo-persistence = { workspace = true }
turbo-rcstr = { workspace = true }
......
......@@ -183,7 +183,8 @@ struct TurboTasksBackendInner<B: BackingStorage> {
storage: Storage,
/// When true, the backing_storage has data that is not in the local storage.
local_is_partial: AtomicBool,
/// This is determined once at startup and never changes.
local_is_partial: bool,
/// Number of executing operations + Highest bit is set when snapshot is
/// requested. When that bit is set, operations should pause until the
......@@ -258,7 +259,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
),
persisted_task_cache_log: need_log.then(|| Sharded::new(shard_amount)),
task_cache: FxDashMap::default(),
local_is_partial: AtomicBool::new(next_task_id != TaskId::MIN),
local_is_partial: next_task_id != TaskId::MIN,
storage: Storage::new(shard_amount, small_preallocation),
in_progress_operations: AtomicUsize::new(0),
snapshot_request: Mutex::new(SnapshotRequest::new()),
......
......@@ -9,7 +9,6 @@ mod update_cell;
mod update_collectible;
use std::{
fmt::{Debug, Display, Formatter},
mem::transmute,
sync::{Arc, atomic::Ordering},
};
......@@ -26,7 +25,7 @@ use crate::{
storage::{SpecificTaskDataCategory, StorageWriteGuard},
storage_schema::{TaskStorage, TaskStorageAccessors},
},
backing_storage::{BackingStorage, BackingStorageSealed},
backing_storage::BackingStorage,
data::{ActivenessState, CollectibleRef, Dirtyness, InProgressState, TransientTask},
};
......@@ -34,11 +33,6 @@ pub trait Operation: Encode + Decode<()> + Default + TryFrom<AnyOperation, Error
fn execute(self, ctx: &mut impl ExecuteContext<'_>);
}
enum TransactionState<'tx, B: BackingStorage> {
None,
Owned(Option<B::ReadTransaction<'tx>>),
}
pub trait ExecuteContext<'e>: Sized {
type TaskGuardImpl: TaskGuard + 'e;
fn child_context<'l, 'r>(&'r self) -> impl ChildExecuteContext<'l> + use<'e, 'l, Self>
......@@ -104,23 +98,15 @@ pub trait ChildExecuteContext<'e>: Send + Sized {
fn create(self) -> impl ExecuteContext<'e>;
}
pub struct ExecuteContextImpl<'e, 'tx, B: BackingStorage>
where
Self: 'e,
'tx: 'e,
{
pub struct ExecuteContextImpl<'e, B: BackingStorage> {
backend: &'e TurboTasksBackendInner<B>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
_operation_guard: Option<OperationGuard<'e, B>>,
transaction: TransactionState<'tx, B>,
#[cfg(debug_assertions)]
active_task_locks: std::sync::Arc<std::sync::atomic::AtomicU8>,
}
impl<'e, 'tx, B: BackingStorage> ExecuteContextImpl<'e, 'tx, B>
where
'tx: 'e,
{
impl<'e, B: BackingStorage> ExecuteContextImpl<'e, B> {
pub(super) fn new(
backend: &'e TurboTasksBackendInner<B>,
turbo_tasks: &'e dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
......@@ -129,50 +115,29 @@ where
backend,
turbo_tasks,
_operation_guard: Some(backend.start_operation()),
transaction: TransactionState::None,
#[cfg(debug_assertions)]
active_task_locks: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(0)),
}
}
fn ensure_transaction(&mut self) -> bool {
if matches!(self.transaction, TransactionState::None) {
let check_backing_storage = self.backend.should_restore()
&& self.backend.local_is_partial.load(Ordering::Acquire);
if !check_backing_storage {
return false;
}
let tx = self.backend.backing_storage.start_read_transaction();
let tx = tx.map(|tx| {
// Safety: `self.backend.backing_storage` lives for at least `'e`. The
// transaction returned by `start_read_transaction()` borrows it for `'e`.
// Since `'tx: 'e`, transmuting to `'tx` is sound because the transaction
// is stored in `self.transaction` (as `Owned`) and dropped with `self`,
// while `self.backend` remains alive for `'e`.
unsafe { transmute::<B::ReadTransaction<'_>, B::ReadTransaction<'tx>>(tx) }
});
self.transaction = TransactionState::Owned(tx);
}
true
fn should_check_backing_storage(&self) -> bool {
self.backend.should_restore() && self.backend.local_is_partial
}
fn restore_task_data(
&mut self,
&self,
task_id: TaskId,
category: SpecificTaskDataCategory,
) -> TaskStorage {
if !self.ensure_transaction() {
if !self.should_check_backing_storage() {
// If we don't need to restore, we can just return an empty storage
return TaskStorage::default();
}
let tx = self.get_tx();
let mut storage = TaskStorage::default();
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
let result = unsafe {
self.backend
.backing_storage
.lookup_data(tx, task_id, category, &mut storage)
};
let result = self
.backend
.backing_storage
.lookup_data(task_id, category, &mut storage);
match result {
Ok(()) => storage,
......@@ -186,7 +151,7 @@ where
}
fn restore_task_data_batch(
&mut self,
&self,
task_ids: &[TaskId],
category: SpecificTaskDataCategory,
) -> Option<Vec<TaskStorage>> {
......@@ -194,17 +159,14 @@ where
task_ids.len() > 1,
"Use restore_task_data_typed for single task"
);
if !self.ensure_transaction() {
if !self.should_check_backing_storage() {
// If we don't need to restore, we return None
return None;
}
let tx = self.get_tx();
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
let result = unsafe {
self.backend
.backing_storage
.batch_lookup_data(tx, task_ids, category)
};
let result = self
.backend
.backing_storage
.batch_lookup_data(task_ids, category);
match result {
Ok(result) => Some(result),
Err(e) => {
......@@ -219,13 +181,6 @@ where
}
}
fn get_tx(&self) -> Option<&<B as BackingStorageSealed>::ReadTransaction<'tx>> {
match &self.transaction {
TransactionState::None => unreachable!(),
TransactionState::Owned(tx) => tx.as_ref(),
}
}
fn prepare_tasks_with_callback(
&mut self,
task_ids: impl IntoIterator<Item = (TaskId, TaskDataCategory)>,
......@@ -396,13 +351,10 @@ where
}
}
impl<'e, 'tx, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, 'tx, B>
where
'tx: 'e,
{
impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> {
type TaskGuardImpl = TaskGuardImpl<'e>;
fn child_context<'l, 'r>(&'r self) -> impl ChildExecuteContext<'l> + use<'e, 'tx, 'l, B>
fn child_context<'l, 'r>(&'r self) -> impl ChildExecuteContext<'l> + use<'e, 'l, B>
where
'e: 'l,
{
......@@ -636,20 +588,16 @@ where
}
fn task_by_type(&mut self, task_type: &CachedTaskType) -> Option<TaskId> {
// Ensure we have a transaction (this will be reused by subsequent task() calls)
if !self.ensure_transaction() {
if !self.should_check_backing_storage() {
return None;
}
let tx = self.get_tx();
// Get candidates from backing storage (hash-based lookup may return multiple)
// Safety: `tx` is a valid transaction from `self.backend.backing_storage`.
let candidates = unsafe {
self.backend
.backing_storage
.lookup_task_candidates(tx, task_type)
.expect("Failed to lookup task ids")
};
let candidates = self
.backend
.backing_storage
.lookup_task_candidates(task_type)
.expect("Failed to lookup task ids");
// Verify each candidate by comparing the stored persistent_task_type.
// Only rarely is there more than one candidate, so no need for parallelization.
......@@ -676,7 +624,6 @@ impl<'e, B: BackingStorage> ChildExecuteContext<'e> for ChildExecuteContextImpl<
backend: self.backend,
turbo_tasks: self.turbo_tasks,
_operation_guard: None,
transaction: TransactionState::None,
#[cfg(debug_assertions)]
active_task_locks: std::sync::Arc::new(std::sync::atomic::AtomicU8::new(0)),
}
......
use std::{any::type_name, sync::Arc};
use std::sync::Arc;
use anyhow::Result;
use either::Either;
......@@ -24,11 +24,11 @@ impl SnapshotItem {
}
/// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by
/// [`default_backing_storage`] or [`noop_backing_storage`].
/// [`turbo_backing_storage`] or [`noop_backing_storage`].
///
/// This trait is [sealed]. External crates are not allowed to implement it.
///
/// [`default_backing_storage`]: crate::default_backing_storage
/// [`turbo_backing_storage`]: crate::turbo_backing_storage
/// [`noop_backing_storage`]: crate::noop_backing_storage
/// [`TurboTasksBackend::new`]: crate::TurboTasksBackend::new
/// [sealed]: https://predr.ag/blog/definitive-guide-to-sealed-traits-in-rust/
......@@ -53,7 +53,6 @@ pub trait BackingStorage: BackingStorageSealed {
/// [`BackingStorage`] is exported for documentation reasons and to expose the public
/// [`BackingStorage::invalidate`] method.
pub trait BackingStorageSealed: 'static + Send + Sync {
type ReadTransaction<'l>;
fn next_free_task_id(&self) -> Result<TaskId>;
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>>;
......@@ -65,27 +64,16 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
) -> Result<()>
where
I: Iterator<Item = SnapshotItem> + Send + Sync;
fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>>;
/// Returns all task IDs that match the given task type (hash collision candidates).
///
/// Since TaskCache uses hash-based keys, multiple task types may (rarely) hash to the same key.
/// The caller must verify each returned TaskId by comparing the stored task type which will
/// require a second database read
///
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
unsafe fn lookup_task_candidates(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType,
) -> Result<SmallVec<[TaskId; 1]>>;
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
unsafe fn lookup_data(
fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>>;
/// Looks up and decodes persisted data for a single task, updating the provided storage with
/// data from the database in the given category.
fn lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
......@@ -93,12 +81,8 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
/// Batch lookup and decode data for multiple tasks directly into TypedStorage instances.
/// Returns a vector of TypedStorage, one for each task_id in the input slice.
/// # Safety
///
/// `tx` must be a transaction from this BackingStorage instance.
unsafe fn batch_lookup_data(
fn batch_lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId],
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>>;
......@@ -123,8 +107,6 @@ where
L: BackingStorageSealed,
R: BackingStorageSealed,
{
type ReadTransaction<'l> = Either<L::ReadTransaction<'l>, R::ReadTransaction<'l>>;
fn next_free_task_id(&self) -> Result<TaskId> {
either::for_both!(self, this => this.next_free_task_id())
}
......@@ -149,100 +131,28 @@ where
))
}
fn start_read_transaction(&self) -> Option<Self::ReadTransaction<'_>> {
Some(match self {
Either::Left(this) => Either::Left(this.start_read_transaction()?),
Either::Right(this) => Either::Right(this.start_read_transaction()?),
})
fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>> {
either::for_both!(self, this => this.lookup_task_candidates(key))
}
unsafe fn lookup_task_candidates(
fn lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
key: &CachedTaskType,
) -> Result<SmallVec<[TaskId; 1]>> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
// Safety: `tx` is unwrapped from `Either::Left`, so it originated from `this`.
unsafe { this.lookup_task_candidates(tx, key) }
}
Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
// Safety: `tx` is unwrapped from `Either::Right`, so it originated from `this`.
unsafe { this.lookup_task_candidates(tx, key) }
}
}
}
unsafe fn lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_id: TaskId,
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
// Safety: `tx` is unwrapped from `Either::Left`, so it originated from `this`.
unsafe { this.lookup_data(tx, task_id, category, storage) }
}
Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
// Safety: `tx` is unwrapped from `Either::Right`, so it originated from `this`.
unsafe { this.lookup_data(tx, task_id, category, storage) }
}
}
either::for_both!(self, this => this.lookup_data(task_id, category, storage))
}
unsafe fn batch_lookup_data(
fn batch_lookup_data(
&self,
tx: Option<&Self::ReadTransaction<'_>>,
task_ids: &[TaskId],
category: SpecificTaskDataCategory,
) -> Result<Vec<TaskStorage>> {
match self {
Either::Left(this) => {
let tx = tx.map(|tx| read_transaction_left_or_panic(tx.as_ref()));
// Safety: `tx` is unwrapped from `Either::Left`, so it originated from `this`.
unsafe { this.batch_lookup_data(tx, task_ids, category) }
}
Either::Right(this) => {
let tx = tx.map(|tx| read_transaction_right_or_panic(tx.as_ref()));
// Safety: `tx` is unwrapped from `Either::Right`, so it originated from `this`.
unsafe { this.batch_lookup_data(tx, task_ids, category) }
}
}
either::for_both!(self, this => this.batch_lookup_data(task_ids, category))
}
fn shutdown(&self) -> Result<()> {
either::for_both!(self, this => this.shutdown())
}
}
// similar to `Either::unwrap_left`, but does not require `R: Debug`.
fn read_transaction_left_or_panic<L, R>(either: Either<L, R>) -> L {
match either {
Either::Left(l) => l,
Either::Right(_) => panic!(
"expected ReadTransaction of Either::Left containing {}, received Either::Right type \
of {}",
type_name::<L>(),
type_name::<R>(),
),
}
}
// similar to `Either::unwrap_right`, but does not require `R: Debug`.
fn read_transaction_right_or_panic<L, R>(either: Either<L, R>) -> R {
match either {
Either::Left(_) => panic!(
"expected ReadTransaction of Either::Right containing {}, received Either::Left type \
of {}",
type_name::<R>(),
type_name::<L>(),
),
Either::Right(r) => r,
}
}
use crate::database::key_value_database::KeySpace;
pub struct ByKeySpace<T> {
infra: T,
task_meta: T,
task_data: T,
task_cache: T,
}
impl<T> ByKeySpace<T> {
pub fn new(mut factory: impl FnMut(KeySpace) -> T) -> Self {
Self {
infra: factory(KeySpace::Infra),
task_meta: factory(KeySpace::TaskMeta),
task_data: factory(KeySpace::TaskData),
task_cache: factory(KeySpace::TaskCache),
}
}
pub fn get(&self, key_space: KeySpace) -> &T {
match key_space {
KeySpace::Infra => &self.infra,
KeySpace::TaskMeta => &self.task_meta,
KeySpace::TaskData => &self.task_data,
KeySpace::TaskCache => &self.task_cache,
}
}
pub fn get_mut(&mut self, key_space: KeySpace) -> &mut T {
match key_space {
KeySpace::Infra => &mut self.infra,
KeySpace::TaskMeta => &mut self.task_meta,
KeySpace::TaskData => &mut self.task_data,
KeySpace::TaskCache => &mut self.task_cache,
}
}
pub fn iter(&self) -> impl Iterator<Item = (KeySpace, &T)> {
[
(KeySpace::Infra, &self.infra),
(KeySpace::TaskMeta, &self.task_meta),
(KeySpace::TaskData, &self.task_data),
(KeySpace::TaskCache, &self.task_cache),
]
.into_iter()
}
}
......@@ -33,8 +33,7 @@ struct InvalidationFile<'a> {
/// If the cache was invalidated, the application may choose to show a warning to the user or log it
/// to telemetry.
///
/// This value is returned by [`crate::turbo_backing_storage`] and
/// [`crate::default_backing_storage`].
/// This value is returned by [`crate::turbo_backing_storage`].
pub enum StartupCacheState {
NoCache,
Cached,
......
use std::{
fs,
path::Path,
sync::atomic::{AtomicBool, Ordering},
};
use anyhow::Result;
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
write_batch::{
BaseWriteBatch, ConcurrentWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer,
},
};
pub fn is_fresh(path: &Path) -> bool {
fs::exists(path).is_ok_and(|exists| !exists)
}
pub struct FreshDbOptimization<T: KeyValueDatabase> {
database: T,
fresh_db: AtomicBool,
}
impl<T: KeyValueDatabase> FreshDbOptimization<T> {
pub fn new(database: T, fresh_db: bool) -> Self {
Self {
database,
fresh_db: AtomicBool::new(fresh_db),
}
}
}
impl<T: KeyValueDatabase> KeyValueDatabase for FreshDbOptimization<T> {
type ReadTransaction<'l>
= T::ReadTransaction<'l>
where
Self: 'l;
fn is_empty(&self) -> bool {
self.fresh_db.load(Ordering::Acquire) || self.database.is_empty()
}
fn begin_read_transaction(&self) -> Result<Self::ReadTransaction<'_>> {
self.database.begin_read_transaction()
}
type ValueBuffer<'l>
= T::ValueBuffer<'l>
where
Self: 'l;
fn get<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
key_space: super::key_value_database::KeySpace,
key: &[u8],
) -> anyhow::Result<Option<Self::ValueBuffer<'l>>> {
if self.fresh_db.load(Ordering::Acquire) {
// Performance optimization when the database was empty
// It's assumed that no cache entries are removed from the memory cache, but we
// might change that in future.
return Ok(None);
}
self.database.get(transaction, key_space, key)
}
type SerialWriteBatch<'l>
= FreshDbOptimizationWriteBatch<'l, T::SerialWriteBatch<'l>>
where
Self: 'l;
type ConcurrentWriteBatch<'l>
= FreshDbOptimizationWriteBatch<'l, T::ConcurrentWriteBatch<'l>>
where
Self: 'l;
fn write_batch(
&self,
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> {
Ok(match self.database.write_batch()? {
WriteBatch::Serial(write_batch) => WriteBatch::serial(FreshDbOptimizationWriteBatch {
write_batch,
fresh_db: &self.fresh_db,
}),
WriteBatch::Concurrent(write_batch, _) => {
WriteBatch::concurrent(FreshDbOptimizationWriteBatch {
write_batch,
fresh_db: &self.fresh_db,
})
}
})
}
}
pub struct FreshDbOptimizationWriteBatch<'a, B> {
write_batch: B,
fresh_db: &'a AtomicBool,
}
impl<'a, B: BaseWriteBatch<'a>> BaseWriteBatch<'a> for FreshDbOptimizationWriteBatch<'a, B> {
type ValueBuffer<'l>
= B::ValueBuffer<'l>
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l,
{
self.write_batch.get(key_space, key)
}
fn commit(self) -> Result<()> {
self.fresh_db.store(false, Ordering::Release);
self.write_batch.commit()
}
}
impl<'a, B: SerialWriteBatch<'a>> SerialWriteBatch<'a> for FreshDbOptimizationWriteBatch<'a, B> {
fn put(
&mut self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
self.write_batch.put(key_space, key, value)
}
fn delete(&mut self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
self.write_batch.delete(key_space, key)
}
fn flush(&mut self, key_space: KeySpace) -> Result<()> {
self.write_batch.flush(key_space)
}
}
impl<'a, B: ConcurrentWriteBatch<'a>> ConcurrentWriteBatch<'a>
for FreshDbOptimizationWriteBatch<'a, B>
{
fn put(&self, key_space: KeySpace, key: WriteBuffer<'_>, value: WriteBuffer<'_>) -> Result<()> {
self.write_batch.put(key_space, key, value)
}
fn delete(&self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
self.write_batch.delete(key_space, key)
}
unsafe fn flush(&self, key_space: KeySpace) -> Result<()> {
unsafe { self.write_batch.flush(key_space) }
}
}
......@@ -2,9 +2,7 @@ use anyhow::Result;
use smallvec::SmallVec;
use turbo_persistence::{FamilyConfig, FamilyKind};
use crate::database::write_batch::{
ConcurrentWriteBatch, SerialWriteBatch, UnimplementedWriteBatch, WriteBatch,
};
use crate::database::write_batch::ConcurrentWriteBatch;
#[derive(Debug, Clone, Copy)]
pub enum KeySpace {
......@@ -29,12 +27,6 @@ impl KeySpace {
}
pub trait KeyValueDatabase {
type ReadTransaction<'l>
where
Self: 'l;
fn begin_read_transaction(&self) -> Result<Self::ReadTransaction<'_>>;
fn is_empty(&self) -> bool {
false
}
......@@ -43,52 +35,38 @@ pub trait KeyValueDatabase {
where
Self: 'l;
fn get<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>>;
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>>;
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
/// The default implementation returns at most one value (from `get`), but implementations
/// that support multiple values per key should override this.
fn get_multiple<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'l>; 1]>> {
Ok(self.get(transaction, key_space, key)?.into_iter().collect())
) -> Result<SmallVec<[Self::ValueBuffer<'_>; 1]>> {
Ok(self.get(key_space, key)?.into_iter().collect())
}
fn batch_get<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
fn batch_get(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'l>>>> {
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> {
let mut results = Vec::with_capacity(keys.len());
for key in keys {
let value = self.get(transaction, key_space, key)?;
let value = self.get(key_space, key)?;
results.push(value);
}
Ok(results)
}
type SerialWriteBatch<'l>: SerialWriteBatch<'l>
= UnimplementedWriteBatch
where
Self: 'l;
type ConcurrentWriteBatch<'l>: ConcurrentWriteBatch<'l>
= UnimplementedWriteBatch
where
Self: 'l;
fn write_batch(
&self,
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>>;
fn write_batch(&self) -> Result<Self::ConcurrentWriteBatch<'_>>;
/// Called when the database has been invalidated via
/// [`crate::backing_storage::BackingStorage::invalidate`]
......
use std::hash::{Hash, Hasher};
use byteorder::ByteOrder;
use lmdb::{Database, RwTransaction, Transaction, WriteFlags};
use rustc_hash::FxHasher;
const MAX_KEY_SIZE: usize = 511;
const SHARED_KEY: usize = MAX_KEY_SIZE - 8;
pub fn get<'tx, T: Transaction>(
tx: &'tx T,
database: Database,
key: &[u8],
) -> lmdb::Result<&'tx [u8]> {
if key.len() > MAX_KEY_SIZE - 1 {
let hashed_key = hashed_key(key);
let data = tx.get(database, &hashed_key)?;
let iter = ExtendedValueIter::new(data);
for (k, v) in iter {
if k == key {
return Ok(v);
}
}
Err(lmdb::Error::NotFound)
} else {
tx.get(database, &key)
}
}
pub fn put(
tx: &mut RwTransaction<'_>,
database: Database,
key: &[u8],
value: &[u8],
flags: WriteFlags,
) -> lmdb::Result<()> {
if key.len() > MAX_KEY_SIZE - 1 {
let hashed_key = hashed_key(key);
let size = key.len() - SHARED_KEY + value.len() + 8;
let old = tx.get(database, &hashed_key);
let old_size = old.map_or(0, |v| v.len());
let mut data = Vec::with_capacity(old_size + size);
data.extend_from_slice(&((key.len() - SHARED_KEY) as u32).to_be_bytes());
data.extend_from_slice(&(value.len() as u32).to_be_bytes());
data.extend_from_slice(&key[SHARED_KEY..]);
data.extend_from_slice(value);
if let Ok(old) = old {
let iter = ExtendedValueIter::new(old);
for (k, v) in iter {
if k != &key[SHARED_KEY..] {
data.extend_from_slice(&(k.len() as u32).to_be_bytes());
data.extend_from_slice(&(v.len() as u32).to_be_bytes());
data.extend_from_slice(k);
data.extend_from_slice(v);
}
}
};
tx.put(database, &hashed_key, &data, flags)?;
Ok(())
} else {
tx.put(database, &key, &value, flags)
}
}
pub fn delete(
tx: &mut RwTransaction<'_>,
database: Database,
key: &[u8],
flags: WriteFlags,
) -> lmdb::Result<()> {
if key.len() > MAX_KEY_SIZE - 1 {
let hashed_key = hashed_key(key);
let old = match tx.get(database, &hashed_key) {
Ok(data) => data,
Err(lmdb::Error::NotFound) => return Ok(()),
Err(err) => return Err(err),
};
let mut data = Vec::with_capacity(old.len());
let iter = ExtendedValueIter::new(old);
for (k, v) in iter {
if k != &key[SHARED_KEY..] {
data.extend_from_slice(&(k.len() as u32).to_be_bytes());
data.extend_from_slice(&(v.len() as u32).to_be_bytes());
data.extend_from_slice(k);
data.extend_from_slice(v);
}
}
if data.is_empty() {
tx.del(database, &hashed_key, None)?;
Ok(())
} else if data.len() != old.len() {
tx.put(database, &hashed_key, &data, flags)?;
Ok(())
} else {
Ok(())
}
} else {
tx.del(database, &key, None)
}
}
fn hashed_key(key: &[u8]) -> [u8; MAX_KEY_SIZE] {
let mut result = [0; MAX_KEY_SIZE];
let mut hash = FxHasher::default();
key.hash(&mut hash);
byteorder::BigEndian::write_u64(&mut result, hash.finish());
result[8..].copy_from_slice(&key[0..SHARED_KEY]);
result
}
struct ExtendedValueIter<'a> {
data: &'a [u8],
pos: usize,
}
impl<'a> Iterator for ExtendedValueIter<'a> {
type Item = (&'a [u8], &'a [u8]);
fn next(&mut self) -> Option<Self::Item> {
if self.pos >= self.data.len() {
return None;
}
let key_len = byteorder::BigEndian::read_u32(&self.data[self.pos..]) as usize;
self.pos += 4;
let value_len = byteorder::BigEndian::read_u32(&self.data[self.pos..]) as usize;
self.pos += 4;
let key = &self.data[self.pos..self.pos + key_len];
self.pos += key_len;
let value = &self.data[self.pos..self.pos + value_len];
self.pos += value_len;
Some((key, value))
}
}
impl<'a> ExtendedValueIter<'a> {
fn new(data: &'a [u8]) -> Self {
Self { data, pos: 0 }
}
}
use std::{fs::create_dir_all, path::Path, thread::available_parallelism};
use anyhow::{Context, Result};
use lmdb::{
Database, DatabaseFlags, Environment, EnvironmentFlags, RoTransaction, RwTransaction,
Transaction, WriteFlags,
};
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
write_batch::{BaseWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer},
};
mod extended_key;
pub struct LmbdKeyValueDatabase {
env: Environment,
infra_db: Database,
data_db: Database,
meta_db: Database,
task_cache_db: Database,
}
impl LmbdKeyValueDatabase {
pub fn new(path: &Path) -> Result<Self> {
create_dir_all(path).context("Creating database directory failed")?;
#[cfg(target_arch = "x86")]
const MAP_SIZE: usize = usize::MAX;
#[cfg(not(target_arch = "x86"))]
const MAP_SIZE: usize = 40 * 1024 * 1024 * 1024;
let env = Environment::new()
.set_flags(
EnvironmentFlags::WRITE_MAP
| EnvironmentFlags::NO_META_SYNC
| EnvironmentFlags::NO_TLS,
)
.set_max_readers((available_parallelism().map_or(16, |v| v.get()) * 8) as u32)
.set_max_dbs(4)
.set_map_size(MAP_SIZE)
.open(path)?;
let infra_db = env.create_db(Some("infra"), DatabaseFlags::INTEGER_KEY)?;
let data_db = env.create_db(Some("data"), DatabaseFlags::INTEGER_KEY)?;
let meta_db = env.create_db(Some("meta"), DatabaseFlags::INTEGER_KEY)?;
let task_cache_db = env.create_db(Some("task_cache"), DatabaseFlags::empty())?;
Ok(LmbdKeyValueDatabase {
env,
infra_db,
data_db,
meta_db,
task_cache_db,
})
}
fn db(&self, key_space: KeySpace) -> Database {
match key_space {
KeySpace::Infra => self.infra_db,
KeySpace::TaskMeta => self.meta_db,
KeySpace::TaskData => self.data_db,
KeySpace::TaskCache => self.task_cache_db,
}
}
}
impl KeyValueDatabase for LmbdKeyValueDatabase {
type ReadTransaction<'l>
= RoTransaction<'l>
where
Self: 'l;
fn begin_read_transaction(&self) -> Result<Self::ReadTransaction<'_>> {
Ok(self.env.begin_ro_txn()?)
}
type ValueBuffer<'l> = &'l [u8];
fn get<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
key_space: super::key_value_database::KeySpace,
key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> {
let db = match key_space {
KeySpace::Infra => self.infra_db,
KeySpace::TaskMeta => self.meta_db,
KeySpace::TaskData => self.data_db,
KeySpace::TaskCache => self.task_cache_db,
};
let value = match extended_key::get(transaction, db, key) {
Ok(result) => result,
Err(err) => {
if err == lmdb::Error::NotFound {
return Ok(None);
} else {
return Err(err.into());
}
}
};
Ok(Some(value))
}
type SerialWriteBatch<'l>
= LmbdWriteBatch<'l>
where
Self: 'l;
fn write_batch(
&self,
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> {
Ok(WriteBatch::serial(LmbdWriteBatch {
tx: self.env.begin_rw_txn()?,
this: self,
}))
}
}
pub struct LmbdWriteBatch<'l> {
tx: RwTransaction<'l>,
this: &'l LmbdKeyValueDatabase,
}
impl<'a> BaseWriteBatch<'a> for LmbdWriteBatch<'a> {
type ValueBuffer<'l>
= &'l [u8]
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l,
{
match extended_key::get(&self.tx, self.this.db(key_space), key) {
Ok(value) => Ok(Some(value)),
Err(err) => {
if err == lmdb::Error::NotFound {
Ok(None)
} else {
Err(err.into())
}
}
}
}
fn commit(self) -> Result<()> {
self.tx.commit()?;
Ok(())
}
}
impl<'a> SerialWriteBatch<'a> for LmbdWriteBatch<'a> {
fn put(
&mut self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
extended_key::put(
&mut self.tx,
self.this.db(key_space),
&key,
&value,
WriteFlags::empty(),
)?;
Ok(())
}
fn delete(&mut self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
extended_key::delete(
&mut self.tx,
self.this.db(key_space),
&key,
WriteFlags::empty(),
)?;
Ok(())
}
fn flush(&mut self, _key_space: KeySpace) -> Result<()> {
// this is an unimplemented optimization, this LMDB implementation is only used in testing
Ok(())
}
}
#[cfg(feature = "lmdb")]
mod by_key_space;
pub mod db_invalidation;
pub mod db_versioning;
#[cfg(feature = "lmdb")]
pub mod fresh_db_optimization;
pub mod key_value_database;
#[cfg(feature = "lmdb")]
pub mod lmdb;
pub mod noop_kv;
#[cfg(feature = "lmdb")]
pub mod read_transaction_cache;
#[cfg(feature = "lmdb")]
pub mod startup_cache;
pub mod turbo;
pub mod write_batch;
......@@ -2,57 +2,34 @@ use anyhow::Result;
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
write_batch::{
BaseWriteBatch, ConcurrentWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer,
},
write_batch::{ConcurrentWriteBatch, WriteBuffer},
};
pub struct NoopKvDb;
impl KeyValueDatabase for NoopKvDb {
type ReadTransaction<'l>
= ()
where
Self: 'l;
fn begin_read_transaction(&self) -> Result<Self::ReadTransaction<'_>> {
Ok(())
}
type ValueBuffer<'l>
= &'l [u8]
where
Self: 'l;
fn get<'l, 'db: 'l>(
&'l self,
_transaction: &'l Self::ReadTransaction<'db>,
_key_space: KeySpace,
_key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> {
fn get(&self, _key_space: KeySpace, _key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>> {
Ok(None)
}
type SerialWriteBatch<'l>
= NoopWriteBatch
where
Self: 'l;
type ConcurrentWriteBatch<'l>
= NoopWriteBatch
where
Self: 'l;
fn write_batch(
&self,
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> {
Ok(WriteBatch::concurrent(NoopWriteBatch))
fn write_batch(&self) -> Result<Self::ConcurrentWriteBatch<'_>> {
Ok(NoopWriteBatch)
}
}
pub struct NoopWriteBatch;
impl<'a> BaseWriteBatch<'a> for NoopWriteBatch {
impl<'a> ConcurrentWriteBatch<'a> for NoopWriteBatch {
type ValueBuffer<'l>
= &'l [u8]
where
......@@ -69,28 +46,7 @@ impl<'a> BaseWriteBatch<'a> for NoopWriteBatch {
fn commit(self) -> Result<()> {
Ok(())
}
}
impl SerialWriteBatch<'_> for NoopWriteBatch {
fn put(
&mut self,
_key_space: KeySpace,
_key: WriteBuffer<'_>,
_value: WriteBuffer<'_>,
) -> Result<()> {
Ok(())
}
fn delete(&mut self, _key_space: KeySpace, _key: WriteBuffer<'_>) -> Result<()> {
Ok(())
}
fn flush(&mut self, _key_space: KeySpace) -> Result<()> {
Ok(())
}
}
impl ConcurrentWriteBatch<'_> for NoopWriteBatch {
fn put(
&self,
_key_space: KeySpace,
......
use std::{cell::UnsafeCell, mem::transmute, sync::Arc};
use anyhow::Result;
use arc_swap::ArcSwap;
use smallvec::SmallVec;
use thread_local::ThreadLocal;
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
write_batch::{
BaseWriteBatch, ConcurrentWriteBatch, SerialWriteBatch, WriteBatch, WriteBuffer,
},
};
struct ThreadLocalReadTransactionsContainer<T: KeyValueDatabase + 'static>(
UnsafeCell<SmallVec<[T::ReadTransaction<'static>; 4]>>,
);
impl<T: KeyValueDatabase> ThreadLocalReadTransactionsContainer<T> {
unsafe fn pop(&self) -> Option<T::ReadTransaction<'static>> {
let vec = unsafe { &mut *self.0.get() };
vec.pop()
}
unsafe fn push(&self, tx: T::ReadTransaction<'static>) {
let vec = unsafe { &mut *self.0.get() };
vec.push(tx)
}
}
// Safety: It's safe to send RoTransaction between threads, but the types don't allow that.
unsafe impl<T: KeyValueDatabase> Send for ThreadLocalReadTransactionsContainer<T> {}
/// Caches read transactions in thread-local storage to avoid creating new ones for every operation.
///
/// # Safety invariant (field ordering)
///
/// `read_transactions_cache` must be declared before `database` so that Rust's field drop order
/// (declaration order) drops the cached transactions before the database. The cached transactions
/// store `T::ReadTransaction<'static>` where the `'static` is a transmuted lie — the true
/// lifetime is tied to `database`. Dropping the cache first ensures all transactions are released
/// before the database they borrow from.
pub struct ReadTransactionCache<T: KeyValueDatabase + 'static> {
// Safety: Must be declared before `database` — see struct-level safety invariant above.
read_transactions_cache: ArcSwap<ThreadLocal<ThreadLocalReadTransactionsContainer<T>>>,
database: T,
}
impl<T: KeyValueDatabase + 'static> ReadTransactionCache<T> {
pub fn new(database: T) -> Self {
Self {
read_transactions_cache: ArcSwap::new(Arc::new(ThreadLocal::new())),
database,
}
}
}
impl<T: KeyValueDatabase + 'static> KeyValueDatabase for ReadTransactionCache<T> {
type ReadTransaction<'l>
= CachedReadTransaction<'l, T>
where
T: 'l;
fn is_empty(&self) -> bool {
self.database.is_empty()
}
fn begin_read_transaction(&self) -> Result<Self::ReadTransaction<'_>> {
let guard = self.read_transactions_cache.load();
let container = guard
.get_or(|| ThreadLocalReadTransactionsContainer(UnsafeCell::new(Default::default())));
// Safety: Since it's a thread local it's safe to take from the container
let tx = if let Some(tx) = unsafe { container.pop() } {
unsafe { transmute::<T::ReadTransaction<'static>, T::ReadTransaction<'_>>(tx) }
} else {
self.database.begin_read_transaction()?
};
let thread_locals = guard.clone();
Ok(CachedReadTransaction::<T> {
tx: Some(tx),
thread_locals,
})
}
type ValueBuffer<'l> = T::ValueBuffer<'l>;
fn get<'l, 'db: 'l>(
&'l self,
transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> anyhow::Result<Option<Self::ValueBuffer<'l>>> {
self.database
.get(transaction.tx.as_ref().unwrap(), key_space, key)
}
type SerialWriteBatch<'l> = ReadTransactionCacheWriteBatch<'l, T, T::SerialWriteBatch<'l>>;
type ConcurrentWriteBatch<'l> =
ReadTransactionCacheWriteBatch<'l, T, T::ConcurrentWriteBatch<'l>>;
fn write_batch(
&self,
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> {
Ok(match self.database.write_batch()? {
WriteBatch::Serial(write_batch) => WriteBatch::serial(ReadTransactionCacheWriteBatch {
write_batch,
read_transactions_cache: &self.read_transactions_cache,
}),
WriteBatch::Concurrent(write_batch, _) => {
WriteBatch::concurrent(ReadTransactionCacheWriteBatch {
write_batch,
read_transactions_cache: &self.read_transactions_cache,
})
}
})
}
}
pub struct CachedReadTransaction<'l, T: KeyValueDatabase + 'static> {
tx: Option<T::ReadTransaction<'l>>,
thread_locals: Arc<ThreadLocal<ThreadLocalReadTransactionsContainer<T>>>,
}
impl<T: KeyValueDatabase> Drop for CachedReadTransaction<'_, T> {
fn drop(&mut self) {
let container = self
.thread_locals
.get_or(|| ThreadLocalReadTransactionsContainer(UnsafeCell::new(Default::default())));
// Safety: We cast to 'static because the thread-local cache stores transactions
// with an erased lifetime. The transaction will be cast back to the database's
// actual lifetime when popped in `begin_read_transaction`. This is sound because
// `ReadTransactionCache` declares `read_transactions_cache` before `database`,
// so Rust drops the cache (releasing all stored transactions) before the database.
let tx = unsafe {
transmute::<T::ReadTransaction<'_>, T::ReadTransaction<'static>>(
self.tx.take().unwrap(),
)
};
// Safety: It's safe to put it back since it's a thread local
unsafe {
container.push(tx);
}
}
}
pub struct ReadTransactionCacheWriteBatch<'l, T: KeyValueDatabase + 'static, B> {
write_batch: B,
read_transactions_cache: &'l ArcSwap<ThreadLocal<ThreadLocalReadTransactionsContainer<T>>>,
}
impl<'a, T: KeyValueDatabase + 'static, B: BaseWriteBatch<'a>> BaseWriteBatch<'a>
for ReadTransactionCacheWriteBatch<'a, T, B>
{
fn commit(self) -> anyhow::Result<()> {
self.write_batch.commit()?;
let _span = tracing::trace_span!("swap read transactions").entered();
// This resets the thread local storage for read transactions, read transactions are
// eventually dropped, allowing DB to free up unused storage.
self.read_transactions_cache
.store(Arc::new(ThreadLocal::new()));
Ok(())
}
type ValueBuffer<'l>
= B::ValueBuffer<'l>
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l,
{
self.write_batch.get(key_space, key)
}
}
impl<'a, T: KeyValueDatabase, B: SerialWriteBatch<'a>> SerialWriteBatch<'a>
for ReadTransactionCacheWriteBatch<'a, T, B>
{
fn put(
&mut self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
self.write_batch.put(key_space, key, value)
}
fn delete(&mut self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
self.write_batch.delete(key_space, key)
}
fn flush(&mut self, key_space: KeySpace) -> Result<()> {
self.write_batch.flush(key_space)
}
}
impl<'a, T: KeyValueDatabase, B: ConcurrentWriteBatch<'a>> ConcurrentWriteBatch<'a>
for ReadTransactionCacheWriteBatch<'a, T, B>
{
fn put(&self, key_space: KeySpace, key: WriteBuffer<'_>, value: WriteBuffer<'_>) -> Result<()> {
self.write_batch.put(key_space, key, value)
}
fn delete(&self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
self.write_batch.delete(key_space, key)
}
unsafe fn flush(&self, key_space: KeySpace) -> Result<()> {
unsafe { self.write_batch.flush(key_space) }
}
}
......@@ -21,7 +21,7 @@ use turbo_tasks::{
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
turbo::parallel_scheduler::TurboTasksParallelScheduler,
write_batch::{BaseWriteBatch, ConcurrentWriteBatch, WriteBatch, WriteBuffer},
write_batch::{ConcurrentWriteBatch, WriteBuffer},
};
mod parallel_scheduler;
......@@ -70,48 +70,32 @@ impl TurboKeyValueDatabase {
}
impl KeyValueDatabase for TurboKeyValueDatabase {
type ReadTransaction<'l>
= ()
where
Self: 'l;
fn is_empty(&self) -> bool {
self.db.is_empty()
}
fn begin_read_transaction(&self) -> Result<Self::ReadTransaction<'_>> {
Ok(())
}
type ValueBuffer<'l>
= ArcBytes
where
Self: 'l;
fn get<'l, 'db: 'l>(
&'l self,
_transaction: &'l Self::ReadTransaction<'db>,
key_space: KeySpace,
key: &[u8],
) -> Result<Option<Self::ValueBuffer<'l>>> {
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>> {
self.db.get(key_space as usize, &key)
}
fn batch_get<'l, 'db: 'l>(
&'l self,
_transaction: &'l Self::ReadTransaction<'db>,
fn batch_get(
&self,
key_space: KeySpace,
keys: &[&[u8]],
) -> Result<Vec<Option<Self::ValueBuffer<'l>>>> {
) -> Result<Vec<Option<Self::ValueBuffer<'_>>>> {
self.db.batch_get(key_space as usize, keys)
}
fn get_multiple<'l, 'db: 'l>(
&'l self,
_transaction: &'l Self::ReadTransaction<'db>,
fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'l>; 1]>> {
) -> Result<SmallVec<[Self::ValueBuffer<'_>; 1]>> {
self.db.get_multiple(key_space as usize, &key)
}
......@@ -120,20 +104,18 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
where
Self: 'l;
fn write_batch(
&self,
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> {
fn write_batch(&self) -> Result<Self::ConcurrentWriteBatch<'_>> {
// Wait for the compaction to finish
if let Some(join_handle) = self.compact_join_handle.lock().take() {
join_handle.join()?;
}
// Start a new write batch
Ok(WriteBatch::concurrent(TurboWriteBatch {
Ok(TurboWriteBatch {
batch: self.db.write_batch()?,
db: &self.db,
compact_join_handle: (!self.is_short_session && !self.db.is_empty())
.then_some(&self.compact_join_handle),
}))
})
}
fn prevent_writes(&self) {}
......@@ -210,7 +192,7 @@ pub struct TurboWriteBatch<'a> {
compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>,
}
impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
impl<'a> ConcurrentWriteBatch<'a> for TurboWriteBatch<'a> {
type ValueBuffer<'l>
= ArcBytes
where
......@@ -243,9 +225,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
Ok(())
}
}
impl<'a> ConcurrentWriteBatch<'a> for TurboWriteBatch<'a> {
fn put(&self, key_space: KeySpace, key: WriteBuffer<'_>, value: WriteBuffer<'_>) -> Result<()> {
self.batch
.put(key_space as u32, key.into_static(), value.into())
......
use std::{
borrow::{Borrow, Cow},
marker::PhantomData,
ops::Deref,
};
use std::{borrow::Cow, ops::Deref};
use anyhow::Result;
use smallvec::SmallVec;
use crate::database::key_value_database::KeySpace;
pub trait BaseWriteBatch<'a> {
type ValueBuffer<'l>: std::borrow::Borrow<[u8]>
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l;
fn commit(self) -> Result<()>;
}
pub enum WriteBuffer<'a> {
Borrowed(&'a [u8]),
Vec(Vec<u8>),
......@@ -58,18 +42,16 @@ impl<'l> From<Cow<'l, [u8]>> for WriteBuffer<'l> {
}
}
pub trait SerialWriteBatch<'a>: BaseWriteBatch<'a> {
fn put(
&mut self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()>;
fn delete(&mut self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()>;
fn flush(&mut self, key_space: KeySpace) -> Result<()>;
}
pub trait ConcurrentWriteBatch<'a>: Sync + Send {
type ValueBuffer<'l>: std::borrow::Borrow<[u8]>
where
Self: 'l,
'a: 'l;
pub trait ConcurrentWriteBatch<'a>: BaseWriteBatch<'a> + Sync + Send {
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l;
fn commit(self) -> Result<()>;
fn put(&self, key_space: KeySpace, key: WriteBuffer<'_>, value: WriteBuffer<'_>) -> Result<()>;
fn delete(&self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()>;
/// Flushes a key space of the write batch, reducing the amount of buffered memory used.
......@@ -79,247 +61,3 @@ pub trait ConcurrentWriteBatch<'a>: BaseWriteBatch<'a> + Sync + Send {
/// flushed key space.
unsafe fn flush(&self, key_space: KeySpace) -> Result<()>;
}
pub enum WriteBatch<'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
Serial(S),
Concurrent(C, PhantomData<&'a ()>),
}
pub enum WriteBatchValueBuffer<S: Borrow<[u8]>, C: Borrow<[u8]>> {
Serial(S),
Concurrent(C),
}
impl<S: Borrow<[u8]>, C: Borrow<[u8]>> Borrow<[u8]> for WriteBatchValueBuffer<S, C> {
fn borrow(&self) -> &[u8] {
match self {
WriteBatchValueBuffer::Serial(s) => s.borrow(),
WriteBatchValueBuffer::Concurrent(c) => c.borrow(),
}
}
}
impl<'a, S, C> WriteBatch<'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
pub fn serial(s: S) -> Self {
WriteBatch::Serial(s)
}
pub fn concurrent(c: C) -> Self {
WriteBatch::Concurrent(c, PhantomData)
}
}
impl<'a, S, C> BaseWriteBatch<'a> for WriteBatch<'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
type ValueBuffer<'l>
= WriteBatchValueBuffer<S::ValueBuffer<'l>, C::ValueBuffer<'l>>
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l,
{
Ok(match self {
WriteBatch::Serial(s) => s.get(key_space, key)?.map(WriteBatchValueBuffer::Serial),
WriteBatch::Concurrent(c, _) => c
.get(key_space, key)?
.map(WriteBatchValueBuffer::Concurrent),
})
}
fn commit(self) -> Result<()> {
match self {
WriteBatch::Serial(s) => s.commit(),
WriteBatch::Concurrent(c, _) => c.commit(),
}
}
}
impl<'a, S, C> SerialWriteBatch<'a> for WriteBatch<'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
fn put(
&mut self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
match self {
WriteBatch::Serial(s) => s.put(key_space, key, value),
WriteBatch::Concurrent(c, _) => c.put(key_space, key, value),
}
}
fn delete(&mut self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
match self {
WriteBatch::Serial(s) => s.delete(key_space, key),
WriteBatch::Concurrent(c, _) => c.delete(key_space, key),
}
}
fn flush(&mut self, key_space: KeySpace) -> Result<()> {
match self {
WriteBatch::Serial(s) => s.flush(key_space),
WriteBatch::Concurrent(c, _) => {
// Safety: the &mut self ensures that no concurrent operation is happening
unsafe { c.flush(key_space) }
}
}
}
}
pub enum WriteBatchRef<'r, 'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
Serial(&'r mut S),
Concurrent(&'r C, PhantomData<&'a ()>),
}
impl<'r, 'a, S, C> WriteBatchRef<'r, 'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
pub fn serial(s: &'r mut S) -> Self {
WriteBatchRef::Serial(s)
}
pub fn concurrent(c: &'r C) -> Self {
WriteBatchRef::Concurrent(c, PhantomData)
}
}
impl<'a, S, C> BaseWriteBatch<'a> for WriteBatchRef<'_, 'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
type ValueBuffer<'l>
= WriteBatchValueBuffer<S::ValueBuffer<'l>, C::ValueBuffer<'l>>
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l,
{
Ok(match self {
WriteBatchRef::Serial(s) => s.get(key_space, key)?.map(WriteBatchValueBuffer::Serial),
WriteBatchRef::Concurrent(c, _) => c
.get(key_space, key)?
.map(WriteBatchValueBuffer::Concurrent),
})
}
fn commit(self) -> Result<()> {
// TODO change the traits to make this a type level constraint
panic!("WriteBatchRef::commit is not usable");
}
}
impl<'a, S, C> SerialWriteBatch<'a> for WriteBatchRef<'_, 'a, S, C>
where
S: SerialWriteBatch<'a>,
C: ConcurrentWriteBatch<'a>,
{
fn put(
&mut self,
key_space: KeySpace,
key: WriteBuffer<'_>,
value: WriteBuffer<'_>,
) -> Result<()> {
match self {
WriteBatchRef::Serial(s) => s.put(key_space, key, value),
WriteBatchRef::Concurrent(c, _) => c.put(key_space, key, value),
}
}
fn delete(&mut self, key_space: KeySpace, key: WriteBuffer<'_>) -> Result<()> {
match self {
WriteBatchRef::Serial(s) => s.delete(key_space, key),
WriteBatchRef::Concurrent(c, _) => c.delete(key_space, key),
}
}
fn flush(&mut self, key_space: KeySpace) -> Result<()> {
match self {
WriteBatchRef::Serial(s) => s.flush(key_space),
WriteBatchRef::Concurrent(c, _) => {
// Safety: the &mut self ensures that no concurrent operation is happening
unsafe { c.flush(key_space) }
}
}
}
}
pub struct UnimplementedWriteBatch;
impl<'a> BaseWriteBatch<'a> for UnimplementedWriteBatch {
type ValueBuffer<'l>
= &'l [u8]
where
Self: 'l,
'a: 'l;
fn get<'l>(&'l self, _key_space: KeySpace, _key: &[u8]) -> Result<Option<Self::ValueBuffer<'l>>>
where
'a: 'l,
{
todo!()
}
fn commit(self) -> Result<()> {
todo!()
}
}
impl SerialWriteBatch<'_> for UnimplementedWriteBatch {
fn put(
&mut self,
_key_space: KeySpace,
_key: WriteBuffer<'_>,
_value: WriteBuffer<'_>,
) -> Result<()> {
todo!()
}
fn delete(&mut self, _key_space: KeySpace, _key: WriteBuffer<'_>) -> Result<()> {
todo!()
}
fn flush(&mut self, _key_space: KeySpace) -> Result<()> {
todo!()
}
}
impl ConcurrentWriteBatch<'_> for UnimplementedWriteBatch {
fn put(
&self,
_key_space: KeySpace,
_key: WriteBuffer<'_>,
_value: WriteBuffer<'_>,
) -> Result<()> {
todo!()
}
fn delete(&self, _key_space: KeySpace, _key: WriteBuffer<'_>) -> Result<()> {
todo!()
}
unsafe fn flush(&self, _key_space: KeySpace) -> Result<()> {
todo!()
}
}
......@@ -25,60 +25,11 @@ pub use crate::{
kv_backing_storage::KeyValueDatabaseBackingStorage,
};
#[cfg(feature = "lmdb")]
pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
database::read_transaction_cache::ReadTransactionCache<
database::startup_cache::StartupCacheLayer<
database::fresh_db_optimization::FreshDbOptimization<
crate::database::lmdb::LmbdKeyValueDatabase,
>,
>,
>,
>;
/// Creates an [`lmdb`]-based `BackingStorage` to be passed to [`TurboTasksBackend::new`].
///
/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
///
/// This is backend is slower than [`turbo_backing_storage`], but it's a known-good database that
/// can be used when reproducing user-reported issues to isolate bugs.
///
/// When the `lmdb` cargo feature is enabled, [`default_backing_storage`] will return this value.
#[cfg(feature = "lmdb")]
pub fn lmdb_backing_storage(
base_path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<(LmdbBackingStorage, StartupCacheState)> {
use crate::database::{
fresh_db_optimization::{FreshDbOptimization, is_fresh},
read_transaction_cache::ReadTransactionCache,
startup_cache::StartupCacheLayer,
};
KeyValueDatabaseBackingStorage::open_versioned_on_disk(
base_path.to_owned(),
version_info,
is_ci,
|versioned_path| {
let fresh_db = is_fresh(&versioned_path);
let database = crate::database::lmdb::LmbdKeyValueDatabase::new(&versioned_path)?;
let database = FreshDbOptimization::new(database, fresh_db);
let database =
StartupCacheLayer::new(database, versioned_path.join("startup.cache"), fresh_db)?;
Ok(ReadTransactionCache::new(database))
},
)
}
pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
///
/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
///
/// This is the fastest most-tested implementation of `BackingStorage`, and is normally returned by
/// [`default_backing_storage`].
pub fn turbo_backing_storage(
base_path: &Path,
version_info: &GitVersionInfo,
......@@ -99,27 +50,3 @@ pub type NoopBackingStorage = KeyValueDatabaseBackingStorage<NoopKvDb>;
pub fn noop_backing_storage() -> NoopBackingStorage {
KeyValueDatabaseBackingStorage::new_in_memory(NoopKvDb)
}
#[cfg(feature = "lmdb")]
pub type DefaultBackingStorage = LmdbBackingStorage;
#[cfg(not(feature = "lmdb"))]
pub type DefaultBackingStorage = TurboBackingStorage;
/// Calls [`turbo_backing_storage`] (recommended) or `lmdb_backing_storage`, depending on if the
/// `lmdb` cargo feature is enabled.
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
is_short_session: bool,
) -> Result<(DefaultBackingStorage, StartupCacheState)> {
#[cfg(feature = "lmdb")]
{
lmdb_backing_storage(path, version_info, is_ci)
}
#[cfg(not(feature = "lmdb"))]
{
turbo_backing_storage(path, version_info, is_ci, is_short_session)
}
}
......@@ -13,7 +13,7 @@
small_preallocation: true,
..Default::default()
},
turbo_tasks_backend::default_backing_storage(
turbo_tasks_backend::turbo_backing_storage(
path.as_path(),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
......
......@@ -12,7 +12,7 @@
small_preallocation: true,
..Default::default()
},
turbo_tasks_backend::default_backing_storage(
turbo_tasks_backend::turbo_backing_storage(
path.as_path(),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
......
......@@ -12,7 +12,7 @@
small_preallocation: true,
..Default::default()
},
turbo_tasks_backend::default_backing_storage(
turbo_tasks_backend::turbo_backing_storage(
path.as_path(),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
......
......@@ -13,7 +13,7 @@
small_preallocation: true,
..Default::default()
},
turbo_tasks_backend::default_backing_storage(
turbo_tasks_backend::turbo_backing_storage(
path.as_path(),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
......
......@@ -304,7 +304,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
node_file_trace(input, "persistent_cache", 2, 240, |directory_path| {
TurboTasks::new(TurboTasksBackend::new(
turbo_tasks_backend::BackendOptions::default(),
turbo_tasks_backend::default_backing_storage(
turbo_tasks_backend::turbo_backing_storage(
&directory_path.join(".cache"),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment