Commit ec9f7f3e authored by Luke Sandberg's avatar Luke Sandberg

fewer copies

parent d8ef88ee
......@@ -182,23 +182,16 @@ impl<'e, B: BackingStorage> ExecuteContextImpl<'e, B> {
self.backend.should_restore() && self.backend.local_is_partial
}
fn restore_task_data(
fn lookup_task_data(
&self,
task_id: TaskId,
category: SpecificTaskDataCategory,
) -> TaskStorage {
) -> Option<turbo_persistence::ArcBytes> {
if !self.should_check_backing_storage() {
// If we don't need to restore, we can just return an empty storage
return TaskStorage::default();
return None;
}
let mut storage = TaskStorage::default();
let result = self
.backend
.backing_storage
.lookup_data(task_id, category, &mut storage);
match result {
Ok(()) => storage,
match self.backend.backing_storage.lookup_data(task_id, category) {
Ok(bytes) => bytes,
Err(e) => {
panic!(
"Failed to restore task data (corrupted database or bug): {:?}",
......@@ -531,28 +524,39 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> {
category.includes_meta() && !task.flags.is_restored(TaskDataCategory::Meta);
if needs_data || needs_meta {
// Avoid holding the lock too long since this can also affect other tasks
// Drop lock once, do all I/O, then re-acquire once
// Avoid holding the lock too long since this can also affect other tasks.
// Drop lock once, do all I/O (returning raw bytes), then re-acquire once
// and decode directly into the live task — no scratch TaskStorage needed.
drop(task);
let storage_data = needs_data
.then(|| self.restore_task_data(task_id, SpecificTaskDataCategory::Data));
let storage_meta = needs_meta
.then(|| self.restore_task_data(task_id, SpecificTaskDataCategory::Meta));
let bytes_data = needs_data
.then(|| self.lookup_task_data(task_id, SpecificTaskDataCategory::Data));
let bytes_meta = needs_meta
.then(|| self.lookup_task_data(task_id, SpecificTaskDataCategory::Meta));
task = self.backend.storage.access_mut(task_id);
// Handle race conditions and merge
if let Some(storage) = storage_data
&& !task.flags.is_restored(TaskDataCategory::Data)
{
task.restore_from(storage, TaskDataCategory::Data);
// Handle race conditions and decode directly into live storage.
// set_restored is called even when no bytes found (task simply has no
// persisted data for this category).
if needs_data && !task.flags.is_restored(TaskDataCategory::Data) {
if let Some(bytes) = bytes_data.flatten() {
let mut decoder = new_turbo_bincode_decoder(&bytes);
task.decode(SpecificTaskDataCategory::Data, &mut decoder)
.unwrap_or_else(|e| {
panic!("Failed to decode Data for {task_id:?}: {e:?}")
});
}
task.flags.set_restored(TaskDataCategory::Data);
}
if let Some(storage) = storage_meta
&& !task.flags.is_restored(TaskDataCategory::Meta)
{
task.restore_from(storage, TaskDataCategory::Meta);
if needs_meta && !task.flags.is_restored(TaskDataCategory::Meta) {
if let Some(bytes) = bytes_meta.flatten() {
let mut decoder = new_turbo_bincode_decoder(&bytes);
task.decode(SpecificTaskDataCategory::Meta, &mut decoder)
.unwrap_or_else(|e| {
panic!("Failed to decode Meta for {task_id:?}: {e:?}")
});
}
task.flags.set_restored(TaskDataCategory::Meta);
}
}
......@@ -625,47 +629,69 @@ impl<'e, B: BackingStorage> ExecuteContext<'e> for ExecuteContextImpl<'e, B> {
category.includes_meta() && !task2.flags.is_restored(TaskDataCategory::Meta);
if needs_data1 || needs_meta1 || needs_data2 || needs_meta2 {
// Avoid holding the lock too long since this can also affect other tasks
// Drop locks once, do all I/O, then re-acquire once
// Avoid holding the lock too long since this can also affect other tasks.
// Drop locks once, do all I/O (returning raw bytes), then re-acquire once
// and decode directly into the live tasks — no scratch TaskStorage needed.
drop(task1);
drop(task2);
let storage_data1 = needs_data1
.then(|| self.restore_task_data(task_id1, SpecificTaskDataCategory::Data));
let storage_meta1 = needs_meta1
.then(|| self.restore_task_data(task_id1, SpecificTaskDataCategory::Meta));
let storage_data2 = needs_data2
.then(|| self.restore_task_data(task_id2, SpecificTaskDataCategory::Data));
let storage_meta2 = needs_meta2
.then(|| self.restore_task_data(task_id2, SpecificTaskDataCategory::Meta));
let bytes_data1 = needs_data1
.then(|| self.lookup_task_data(task_id1, SpecificTaskDataCategory::Data));
let bytes_meta1 = needs_meta1
.then(|| self.lookup_task_data(task_id1, SpecificTaskDataCategory::Meta));
let bytes_data2 = needs_data2
.then(|| self.lookup_task_data(task_id2, SpecificTaskDataCategory::Data));
let bytes_meta2 = needs_meta2
.then(|| self.lookup_task_data(task_id2, SpecificTaskDataCategory::Meta));
let (t1, t2) = self.backend.storage.access_pair_mut(task_id1, task_id2);
task1 = t1;
task2 = t2;
// Merge results, handling race conditions
if let Some(storage) = storage_data1
&& !task1.flags.is_restored(TaskDataCategory::Data)
{
task1.restore_from(storage, TaskDataCategory::Data);
// Decode directly into live storage, handling race conditions.
// set_restored is called even when no bytes found.
if needs_data1 && !task1.flags.is_restored(TaskDataCategory::Data) {
if let Some(bytes) = bytes_data1.flatten() {
let mut decoder = new_turbo_bincode_decoder(&bytes);
task1
.decode(SpecificTaskDataCategory::Data, &mut decoder)
.unwrap_or_else(|e| {
panic!("Failed to decode Data for {task_id1:?}: {e:?}")
});
}
task1.flags.set_restored(TaskDataCategory::Data);
}
if let Some(storage) = storage_meta1
&& !task1.flags.is_restored(TaskDataCategory::Meta)
{
task1.restore_from(storage, TaskDataCategory::Meta);
if needs_meta1 && !task1.flags.is_restored(TaskDataCategory::Meta) {
if let Some(bytes) = bytes_meta1.flatten() {
let mut decoder = new_turbo_bincode_decoder(&bytes);
task1
.decode(SpecificTaskDataCategory::Meta, &mut decoder)
.unwrap_or_else(|e| {
panic!("Failed to decode Meta for {task_id1:?}: {e:?}")
});
}
task1.flags.set_restored(TaskDataCategory::Meta);
}
if let Some(storage) = storage_data2
&& !task2.flags.is_restored(TaskDataCategory::Data)
{
task2.restore_from(storage, TaskDataCategory::Data);
if needs_data2 && !task2.flags.is_restored(TaskDataCategory::Data) {
if let Some(bytes) = bytes_data2.flatten() {
let mut decoder = new_turbo_bincode_decoder(&bytes);
task2
.decode(SpecificTaskDataCategory::Data, &mut decoder)
.unwrap_or_else(|e| {
panic!("Failed to decode Data for {task_id2:?}: {e:?}")
});
}
task2.flags.set_restored(TaskDataCategory::Data);
}
if let Some(storage) = storage_meta2
&& !task2.flags.is_restored(TaskDataCategory::Meta)
{
task2.restore_from(storage, TaskDataCategory::Meta);
if needs_meta2 && !task2.flags.is_restored(TaskDataCategory::Meta) {
if let Some(bytes) = bytes_meta2.flatten() {
let mut decoder = new_turbo_bincode_decoder(&bytes);
task2
.decode(SpecificTaskDataCategory::Meta, &mut decoder)
.unwrap_or_else(|e| {
panic!("Failed to decode Meta for {task_id2:?}: {e:?}")
});
}
task2.flags.set_restored(TaskDataCategory::Meta);
}
}
......
......@@ -4,10 +4,11 @@ use anyhow::Result;
use either::Either;
use smallvec::SmallVec;
use turbo_bincode::TurboBincodeBuffer;
use turbo_persistence::ArcBytes;
use turbo_tasks::{TaskId, backend::CachedTaskType};
use crate::{
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backend::{AnyOperation, SpecificTaskDataCategory},
utils::chunked_vec::ChunkedVec,
};
......@@ -69,14 +70,16 @@ pub trait BackingStorageSealed: 'static + Send + Sync {
/// The caller must verify each returned TaskId by comparing the stored task type which will
/// require a second database read
fn lookup_task_candidates(&self, key: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>>;
/// Looks up and decodes persisted data for a single task, updating the provided storage with
/// data from the database in the given category.
/// Looks up raw persisted bytes for a single task.
///
/// Returns `None` if the task has no persisted data for the given category.
/// The caller is responsible for decoding the returned bytes. The returned
/// [`ArcBytes`] is ref-counted and zero-copy when backed by a memory-mapped file.
fn lookup_data(
&self,
task_id: TaskId,
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()>;
) -> Result<Option<ArcBytes>>;
/// Batch lookup raw bytes for multiple tasks, calling
/// `callback(index, Option<&[u8]>)` for every task in `task_ids` immediately after its
......@@ -151,9 +154,8 @@ where
&self,
task_id: TaskId,
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()> {
either::for_both!(self, this => this.lookup_data(task_id, category, storage))
) -> Result<Option<ArcBytes>> {
either::for_both!(self, this => this.lookup_data(task_id, category))
}
fn batch_lookup_data(
......
use anyhow::Result;
use smallvec::SmallVec;
use turbo_persistence::{FamilyConfig, FamilyKind};
use turbo_persistence::{ArcBytes, FamilyConfig, FamilyKind};
use crate::database::write_batch::ConcurrentWriteBatch;
......@@ -57,21 +57,13 @@ pub trait KeyValueDatabase {
false
}
type ValueBuffer<'l>: std::borrow::Borrow<[u8]>
where
Self: 'l;
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>>;
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>>;
/// Looks up a key and returns all matching values.
///
/// Useful for keyspaces where keys are hashes and collisions are possible (e.g., TaskCache).
/// The default implementation returns at most one value (from `get`), but implementations
/// that support multiple values per key should override this.
fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'_>; 1]>> {
fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
Ok(self.get(key_space, key)?.into_iter().collect())
}
......@@ -89,7 +81,7 @@ pub trait KeyValueDatabase {
) -> Result<()> {
for (index, key) in keys.iter().enumerate() {
let value = self.get(key_space, key)?;
let opt_bytes = value.as_ref().map(std::borrow::Borrow::<[u8]>::borrow);
let opt_bytes = value.as_deref();
callback(index, opt_bytes)?;
}
Ok(())
......
use anyhow::Result;
use turbo_persistence::ArcBytes;
use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase},
......@@ -8,12 +9,7 @@ use crate::database::{
pub struct NoopKvDb;
impl KeyValueDatabase for NoopKvDb {
type ValueBuffer<'l>
= &'l [u8]
where
Self: 'l;
fn get(&self, _key_space: KeySpace, _key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>> {
fn get(&self, _key_space: KeySpace, _key: &[u8]) -> Result<Option<ArcBytes>> {
Ok(None)
}
......
......@@ -87,12 +87,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.is_empty()
}
type ValueBuffer<'l>
= ArcBytes
where
Self: 'l;
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<Self::ValueBuffer<'_>>> {
fn get(&self, key_space: KeySpace, key: &[u8]) -> Result<Option<ArcBytes>> {
self.db.get(key_space as usize, &key)
}
......@@ -105,11 +100,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
self.db.batch_get_with(key_space as usize, keys, callback)
}
fn get_multiple(
&self,
key_space: KeySpace,
key: &[u8],
) -> Result<SmallVec<[Self::ValueBuffer<'_>; 1]>> {
fn get_multiple(&self, key_space: KeySpace, key: &[u8]) -> Result<SmallVec<[ArcBytes; 1]>> {
self.db.get_multiple(key_space as usize, &key)
}
......
......@@ -7,7 +7,8 @@ use std::{
use anyhow::{Context, Result};
use smallvec::SmallVec;
use turbo_bincode::{new_turbo_bincode_decoder, turbo_bincode_decode, turbo_bincode_encode};
use turbo_bincode::{turbo_bincode_decode, turbo_bincode_encode};
use turbo_persistence::ArcBytes;
use turbo_tasks::{
TaskId,
backend::CachedTaskType,
......@@ -18,7 +19,7 @@ use turbo_tasks_hash::Xxh3Hash64Hasher;
use crate::{
GitVersionInfo,
backend::{AnyOperation, SpecificTaskDataCategory, storage_schema::TaskStorage},
backend::{AnyOperation, SpecificTaskDataCategory},
backing_storage::{BackingStorage, BackingStorageSealed, SnapshotItem},
database::{
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
......@@ -47,11 +48,6 @@ impl AsRef<[u8]> for IntKey {
}
}
fn as_u32(bytes: impl Borrow<[u8]>) -> Result<u32> {
let n = u32::from_le_bytes(bytes.borrow().try_into()?);
Ok(n)
}
// We want to invalidate the cache on panic for most users, but this is a band-aid to underlying
// problems in turbo-tasks.
//
......@@ -187,7 +183,7 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorageInner<T> {
fn get_infra_u32(&self, key: u32) -> Result<Option<u32>> {
self.database
.get(KeySpace::Infra, IntKey::new(key).as_ref())?
.map(as_u32)
.map(|b| Ok(u32::from_le_bytes((*b).try_into()?)))
.transpose()
}
}
......@@ -212,16 +208,13 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
}
fn uncompleted_operations(&self) -> Result<Vec<AnyOperation>> {
fn get(database: &impl KeyValueDatabase) -> Result<Vec<AnyOperation>> {
let Some(operations) =
database.get(KeySpace::Infra, IntKey::new(META_KEY_OPERATIONS).as_ref())?
else {
return Ok(Vec::new());
};
let operations = turbo_bincode_decode(operations.borrow())?;
Ok(operations)
}
get(&self.inner.database).context("Unable to read uncompleted operations from database")
self.inner
.database
.get(KeySpace::Infra, IntKey::new(META_KEY_OPERATIONS).as_ref())?
.map(|b| turbo_bincode_decode(&b))
.transpose()
.map(Option::unwrap_or_default)
.context("Unable to read uncompleted operations from database")
}
fn save_snapshot<I>(
......@@ -298,14 +291,14 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
}
fn lookup_task_candidates(&self, task_type: &CachedTaskType) -> Result<SmallVec<[TaskId; 1]>> {
let inner = &*self.inner;
if inner.database.is_empty() {
if self.inner.database.is_empty() {
// Checking if the database is empty is a performance optimization
// to avoid computing the hash.
return Ok(SmallVec::new());
}
let hash = compute_task_type_hash(task_type);
let buffers = inner
let buffers = self
.inner
.database
.get_multiple(KeySpace::TaskCache, &hash.to_le_bytes())
.with_context(|| {
......@@ -314,7 +307,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
let mut task_ids = SmallVec::with_capacity(buffers.len());
for bytes in buffers {
let bytes = bytes.borrow().try_into()?;
let bytes = (*bytes).try_into()?;
let id = TaskId::try_from(u32::from_le_bytes(bytes)).unwrap();
task_ids.push(id);
}
......@@ -325,22 +318,11 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
&self,
task_id: TaskId,
category: SpecificTaskDataCategory,
storage: &mut TaskStorage,
) -> Result<()> {
let inner = &*self.inner;
let Some(bytes) = inner
) -> Result<Option<ArcBytes>> {
self.inner
.database
.get(category.key_space(), IntKey::new(*task_id).as_ref())
.with_context(|| {
format!("Looking up task storage for {task_id} from database failed")
})?
else {
return Ok(());
};
let mut decoder = new_turbo_bincode_decoder(bytes.borrow());
storage
.decode(category, &mut decoder)
.map_err(|e| anyhow::anyhow!("Failed to decode {category:?}: {e:?}"))
.with_context(|| format!("Looking up task storage for {task_id} from database failed"))
}
fn batch_lookup_data(
......@@ -349,10 +331,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
category: SpecificTaskDataCategory,
callback: &mut dyn FnMut(usize, Option<&[u8]>),
) -> Result<()> {
let inner = &*self.inner;
let int_keys: Vec<_> = task_ids.iter().map(|&id| IntKey::new(*id)).collect();
let keys = int_keys.iter().map(|k| k.as_ref()).collect::<Vec<_>>();
inner
self.inner
.database
.batch_get_with(category.key_space(), &keys, |index, opt_bytes| {
callback(index, opt_bytes);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment