Unverified Commit d898c8f5 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

introduce an options struct for constructing backend storage (#97284)

Improve legibility a bit, requested by @bgw here
https://github.com/vercel/next.js/pull/95976#discussion_r3762093713
parent 74a30b10
......@@ -57,7 +57,7 @@ use turbo_tasks::{
trace::TraceRawVcs,
unmark_top_level_task_may_leak_eventually_consistent_state,
};
use turbo_tasks_backend::db_invalidation::invalidation_reasons;
use turbo_tasks_backend::{BackingStorageOptions, db_invalidation::invalidation_reasons};
#[cfg(windows)]
use turbo_tasks_fs::windows::to_verbatim_with_case_folded_disk;
use turbo_tasks_fs::{
......@@ -576,18 +576,17 @@ pub fn project_new<'env>(
env.spawn_future(
async move {
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
let skip_compaction = turbo_engine_options.skip_compaction.unwrap_or(false);
let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction;
let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir),
&options.next_version,
options.is_persistent_caching_enabled,
dependency_tracking,
is_ci,
is_short_session,
skip_compaction,
BackingStorageOptions {
is_ci: turbo_engine_options.is_ci.unwrap_or(false),
is_short_session: turbo_engine_options.is_short_session.unwrap_or(false),
skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false),
},
turbopack_memory_eviction,
)?;
let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
......
......@@ -25,8 +25,9 @@ use turbo_tasks::{
message_queue::{CompilationEvent, Severity},
};
use turbo_tasks_backend::{
BackendOptions, EvictionMode, GitVersionInfo, StartupCacheState, TurboTasksBackend,
db_invalidation::invalidation_reasons, noop_backing_storage, turbo_backing_storage,
BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, StartupCacheState,
TurboTasksBackend, db_invalidation::invalidation_reasons, noop_backing_storage,
turbo_backing_storage,
};
pub type NextTurboTasks = Arc<TurboTasks<TurboTasksBackend>>;
......@@ -289,20 +290,21 @@ pub fn create_turbo_tasks(
next_version: &str,
persistent_caching: bool,
dependency_tracking: bool,
is_ci: bool,
is_short_session: bool,
skip_compaction: bool,
storage_options: BackingStorageOptions,
turbopack_memory_eviction: MemoryEvictionMode,
) -> Result<NextTurboTasks> {
let BackingStorageOptions {
is_ci,
is_short_session,
..
} = storage_options;
Ok(if persistent_caching {
let describe = cache_describe(next_version);
let version_info = git_version_info(&describe);
let (backing_storage, cache_state) = turbo_backing_storage(
&output_path.join("cache").join("turbopack"),
&version_info,
is_ci,
is_short_session,
skip_compaction,
storage_options,
)?;
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
......
......@@ -17,7 +17,10 @@ use turbo_tasks::{
turbo_tasks,
};
use crate::database::{key_value_database::KeySpace, write_batch::WriteBuffer};
use crate::{
BackingStorageOptions,
database::{key_value_database::KeySpace, write_batch::WriteBuffer},
};
mod parallel_scheduler;
pub(crate) use parallel_scheduler::TurboTasksParallelScheduler;
......@@ -49,31 +52,22 @@ pub const COMPACT_CONFIG: CompactConfig = CompactConfig {
pub struct TurboKeyValueDatabase {
db: TurboPersistence<TurboTasksParallelScheduler, FAMILIES>,
is_ci: bool,
is_short_session: bool,
options: BackingStorageOptions,
is_fresh: bool,
skip_compaction: bool,
}
impl TurboKeyValueDatabase {
pub fn new(
versioned_path: PathBuf,
is_ci: bool,
is_short_session: bool,
skip_compaction: bool,
) -> Result<Self> {
pub fn new(versioned_path: PathBuf, options: BackingStorageOptions) -> Result<Self> {
assert!(
!skip_compaction || is_short_session,
!options.skip_compaction || options.is_short_session,
"skip_compaction=true requires is_short_session=true"
);
let db = TurboPersistence::open_with_config(versioned_path, db_config())?;
let is_fresh = db.is_empty();
Ok(Self {
db,
is_ci,
is_short_session,
options,
is_fresh,
skip_compaction,
})
}
......@@ -83,10 +77,12 @@ impl TurboKeyValueDatabase {
pub fn empty_in_memory() -> Self {
Self {
db: TurboPersistence::empty_in_memory_with_config(db_config()),
is_ci: false,
is_short_session: true,
options: BackingStorageOptions {
is_ci: false,
is_short_session: true,
skip_compaction: true,
},
is_fresh: true,
skip_compaction: true,
}
}
......@@ -129,7 +125,7 @@ impl TurboKeyValueDatabase {
/// Returns `Ok(Some(stats))` with the bytes written/deleted if compaction actually merged
/// files, `Ok(None)` if there was nothing to compact.
pub fn compact(&self) -> Result<Option<CommitStats>> {
if self.is_short_session || self.db.is_empty() {
if self.options.is_short_session || self.db.is_empty() {
return Ok(None);
}
do_compact(
......@@ -148,8 +144,8 @@ impl TurboKeyValueDatabase {
pub fn shutdown(&self) -> Result<()> {
// Compact the database on shutdown
// (Avoid compacting a fresh database since we don't have any usage info yet)
if !self.is_fresh && !self.skip_compaction {
if self.is_ci {
if !self.is_fresh && !self.options.skip_compaction {
if self.options.is_ci {
// Fully compact in CI to reduce cache size
do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)?;
} else {
......
......@@ -491,7 +491,18 @@ mod tests {
use turbo_tasks::TaskId;
use super::*;
use crate::database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer};
use crate::{
BackingStorageOptions,
database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer},
};
/// Options used by these tests. `is_short_session` disables background compaction, which
/// requires a turbo-tasks context that these tests don't set up.
const TEST_STORAGE_OPTIONS: BackingStorageOptions = BackingStorageOptions {
is_ci: false,
is_short_session: true,
skip_compaction: false,
};
/// Helper to write to the database using the concurrent batch API.
fn write_task_cache_entry(
......@@ -533,9 +544,7 @@ mod tests {
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Use is_short_session=true to disable background compaction (which requires turbo-tasks
// context)
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
// Simulate a hash collision by writing multiple TaskIds with the same hash key
let collision_hash: u64 = 0xDEADBEEF;
......@@ -575,7 +584,7 @@ mod tests {
// Write all entries in a single batch with flush (like save_snapshot does)
{
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
let batch = db.write_batch()?;
for (hash, task_id) in hashes.iter().zip(task_ids.iter()) {
......@@ -594,7 +603,7 @@ mod tests {
// Reopen and verify all entries are readable
{
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
let mut found = 0;
let mut missing = 0;
for (hash, expected_id) in hashes.iter().zip(task_ids.iter()) {
......@@ -632,7 +641,14 @@ mod tests {
let survivor_id = TaskId::try_from(222u32).unwrap();
let deleted_key = (*deleted_id).to_le_bytes();
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
let db = TurboKeyValueDatabase::new(
path.to_path_buf(),
BackingStorageOptions {
is_ci: false,
is_short_session: true,
skip_compaction: false,
},
)?;
// Both ids collide in one TaskCache bucket, purely on disk; the deleted task also has
// meta and data entries.
......
......@@ -25,19 +25,33 @@ pub use crate::{
kv_backing_storage::TurboBackingStorage,
};
/// Options controlling how the on-disk persistent cache database is opened and compacted.
#[derive(Clone, Copy, Debug, Default)]
pub struct BackingStorageOptions {
/// Whether the process is running in a CI environment. Enables more aggressive (full)
/// compaction on shutdown to reduce the size of the cache that gets uploaded.
pub is_ci: bool,
/// Whether this is a short-lived session (e.g. a single build). Disables background
/// persistence during the session
pub is_short_session: bool,
/// Whether to skip database compaction on shutdown entirely
pub skip_compaction: bool,
}
/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
///
/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
pub fn turbo_backing_storage(
base_path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
is_short_session: bool,
skip_compaction: bool,
options: BackingStorageOptions,
) -> Result<(TurboBackingStorage, StartupCacheState)> {
TurboBackingStorage::open_versioned_on_disk(base_path.to_owned(), version_info, is_ci, |path| {
TurboKeyValueDatabase::new(path, is_ci, is_short_session, skip_compaction)
})
TurboBackingStorage::open_versioned_on_disk(
base_path.to_owned(),
version_info,
options.is_ci,
|path| TurboKeyValueDatabase::new(path, options),
)
}
/// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by
......
......@@ -11,7 +11,9 @@ use anyhow::Result;
use turbo_tasks::{
ResolvedVc, State, TurboTasks, Vc, unmark_top_level_task_may_leak_eventually_consistent_state,
};
use turbo_tasks_backend::{BackendOptions, EvictionMode, GitVersionInfo, TurboTasksBackend};
use turbo_tasks_backend::{
BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, TurboTasksBackend,
};
/// Creates a fresh per-call persistence directory in the OS temp dir, with the test `name` as a
/// prefix so a leaked directory from a failed run is identifiable. The unique suffix from
......@@ -48,9 +50,11 @@ fn create_tt_with_workers(
describe: "test-unversioned",
dirty: false,
},
false,
true,
true,
BackingStorageOptions {
is_short_session: true,
skip_compaction: true,
..Default::default()
},
)
.unwrap()
.0,
......
......@@ -20,9 +20,10 @@
describe: "test-unversioned",
dirty: false,
},
false,
true,
false,
turbo_tasks_backend::BackingStorageOptions {
is_short_session: true,
..Default::default()
},
).unwrap().0
)
)
......
......@@ -19,9 +19,10 @@
describe: "test-unversioned",
dirty: false,
},
false,
true,
false,
turbo_tasks_backend::BackingStorageOptions {
is_short_session: true,
..Default::default()
},
).unwrap().0
)
)
......
......@@ -19,9 +19,10 @@
describe: "test-unversioned",
dirty: false,
},
false,
true,
false,
turbo_tasks_backend::BackingStorageOptions {
is_short_session: true,
..Default::default()
},
).unwrap().0
)
)
......
......@@ -14,8 +14,8 @@ use turbo_tasks::{
read_strongly_consistent_and_apply_effects, take_effects,
};
use turbo_tasks_backend::{
BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend,
noop_backing_storage, turbo_backing_storage,
BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode,
TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
};
use turbo_tasks_fs::FileSystem;
use turbo_unix_path::join_path;
......@@ -543,8 +543,15 @@ pub async fn build(args: &BuildArguments) -> Result<()> {
.cache_dir
.clone()
.unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache"));
let (backing_storage, cache_state) =
turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?;
let (backing_storage, cache_state) = turbo_backing_storage(
&cache_dir,
&version_info,
BackingStorageOptions {
is_ci,
is_short_session,
skip_compaction: false,
},
)?;
let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
StorageMode::ReadOnly
} else if is_ci || is_short_session {
......
......@@ -18,8 +18,8 @@ use turbo_tasks::{
util::{FormatBytes, FormatDuration},
};
use turbo_tasks_backend::{
BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend,
noop_backing_storage, turbo_backing_storage,
BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode,
TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
};
use turbo_tasks_fs::FileSystem;
use turbo_tasks_malloc::TurboMalloc;
......@@ -390,8 +390,15 @@ pub async fn start_server(args: &DevArguments) -> Result<()> {
.cache_dir
.clone()
.unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache"));
let (backing_storage, cache_state) =
turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?;
let (backing_storage, cache_state) = turbo_backing_storage(
&cache_dir,
&version_info,
BackingStorageOptions {
is_ci,
is_short_session,
skip_compaction: false,
},
)?;
let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
StorageMode::ReadOnly
} else if is_ci || is_short_session {
......
......@@ -20,9 +20,10 @@
describe: "test-unversioned",
dirty: false,
},
false,
true,
false,
turbo_tasks_backend::BackingStorageOptions {
is_short_session: true,
..Default::default()
},
).unwrap().0
)
)
......
......@@ -331,9 +331,10 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
describe: "test-unversioned",
dirty: false,
},
false,
true,
false,
turbo_tasks_backend::BackingStorageOptions {
is_short_session: true,
..Default::default()
},
)
.unwrap()
.0,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment