Unverified Commit d898c8f5 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

introduce an options struct for constructing backend storage (#97284)

Improve legibility a bit, requested by @bgw here
https://github.com/vercel/next.js/pull/95976#discussion_r3762093713
parent 74a30b10
...@@ -57,7 +57,7 @@ use turbo_tasks::{ ...@@ -57,7 +57,7 @@ use turbo_tasks::{
trace::TraceRawVcs, trace::TraceRawVcs,
unmark_top_level_task_may_leak_eventually_consistent_state, unmark_top_level_task_may_leak_eventually_consistent_state,
}; };
use turbo_tasks_backend::db_invalidation::invalidation_reasons; use turbo_tasks_backend::{BackingStorageOptions, db_invalidation::invalidation_reasons};
#[cfg(windows)] #[cfg(windows)]
use turbo_tasks_fs::windows::to_verbatim_with_case_folded_disk; use turbo_tasks_fs::windows::to_verbatim_with_case_folded_disk;
use turbo_tasks_fs::{ use turbo_tasks_fs::{
...@@ -576,18 +576,17 @@ pub fn project_new<'env>( ...@@ -576,18 +576,17 @@ pub fn project_new<'env>(
env.spawn_future( env.spawn_future(
async move { async move {
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true); let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
let skip_compaction = turbo_engine_options.skip_compaction.unwrap_or(false);
let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction; let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction;
let turbo_tasks = create_turbo_tasks( let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir), PathBuf::from(&options.dist_dir),
&options.next_version, &options.next_version,
options.is_persistent_caching_enabled, options.is_persistent_caching_enabled,
dependency_tracking, dependency_tracking,
is_ci, BackingStorageOptions {
is_short_session, is_ci: turbo_engine_options.is_ci.unwrap_or(false),
skip_compaction, is_short_session: turbo_engine_options.is_short_session.unwrap_or(false),
skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false),
},
turbopack_memory_eviction, turbopack_memory_eviction,
)?; )?;
let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks); let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
......
...@@ -25,8 +25,9 @@ use turbo_tasks::{ ...@@ -25,8 +25,9 @@ use turbo_tasks::{
message_queue::{CompilationEvent, Severity}, message_queue::{CompilationEvent, Severity},
}; };
use turbo_tasks_backend::{ use turbo_tasks_backend::{
BackendOptions, EvictionMode, GitVersionInfo, StartupCacheState, TurboTasksBackend, BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, StartupCacheState,
db_invalidation::invalidation_reasons, noop_backing_storage, turbo_backing_storage, TurboTasksBackend, db_invalidation::invalidation_reasons, noop_backing_storage,
turbo_backing_storage,
}; };
pub type NextTurboTasks = Arc<TurboTasks<TurboTasksBackend>>; pub type NextTurboTasks = Arc<TurboTasks<TurboTasksBackend>>;
...@@ -289,20 +290,21 @@ pub fn create_turbo_tasks( ...@@ -289,20 +290,21 @@ pub fn create_turbo_tasks(
next_version: &str, next_version: &str,
persistent_caching: bool, persistent_caching: bool,
dependency_tracking: bool, dependency_tracking: bool,
is_ci: bool, storage_options: BackingStorageOptions,
is_short_session: bool,
skip_compaction: bool,
turbopack_memory_eviction: MemoryEvictionMode, turbopack_memory_eviction: MemoryEvictionMode,
) -> Result<NextTurboTasks> { ) -> Result<NextTurboTasks> {
let BackingStorageOptions {
is_ci,
is_short_session,
..
} = storage_options;
Ok(if persistent_caching { Ok(if persistent_caching {
let describe = cache_describe(next_version); let describe = cache_describe(next_version);
let version_info = git_version_info(&describe); let version_info = git_version_info(&describe);
let (backing_storage, cache_state) = turbo_backing_storage( let (backing_storage, cache_state) = turbo_backing_storage(
&output_path.join("cache").join("turbopack"), &output_path.join("cache").join("turbopack"),
&version_info, &version_info,
is_ci, storage_options,
is_short_session,
skip_compaction,
)?; )?;
let tt = TurboTasks::new(TurboTasksBackend::new( let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions { BackendOptions {
......
...@@ -17,7 +17,10 @@ use turbo_tasks::{ ...@@ -17,7 +17,10 @@ use turbo_tasks::{
turbo_tasks, turbo_tasks,
}; };
use crate::database::{key_value_database::KeySpace, write_batch::WriteBuffer}; use crate::{
BackingStorageOptions,
database::{key_value_database::KeySpace, write_batch::WriteBuffer},
};
mod parallel_scheduler; mod parallel_scheduler;
pub(crate) use parallel_scheduler::TurboTasksParallelScheduler; pub(crate) use parallel_scheduler::TurboTasksParallelScheduler;
...@@ -49,31 +52,22 @@ pub const COMPACT_CONFIG: CompactConfig = CompactConfig { ...@@ -49,31 +52,22 @@ pub const COMPACT_CONFIG: CompactConfig = CompactConfig {
pub struct TurboKeyValueDatabase { pub struct TurboKeyValueDatabase {
db: TurboPersistence<TurboTasksParallelScheduler, FAMILIES>, db: TurboPersistence<TurboTasksParallelScheduler, FAMILIES>,
is_ci: bool, options: BackingStorageOptions,
is_short_session: bool,
is_fresh: bool, is_fresh: bool,
skip_compaction: bool,
} }
impl TurboKeyValueDatabase { impl TurboKeyValueDatabase {
pub fn new( pub fn new(versioned_path: PathBuf, options: BackingStorageOptions) -> Result<Self> {
versioned_path: PathBuf,
is_ci: bool,
is_short_session: bool,
skip_compaction: bool,
) -> Result<Self> {
assert!( assert!(
!skip_compaction || is_short_session, !options.skip_compaction || options.is_short_session,
"skip_compaction=true requires is_short_session=true" "skip_compaction=true requires is_short_session=true"
); );
let db = TurboPersistence::open_with_config(versioned_path, db_config())?; let db = TurboPersistence::open_with_config(versioned_path, db_config())?;
let is_fresh = db.is_empty(); let is_fresh = db.is_empty();
Ok(Self { Ok(Self {
db, db,
is_ci, options,
is_short_session,
is_fresh, is_fresh,
skip_compaction,
}) })
} }
...@@ -83,10 +77,12 @@ impl TurboKeyValueDatabase { ...@@ -83,10 +77,12 @@ impl TurboKeyValueDatabase {
pub fn empty_in_memory() -> Self { pub fn empty_in_memory() -> Self {
Self { Self {
db: TurboPersistence::empty_in_memory_with_config(db_config()), db: TurboPersistence::empty_in_memory_with_config(db_config()),
is_ci: false, options: BackingStorageOptions {
is_short_session: true, is_ci: false,
is_short_session: true,
skip_compaction: true,
},
is_fresh: true, is_fresh: true,
skip_compaction: true,
} }
} }
...@@ -129,7 +125,7 @@ impl TurboKeyValueDatabase { ...@@ -129,7 +125,7 @@ impl TurboKeyValueDatabase {
/// Returns `Ok(Some(stats))` with the bytes written/deleted if compaction actually merged /// Returns `Ok(Some(stats))` with the bytes written/deleted if compaction actually merged
/// files, `Ok(None)` if there was nothing to compact. /// files, `Ok(None)` if there was nothing to compact.
pub fn compact(&self) -> Result<Option<CommitStats>> { pub fn compact(&self) -> Result<Option<CommitStats>> {
if self.is_short_session || self.db.is_empty() { if self.options.is_short_session || self.db.is_empty() {
return Ok(None); return Ok(None);
} }
do_compact( do_compact(
...@@ -148,8 +144,8 @@ impl TurboKeyValueDatabase { ...@@ -148,8 +144,8 @@ impl TurboKeyValueDatabase {
pub fn shutdown(&self) -> Result<()> { pub fn shutdown(&self) -> Result<()> {
// Compact the database on shutdown // Compact the database on shutdown
// (Avoid compacting a fresh database since we don't have any usage info yet) // (Avoid compacting a fresh database since we don't have any usage info yet)
if !self.is_fresh && !self.skip_compaction { if !self.is_fresh && !self.options.skip_compaction {
if self.is_ci { if self.options.is_ci {
// Fully compact in CI to reduce cache size // Fully compact in CI to reduce cache size
do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)?; do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)?;
} else { } else {
......
...@@ -491,7 +491,18 @@ mod tests { ...@@ -491,7 +491,18 @@ mod tests {
use turbo_tasks::TaskId; use turbo_tasks::TaskId;
use super::*; use super::*;
use crate::database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer}; use crate::{
BackingStorageOptions,
database::{turbo::TurboKeyValueDatabase, write_batch::WriteBuffer},
};
/// Options used by these tests. `is_short_session` disables background compaction, which
/// requires a turbo-tasks context that these tests don't set up.
const TEST_STORAGE_OPTIONS: BackingStorageOptions = BackingStorageOptions {
is_ci: false,
is_short_session: true,
skip_compaction: false,
};
/// Helper to write to the database using the concurrent batch API. /// Helper to write to the database using the concurrent batch API.
fn write_task_cache_entry( fn write_task_cache_entry(
...@@ -533,9 +544,7 @@ mod tests { ...@@ -533,9 +544,7 @@ mod tests {
let tempdir = tempfile::tempdir()?; let tempdir = tempfile::tempdir()?;
let path = tempdir.path(); let path = tempdir.path();
// Use is_short_session=true to disable background compaction (which requires turbo-tasks let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
// context)
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?;
// Simulate a hash collision by writing multiple TaskIds with the same hash key // Simulate a hash collision by writing multiple TaskIds with the same hash key
let collision_hash: u64 = 0xDEADBEEF; let collision_hash: u64 = 0xDEADBEEF;
...@@ -575,7 +584,7 @@ mod tests { ...@@ -575,7 +584,7 @@ mod tests {
// Write all entries in a single batch with flush (like save_snapshot does) // Write all entries in a single batch with flush (like save_snapshot does)
{ {
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
let batch = db.write_batch()?; let batch = db.write_batch()?;
for (hash, task_id) in hashes.iter().zip(task_ids.iter()) { for (hash, task_id) in hashes.iter().zip(task_ids.iter()) {
...@@ -594,7 +603,7 @@ mod tests { ...@@ -594,7 +603,7 @@ mod tests {
// Reopen and verify all entries are readable // Reopen and verify all entries are readable
{ {
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; let db = TurboKeyValueDatabase::new(path.to_path_buf(), TEST_STORAGE_OPTIONS)?;
let mut found = 0; let mut found = 0;
let mut missing = 0; let mut missing = 0;
for (hash, expected_id) in hashes.iter().zip(task_ids.iter()) { for (hash, expected_id) in hashes.iter().zip(task_ids.iter()) {
...@@ -632,7 +641,14 @@ mod tests { ...@@ -632,7 +641,14 @@ mod tests {
let survivor_id = TaskId::try_from(222u32).unwrap(); let survivor_id = TaskId::try_from(222u32).unwrap();
let deleted_key = (*deleted_id).to_le_bytes(); let deleted_key = (*deleted_id).to_le_bytes();
let db = TurboKeyValueDatabase::new(path.to_path_buf(), false, true, false)?; let db = TurboKeyValueDatabase::new(
path.to_path_buf(),
BackingStorageOptions {
is_ci: false,
is_short_session: true,
skip_compaction: false,
},
)?;
// Both ids collide in one TaskCache bucket, purely on disk; the deleted task also has // Both ids collide in one TaskCache bucket, purely on disk; the deleted task also has
// meta and data entries. // meta and data entries.
......
...@@ -25,19 +25,33 @@ pub use crate::{ ...@@ -25,19 +25,33 @@ pub use crate::{
kv_backing_storage::TurboBackingStorage, kv_backing_storage::TurboBackingStorage,
}; };
/// Options controlling how the on-disk persistent cache database is opened and compacted.
#[derive(Clone, Copy, Debug, Default)]
pub struct BackingStorageOptions {
/// Whether the process is running in a CI environment. Enables more aggressive (full)
/// compaction on shutdown to reduce the size of the cache that gets uploaded.
pub is_ci: bool,
/// Whether this is a short-lived session (e.g. a single build). Disables background
/// persistence during the session
pub is_short_session: bool,
/// Whether to skip database compaction on shutdown entirely
pub skip_compaction: bool,
}
/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`]. /// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
/// ///
/// Information about the state of the on-disk cache is returned using [`StartupCacheState`]. /// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
pub fn turbo_backing_storage( pub fn turbo_backing_storage(
base_path: &Path, base_path: &Path,
version_info: &GitVersionInfo, version_info: &GitVersionInfo,
is_ci: bool, options: BackingStorageOptions,
is_short_session: bool,
skip_compaction: bool,
) -> Result<(TurboBackingStorage, StartupCacheState)> { ) -> Result<(TurboBackingStorage, StartupCacheState)> {
TurboBackingStorage::open_versioned_on_disk(base_path.to_owned(), version_info, is_ci, |path| { TurboBackingStorage::open_versioned_on_disk(
TurboKeyValueDatabase::new(path, is_ci, is_short_session, skip_compaction) base_path.to_owned(),
}) version_info,
options.is_ci,
|path| TurboKeyValueDatabase::new(path, options),
)
} }
/// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by /// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by
......
...@@ -11,7 +11,9 @@ use anyhow::Result; ...@@ -11,7 +11,9 @@ use anyhow::Result;
use turbo_tasks::{ use turbo_tasks::{
ResolvedVc, State, TurboTasks, Vc, unmark_top_level_task_may_leak_eventually_consistent_state, ResolvedVc, State, TurboTasks, Vc, unmark_top_level_task_may_leak_eventually_consistent_state,
}; };
use turbo_tasks_backend::{BackendOptions, EvictionMode, GitVersionInfo, TurboTasksBackend}; use turbo_tasks_backend::{
BackendOptions, BackingStorageOptions, EvictionMode, GitVersionInfo, TurboTasksBackend,
};
/// Creates a fresh per-call persistence directory in the OS temp dir, with the test `name` as a /// Creates a fresh per-call persistence directory in the OS temp dir, with the test `name` as a
/// prefix so a leaked directory from a failed run is identifiable. The unique suffix from /// prefix so a leaked directory from a failed run is identifiable. The unique suffix from
...@@ -48,9 +50,11 @@ fn create_tt_with_workers( ...@@ -48,9 +50,11 @@ fn create_tt_with_workers(
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false, BackingStorageOptions {
true, is_short_session: true,
true, skip_compaction: true,
..Default::default()
},
) )
.unwrap() .unwrap()
.0, .0,
......
...@@ -20,9 +20,10 @@ ...@@ -20,9 +20,10 @@
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false, turbo_tasks_backend::BackingStorageOptions {
true, is_short_session: true,
false, ..Default::default()
},
).unwrap().0 ).unwrap().0
) )
) )
......
...@@ -19,9 +19,10 @@ ...@@ -19,9 +19,10 @@
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false, turbo_tasks_backend::BackingStorageOptions {
true, is_short_session: true,
false, ..Default::default()
},
).unwrap().0 ).unwrap().0
) )
) )
......
...@@ -19,9 +19,10 @@ ...@@ -19,9 +19,10 @@
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false, turbo_tasks_backend::BackingStorageOptions {
true, is_short_session: true,
false, ..Default::default()
},
).unwrap().0 ).unwrap().0
) )
) )
......
...@@ -14,8 +14,8 @@ use turbo_tasks::{ ...@@ -14,8 +14,8 @@ use turbo_tasks::{
read_strongly_consistent_and_apply_effects, take_effects, read_strongly_consistent_and_apply_effects, take_effects,
}; };
use turbo_tasks_backend::{ use turbo_tasks_backend::{
BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend, BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode,
noop_backing_storage, turbo_backing_storage, TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
}; };
use turbo_tasks_fs::FileSystem; use turbo_tasks_fs::FileSystem;
use turbo_unix_path::join_path; use turbo_unix_path::join_path;
...@@ -543,8 +543,15 @@ pub async fn build(args: &BuildArguments) -> Result<()> { ...@@ -543,8 +543,15 @@ pub async fn build(args: &BuildArguments) -> Result<()> {
.cache_dir .cache_dir
.clone() .clone()
.unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache")); .unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache"));
let (backing_storage, cache_state) = let (backing_storage, cache_state) = turbo_backing_storage(
turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?; &cache_dir,
&version_info,
BackingStorageOptions {
is_ci,
is_short_session,
skip_compaction: false,
},
)?;
let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
StorageMode::ReadOnly StorageMode::ReadOnly
} else if is_ci || is_short_session { } else if is_ci || is_short_session {
......
...@@ -18,8 +18,8 @@ use turbo_tasks::{ ...@@ -18,8 +18,8 @@ use turbo_tasks::{
util::{FormatBytes, FormatDuration}, util::{FormatBytes, FormatDuration},
}; };
use turbo_tasks_backend::{ use turbo_tasks_backend::{
BackendOptions, GitVersionInfo, StartupCacheState, StorageMode, TurboTasksBackend, BackendOptions, BackingStorageOptions, GitVersionInfo, StartupCacheState, StorageMode,
noop_backing_storage, turbo_backing_storage, TurboTasksBackend, noop_backing_storage, turbo_backing_storage,
}; };
use turbo_tasks_fs::FileSystem; use turbo_tasks_fs::FileSystem;
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
...@@ -390,8 +390,15 @@ pub async fn start_server(args: &DevArguments) -> Result<()> { ...@@ -390,8 +390,15 @@ pub async fn start_server(args: &DevArguments) -> Result<()> {
.cache_dir .cache_dir
.clone() .clone()
.unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache")); .unwrap_or_else(|| PathBuf::from(&*project_dir).join(".turbopack/cache"));
let (backing_storage, cache_state) = let (backing_storage, cache_state) = turbo_backing_storage(
turbo_backing_storage(&cache_dir, &version_info, is_ci, is_short_session, false)?; &cache_dir,
&version_info,
BackingStorageOptions {
is_ci,
is_short_session,
skip_compaction: false,
},
)?;
let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { let storage_mode = if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
StorageMode::ReadOnly StorageMode::ReadOnly
} else if is_ci || is_short_session { } else if is_ci || is_short_session {
......
...@@ -20,9 +20,10 @@ ...@@ -20,9 +20,10 @@
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false, turbo_tasks_backend::BackingStorageOptions {
true, is_short_session: true,
false, ..Default::default()
},
).unwrap().0 ).unwrap().0
) )
) )
......
...@@ -331,9 +331,10 @@ fn node_file_trace_persistent(#[case] input: CaseInput) { ...@@ -331,9 +331,10 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false, turbo_tasks_backend::BackingStorageOptions {
true, is_short_session: true,
false, ..Default::default()
},
) )
.unwrap() .unwrap()
.0, .0,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment