Unverified Commit df1f762a authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: Compact only at the end for short sessions (#82224)

### What?

Change the compaction behavior for "short sessions" (like next build). A longer "short session" might commit to the DB multiple times (every 60s currently), which previously resulted in compactions happening after multiple commits to the database. As compactions are expensive and memory hungry, we want to avoid them until the end of the session. At the end of the session we can run it in parallel to node.js SSG.
parent 8377abe4
...@@ -253,6 +253,8 @@ pub struct NapiTurboEngineOptions { ...@@ -253,6 +253,8 @@ pub struct NapiTurboEngineOptions {
pub dependency_tracking: Option<bool>, pub dependency_tracking: Option<bool>,
/// Whether the project is running in a CI environment. /// Whether the project is running in a CI environment.
pub is_ci: Option<bool>, pub is_ci: Option<bool>,
/// Whether the project is running in a short session.
pub is_short_session: Option<bool>,
} }
impl From<NapiWatchOptions> for WatchOptions { impl From<NapiWatchOptions> for WatchOptions {
...@@ -434,12 +436,14 @@ pub fn project_new( ...@@ -434,12 +436,14 @@ pub fn project_new(
let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default(); let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default();
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true); let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let is_ci = turbo_engine_options.is_ci.unwrap_or(false); let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
let turbo_tasks = create_turbo_tasks( let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir), PathBuf::from(&options.dist_dir),
persistent_caching, persistent_caching,
memory_limit, memory_limit,
dependency_tracking, dependency_tracking,
is_ci, is_ci,
is_short_session,
)?; )?;
let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks); let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
......
...@@ -185,6 +185,7 @@ pub fn create_turbo_tasks( ...@@ -185,6 +185,7 @@ pub fn create_turbo_tasks(
_memory_limit: usize, _memory_limit: usize,
dependency_tracking: bool, dependency_tracking: bool,
is_ci: bool, is_ci: bool,
is_short_session: bool,
) -> Result<NextTurboTasks> { ) -> Result<NextTurboTasks> {
Ok(if persistent_caching { Ok(if persistent_caching {
let version_info = GitVersionInfo { let version_info = GitVersionInfo {
...@@ -192,8 +193,12 @@ pub fn create_turbo_tasks( ...@@ -192,8 +193,12 @@ pub fn create_turbo_tasks(
dirty: option_env!("CI").is_none_or(|value| value.is_empty()) dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true", && env!("VERGEN_GIT_DIRTY") == "true",
}; };
let (backing_storage, cache_state) = let (backing_storage, cache_state) = default_backing_storage(
default_backing_storage(&output_path.join("cache/turbopack"), &version_info, is_ci)?; &output_path.join("cache/turbopack"),
&version_info,
is_ci,
is_short_session,
)?;
let tt = TurboTasks::new(TurboTasksBackend::new( let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions { BackendOptions {
storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
......
...@@ -212,6 +212,8 @@ export interface NapiTurboEngineOptions { ...@@ -212,6 +212,8 @@ export interface NapiTurboEngineOptions {
dependencyTracking?: boolean dependencyTracking?: boolean
/** Whether the project is running in a CI environment. */ /** Whether the project is running in a CI environment. */
isCi?: boolean isCi?: boolean
/** Whether the project is running in a short session. */
isShortSession?: boolean
} }
export declare function projectNew( export declare function projectNew(
options: NapiProjectOptions, options: NapiProjectOptions,
......
...@@ -91,6 +91,7 @@ export async function turbopackBuild(): Promise<{ ...@@ -91,6 +91,7 @@ export async function turbopackBuild(): Promise<{
memoryLimit: config.experimental?.turbopackMemoryLimit, memoryLimit: config.experimental?.turbopackMemoryLimit,
dependencyTracking: persistentCaching, dependencyTracking: persistentCaching,
isCi: isCI, isCi: isCI,
isShortSession: true,
} }
) )
try { try {
......
...@@ -254,6 +254,7 @@ export async function createHotReloaderTurbopack( ...@@ -254,6 +254,7 @@ export async function createHotReloaderTurbopack(
{ {
persistentCaching: isPersistentCachingEnabled(opts.nextConfig), persistentCaching: isPersistentCachingEnabled(opts.nextConfig),
memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit, memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit,
isShortSession: false,
} }
) )
backgroundLogCompilationEvents(project, { backgroundLogCompilationEvents(project, {
......
...@@ -30,14 +30,18 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig { ...@@ -30,14 +30,18 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig {
pub struct TurboKeyValueDatabase { pub struct TurboKeyValueDatabase {
db: Arc<TurboPersistence>, db: Arc<TurboPersistence>,
compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>, compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>,
is_ci: bool,
is_short_session: bool,
} }
impl TurboKeyValueDatabase { impl TurboKeyValueDatabase {
pub fn new(versioned_path: PathBuf) -> Result<Self> { pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
let db = Arc::new(TurboPersistence::open(versioned_path)?); let db = Arc::new(TurboPersistence::open(versioned_path)?);
let mut this = Self { let mut this = Self {
db: db.clone(), db: db.clone(),
compact_join_handle: Mutex::new(None), compact_join_handle: Mutex::new(None),
is_ci,
is_short_session,
}; };
// start compaction in background if the database is not empty // start compaction in background if the database is not empty
if !db.is_empty() { if !db.is_empty() {
...@@ -98,8 +102,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -98,8 +102,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
Ok(WriteBatch::concurrent(TurboWriteBatch { Ok(WriteBatch::concurrent(TurboWriteBatch {
batch: self.db.write_batch()?, batch: self.db.write_batch()?,
db: &self.db, db: &self.db,
compact_join_handle: &self.compact_join_handle, compact_join_handle: (!self.is_short_session && !self.db.is_empty())
initial_write: self.db.is_empty(), .then_some(&self.compact_join_handle),
})) }))
} }
...@@ -110,6 +114,16 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -110,6 +114,16 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
if let Some(join_handle) = self.compact_join_handle.lock().take() { if let Some(join_handle) = self.compact_join_handle.lock().take() {
join_handle.join().unwrap()?; join_handle.join().unwrap()?;
} }
// Compact the database on shutdown
self.db.compact(&CompactConfig {
max_merge_segment_count: if self.is_ci {
// Fully compact in CI to reduce cache size
usize::MAX
} else {
available_parallelism().map_or(4, |c| max(4, c.get()))
},
..COMPACT_CONFIG
})?;
// Shutdown the database // Shutdown the database
self.db.shutdown() self.db.shutdown()
} }
...@@ -118,8 +132,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -118,8 +132,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
pub struct TurboWriteBatch<'a> { pub struct TurboWriteBatch<'a> {
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>, batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>,
db: &'a Arc<TurboPersistence>, db: &'a Arc<TurboPersistence>,
compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>, compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>,
initial_write: bool,
} }
impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> { impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
...@@ -140,7 +153,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> { ...@@ -140,7 +153,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
// Commit the write batch // Commit the write batch
self.db.commit_write_batch(self.batch)?; self.db.commit_write_batch(self.batch)?;
if !self.initial_write { if let Some(compact_join_handle) = self.compact_join_handle {
// Start a new compaction in the background // Start a new compaction in the background
let db = self.db.clone(); let db = self.db.clone();
let handle = spawn(move || { let handle = spawn(move || {
...@@ -150,7 +163,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> { ...@@ -150,7 +163,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
..COMPACT_CONFIG ..COMPACT_CONFIG
}) })
}); });
self.compact_join_handle.lock().replace(handle); compact_join_handle.lock().replace(handle);
} }
Ok(()) Ok(())
......
...@@ -83,12 +83,13 @@ pub fn turbo_backing_storage( ...@@ -83,12 +83,13 @@ pub fn turbo_backing_storage(
base_path: &Path, base_path: &Path,
version_info: &GitVersionInfo, version_info: &GitVersionInfo,
is_ci: bool, is_ci: bool,
is_short_session: bool,
) -> Result<(TurboBackingStorage, StartupCacheState)> { ) -> Result<(TurboBackingStorage, StartupCacheState)> {
KeyValueDatabaseBackingStorage::open_versioned_on_disk( KeyValueDatabaseBackingStorage::open_versioned_on_disk(
base_path.to_owned(), base_path.to_owned(),
version_info, version_info,
is_ci, is_ci,
TurboKeyValueDatabase::new, |path| TurboKeyValueDatabase::new(path, is_ci, is_short_session),
) )
} }
...@@ -111,6 +112,7 @@ pub fn default_backing_storage( ...@@ -111,6 +112,7 @@ pub fn default_backing_storage(
path: &Path, path: &Path,
version_info: &GitVersionInfo, version_info: &GitVersionInfo,
is_ci: bool, is_ci: bool,
is_short_session: bool,
) -> Result<(DefaultBackingStorage, StartupCacheState)> { ) -> Result<(DefaultBackingStorage, StartupCacheState)> {
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
{ {
...@@ -118,6 +120,6 @@ pub fn default_backing_storage( ...@@ -118,6 +120,6 @@ pub fn default_backing_storage(
} }
#[cfg(not(feature = "lmdb"))] #[cfg(not(feature = "lmdb"))]
{ {
turbo_backing_storage(path, version_info, is_ci) turbo_backing_storage(path, version_info, is_ci, is_short_session)
} }
} }
...@@ -16,7 +16,8 @@ ...@@ -16,7 +16,8 @@
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false false,
true,
).unwrap().0 ).unwrap().0
) )
) )
......
...@@ -16,7 +16,8 @@ ...@@ -16,7 +16,8 @@
describe: "test-unversioned", describe: "test-unversioned",
dirty: false, dirty: false,
}, },
false false,
true,
).unwrap().0 ).unwrap().0
) )
) )
......
...@@ -292,6 +292,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) { ...@@ -292,6 +292,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
dirty: false, dirty: false,
}, },
false, false,
true,
) )
.unwrap() .unwrap()
.0, .0,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment