Unverified Commit df1f762a authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: Compact only at the end for short sessions (#82224)

### What?

Change the compaction behavior for "short sessions" (like next build). A longer "short session" might commit to the DB multiple times (every 60s currently), which previously resulted in compactions happening after multiple commits to the database. As compactions are expensive and memory hungry, we want to avoid them until the end of the session. At the end of the session we can run it in parallel to node.js SSG.
parent 8377abe4
......@@ -253,6 +253,8 @@ pub struct NapiTurboEngineOptions {
pub dependency_tracking: Option<bool>,
/// Whether the project is running in a CI environment.
pub is_ci: Option<bool>,
/// Whether the project is running in a short session.
pub is_short_session: Option<bool>,
}
impl From<NapiWatchOptions> for WatchOptions {
......@@ -434,12 +436,14 @@ pub fn project_new(
let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default();
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
let is_short_session = turbo_engine_options.is_short_session.unwrap_or(false);
let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir),
persistent_caching,
memory_limit,
dependency_tracking,
is_ci,
is_short_session,
)?;
let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
......
......@@ -185,6 +185,7 @@ pub fn create_turbo_tasks(
_memory_limit: usize,
dependency_tracking: bool,
is_ci: bool,
is_short_session: bool,
) -> Result<NextTurboTasks> {
Ok(if persistent_caching {
let version_info = GitVersionInfo {
......@@ -192,8 +193,12 @@ pub fn create_turbo_tasks(
dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true",
};
let (backing_storage, cache_state) =
default_backing_storage(&output_path.join("cache/turbopack"), &version_info, is_ci)?;
let (backing_storage, cache_state) = default_backing_storage(
&output_path.join("cache/turbopack"),
&version_info,
is_ci,
is_short_session,
)?;
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
......
......@@ -212,6 +212,8 @@ export interface NapiTurboEngineOptions {
dependencyTracking?: boolean
/** Whether the project is running in a CI environment. */
isCi?: boolean
/** Whether the project is running in a short session. */
isShortSession?: boolean
}
export declare function projectNew(
options: NapiProjectOptions,
......
......@@ -91,6 +91,7 @@ export async function turbopackBuild(): Promise<{
memoryLimit: config.experimental?.turbopackMemoryLimit,
dependencyTracking: persistentCaching,
isCi: isCI,
isShortSession: true,
}
)
try {
......
......@@ -254,6 +254,7 @@ export async function createHotReloaderTurbopack(
{
persistentCaching: isPersistentCachingEnabled(opts.nextConfig),
memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit,
isShortSession: false,
}
)
backgroundLogCompilationEvents(project, {
......
......@@ -30,14 +30,18 @@ const COMPACT_CONFIG: CompactConfig = CompactConfig {
pub struct TurboKeyValueDatabase {
db: Arc<TurboPersistence>,
compact_join_handle: Mutex<Option<JoinHandle<Result<()>>>>,
is_ci: bool,
is_short_session: bool,
}
impl TurboKeyValueDatabase {
pub fn new(versioned_path: PathBuf) -> Result<Self> {
pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
let db = Arc::new(TurboPersistence::open(versioned_path)?);
let mut this = Self {
db: db.clone(),
compact_join_handle: Mutex::new(None),
is_ci,
is_short_session,
};
// start compaction in background if the database is not empty
if !db.is_empty() {
......@@ -98,8 +102,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
Ok(WriteBatch::concurrent(TurboWriteBatch {
batch: self.db.write_batch()?,
db: &self.db,
compact_join_handle: &self.compact_join_handle,
initial_write: self.db.is_empty(),
compact_join_handle: (!self.is_short_session && !self.db.is_empty())
.then_some(&self.compact_join_handle),
}))
}
......@@ -110,6 +114,16 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
if let Some(join_handle) = self.compact_join_handle.lock().take() {
join_handle.join().unwrap()?;
}
// Compact the database on shutdown
self.db.compact(&CompactConfig {
max_merge_segment_count: if self.is_ci {
// Fully compact in CI to reduce cache size
usize::MAX
} else {
available_parallelism().map_or(4, |c| max(4, c.get()))
},
..COMPACT_CONFIG
})?;
// Shutdown the database
self.db.shutdown()
}
......@@ -118,8 +132,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
pub struct TurboWriteBatch<'a> {
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>,
db: &'a Arc<TurboPersistence>,
compact_join_handle: &'a Mutex<Option<JoinHandle<Result<()>>>>,
initial_write: bool,
compact_join_handle: Option<&'a Mutex<Option<JoinHandle<Result<()>>>>>,
}
impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
......@@ -140,7 +153,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
// Commit the write batch
self.db.commit_write_batch(self.batch)?;
if !self.initial_write {
if let Some(compact_join_handle) = self.compact_join_handle {
// Start a new compaction in the background
let db = self.db.clone();
let handle = spawn(move || {
......@@ -150,7 +163,7 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
..COMPACT_CONFIG
})
});
self.compact_join_handle.lock().replace(handle);
compact_join_handle.lock().replace(handle);
}
Ok(())
......
......@@ -83,12 +83,13 @@ pub fn turbo_backing_storage(
base_path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
is_short_session: bool,
) -> Result<(TurboBackingStorage, StartupCacheState)> {
KeyValueDatabaseBackingStorage::open_versioned_on_disk(
base_path.to_owned(),
version_info,
is_ci,
TurboKeyValueDatabase::new,
|path| TurboKeyValueDatabase::new(path, is_ci, is_short_session),
)
}
......@@ -111,6 +112,7 @@ pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
is_short_session: bool,
) -> Result<(DefaultBackingStorage, StartupCacheState)> {
#[cfg(feature = "lmdb")]
{
......@@ -118,6 +120,6 @@ pub fn default_backing_storage(
}
#[cfg(not(feature = "lmdb"))]
{
turbo_backing_storage(path, version_info, is_ci)
turbo_backing_storage(path, version_info, is_ci, is_short_session)
}
}
......@@ -16,7 +16,8 @@
describe: "test-unversioned",
dirty: false,
},
false
false,
true,
).unwrap().0
)
)
......
......@@ -16,7 +16,8 @@
describe: "test-unversioned",
dirty: false,
},
false
false,
true,
).unwrap().0
)
)
......
......@@ -292,6 +292,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
dirty: false,
},
false,
true,
)
.unwrap()
.0,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment