Unverified Commit bf13ab8a authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Introduce config options for Turbopack GC (#98378)

### What?

Replaces the Turbopack GC environment variables with an
`experimental.turbopackGc`
config option

```js
// next.config.js
experimental: {
  turbopackGc: true,                                      // on, defaults
  turbopackGc: { minProgressMs: 100, rootTtlMs: 432e6 },  // on, tuned
}
```

This replaces the env vars which were unused and not the ergonomic
parent a30ad8fa
...@@ -90,7 +90,7 @@ use crate::{ ...@@ -90,7 +90,7 @@ use crate::{
endpoint::ExternalEndpoint, endpoint::ExternalEndpoint,
turbopack_ctx::{ turbopack_ctx::{
MemoryEvictionMode, NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject, MemoryEvictionMode, NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject,
NextTurboTasks, NextTurbopackContext, create_turbo_tasks, NapiTurbopackGcOptions, NextTurboTasks, NextTurbopackContext, create_turbo_tasks,
}, },
utils::{ utils::{
DetachedVc, NapiIssue, NapiUsedFeature, SubscriptionTask, TurbopackResult, get_issues, DetachedVc, NapiIssue, NapiUsedFeature, SubscriptionTask, TurbopackResult, get_issues,
...@@ -273,6 +273,8 @@ pub struct NapiTurboEngineOptions { ...@@ -273,6 +273,8 @@ pub struct NapiTurboEngineOptions {
pub skip_compaction: Option<bool>, pub skip_compaction: Option<bool>,
/// Turbopack memory eviction mode for the persistent cache. /// Turbopack memory eviction mode for the persistent cache.
pub turbopack_memory_eviction: MemoryEvictionMode, pub turbopack_memory_eviction: MemoryEvictionMode,
/// Tuning for Turbopack's reference-counting GC. `None` disables the GC.
pub gc: Option<NapiTurbopackGcOptions>,
} }
impl From<NapiWatchOptions> for WatchOptions { impl From<NapiWatchOptions> for WatchOptions {
...@@ -564,7 +566,6 @@ pub fn project_new<'env>( ...@@ -564,7 +566,6 @@ pub fn project_new<'env>(
env.spawn_future( env.spawn_future(
async move { async move {
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true); let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction;
let turbo_tasks = create_turbo_tasks( let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir), PathBuf::from(&options.dist_dir),
&options.next_version, &options.next_version,
...@@ -575,7 +576,8 @@ pub fn project_new<'env>( ...@@ -575,7 +576,8 @@ pub fn project_new<'env>(
is_short_session: turbo_engine_options.is_short_session.unwrap_or(false), is_short_session: turbo_engine_options.is_short_session.unwrap_or(false),
skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false), skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false),
}, },
turbopack_memory_eviction, turbo_engine_options.turbopack_memory_eviction,
turbo_engine_options.gc,
)?; )?;
let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks); let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
......
...@@ -6,7 +6,7 @@ use std::{ ...@@ -6,7 +6,7 @@ use std::{
io::{self, BufRead, Write}, io::{self, BufRead, Write},
path::PathBuf, path::PathBuf,
sync::{Arc, LazyLock, Mutex}, sync::{Arc, LazyLock, Mutex},
time::Instant, time::{Duration, Instant},
}; };
use anyhow::Result; use anyhow::Result;
...@@ -285,6 +285,27 @@ impl From<MemoryEvictionMode> for EvictionMode { ...@@ -285,6 +285,27 @@ impl From<MemoryEvictionMode> for EvictionMode {
} }
} }
#[napi(object)]
#[derive(Debug, Clone, Copy)]
pub struct NapiTurbopackGcOptions {
/// How long a GC pass runs before it will honour an interrupt, in milliseconds.
pub min_progress_ms: Option<f64>,
/// How long a GC root may go un-anchored before it ages out, in milliseconds.
pub root_ttl_ms: Option<f64>,
}
/// Converts a millisecond count that crossed the napi boundary into a `Duration`.
fn duration_from_millis_f64(ms: Option<f64>) -> Option<Duration> {
if let Some(ms) = ms
&& ms.is_finite()
&& ms >= 0.0
{
Duration::try_from_secs_f64(ms / 1000.0).ok()
} else {
None
}
}
pub fn create_turbo_tasks( pub fn create_turbo_tasks(
output_path: PathBuf, output_path: PathBuf,
next_version: &str, next_version: &str,
...@@ -292,6 +313,7 @@ pub fn create_turbo_tasks( ...@@ -292,6 +313,7 @@ pub fn create_turbo_tasks(
dependency_tracking: bool, dependency_tracking: bool,
storage_options: BackingStorageOptions, storage_options: BackingStorageOptions,
turbopack_memory_eviction: MemoryEvictionMode, turbopack_memory_eviction: MemoryEvictionMode,
gc: Option<NapiTurbopackGcOptions>,
) -> Result<NextTurboTasks> { ) -> Result<NextTurboTasks> {
let BackingStorageOptions { let BackingStorageOptions {
is_ci, is_ci,
...@@ -306,6 +328,17 @@ pub fn create_turbo_tasks( ...@@ -306,6 +328,17 @@ pub fn create_turbo_tasks(
&version_info, &version_info,
storage_options, storage_options,
)?; )?;
let (gc, gc_min_progress, gc_root_ttl) = match gc {
Some(NapiTurbopackGcOptions {
min_progress_ms,
root_ttl_ms,
}) => (
Some(true),
duration_from_millis_f64(min_progress_ms),
duration_from_millis_f64(root_ttl_ms),
),
None => (None, None, None),
};
let tt = TurboTasks::new(TurboTasksBackend::new( let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions { BackendOptions {
storage_mode: Some(if env::var("TURBO_ENGINE_READ_ONLY").is_ok() { storage_mode: Some(if env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
...@@ -318,6 +351,9 @@ pub fn create_turbo_tasks( ...@@ -318,6 +351,9 @@ pub fn create_turbo_tasks(
dependency_tracking, dependency_tracking,
num_workers: Some(tokio::runtime::Handle::current().metrics().num_workers()), num_workers: Some(tokio::runtime::Handle::current().metrics().num_workers()),
eviction_mode: EvictionMode::from(turbopack_memory_eviction), eviction_mode: EvictionMode::from(turbopack_memory_eviction),
gc,
gc_min_progress,
gc_root_ttl,
..Default::default() ..Default::default()
}, },
backing_storage, backing_storage,
...@@ -331,6 +367,8 @@ pub fn create_turbo_tasks( ...@@ -331,6 +367,8 @@ pub fn create_turbo_tasks(
BackendOptions { BackendOptions {
storage_mode: None, storage_mode: None,
dependency_tracking, dependency_tracking,
// GC is deliberately left at its default (off) here: without backing storage
// there is nothing to reclaim from disk, so `gc` would have no effect.
..Default::default() ..Default::default()
}, },
noop_backing_storage(), noop_backing_storage(),
......
...@@ -505,6 +505,19 @@ export interface NapiTurboEngineOptions { ...@@ -505,6 +505,19 @@ export interface NapiTurboEngineOptions {
skipCompaction?: boolean skipCompaction?: boolean
/** Turbopack memory eviction mode for the persistent cache. */ /** Turbopack memory eviction mode for the persistent cache. */
turbopackMemoryEviction: MemoryEvictionMode turbopackMemoryEviction: MemoryEvictionMode
/** Tuning for Turbopack's reference-counting GC. `None` disables the GC. */
gc?: NapiTurbopackGcOptions
}
/**
* Tuning for Turbopack's reference-counting GC, mirroring the
* `experimental.turbopackGc` config option.
*/
export interface NapiTurbopackGcOptions {
/** How long a GC pass runs before it will honour an interrupt, in milliseconds. */
minProgressMs?: number
/** How long a GC root may go un-anchored before it ages out, in milliseconds. */
rootTtlMs?: number
} }
export interface NapiUpdateInfo { export interface NapiUpdateInfo {
......
...@@ -13,6 +13,7 @@ import type { ...@@ -13,6 +13,7 @@ import type {
TraceQueryOptions, TraceQueryOptions,
TraceQueryResult, TraceQueryResult,
MemoryEvictionMode, MemoryEvictionMode,
NapiTurbopackGcOptions,
ServerHmrVersion as NativeServerHmrVersion, ServerHmrVersion as NativeServerHmrVersion,
projectCompilationEventsSubscribe, projectCompilationEventsSubscribe,
projectFeatureUsage, projectFeatureUsage,
...@@ -28,6 +29,8 @@ export type { NapiTurboEngineOptions as TurboEngineOptions } ...@@ -28,6 +29,8 @@ export type { NapiTurboEngineOptions as TurboEngineOptions }
export type { MemoryEvictionMode } export type { MemoryEvictionMode }
export type { NapiTurbopackGcOptions as TurbopackGcOptions }
export type Lockfile = { __napiType: 'Lockfile' } export type Lockfile = { __napiType: 'Lockfile' }
export interface TurbopackProjectCallbacks { export interface TurbopackProjectCallbacks {
......
...@@ -94,6 +94,7 @@ export async function turbopackAnalyze( ...@@ -94,6 +94,7 @@ export async function turbopackAnalyze(
}, },
{ {
turbopackMemoryEviction: config.experimental.turbopackMemoryEvictionMode, turbopackMemoryEviction: config.experimental.turbopackMemoryEvictionMode,
gc: config.experimental.turbopackGcOptions,
dependencyTracking: persistentCaching, dependencyTracking: persistentCaching,
isCi: isCI, isCi: isCI,
isShortSession: true, isShortSession: true,
......
...@@ -113,6 +113,7 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{ ...@@ -113,6 +113,7 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{
const sharedTurboOptions = { const sharedTurboOptions = {
turbopackMemoryEviction: config.experimental.turbopackMemoryEvictionMode, turbopackMemoryEviction: config.experimental.turbopackMemoryEvictionMode,
gc: config.experimental.turbopackGcOptions,
dependencyTracking: persistentCaching || hasDeferredEntries, dependencyTracking: persistentCaching || hasDeferredEntries,
isCi: isCI, isCi: isCI,
isShortSession: true, isShortSession: true,
......
...@@ -377,6 +377,15 @@ export const experimentalSchema = { ...@@ -377,6 +377,15 @@ export const experimentalSchema = {
turbopackMemoryEviction: z turbopackMemoryEviction: z
.union([z.literal(false), z.literal('full'), z.literal('auto')]) .union([z.literal(false), z.literal('full'), z.literal('auto')])
.optional(), .optional(),
turbopackGc: z
.union([
z.boolean(),
z.strictObject({
minProgressMs: z.number().min(0).finite().optional(),
rootTtlMs: z.number().min(0).finite().optional(),
}),
])
.optional(),
turbopackPluginRuntimeStrategy: z turbopackPluginRuntimeStrategy: z
.enum(['workerThreads', 'childProcesses', 'forceWorkerThreads']) .enum(['workerThreads', 'childProcesses', 'forceWorkerThreads'])
.optional(), .optional(),
......
...@@ -14,7 +14,7 @@ import type { SupportedTestRunners } from '../cli/next-test' ...@@ -14,7 +14,7 @@ import type { SupportedTestRunners } from '../cli/next-test'
import { INFINITE_CACHE } from '../lib/constants' import { INFINITE_CACHE } from '../lib/constants'
import { isStableBuild } from '../shared/lib/errors/canary-only-config-error' import { isStableBuild } from '../shared/lib/errors/canary-only-config-error'
import type { FallbackRouteParam } from '../build/static-paths/types' import type { FallbackRouteParam } from '../build/static-paths/types'
import type { MemoryEvictionMode } from '../build/swc/types' import type { MemoryEvictionMode, TurbopackGcOptions } from '../build/swc/types'
import type { CacheLife } from './use-cache/cache-life' import type { CacheLife } from './use-cache/cache-life'
/** /**
...@@ -79,6 +79,9 @@ export type NextConfigComplete = Required< ...@@ -79,6 +79,9 @@ export type NextConfigComplete = Required<
instantInsights: { validationLevel: ValidationLevel } instantInsights: { validationLevel: ValidationLevel }
// Normalized by finalized config with a default and the expected type // Normalized by finalized config with a default and the expected type
turbopackMemoryEvictionMode: MemoryEvictionMode turbopackMemoryEvictionMode: MemoryEvictionMode
// Normalized by config.ts: `false`/unset becomes `undefined` (GC off),
// `true` becomes `{}` (GC on with default timings)
turbopackGcOptions: TurbopackGcOptions | undefined
} }
// The root directory of the distDir. In development mode, this is the parent directory of `distDir` // The root directory of the distDir. In development mode, this is the parent directory of `distDir`
// since development builds use `{distDir}/dev`. This is used to ensure that the bundler doesn't // since development builds use `{distDir}/dev`. This is used to ensure that the bundler doesn't
...@@ -779,6 +782,21 @@ export interface ExperimentalConfig { ...@@ -779,6 +782,21 @@ export interface ExperimentalConfig {
*/ */
turbopackMemoryEviction?: false | 'full' | 'auto' turbopackMemoryEviction?: false | 'full' | 'auto'
/**
* Enables Turbopack's garbage collector, which deletes unreachable
* tasks from the persistent cache and from memory.
*
*
* - `false` (default): never collect.
* - `true`: collect
* - An object: collect, overriding individual timings.
* - `minProgressMs`: how long a GC pass runs before it will honour an
* interrupt. Defaults to 100ms.
* - `rootTtlMs`: how long a GC root may go un-anchored before it ages out.
* Defaults to 3 days.
*/
turbopackGc?: boolean | { minProgressMs?: number; rootTtlMs?: number }
/** /**
* Selects the backend used by Turbopack for Node.js evaluation, e.g. webpack * Selects the backend used by Turbopack for Node.js evaluation, e.g. webpack
* loaders, Babel, or PostCSS. * loaders, Babel, or PostCSS.
......
...@@ -58,7 +58,7 @@ import type { NextAdapter } from '../build/adapter/build-complete' ...@@ -58,7 +58,7 @@ import type { NextAdapter } from '../build/adapter/build-complete'
import { HardDeprecatedConfigError } from '../shared/lib/errors/hard-deprecated-config-error' import { HardDeprecatedConfigError } from '../shared/lib/errors/hard-deprecated-config-error'
import { NextInstanceErrorState } from './mcp/tools/next-instance-error-state' import { NextInstanceErrorState } from './mcp/tools/next-instance-error-state'
import { Bundler } from '../lib/bundler' import { Bundler } from '../lib/bundler'
import type { MemoryEvictionMode } from '../build/swc/types' import type { MemoryEvictionMode, TurbopackGcOptions } from '../build/swc/types'
import { hrtimeBigIntDurationToString } from '../build/duration-to-string' import { hrtimeBigIntDurationToString } from '../build/duration-to-string'
export { normalizeConfig } from './config-shared' export { normalizeConfig } from './config-shared'
...@@ -507,6 +507,23 @@ function assignDefaultsAndValidate( ...@@ -507,6 +507,23 @@ function assignDefaultsAndValidate(
;(result as NextConfigComplete).experimental.turbopackMemoryEvictionMode = ;(result as NextConfigComplete).experimental.turbopackMemoryEvictionMode =
turbopackMemoryEvictionMode as MemoryEvictionMode turbopackMemoryEvictionMode as MemoryEvictionMode
// Normalize the user-facing `turbopackGc` (`boolean | { minProgressMs?,
// rootTtlMs? } | undefined`) into the object napi expects
const turbopackGc = result.experimental.turbopackGc
let turbopackGcOptions: TurbopackGcOptions | undefined
if (turbopackGc === true) {
turbopackGcOptions = {}
} else if (typeof turbopackGc === 'object' && turbopackGc !== null) {
turbopackGcOptions = {
minProgressMs: turbopackGc.minProgressMs,
rootTtlMs: turbopackGc.rootTtlMs,
}
} else {
turbopackGcOptions = undefined
}
;(result as NextConfigComplete).experimental.turbopackGcOptions =
turbopackGcOptions
// Normalize experimental.browserDebugInfoInTerminal to logging.browserToTerminal // Normalize experimental.browserDebugInfoInTerminal to logging.browserToTerminal
if ( if (
result.logging !== false && result.logging !== false &&
......
...@@ -535,6 +535,7 @@ export async function createHotReloaderTurbopack( ...@@ -535,6 +535,7 @@ export async function createHotReloaderTurbopack(
{ {
turbopackMemoryEviction: turbopackMemoryEviction:
opts.nextConfig.experimental.turbopackMemoryEvictionMode, opts.nextConfig.experimental.turbopackMemoryEvictionMode,
gc: opts.nextConfig.experimental.turbopackGcOptions,
isShortSession: false, isShortSession: false,
} }
) )
......
...@@ -377,8 +377,7 @@ impl TurboTasksBackend { ...@@ -377,8 +377,7 @@ impl TurboTasksBackend {
// A pass sets `deleted` flags, and the persist path only knows how to tombstone those when // A pass sets `deleted` flags, and the persist path only knows how to tombstone those when
// GC is enabled. Running a pass on a GC-disabled backend would leave soft-deleted tasks // GC is enabled. Running a pass on a GC-disabled backend would leave soft-deleted tasks
// that persistence refuses to handle, so require the backend to be configured for GC // that persistence refuses to handle, so require the backend to be configured for GC
// (`BackendOptions::gc` or `TURBO_ENGINE_GC`) rather than silently diverging from // (`BackendOptions::gc`) rather than silently diverging from production.
// production.
assert!( assert!(
self.gc_enabled, self.gc_enabled,
"gc_for_testing requires a GC-enabled backend: set `BackendOptions::gc = Some(true)`" "gc_for_testing requires a GC-enabled backend: set `BackendOptions::gc = Some(true)`"
......
...@@ -157,16 +157,17 @@ pub struct BackendOptions { ...@@ -157,16 +157,17 @@ pub struct BackendOptions {
/// This reclaims memory by clearing persisted data that can be re-loaded from disk on demand. /// This reclaims memory by clearing persisted data that can be re-loaded from disk on demand.
pub eviction_mode: EvictionMode, pub eviction_mode: EvictionMode,
/// Overrides whether the reference-counting GC runs for this backend. `None` (default) derives /// Whether the reference-counting GC runs for this backend. `None` (default) leaves it off.
/// it from the `TURBO_ENGINE_GC` env var; ///
/// In Next.js this is driven by the `experimental.turbopackGc` config option.
pub gc: Option<bool>, pub gc: Option<bool>,
/// Overrides how long a GC root may go un-anchored before it ages out. `None` (default) /// How long a GC root may go un-anchored before it ages out. `None` (default) uses
/// derives it from the `TURBO_ENGINE_GC_ROOT_TTL_MS` env var, falling back to
/// [`DEFAULT_GC_ROOT_TTL`]. /// [`DEFAULT_GC_ROOT_TTL`].
pub gc_root_ttl: Option<Duration>, pub gc_root_ttl: Option<Duration>,
/// Overrides how long a GC pass runs before it will honour an interrupt. /// How long a GC pass runs before it will honour an interrupt. `None` (default) uses
/// [`GC_MIN_PROGRESS`].
pub gc_min_progress: Option<Duration>, pub gc_min_progress: Option<Duration>,
} }
...@@ -311,58 +312,25 @@ impl TurboTasksBackend { ...@@ -311,58 +312,25 @@ impl TurboTasksBackend {
options.active_tracking = false; options.active_tracking = false;
} }
let small_preallocation = options.small_preallocation; let small_preallocation = options.small_preallocation;
let gc_root_ttl = options.gc_root_ttl.unwrap_or(DEFAULT_GC_ROOT_TTL);
let gc_min_progress = options.gc_min_progress.unwrap_or(GC_MIN_PROGRESS);
let next_task_id = backing_storage let next_task_id = backing_storage
.next_free_task_id() .next_free_task_id()
.expect("Failed to get task id"); .expect("Failed to get task id");
let mut gc_enabled = options.gc.unwrap_or_else(|| { let mut gc_enabled = options.gc.unwrap_or(false);
std::env::var_os("TURBO_ENGINE_GC")
.is_some_and(|v| matches!(v.to_str(), Some("1" | "true" | "yes")))
});
if gc_enabled if gc_enabled
&& options.storage_mode == Some(StorageMode::ReadWrite) && options.storage_mode == Some(StorageMode::ReadWrite)
&& options.eviction_mode == EvictionMode::Off && options.eviction_mode == EvictionMode::Off
{ {
eprintln!( eprintln!(
"warning: GC is enabled but eviction is disabled on a ReadWrite backend; GC would \ "warning: GC is enabled but eviction is disabled; GC would leave collected tasks \
leave collected tasks resident forever. Forcing GC off. Enable eviction \ resident forever. Forcing GC off. Enable eviction ('auto'/'full') to use GC in \
('auto'/'full') to use GC in this mode." this mode."
); );
gc_enabled = false; gc_enabled = false;
} }
let gc_min_progress = options.gc_min_progress.unwrap_or_else(|| {
match std::env::var("TURBO_ENGINE_GC_MIN_PROGRESS_MS") {
Ok(v) => match v.parse::<u64>() {
Ok(ms) => Duration::from_millis(ms),
Err(e) => {
eprintln!(
"warning: TURBO_ENGINE_GC_MIN_PROGRESS_MS set but is not parsable: \
{e}. Using the default instead."
);
GC_MIN_PROGRESS
}
},
Err(_) => GC_MIN_PROGRESS,
}
});
let gc_root_ttl = options.gc_root_ttl.unwrap_or_else(|| {
match std::env::var("TURBO_ENGINE_GC_ROOT_TTL_MS") {
Ok(v) => match v.parse::<u64>() {
Ok(ms) => Duration::from_millis(ms),
Err(e) => {
eprintln!(
"warning: TURBO_ENGINE_GC_ROOT_TTL_MS set but is not parsable: {e}. \
Using the default instead."
);
DEFAULT_GC_ROOT_TTL
}
},
Err(_) => DEFAULT_GC_ROOT_TTL,
}
});
Self { Self {
options, options,
gc_enabled, gc_enabled,
......
...@@ -26,10 +26,10 @@ fn open_tt_at(path: &Path, num_workers: usize) -> Arc<TurboTasks<TurboTasksBacke ...@@ -26,10 +26,10 @@ fn open_tt_at(path: &Path, num_workers: usize) -> Arc<TurboTasks<TurboTasksBacke
open_tt_at_with_gc(path, num_workers, Some(true), None, None) open_tt_at_with_gc(path, num_workers, Some(true), None, None)
} }
/// Like [`open_tt_at`], but forces the GC on or off for this backend instead of deriving it from /// Like [`open_tt_at`], but sets the GC on or off for this backend explicitly, and optionally
/// the `TURBO_ENGINE_GC` env var, which every test in the binary would share. /// pins the GC timings. A `None` timing leaves the backend default in place.
/// ///
/// A test that depends on the persisted GC roots map must force it on: the map is only written by /// A test that depends on the persisted GC roots map must turn it on: the map is only written by
/// the GC branch of `snapshot_and_persist`, so with GC off a session persists an empty root set and /// the GC branch of `snapshot_and_persist`, so with GC off a session persists an empty root set and
/// the cross-session behaviour under test silently never engages. /// the cross-session behaviour under test silently never engages.
fn open_tt_at_with_gc( fn open_tt_at_with_gc(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment