Unverified Commit 24258f0c authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Introduce config options for GC

Replace the TURBO_ENGINE_GC, TURBO_ENGINE_GC_MIN_PROGRESS_MS and
TURBO_ENGINE_GC_ROOT_TTL_MS env vars with an experimental.turbopackGc
config option, plumbed to the backend over napi.

The env vars are removed outright rather than deprecated; GC never
launched, so there is nothing to stay compatible with.
parent 2994f6be
......@@ -90,7 +90,7 @@ use crate::{
endpoint::ExternalEndpoint,
turbopack_ctx::{
MemoryEvictionMode, NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject,
NextTurboTasks, NextTurbopackContext, create_turbo_tasks,
NapiTurbopackGcOptions, NextTurboTasks, NextTurbopackContext, create_turbo_tasks,
},
utils::{
DetachedVc, NapiIssue, NapiUsedFeature, SubscriptionTask, TurbopackResult, get_issues,
......@@ -290,6 +290,8 @@ pub struct NapiTurboEngineOptions {
pub skip_compaction: Option<bool>,
/// Turbopack memory eviction mode for the persistent cache.
pub turbopack_memory_eviction: MemoryEvictionMode,
/// Tuning for Turbopack's reference-counting GC. `None` disables the GC.
pub gc: Option<NapiTurbopackGcOptions>,
}
impl From<NapiWatchOptions> for WatchOptions {
......@@ -581,7 +583,7 @@ pub fn project_new<'env>(
env.spawn_future(
async move {
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let turbopack_memory_eviction = turbo_engine_options.turbopack_memory_eviction;
let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir),
&options.next_version,
......@@ -592,7 +594,8 @@ pub fn project_new<'env>(
is_short_session: turbo_engine_options.is_short_session.unwrap_or(false),
skip_compaction: turbo_engine_options.skip_compaction.unwrap_or(false),
},
turbopack_memory_eviction,
turbo_engine_options.turbopack_memory_eviction,
turbo_engine_options.gc,
)?;
let turbopack_ctx = NextTurbopackContext::new(turbo_tasks.clone(), napi_callbacks);
......
......@@ -6,7 +6,7 @@ use std::{
io::{self, BufRead, Write},
path::PathBuf,
sync::{Arc, LazyLock, Mutex},
time::Instant,
time::{Duration, Instant},
};
use anyhow::Result;
......@@ -285,6 +285,27 @@ impl From<MemoryEvictionMode> for EvictionMode {
}
}
#[napi(object)]
#[derive(Debug, Clone, Copy)]
pub struct NapiTurbopackGcOptions {
/// How long a GC pass runs before it will honour an interrupt, in milliseconds.
pub min_progress_ms: Option<f64>,
/// How long a GC root may go un-anchored before it ages out, in milliseconds.
pub root_ttl_ms: Option<f64>,
}
/// Converts a millisecond count that crossed the napi boundary into a `Duration`.
fn duration_from_millis_f64(ms: Option<f64>) -> Option<Duration> {
if let Some(ms) = ms
&& ms.is_finite()
&& ms >= 0.0
{
Some(Duration::from_secs_f64(ms / 1000.0))
} else {
None
}
}
pub fn create_turbo_tasks(
output_path: PathBuf,
next_version: &str,
......@@ -292,6 +313,7 @@ pub fn create_turbo_tasks(
dependency_tracking: bool,
storage_options: BackingStorageOptions,
turbopack_memory_eviction: MemoryEvictionMode,
gc: Option<NapiTurbopackGcOptions>,
) -> Result<NextTurboTasks> {
let BackingStorageOptions {
is_ci,
......@@ -306,6 +328,17 @@ pub fn create_turbo_tasks(
&version_info,
storage_options,
)?;
let (gc, gc_min_progress, gc_root_ttl) = match gc {
Some(NapiTurbopackGcOptions {
min_progress_ms,
root_ttl_ms,
}) => (
Some(true),
duration_from_millis_f64(min_progress_ms),
duration_from_millis_f64(root_ttl_ms),
),
None => (None, None, None),
};
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
storage_mode: Some(if env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
......@@ -318,6 +351,9 @@ pub fn create_turbo_tasks(
dependency_tracking,
num_workers: Some(tokio::runtime::Handle::current().metrics().num_workers()),
eviction_mode: EvictionMode::from(turbopack_memory_eviction),
gc,
gc_min_progress,
gc_root_ttl,
..Default::default()
},
backing_storage,
......@@ -331,6 +367,8 @@ pub fn create_turbo_tasks(
BackendOptions {
storage_mode: None,
dependency_tracking,
// GC is deliberately left at its default (off) here: without backing storage
// there is nothing to reclaim from disk, so `gc` would have no effect.
..Default::default()
},
noop_backing_storage(),
......
......@@ -528,6 +528,22 @@ export interface NapiTurboEngineOptions {
skipCompaction?: boolean
/** Turbopack memory eviction mode for the persistent cache. */
turbopackMemoryEviction: MemoryEvictionMode
/** Tuning for Turbopack's reference-counting GC. `None` disables the GC. */
gc?: NapiTurbopackGcOptions
}
/**
* Tuning for Turbopack's reference-counting GC, mirroring the
* `experimental.turbopackGc` config option.
*
* The presence of this object means GC is enabled; each field is optional and
* falls back to the backend default when omitted.
*/
export interface NapiTurbopackGcOptions {
/** How long a GC pass runs before it will honour an interrupt, in milliseconds. */
minProgressMs?: number
/** How long a GC root may go un-anchored before it ages out, in milliseconds. */
rootTtlMs?: number
}
export interface NapiUpdateInfo {
......
......@@ -13,6 +13,7 @@ import type {
TraceQueryOptions,
TraceQueryResult,
MemoryEvictionMode,
NapiTurbopackGcOptions,
ServerHmrVersion as NativeServerHmrVersion,
} from './generated-native'
......@@ -22,6 +23,8 @@ export type { NapiTurboEngineOptions as TurboEngineOptions }
export type { MemoryEvictionMode }
export type { NapiTurbopackGcOptions as TurbopackGcOptions }
export type Lockfile = { __napiType: 'Lockfile' }
export interface TurbopackProjectCallbacks {
......
......@@ -94,6 +94,7 @@ export async function turbopackAnalyze(
},
{
turbopackMemoryEviction: config.experimental.turbopackMemoryEvictionMode,
gc: config.experimental.turbopackGcOptions,
dependencyTracking: persistentCaching,
isCi: isCI,
isShortSession: true,
......
......@@ -117,6 +117,7 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{
const sharedTurboOptions = {
turbopackMemoryEviction: config.experimental.turbopackMemoryEvictionMode,
gc: config.experimental.turbopackGcOptions,
dependencyTracking: persistentCaching || hasDeferredEntries,
isCi: isCI,
isShortSession: true,
......
......@@ -374,6 +374,15 @@ export const experimentalSchema = {
turbopackMemoryEviction: z
.union([z.literal(false), z.literal('full'), z.literal('auto')])
.optional(),
turbopackGc: z
.union([
z.boolean(),
z.strictObject({
minProgressMs: z.number().min(0).finite().optional(),
rootTtlMs: z.number().min(0).finite().optional(),
}),
])
.optional(),
turbopackPluginRuntimeStrategy: z
.enum(['workerThreads', 'childProcesses'])
.optional(),
......
......@@ -14,7 +14,7 @@ import type { SupportedTestRunners } from '../cli/next-test'
import { INFINITE_CACHE } from '../lib/constants'
import { isStableBuild } from '../shared/lib/errors/canary-only-config-error'
import type { FallbackRouteParam } from '../build/static-paths/types'
import type { MemoryEvictionMode } from '../build/swc/types'
import type { MemoryEvictionMode, TurbopackGcOptions } from '../build/swc/types'
import type { CacheLife } from './use-cache/cache-life'
/**
......@@ -79,6 +79,9 @@ export type NextConfigComplete = Required<
instantInsights: { validationLevel: ValidationLevel }
// Normalized by finalized config with a default and the expected type
turbopackMemoryEvictionMode: MemoryEvictionMode
// Normalized by config.ts: `false`/unset becomes `undefined` (GC off),
// `true` becomes `{}` (GC on with default timings)
turbopackGcOptions: TurbopackGcOptions | undefined
}
// The root directory of the distDir. In development mode, this is the parent directory of `distDir`
// since development builds use `{distDir}/dev`. This is used to ensure that the bundler doesn't
......@@ -777,6 +780,26 @@ export interface ExperimentalConfig {
*/
turbopackMemoryEviction?: false | 'full' | 'auto'
/**
* Enables Turbopack's reference-counting garbage collector, which deletes
* unreachable tasks from the persistent cache and from memory.
*
* Only effective in dev sessions where
* `experimental.turbopackFileSystemCacheForDev` is enabled (which it is by
* default), and where `experimental.turbopackMemoryEviction` is not disabled:
* the GC reclaims memory by way of eviction, so with eviction off it would
* leave collected tasks resident forever.
*
* - `false` (default): never collect.
* - `true`: collect, using the default timings.
* - An object: collect, overriding individual timings.
* - `minProgressMs`: how long a GC pass runs before it will honour an
* interrupt. Defaults to 100ms.
* - `rootTtlMs`: how long a GC root may go un-anchored before it ages out.
* Defaults to 3 days.
*/
turbopackGc?: boolean | { minProgressMs?: number; rootTtlMs?: number }
/**
* Selects the backend used by Turbopack for Node.js evaluation, e.g. webpack
* loaders, Babel, or PostCSS.
......
......@@ -56,7 +56,7 @@ import type { NextAdapter } from '../build/adapter/build-complete'
import { HardDeprecatedConfigError } from '../shared/lib/errors/hard-deprecated-config-error'
import { NextInstanceErrorState } from './mcp/tools/next-instance-error-state'
import { Bundler } from '../lib/bundler'
import type { MemoryEvictionMode } from '../build/swc/types'
import type { MemoryEvictionMode, TurbopackGcOptions } from '../build/swc/types'
import { hrtimeBigIntDurationToString } from '../build/duration-to-string'
export { normalizeConfig } from './config-shared'
......@@ -505,6 +505,23 @@ function assignDefaultsAndValidate(
;(result as NextConfigComplete).experimental.turbopackMemoryEvictionMode =
turbopackMemoryEvictionMode as MemoryEvictionMode
// Normalize the user-facing `turbopackGc` (`boolean | { minProgressMs?,
// rootTtlMs? } | undefined`) into the object napi expects
const turbopackGc = result.experimental.turbopackGc
let turbopackGcOptions: TurbopackGcOptions | undefined
if (turbopackGc === true) {
turbopackGcOptions = {}
} else if (typeof turbopackGc === 'object' && turbopackGc !== null) {
turbopackGcOptions = {
minProgressMs: turbopackGc.minProgressMs,
rootTtlMs: turbopackGc.rootTtlMs,
}
} else {
turbopackGcOptions = undefined
}
;(result as NextConfigComplete).experimental.turbopackGcOptions =
turbopackGcOptions
// Normalize experimental.browserDebugInfoInTerminal to logging.browserToTerminal
if (
result.logging !== false &&
......
......@@ -529,6 +529,7 @@ export async function createHotReloaderTurbopack(
{
turbopackMemoryEviction:
opts.nextConfig.experimental.turbopackMemoryEvictionMode,
gc: opts.nextConfig.experimental.turbopackGcOptions,
isShortSession: false,
}
)
......
......@@ -377,8 +377,7 @@ impl TurboTasksBackend {
// A pass sets `deleted` flags, and the persist path only knows how to tombstone those when
// GC is enabled. Running a pass on a GC-disabled backend would leave soft-deleted tasks
// that persistence refuses to handle, so require the backend to be configured for GC
// (`BackendOptions::gc` or `TURBO_ENGINE_GC`) rather than silently diverging from
// production.
// (`BackendOptions::gc`) rather than silently diverging from production.
assert!(
self.gc_enabled,
"gc_for_testing requires a GC-enabled backend: set `BackendOptions::gc = Some(true)`"
......
......@@ -157,16 +157,17 @@ pub struct BackendOptions {
/// This reclaims memory by clearing persisted data that can be re-loaded from disk on demand.
pub eviction_mode: EvictionMode,
/// Overrides whether the reference-counting GC runs for this backend. `None` (default) derives
/// it from the `TURBO_ENGINE_GC` env var;
/// Whether the reference-counting GC runs for this backend. `None` (default) leaves it off.
///
/// In Next.js this is driven by the `experimental.turbopackGc` config option.
pub gc: Option<bool>,
/// Overrides how long a GC root may go un-anchored before it ages out. `None` (default)
/// derives it from the `TURBO_ENGINE_GC_ROOT_TTL_MS` env var, falling back to
/// How long a GC root may go un-anchored before it ages out. `None` (default) uses
/// [`DEFAULT_GC_ROOT_TTL`].
pub gc_root_ttl: Option<Duration>,
/// Overrides how long a GC pass runs before it will honour an interrupt.
/// How long a GC pass runs before it will honour an interrupt. `None` (default) uses
/// [`GC_MIN_PROGRESS`].
pub gc_min_progress: Option<Duration>,
}
......@@ -311,58 +312,25 @@ impl TurboTasksBackend {
options.active_tracking = false;
}
let small_preallocation = options.small_preallocation;
let gc_root_ttl = options.gc_root_ttl.unwrap_or(DEFAULT_GC_ROOT_TTL);
let gc_min_progress = options.gc_min_progress.unwrap_or(GC_MIN_PROGRESS);
let next_task_id = backing_storage
.next_free_task_id()
.expect("Failed to get task id");
let mut gc_enabled = options.gc.unwrap_or_else(|| {
std::env::var_os("TURBO_ENGINE_GC")
.is_some_and(|v| matches!(v.to_str(), Some("1" | "true" | "yes")))
});
let mut gc_enabled = options.gc.unwrap_or(false);
if gc_enabled
&& options.storage_mode == Some(StorageMode::ReadWrite)
&& options.eviction_mode == EvictionMode::Off
{
eprintln!(
"warning: GC is enabled but eviction is disabled on a ReadWrite backend; GC would \
leave collected tasks resident forever. Forcing GC off. Enable eviction \
('auto'/'full') to use GC in this mode."
"warning: GC is enabled but eviction is disabled; GC would leave collected tasks \
resident forever. Forcing GC off. Enable eviction ('auto'/'full') to use GC in \
this mode."
);
gc_enabled = false;
}
let gc_min_progress = options.gc_min_progress.unwrap_or_else(|| {
match std::env::var("TURBO_ENGINE_GC_MIN_PROGRESS_MS") {
Ok(v) => match v.parse::<u64>() {
Ok(ms) => Duration::from_millis(ms),
Err(e) => {
eprintln!(
"warning: TURBO_ENGINE_GC_MIN_PROGRESS_MS set but is not parsable: \
{e}. Using the default instead."
);
GC_MIN_PROGRESS
}
},
Err(_) => GC_MIN_PROGRESS,
}
});
let gc_root_ttl = options.gc_root_ttl.unwrap_or_else(|| {
match std::env::var("TURBO_ENGINE_GC_ROOT_TTL_MS") {
Ok(v) => match v.parse::<u64>() {
Ok(ms) => Duration::from_millis(ms),
Err(e) => {
eprintln!(
"warning: TURBO_ENGINE_GC_ROOT_TTL_MS set but is not parsable: {e}. \
Using the default instead."
);
DEFAULT_GC_ROOT_TTL
}
},
Err(_) => DEFAULT_GC_ROOT_TTL,
}
});
Self {
options,
gc_enabled,
......
......@@ -26,10 +26,10 @@ fn open_tt_at(path: &Path, num_workers: usize) -> Arc<TurboTasks<TurboTasksBacke
open_tt_at_with_gc(path, num_workers, Some(true), None, None)
}
/// Like [`open_tt_at`], but forces the GC on or off for this backend instead of deriving it from
/// the `TURBO_ENGINE_GC` env var, which every test in the binary would share.
/// Like [`open_tt_at`], but sets the GC on or off for this backend explicitly, and optionally
/// pins the GC timings. A `None` timing leaves the backend default in place.
///
/// A test that depends on the persisted GC roots map must force it on: the map is only written by
/// A test that depends on the persisted GC roots map must turn it on: the map is only written by
/// the GC branch of `snapshot_and_persist`, so with GC off a session persists an empty root set and
/// the cross-session behaviour under test silently never engages.
fn open_tt_at_with_gc(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment