Unverified Commit 5f2ef546 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Turbopack: add opt-in build snapshot and eviction

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 79107ebd
......@@ -53,9 +53,9 @@ use tracing::Instrument;
use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Effects, FxIndexSet, GcRoot, OperationValue, OperationVc, PrettyPrintError, ReadRef,
ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasksApi, TurboTasksCallApi, UpdateInfo,
Vc, mark_top_level_task,
Completion, Effects, FxIndexSet, GcRoot, OperationValue, OperationVc, PrettyPrintError,
ReadRef, ResolvedVc, TransientInstance, TryJoinIterExt, TurboTasksApi, TurboTasksCallApi,
UpdateInfo, Vc, mark_top_level_task,
message_queue::{CompilationEvent, Severity, TraceEvent},
read_strongly_consistent_and_apply_effects, take_effects, turbo_tasks,
unmark_top_level_task_may_leak_eventually_consistent_state,
......@@ -1495,6 +1495,7 @@ pub async fn project_write_all_entrypoints_to_disk(
EntrypointsWritePhase::All
};
let graph_gc_root = snapshot_and_evict_before_output_phase(ctx, container, first_phase).await;
let (mut entrypoints, mut issues) = tt
.run(async move {
let entrypoints_with_issues_op = get_all_written_entrypoints_with_issues_operation(
......@@ -1521,6 +1522,7 @@ pub async fn project_write_all_entrypoints_to_disk(
})
.or_else(|e| ctx.throw_turbopack_internal_result(&e.into()))
.await?;
drop(graph_gc_root);
if has_deferred_entrypoints {
ctx.on_before_deferred_entries().await?;
......@@ -1559,6 +1561,9 @@ pub async fn project_write_all_entrypoints_to_disk(
.await?;
}
let graph_gc_root =
snapshot_and_evict_before_output_phase(ctx, container, EntrypointsWritePhase::Deferred)
.await;
let (deferred_entrypoints, deferred_issues) = tt
.run(async move {
let entrypoints_with_issues_op = get_all_written_entrypoints_with_issues_operation(
......@@ -1585,6 +1590,7 @@ pub async fn project_write_all_entrypoints_to_disk(
})
.or_else(|e| ctx.throw_turbopack_internal_result(&e.into()))
.await?;
drop(graph_gc_root);
if deferred_entrypoints.is_some() {
entrypoints = deferred_entrypoints;
......@@ -1620,6 +1626,116 @@ pub async fn project_write_all_entrypoints_to_disk(
})
}
/// Whether `next build` should persist a snapshot and evict persisted tasks from memory once the
/// module graph is final, before chunking and code generation start. This trades build time for a
/// lower peak memory usage. Prototype: only enabled via
/// `NEXT_TURBOPACK_BUILD_SNAPSHOT_AND_EVICT=1`.
fn is_build_snapshot_and_evict_enabled() -> bool {
static ENABLED: LazyLock<bool> = LazyLock::new(|| {
std::env::var("NEXT_TURBOPACK_BUILD_SNAPSHOT_AND_EVICT").is_ok_and(|v| v == "1")
});
*ENABLED
}
#[turbo_tasks::function(operation, root)]
async fn whole_app_module_graphs_operation(
container: ResolvedVc<ProjectContainer>,
) -> Result<Vc<Completion>> {
container
.project()
.whole_app_module_graphs()
.as_side_effect()
.await?;
Ok(Completion::new())
}
#[turbo_tasks::function(operation, root)]
async fn whole_app_module_graphs_with_issues_operation(
container: ResolvedVc<ProjectContainer>,
) -> Result<Vc<OperationResult>> {
let filter = container.project().issue_filter().await?;
let (_, issues, effects) = strongly_consistent_catch_collectables(
whole_app_module_graphs_operation(container),
&filter,
)
.await?;
Ok(OperationResult { issues, effects }.cell())
}
/// If enabled (see [`is_build_snapshot_and_evict_enabled`]), computes the whole app module graph
/// for the given write phase, applies its effects, then persists a snapshot and evicts persisted
/// tasks from memory before the output phase starts chunking and code generation.
///
/// The module graph's issues are collected here, but not returned: the following output phase
/// reads the same tasks and reports them, so returning them here would report them twice. If
/// computing the graph or applying its effects fails, the snapshot is skipped and the output phase
/// reports the failure. Failing to persist is reported as a warning and never fails the build.
///
/// Returns a [`GcRoot`] that keeps the module graph from being garbage collected. Hold it until
/// the output phase has completed.
async fn snapshot_and_evict_before_output_phase(
ctx: &NextTurbopackContext,
container: ResolvedVc<ProjectContainer>,
write_phase: EntrypointsWritePhase,
) -> Option<GcRoot<OperationResult>> {
if !is_build_snapshot_and_evict_enabled() {
return None;
}
let tt = ctx.turbo_tasks().clone();
let start = Instant::now();
let module_graphs_op = match tt
.run(async move {
let op = whole_app_module_graphs_with_issues_operation(container);
read_strongly_consistent_and_apply_effects(op, |v| &v.effects).await?;
Ok(op)
})
.await
{
Ok(op) => op,
Err(err) => {
eprintln!(
"Turbopack snapshot and evict ({write_phase:?}): skipped, computing the module \
graph failed: {err}"
);
return None;
}
};
let gc_root = GcRoot::pin(tt.clone(), module_graphs_op);
let graph_duration = start.elapsed();
let start = Instant::now();
let result = {
let tt = tt.clone();
tokio::task::spawn_blocking(move || tt.backend().snapshot_and_evict(&tt)).await
};
let duration = start.elapsed();
match result {
Ok(Ok(Some(counts))) => eprintln!(
"Turbopack snapshot and evict ({write_phase:?}): evicted {} tasks fully, {} data and \
meta, {} data only, {} meta only, {} task cache keys in {duration:.2?} (module graph \
{graph_duration:.2?})",
counts.full,
counts.data_and_meta,
counts.data_only,
counts.meta_only,
counts.key_evictions,
),
Ok(Ok(None)) => eprintln!(
"Turbopack snapshot and evict ({write_phase:?}): not attempted, persistence is \
disabled"
),
Ok(Err(err)) => eprintln!(
"WARNING: Turbopack snapshot and evict ({write_phase:?}) failed, continuing without \
it: {err:#}"
),
Err(err) => eprintln!(
"WARNING: Turbopack snapshot and evict ({write_phase:?}) panicked, continuing without \
it: {err}"
),
}
Some(gc_root)
}
#[turbo_tasks::function(operation, root)]
async fn get_all_written_entrypoints_with_issues_operation(
container: ResolvedVc<ProjectContainer>,
......
......@@ -194,6 +194,9 @@ pub enum TurboTasksBackendJob {
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SnapshotReason {
Test,
/// An explicit [`TurboTasksBackend::snapshot_and_evict`] call, e.g. at a phase boundary of a
/// short session.
SnapshotAndEvict,
Stop,
InitialSnapshotTimeout,
RegularSnapshotInterval,
......@@ -204,6 +207,7 @@ impl SnapshotReason {
fn as_str(self) -> &'static str {
match self {
SnapshotReason::Test => "test",
SnapshotReason::SnapshotAndEvict => "snapshot and evict",
SnapshotReason::Stop => "stop",
SnapshotReason::InitialSnapshotTimeout => "initial snapshot timeout",
SnapshotReason::RegularSnapshotInterval => "regular snapshot interval",
......@@ -258,8 +262,11 @@ pub struct TurboTasksBackend {
snapshot_coord: SnapshotCoordinator,
/// Serializes calls to `snapshot_and_persist` (and `gc_for_testing`). The coordinator's
/// `begin_exclusion` asserts that exclusive phases don't overlap; this mutex
/// enforces that contract for our callers (background loop and
/// `stop_and_wait`).
/// enforces that contract for our callers (background loop, `stop_and_wait` and
/// `snapshot_and_evict`).
///
/// Eviction sweeps (`Storage::evict_after_snapshot`) also hold it, so that they never run
/// while another caller's snapshot is in snapshot mode.
///
/// Also holds the [`LastPersistenceStatus`], which disables persisting after a failure.
snapshot_in_progress: Mutex<LastPersistenceStatus>,
......@@ -440,10 +447,66 @@ impl TurboTasksBackend {
)
}
/// Persists a snapshot of all modified tasks and then evicts every evictable task from memory,
/// so that it is restored from the backing storage on demand. This lets short sessions (which
/// use [`StorageMode::ReadWriteOnShutdown`] and never run the background snapshot loop)
/// reclaim memory at a point of their choosing, e.g. after an expensive phase whose results
/// are mostly no longer needed in memory.
///
/// The snapshot waits for all in-flight backend operations to settle before capturing the
/// task graph, using the same commit protocol as every other snapshot. Task execution may
/// continue concurrently; only backend operations are excluded during the capture. Tasks that
/// are executing (or otherwise not evictable) stay in memory. After evicting, the allocator is
/// asked to return freed memory to the OS.
///
/// Returns:
/// - `Ok(Some(counts))` when the snapshot succeeded and the eviction sweep ran (the counts may
/// be zero).
/// - `Ok(None)` when nothing was attempted: persistence is disabled or read-only, or an earlier
/// persist failed (which disables persisting for the rest of the session).
/// - `Err(..)` when persisting failed during this call. Nothing is evicted, and persisting is
/// disabled for the rest of the session.
///
/// This blocks the calling thread. Call it from outside task execution and backend operations
/// (e.g. between top-level `TurboTasks::run` calls), and not after the backend started
/// stopping. Calling it from within a backend operation would wait for that operation.
pub fn snapshot_and_evict(
&self,
turbo_tasks: &TurboTasks<TurboTasksBackend>,
) -> Result<Option<EvictionCounts>> {
if !self.should_persist() {
return Ok(None);
}
let span = tracing::info_span!("snapshot and evict");
// Hold the snapshot lock across the eviction sweep too, so that no other snapshot can
// enter snapshot mode between this snapshot and its sweep.
let mut status = self.snapshot_in_progress.lock();
if *status == LastPersistenceStatus::Failed {
return Ok(None);
}
// Same failure handling as `snapshot_and_persist`: mark the cycle failed up front and only
// reset it on success, as encoding failures panic.
*status = LastPersistenceStatus::Failed;
let Some(_) = self.snapshot_and_persist_locked(
span.id(),
SnapshotReason::SnapshotAndEvict,
turbo_tasks,
)?
else {
unreachable!("snapshot and evict waits for operations, so it is never skipped");
};
*status = LastPersistenceStatus::Ok;
let eviction_counts = self.storage.evict_after_snapshot(span.id());
drop(status);
TurboMalloc::collect(true);
Ok(Some(eviction_counts))
}
/// Perform a snapshot and then evict all evictable tasks from memory.
///
/// This is exposed for integration tests that need to verify the snapshot → evict → restore
/// cycle works correctly.
/// cycle works correctly. Unlike [`Self::snapshot_and_evict`], GC passes run by this snapshot
/// are interruptible and the outcome includes test-only details.
#[doc(hidden)]
pub fn snapshot_and_evict_for_testing(
&self,
......@@ -465,7 +528,11 @@ impl TurboTasksBackend {
return TestSnapshotOutcome::default();
}
};
let eviction_counts = self.storage.evict_after_snapshot(None);
let eviction_counts = {
// Don't race with another caller's snapshot, see `snapshot_in_progress`.
let _guard = self.snapshot_in_progress.lock();
self.storage.evict_after_snapshot(None)
};
TestSnapshotOutcome {
had_new_data,
eviction_counts,
......@@ -3249,6 +3316,9 @@ impl TurboTasksBackend {
// when enabled we should expect it to reclaim substantial
// memory so racing with execution is as likely to save time as
// cost it.
// Don't race with another caller's snapshot, see
// `snapshot_in_progress`.
let _guard = self.snapshot_in_progress.lock();
self.storage.evict_after_snapshot(background_span.id());
true
} else {
......@@ -4127,4 +4197,51 @@ mod persist_failure_tests {
);
assert!(!task.flags.any_snapshot_pending());
}
/// `snapshot_and_evict` must wait for in-flight operations (never skip or capture a partial
/// graph), run GC to completion and must not drain the map.
#[test]
fn snapshot_and_evict_reason_waits_for_operations() {
let reason = SnapshotReason::SnapshotAndEvict;
assert!(reason.waits_for_operations());
assert!(!reason.gc_is_interruptible());
assert!(!reason.drain_entries());
}
/// A failed persist during `snapshot_and_evict` is reported as an error and skips eviction;
/// later calls are not attempted.
#[tokio::test(flavor = "multi_thread")]
async fn snapshot_and_evict_reports_persist_failure() {
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
storage_mode: Some(StorageMode::ReadWrite),
num_workers: Some(1),
small_preallocation: true,
..Default::default()
},
noop_backing_storage(),
));
let backend = tt.backend();
let task_id = TaskId::new(1).unwrap();
{
let mut task = backend.storage.access_mut(task_id);
let _ = task.track_modification(SpecificTaskDataCategory::Data, "test");
}
assert!(backend.snapshot_and_evict(&tt).is_err());
assert_eq!(
*backend.snapshot_in_progress.lock(),
LastPersistenceStatus::Failed
);
assert!(
backend.storage.with_task(task_id, |_| ()).is_some(),
"a failed snapshot must not evict"
);
assert!(
backend
.snapshot_and_evict(&tt)
.expect("a disabled persist is skipped, not an error")
.is_none()
);
}
}
......@@ -18,8 +18,8 @@ use turbo_persistence::{CompactConfig, TurboPersistence};
use crate::database::turbo::{self, TurboKeyValueDatabase};
pub use crate::{
backend::{
BackendOptions, EvictionMode, GcPassResult, GcStats, StorageMode, TestSnapshotOutcome,
TtlCounter, TurboTasksBackend,
BackendOptions, EvictionCounts, EvictionMode, GcPassResult, GcStats, StorageMode,
TestSnapshotOutcome, TtlCounter, TurboTasksBackend,
},
database::{
db_invalidation,
......
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this
//! Tests for [`TurboTasksBackend::snapshot_and_evict`], the explicit snapshot + eviction entry
//! point used by short sessions (e.g. production builds) at phase boundaries.
mod util;
use std::sync::atomic::{AtomicU32, Ordering};
use anyhow::Result;
use turbo_tasks::{GcRoot, OperationVc, ResolvedVc, State, TurboTasks, Vc};
use turbo_tasks_backend::{
BackendOptions, EvictionCounts, StorageMode, TurboTasksBackend, noop_backing_storage,
};
use crate::util::create_tt;
fn total_evicted(counts: &EvictionCounts) -> usize {
counts.full + counts.data_and_meta + counts.data_only + counts.meta_only
}
/// Evicted tasks are restored from disk and produce the same results.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_and_evict_preserves_results() {
let (tt, _dir) = create_tt("snapshot_and_evict_preserves_results");
let (state_op, first) = tt
.run_once(async move {
let state_op = create_state(10);
let state_vc = state_op.resolve().strongly_consistent().await?;
let output = chain(state_vc).read_strongly_consistent().await?;
Ok((state_op, (output.value, output.random)))
})
.await
.unwrap();
assert_eq!(first.0, 43);
let counts = tt
.backend()
.snapshot_and_evict(&tt)
.expect("persisting must succeed")
.expect("persistence is enabled");
assert!(
total_evicted(&counts) > 0,
"expected some tasks to be evicted, got {counts:?}"
);
let second = tt
.run_once(async move {
let state_vc = state_op.resolve().strongly_consistent().await?;
let output = chain(state_vc).read_strongly_consistent().await?;
Ok((output.value, output.random))
})
.await
.unwrap();
// Same value and the same `random`: the output was restored, not recomputed.
assert_eq!(second, first);
tt.stop_and_wait().await;
}
/// Repeated calls with invalidations in between keep results correct.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_and_evict_repeated_with_invalidation() {
let (tt, _dir) = create_tt("snapshot_and_evict_repeated_with_invalidation");
let state_op = tt
.run_once(async move {
let state_op = create_state(10);
let state_vc = state_op.resolve().strongly_consistent().await?;
assert_eq!(chain(state_vc).read_strongly_consistent().await?.value, 43);
Ok(state_op)
})
.await
.unwrap();
for (input, expected) in [(20, 73), (0, 13), (5, 28)] {
let counts = tt
.backend()
.snapshot_and_evict(&tt)
.expect("persisting must succeed")
.expect("persistence is enabled");
println!("evicted before setting {input}: {counts:?}");
let value = tt
.run_once(async move {
let state_vc = state_op.resolve().strongly_consistent().await?;
state_op.read_strongly_consistent().await?.set(input);
Ok(chain(state_vc).read_strongly_consistent().await?.value)
})
.await
.unwrap();
assert_eq!(value, expected);
}
tt.stop_and_wait().await;
}
/// Without persistence (disabled or read-only) nothing is attempted and nothing is lost.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_and_evict_without_persistence() {
for (storage_mode, name) in [
(None, "disabled"),
(Some(StorageMode::ReadOnly), "read-only"),
] {
let tt = TurboTasks::new(TurboTasksBackend::new(
BackendOptions {
storage_mode,
small_preallocation: true,
..Default::default()
},
noop_backing_storage(),
));
let state_op = tt
.run_once(async move {
let state_op = create_state(10);
let state_vc = state_op.resolve().strongly_consistent().await?;
assert_eq!(chain(state_vc).read_strongly_consistent().await?.value, 43);
Ok(state_op)
})
.await
.unwrap();
assert!(
tt.backend()
.snapshot_and_evict(&tt)
.expect("nothing is attempted, so nothing can fail")
.is_none(),
"{name}"
);
let value = tt
.run_once(async move {
let state_vc = state_op.resolve().strongly_consistent().await?;
Ok(chain(state_vc).read_strongly_consistent().await?.value)
})
.await
.unwrap();
assert_eq!(value, 43);
tt.stop_and_wait().await;
}
}
/// Concurrent callers are serialized (each snapshot is followed by its own sweep before another
/// snapshot starts), and results stay correct while tasks execute.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn snapshot_and_evict_concurrent_callers() {
let (tt, _dir) = create_tt("snapshot_and_evict_concurrent_callers");
let state_op = tt
.run_once(async move {
let state_op = create_state(10);
let state_vc = state_op.resolve().strongly_consistent().await?;
assert_eq!(chain(state_vc).read_strongly_consistent().await?.value, 43);
Ok(state_op)
})
.await
.unwrap();
let callers = (0..4)
.map(|_| {
let tt = tt.clone();
tokio::task::spawn_blocking(move || {
for _ in 0..5 {
tt.backend()
.snapshot_and_evict(&tt)
.expect("persisting must succeed")
.expect("persistence is enabled");
}
})
})
.collect::<Vec<_>>();
for (input, expected) in [(20, 73), (0, 13), (5, 28)] {
let value = tt
.run_once(async move {
let state_vc = state_op.resolve().strongly_consistent().await?;
state_op.read_strongly_consistent().await?.set(input);
Ok(chain(state_vc).read_strongly_consistent().await?.value)
})
.await
.unwrap();
assert_eq!(value, expected);
}
for caller in callers {
caller.await.unwrap();
}
tt.stop_and_wait().await;
}
static PINNED_ROOT_EXECUTIONS: AtomicU32 = AtomicU32::new(0);
#[turbo_tasks::function]
fn pinned_leaf(n: u32) -> Vc<u32> {
Vc::cell(n * 2)
}
#[turbo_tasks::function(operation, root)]
async fn pinned_root(n: u32) -> Result<Vc<u32>> {
PINNED_ROOT_EXECUTIONS.fetch_add(1, Ordering::Relaxed);
Ok(Vc::cell(*pinned_leaf(n).await? + 1))
}
/// With GC enabled, an operation pinned across repeated calls (each running a GC pass) is restored
/// from disk afterwards instead of being collected and re-executed.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn snapshot_and_evict_keeps_pinned_gc_root() {
// `create_tt` enables GC.
let (tt, _dir) = create_tt("snapshot_and_evict_keeps_pinned_gc_root");
let op: OperationVc<u32> = tt
.run_once(async move {
let op = pinned_root(21);
assert_eq!(*op.read_strongly_consistent().await?, 43);
Ok(op)
})
.await
.unwrap();
assert_eq!(PINNED_ROOT_EXECUTIONS.load(Ordering::Relaxed), 1);
let pin = GcRoot::pin(tt.clone(), op);
for _ in 0..3 {
let counts = tt
.backend()
.snapshot_and_evict(&tt)
.expect("persisting must succeed")
.expect("persistence is enabled");
println!("evicted: {counts:?}");
}
let value = tt
.run_once(async move { Ok(*op.read_strongly_consistent().await?) })
.await
.unwrap();
assert_eq!(value, 43);
assert_eq!(
PINNED_ROOT_EXECUTIONS.load(Ordering::Relaxed),
1,
"the pinned root must be restored, not collected and re-executed"
);
drop(pin);
tt.stop_and_wait().await;
}
#[turbo_tasks::value(transparent)]
struct Step(State<u32>);
#[turbo_tasks::function(operation, root)]
fn create_state(initial: u32) -> Vc<Step> {
Step(State::new(initial)).cell()
}
#[turbo_tasks::value]
struct Output {
value: u32,
random: u32,
}
#[turbo_tasks::function]
async fn add_one(input: ResolvedVc<Step>) -> Result<Vc<u32>> {
Ok(Vc::cell(*input.await?.get() + 1))
}
#[turbo_tasks::function]
async fn times_three(input: Vc<u32>) -> Result<Vc<u32>> {
Ok(Vc::cell(*input.await? * 3))
}
/// `(input + 1) * 3 + 10`, with a random tag that only changes when this task re-executes.
#[turbo_tasks::function(operation, root)]
async fn chain(input: ResolvedVc<Step>) -> Result<Vc<Output>> {
let value = *times_three(add_one(*input)).await? + 10;
Ok(Output {
value,
random: rand::random(),
}
.cell())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment