Unverified Commit 228df5fd authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[backport] Retain fewer stale cache versions and use a TTL, plus the mtime fallback (#97304)

Backports the following to `next-16-3`, as a pair:

- #96941 — "[turbopack] Retain fewer stale cache versions and use a TTL"
(`ae1e53a1`)
- #97268 — "Add the mtime fallback back" (`52a3410d`)

The second fixes a bug in the first: #96941 changes the `CURRENT` file
format from a bare
big-endian `u32` to a JSON object, and an existing cache directory in
the old format could no
longer be aged. #97268 adds the mtime fallback so an unparsable
`CURRENT` is aged by its mtime
instead of failing. They only make sense together, hence one branch.

Both cherry-picked with `-x`, no conflicts. Every touched source file is
byte-identical to canary.

### Verification on this branch

- `cargo check`, `cargo clippy --all-targets`, `cargo fmt --check` on
`turbo-persistence` and `turbo-tasks-backend` — clean
- `cargo test -p turbo-persistence -p turbo-tasks-backend` — all pass, 0
failures
- includes the `db_versioning` tests, notably
`test_unparsable_current_falls_back_to_mtime::case_1_old_u32_format`
(the old-format compatibility case #97268 restores)
- `cargo metadata --locked` — `Cargo.lock` consistent with the
manifests, no lockfile update needed
- `pnpm --filter=next types`, prettier, eslint — clean
- `pnpm build-all` — 18/18 tasks
- `test/production/next-server-nft` — 5/5 tests, 5/5 snapshots pass
(covers the traced `turbopack-cache-seed.js`)

### End-to-end verification against a real old-format cache

`create-next-app@16.3.0` + `next build` (with
`turbopackFileSystemCacheForBuild`) on released
`next@16.3.0` writes a `CURRENT` in the old pre-JSON format — a bare
big-endian `u32`
(`0000 0008`), 30M cache. This branch was then applied with
`pnpm pack-next -p <app> -- --release` (release binary confirmed to
contain
`TURBO_ENGINE_VERSION_TTL_DAYS` and the new `CURRENT` field names), and
two arms were run from
an identical starting state — same cache, same code, only the env var
differing:

| Arm | Build | Cache dirs after |
| --- | --- | --- |
| `TURBO_ENGINE_VERSION_TTL_DAYS=0` | exit 0 | old dir **deleted**, 1
remains |
| default TTL (3 days) | exit 0 | old dir **retained**, intact at 30M, 2
remain |

The retention arm is the meaningful control: without #97268's fallback,
`read_current_version`
would error on the bare-`u32` file, the age would become
`Duration::MAX`, and the directory would
be evicted in *both* arms. Retention under the default TTL is direct
evidence the mtime fallback
works.

Also confirmed:

- new `CURRENT` is written as JSON:
`{"max_sequence_number":8,"commit_time":"..."}`
- retention cap holds: seeded 4 version dirs → build left exactly 2
(current + one most-recently-used other)
- clean deletion, no `DELETION_PREFIX` leftovers in any run
- `TURBO_ENGINE_VERSION_TTL_DAYS=notanumber` warns (`expected a whole
number of days`) and falls back to the 3-day default rather than failing

<!-- NEXT_JS_LLM -->
parent 16eab3ac
...@@ -9992,6 +9992,8 @@ dependencies = [ ...@@ -9992,6 +9992,8 @@ dependencies = [
"rand 0.10.1", "rand 0.10.1",
"rayon", "rayon",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde",
"serde_json",
"smallvec", "smallvec",
"tempfile", "tempfile",
"thread_local", "thread_local",
...@@ -10096,6 +10098,7 @@ dependencies = [ ...@@ -10096,6 +10098,7 @@ dependencies = [
"hashbrown 0.14.5", "hashbrown 0.14.5",
"indexmap 2.13.0", "indexmap 2.13.0",
"indoc", "indoc",
"jiff",
"lzzzz", "lzzzz",
"parking_lot", "parking_lot",
"rand 0.10.1", "rand 0.10.1",
......
...@@ -76,10 +76,10 @@ function findSeedSource( ...@@ -76,10 +76,10 @@ function findSeedSource(
const currentWorktree = path.resolve(worktreeInfo.worktreeRoot) const currentWorktree = path.resolve(worktreeInfo.worktreeRoot)
// We are going to find the best candidate worktree // We are going to find the best candidate worktree
// based on the newest mtime of the CURRENT file in the cache directory. // based on the most recently written cache directory.
// We only look at our version // We only look at our version
let best: { versionDir: string; mtimeMs: number } | undefined let best: { versionDir: string; commitTimeMs: number } | undefined
for (const root of [ for (const root of [
worktreeInfo.mainRepoRoot, worktreeInfo.mainRepoRoot,
...listLinkedWorktreeRoots(worktreeInfo.mainRepoRoot), ...listLinkedWorktreeRoots(worktreeInfo.mainRepoRoot),
...@@ -93,21 +93,31 @@ function findSeedSource( ...@@ -93,21 +93,31 @@ function findSeedSource(
'turbopack', 'turbopack',
version version
) )
const mtimeMs = currentMtimeMs(versionDir) const commitTimeMs = currentCommitTimeMs(versionDir)
if (mtimeMs === undefined) continue if (commitTimeMs === undefined) continue
if (!best || mtimeMs > best.mtimeMs) { if (!best || commitTimeMs > best.commitTimeMs) {
best = { versionDir, mtimeMs } best = { versionDir, commitTimeMs }
} }
} }
return best?.versionDir return best?.versionDir
} }
function currentMtimeMs(versionDir: string): number | undefined { // When the cache in `versionDir` was last committed to, in epoch milliseconds, or undefined if
// there is no usable cache there. Read from the `commit_time` the persistence layer records in
// CURRENT.
function currentCommitTimeMs(versionDir: string): number | undefined {
let commitTime
try { try {
return fs.statSync(path.join(versionDir, 'CURRENT')).mtimeMs commitTime = JSON.parse(
fs.readFileSync(path.join(versionDir, 'CURRENT'), 'utf8')
).commit_time
} catch { } catch {
// missing, unreadable, or not valid JSON - treat it as not a seed candidate
return undefined return undefined
} }
if (typeof commitTime !== 'string') return undefined
const parsed = Date.parse(commitTime)
return Number.isNaN(parsed) ? undefined : parsed
} }
function dirHasEntries(dir: string): boolean { function dirHasEntries(dir: string): boolean {
......
...@@ -20,7 +20,7 @@ crc32fast = { workspace = true } ...@@ -20,7 +20,7 @@ crc32fast = { workspace = true }
dashmap = { workspace = true} dashmap = { workspace = true}
either = { workspace = true } either = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }
jiff = "0.2.10" jiff = { version = "0.2.10", features = ["serde"] }
lzzzz = { workspace = true } lzzzz = { workspace = true }
memmap2 = "0.9.5" memmap2 = "0.9.5"
nohash-hasher = { workspace = true } nohash-hasher = { workspace = true }
...@@ -31,6 +31,8 @@ postcard = { workspace = true, features = ["alloc", "use-std"] } ...@@ -31,6 +31,8 @@ postcard = { workspace = true, features = ["alloc", "use-std"] }
zerocopy = { version = "0.8", features = ["derive"] } zerocopy = { version = "0.8", features = ["derive"] }
quick_cache = { workspace = true } quick_cache = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true }
thread_local = { workspace = true } thread_local = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
......
...@@ -12,7 +12,7 @@ It supports having multiple key families, which are stored in separate files, bu ...@@ -12,7 +12,7 @@ It supports having multiple key families, which are stored in separate files, bu
## On disk format ## On disk format
There is a single `CURRENT` file which stores the latest committed sequence number. There is a single `CURRENT` file, a small JSON object holding the latest committed sequence number (`max_sequence_number`) and when that commit happened (`commit_time`). The commit time is stored in the file rather than taken from its mtime so that it survives the directory being copied or restored. External tools read this file, so its field names are a stable contract.
All other files have a sequence number as file name, e. g. `0000123.sst`. All files are immutable once their sequence number is <= the committed sequence number. But they might be deleted when they are superseded by other committed files. All other files have a sequence number as file name, e. g. `0000123.sst`. All files are immutable once their sequence number is <= the committed sequence number. But they might be deleted when they are superseded by other committed files.
......
...@@ -24,6 +24,7 @@ use turbo_persistence::{ ...@@ -24,6 +24,7 @@ use turbo_persistence::{
BLOCK_HEADER_SIZE, checksum_block, BLOCK_HEADER_SIZE, checksum_block,
meta_file::MetaFile, meta_file::MetaFile,
mmap_helper::advise_mmap_for_persistence, mmap_helper::advise_mmap_for_persistence,
read_current_version,
sst_filter::SstFilter, sst_filter::SstFilter,
static_sorted_file::{ static_sorted_file::{
BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH,
...@@ -211,9 +212,9 @@ fn format_bytes(bytes: u64) -> String { ...@@ -211,9 +212,9 @@ fn format_bytes(bytes: u64) -> String {
/// and apply SstFilter to skip superseded entries. /// and apply SstFilter to skip superseded entries.
fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> { fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
// Read the CURRENT sequence number — only files with seq <= current are valid. // Read the CURRENT sequence number — only files with seq <= current are valid.
let current: u32 = File::open(db_path.join("CURRENT"))? let current = read_current_version(db_path)?
.read_u32::<BE>() .context("CURRENT file is missing")?
.context("Failed to read CURRENT file")?; .max_sequence_number;
// Read .del files to find sequences that were deleted but not yet cleaned up. // Read .del files to find sequences that were deleted but not yet cleaned up.
let mut deleted_seqs: HashSet<u32> = HashSet::new(); let mut deleted_seqs: HashSet<u32> = HashSet::new();
......
...@@ -2,7 +2,7 @@ use std::{ ...@@ -2,7 +2,7 @@ use std::{
borrow::Cow, borrow::Cow,
collections::HashSet, collections::HashSet,
fmt::Display, fmt::Display,
io::{BufWriter, Write}, io::{BufWriter, ErrorKind, Write},
mem::take, mem::take,
ops::RangeInclusive, ops::RangeInclusive,
path::{Path, PathBuf}, path::{Path, PathBuf},
...@@ -20,6 +20,7 @@ use jiff::Timestamp; ...@@ -20,6 +20,7 @@ use jiff::Timestamp;
use memmap2::Mmap; use memmap2::Mmap;
use nohash_hasher::BuildNoHashHasher; use nohash_hasher::BuildNoHashHasher;
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
use serde::{Deserialize, Serialize};
use smallvec::SmallVec; use smallvec::SmallVec;
use tracing::span::EnteredSpan; use tracing::span::EnteredSpan;
...@@ -159,24 +160,76 @@ impl WriteOperationGuard<'_> { ...@@ -159,24 +160,76 @@ impl WriteOperationGuard<'_> {
} }
} }
/// The contents of the `CURRENT` file: which sequence number is committed, and when that commit
/// happened.
///
/// # Compatibility
///
/// Unlike other parts of the persistent database the `CURRENT` file is occasionally read by other
/// versions of turbopack, so we should be careful when updating this struct
///
/// - Never rename a field. This will break readers from other versions
/// - Never remove a field, unless it has always had `[serde(default)]`
/// - Never change the type of a field
/// - New fields should be `#[serde(default)]` and semantically optional to readers from other
/// versions
/// - Never add `#[serde(deny_unknown_fields)]`.
///
/// Field names are also parsed outside this crate (next.js reads `CURRENT` directly, in
/// `turbopack-cache-seed.ts`), so a rename would have to move in lockstep there too.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CurrentDbVersion {
/// The highest sequence number that is part of the committed database.
pub max_sequence_number: u32,
/// When this database was last committed to.
pub commit_time: Timestamp,
}
/// Reads the `CURRENT` file in the database directory `path`.
///
/// Returns `Ok(None)` if the file doesn't exist, which for a writable database means "not
/// initialized yet".
pub fn read_current_version(path: &Path) -> Result<Option<CurrentDbVersion>> {
let current_path = path.join("CURRENT");
let content = match fs::read(&current_path) {
Ok(content) => content,
Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e).context("Failed to read CURRENT file"),
};
serde_json::from_slice::<CurrentDbVersion>(&content)
.with_context(|| {
format!(
"CURRENT file at {} is corrupt ({} bytes)",
current_path.display(),
content.len()
)
})
.map(Some)
}
/// Durably and atomically updates the `CURRENT` file in the database directory `path` to point at /// Durably and atomically updates the `CURRENT` file in the database directory `path` to point at
/// `seq`. /// `seq`, stamping it as used now.
/// ///
/// The write is made atomic by writing `seq` to a temporary `CURRENT.next` file, flushing it, and /// The write is made atomic by writing to a temporary `CURRENT.next` file, flushing it, and then
/// then `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and /// `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and replaces the
/// replaces the destination on Windows, so a concurrent or crashing writer can never observe a /// destination on Windows, so a concurrent or crashing writer can never observe a torn `CURRENT`
/// torn `CURRENT` (in-place overwrites, by contrast, can leave a partially-written value on a /// (in-place overwrites, by contrast, can leave a partially-written value on a crash mid-write).
/// crash mid-write). After the rename we fsync the directory so the new `CURRENT` → inode mapping /// After the rename we fsync the directory so the new `CURRENT` → inode mapping survives a crash.
/// survives a crash.
fn commit_current(path: &Path, seq: u32) -> Result<()> { fn commit_current(path: &Path, seq: u32) -> Result<()> {
let version: &CurrentDbVersion = &CurrentDbVersion {
max_sequence_number: seq,
commit_time: Timestamp::now(),
};
let mut contents =
serde_json::to_string(version).context("Failed to serialize the CURRENT file")?;
contents.push('\n');
let next_path = path.join("CURRENT.next"); let next_path = path.join("CURRENT.next");
let mut next_file = File::create(&next_path)?; let mut next_file = File::create(&next_path)?;
next_file.write_u32::<BE>(seq)?; next_file.write_all(contents.as_bytes())?;
next_file.sync_data()?; next_file.sync_data()?;
drop(next_file); drop(next_file);
fs::rename(&next_path, path.join("CURRENT"))?; fs::rename(&next_path, path.join("CURRENT"))?;
// Fsync the directory. This is the single durability barrier for a commit: by the time we get // Fsync the directory. This is the single durability barrier for a commit: by the time we get
// here every file created earlier in the commit (SST/meta/blob and any `.del` file) already // here every file created earlier in the commit (SST/meta/blob and any `.del` file) already
// exists, so this one fsync flushes *all* of their directory entries together with the CURRENT // exists, so this one fsync flushes *all* of their directory entries together with the CURRENT
...@@ -459,7 +512,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -459,7 +512,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok(()) Ok(())
} }
Err(e) => { Err(e) => {
if !read_only && e.kind() == std::io::ErrorKind::NotFound { if !read_only && e.kind() == ErrorKind::NotFound {
self.create_and_init_directory() self.create_and_init_directory()
.context("Creating and initializing persistence directory failed")?; .context("Creating and initializing persistence directory failed")?;
Ok(()) Ok(())
...@@ -479,18 +532,11 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -479,18 +532,11 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Loads an existing database directory and performs cleanup if necessary. /// Loads an existing database directory and performs cleanup if necessary.
fn load_directory(&mut self, entries: ReadDir, read_only: bool) -> Result<bool> { fn load_directory(&mut self, entries: ReadDir, read_only: bool) -> Result<bool> {
let mut meta_files = Vec::new(); let mut meta_files = Vec::new();
let mut current_file = match File::open(self.path.join("CURRENT")) { let current = match read_current_version(&self.path)? {
Ok(file) => file, Some(version) => version.max_sequence_number,
Err(e) => { None if !read_only => return Ok(false),
if !read_only && e.kind() == std::io::ErrorKind::NotFound { None => bail!("Failed to open database: CURRENT file is missing"),
return Ok(false);
} else {
return Err(e).context("Failed to open CURRENT file");
}
}
}; };
let current = current_file.read_u32::<BE>()?;
drop(current_file);
let mut deleted_files = HashSet::new(); let mut deleted_files = HashSet::new();
for entry in entries { for entry in entries {
......
...@@ -30,7 +30,10 @@ mod tests; ...@@ -30,7 +30,10 @@ mod tests;
pub use arc_bytes::ArcBytes; pub use arc_bytes::ArcBytes;
pub use compression::checksum_block; pub use compression::checksum_block;
pub use db::{CommitStats, CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence}; pub use db::{
CommitStats, CompactConfig, CurrentDbVersion, MetaFileEntryInfo, MetaFileInfo,
TurboPersistence, read_current_version,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FamilyKind { pub enum FamilyKind {
......
...@@ -6,7 +6,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; ...@@ -6,7 +6,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
use crate::{ use crate::{
DbConfig, FamilyConfig, FamilyKind, DbConfig, FamilyConfig, FamilyKind,
constants::{MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::{MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
db::{CompactConfig, TurboPersistence}, db::{CompactConfig, TurboPersistence, read_current_version},
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
write_batch::WriteBatch, write_batch::WriteBatch,
}; };
...@@ -2229,3 +2229,38 @@ fn stale_current_next_is_recovered() -> Result<()> { ...@@ -2229,3 +2229,38 @@ fn stale_current_next_is_recovered() -> Result<()> {
Ok(()) Ok(())
} }
/// `CURRENT` round-trips through JSON, recording both the sequence number and the commit time.
#[test]
fn current_file_is_json_with_commit_time() -> Result<()> {
use crate::parallel_scheduler::SerialScheduler;
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
let before = jiff::Timestamp::now();
{
let db = TurboPersistence::<SerialScheduler, 1>::open(path.to_path_buf())?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![42u8].into())?;
db.commit_write_batch(batch)?;
db.shutdown()?;
}
let after = jiff::Timestamp::now();
// next.js parses `CURRENT` without going through this crate, so these field names are a
// public contract.
let raw = fs::read_to_string(path.join("CURRENT"))?;
assert!(raw.contains("max_sequence_number"), "got: {raw}");
assert!(raw.contains("commit_time"), "got: {raw}");
let version = read_current_version(path)?.expect("CURRENT should exist");
assert!(version.max_sequence_number > 0);
assert!(
version.commit_time >= before && version.commit_time <= after,
"commit_time {} outside [{before}, {after}]",
version.commit_time
);
Ok(())
}
...@@ -45,6 +45,7 @@ dashmap = { workspace = true, features = ["raw-api"]} ...@@ -45,6 +45,7 @@ dashmap = { workspace = true, features = ["raw-api"]}
fs-err = { workspace = true } fs-err = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] } hashbrown = { workspace = true, features = ["raw"] }
indexmap = { workspace = true } indexmap = { workspace = true }
jiff = "0.2.10"
lzzzz = { workspace = true, optional = true } lzzzz = { workspace = true, optional = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
rand = { workspace = true } rand = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment