Unverified Commit ae1e53a1 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Retain fewer stale cache versions and use a TTL (#96941)

In CI we do not retain old versions of the db at all, but in dev or
non-CI settings we would retain up to 2 indefinitely.

Given how large caches can grow, this PR reduces it to no more than 1
old version, and then only for a certain amount of time, currently 3
days. The idea being to support switching back and forth between feature
branches and a next.js upgrade branch, 3 days allows this worflow to
survive a weekend.

To support this the `CURRENT` file format is modified. Instead of just
storing a sequence number it now stores a small json object containing
the `max_sequence_number` and the mtime. We store the last_write_time in
the file to support the fact that we anticipate `cache` directories to
be copied around and the native mtime would record the copy time instead
of the commit time.

This should reduce dev time disk usage in the common case.
parent 4991d120
......@@ -10026,6 +10026,8 @@ dependencies = [
"rand 0.10.1",
"rayon",
"rustc-hash 2.1.1",
"serde",
"serde_json",
"smallvec",
"tempfile",
"thread_local",
......@@ -10130,6 +10132,7 @@ dependencies = [
"hashbrown 0.14.5",
"indexmap 2.13.0",
"indoc",
"jiff",
"lzzzz",
"parking_lot",
"rand 0.10.1",
......
......@@ -76,10 +76,10 @@ function findSeedSource(
const currentWorktree = path.resolve(worktreeInfo.worktreeRoot)
// We are going to find the best candidate worktree
// based on the newest mtime of the CURRENT file in the cache directory.
// based on the most recently written cache directory.
// We only look at our version
let best: { versionDir: string; mtimeMs: number } | undefined
let best: { versionDir: string; commitTimeMs: number } | undefined
for (const root of [
worktreeInfo.mainRepoRoot,
...listLinkedWorktreeRoots(worktreeInfo.mainRepoRoot),
......@@ -93,21 +93,31 @@ function findSeedSource(
'turbopack',
version
)
const mtimeMs = currentMtimeMs(versionDir)
if (mtimeMs === undefined) continue
if (!best || mtimeMs > best.mtimeMs) {
best = { versionDir, mtimeMs }
const commitTimeMs = currentCommitTimeMs(versionDir)
if (commitTimeMs === undefined) continue
if (!best || commitTimeMs > best.commitTimeMs) {
best = { versionDir, commitTimeMs }
}
}
return best?.versionDir
}
function currentMtimeMs(versionDir: string): number | undefined {
// When the cache in `versionDir` was last committed to, in epoch milliseconds, or undefined if
// there is no usable cache there. Read from the `commit_time` the persistence layer records in
// CURRENT.
function currentCommitTimeMs(versionDir: string): number | undefined {
let commitTime
try {
return fs.statSync(path.join(versionDir, 'CURRENT')).mtimeMs
commitTime = JSON.parse(
fs.readFileSync(path.join(versionDir, 'CURRENT'), 'utf8')
).commit_time
} catch {
// missing, unreadable, or not valid JSON - treat it as not a seed candidate
return undefined
}
if (typeof commitTime !== 'string') return undefined
const parsed = Date.parse(commitTime)
return Number.isNaN(parsed) ? undefined : parsed
}
function dirHasEntries(dir: string): boolean {
......
......@@ -20,7 +20,7 @@ crc32fast = { workspace = true }
dashmap = { workspace = true}
either = { workspace = true }
fs-err = { workspace = true }
jiff = "0.2.10"
jiff = { version = "0.2.10", features = ["serde"] }
lzzzz = { workspace = true }
memmap2 = "0.9.5"
nohash-hasher = { workspace = true }
......@@ -31,6 +31,8 @@ postcard = { workspace = true, features = ["alloc", "use-std"] }
zerocopy = { version = "0.8", features = ["derive"] }
quick_cache = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
smallvec = { workspace = true }
thread_local = { workspace = true }
tracing = { workspace = true }
......
......@@ -12,7 +12,7 @@ It supports having multiple key families, which are stored in separate files, bu
## On disk format
There is a single `CURRENT` file which stores the latest committed sequence number.
There is a single `CURRENT` file, a small JSON object holding the latest committed sequence number (`max_sequence_number`) and when that commit happened (`commit_time`). The commit time is stored in the file rather than taken from its mtime so that it survives the directory being copied or restored. External tools read this file, so its field names are a stable contract.
All other files have a sequence number as file name, e. g. `0000123.sst`. All files are immutable once their sequence number is <= the committed sequence number. But they might be deleted when they are superseded by other committed files.
......
......@@ -24,6 +24,7 @@ use turbo_persistence::{
BLOCK_HEADER_SIZE, checksum_block,
meta_file::MetaFile,
mmap_helper::advise_mmap_for_persistence,
read_current_version,
sst_filter::SstFilter,
static_sorted_file::{
BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH,
......@@ -211,9 +212,9 @@ fn format_bytes(bytes: u64) -> String {
/// and apply SstFilter to skip superseded entries.
fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
// Read the CURRENT sequence number — only files with seq <= current are valid.
let current: u32 = File::open(db_path.join("CURRENT"))?
.read_u32::<BE>()
.context("Failed to read CURRENT file")?;
let current = read_current_version(db_path)?
.context("CURRENT file is missing")?
.max_sequence_number;
// Read .del files to find sequences that were deleted but not yet cleaned up.
let mut deleted_seqs: HashSet<u32> = HashSet::new();
......
......@@ -2,7 +2,7 @@ use std::{
borrow::Cow,
collections::HashSet,
fmt::Display,
io::{BufWriter, Write},
io::{BufWriter, ErrorKind, Write},
mem::take,
ops::RangeInclusive,
path::{Path, PathBuf},
......@@ -20,6 +20,7 @@ use jiff::Timestamp;
use memmap2::Mmap;
use nohash_hasher::BuildNoHashHasher;
use parking_lot::{Mutex, RwLock};
use serde::{Deserialize, Serialize};
use smallvec::SmallVec;
use tracing::span::EnteredSpan;
......@@ -159,24 +160,61 @@ impl WriteOperationGuard<'_> {
}
}
/// The contents of the `CURRENT` file: which sequence number is committed, and when that commit
/// happened.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CurrentDbVersion {
/// The highest sequence number that is part of the committed database.
pub max_sequence_number: u32,
/// When this database was last committed to.
pub commit_time: Timestamp,
}
/// Reads the `CURRENT` file in the database directory `path`.
///
/// Returns `Ok(None)` if the file doesn't exist, which for a writable database means "not
/// initialized yet".
pub fn read_current_version(path: &Path) -> Result<Option<CurrentDbVersion>> {
let current_path = path.join("CURRENT");
let content = match fs::read(&current_path) {
Ok(content) => content,
Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e).context("Failed to read CURRENT file"),
};
serde_json::from_slice::<CurrentDbVersion>(&content)
.with_context(|| {
format!(
"CURRENT file at {} is corrupt ({} bytes)",
current_path.display(),
content.len()
)
})
.map(Some)
}
/// Durably and atomically updates the `CURRENT` file in the database directory `path` to point at
/// `seq`.
/// `seq`, stamping it as used now.
///
/// The write is made atomic by writing `seq` to a temporary `CURRENT.next` file, flushing it, and
/// then `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and
/// replaces the destination on Windows, so a concurrent or crashing writer can never observe a
/// torn `CURRENT` (in-place overwrites, by contrast, can leave a partially-written value on a
/// crash mid-write). After the rename we fsync the directory so the new `CURRENT` → inode mapping
/// survives a crash.
/// The write is made atomic by writing to a temporary `CURRENT.next` file, flushing it, and then
/// `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and replaces the
/// destination on Windows, so a concurrent or crashing writer can never observe a torn `CURRENT`
/// (in-place overwrites, by contrast, can leave a partially-written value on a crash mid-write).
/// After the rename we fsync the directory so the new `CURRENT` → inode mapping survives a crash.
fn commit_current(path: &Path, seq: u32) -> Result<()> {
let version: &CurrentDbVersion = &CurrentDbVersion {
max_sequence_number: seq,
commit_time: Timestamp::now(),
};
let mut contents =
serde_json::to_string(version).context("Failed to serialize the CURRENT file")?;
contents.push('\n');
let next_path = path.join("CURRENT.next");
let mut next_file = File::create(&next_path)?;
next_file.write_u32::<BE>(seq)?;
next_file.write_all(contents.as_bytes())?;
next_file.sync_data()?;
drop(next_file);
fs::rename(&next_path, path.join("CURRENT"))?;
// Fsync the directory. This is the single durability barrier for a commit: by the time we get
// here every file created earlier in the commit (SST/meta/blob and any `.del` file) already
// exists, so this one fsync flushes *all* of their directory entries together with the CURRENT
......@@ -459,7 +497,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
Ok(())
}
Err(e) => {
if !read_only && e.kind() == std::io::ErrorKind::NotFound {
if !read_only && e.kind() == ErrorKind::NotFound {
self.create_and_init_directory()
.context("Creating and initializing persistence directory failed")?;
Ok(())
......@@ -479,18 +517,11 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Loads an existing database directory and performs cleanup if necessary.
fn load_directory(&mut self, entries: ReadDir, read_only: bool) -> Result<bool> {
let mut meta_files = Vec::new();
let mut current_file = match File::open(self.path.join("CURRENT")) {
Ok(file) => file,
Err(e) => {
if !read_only && e.kind() == std::io::ErrorKind::NotFound {
return Ok(false);
} else {
return Err(e).context("Failed to open CURRENT file");
}
}
let current = match read_current_version(&self.path)? {
Some(version) => version.max_sequence_number,
None if !read_only => return Ok(false),
None => bail!("Failed to open database: CURRENT file is missing"),
};
let current = current_file.read_u32::<BE>()?;
drop(current_file);
let mut deleted_files = HashSet::new();
for entry in entries {
......
......@@ -30,7 +30,10 @@ mod tests;
pub use arc_bytes::ArcBytes;
pub use compression::checksum_block;
pub use db::{CommitStats, CompactConfig, MetaFileEntryInfo, MetaFileInfo, TurboPersistence};
pub use db::{
CommitStats, CompactConfig, CurrentDbVersion, MetaFileEntryInfo, MetaFileInfo,
TurboPersistence, read_current_version,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FamilyKind {
......
......@@ -6,7 +6,7 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
use crate::{
DbConfig, FamilyConfig, FamilyKind,
constants::{MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
db::{CompactConfig, TurboPersistence},
db::{CompactConfig, TurboPersistence, read_current_version},
parallel_scheduler::ParallelScheduler,
write_batch::WriteBatch,
};
......@@ -2229,3 +2229,38 @@ fn stale_current_next_is_recovered() -> Result<()> {
Ok(())
}
/// `CURRENT` round-trips through JSON, recording both the sequence number and the commit time.
#[test]
fn current_file_is_json_with_commit_time() -> Result<()> {
use crate::parallel_scheduler::SerialScheduler;
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
let before = jiff::Timestamp::now();
{
let db = TurboPersistence::<SerialScheduler, 1>::open(path.to_path_buf())?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![42u8].into())?;
db.commit_write_batch(batch)?;
db.shutdown()?;
}
let after = jiff::Timestamp::now();
// next.js parses `CURRENT` without going through this crate, so these field names are a
// public contract.
let raw = fs::read_to_string(path.join("CURRENT"))?;
assert!(raw.contains("max_sequence_number"), "got: {raw}");
assert!(raw.contains("commit_time"), "got: {raw}");
let version = read_current_version(path)?.expect("CURRENT should exist");
assert!(version.max_sequence_number > 0);
assert!(
version.commit_time >= before && version.commit_time <= after,
"commit_time {} outside [{before}, {after}]",
version.commit_time
);
Ok(())
}
......@@ -45,6 +45,7 @@ dashmap = { workspace = true, features = ["raw-api"]}
fs-err = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] }
indexmap = { workspace = true }
jiff = "0.2.10"
lzzzz = { workspace = true, optional = true }
parking_lot = { workspace = true }
rand = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment