Unverified Commit becee66e authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

fix(turbo-persistence): preserve write errors on writer drop (#98840)

### What?

Preserve write failures from streaming SST construction instead of
replacing them with debug-only writer lifecycle assertions. This adds
explicit cancellation, immediate best-effort cleanup of incomplete SSTs,
and focused coverage for failed writes and finalization.

### Why?

`std::thread::panicking()` is false while an error is propagated through
`Result`. On disk-full and similar failures, writers and their owning
collectors are therefore dropped before a caller begins panic unwinding,
allowing lifecycle assertions to obscure the original I/O error. Leaving
partial files until startup recovery also delays reclaiming scarce disk
space.

### How?

`StreamingSstWriter::cancel()` consumes an incomplete writer, discards
buffered bytes without flushing or finalizing, closes the raw handle,
and best-effort removes the partial SST. Failed close attempts and
unfinished Drop paths use the same cleanup, while deletion failures are
ignored so they cannot replace the root error; existing startup recovery
remains the fallback.

Writer-owning scopes preserve the original error, explicitly cancel
their writers, and then return that error. The compaction merge scope
cancels both collectors on every error exit so a sibling collector
cannot mask the failure. Successful close marks the finalized SST for
preservation, and healthy writers still have to call `close()` or
`cancel()` to satisfy the debug lifecycle invariant.

The regression tests substitute an unbuffered read-only output handle to
force deterministic add and close failures without requiring disk
pressure.

### Verification

- `cargo test -p turbo-persistence` (114 passed)
- `cargo check --release -p turbo-persistence`
- `cargo fmt --package turbo-persistence -- --check`

<!-- NEXT_JS_LLM -->

<!-- fleet da0c4e98-b86a-4c93-a519-8a63147e3f5c -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent cdecbe09
......@@ -2,7 +2,7 @@ use std::{
borrow::Cow,
collections::VecDeque,
io::{BufWriter, Write},
path::Path,
path::{Path, PathBuf},
};
use anyhow::{Context, Result};
......@@ -355,7 +355,10 @@ pub fn write_static_stored_file<E: Entry>(
);
let mut writer = StreamingSstWriter::new(file, flags, entries.len() as u64, compression)?;
for entry in entries {
writer.add(entry)?;
if let Err(err) = writer.add(entry) {
writer.cancel();
return Err(err);
}
}
writer.close()
}
......@@ -544,6 +547,9 @@ pub struct StreamingSstWriter<E: Entry> {
// File I/O. Wrapped in Option so close() can take ownership without a partial-move
// compile error (partial moves are forbidden when the type has a Drop impl).
file: Option<BufWriter<File>>,
file_path: PathBuf,
/// Whether close() successfully finalized the SST, so Drop must preserve the file.
preserve_file: bool,
compress_buffer: Vec<u8>,
block_offsets: Vec<u32>,
compressor: Compressor,
......@@ -615,7 +621,8 @@ pub struct StreamingSstWriter<E: Entry> {
/// State of the current incomplete key block at the tail of the resolved prefix.
current_key_block: KeyBlockAccumulator,
/// Set to `true` by `close()` so the Drop guard can detect writers dropped without closing.
/// Set to `true` by `close()` or `cancel()` so the Drop guard can detect writers dropped
/// without completing their lifecycle.
#[cfg(debug_assertions)]
finished: bool,
}
......@@ -630,6 +637,7 @@ impl<E: Entry> StreamingSstWriter<E> {
max_entry_count: u64,
compression: Compression,
) -> Result<Self> {
let file_path = file.to_owned();
let file = BufWriter::new(File::create(file)?);
let compressor = Compressor::new(compression)?;
......@@ -648,6 +656,8 @@ impl<E: Entry> StreamingSstWriter<E> {
Ok(Self {
file: Some(file),
file_path,
preserve_file: false,
compress_buffer: Vec::with_capacity(MIN_SMALL_VALUE_BLOCK_SIZE + MAX_SMALL_VALUE_SIZE),
block_offsets: Vec::with_capacity(estimated_total_blocks),
compressor,
......@@ -818,6 +828,27 @@ impl<E: Entry> StreamingSstWriter<E> {
self.try_flush_key_blocks()
}
/// Abandons this writer without flushing buffered data or finalizing the SST file.
pub fn cancel(mut self) {
self.discard_partial_file();
#[cfg(debug_assertions)]
{
self.finished = true;
}
}
/// Closes the raw handle without flushing its buffer and best-effort removes the partial SST.
fn discard_partial_file(&mut self) {
if let Some(file) = self.file.take() {
// Unlike dropping BufWriter, into_parts() does not attempt to flush its buffer.
let (file, _) = file.into_parts();
drop(file);
}
// Startup recovery is the fallback if deletion itself fails (for example on Windows if
// another handle is still open). Never replace the error that caused cancellation.
let _ = fs_err::remove_file(&self.file_path);
}
/// Appends a new entry to the pending-keys queue.
fn push_pending_key_entry(&mut self, entry: E, value_ref: ValueRef) {
self.pending_keys
......@@ -1121,7 +1152,9 @@ impl<E: Entry> StreamingSstWriter<E> {
entries: self.entry_count,
};
Ok((meta, file.into_inner()?))
let file = file.into_inner()?;
self.preserve_file = true;
Ok((meta, file))
}
/// Flushes all remaining entries as key blocks. Called from `close()` after all small value
......@@ -1175,14 +1208,18 @@ impl<E: Entry> StreamingSstWriter<E> {
}
}
#[cfg(debug_assertions)]
impl<E: Entry> Drop for StreamingSstWriter<E> {
fn drop(&mut self) {
if !self.preserve_file {
self.discard_partial_file();
}
// Skip assertion during panic unwinding to avoid a double-panic (which would abort).
#[cfg(debug_assertions)]
if !std::thread::panicking() {
assert!(
self.finished || self.entry_count == 0,
"StreamingSstWriter dropped without calling close()"
"StreamingSstWriter dropped without calling close() or cancel()"
);
}
}
......@@ -1580,7 +1617,10 @@ mod tests {
let mut writer =
StreamingSstWriter::new(&sst_path, flags, entries.len() as u64, Compression::Lz4)?;
for entry in entries {
writer.add(entry)?;
if let Err(err) = writer.add(entry) {
writer.cancel();
return Err(err);
}
}
let (meta, _file) = writer.close()?;
Ok(meta)
......@@ -1809,6 +1849,54 @@ mod tests {
Ok(())
}
#[test]
#[cfg(debug_assertions)]
fn cancel_after_failed_add_does_not_trigger_drop_assertion() {
let dir = tempfile::tempdir().unwrap();
let sst_path = dir.path().join("test.sst");
let mut writer =
StreamingSstWriter::new(&sst_path, MetaEntryFlags::default(), 1, Compression::Lz4)
.unwrap();
// Replace the output with an unbuffered, read-only handle to force a deterministic write
// error without relying on the filesystem being full.
drop(writer.file.take());
writer.file = Some(BufWriter::with_capacity(0, File::open(&sst_path).unwrap()));
let error = writer
.add(TestEntry::medium(b"key", &[0; 8192]))
.unwrap_err();
assert!(format!("{error:#}").contains("Failed to write value block"));
let cancel_result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| writer.cancel()));
assert!(
cancel_result.is_ok(),
"cancelling a writer after an add error must not trigger the lifecycle assertion"
);
assert!(!sst_path.exists(), "cancel should remove the partial SST");
}
#[test]
fn failed_close_removes_partial_file() {
let dir = tempfile::tempdir().unwrap();
let sst_path = dir.path().join("test.sst");
let mut writer =
StreamingSstWriter::new(&sst_path, MetaEntryFlags::default(), 1, Compression::Lz4)
.unwrap();
writer.add(TestEntry::inline(b"key", b"value")).unwrap();
// Force close() to fail while flushing the pending key block.
drop(writer.file.take());
writer.file = Some(BufWriter::with_capacity(0, File::open(&sst_path).unwrap()));
let error = writer.close().unwrap_err();
assert!(format!("{error:#}").contains("Failed to write key block"));
assert!(
!sst_path.exists(),
"a failed close should remove the partial SST"
);
}
#[test]
fn is_full_entry_count_limit() {
let dir = tempfile::tempdir().unwrap();
......@@ -1894,7 +1982,10 @@ mod tests {
Compression::Lz4,
)?;
for entry in &entries {
writer.add(entry)?;
if let Err(err) = writer.add(entry) {
writer.cancel();
return Err(err);
}
}
let (meta2, _) = writer.close()?;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment