Unverified Commit fce9c2fe authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

turbopack: report the cause when saving the filesystem cache fails (#99823)

## What

Builds with the Turbopack filesystem cache didn't give useful output
when the cache couldn't be written during shutdown, for example because
of a read-only directory or a full disk. The build succeeded, but it
printed a bare `Shutting down failed: <message>` which was not useful

Before:
```
Persisting failed during shutdown: Unable to write SST file 00000020.sst

Caused by:
: Permission denied (os error 13)te/var/folders/p6/ytr7tcts5ndf72xw136ky30m0000gn/T/next-install-6cb7b5bc6e3b95a18ae07aa856827db93efc662fe73934ecebae8de778735e0c/.next/cache/turbopack/v16.5.0-canary.2-2c48f075ef/00000020.sst`
Shutting down failed: A previous write operation failed with an unrecoverable error; no further writes are possible
```

Now:

```
WARNING: Saving the filesystem cache failed:
    Unable to write SST file 00000020.sst: failed to create file `/Users/lukesandberg/projects/next_2/next.js/test/tmp/next-test-1791392696220-115/.next/cache/turbopack/unversioned/00000020.sst`: Permission denied (os error 13)
```

The "Shutting down failed" error is gone and the filesystem cache error
is slightly improved.

Also:
- **Compaction is skipped once a write has already failed.** That avoids
a second, redundant failure.
- **Database shutdown can't fail anymore.** Compaction errors during
shutdown are logged as warnings.

---------
Co-authored-by: 's avatarvercel-gh-bot-5[bot] <312521305+vercel-gh-bot-5[bot]@users.noreply.github.com>
parent eeb353be
...@@ -180,7 +180,7 @@ fn prefill_database(path: &Path, config: &DbConfig) -> Result<Vec<Box<[u8]>>> { ...@@ -180,7 +180,7 @@ fn prefill_database(path: &Path, config: &DbConfig) -> Result<Vec<Box<[u8]>>> {
} }
} }
db.shutdown()?; db.shutdown();
keys.shuffle(&mut rng); keys.shuffle(&mut rng);
Ok(keys) Ok(keys)
} }
...@@ -333,7 +333,7 @@ fn bench_write(c: &mut Criterion) { ...@@ -333,7 +333,7 @@ fn bench_write(c: &mut Criterion) {
db.commit_write_batch(batch).unwrap(); db.commit_write_batch(batch).unwrap();
} }
db.shutdown().unwrap(); db.shutdown();
tempdir tempdir
}, },
BatchSize::PerIteration, BatchSize::PerIteration,
...@@ -668,7 +668,7 @@ fn prefill_multi_value_database( ...@@ -668,7 +668,7 @@ fn prefill_multi_value_database(
} }
} }
db.shutdown()?; db.shutdown();
keys.shuffle(&mut rng); keys.shuffle(&mut rng);
Ok(keys) Ok(keys)
} }
...@@ -1067,7 +1067,7 @@ fn bench_write_multi_value(c: &mut Criterion) { ...@@ -1067,7 +1067,7 @@ fn bench_write_multi_value(c: &mut Criterion) {
batch.put(0, &**key, value.into()).unwrap(); batch.put(0, &**key, value.into()).unwrap();
} }
db.commit_write_batch(batch).unwrap(); db.commit_write_batch(batch).unwrap();
db.shutdown().unwrap(); db.shutdown();
tempdir tempdir
}, },
BatchSize::PerIteration, BatchSize::PerIteration,
......
...@@ -1379,6 +1379,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1379,6 +1379,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Returns `Some(stats)` describing the bytes written/deleted if a compaction commit happened, /// Returns `Some(stats)` describing the bytes written/deleted if a compaction commit happened,
/// or `None` if there was nothing to compact. /// or `None` if there was nothing to compact.
pub fn compact(&self, compact_config: &CompactConfig) -> Result<Option<CommitStats>> { pub fn compact(&self, compact_config: &CompactConfig) -> Result<Option<CommitStats>> {
if self.has_unrecoverable_write_error() {
// If a previous write failed just skip compaction.
return Ok(None);
}
let mut guard = self.acquire_write_operation("compaction")?; let mut guard = self.acquire_write_operation("compaction")?;
// Free block caches and SST mmaps before compaction. The block caches // Free block caches and SST mmaps before compaction. The block caches
...@@ -2548,11 +2552,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -2548,11 +2552,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Shuts down the database. This will print statistics if the `print_stats` feature is enabled. /// Shuts down the database. This will print statistics if the `print_stats` feature is enabled.
/// Retries deletion of all previously-deferred files and clears successfully deleted batches. /// Retries deletion of all previously-deferred files and clears successfully deleted batches.
pub fn shutdown(&self) -> Result<()> { pub fn shutdown(&self) {
#[cfg(feature = "print_stats")] #[cfg(feature = "print_stats")]
println!("{:#?}", self.statistics()); println!("{:#?}", self.statistics());
self.retry_deferred_deletions(); self.retry_deferred_deletions();
Ok(())
} }
/// Attempts to delete files with the given extension, returning an iterator of sequence /// Attempts to delete files with the given extension, returning an iterator of sequence
......
...@@ -1645,12 +1645,12 @@ impl TurboTasksBackend { ...@@ -1645,12 +1645,12 @@ impl TurboTasksBackend {
&& let Err(err) = && let Err(err) =
self.snapshot_and_persist(Span::current().into(), SnapshotReason::Stop, turbo_tasks) self.snapshot_and_persist(Span::current().into(), SnapshotReason::Stop, turbo_tasks)
{ {
eprintln!("Persisting failed during shutdown: {err:?}"); // We don't treat this as a failure, just warn that the cache is broken and keep going.
eprintln!("WARNING: Saving the filesystem cache failed:\n {err:#}");
} }
self.storage.drop_contents(); self.storage.drop_contents();
if let Err(err) = self.backing_storage.shutdown() { // Do final cleanup, this is best effort and unfailable
println!("Shutting down failed: {err}"); self.backing_storage.shutdown();
}
} }
#[allow(unused_variables)] #[allow(unused_variables)]
......
...@@ -139,20 +139,24 @@ impl TurboKeyValueDatabase { ...@@ -139,20 +139,24 @@ impl TurboKeyValueDatabase {
self.db.has_unrecoverable_write_error() self.db.has_unrecoverable_write_error()
} }
pub fn shutdown(&self) -> Result<()> { pub fn shutdown(&self) {
// Compact the database on shutdown // Compact the database on shutdown
// (Avoid compacting a fresh database since we don't have any usage info yet) // (Avoid compacting a fresh database since we don't have any usage info yet)
if !self.is_fresh && !self.options.skip_compaction { if !self.is_fresh && !self.options.skip_compaction {
if self.options.is_ci { let result = if self.options.is_ci {
// Fully compact in CI to reduce cache size // Fully compact in CI to reduce cache size
do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)?; do_compact(&self.db, COMPACTION_MESSAGE, usize::MAX)
} else { } else {
// Compact with a reasonable limit in non-CI environments // Compact with a reasonable limit in non-CI environments
do_compact( do_compact(
&self.db, &self.db,
COMPACTION_MESSAGE, COMPACTION_MESSAGE,
available_parallelism().map_or(4, |c| max(4, c.get())), available_parallelism().map_or(4, |c| max(4, c.get())),
)?; )
};
if let Err(err) = result {
// Compacting failures are generally not fatal, so just log them
eprintln!("WARNING: Compacting the database failed {err}");
} }
} }
// Shutdown the database // Shutdown the database
......
...@@ -460,7 +460,7 @@ impl TurboBackingStorage { ...@@ -460,7 +460,7 @@ impl TurboBackingStorage {
self.inner.database.compact() self.inner.database.compact()
} }
pub(crate) fn shutdown(&self) -> Result<()> { pub(crate) fn shutdown(&self) {
self.inner.database.shutdown() self.inner.database.shutdown()
} }
...@@ -588,7 +588,7 @@ mod tests { ...@@ -588,7 +588,7 @@ mod tests {
"Should return all 3 task IDs for the colliding hash" "Should return all 3 task IDs for the colliding hash"
); );
db.shutdown()?; db.shutdown();
Ok(()) Ok(())
} }
...@@ -623,7 +623,7 @@ mod tests { ...@@ -623,7 +623,7 @@ mod tests {
unsafe { batch.flush(KeySpace::TaskCache) }?; unsafe { batch.flush(KeySpace::TaskCache) }?;
batch.commit()?; batch.commit()?;
db.shutdown()?; db.shutdown();
} }
// Reopen and verify all entries are readable // Reopen and verify all entries are readable
...@@ -643,7 +643,7 @@ mod tests { ...@@ -643,7 +643,7 @@ mod tests {
} }
} }
assert_eq!(missing, 0, "Found {found}/{n} entries, missing {missing}"); assert_eq!(missing, 0, "Found {found}/{n} entries, missing {missing}");
db.shutdown()?; db.shutdown();
} }
Ok(()) Ok(())
...@@ -726,7 +726,7 @@ mod tests { ...@@ -726,7 +726,7 @@ mod tests {
"save_snapshot should delete only the named id from the bucket" "save_snapshot should delete only the named id from the bucket"
); );
db.shutdown()?; db.shutdown();
Ok(()) Ok(())
} }
} }
...@@ -100,5 +100,6 @@ pub fn compact_database( ...@@ -100,5 +100,6 @@ pub fn compact_database(
max_merge_jobs: usize::MAX, max_merge_jobs: usize::MAX,
..turbo::COMPACT_CONFIG ..turbo::COMPACT_CONFIG
})?; })?;
db.shutdown() db.shutdown();
Ok(())
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment