Unverified Commit cd243042 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Turbopack: add more context to persistence file errors (#95318)

### What?

Adds path- and operation-level context to the filesystem errors raised
by the `turbo-persistence` commit/rollback flow and the
`turbo-tasks-backend` database invalidation/versioning helpers.

### Why?

A user reported a bare `Access is denied. (OS error 5)` on Windows 11
that surfaced out of the cache `commit` flow with no indication of which
file or which filesystem operation failed:
https://github.com/vercel/next.js/discussions/95130#discussioncomment-17479429
The underlying issue was fixed in #95495 but this adds more information
for future issues.

Raw `io::Error`s from `?` carry an OS error code but no path, so there
was no way to tell whether the failure was on `CURRENT`, a `.del` file,
a per-file `fsync`, the directory `fsync`, or a cleanup `remove_file`
etc.

### How?

- Adopted the `fs_err` crate for all fs operations in the persistence
layer
- Added additional context to some flows
- Unified `CURRENT` file writing and adopted an 'atomic write' pattern.

<!-- NEXT_JS_LLM_PR -->
parent c3dd28fd
...@@ -2809,6 +2809,15 @@ dependencies = [ ...@@ -2809,6 +2809,15 @@ dependencies = [
"syn 2.0.104", "syn 2.0.104",
] ]
[[package]]
name = "fs-err"
version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "fs-set-times" name = "fs-set-times"
version = "0.20.3" version = "0.20.3"
...@@ -9962,6 +9971,7 @@ dependencies = [ ...@@ -9962,6 +9971,7 @@ dependencies = [
"crc32fast", "crc32fast",
"dashmap 6.1.0", "dashmap 6.1.0",
"either", "either",
"fs-err",
"jiff", "jiff",
"lzzzz", "lzzzz",
"memmap2 0.9.5", "memmap2 0.9.5",
...@@ -10072,6 +10082,7 @@ dependencies = [ ...@@ -10072,6 +10082,7 @@ dependencies = [
"codspeed-criterion-compat", "codspeed-criterion-compat",
"crossbeam-utils", "crossbeam-utils",
"dashmap 6.1.0", "dashmap 6.1.0",
"fs-err",
"futures", "futures",
"hashbrown 0.14.5", "hashbrown 0.14.5",
"indexmap 2.13.0", "indexmap 2.13.0",
......
...@@ -273,6 +273,7 @@ dunce = "1.0.3" ...@@ -273,6 +273,7 @@ dunce = "1.0.3"
either = "1.15.0" either = "1.15.0"
erased-serde = "0.4.5" erased-serde = "0.4.5"
flate2 = "1.0.28" flate2 = "1.0.28"
fs-err = "3.1.1"
futures = "0.3.31" futures = "0.3.31"
futures-retry = "0.6.0" futures-retry = "0.6.0"
futures-util = "0.3.31" futures-util = "0.3.31"
......
...@@ -19,6 +19,7 @@ byteorder = { workspace = true } ...@@ -19,6 +19,7 @@ byteorder = { workspace = true }
crc32fast = { workspace = true } crc32fast = { workspace = true }
dashmap = { workspace = true} dashmap = { workspace = true}
either = { workspace = true } either = { workspace = true }
fs-err = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
lzzzz = { workspace = true } lzzzz = { workspace = true }
memmap2 = "0.9.5" memmap2 = "0.9.5"
......
...@@ -12,12 +12,12 @@ ...@@ -12,12 +12,12 @@
use std::{ use std::{
collections::{BTreeMap, HashSet}, collections::{BTreeMap, HashSet},
fs::{self, File},
path::{Path, PathBuf}, path::{Path, PathBuf},
}; };
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use fs_err::{self as fs, File};
use lzzzz::lz4::decompress; use lzzzz::lz4::decompress;
use memmap2::Mmap; use memmap2::Mmap;
use turbo_persistence::{ use turbo_persistence::{
...@@ -211,8 +211,7 @@ fn format_bytes(bytes: u64) -> String { ...@@ -211,8 +211,7 @@ fn format_bytes(bytes: u64) -> String {
/// and apply SstFilter to skip superseded entries. /// and apply SstFilter to skip superseded entries.
fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> { fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
// Read the CURRENT sequence number — only files with seq <= current are valid. // Read the CURRENT sequence number — only files with seq <= current are valid.
let current: u32 = File::open(db_path.join("CURRENT")) let current: u32 = File::open(db_path.join("CURRENT"))?
.context("Failed to open CURRENT file")?
.read_u32::<BE>() .read_u32::<BE>()
.context("Failed to read CURRENT file")?; .context("Failed to read CURRENT file")?;
...@@ -425,9 +424,9 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result<SstStats> { ...@@ -425,9 +424,9 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result<SstStats> {
let filename = format!("{:08}.sst", info.sequence_number); let filename = format!("{:08}.sst", info.sequence_number);
let path = db_path.join(&filename); let path = db_path.join(&filename);
let file = File::open(&path).with_context(|| format!("Failed to open {}", filename))?; let file = File::open(&path)?;
let file_size = file.metadata()?.len(); let file_size = file.metadata()?.len();
let mmap = unsafe { Mmap::map(&file)? }; let mmap = unsafe { Mmap::map(file.file())? };
advise_mmap_for_persistence(&mmap)?; advise_mmap_for_persistence(&mmap)?;
let mut stats = SstStats { let mut stats = SstStats {
......
...@@ -2,7 +2,6 @@ use std::{ ...@@ -2,7 +2,6 @@ use std::{
borrow::Cow, borrow::Cow,
collections::HashSet, collections::HashSet,
fmt::Display, fmt::Display,
fs::{self, File, OpenOptions, ReadDir},
io::{BufWriter, Write}, io::{BufWriter, Write},
mem::take, mem::take,
ops::RangeInclusive, ops::RangeInclusive,
...@@ -16,6 +15,7 @@ use std::{ ...@@ -16,6 +15,7 @@ use std::{
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use byteorder::{BE, ReadBytesExt, WriteBytesExt}; use byteorder::{BE, ReadBytesExt, WriteBytesExt};
use dashmap::DashSet; use dashmap::DashSet;
use fs_err::{self as fs, File, OpenOptions, ReadDir};
use jiff::Timestamp; use jiff::Timestamp;
use memmap2::Mmap; use memmap2::Mmap;
use nohash_hasher::BuildNoHashHasher; use nohash_hasher::BuildNoHashHasher;
...@@ -159,21 +159,50 @@ impl WriteOperationGuard<'_> { ...@@ -159,21 +159,50 @@ impl WriteOperationGuard<'_> {
} }
} }
/// Durably and atomically updates the `CURRENT` file in the database directory `path` to point at
/// `seq`.
///
/// The write is made atomic by writing `seq` to a temporary `CURRENT.next` file, flushing it, and
/// then `rename`ing it over `CURRENT`. A `rename` within a directory is atomic on POSIX and
/// replaces the destination on Windows, so a concurrent or crashing writer can never observe a
/// torn `CURRENT` (in-place overwrites, by contrast, can leave a partially-written value on a
/// crash mid-write). After the rename we fsync the directory so the new `CURRENT` → inode mapping
/// survives a crash.
fn commit_current(path: &Path, seq: u32) -> Result<()> {
let next_path = path.join("CURRENT.next");
let mut next_file = File::create(&next_path)?;
next_file.write_u32::<BE>(seq)?;
next_file.sync_data()?;
drop(next_file);
fs::rename(&next_path, path.join("CURRENT"))?;
// Fsync the directory. This is the single durability barrier for a commit: by the time we get
// here every file created earlier in the commit (SST/meta/blob and any `.del` file) already
// exists, so this one fsync flushes *all* of their directory entries together with the CURRENT
// rename. Because the file *contents* were already `sync_data`'d before this call and the
// rename is the last directory mutation, a crash can never leave a durable CURRENT pointing at
// files whose directory entries were lost. Callers therefore do not need a separate directory
// fsync before invoking this.
//
// Skipped on Windows: `sync_data` on a directory handle fails with ERROR_ACCESS_DENIED (the
// handle `File::open` returns for a directory has no write access).Apparently metadata changes
// are always atomic on windows so this is simply unneeded.
#[cfg(not(windows))]
File::open(path)
.and_then(|dir| dir.sync_data())
.context("Failed to sync database directory after updating CURRENT")?;
Ok(())
}
/// Deletes all files in `path` whose numeric stem is greater than `seq_before`. /// Deletes all files in `path` whose numeric stem is greater than `seq_before`.
/// ///
/// Called on rollback to clean up any SST, meta, blob, or del files written during a /// Called on rollback to clean up any SST, meta, blob, or del files written during a
/// failed write operation or compaction. /// failed write operation or compaction.
fn delete_orphan_files(path: &Path, seq_before: u32) -> Result<()> { fn delete_orphan_files(path: &Path, seq_before: u32) -> Result<()> {
// Restore CURRENT to seq_before first. The failure may have happened mid-write // Restore CURRENT to seq_before first, so the on-disk state is consistent before we start
// to CURRENT, leaving it partially written. Writing seq_before makes the // deleting the orphan files that a failed write/compaction left behind.
// on-disk state consistent before we start deleting orphan files. commit_current(path, seq_before).context("Unable to restore CURRENT file")?;
let mut current_file = OpenOptions::new()
.write(true)
.truncate(false)
.read(false)
.open(path.join("CURRENT"))?;
current_file.write_u32::<BE>(seq_before)?;
current_file.sync_all()?;
for entry in fs::read_dir(path)? { for entry in fs::read_dir(path)? {
let entry = entry?; let entry = entry?;
...@@ -424,7 +453,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -424,7 +453,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
if read_only { if read_only {
bail!("Failed to open database"); bail!("Failed to open database");
} }
self.init_directory() commit_current(&self.path, 0)
.context("Initializing persistence directory failed")?; .context("Initializing persistence directory failed")?;
} }
Ok(()) Ok(())
...@@ -444,15 +473,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -444,15 +473,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
/// Creates the directory and initializes it. /// Creates the directory and initializes it.
fn create_and_init_directory(&mut self) -> Result<()> { fn create_and_init_directory(&mut self) -> Result<()> {
fs::create_dir_all(&self.path)?; fs::create_dir_all(&self.path)?;
self.init_directory() commit_current(&self.path, 0)
}
/// Initializes the directory by creating the CURRENT file.
fn init_directory(&mut self) -> Result<()> {
let mut current = File::create(self.path.join("CURRENT"))?;
current.write_u32::<BE>(0)?;
current.flush()?;
Ok(())
} }
/// Loads an existing database directory and performs cleanup if necessary. /// Loads an existing database directory and performs cleanup if necessary.
...@@ -476,6 +497,15 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -476,6 +497,15 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
let entry = entry?; let entry = entry?;
let path = entry.path(); let path = entry.path();
if let Some(ext) = path.extension().and_then(|s| s.to_str()) { if let Some(ext) = path.extension().and_then(|s| s.to_str()) {
// A leftover `CURRENT.next` means a crash interrupted a `commit_current` before
// the rename onto `CURRENT` completed. The current `CURRENT` (already read above)
// is authoritative, so the stale temp file is just deleted.
if path.file_stem().and_then(|s| s.to_str()) == Some("CURRENT") {
if !read_only {
fs::remove_file(&path)?;
}
continue;
}
let seq: u32 = path let seq: u32 = path
.file_stem() .file_stem()
.context("File has no file stem")? .context("File has no file stem")?
...@@ -576,9 +606,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -576,9 +606,8 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
#[tracing::instrument(level = "info", name = "reading database blob", skip_all)] #[tracing::instrument(level = "info", name = "reading database blob", skip_all)]
fn read_blob(&self, seq: u32) -> Result<ArcBytes> { fn read_blob(&self, seq: u32) -> Result<ArcBytes> {
let path = self.path.join(format!("{seq:08}.blob")); let path = self.path.join(format!("{seq:08}.blob"));
let file = File::open(&path) let file = File::open(&path)?;
.with_context(|| format!("Failed to open blob file {}", path.display()))?; let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| {
let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
format!( format!(
"Failed to mmap blob file {} ({} bytes)", "Failed to mmap blob file {} ({} bytes)",
path.display(), path.display(),
...@@ -876,15 +905,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -876,15 +905,10 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
sst_filter.apply_filter(meta_file); sst_filter.apply_filter(meta_file);
} }
// Sync the directory to ensure the new directory entries (file name → inode mappings) // Note: the file *contents* were made durable by the `sync_data()` calls above. The
// are durable before we update CURRENT. Without this, a crash could leave CURRENT pointing // directory entries (file name → inode mappings) are made durable by the single directory
// to files whose directory entries were lost even though their data was flushed. // fsync inside `commit_current` below, which also commits the CURRENT rename. See
// Skipped on Windows: `sync_data` calls `FlushFileBuffers`, which needs a handle with // `commit_current` for why one trailing fsync is sufficient.
// write access and always fails with ERROR_ACCESS_DENIED on the read-only handle that
// `File::open` returns for a directory. Other storage engines (LevelDB, SQLite) also
// skip directory syncing on Windows.
#[cfg(not(windows))]
File::open(&self.path)?.sync_data()?;
drop(sync_span); drop(sync_span);
let new_meta_info = new_meta_files let new_meta_info = new_meta_files
...@@ -1013,18 +1037,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -1013,18 +1037,13 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
for seq in blob_seq_numbers_to_delete.iter() { for seq in blob_seq_numbers_to_delete.iter() {
buf.write_u32::<BE>(*seq)?; buf.write_u32::<BE>(*seq)?;
} }
let mut file = File::create(self.path.join(format!("{seq:08}.del")))?; let del_path = self.path.join(format!("{seq:08}.del"));
let mut file = File::create(&del_path)?;
file.write_all(&buf)?; file.write_all(&buf)?;
file.sync_data()?; file.sync_data()?;
} }
let mut current_file = OpenOptions::new() commit_current(&self.path, seq).context("Committing CURRENT file failed")?;
.write(true)
.truncate(false)
.read(false)
.open(self.path.join("CURRENT"))?;
current_file.write_u32::<BE>(seq)?;
current_file.sync_data()?;
// ── Point of no return ────────────────────────────────────────── // ── Point of no return ──────────────────────────────────────────
// //
......
use std::{ use std::{
cmp::Ordering, cmp::Ordering,
fmt::Display, fmt::Display,
fs::File,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::OnceLock, sync::OnceLock,
}; };
...@@ -9,6 +8,7 @@ use std::{ ...@@ -9,6 +8,7 @@ use std::{
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use bitfield::bitfield; use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
use fs_err::File;
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use smallvec::SmallVec; use smallvec::SmallVec;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be};
...@@ -267,8 +267,8 @@ impl MetaFile { ...@@ -267,8 +267,8 @@ impl MetaFile {
} }
fn open_internal(db_path: PathBuf, sequence_number: u32, path: &Path) -> Result<Self> { fn open_internal(db_path: PathBuf, sequence_number: u32, path: &Path) -> Result<Self> {
let file = File::open(path).context("Failed to open meta file")?; let file = File::open(path)?;
let mmap = unsafe { MmapOptions::new().map(&file) }.context("Failed to mmap")?; let mmap = unsafe { MmapOptions::new().map(file.file()) }.context("Failed to mmap")?;
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::Random) mmap.advise(memmap2::Advice::Random)
.context("Failed to advise mmap")?; .context("Failed to advise mmap")?;
......
use std::{ use std::{
fs::File,
io::{self, BufWriter, Write}, io::{self, BufWriter, Write},
path::Path, path::Path,
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use byteorder::{BE, WriteBytesExt}; use byteorder::{BE, WriteBytesExt};
use fs_err::File;
use qfilter::Filter; use qfilter::Filter;
use zerocopy::IntoBytes; use zerocopy::IntoBytes;
......
use std::{ use std::{
cmp::Ordering, cmp::Ordering,
fs::File,
hash::BuildHasherDefault, hash::BuildHasherDefault,
path::Path, path::Path,
rc::Rc, rc::Rc,
...@@ -11,6 +10,7 @@ use std::{ ...@@ -11,6 +10,7 @@ use std::{
}; };
use anyhow::{Context, Result, bail, ensure}; use anyhow::{Context, Result, bail, ensure};
use fs_err::File;
use memmap2::Mmap; use memmap2::Mmap;
use quick_cache::{Lifecycle, sync::GuardResult}; use quick_cache::{Lifecycle, sync::GuardResult};
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
...@@ -219,9 +219,8 @@ impl StaticSortedFile { ...@@ -219,9 +219,8 @@ impl StaticSortedFile {
pub fn open(db_path: &Path, meta: StaticSortedFileMetaData) -> Result<Self> { pub fn open(db_path: &Path, meta: StaticSortedFileMetaData) -> Result<Self> {
let filename = format!("{:08}.sst", meta.sequence_number); let filename = format!("{:08}.sst", meta.sequence_number);
let path = db_path.join(&filename); let path = db_path.join(&filename);
let file = File::open(&path) let file = File::open(&path)?;
.with_context(|| format!("Failed to open SST file {}", path.display()))?; let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| {
let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
format!( format!(
"Failed to mmap SST file {} ({} bytes)", "Failed to mmap SST file {} ({} bytes)",
path.display(), path.display(),
...@@ -781,9 +780,8 @@ impl StaticSortedFileIter { ...@@ -781,9 +780,8 @@ impl StaticSortedFileIter {
pub fn open(db_path: &Path, meta: StaticSortedFileMetaData) -> Result<Self> { pub fn open(db_path: &Path, meta: StaticSortedFileMetaData) -> Result<Self> {
let filename = format!("{:08}.sst", meta.sequence_number); let filename = format!("{:08}.sst", meta.sequence_number);
let path = db_path.join(&filename); let path = db_path.join(&filename);
let file = File::open(&path) let file = File::open(&path)?;
.with_context(|| format!("Failed to open SST file {}", path.display()))?; let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| {
let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
format!( format!(
"Failed to mmap SST file {} ({} bytes)", "Failed to mmap SST file {} ({} bytes)",
path.display(), path.display(),
......
use std::{ use std::{
borrow::Cow, borrow::Cow,
collections::VecDeque, collections::VecDeque,
fs::File,
io::{BufWriter, Write}, io::{BufWriter, Write},
path::Path, path::Path,
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use byteorder::{BE, ByteOrder, WriteBytesExt}; use byteorder::{BE, ByteOrder, WriteBytesExt};
use fs_err::File;
use crate::{ use crate::{
compression::{checksum_block, compress_into_buffer}, compression::{checksum_block, compress_into_buffer},
......
...@@ -2192,3 +2192,40 @@ fn compaction_preserves_active_blob() -> Result<()> { ...@@ -2192,3 +2192,40 @@ fn compaction_preserves_active_blob() -> Result<()> {
db.shutdown()?; db.shutdown()?;
Ok(()) Ok(())
} }
/// A `CURRENT.next` file left behind by a crash mid-`commit_current` (before the rename onto
/// `CURRENT` completed) must not break opening the database: it should be ignored/cleaned up and
/// the last committed `CURRENT` should remain authoritative.
#[test]
fn stale_current_next_is_recovered() -> Result<()> {
use crate::parallel_scheduler::SerialScheduler;
let tempdir = tempfile::tempdir()?;
let path = tempdir.path();
// Commit a value so CURRENT points at a real sequence number.
{
let db = TurboPersistence::<SerialScheduler, 1>::open(path.to_path_buf())?;
let batch = db.write_batch()?;
batch.put(0, vec![1u8], vec![42u8].into())?;
db.commit_write_batch(batch)?;
db.shutdown()?;
}
// Simulate a crash partway through a later CURRENT update: a stray, even garbage-length,
// CURRENT.next is present on disk while CURRENT itself is untouched.
fs::write(path.join("CURRENT.next"), b"\xAA")?;
// Opening must succeed, remove the stale temp file, and still read the committed value.
{
let db = TurboPersistence::<SerialScheduler, 1>::open(path.to_path_buf())?;
assert_eq!(db.get(0, &vec![1u8])?.as_deref(), Some(&[42u8][..]));
db.shutdown()?;
}
assert!(
!path.join("CURRENT.next").exists(),
"stale CURRENT.next should be cleaned up on open"
);
Ok(())
}
use std::{ use std::{
cell::SyncUnsafeCell, cell::SyncUnsafeCell,
fs::File,
io::Write, io::Write,
mem::{replace, take}, mem::{replace, take},
path::PathBuf, path::PathBuf,
...@@ -10,6 +9,7 @@ use std::{ ...@@ -10,6 +9,7 @@ use std::{
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use byteorder::{BE, WriteBytesExt}; use byteorder::{BE, WriteBytesExt};
use either::Either; use either::Either;
use fs_err::File;
use parking_lot::Mutex; use parking_lot::Mutex;
use smallvec::SmallVec; use smallvec::SmallVec;
use thread_local::ThreadLocal; use thread_local::ThreadLocal;
...@@ -460,10 +460,9 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize ...@@ -460,10 +460,9 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
let size = buffer.len() as u64; let size = buffer.len() as u64;
let file = self.db_path.join(format!("{seq:08}.blob")); let file = self.db_path.join(format!("{seq:08}.blob"));
let mut file = File::create(&file).context("Unable to create blob file")?; let mut file = File::create(&file)?;
file.write_all(&buffer) file.write_all(&buffer)?;
.context("Unable to write blob file")?; file.flush()?;
file.flush().context("Unable to flush blob file")?;
Ok(NewFile { seq, file, size }) Ok(NewFile { seq, file, size })
} }
......
...@@ -42,6 +42,7 @@ auto-hash-map = { workspace = true } ...@@ -42,6 +42,7 @@ auto-hash-map = { workspace = true }
bitfield = { workspace = true } bitfield = { workspace = true }
crossbeam-utils = { workspace = true } crossbeam-utils = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]} dashmap = { workspace = true, features = ["raw-api"]}
fs-err = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] } hashbrown = { workspace = true, features = ["raw"] }
indexmap = { workspace = true } indexmap = { workspace = true }
lzzzz = { workspace = true, optional = true } lzzzz = { workspace = true, optional = true }
......
use std::{ use std::{
borrow::Cow, borrow::Cow,
fs::{self, File, read_dir}, io::{BufReader, BufWriter, ErrorKind, Write},
io::{self, BufReader, BufWriter, ErrorKind, Write},
path::Path, path::Path,
}; };
use anyhow::Context; use anyhow::Context;
use fs_err::{self as fs, File, read_dir};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db"; const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db";
...@@ -156,7 +156,7 @@ pub(crate) fn cleanup_db(base_path: &Path) -> anyhow::Result<()> { ...@@ -156,7 +156,7 @@ pub(crate) fn cleanup_db(base_path: &Path) -> anyhow::Result<()> {
}) })
} }
fn cleanup_db_inner(base_path: &Path) -> io::Result<()> { fn cleanup_db_inner(base_path: &Path) -> anyhow::Result<()> {
let Ok(contents) = read_dir(base_path) else { let Ok(contents) = read_dir(base_path) else {
return Ok(()); return Ok(());
}; };
...@@ -164,11 +164,12 @@ fn cleanup_db_inner(base_path: &Path) -> io::Result<()> { ...@@ -164,11 +164,12 @@ fn cleanup_db_inner(base_path: &Path) -> io::Result<()> {
// delete everything except the invalidation marker // delete everything except the invalidation marker
for entry in contents { for entry in contents {
let entry = entry?; let entry = entry?;
let path = entry.path();
if entry.file_name() != INVALIDATION_MARKER { if entry.file_name() != INVALIDATION_MARKER {
if entry.file_type()?.is_dir() { if entry.file_type()?.is_dir() {
fs::remove_dir_all(entry.path())?; fs::remove_dir_all(&path)?;
} else { } else {
fs::remove_file(entry.path())?; fs::remove_file(&path)?;
} }
} }
} }
......
use std::{ use std::{
env, env,
ffi::{OsStr, OsString}, ffi::{OsStr, OsString},
fs::{DirEntry, read_dir, remove_dir_all, rename},
path::{Path, PathBuf}, path::{Path, PathBuf},
time::Duration, time::Duration,
}; };
use anyhow::Result; use anyhow::Result;
use fs_err::{DirEntry, read_dir, remove_dir_all, rename};
/// Information gathered by `vergen_gitcl` in the top-level binary crate and passed down. This /// Information gathered by `vergen_gitcl` in the top-level binary crate and passed down. This
/// information must be computed in the top-level crate for cargo incremental compilation to work /// information must be computed in the top-level crate for cargo incremental compilation to work
......
...@@ -313,7 +313,7 @@ impl TurboBackingStorage { ...@@ -313,7 +313,7 @@ impl TurboBackingStorage {
let _span = tracing::trace_span!("commit").entered(); let _span = tracing::trace_span!("commit").entered();
// Byte totals are the physical on-disk bytes (post-compression, including .sst / // Byte totals are the physical on-disk bytes (post-compression, including .sst /
// .blob / .meta files) produced and removed by the commit. // .blob / .meta files) produced and removed by the commit.
let stats = batch.commit().context("Unable to commit operations")?; let stats = batch.commit().context("Unable to commit snapshot")?;
snapshot_meta.bytes_written = stats.bytes_written; snapshot_meta.bytes_written = stats.bytes_written;
snapshot_meta.bytes_deleted = stats.bytes_deleted; snapshot_meta.bytes_deleted = stats.bytes_deleted;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment