Unverified Commit 6f763ecf authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add anyhow::Context to turbo-persistence mmap, file open, and…

Turbopack: add anyhow::Context to turbo-persistence mmap, file open, and decompress operations (#90769)

### What?

Adds `anyhow::Context` (`.context()` / `.with_context()`) to error-prone
operations in the `turbo-persistence` crate: file opens, memory mapping
(`mmap`), and LZ4 decompression.

### Why?

`get` / `batch_get` sometimes fail with a bare "Cannot allocate memory"
error, but the anyhow error chain has no context about *which* file,
block, or operation caused it. This makes debugging difficult since the
raw OS error propagates through bare `?` operators without any
identifying information.

### How?

Split double-`?` expressions (e.g. `Mmap::map(&File::open(&path)?)?`)
into separate statements so each operation gets its own context. Added
`.with_context()` to:

- **`compression.rs`** — `decompress()` call in `decompress_into_arc()`,
reporting compressed and uncompressed sizes
- **`db.rs`** — `File::open`, `Mmap::map`, and `read_u32` in
`read_blob()`, reporting the blob file path and size
- **`static_sorted_file.rs`** — `File::open` and `Mmap::map` in
`open_internal()`, `get_raw_block_slice` and `decompress_into_arc` in
`read_block()`, and block offset reads in `get_raw_block_slice()`,
reporting SST filename and block index
- **`meta_file.rs`** — `options.map()` in `open_internal()`, reporting
the meta file path

An ENOMEM error will now produce a chain like:
```
Unable to open static sorted file 00000042.sst

Caused by:
    Failed to mmap SST file /path/to/00000042.sst (1234567 bytes)

    Caused by:
        Cannot allocate memory (os error 12)
```
parent c2b4c081
...@@ -21,7 +21,13 @@ pub fn decompress_into_arc(uncompressed_length: u32, block: &[u8]) -> Result<Arc ...@@ -21,7 +21,13 @@ pub fn decompress_into_arc(uncompressed_length: u32, block: &[u8]) -> Result<Arc
let mut buffer = unsafe { buffer.assume_init() }; let mut buffer = unsafe { buffer.assume_init() };
// We just created this Arc so refcount is 1; get_mut always succeeds. // We just created this Arc so refcount is 1; get_mut always succeeds.
let decompressed = Arc::get_mut(&mut buffer).expect("Arc refcount should be 1"); let decompressed = Arc::get_mut(&mut buffer).expect("Arc refcount should be 1");
let bytes_written = decompress(block, decompressed)?; let bytes_written = decompress(block, decompressed).with_context(|| {
format!(
"Failed to decompress block ({} bytes compressed, {} bytes uncompressed)",
block.len(),
uncompressed_length
)
})?;
assert_eq!( assert_eq!(
bytes_written, uncompressed_length as usize, bytes_written, uncompressed_length as usize,
"Decompressed length does not match expected length" "Decompressed length does not match expected length"
......
...@@ -402,7 +402,15 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -402,7 +402,15 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
#[tracing::instrument(level = "info", name = "reading database blob", skip_all)] #[tracing::instrument(level = "info", name = "reading database blob", skip_all)]
fn read_blob(&self, seq: u32) -> Result<ArcBytes> { fn read_blob(&self, seq: u32) -> Result<ArcBytes> {
let path = self.path.join(format!("{seq:08}.blob")); let path = self.path.join(format!("{seq:08}.blob"));
let mmap = unsafe { Mmap::map(&File::open(&path)?)? }; let file = File::open(&path)
.with_context(|| format!("Failed to open blob file {}", path.display()))?;
let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
format!(
"Failed to mmap blob file {} ({} bytes)",
path.display(),
file.metadata().map(|m| m.len()).unwrap_or(0)
)
})?;
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::Sequential)?; mmap.advise(memmap2::Advice::Sequential)?;
#[cfg(unix)] #[cfg(unix)]
...@@ -412,7 +420,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -412,7 +420,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
mmap.advise(memmap2::Advice::Unmergeable)?; mmap.advise(memmap2::Advice::Unmergeable)?;
let mut reader = &mmap[..]; let mut reader = &mmap[..];
let uncompressed_length = reader.read_u32::<BE>()?; let uncompressed_length = reader
.read_u32::<BE>()
.context("Failed to read uncompressed length from blob file")?;
let expected_checksum = reader.read_u32::<BE>()?; let expected_checksum = reader.read_u32::<BE>()?;
// Verify checksum on the compressed on-disk data before decompression. // Verify checksum on the compressed on-disk data before decompression.
......
...@@ -284,7 +284,8 @@ impl MetaFile { ...@@ -284,7 +284,8 @@ impl MetaFile {
let file = file.into_inner(); let file = file.into_inner();
let mut options = MmapOptions::new(); let mut options = MmapOptions::new();
options.offset(offset); options.offset(offset);
let mmap = unsafe { options.map(&file)? }; let mmap = unsafe { options.map(&file) }
.with_context(|| format!("Failed to mmap meta file {}", path.display()))?;
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::Random)?; mmap.advise(memmap2::Advice::Random)?;
let file = Self { let file = Self {
......
...@@ -168,7 +168,15 @@ impl StaticSortedFile { ...@@ -168,7 +168,15 @@ impl StaticSortedFile {
meta: StaticSortedFileMetaData, meta: StaticSortedFileMetaData,
sequential: bool, sequential: bool,
) -> Result<Self> { ) -> Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(&path)?)? }; let file = File::open(&path)
.with_context(|| format!("Failed to open SST file {}", path.display()))?;
let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
format!(
"Failed to mmap SST file {} ({} bytes)",
path.display(),
file.metadata().map(|m| m.len()).unwrap_or(0)
)
})?;
#[cfg(unix)] #[cfg(unix)]
if sequential { if sequential {
mmap.advise(memmap2::Advice::Sequential)?; mmap.advise(memmap2::Advice::Sequential)?;
...@@ -465,7 +473,12 @@ impl StaticSortedFile { ...@@ -465,7 +473,12 @@ impl StaticSortedFile {
#[tracing::instrument(level = "info", name = "reading database block", skip_all)] #[tracing::instrument(level = "info", name = "reading database block", skip_all)]
fn read_block(&self, block_index: u16) -> Result<ArcBytes> { fn read_block(&self, block_index: u16) -> Result<ArcBytes> {
let (uncompressed_length, expected_checksum, block) = let (uncompressed_length, expected_checksum, block) =
self.get_raw_block_slice(block_index)?; self.get_raw_block_slice(block_index).with_context(|| {
format!(
"Failed to read raw block {} from {:08}.sst",
block_index, self.meta.sequence_number
)
})?;
// Verify checksum on the raw on-disk data before decompression. // Verify checksum on the raw on-disk data before decompression.
self.verify_checksum(block, expected_checksum, block_index)?; self.verify_checksum(block, expected_checksum, block_index)?;
...@@ -486,7 +499,12 @@ impl StaticSortedFile { ...@@ -486,7 +499,12 @@ impl StaticSortedFile {
block.len(), block.len(),
); );
let buffer = decompress_into_arc(uncompressed_length, block)?; let buffer = decompress_into_arc(uncompressed_length, block).with_context(|| {
format!(
"Failed to decompress block {} from {:08}.sst ({} bytes uncompressed)",
block_index, self.meta.sequence_number, uncompressed_length
)
})?;
Ok(ArcBytes::from(buffer)) Ok(ArcBytes::from(buffer))
} }
...@@ -537,9 +555,23 @@ impl StaticSortedFile { ...@@ -537,9 +555,23 @@ impl StaticSortedFile {
let block_start = if block_index == 0 { let block_start = if block_index == 0 {
0 0
} else { } else {
(&self.mmap[offset - 4..offset]).read_u32::<BE>()? as usize (&self.mmap[offset - 4..offset])
.read_u32::<BE>()
.with_context(|| {
format!(
"Failed to read block_start offset for block {} in {:08}.sst",
block_index, self.meta.sequence_number
)
})? as usize
}; };
let block_end = (&self.mmap[offset..offset + 4]).read_u32::<BE>()? as usize; let block_end = (&self.mmap[offset..offset + 4])
.read_u32::<BE>()
.with_context(|| {
format!(
"Failed to read block_end offset for block {} in {:08}.sst",
block_index, self.meta.sequence_number
)
})? as usize;
#[cfg(feature = "strict_checks")] #[cfg(feature = "strict_checks")]
if block_end > self.mmap.len() || block_start > self.mmap.len() { if block_end > self.mmap.len() || block_start > self.mmap.len() {
bail!( bail!(
...@@ -552,9 +584,26 @@ impl StaticSortedFile { ...@@ -552,9 +584,26 @@ impl StaticSortedFile {
self.meta.block_offsets_start(self.mmap.len()), self.meta.block_offsets_start(self.mmap.len()),
); );
} }
let uncompressed_length = let uncompressed_length = u32::from_be_bytes(
u32::from_be_bytes(self.mmap[block_start..block_start + 4].try_into()?); self.mmap[block_start..block_start + 4]
let checksum = u32::from_be_bytes(self.mmap[block_start + 4..block_start + 8].try_into()?); .try_into()
.with_context(|| {
format!(
"Failed to read uncompressed_length from block {} header in {:08}.sst",
block_index, self.meta.sequence_number
)
})?,
);
let checksum = u32::from_be_bytes(
self.mmap[block_start + 4..block_start + 8]
.try_into()
.with_context(|| {
format!(
"Failed to read checksum from block {} header in {:08}.sst",
block_index, self.meta.sequence_number
)
})?,
);
let block = &self.mmap[block_start + BLOCK_HEADER_SIZE..block_end]; let block = &self.mmap[block_start + BLOCK_HEADER_SIZE..block_end];
Ok((uncompressed_length, checksum, block)) Ok((uncompressed_length, checksum, block))
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment