Unverified Commit 0f089d5f authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

turbo-persistence: fix mmap page alignment and improve error context in…

turbo-persistence: fix mmap page alignment and improve error context in MetaFile::open_internal (#91640)

### What?

Two changes to `turbopack/crates/turbo-persistence/src/meta_file.rs` and
`mmap_helper.rs`:

1. **Bug fix: mmap page alignment** — `MmapOptions::offset()` requires
the offset to be page-aligned (typically 4096 bytes), but the code was
passing `file.stream_position()` (the byte position after reading the
variable-length header), which is not guaranteed to be page-aligned.
This would cause `mmap` to fail or behave incorrectly on some inputs.

2. **Improved error context** — Add `.context("...")` to all `?`
operators in `open_internal` (except read calls) and inside
`advise_mmap_for_persistence`, so errors carry actionable context. The
filename is already attached by the `MetaFile::open` wrapper, so no
filename is repeated in these messages.

### Why?

The mmap offset bug could cause silent failures or panics when the meta
file header length is not a multiple of the OS page size. The missing
error context made it hard to diagnose failures in production.

### How?

**mmap fix (`meta_file.rs`):**
- Remove `options.offset(offset)` — map the entire file from byte 0
instead.
- Add a new `amqf_data_start: usize` field to `MetaFile` storing the
header-end byte offset.
- `amqf_data()` now returns `&self.mmap[self.amqf_data_start..]`,
exposing only the AMQF data region. All existing callers that index into
`amqf_data()` continue to work unchanged since their offsets are already
relative to the start of AMQF data.

**Error context (`meta_file.rs`):**
- `File::open(path)?` → `.context("Failed to open meta file")`
- `file.stream_position()?` → `.context("Failed to get stream
position")`
- `unsafe { options.map(&file) }.with_context(|| format!(...))` →
`.context("Failed to mmap")` (drop the closure and filename, already
provided by the outer wrapper)
- `mmap.advise(Advice::Random)?` → `.context("Failed to advise mmap")`

**Error context (`mmap_helper.rs`):**
- `Advice::DontFork` → `.context("Failed to advise mmap DontFork")`
- `Advice::Unmergeable` → `.context("Failed to advise mmap
Unmergeable")`
Co-authored-by: 's avatarTobias Koppers <sokra@users.noreply.github.com>
Co-authored-by: 's avatarClaude <noreply@anthropic.com>
parent bdb2f2ce
...@@ -229,6 +229,8 @@ pub struct MetaFile { ...@@ -229,6 +229,8 @@ pub struct MetaFile {
/// The offset of the end of the "used keys" AMQF data in the the meta file relative to the end /// The offset of the end of the "used keys" AMQF data in the the meta file relative to the end
/// of the header. /// of the header.
end_of_used_keys_amqf_data_offset: u32, end_of_used_keys_amqf_data_offset: u32,
/// Byte offset of the start of AMQF data within the mmap (= end of the header).
amqf_data_start: usize,
/// The memory mapped file. /// The memory mapped file.
mmap: Mmap, mmap: Mmap,
} }
...@@ -244,7 +246,7 @@ impl MetaFile { ...@@ -244,7 +246,7 @@ impl MetaFile {
} }
fn open_internal(db_path: PathBuf, sequence_number: u32, path: &Path) -> Result<Self> { fn open_internal(db_path: PathBuf, sequence_number: u32, path: &Path) -> Result<Self> {
let mut file = BufReader::new(File::open(path)?); let mut file = BufReader::new(File::open(path).context("Failed to open meta file")?);
let magic = file.read_u32::<BE>()?; let magic = file.read_u32::<BE>()?;
if magic != 0xFE4ADA4A { if magic != 0xFE4ADA4A {
bail!("Invalid magic number"); bail!("Invalid magic number");
...@@ -281,14 +283,14 @@ impl MetaFile { ...@@ -281,14 +283,14 @@ impl MetaFile {
let start_of_used_keys_amqf_data_offset = start_of_amqf_data_offset; let start_of_used_keys_amqf_data_offset = start_of_amqf_data_offset;
let end_of_used_keys_amqf_data_offset = file.read_u32::<BE>()?; let end_of_used_keys_amqf_data_offset = file.read_u32::<BE>()?;
let offset = file.stream_position()?; let offset = file
.stream_position()
.context("Failed to get stream position")?;
let file = file.into_inner(); let file = file.into_inner();
let mut options = MmapOptions::new(); let mmap = unsafe { MmapOptions::new().map(&file) }.context("Failed to mmap")?;
options.offset(offset);
let mmap = unsafe { options.map(&file) }
.with_context(|| format!("Failed to mmap meta file {}", path.display()))?;
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::Random)?; mmap.advise(memmap2::Advice::Random)
.context("Failed to advise mmap")?;
advise_mmap_for_persistence(&mmap)?; advise_mmap_for_persistence(&mmap)?;
let file = Self { let file = Self {
db_path, db_path,
...@@ -299,6 +301,7 @@ impl MetaFile { ...@@ -299,6 +301,7 @@ impl MetaFile {
obsolete_sst_files, obsolete_sst_files,
start_of_used_keys_amqf_data_offset, start_of_used_keys_amqf_data_offset,
end_of_used_keys_amqf_data_offset, end_of_used_keys_amqf_data_offset,
amqf_data_start: offset as usize,
mmap, mmap,
}; };
Ok(file) Ok(file)
...@@ -336,7 +339,7 @@ impl MetaFile { ...@@ -336,7 +339,7 @@ impl MetaFile {
} }
pub fn amqf_data(&self) -> &[u8] { pub fn amqf_data(&self) -> &[u8] {
&self.mmap &self.mmap[self.amqf_data_start..]
} }
pub fn deserialize_used_key_hashes_amqf(&self) -> Result<Option<qfilter::Filter>> { pub fn deserialize_used_key_hashes_amqf(&self) -> Result<Option<qfilter::Filter>> {
......
use anyhow::Context;
/// Apply Linux-specific mmap advice flags that should be set on all persistent mmaps. /// Apply Linux-specific mmap advice flags that should be set on all persistent mmaps.
/// ///
/// - `DontFork`: prevents mmap regions from being copied into child processes on `fork()`, avoiding /// - `DontFork`: prevents mmap regions from being copied into child processes on `fork()`, avoiding
...@@ -6,8 +8,10 @@ ...@@ -6,8 +8,10 @@
/// compressed content that won't benefit from deduplication scanning. /// compressed content that won't benefit from deduplication scanning.
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
pub fn advise_mmap_for_persistence(mmap: &memmap2::Mmap) -> anyhow::Result<()> { pub fn advise_mmap_for_persistence(mmap: &memmap2::Mmap) -> anyhow::Result<()> {
mmap.advise(memmap2::Advice::DontFork)?; mmap.advise(memmap2::Advice::DontFork)
mmap.advise(memmap2::Advice::Unmergeable)?; .context("Failed to advise mmap DontFork")?;
mmap.advise(memmap2::Advice::Unmergeable)
.context("Failed to advise mmap Unmergeable")?;
Ok(()) Ok(())
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment