Unverified Commit 83ae8ad6 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Set DontFork and Unmergeable on all mmap sites in turbo-persistence (#90941)

## Summary

- Add `madvise(MADV_DONTFORK)` and `madvise(MADV_UNMERGEABLE)` to all mmap sites in turbo-persistence
- Extract a shared `advise_mmap_for_persistence` helper into a dedicated `mmap_helper` module

## Why

`read_blob` in `db.rs` already correctly sets `DontFork` and `Unmergeable` on its mmap, but three other mmap sites were missing these flags:

- **`static_sorted_file.rs`** (`open_internal`) — SST file mmaps used for lookups and compaction
- **`meta_file.rs`** (`open_internal`) — meta file mmaps storing AMQF filters and SST metadata
- **`sst_inspect.rs`** (`analyze_sst_file`) — diagnostic tool mmap

`MADV_DONTFORK` prevents mmap regions from being copied into child processes on `fork()`, avoiding unnecessary memory duplication and potential SIGBUS issues if the parent unmaps before the child accesses the page. `MADV_UNMERGEABLE` opts the pages out of KSM (Kernel Same-page Merging), avoiding the overhead of scanning these pages for deduplication since they contain unique compressed data.

Both flags are Linux-only (`#[cfg(target_os = "linux")]`), matching the existing pattern in `read_blob`.

## Implementation

Rather than duplicating `#[cfg(target_os = "linux")]` madvise calls at each mmap site, a shared helper `advise_mmap_for_persistence()` is extracted into `src/mmap_helper.rs`. The function applies both flags on Linux and is a no-op on other platforms. All four mmap sites (`db.rs`, `static_sorted_file.rs`, `meta_file.rs`, `sst_inspect.rs`) now call this single helper.

## Test Plan

- `cargo check -p turbo-persistence` passes
- `cargo fmt` clean
- No behavioral change on non-Linux platforms (flags are gated behind `#[cfg(target_os = "linux")]`)
parent 16206a7f
...@@ -27,7 +27,10 @@ use turbo_persistence::static_sorted_file::{ ...@@ -27,7 +27,10 @@ use turbo_persistence::static_sorted_file::{
KEY_BLOCK_ENTRY_TYPE_DELETED, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_DELETED, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_MEDIUM,
KEY_BLOCK_ENTRY_TYPE_SMALL, KEY_BLOCK_ENTRY_TYPE_SMALL,
}; };
use turbo_persistence::{BLOCK_HEADER_SIZE, checksum_block, meta_file::MetaFile}; use turbo_persistence::{
BLOCK_HEADER_SIZE, checksum_block, meta_file::MetaFile,
mmap_helper::advise_mmap_for_persistence,
};
/// Block size information /// Block size information
#[derive(Default, Debug, Clone)] #[derive(Default, Debug, Clone)]
...@@ -230,6 +233,7 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result<SstStats> { ...@@ -230,6 +233,7 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result<SstStats> {
let file = File::open(&path).with_context(|| format!("Failed to open {}", filename))?; let file = File::open(&path).with_context(|| format!("Failed to open {}", filename))?;
let file_size = file.metadata()?.len(); let file_size = file.metadata()?.len();
let mmap = unsafe { Mmap::map(&file)? }; let mmap = unsafe { Mmap::map(&file)? };
advise_mmap_for_persistence(&mmap)?;
let mut stats = SstStats { let mut stats = SstStats {
block_directory_size: info.block_count as u64 * 4, block_directory_size: info.block_count as u64 * 4,
......
...@@ -34,6 +34,7 @@ use crate::{ ...@@ -34,6 +34,7 @@ use crate::{
merge_iter::MergeIter, merge_iter::MergeIter,
meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange}, meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange},
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
mmap_helper::advise_mmap_for_persistence,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
sst_filter::SstFilter, sst_filter::SstFilter,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile},
...@@ -415,10 +416,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -415,10 +416,7 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
mmap.advise(memmap2::Advice::Sequential)?; mmap.advise(memmap2::Advice::Sequential)?;
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::WillNeed)?; mmap.advise(memmap2::Advice::WillNeed)?;
#[cfg(target_os = "linux")] advise_mmap_for_persistence(&mmap)?;
mmap.advise(memmap2::Advice::DontFork)?;
#[cfg(target_os = "linux")]
mmap.advise(memmap2::Advice::Unmergeable)?;
let mut reader = &mmap[..]; let mut reader = &mmap[..];
let uncompressed_length = reader let uncompressed_length = reader
.read_u32::<BE>() .read_u32::<BE>()
......
...@@ -14,6 +14,7 @@ mod lookup_entry; ...@@ -14,6 +14,7 @@ mod lookup_entry;
mod merge_iter; mod merge_iter;
pub mod meta_file; pub mod meta_file;
mod meta_file_builder; mod meta_file_builder;
pub mod mmap_helper;
mod parallel_scheduler; mod parallel_scheduler;
mod sst_filter; mod sst_filter;
pub mod static_sorted_file; pub mod static_sorted_file;
......
...@@ -19,6 +19,7 @@ use turbo_bincode::turbo_bincode_decode; ...@@ -19,6 +19,7 @@ use turbo_bincode::turbo_bincode_decode;
use crate::{ use crate::{
QueryKey, QueryKey,
lookup_entry::LookupValue, lookup_entry::LookupValue,
mmap_helper::advise_mmap_for_persistence,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
}; };
...@@ -288,6 +289,7 @@ impl MetaFile { ...@@ -288,6 +289,7 @@ impl MetaFile {
.with_context(|| format!("Failed to mmap meta file {}", path.display()))?; .with_context(|| format!("Failed to mmap meta file {}", path.display()))?;
#[cfg(unix)] #[cfg(unix)]
mmap.advise(memmap2::Advice::Random)?; mmap.advise(memmap2::Advice::Random)?;
advise_mmap_for_persistence(&mmap)?;
let file = Self { let file = Self {
db_path, db_path,
sequence_number, sequence_number,
......
/// Apply Linux-specific mmap advice flags that should be set on all persistent mmaps.
///
/// - `DontFork`: prevents mmap regions from being copied into child processes on `fork()`, avoiding
/// unnecessary memory duplication and potential SIGBUS.
/// - `Unmergeable`: opts pages out of KSM (Kernel Same-page Merging) since our data is unique
/// compressed content that won't benefit from deduplication scanning.
#[cfg(target_os = "linux")]
pub fn advise_mmap_for_persistence(mmap: &memmap2::Mmap) -> anyhow::Result<()> {
mmap.advise(memmap2::Advice::DontFork)?;
mmap.advise(memmap2::Advice::Unmergeable)?;
Ok(())
}
#[cfg(not(target_os = "linux"))]
pub fn advise_mmap_for_persistence(_mmap: &memmap2::Mmap) -> anyhow::Result<()> {
Ok(())
}
...@@ -19,6 +19,7 @@ use crate::{ ...@@ -19,6 +19,7 @@ use crate::{
compression::{checksum_block, decompress_into_arc}, compression::{checksum_block, decompress_into_arc},
constants::MAX_INLINE_VALUE_SIZE, constants::MAX_INLINE_VALUE_SIZE,
lookup_entry::{LazyLookupValue, LookupEntry, LookupValue}, lookup_entry::{LazyLookupValue, LookupEntry, LookupValue},
mmap_helper::advise_mmap_for_persistence,
static_sorted_file_builder::BLOCK_HEADER_SIZE, static_sorted_file_builder::BLOCK_HEADER_SIZE,
}; };
...@@ -185,6 +186,7 @@ impl StaticSortedFile { ...@@ -185,6 +186,7 @@ impl StaticSortedFile {
let offset = meta.block_offsets_start(mmap.len()); let offset = meta.block_offsets_start(mmap.len());
let _ = mmap.advise_range(memmap2::Advice::Sequential, offset, mmap.len() - offset); let _ = mmap.advise_range(memmap2::Advice::Sequential, offset, mmap.len() - offset);
} }
advise_mmap_for_persistence(&mmap)?;
let file = Self { let file = Self {
meta, meta,
mmap: Arc::new(mmap), mmap: Arc::new(mmap),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment