Unverified Commit 3bf71ee3 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

test: run cargo unit tests with miri as well (#97535)

### What?

Adds a CI job that runs selected Rust unit tests under
[Miri](https://github.com/rust-lang/miri) to detect undefined behavior
in unsafe code.

The job currently covers the low-level crates that are compatible with
Miri:

- `turbo-persistence`
- `turbo-rcstr`
- `turbo-tasks-malloc`

It also makes those crates Miri-compatible by:

- using provenance-free tagged-pointer operations in `turbo-rcstr`;
- disabling mimalloc and native compression under Miri;
- making mmap support a default-enabled Cargo feature that is disabled
for Miri and wasm;
- running persistence through file I/O when mmap is disabled;
- skipping only tests measured to exceed the Miri time budget;
- removing leaked test arenas from the analyzer predicate tests.

### Why?

Miri can detect invalid memory access and other undefined behavior that
normal Rust tests may not expose. Running it in CI gives low-level
unsafe code an additional correctness check.

The job uses an explicit package allowlist because Turbo Tasks builds
its generated registry from linker sections, and Miri does not support
the linker-defined section symbols required by that registry.

### How?

- Installs the `miri` Rust component in CI and the development
container.
- Adds a dedicated `test-cargo-unit-miri` task and reusable workflow
configuration.
- Makes `memmap2` optional behind the default `mmap` feature and
disables that feature in Miri CI and the wasm dependency graph.
- Uses structural `cfg(miri)` branches for allocator and compression
paths Miri cannot execute.
- Re-enables three persistence compaction tests after measuring them
successfully under Miri.
- Keeps normal native behavior unchanged and documents measured Miri
exclusions at the affected tests.

Verification included focused normal and Miri tests for persistence,
rcstr, allocation accounting, compression, and the refactored leak-free
predicate cases.

<!-- NEXT_JS_LLM -->


<!-- fleet fb42942f-173a-484c-b4de-4e18354834c6 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent deab0fec
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
// this should match the `rust-toolchain.toml` // this should match the `rust-toolchain.toml`
"version": "nightly-2026-08-20", "version": "nightly-2026-08-20",
"profile": "minimal", "profile": "minimal",
"components": "rustfmt,clippy,rust-analyzer" "components": "rustfmt,clippy,rust-analyzer,miri"
} }
} }
} }
...@@ -300,6 +300,7 @@ jobs: ...@@ -300,6 +300,7 @@ jobs:
with: with:
needsRust: 'yes' needsRust: 'yes'
needsNextest: 'yes' needsNextest: 'yes'
needsRustJsInstall: 'yes'
skipNativeBuild: 'yes' skipNativeBuild: 'yes'
skipInstallBuild: 'yes' skipInstallBuild: 'yes'
afterBuild: pnpm dlx turbo@${TURBO_VERSION} run test-cargo-unit ${TURBO_ARGS} afterBuild: pnpm dlx turbo@${TURBO_VERSION} run test-cargo-unit ${TURBO_ARGS}
...@@ -313,6 +314,23 @@ jobs: ...@@ -313,6 +314,23 @@ jobs:
runs_on_labels: '["ubuntu-latest-16-core-oss"]' runs_on_labels: '["ubuntu-latest-16-core-oss"]'
secrets: inherit secrets: inherit
# Turbo Tasks' link-section registry is unavailable under Miri, so this runs the explicit
# low-level package allowlist in packages/next-swc/package.json instead of the whole workspace.
test-cargo-unit-miri:
name: test cargo unit (miri)
needs: ['changes', 'build-next']
if: ${{ needs.changes.outputs.docs-only == 'false' }}
uses: ./.github/workflows/build_reusable.yml
with:
needsRust: 'yes'
needsNextest: 'yes'
skipNativeBuild: 'yes'
skipInstallBuild: 'yes'
afterBuild: pnpm dlx turbo@${TURBO_VERSION} run test-cargo-unit-miri ${TURBO_ARGS}
stepName: 'test-cargo-unit-miri'
secrets: inherit
test-bench: test-bench:
name: test cargo benches name: test cargo benches
needs: ['optimize-ci', 'changes', 'build-next'] needs: ['optimize-ci', 'changes', 'build-next']
......
...@@ -36,6 +36,10 @@ on: ...@@ -36,6 +36,10 @@ on:
required: false required: false
description: 'if nextest rust dep is needed' description: 'if nextest rust dep is needed'
type: string type: string
needsRustJsInstall:
required: false
description: 'if Rust JS dependencies should be installed'
type: string
rustBuildProfile: rustBuildProfile:
required: false required: false
description: 'The profile to use for the build, default is `release-with-assertions`, also supports `` for debug and `release` for normal release' description: 'The profile to use for the build, default is `release-with-assertions`, also supports `` for debug and `release` for normal release'
...@@ -360,7 +364,7 @@ jobs: ...@@ -360,7 +364,7 @@ jobs:
# risk caching an incomplete store # risk caching an incomplete store
# If keep conditions in sync breaks, we can split into restore and save # If keep conditions in sync breaks, we can split into restore and save
# steps where saving runs based on the outcome of the install step # steps where saving runs based on the outcome of the install step
if: ${{ inputs.skipInstallBuild != 'yes' || inputs.needsNextest == 'yes' }} if: ${{ inputs.skipInstallBuild != 'yes' || inputs.needsRustJsInstall == 'yes' }}
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
timeout-minutes: 5 timeout-minutes: 5
id: cache-pnpm-store id: cache-pnpm-store
...@@ -371,10 +375,10 @@ jobs: ...@@ -371,10 +375,10 @@ jobs:
- run: pnpm install - run: pnpm install
# condititions must be superset of cache step, otherwise we risk running install without a cache and then caching the incomplete store. # condititions must be superset of cache step, otherwise we risk running install without a cache and then caching the incomplete store.
if: ${{ inputs.skipInstallBuild != 'yes' || inputs.needsNextest == 'yes' }} if: ${{ inputs.skipInstallBuild != 'yes' || inputs.needsRustJsInstall == 'yes' }}
- name: Install node-file-trace test dependencies - name: Install node-file-trace test dependencies
if: ${{ inputs.needsNextest == 'yes' }} if: ${{ inputs.needsRustJsInstall == 'yes' }}
working-directory: turbopack/crates/turbopack-tracing/tests/node-file-trace working-directory: turbopack/crates/turbopack-tracing/tests/node-file-trace
run: pnpm install --recursive run: pnpm install --recursive
......
...@@ -19,7 +19,8 @@ ...@@ -19,7 +19,8 @@
"rust-check-doc": "RUSTDOCFLAGS='-Zunstable-options --output-format=json' cargo doc --no-deps --workspace", "rust-check-doc": "RUSTDOCFLAGS='-Zunstable-options --output-format=json' cargo doc --no-deps --workspace",
"rust-check-fmt": "cd ../..; cargo fmt -- --check", "rust-check-fmt": "cd ../..; cargo fmt -- --check",
"rust-check-napi": "cargo check -p next-napi-bindings", "rust-check-napi": "cargo check -p next-napi-bindings",
"test-cargo-unit": "cargo nextest run --workspace --exclude next-napi-bindings --exclude turbo-tasks-macros --cargo-profile release-with-assertions --no-fail-fast && cargo test --workspace --doc --profile=release-with-assertions --no-fail-fast" "test-cargo-unit": "cargo nextest run --workspace --exclude next-napi-bindings --exclude turbo-tasks-macros --cargo-profile release-with-assertions --no-fail-fast && cargo test --workspace --doc --profile=release-with-assertions --no-fail-fast",
"test-cargo-unit-miri": "MIRIFLAGS='-Zmiri-disable-isolation -Zmiri-permissive-provenance' cargo miri nextest run -p turbo-persistence -p turbo-rcstr -p turbo-tasks-malloc --no-default-features --features turbo-tasks-malloc/custom_allocator --no-fail-fast"
}, },
"napi": { "napi": {
"binaryName": "next-swc", "binaryName": "next-swc",
......
...@@ -91,5 +91,14 @@ ...@@ -91,5 +91,14 @@
"!../../turbopack/crates/turbopack-tests/tests/execution/**/output/*", "!../../turbopack/crates/turbopack-tests/tests/execution/**/output/*",
], ],
}, },
"test-cargo-unit-miri": {
"dependsOn": ["rust-fingerprint"],
"inputs": [
"../../target/.rust-fingerprint",
"../../crates/*/tests/**",
"../../turbopack/crates/*/tests/**",
"!../../turbopack/crates/turbopack-tests/tests/execution/**/output/*",
],
},
}, },
} }
...@@ -2,5 +2,5 @@ ...@@ -2,5 +2,5 @@
# if you move the file, also update any turbo.json inputs that references it. # if you move the file, also update any turbo.json inputs that references it.
[toolchain] [toolchain]
channel = "nightly-2026-08-20" channel = "nightly-2026-08-20"
components = ["rustfmt", "clippy", "rust-analyzer"] components = ["rustfmt", "clippy", "rust-analyzer", "miri"]
profile = "minimal" profile = "minimal"
...@@ -5,7 +5,8 @@ edition = "2024" ...@@ -5,7 +5,8 @@ edition = "2024"
license = "MIT" license = "MIT"
[features] [features]
default = [] default = ["mmap"]
mmap = ["dep:memmap2"]
verify_sst_content = [] verify_sst_content = []
strict_checks = [] strict_checks = []
stats = ["quick_cache/stats"] stats = ["quick_cache/stats"]
...@@ -23,7 +24,7 @@ either = { workspace = true } ...@@ -23,7 +24,7 @@ either = { workspace = true }
fs-err = { workspace = true } fs-err = { workspace = true }
jiff = { version = "0.2.10", features = ["serde"] } jiff = { version = "0.2.10", features = ["serde"] }
lz4_flex = { workspace = true } lz4_flex = { workspace = true }
memmap2 = "0.9.5" memmap2 = { version = "0.9.5", optional = true }
nohash-hasher = { workspace = true } nohash-hasher = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
# See https://github.com/vercel/next.js/issues/91708 for why we need the legacy_x86_64_support feature # See https://github.com/vercel/next.js/issues/91708 for why we need the legacy_x86_64_support feature
...@@ -51,6 +52,7 @@ turbo-tasks-malloc = { workspace = true, features = ["custom_allocator"] } ...@@ -51,6 +52,7 @@ turbo-tasks-malloc = { workspace = true, features = ["custom_allocator"] }
[[bin]] [[bin]]
name = "sst_inspect" name = "sst_inspect"
path = "src/bin/sst_inspect.rs" path = "src/bin/sst_inspect.rs"
required-features = ["mmap"]
[lints] [lints]
workspace = true workspace = true
...@@ -58,3 +60,4 @@ workspace = true ...@@ -58,3 +60,4 @@ workspace = true
[[bench]] [[bench]]
name = "mod" name = "mod"
harness = false harness = false
required-features = ["mmap"]
#![cfg(feature = "mmap")]
use std::{cell::UnsafeCell, path::Path, sync::LazyLock, time::Duration}; use std::{cell::UnsafeCell, path::Path, sync::LazyLock, time::Duration};
use anyhow::Result; use anyhow::Result;
......
...@@ -6,6 +6,7 @@ use std::{ ...@@ -6,6 +6,7 @@ use std::{
sync::Arc, sync::Arc,
}; };
#[cfg(feature = "mmap")]
use memmap2::Mmap; use memmap2::Mmap;
use crate::{ use crate::{
...@@ -24,6 +25,7 @@ enum Repr { ...@@ -24,6 +25,7 @@ enum Repr {
data: *const [u8], data: *const [u8],
_backing: Arc<[u8]>, _backing: Arc<[u8]>,
}, },
#[cfg(feature = "mmap")]
Mmap { Mmap {
data: *const [u8], data: *const [u8],
_backing: Arc<Mmap>, _backing: Arc<Mmap>,
...@@ -45,6 +47,7 @@ impl ArcBytes { ...@@ -45,6 +47,7 @@ impl ArcBytes {
fn backing_bytes(&self) -> Option<&[u8]> { fn backing_bytes(&self) -> Option<&[u8]> {
match &self.repr { match &self.repr {
Repr::Arc { _backing, .. } => Some(_backing), Repr::Arc { _backing, .. } => Some(_backing),
#[cfg(feature = "mmap")]
Repr::Mmap { _backing, .. } => Some(_backing), Repr::Mmap { _backing, .. } => Some(_backing),
Repr::Inline { .. } => None, Repr::Inline { .. } => None,
} }
...@@ -78,7 +81,9 @@ impl Deref for ArcBytes { ...@@ -78,7 +81,9 @@ impl Deref for ArcBytes {
match &self.repr { match &self.repr {
// SAFETY: `data` points into the backing held by the same variant, which keeps it // SAFETY: `data` points into the backing held by the same variant, which keeps it
// alive for as long as `self`. // alive for as long as `self`.
Repr::Arc { data, .. } | Repr::Mmap { data, .. } => unsafe { &**data }, Repr::Arc { data, .. } => unsafe { &**data },
#[cfg(feature = "mmap")]
Repr::Mmap { data, .. } => unsafe { &**data },
// Borrowed from `self`, so this is recomputed after a move rather than stored. // Borrowed from `self`, so this is recomputed after a move rather than stored.
Repr::Inline { buf, len } => &buf[..*len as usize], Repr::Inline { buf, len } => &buf[..*len as usize],
} }
...@@ -114,7 +119,11 @@ impl Eq for ArcBytes {} ...@@ -114,7 +119,11 @@ impl Eq for ArcBytes {}
impl ArcBytes { impl ArcBytes {
/// Returns `true` if this `ArcBytes` is backed by a memory-mapped file. /// Returns `true` if this `ArcBytes` is backed by a memory-mapped file.
pub fn is_mmap_backed(&self) -> bool { pub fn is_mmap_backed(&self) -> bool {
matches!(self.repr, Repr::Mmap { .. }) #[cfg(feature = "mmap")]
return matches!(self.repr, Repr::Mmap { .. });
#[cfg(not(feature = "mmap"))]
false
} }
/// Returns `true` if the backing `Arc` allocation is shared (i.e., there /// Returns `true` if the backing `Arc` allocation is shared (i.e., there
...@@ -124,12 +133,15 @@ impl ArcBytes { ...@@ -124,12 +133,15 @@ impl ArcBytes {
pub fn is_shared_arc(&self) -> bool { pub fn is_shared_arc(&self) -> bool {
match &self.repr { match &self.repr {
Repr::Arc { _backing, .. } => Arc::strong_count(_backing) > 1, Repr::Arc { _backing, .. } => Arc::strong_count(_backing) > 1,
Repr::Mmap { .. } | Repr::Inline { .. } => false, #[cfg(feature = "mmap")]
Repr::Mmap { .. } => false,
Repr::Inline { .. } => false,
} }
} }
} }
impl SharedBytes for ArcBytes { impl SharedBytes for ArcBytes {
#[cfg(feature = "mmap")]
type MmapHandle = Arc<Mmap>; type MmapHandle = Arc<Mmap>;
fn slice(self, range: Range<usize>) -> Self { fn slice(self, range: Range<usize>) -> Self {
...@@ -142,6 +154,7 @@ impl SharedBytes for ArcBytes { ...@@ -142,6 +154,7 @@ impl SharedBytes for ArcBytes {
Self { Self {
repr: match self.repr { repr: match self.repr {
Repr::Arc { _backing, .. } => Repr::Arc { data, _backing }, Repr::Arc { _backing, .. } => Repr::Arc { data, _backing },
#[cfg(feature = "mmap")]
Repr::Mmap { _backing, .. } => Repr::Mmap { data, _backing }, Repr::Mmap { _backing, .. } => Repr::Mmap { data, _backing },
Repr::Inline { .. } => unreachable!("handled above"), Repr::Inline { .. } => unreachable!("handled above"),
}, },
...@@ -168,6 +181,7 @@ impl SharedBytes for ArcBytes { ...@@ -168,6 +181,7 @@ impl SharedBytes for ArcBytes {
data, data,
_backing: _backing.clone(), _backing: _backing.clone(),
}, },
#[cfg(feature = "mmap")]
Repr::Mmap { _backing, .. } => Repr::Mmap { Repr::Mmap { _backing, .. } => Repr::Mmap {
data, data,
_backing: _backing.clone(), _backing: _backing.clone(),
...@@ -179,6 +193,7 @@ impl SharedBytes for ArcBytes { ...@@ -179,6 +193,7 @@ impl SharedBytes for ArcBytes {
} }
} }
#[cfg(feature = "mmap")]
unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self { unsafe fn from_mmap(mmap: &Arc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!( debug_assert!(
is_subslice_of(subslice, mmap), is_subslice_of(subslice, mmap),
......
...@@ -264,10 +264,14 @@ fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> { ...@@ -264,10 +264,14 @@ fn collect_sst_info(db_path: &Path) -> Result<BTreeMap<u32, Vec<SstInfo>>> {
meta_seqs.sort_unstable(); meta_seqs.sort_unstable();
#[cfg(feature = "mmap")]
let access_mode = turbo_persistence::AccessMode::Mmap;
#[cfg(not(feature = "mmap"))]
let access_mode = turbo_persistence::AccessMode::File;
let mut meta_files: Vec<MetaFile> = meta_seqs let mut meta_files: Vec<MetaFile> = meta_seqs
.iter() .iter()
.map(|&seq| { .map(|&seq| {
MetaFile::open(db_path, seq, None, turbo_persistence::AccessMode::Mmap) MetaFile::open(db_path, seq, None, access_mode)
.with_context(|| format!("Failed to open {seq:08}.meta")) .with_context(|| format!("Failed to open {seq:08}.meta"))
}) })
.collect::<Result<_>>()?; .collect::<Result<_>>()?;
......
...@@ -380,6 +380,7 @@ mod tests { ...@@ -380,6 +380,7 @@ mod tests {
} }
} }
#[cfg(not(miri))]
#[test] #[test]
fn test_exhaustive_replace_versus_naive() { fn test_exhaustive_replace_versus_naive() {
for_all_tiny_int_ranges([0, 1], |a| { for_all_tiny_int_ranges([0, 1], |a| {
...@@ -400,6 +401,7 @@ mod tests { ...@@ -400,6 +401,7 @@ mod tests {
}); });
} }
#[cfg(not(miri))]
#[test] #[test]
fn test_exhaustive_update_versus_naive() { fn test_exhaustive_update_versus_naive() {
for_all_tiny_int_ranges([1, 2], |a| { for_all_tiny_int_ranges([1, 2], |a| {
......
...@@ -588,6 +588,7 @@ mod tests { ...@@ -588,6 +588,7 @@ mod tests {
} }
} }
#[cfg(not(miri))]
#[test] #[test]
fn simulate_compactions() { fn simulate_compactions() {
const KEY_RANGE: u64 = 10000; const KEY_RANGE: u64 = 10000;
......
use std::{cell::RefCell, mem::MaybeUninit, rc::Rc, sync::Arc}; #[cfg(not(miri))]
use std::cell::RefCell;
use std::{mem::MaybeUninit, rc::Rc, sync::Arc};
use anyhow::{Context, Result, ensure}; #[cfg(not(miri))]
use anyhow::Context;
use anyhow::{Result, ensure};
#[cfg(not(miri))]
use lz4_flex::block::{ use lz4_flex::block::{
CompressTable, compress_into_with_table, decompress_into, get_maximum_output_size, CompressTable, compress_into_with_table, decompress_into, get_maximum_output_size,
}; };
...@@ -16,6 +21,7 @@ pub enum Compression { ...@@ -16,6 +21,7 @@ pub enum Compression {
Zstd3 = 1, Zstd3 = 1,
} }
#[cfg(not(miri))]
thread_local! { thread_local! {
/// Reuse lz4_flex's large hash table across independent blocks. Starting large improves /// Reuse lz4_flex's large hash table across independent blocks. Starting large improves
/// compression speed and produces faster-to-decode streams for typical persistence blocks. /// compression speed and produces faster-to-decode streams for typical persistence blocks.
...@@ -37,30 +43,43 @@ fn decompress_block( ...@@ -37,30 +43,43 @@ fn decompress_block(
) -> Result<()> { ) -> Result<()> {
debug_assert!( debug_assert!(
expected_len > 0, expected_len > 0,
"decompress_block called with uncompressed_length=0; uncompressed blocks should use \ "decompress_block called with uncompressed_length=0; uncompressed blocks are served \
zero-copy mmap path" directly from their backing"
); );
let bytes_written = match compression { #[cfg(not(miri))]
Compression::Lz4 => decompress_into(block, dest).map_err(anyhow::Error::from), {
Compression::Zstd3 => ZSTD_DECOMPRESSOR.with_borrow_mut(|decompressor| { let bytes_written = match compression {
decompressor Compression::Lz4 => decompress_into(block, dest).map_err(anyhow::Error::from),
.decompress_to_buffer(block, dest) Compression::Zstd3 => ZSTD_DECOMPRESSOR.with_borrow_mut(|decompressor| {
.map_err(anyhow::Error::from) decompressor
}), .decompress_to_buffer(block, dest)
.map_err(anyhow::Error::from)
}),
}
.with_context(|| {
format!(
"Failed to decompress {compression:?} block ({} bytes compressed, {} bytes \
uncompressed)",
block.len(),
expected_len
)
})?;
ensure!(
bytes_written == expected_len as usize,
"Decompressed length does not match expected length: decompressed {bytes_written} \
bytes, expected {expected_len}"
);
}
#[cfg(miri)]
{
// Compression is skipped under Miri, so Miri-created blob payloads are verbatim.
let _ = compression;
ensure!(
block.len() == expected_len as usize,
"Miri builds skip compression, so a compressed block cannot be read under Miri"
);
dest.copy_from_slice(block);
} }
.with_context(|| {
format!(
"Failed to decompress {compression:?} block ({} bytes compressed, {} bytes \
uncompressed)",
block.len(),
expected_len
)
})?;
ensure!(
bytes_written == expected_len as usize,
"Decompressed length does not match expected length: decompressed {bytes_written} bytes, \
expected {expected_len}"
);
Ok(()) Ok(())
} }
...@@ -108,18 +127,24 @@ pub fn checksum_block(data: &[u8]) -> u32 { ...@@ -108,18 +127,24 @@ pub fn checksum_block(data: &[u8]) -> u32 {
/// Reusable compressor for a stream of blocks using the same family configuration. /// Reusable compressor for a stream of blocks using the same family configuration.
pub(crate) struct Compressor { pub(crate) struct Compressor {
compression: Compression, compression: Compression,
#[cfg(not(miri))]
zstd: Option<zstd::bulk::Compressor<'static>>, zstd: Option<zstd::bulk::Compressor<'static>>,
} }
impl Compressor { impl Compressor {
pub(crate) fn new(compression: Compression) -> Result<Self> { pub(crate) fn new(compression: Compression) -> Result<Self> {
#[cfg(not(miri))]
let zstd = match compression { let zstd = match compression {
Compression::Zstd3 => { Compression::Zstd3 => {
Some(zstd::bulk::Compressor::new(3).context("Failed to create zstd compressor")?) Some(zstd::bulk::Compressor::new(3).context("Failed to create zstd compressor")?)
} }
Compression::Lz4 => None, Compression::Lz4 => None,
}; };
Ok(Self { compression, zstd }) Ok(Self {
compression,
#[cfg(not(miri))]
zstd,
})
} }
/// Compresses `block` into reusable storage, replacing its contents. /// Compresses `block` into reusable storage, replacing its contents.
...@@ -130,6 +155,7 @@ impl Compressor { ...@@ -130,6 +155,7 @@ impl Compressor {
buffer: &mut Vec<u8>, buffer: &mut Vec<u8>,
) -> Result<()> { ) -> Result<()> {
buffer.clear(); buffer.clear();
#[cfg(not(miri))]
match self.compression { match self.compression {
Compression::Lz4 => { Compression::Lz4 => {
let max_output_size = get_maximum_output_size(block.len()); let max_output_size = get_maximum_output_size(block.len());
...@@ -156,6 +182,14 @@ impl Compressor { ...@@ -156,6 +182,14 @@ impl Compressor {
.context("zstd compression failed")?; .context("zstd compression failed")?;
} }
} }
#[cfg(miri)]
{
// Compression is deliberately skipped under Miri. This avoids native Zstd, and using
// the same raw representation for both algorithms keeps blob reads on the matching
// copy path above. The caller's savings check stores SST blocks as uncompressed.
let _ = self.compression;
buffer.extend_from_slice(block);
}
Ok(()) Ok(())
} }
} }
......
...@@ -17,9 +17,11 @@ use anyhow::{Context, Result, bail}; ...@@ -17,9 +17,11 @@ use anyhow::{Context, Result, bail};
use auto_hash_map::AutoSet; use auto_hash_map::AutoSet;
use byteorder::{BE, ReadBytesExt, WriteBytesExt}; use byteorder::{BE, ReadBytesExt, WriteBytesExt};
use dashmap::DashSet; use dashmap::DashSet;
#[cfg(feature = "mmap")]
use either::Either; use either::Either;
use fs_err::{self as fs, File, OpenOptions, ReadDir}; use fs_err::{self as fs, File, OpenOptions, ReadDir};
use jiff::Timestamp; use jiff::Timestamp;
#[cfg(feature = "mmap")]
use memmap2::Mmap; use memmap2::Mmap;
use nohash_hasher::BuildNoHashHasher; use nohash_hasher::BuildNoHashHasher;
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
...@@ -29,8 +31,10 @@ use smallvec::SmallVec; ...@@ -29,8 +31,10 @@ use smallvec::SmallVec;
use tracing::span::EnteredSpan; use tracing::span::EnteredSpan;
pub use crate::compaction::selector::CompactConfig; pub use crate::compaction::selector::CompactConfig;
#[cfg(feature = "mmap")]
use crate::{AccessMode, mmap_helper::advise_mmap_for_persistence};
use crate::{ use crate::{
AccessMode, DbConfig, FamilyKind, QueryKey, DbConfig, FamilyKind, QueryKey,
arc_bytes::ArcBytes, arc_bytes::ArcBytes,
compaction::selector::{Compactable, get_merge_segments}, compaction::selector::{Compactable, get_merge_segments},
compression::{Compression, checksum_block, decompress_into_arc}, compression::{Compression, checksum_block, decompress_into_arc},
...@@ -43,7 +47,6 @@ use crate::{ ...@@ -43,7 +47,6 @@ use crate::{
merge_iter::MergeIter, merge_iter::MergeIter,
meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange}, meta_file::{MetaEntryFlags, MetaFile, MetaLookupResult, StaticSortedFileRange},
meta_file_builder::MetaFileBuilder, meta_file_builder::MetaFileBuilder,
mmap_helper::advise_mmap_for_persistence,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
rc_bytes::RcBytes, rc_bytes::RcBytes,
sst_filter::SstFilter, sst_filter::SstFilter,
...@@ -703,7 +706,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -703,7 +706,9 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
#[tracing::instrument(level = "info", name = "reading database blob", skip_all)] #[tracing::instrument(level = "info", name = "reading database blob", skip_all)]
fn read_blob(&self, seq: u32, compression: Compression) -> Result<ArcBytes> { fn read_blob(&self, seq: u32, compression: Compression) -> Result<ArcBytes> {
let path = self.path.join(format!("{seq:08}.blob")); let path = self.path.join(format!("{seq:08}.blob"));
#[cfg(feature = "mmap")]
let file = File::open(&path)?; let file = File::open(&path)?;
#[cfg(feature = "mmap")]
let data: Either<Mmap, Vec<u8>> = match self.config.access_mode { let data: Either<Mmap, Vec<u8>> = match self.config.access_mode {
AccessMode::Mmap => { AccessMode::Mmap => {
let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| { let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| {
...@@ -722,10 +727,16 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES> ...@@ -722,10 +727,16 @@ impl<S: ParallelScheduler, const FAMILIES: usize> TurboPersistence<S, FAMILIES>
} }
AccessMode::File => Either::Right(fs::read(&path)?), AccessMode::File => Either::Right(fs::read(&path)?),
}; };
#[cfg(feature = "mmap")]
let mut reader: &[u8] = match &data { let mut reader: &[u8] = match &data {
Either::Left(mmap) => mmap, Either::Left(mmap) => mmap,
Either::Right(bytes) => bytes, Either::Right(bytes) => bytes,
}; };
// Without mmap support, read the whole blob into memory.
#[cfg(not(feature = "mmap"))]
let data = fs::read(&path)?;
#[cfg(not(feature = "mmap"))]
let mut reader: &[u8] = &data;
let uncompressed_length = reader let uncompressed_length = reader
.read_u32::<BE>() .read_u32::<BE>()
.context("Failed to read uncompressed length from blob file")?; .context("Failed to read uncompressed length from blob file")?;
......
...@@ -15,6 +15,7 @@ mod lookup_entry; ...@@ -15,6 +15,7 @@ mod lookup_entry;
mod merge_iter; mod merge_iter;
pub mod meta_file; pub mod meta_file;
mod meta_file_builder; mod meta_file_builder;
#[cfg(feature = "mmap")]
pub mod mmap_helper; pub mod mmap_helper;
mod parallel_scheduler; mod parallel_scheduler;
mod rc_bytes; mod rc_bytes;
...@@ -40,6 +41,7 @@ pub use db::{ ...@@ -40,6 +41,7 @@ pub use db::{
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AccessMode { pub enum AccessMode {
/// Memory-map the file and access blocks via the mapped region. /// Memory-map the file and access blocks via the mapped region.
#[cfg(feature = "mmap")]
Mmap, Mmap,
/// Read blocks directly from the file via pread (no mmap). /// Read blocks directly from the file via pread (no mmap).
File, File,
...@@ -78,8 +80,32 @@ pub struct DbConfig<const FAMILIES: usize> { ...@@ -78,8 +80,32 @@ pub struct DbConfig<const FAMILIES: usize> {
pub access_mode: AccessMode, pub access_mode: AccessMode,
} }
/// Reads the `TURBO_PERSISTENCE_MMAP` env var (cached). Returns `AccessMode::File` when the var /// Returns the default access mode for this execution environment.
/// is set to `"0"`, `AccessMode::Mmap` otherwise. ///
/// Builds without mmap support always use file I/O. Builds with mmap support honor
/// `TURBO_PERSISTENCE_MMAP=0`; mmap remains the default otherwise.
fn default_access_mode() -> AccessMode {
#[cfg(not(feature = "mmap"))]
return AccessMode::File;
#[cfg(feature = "mmap")]
access_mode_env_var()
}
/// Returns mmap mode when the feature is enabled, and file mode otherwise.
///
/// Call sites that specifically want mmap use this helper because `AccessMode::Mmap` does not
/// exist without the feature; they fall back to file I/O and still exercise surrounding logic.
#[cfg(any(test, feature = "verify_sst_content"))]
pub(crate) fn mmap_access_mode() -> AccessMode {
#[cfg(not(feature = "mmap"))]
return AccessMode::File;
#[cfg(feature = "mmap")]
AccessMode::Mmap
}
#[cfg(feature = "mmap")]
fn access_mode_env_var() -> AccessMode { fn access_mode_env_var() -> AccessMode {
static ACCESS_MODE_ENV: std::sync::LazyLock<AccessMode> = std::sync::LazyLock::new(|| { static ACCESS_MODE_ENV: std::sync::LazyLock<AccessMode> = std::sync::LazyLock::new(|| {
if std::env::var("TURBO_PERSISTENCE_MMAP") if std::env::var("TURBO_PERSISTENCE_MMAP")
...@@ -95,8 +121,7 @@ fn access_mode_env_var() -> AccessMode { ...@@ -95,8 +121,7 @@ fn access_mode_env_var() -> AccessMode {
} }
impl<const FAMILIES: usize> DbConfig<FAMILIES> { impl<const FAMILIES: usize> DbConfig<FAMILIES> {
/// Returns a config with all defaults, reading the `TURBO_PERSISTENCE_MMAP` env var /// Returns a config with all defaults, using the execution environment's default access mode.
/// to determine the access mode.
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
family_configs: [FamilyConfig { family_configs: [FamilyConfig {
...@@ -104,7 +129,7 @@ impl<const FAMILIES: usize> DbConfig<FAMILIES> { ...@@ -104,7 +129,7 @@ impl<const FAMILIES: usize> DbConfig<FAMILIES> {
kind: FamilyKind::SingleValue, kind: FamilyKind::SingleValue,
compression: Compression::Lz4, compression: Compression::Lz4,
}; FAMILIES], }; FAMILIES],
access_mode: access_mode_env_var(), access_mode: default_access_mode(),
} }
} }
} }
......
...@@ -4,21 +4,24 @@ use std::{ ...@@ -4,21 +4,24 @@ use std::{
mem::take, mem::take,
ops::Deref, ops::Deref,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::OnceLock, sync::{Arc, OnceLock},
}; };
use anyhow::{Context, Result, bail, ensure}; use anyhow::{Context, Result, bail, ensure};
use bitfield::bitfield; use bitfield::bitfield;
use byteorder::{BE, ReadBytesExt}; use byteorder::{BE, ReadBytesExt};
#[cfg(feature = "mmap")]
use fs_err::File; use fs_err::File;
#[cfg(feature = "mmap")]
use memmap2::{Mmap, MmapOptions}; use memmap2::{Mmap, MmapOptions};
use smallvec::SmallVec; use smallvec::SmallVec;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be};
#[cfg(feature = "mmap")]
use crate::mmap_helper::advise_mmap_for_persistence;
use crate::{ use crate::{
AccessMode, Compression, FamilyConfig, QueryKey, AccessMode, Compression, FamilyConfig, QueryKey,
lookup_entry::LookupValue, lookup_entry::LookupValue,
mmap_helper::advise_mmap_for_persistence,
static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData},
}; };
...@@ -229,8 +232,15 @@ impl StaticSortedFileRange { ...@@ -229,8 +232,15 @@ impl StaticSortedFileRange {
} }
enum MetaFileBacking { enum MetaFileBacking {
#[cfg(feature = "mmap")]
Mmap(Mmap), Mmap(Mmap),
Bytes(Box<[u8]>), /// Heap bytes for [`AccessMode::File`].
///
/// This is an `Arc<[u8]>` rather than a `Box<[u8]>` so that moving the backing into
/// [`MetaFile`] does not reborrow the bytes: a `Box` is a unique pointer, so the move
/// invalidates the `FilterRef`s that already borrow from it, which Miri reports as undefined
/// behavior under Stacked Borrows. An `Arc` moves its handle without retagging the allocation.
Bytes(Arc<[u8]>),
} }
impl Deref for MetaFileBacking { impl Deref for MetaFileBacking {
...@@ -238,6 +248,7 @@ impl Deref for MetaFileBacking { ...@@ -238,6 +248,7 @@ impl Deref for MetaFileBacking {
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
match self { match self {
#[cfg(feature = "mmap")]
MetaFileBacking::Mmap(mmap) => mmap, MetaFileBacking::Mmap(mmap) => mmap,
MetaFileBacking::Bytes(bytes) => bytes, MetaFileBacking::Bytes(bytes) => bytes,
} }
...@@ -307,6 +318,7 @@ impl MetaFile { ...@@ -307,6 +318,7 @@ impl MetaFile {
access_mode: AccessMode, access_mode: AccessMode,
) -> Result<Self> { ) -> Result<Self> {
let backing = match access_mode { let backing = match access_mode {
#[cfg(feature = "mmap")]
AccessMode::Mmap => { AccessMode::Mmap => {
let file = File::open(path)?; let file = File::open(path)?;
let mmap = unsafe { MmapOptions::new().map(file.file()) } let mmap = unsafe { MmapOptions::new().map(file.file()) }
...@@ -317,7 +329,7 @@ impl MetaFile { ...@@ -317,7 +329,7 @@ impl MetaFile {
advise_mmap_for_persistence(&mmap)?; advise_mmap_for_persistence(&mmap)?;
MetaFileBacking::Mmap(mmap) MetaFileBacking::Mmap(mmap)
} }
AccessMode::File => MetaFileBacking::Bytes(fs_err::read(path)?.into_boxed_slice()), AccessMode::File => MetaFileBacking::Bytes(fs_err::read(path)?.into()),
}; };
// Parse the header from stable backing bytes via ReadBytesExt on &[u8]. // Parse the header from stable backing bytes via ReadBytesExt on &[u8].
let mut reader: &[u8] = &backing; let mut reader: &[u8] = &backing;
......
...@@ -6,6 +6,7 @@ use std::{ ...@@ -6,6 +6,7 @@ use std::{
rc::Rc, rc::Rc,
}; };
#[cfg(feature = "mmap")]
use memmap2::Mmap; use memmap2::Mmap;
use crate::{ use crate::{
...@@ -24,6 +25,7 @@ enum Repr { ...@@ -24,6 +25,7 @@ enum Repr {
data: *const [u8], data: *const [u8],
_backing: Rc<[u8]>, _backing: Rc<[u8]>,
}, },
#[cfg(feature = "mmap")]
Mmap { Mmap {
data: *const [u8], data: *const [u8],
_backing: Rc<Mmap>, _backing: Rc<Mmap>,
...@@ -48,6 +50,7 @@ impl RcBytes { ...@@ -48,6 +50,7 @@ impl RcBytes {
fn backing_bytes(&self) -> Option<&[u8]> { fn backing_bytes(&self) -> Option<&[u8]> {
match &self.repr { match &self.repr {
Repr::Rc { _backing, .. } => Some(_backing), Repr::Rc { _backing, .. } => Some(_backing),
#[cfg(feature = "mmap")]
Repr::Mmap { _backing, .. } => Some(_backing), Repr::Mmap { _backing, .. } => Some(_backing),
Repr::Inline { .. } => None, Repr::Inline { .. } => None,
} }
...@@ -78,7 +81,9 @@ impl Deref for RcBytes { ...@@ -78,7 +81,9 @@ impl Deref for RcBytes {
match &self.repr { match &self.repr {
// SAFETY: `data` points into the backing held by the same variant, which keeps it // SAFETY: `data` points into the backing held by the same variant, which keeps it
// alive for as long as `self`. // alive for as long as `self`.
Repr::Rc { data, .. } | Repr::Mmap { data, .. } => unsafe { &**data }, Repr::Rc { data, .. } => unsafe { &**data },
#[cfg(feature = "mmap")]
Repr::Mmap { data, .. } => unsafe { &**data },
// Borrowed from `self`, so this is recomputed after a move rather than stored. // Borrowed from `self`, so this is recomputed after a move rather than stored.
Repr::Inline { buf, len } => &buf[..*len as usize], Repr::Inline { buf, len } => &buf[..*len as usize],
} }
...@@ -112,6 +117,7 @@ impl Hash for RcBytes { ...@@ -112,6 +117,7 @@ impl Hash for RcBytes {
} }
impl SharedBytes for RcBytes { impl SharedBytes for RcBytes {
#[cfg(feature = "mmap")]
type MmapHandle = Rc<Mmap>; type MmapHandle = Rc<Mmap>;
fn slice(self, range: Range<usize>) -> Self { fn slice(self, range: Range<usize>) -> Self {
...@@ -124,6 +130,7 @@ impl SharedBytes for RcBytes { ...@@ -124,6 +130,7 @@ impl SharedBytes for RcBytes {
Self { Self {
repr: match self.repr { repr: match self.repr {
Repr::Rc { _backing, .. } => Repr::Rc { data, _backing }, Repr::Rc { _backing, .. } => Repr::Rc { data, _backing },
#[cfg(feature = "mmap")]
Repr::Mmap { _backing, .. } => Repr::Mmap { data, _backing }, Repr::Mmap { _backing, .. } => Repr::Mmap { data, _backing },
Repr::Inline { .. } => unreachable!("handled above"), Repr::Inline { .. } => unreachable!("handled above"),
}, },
...@@ -149,6 +156,7 @@ impl SharedBytes for RcBytes { ...@@ -149,6 +156,7 @@ impl SharedBytes for RcBytes {
data, data,
_backing: _backing.clone(), _backing: _backing.clone(),
}, },
#[cfg(feature = "mmap")]
Repr::Mmap { _backing, .. } => Repr::Mmap { Repr::Mmap { _backing, .. } => Repr::Mmap {
data, data,
_backing: _backing.clone(), _backing: _backing.clone(),
...@@ -160,6 +168,7 @@ impl SharedBytes for RcBytes { ...@@ -160,6 +168,7 @@ impl SharedBytes for RcBytes {
} }
} }
#[cfg(feature = "mmap")]
unsafe fn from_mmap(mmap: &Rc<Mmap>, subslice: &[u8]) -> Self { unsafe fn from_mmap(mmap: &Rc<Mmap>, subslice: &[u8]) -> Self {
debug_assert!( debug_assert!(
is_subslice_of(subslice, mmap), is_subslice_of(subslice, mmap),
......
use std::ops::{Deref, Range}; use std::ops::{Deref, Range};
#[cfg(feature = "mmap")]
use memmap2::Mmap; use memmap2::Mmap;
use crate::Compression; use crate::Compression;
...@@ -30,6 +31,7 @@ const _: () = assert!( ...@@ -30,6 +31,7 @@ const _: () = assert!(
/// once and used for both lookup (ArcBytes) and iteration (RcBytes) paths. /// once and used for both lookup (ArcBytes) and iteration (RcBytes) paths.
pub trait SharedBytes: Clone + Deref<Target = [u8]> + Sized { pub trait SharedBytes: Clone + Deref<Target = [u8]> + Sized {
/// The ref-counted handle to the memory-mapped file. /// The ref-counted handle to the memory-mapped file.
#[cfg(feature = "mmap")]
type MmapHandle: Deref<Target = Mmap>; type MmapHandle: Deref<Target = Mmap>;
/// Returns a new instance that points to a sub-range of the current slice. /// Returns a new instance that points to a sub-range of the current slice.
...@@ -50,6 +52,7 @@ pub trait SharedBytes: Clone + Deref<Target = [u8]> + Sized { ...@@ -50,6 +52,7 @@ pub trait SharedBytes: Clone + Deref<Target = [u8]> + Sized {
/// ///
/// The caller must ensure that `subslice` points to memory within the /// The caller must ensure that `subslice` points to memory within the
/// given `mmap`. /// given `mmap`.
#[cfg(feature = "mmap")]
unsafe fn from_mmap(mmap: &Self::MmapHandle, subslice: &[u8]) -> Self; unsafe fn from_mmap(mmap: &Self::MmapHandle, subslice: &[u8]) -> Self;
/// Creates an instance from a decompressed block. /// Creates an instance from a decompressed block.
......
#[cfg(feature = "mmap")]
use std::ops::Range;
use std::{ use std::{
borrow::Cow, borrow::Cow,
cmp::Ordering, cmp::Ordering,
hash::BuildHasherDefault, hash::BuildHasherDefault,
io, io,
ops::Range, marker::PhantomData,
path::Path, path::Path,
rc::Rc, rc::Rc,
sync::{ sync::{
...@@ -14,11 +16,14 @@ use std::{ ...@@ -14,11 +16,14 @@ use std::{
use anyhow::{Context, Result, bail, ensure}; use anyhow::{Context, Result, bail, ensure};
use fs_err::File; use fs_err::File;
#[cfg(feature = "mmap")]
use memmap2::Mmap; use memmap2::Mmap;
use quick_cache::{Lifecycle, sync::GuardResult}; use quick_cache::{Lifecycle, sync::GuardResult};
use rustc_hash::FxHasher; use rustc_hash::FxHasher;
use smallvec::SmallVec; use smallvec::SmallVec;
#[cfg(feature = "mmap")]
use crate::mmap_helper::advise_mmap_for_persistence;
use crate::{ use crate::{
AccessMode, Compression, QueryKey, AccessMode, Compression, QueryKey,
arc_bytes::ArcBytes, arc_bytes::ArcBytes,
...@@ -26,7 +31,6 @@ use crate::{ ...@@ -26,7 +31,6 @@ use crate::{
compression::checksum_block, compression::checksum_block,
constants::MAX_INLINE_VALUE_SIZE, constants::MAX_INLINE_VALUE_SIZE,
lookup_entry::{IterValue, LookupEntry, LookupValue}, lookup_entry::{IterValue, LookupEntry, LookupValue},
mmap_helper::advise_mmap_for_persistence,
rc_bytes::RcBytes, rc_bytes::RcBytes,
shared_bytes::SharedBytes, shared_bytes::SharedBytes,
static_sorted_file_builder::{ static_sorted_file_builder::{
...@@ -315,6 +319,7 @@ impl StaticSortedFileMetaData { ...@@ -315,6 +319,7 @@ impl StaticSortedFileMetaData {
} }
enum StaticSortedFileBacking { enum StaticSortedFileBacking {
#[cfg(feature = "mmap")]
Mmap(Arc<Mmap>), Mmap(Arc<Mmap>),
File { File {
file: Arc<File>, file: Arc<File>,
...@@ -360,6 +365,7 @@ enum IndexEntries { ...@@ -360,6 +365,7 @@ enum IndexEntries {
/// borrow from its own `backing` field, and an `ArcBytes` would bump and drop the `mmap` /// borrow from its own `backing` field, and an `ArcBytes` would bump and drop the `mmap`
/// refcount on every lookup. All readers of a file share that one counter, so the contention /// refcount on every lookup. All readers of a file share that one counter, so the contention
/// scales with reader threads — measured ~3 ns single-threaded but ~70 ns at 8 threads. /// scales with reader threads — measured ~3 ns single-threaded but ~70 ns at 8 threads.
#[cfg(feature = "mmap")]
Mmap(Range<usize>), Mmap(Range<usize>),
/// Read into memory at open time, for the non-mmap backing, which has nothing to borrow from. /// Read into memory at open time, for the non-mmap backing, which has nothing to borrow from.
Owned(Box<[u8]>), Owned(Box<[u8]>),
...@@ -419,6 +425,7 @@ impl IndexBlock { ...@@ -419,6 +425,7 @@ impl IndexBlock {
let entries = match backing { let entries = match backing {
// Store a range, not the slice: `StaticSortedFile` owns the mmap these bytes live in. // Store a range, not the slice: `StaticSortedFile` owns the mmap these bytes live in.
#[cfg(feature = "mmap")]
StaticSortedFileBacking::Mmap(mmap) => { StaticSortedFileBacking::Mmap(mmap) => {
let start = entry_bytes.as_ptr() as usize - mmap.as_ptr() as usize; let start = entry_bytes.as_ptr() as usize - mmap.as_ptr() as usize;
IndexEntries::Mmap(start..start + entry_bytes.len()) IndexEntries::Mmap(start..start + entry_bytes.len())
...@@ -447,6 +454,7 @@ impl StaticSortedFile { ...@@ -447,6 +454,7 @@ impl StaticSortedFile {
let path = db_path.join(&filename); let path = db_path.join(&filename);
let file = File::open(&path)?; let file = File::open(&path)?;
let backing = match access_mode { let backing = match access_mode {
#[cfg(feature = "mmap")]
AccessMode::Mmap => { AccessMode::Mmap => {
let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| { let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| {
format!( format!(
...@@ -504,12 +512,14 @@ impl StaticSortedFile { ...@@ -504,12 +512,14 @@ impl StaticSortedFile {
#[inline] #[inline]
fn index_entries(&self) -> &[[u8; INDEX_BLOCK_ENTRY_SIZE]] { fn index_entries(&self) -> &[[u8; INDEX_BLOCK_ENTRY_SIZE]] {
let bytes = match (&self.index.entries, &self.backing) { let bytes = match (&self.index.entries, &self.backing) {
#[cfg(feature = "mmap")]
(IndexEntries::Mmap(range), StaticSortedFileBacking::Mmap(mmap)) => { (IndexEntries::Mmap(range), StaticSortedFileBacking::Mmap(mmap)) => {
&mmap[range.clone()] &mmap[range.clone()]
} }
(IndexEntries::Owned(bytes), _) => &bytes[..], (IndexEntries::Owned(bytes), _) => &bytes[..],
// `IndexBlock::parse` only produces `Mmap` entries for an mmap backing, and the // `IndexBlock::parse` only produces `Mmap` entries for an mmap backing, and the
// backing never changes after open. // backing never changes after open.
#[cfg(feature = "mmap")]
(IndexEntries::Mmap(_), StaticSortedFileBacking::File { .. }) => unreachable!( (IndexEntries::Mmap(_), StaticSortedFileBacking::File { .. }) => unreachable!(
"mmap-ranged index entries with a file backing in {:08}.sst", "mmap-ranged index entries with a file backing in {:08}.sst",
self.meta.sequence_number self.meta.sequence_number
...@@ -749,17 +759,20 @@ impl StaticSortedFile { ...@@ -749,17 +759,20 @@ impl StaticSortedFile {
/// its refcount belongs to one cache entry rather than the whole file. /// its refcount belongs to one cache entry rather than the whole file.
enum BlockRef<'l> { enum BlockRef<'l> {
/// Borrowed from the memory-mapped file. /// Borrowed from the memory-mapped file.
#[cfg(feature = "mmap")]
Mmap(&'l [u8]), Mmap(&'l [u8]),
/// Owned, and shared with the block cache. /// Owned, and shared with the block cache. The zero-sized marker keeps the backing borrow
Cached(ArcBytes), /// lifetime represented in builds where the mmap variant is disabled.
Cached(ArcBytes, PhantomData<&'l ()>),
} }
impl BlockRef<'_> { impl BlockRef<'_> {
#[inline] #[inline]
fn as_slice(&self) -> &[u8] { fn as_slice(&self) -> &[u8] {
match self { match self {
#[cfg(feature = "mmap")]
BlockRef::Mmap(data) => data, BlockRef::Mmap(data) => data,
BlockRef::Cached(block) => block, BlockRef::Cached(block, _) => block,
} }
} }
...@@ -767,8 +780,10 @@ impl BlockRef<'_> { ...@@ -767,8 +780,10 @@ impl BlockRef<'_> {
/// ///
/// Only needed by callers that hand the bytes to something outliving the lookup. /// Only needed by callers that hand the bytes to something outliving the lookup.
#[inline] #[inline]
#[cfg_attr(not(feature = "mmap"), allow(unused_variables))]
fn into_owned(self, backing: &StaticSortedFileBacking) -> ArcBytes { fn into_owned(self, backing: &StaticSortedFileBacking) -> ArcBytes {
match self { match self {
#[cfg(feature = "mmap")]
BlockRef::Mmap(data) => { BlockRef::Mmap(data) => {
let StaticSortedFileBacking::Mmap(mmap) = backing else { let StaticSortedFileBacking::Mmap(mmap) = backing else {
// `get_or_read_block` only borrows from an mmap backing. // `get_or_read_block` only borrows from an mmap backing.
...@@ -777,7 +792,7 @@ impl BlockRef<'_> { ...@@ -777,7 +792,7 @@ impl BlockRef<'_> {
// SAFETY: the borrow came from this mmap, via `get_or_read_block`. // SAFETY: the borrow came from this mmap, via `get_or_read_block`.
unsafe { ArcBytes::from_mmap(mmap, data) } unsafe { ArcBytes::from_mmap(mmap, data) }
} }
BlockRef::Cached(block) => block, BlockRef::Cached(block, _) => block,
} }
} }
} }
...@@ -799,6 +814,11 @@ fn get_or_read_block<'l>( ...@@ -799,6 +814,11 @@ fn get_or_read_block<'l>(
verified_blocks: &[AtomicU64], verified_blocks: &[AtomicU64],
compression: Compression, compression: Compression,
) -> Result<BlockRef<'l>> { ) -> Result<BlockRef<'l>> {
// `verified_blocks` tracks which mmap-backed blocks already passed their checksum. There is
// no mmap backing without the feature, so the bitmap is unused there.
#[cfg(not(feature = "mmap"))]
let _ = verified_blocks;
#[cfg(feature = "mmap")]
let mmap_block = if let StaticSortedFileBacking::Mmap(mmap) = backing { let mmap_block = if let StaticSortedFileBacking::Mmap(mmap) = backing {
let (uncompressed_length, checksum, block_data) = let (uncompressed_length, checksum, block_data) =
get_raw_block_slice(mmap, meta, block_index).with_context(|| { get_raw_block_slice(mmap, meta, block_index).with_context(|| {
...@@ -818,6 +838,10 @@ fn get_or_read_block<'l>( ...@@ -818,6 +838,10 @@ fn get_or_read_block<'l>(
} else { } else {
None None
}; };
// Without an mmap backing there is never a zero-copy block to borrow, so every block goes
// through the decompress/cache path below.
#[cfg(not(feature = "mmap"))]
let mmap_block: Option<(u32, u32, &[u8])> = None;
// Compressed: check cache; decompress and insert on miss. // Compressed: check cache; decompress and insert on miss.
// File-backed blocks use the same cache, including uncompressed ones. // File-backed blocks use the same cache, including uncompressed ones.
...@@ -834,6 +858,7 @@ fn get_or_read_block<'l>( ...@@ -834,6 +858,7 @@ fn get_or_read_block<'l>(
// A cached block may have been evicted, so re-reading still // A cached block may have been evicted, so re-reading still
// benefits from the bitmap to skip redundant CRC verification. // benefits from the bitmap to skip redundant CRC verification.
match backing { match backing {
#[cfg(feature = "mmap")]
StaticSortedFileBacking::Mmap(_) => verify_checksum_once( StaticSortedFileBacking::Mmap(_) => verify_checksum_once(
meta, meta,
&block_data, &block_data,
...@@ -862,11 +887,13 @@ fn get_or_read_block<'l>( ...@@ -862,11 +887,13 @@ fn get_or_read_block<'l>(
} }
GuardResult::Timeout => unreachable!(), GuardResult::Timeout => unreachable!(),
}, },
PhantomData,
)) ))
} }
/// Gets the raw block slice directly from a memory-mapped file. /// Gets the raw block slice directly from a memory-mapped file.
/// Returns `(uncompressed_length, checksum, block_data)`. /// Returns `(uncompressed_length, checksum, block_data)`.
#[cfg(feature = "mmap")]
fn get_raw_block_slice<'a>( fn get_raw_block_slice<'a>(
mmap: &'a Mmap, mmap: &'a Mmap,
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
...@@ -932,6 +959,7 @@ fn get_raw_block<'a>( ...@@ -932,6 +959,7 @@ fn get_raw_block<'a>(
block_index: u16, block_index: u16,
) -> Result<(u32, u32, Cow<'a, [u8]>)> { ) -> Result<(u32, u32, Cow<'a, [u8]>)> {
match backing { match backing {
#[cfg(feature = "mmap")]
StaticSortedFileBacking::Mmap(mmap) => { StaticSortedFileBacking::Mmap(mmap) => {
let (uncompressed_length, checksum, block) = let (uncompressed_length, checksum, block) =
get_raw_block_slice(mmap, meta, block_index)?; get_raw_block_slice(mmap, meta, block_index)?;
...@@ -1026,6 +1054,7 @@ fn verify_checksum( ...@@ -1026,6 +1054,7 @@ fn verify_checksum(
/// since the check is deterministic and idempotent. Verification failures are /// since the check is deterministic and idempotent. Verification failures are
/// *not* recorded in the bitmap, so a corrupted block will be re-checked (and /// *not* recorded in the bitmap, so a corrupted block will be re-checked (and
/// fail again) on every access. /// fail again) on every access.
#[cfg_attr(not(feature = "mmap"), allow(dead_code))]
fn verify_checksum_once( fn verify_checksum_once(
meta: &StaticSortedFileMetaData, meta: &StaticSortedFileMetaData,
data: &[u8], data: &[u8],
...@@ -1054,6 +1083,7 @@ fn read_block_lookup( ...@@ -1054,6 +1083,7 @@ fn read_block_lookup(
verify_checksum(meta, &block, checksum, block_index)?; verify_checksum(meta, &block, checksum, block_index)?;
if uncompressed_length == 0 { if uncompressed_length == 0 {
return match (backing, block) { return match (backing, block) {
#[cfg(feature = "mmap")]
(StaticSortedFileBacking::Mmap(mmap), Cow::Borrowed(block)) => { (StaticSortedFileBacking::Mmap(mmap), Cow::Borrowed(block)) => {
// SAFETY: block points into mmap. // SAFETY: block points into mmap.
Ok(unsafe { ArcBytes::from_mmap(mmap, block) }) Ok(unsafe { ArcBytes::from_mmap(mmap, block) })
...@@ -1077,6 +1107,7 @@ fn get_raw_block_iter( ...@@ -1077,6 +1107,7 @@ fn get_raw_block_iter(
block_index: u16, block_index: u16,
) -> Result<(u32, u32, RcBytes)> { ) -> Result<(u32, u32, RcBytes)> {
match backing { match backing {
#[cfg(feature = "mmap")]
StaticSortedFileIterBacking::Mmap(mmap) => { StaticSortedFileIterBacking::Mmap(mmap) => {
let (uncompressed_length, checksum, block) = let (uncompressed_length, checksum, block) =
get_raw_block_slice(mmap, meta, block_index)?; get_raw_block_slice(mmap, meta, block_index)?;
...@@ -1184,6 +1215,7 @@ fn handle_key_match_generic<B: SharedBytes>( ...@@ -1184,6 +1215,7 @@ fn handle_key_match_generic<B: SharedBytes>(
} }
enum StaticSortedFileIterBacking { enum StaticSortedFileIterBacking {
#[cfg(feature = "mmap")]
Mmap(Rc<Mmap>), Mmap(Rc<Mmap>),
File { File {
file: Rc<File>, file: Rc<File>,
...@@ -1290,6 +1322,7 @@ impl StaticSortedFileIter { ...@@ -1290,6 +1322,7 @@ impl StaticSortedFileIter {
let path = db_path.join(&filename); let path = db_path.join(&filename);
let file = File::open(&path)?; let file = File::open(&path)?;
let backing = match access_mode { let backing = match access_mode {
#[cfg(feature = "mmap")]
AccessMode::Mmap => { AccessMode::Mmap => {
let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| { let mmap = unsafe { Mmap::map(file.file()) }.with_context(|| {
format!( format!(
......
...@@ -1434,7 +1434,6 @@ impl<W: Write> IndexBlockBuilder<W> { ...@@ -1434,7 +1434,6 @@ impl<W: Write> IndexBlockBuilder<W> {
mod tests { mod tests {
use super::*; use super::*;
use crate::{ use crate::{
AccessMode,
key::hash_key, key::hash_key,
lookup_entry::LookupValue, lookup_entry::LookupValue,
static_sorted_file::{ static_sorted_file::{
...@@ -1566,7 +1565,7 @@ mod tests { ...@@ -1566,7 +1565,7 @@ mod tests {
block_count: meta.block_count, block_count: meta.block_count,
}, },
Compression::Lz4, Compression::Lz4,
AccessMode::Mmap, crate::mmap_access_mode(),
) )
} }
...@@ -1913,7 +1912,7 @@ mod tests { ...@@ -1913,7 +1912,7 @@ mod tests {
block_count: meta1.block_count, block_count: meta1.block_count,
}, },
Compression::Lz4, Compression::Lz4,
AccessMode::Mmap, crate::mmap_access_mode(),
)?; )?;
let sst2 = StaticSortedFile::open( let sst2 = StaticSortedFile::open(
dir.path(), dir.path(),
...@@ -1922,7 +1921,7 @@ mod tests { ...@@ -1922,7 +1921,7 @@ mod tests {
block_count: meta2.block_count, block_count: meta2.block_count,
}, },
Compression::Lz4, Compression::Lz4,
AccessMode::Mmap, crate::mmap_access_mode(),
)?; )?;
let kc = make_cache(); let kc = make_cache();
let vc = make_cache(); let vc = make_cache();
......
use std::{fs, path::Path, time::Instant}; #[cfg(not(miri))]
use std::time::Instant;
use std::{fs, path::Path};
use anyhow::Result; use anyhow::Result;
#[cfg(not(miri))]
use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rayon::iter::{IntoParallelIterator, ParallelIterator};
use rstest::rstest; use rstest::rstest;
// Miri rejects a process that exits while rayon's global worker threads are still parked, so
// run the same tests on the serial scheduler there. The alias keeps the test bodies identical.
#[cfg(miri)]
use crate::parallel_scheduler::SerialScheduler as RayonParallelScheduler;
use crate::{ use crate::{
AccessMode, Compression, DbConfig, FamilyConfig, FamilyKind, AccessMode, Compression, DbConfig, FamilyConfig, FamilyKind,
constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, constants::MAX_INLINE_VALUE_SIZE,
db::{CompactConfig, TurboPersistence, read_current_version}, db::{CompactConfig, TurboPersistence, read_current_version},
lookup_entry::IterValue, lookup_entry::IterValue,
static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData},
};
#[cfg(not(miri))]
use crate::{
constants::{MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE},
meta_file::MetaFile, meta_file::MetaFile,
parallel_scheduler::ParallelScheduler, parallel_scheduler::ParallelScheduler,
static_sorted_file::{StaticSortedFileIter, StaticSortedFileMetaData},
write_batch::WriteBatch, write_batch::WriteBatch,
}; };
#[cfg(not(miri))]
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
struct RayonParallelScheduler; struct RayonParallelScheduler;
#[cfg(not(miri))]
impl ParallelScheduler for RayonParallelScheduler { impl ParallelScheduler for RayonParallelScheduler {
fn block_in_place<R>(&self, f: impl FnOnce() -> R + Send) -> R fn block_in_place<R>(&self, f: impl FnOnce() -> R + Send) -> R
where where
...@@ -108,6 +121,7 @@ impl ParallelScheduler for RayonParallelScheduler { ...@@ -108,6 +121,7 @@ impl ParallelScheduler for RayonParallelScheduler {
} }
} }
#[cfg(not(miri))]
fn tuple_key(prefix: u8, suffix: [u8; 4]) -> Box<[u8]> { fn tuple_key(prefix: u8, suffix: [u8; 4]) -> Box<[u8]> {
let mut key = Vec::with_capacity(1 + suffix.len()); let mut key = Vec::with_capacity(1 + suffix.len());
key.push(prefix); key.push(prefix);
...@@ -118,7 +132,7 @@ fn tuple_key(prefix: u8, suffix: [u8; 4]) -> Box<[u8]> { ...@@ -118,7 +132,7 @@ fn tuple_key(prefix: u8, suffix: [u8; 4]) -> Box<[u8]> {
fn config_with_mmap<const F: usize>(mmap: bool) -> DbConfig<F> { fn config_with_mmap<const F: usize>(mmap: bool) -> DbConfig<F> {
DbConfig { DbConfig {
access_mode: if mmap { access_mode: if mmap {
AccessMode::Mmap crate::mmap_access_mode()
} else { } else {
AccessMode::File AccessMode::File
}, },
...@@ -152,7 +166,7 @@ fn multi_value_config_with_mmap(mmap: bool) -> DbConfig<1> { ...@@ -152,7 +166,7 @@ fn multi_value_config_with_mmap(mmap: bool) -> DbConfig<1> {
compression: Compression::Lz4, compression: Compression::Lz4,
}], }],
access_mode: if mmap { access_mode: if mmap {
AccessMode::Mmap crate::mmap_access_mode()
} else { } else {
AccessMode::File AccessMode::File
}, },
...@@ -166,6 +180,9 @@ fn open_multi_value_db( ...@@ -166,6 +180,9 @@ fn open_multi_value_db(
open_db_with_config(path, multi_value_config_with_mmap(mmap)) open_db_with_config(path, multi_value_config_with_mmap(mmap))
} }
// This stress test writes more than ten million entries and uses Rayon directly, so it is too slow
// to run under Miri and depends on concurrency that Miri cannot provide efficiently.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -552,6 +569,9 @@ fn full_cycle(#[case] mmap: bool) -> Result<()> { ...@@ -552,6 +569,9 @@ fn full_cycle(#[case] mmap: bool) -> Result<()> {
Ok(()) Ok(())
} }
// This test exceeded 20 minutes under Miri because it writes and repeatedly reads tens of
// thousands of entries across multiple reopen/compaction cycles.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -664,6 +684,8 @@ fn persist_changes(#[case] mmap: bool) -> Result<()> { ...@@ -664,6 +684,8 @@ fn persist_changes(#[case] mmap: bool) -> Result<()> {
Ok(()) Ok(())
} }
// This 50-iteration compaction/restore stress test exceeded 20 minutes under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -751,6 +773,8 @@ fn partial_compaction(#[case] mmap: bool) -> Result<()> { ...@@ -751,6 +773,8 @@ fn partial_compaction(#[case] mmap: bool) -> Result<()> {
Ok(()) Ok(())
} }
// This repeated large-file merge/removal stress test exceeded 20 minutes under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -1047,6 +1071,8 @@ fn batch_get_large_batch(#[case] mmap: bool) -> Result<()> { ...@@ -1047,6 +1071,8 @@ fn batch_get_large_batch(#[case] mmap: bool) -> Result<()> {
Ok(()) Ok(())
} }
// Crossing the real blob-size boundary makes this test exceed 20 minutes under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -1372,6 +1398,8 @@ fn batch_get_after_restore(#[case] mmap: bool) -> Result<()> { ...@@ -1372,6 +1398,8 @@ fn batch_get_after_restore(#[case] mmap: bool) -> Result<()> {
/// Test that compaction works with many small values without overflowing block indices. /// Test that compaction works with many small values without overflowing block indices.
/// Reproduces a CI benchmark failure with key_4/value_512/entries_1.98Mi/compacted. /// Reproduces a CI benchmark failure with key_4/value_512/entries_1.98Mi/compacted.
// Miri was killed while processing this production-scale, two-million-entry workload.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -1418,6 +1446,8 @@ fn many_small_values_compaction(#[case] mmap: bool) -> Result<()> { ...@@ -1418,6 +1446,8 @@ fn many_small_values_compaction(#[case] mmap: bool) -> Result<()> {
/// Test compaction with MAX_SMALL_VALUE_SIZE (4096-byte) values. /// Test compaction with MAX_SMALL_VALUE_SIZE (4096-byte) values.
/// Worst case for small value blocks: fewest entries per block. /// Worst case for small value blocks: fewest entries per block.
// This production-scale 512K-entry workload exceeded 20 minutes under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -1463,6 +1493,8 @@ fn many_max_small_values_compaction(#[case] mmap: bool) -> Result<()> { ...@@ -1463,6 +1493,8 @@ fn many_max_small_values_compaction(#[case] mmap: bool) -> Result<()> {
/// Test compaction with 4097-byte values (minimum medium size). /// Test compaction with 4097-byte values (minimum medium size).
/// Each medium value gets its own dedicated block, so this is the worst case for block count. /// Each medium value gets its own dedicated block, so this is the worst case for block count.
// This production-scale 128K-entry workload exceeded 20 minutes under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -1974,6 +2006,7 @@ fn multi_value_tombstone_shadows_older_sst_only(#[case] mmap: bool) -> Result<() ...@@ -1974,6 +2006,7 @@ fn multi_value_tombstone_shadows_older_sst_only(#[case] mmap: bool) -> Result<()
} }
/// Returns the number of `.blob` files in the given directory. /// Returns the number of `.blob` files in the given directory.
#[cfg(not(miri))]
fn count_blob_files(dir: &Path) -> usize { fn count_blob_files(dir: &Path) -> usize {
fs::read_dir(dir) fs::read_dir(dir)
.unwrap() .unwrap()
...@@ -1984,6 +2017,9 @@ fn count_blob_files(dir: &Path) -> usize { ...@@ -1984,6 +2017,9 @@ fn count_blob_files(dir: &Path) -> usize {
/// Test that compaction deletes blob files when their entries are superseded /// Test that compaction deletes blob files when their entries are superseded
/// by newer values (SingleValue family). /// by newer values (SingleValue family).
// The first access-mode variant exceeded 20 minutes under Miri while processing the production-size
// blob boundary.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -2043,6 +2079,9 @@ fn compaction_deletes_superseded_blob(#[case] mmap: bool) -> Result<()> { ...@@ -2043,6 +2079,9 @@ fn compaction_deletes_superseded_blob(#[case] mmap: bool) -> Result<()> {
/// Test that compaction deletes blob files when a key is deleted via tombstone /// Test that compaction deletes blob files when a key is deleted via tombstone
/// (SingleValue family). /// (SingleValue family).
// The first access-mode variant exceeded 20 minutes under Miri while processing the production-size
// blob boundary.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -2094,6 +2133,9 @@ fn compaction_deletes_blob_on_tombstone(#[case] mmap: bool) -> Result<()> { ...@@ -2094,6 +2133,9 @@ fn compaction_deletes_blob_on_tombstone(#[case] mmap: bool) -> Result<()> {
/// Test that compaction deletes blob files for MultiValue families when a /// Test that compaction deletes blob files for MultiValue families when a
/// tombstone prunes older blob entries. /// tombstone prunes older blob entries.
// The first access-mode variant exceeded 20 minutes under Miri while processing the production-size
// blob boundary.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -2143,6 +2185,9 @@ fn compaction_deletes_blob_multi_value_tombstone(#[case] mmap: bool) -> Result<( ...@@ -2143,6 +2185,9 @@ fn compaction_deletes_blob_multi_value_tombstone(#[case] mmap: bool) -> Result<(
/// Test that compaction preserves blob files that are still referenced /// Test that compaction preserves blob files that are still referenced
/// (not superseded). /// (not superseded).
// The first access-mode variant exceeded 20 minutes under Miri while processing the production-size
// blob boundary.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -2457,7 +2502,9 @@ fn count_tombstones( ...@@ -2457,7 +2502,9 @@ fn count_tombstones(
sequence_number: entry.sequence_number, sequence_number: entry.sequence_number,
block_count: entry.block_count, block_count: entry.block_count,
}; };
for item in StaticSortedFileIter::open(path, sst, Compression::Lz4, AccessMode::Mmap)? { for item in
StaticSortedFileIter::open(path, sst, Compression::Lz4, crate::mmap_access_mode())?
{
if matches!( if matches!(
item?.value, item?.value,
IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. }
...@@ -2634,6 +2681,8 @@ fn compaction_keeps_tombstone_when_older_sst_has_the_key() -> Result<()> { ...@@ -2634,6 +2681,8 @@ fn compaction_keeps_tombstone_when_older_sst_has_the_key() -> Result<()> {
/// ///
/// This is the case a sequence-number threshold gets wrong — it would treat the skipped SST as /// This is the case a sequence-number threshold gets wrong — it would treat the skipped SST as
/// part of the job and drop a tombstone that is still load-bearing. /// part of the job and drop a tombstone that is still load-bearing.
// This multi-SST partial-compaction scenario exceeded 20 minutes under Miri.
#[cfg(not(miri))]
#[test] #[test]
fn compaction_keeps_tombstone_when_skipped_sst_has_the_key() -> Result<()> { fn compaction_keeps_tombstone_when_skipped_sst_has_the_key() -> Result<()> {
let tempdir = tempfile::tempdir()?; let tempdir = tempfile::tempdir()?;
...@@ -2814,6 +2863,8 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> { ...@@ -2814,6 +2863,8 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> {
Ok(()) Ok(())
} }
// This 8,000-key meta-sharding compaction test exceeded 20 minutes under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case(true)] #[case(true)]
#[case(false)] #[case(false)]
...@@ -2821,7 +2872,7 @@ fn partial_compaction_retires_fully_consumed_meta_files(#[case] mmap: bool) -> R ...@@ -2821,7 +2872,7 @@ fn partial_compaction_retires_fully_consumed_meta_files(#[case] mmap: bool) -> R
let tempdir = tempfile::tempdir()?; let tempdir = tempfile::tempdir()?;
let path = tempdir.path(); let path = tempdir.path();
let access_mode = if mmap { let access_mode = if mmap {
AccessMode::Mmap crate::mmap_access_mode()
} else { } else {
AccessMode::File AccessMode::File
}; };
......
...@@ -533,7 +533,6 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize ...@@ -533,7 +533,6 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
use core::panic; use core::panic;
use crate::{ use crate::{
AccessMode,
collector_entry::CollectorEntryValue, collector_entry::CollectorEntryValue,
key::hash_key, key::hash_key,
lookup_entry::LookupValue, lookup_entry::LookupValue,
...@@ -551,7 +550,7 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize ...@@ -551,7 +550,7 @@ impl<'db, K: StoreKey + Send + Sync, S: ParallelScheduler, const FAMILIES: usize
block_count: meta.block_count, block_count: meta.block_count,
}, },
self.family_configs[usize_from_u32(family)].compression, self.family_configs[usize_from_u32(family)].compression,
AccessMode::Mmap, crate::mmap_access_mode(),
)?; )?;
let cache2 = BlockCache::with( let cache2 = BlockCache::with(
10, 10,
......
...@@ -77,12 +77,12 @@ pub(crate) fn new_atom(text: Cow<'_, str>) -> RcStr { ...@@ -77,12 +77,12 @@ pub(crate) fn new_atom(text: Cow<'_, str>) -> RcStr {
/// Construct a new dynamic RcStr from a DynamicPrehashedString /// Construct a new dynamic RcStr from a DynamicPrehashedString
pub(crate) fn new_atom_from_prehashed(prehashed: DynamicPrehashedString) -> RcStr { pub(crate) fn new_atom_from_prehashed(prehashed: DynamicPrehashedString) -> RcStr {
let entry: Arc<DynamicPrehashedString> = Arc::new(prehashed); let entry: Arc<DynamicPrehashedString> = Arc::new(prehashed);
let mut entry = Arc::into_raw(entry); let entry = Arc::into_raw(entry);
debug_assert!(0 == entry as u8 & TAG_MASK); debug_assert_eq!(entry.addr() & TAG_MASK as usize, 0);
entry = ((entry as usize) | DYNAMIC_TAG as usize) as *mut DynamicPrehashedString; let entry = entry.map_addr(|addr| addr | DYNAMIC_TAG as usize);
let ptr: NonNull<DynamicPrehashedString> = unsafe { let ptr: NonNull<DynamicPrehashedString> = unsafe {
// Safety: Arc::into_raw returns a non-null pointer // Safety: Arc::into_raw returns a non-null pointer
NonNull::new_unchecked(entry as *mut _) NonNull::new_unchecked(entry.cast_mut())
}; };
RcStr { RcStr {
......
...@@ -951,6 +951,8 @@ mod tests { ...@@ -951,6 +951,8 @@ mod tests {
let decoded: RcStr = turbo_bincode_decode(&encoded).unwrap(); let decoded: RcStr = turbo_bincode_decode(&encoded).unwrap();
assert_eq!(decoded.as_str(), STATIC_STR); assert_eq!(decoded.as_str(), STATIC_STR);
// Decoded via peek_read path should find the static constant // Decoded via peek_read path should find the static constant
// Miri does not populate the linker section used by scattered-collect.
#[cfg(not(miri))]
assert_eq!(decoded.tag(), STATIC_TAG); assert_eq!(decoded.tag(), STATIC_TAG);
} }
......
...@@ -37,6 +37,13 @@ ...@@ -37,6 +37,13 @@
//! Reading the value back as an integer (`get_ptr`, `get_value`, `tag_byte`) only ever happens at //! Reading the value back as an integer (`get_ptr`, `get_value`, `tag_byte`) only ever happens at
//! run time, where pointer → integer is perfectly legal. //! run time, where pointer → integer is perfectly legal.
#[cfg(not(any(
target_pointer_width = "32",
target_pointer_width = "16",
feature = "atom_size_64",
feature = "atom_size_128"
)))]
use std::num::NonZeroUsize;
use std::{num::NonZeroU8, os::raw::c_void, ptr::NonNull, slice}; use std::{num::NonZeroU8, os::raw::c_void, ptr::NonNull, slice};
use self::raw_types::*; use self::raw_types::*;
...@@ -209,11 +216,35 @@ impl TaggedValue { ...@@ -209,11 +216,35 @@ impl TaggedValue {
#[inline(always)] #[inline(always)]
pub const fn new_tag(value: NonZeroU8) -> Self { pub const fn new_tag(value: NonZeroU8) -> Self {
// An integer → pointer transmute, which const evaluation permits. #[cfg(not(any(
let value = value.get() as RawTaggedValue; target_pointer_width = "32",
Self { target_pointer_width = "16",
#[allow(clippy::transmute_int_to_non_zero)] feature = "atom_size_64",
value: unsafe { std::mem::transmute::<RawTaggedValue, RawTaggedNonZeroValue>(value) }, feature = "atom_size_128"
)))]
{
// A provenance-free integer → pointer conversion for the pointer representation.
Self {
value: NonNull::without_provenance(
NonZeroUsize::new(value.get() as usize).unwrap(),
),
}
}
#[cfg(any(
target_pointer_width = "32",
target_pointer_width = "16",
feature = "atom_size_64",
feature = "atom_size_128"
))]
{
// An integer → pointer transmute, which const evaluation permits.
let value = value.get() as RawTaggedValue;
Self {
#[allow(clippy::transmute_int_to_non_zero)]
value: unsafe {
std::mem::transmute::<RawTaggedValue, RawTaggedNonZeroValue>(value)
},
}
} }
} }
...@@ -240,12 +271,26 @@ impl TaggedValue { ...@@ -240,12 +271,26 @@ impl TaggedValue {
feature = "atom_size_128" feature = "atom_size_128"
)))] )))]
{ {
(self.value.as_ptr() as usize & !(TAG_MASK as usize)) as _ self.value
.as_ptr()
.map_addr(|addr| addr & !(TAG_MASK as usize))
.cast_const()
.cast()
} }
} }
#[inline(always)] #[inline(always)]
fn get_value(&self) -> RawTaggedValue { fn get_value(&self) -> RawTaggedValue {
#[cfg(not(any(
target_pointer_width = "32",
target_pointer_width = "16",
feature = "atom_size_64",
feature = "atom_size_128"
)))]
{
// Read the pointer representation's address without exposing provenance.
self.value.addr().get()
}
#[cfg(all( #[cfg(all(
any(target_pointer_width = "32", target_pointer_width = "16"), any(target_pointer_width = "32", target_pointer_width = "16"),
not(feature = "atom_size_128") not(feature = "atom_size_128")
...@@ -255,10 +300,18 @@ impl TaggedValue { ...@@ -255,10 +300,18 @@ impl TaggedValue {
// value as an integer is well defined. Again, run time only. // value as an integer is well defined. Again, run time only.
unsafe { std::mem::transmute::<RawTaggedNonZeroValue, RawTaggedValue>(self.value) } unsafe { std::mem::transmute::<RawTaggedNonZeroValue, RawTaggedValue>(self.value) }
} }
#[cfg(not(all( #[cfg(all(
any(target_pointer_width = "32", target_pointer_width = "16"), any(
not(feature = "atom_size_128") target_pointer_width = "32",
)))] target_pointer_width = "16",
feature = "atom_size_64",
feature = "atom_size_128"
),
not(all(
any(target_pointer_width = "32", target_pointer_width = "16"),
not(feature = "atom_size_128")
))
))]
{ {
unsafe { unsafe {
std::mem::transmute::<Option<RawTaggedNonZeroValue>, RawTaggedValue>(Some( std::mem::transmute::<Option<RawTaggedNonZeroValue>, RawTaggedValue>(Some(
......
...@@ -59,13 +59,18 @@ smallvec = { workspace = true } ...@@ -59,13 +59,18 @@ smallvec = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
turbo-bincode = { workspace = true } turbo-bincode = { workspace = true }
turbo-persistence = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true } turbo-tasks-hash = { workspace = true }
turbo-tasks-malloc = { workspace = true, default-features = false } turbo-tasks-malloc = { workspace = true, default-features = false }
thread_local = { workspace = true } thread_local = { workspace = true }
[target.'cfg(not(target_family = "wasm"))'.dependencies]
turbo-persistence = { workspace = true }
[target.'cfg(target_family = "wasm")'.dependencies]
turbo-persistence = { workspace = true, default-features = false }
[dev-dependencies] [dev-dependencies]
async-trait = { workspace = true } async-trait = { workspace = true }
criterion = { workspace = true, features = ["async_tokio"] } criterion = { workspace = true, features = ["async_tokio"] }
......
...@@ -620,6 +620,8 @@ mod tests { ...@@ -620,6 +620,8 @@ mod tests {
/// Tests that multiple distinct keys written in a single batch with flush can be read back. /// Tests that multiple distinct keys written in a single batch with flush can be read back.
/// This mirrors the actual save_snapshot pattern: write many TaskCache entries, flush, commit. /// This mirrors the actual save_snapshot pattern: write many TaskCache entries, flush, commit.
// This test is too slow to run under Miri.
#[cfg(not(miri))]
#[tokio::test(flavor = "multi_thread")] #[tokio::test(flavor = "multi_thread")]
async fn test_batch_write_with_flush_and_reopen() -> Result<()> { async fn test_batch_write_with_flush_and_reopen() -> Result<()> {
let tempdir = tempfile::tempdir()?; let tempdir = tempfile::tempdir()?;
......
...@@ -228,6 +228,8 @@ mod tests { ...@@ -228,6 +228,8 @@ mod tests {
use super::*; use super::*;
// This test is too slow to run under Miri.
#[cfg(not(miri))]
#[test] #[test]
fn stress_deadlock() { fn stress_deadlock() {
const N: usize = 100000; const N: usize = 100000;
......
...@@ -358,6 +358,8 @@ async fn counted_leaf(nonce: u32) -> Result<Vc<Value>> { ...@@ -358,6 +358,8 @@ async fn counted_leaf(nonce: u32) -> Result<Vc<Value>> {
/// Inline execution nests: reading the deepest task of a chain of uncomputed tasks executes them /// Inline execution nests: reading the deepest task of a chain of uncomputed tasks executes them
/// one inside the other. The nesting cap keeps that from growing the stack without bounds. /// one inside the other. The nesting cap keeps that from growing the stack without bounds.
// This test is too slow to run under Miri.
#[cfg(not(miri))]
#[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn test_deep_dependency_chain() { async fn test_deep_dependency_chain() {
let mut nonce = 0; let mut nonce = 0;
......
...@@ -1134,6 +1134,9 @@ mod tests { ...@@ -1134,6 +1134,9 @@ mod tests {
/// `recursive_mode` is set explicitly rather than left to the platform default so that both /// `recursive_mode` is set explicitly rather than left to the platform default so that both
/// watching strategies are covered on every host. `TURBO_TASKS_FORCE_WATCH_MODE` still /// watching strategies are covered on every host. `TURBO_TASKS_FORCE_WATCH_MODE` still
/// overrides it, collapsing these into two cases. /// overrides it, collapsing these into two cases.
// Miri cannot run the native cases because inotify is unsupported, while the polling cases
// require Turbo Tasks' link-section registry, which is unavailable under Miri.
#[cfg(not(miri))]
#[rstest] #[rstest]
#[case::native_recursive(None, DiskWatcherRecursiveMode::Recursive)] #[case::native_recursive(None, DiskWatcherRecursiveMode::Recursive)]
#[case::native_non_recursive(None, DiskWatcherRecursiveMode::NonRecursive)] #[case::native_non_recursive(None, DiskWatcherRecursiveMode::NonRecursive)]
......
// Miri cannot run trybuild because its subprocesses require the unsupported `posix_spawnattr_init`.
#![cfg(not(miri))]
// Unset RUSTC_WRAPPER before trybuild tests run. When sccache wraps rustc, it // Unset RUSTC_WRAPPER before trybuild tests run. When sccache wraps rustc, it
// emits "warning: ignoring -C extra-filename flag due to -o flag" which pollutes // emits "warning: ignoring -C extra-filename flag due to -o flag" which pollutes
// trybuild's stderr snapshot comparisons. Unsetting it here means only the // trybuild's stderr snapshot comparisons. Unsetting it here means only the
......
...@@ -18,18 +18,18 @@ harness = false ...@@ -18,18 +18,18 @@ harness = false
[dev-dependencies] [dev-dependencies]
criterion = { workspace = true } criterion = { workspace = true }
[target.'cfg(not(target_family = "wasm"))'.dependencies] [target.'cfg(all(not(target_family = "wasm"), not(miri)))'.dependencies]
libmimalloc-sys = { version = "0.1.44", features = [ libmimalloc-sys = { version = "0.1.44", features = [
"extended", "extended",
], optional = true } ], optional = true }
[target.'cfg(not(any(target_os = "linux", target_family = "wasm")))'.dependencies] [target.'cfg(not(any(target_os = "linux", target_family = "wasm", miri)))'.dependencies]
mimalloc = { version = "0.1.48", features = [ mimalloc = { version = "0.1.48", features = [
"v3", "v3",
"extended", "extended",
], optional = true } ], optional = true }
[target.'cfg(all(target_os = "linux", not(target_family = "wasm")))'.dependencies] [target.'cfg(all(target_os = "linux", not(target_family = "wasm"), not(miri)))'.dependencies]
mimalloc = { version = "0.1.48", features = [ mimalloc = { version = "0.1.48", features = [
"v3", "v3",
"extended", "extended",
......
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
use std::{cell::UnsafeCell, ptr::NonNull}; use std::{cell::UnsafeCell, ptr::NonNull};
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
pub use self::global::get; pub use self::global::get;
use crate::AllocationCounters; use crate::AllocationCounters;
...@@ -20,7 +20,7 @@ use crate::AllocationCounters; ...@@ -20,7 +20,7 @@ use crate::AllocationCounters;
/// Only compiled without the `custom_allocator` feature; see the module docs. Each thread holds /// Only compiled without the `custom_allocator` feature; see the module docs. Each thread holds
/// its buffer in its own [`ThreadLocalCounter`] and passes it in, so the counter's state lives in /// its buffer in its own [`ThreadLocalCounter`] and passes it in, so the counter's state lives in
/// exactly one place. /// exactly one place.
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
mod global { mod global {
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
...@@ -105,7 +105,7 @@ struct ThreadLocalCounter { ...@@ -105,7 +105,7 @@ struct ThreadLocalCounter {
/// global counter desprite not being allocated yet. It is unsigned so that /// global counter desprite not being allocated yet. It is unsigned so that
/// means the global counter is always equal or greater than the real /// means the global counter is always equal or greater than the real
/// value. /// value.
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
buffer: usize, buffer: usize,
allocation_counters: AllocationCounters, allocation_counters: AllocationCounters,
} }
...@@ -113,7 +113,11 @@ struct ThreadLocalCounter { ...@@ -113,7 +113,11 @@ struct ThreadLocalCounter {
impl ThreadLocalCounter { impl ThreadLocalCounter {
const fn new() -> Self { const fn new() -> Self {
Self { Self {
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(
feature = "custom_allocator",
not(target_family = "wasm"),
not(miri)
)))]
buffer: 0, buffer: 0,
allocation_counters: AllocationCounters::new(), allocation_counters: AllocationCounters::new(),
} }
...@@ -124,7 +128,7 @@ impl ThreadLocalCounter { ...@@ -124,7 +128,7 @@ impl ThreadLocalCounter {
self.allocation_counters.allocations += size; self.allocation_counters.allocations += size;
self.allocation_counters.allocation_count += 1; self.allocation_counters.allocation_count += 1;
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
self.buffered_add(size); self.buffered_add(size);
} }
...@@ -133,7 +137,7 @@ impl ThreadLocalCounter { ...@@ -133,7 +137,7 @@ impl ThreadLocalCounter {
self.allocation_counters.deallocations += size; self.allocation_counters.deallocations += size;
self.allocation_counters.deallocation_count += 1; self.allocation_counters.deallocation_count += 1;
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
self.buffered_remove(size); self.buffered_remove(size);
} }
...@@ -144,7 +148,7 @@ impl ThreadLocalCounter { ...@@ -144,7 +148,7 @@ impl ThreadLocalCounter {
self.allocation_counters.allocations += new_size; self.allocation_counters.allocations += new_size;
self.allocation_counters.allocation_count += 1; self.allocation_counters.allocation_count += 1;
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
{ {
match old_size.cmp(&new_size) { match old_size.cmp(&new_size) {
std::cmp::Ordering::Equal => {} std::cmp::Ordering::Equal => {}
...@@ -155,7 +159,7 @@ impl ThreadLocalCounter { ...@@ -155,7 +159,7 @@ impl ThreadLocalCounter {
} }
fn unload(&mut self) { fn unload(&mut self) {
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
global::flush_all(&mut self.buffer); global::flush_all(&mut self.buffer);
self.allocation_counters = AllocationCounters::default(); self.allocation_counters = AllocationCounters::default();
} }
...@@ -279,7 +283,7 @@ mod tests { ...@@ -279,7 +283,7 @@ mod tests {
/// read — it is process-wide, and this binary installs [`crate::TurboMalloc`] as its global /// read — it is process-wide, and this binary installs [`crate::TurboMalloc`] as its global
/// allocator, so every other thread moves it concurrently. `buffer` is thread-local and /// allocator, so every other thread moves it concurrently. `buffer` is thread-local and
/// exact. /// exact.
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
#[test] #[test]
fn counting() { fn counting() {
use super::global::{MAX_BUFFER, TARGET_BUFFER}; use super::global::{MAX_BUFFER, TARGET_BUFFER};
......
...@@ -96,7 +96,7 @@ impl TurboMalloc { ...@@ -96,7 +96,7 @@ impl TurboMalloc {
/// Without the `custom_allocator` feature this is a process-wide live-bytes counter instead, /// Without the `custom_allocator` feature this is a process-wide live-bytes counter instead,
/// which is approximate because threads buffer their updates. /// which is approximate because threads buffer their updates.
pub fn memory_usage() -> usize { pub fn memory_usage() -> usize {
#[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] #[cfg(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri)))]
{ {
// `current_commit` is a relaxed atomic load, but `mi_process_info` also calls // `current_commit` is a relaxed atomic load, but `mi_process_info` also calls
// `_mi_prim_process_info`, which is a `getrusage` (plus a `task_info` on macOS). All // `_mi_prim_process_info`, which is a `getrusage` (plus a `task_info` on macOS). All
...@@ -117,7 +117,7 @@ impl TurboMalloc { ...@@ -117,7 +117,7 @@ impl TurboMalloc {
} }
current_commit current_commit
} }
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
{ {
self::counter::get() self::counter::get()
} }
...@@ -139,11 +139,11 @@ impl TurboMalloc { ...@@ -139,11 +139,11 @@ impl TurboMalloc {
/// force=true: do all the work of `process=false` and then process global shared structures and /// force=true: do all the work of `process=false` and then process global shared structures and
/// return memory to the OS if possible, this is much slower and should only be done rarely. /// return memory to the OS if possible, this is much slower and should only be done rarely.
pub fn collect(force: bool) { pub fn collect(force: bool) {
#[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] #[cfg(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri)))]
unsafe { unsafe {
libmimalloc_sys::mi_collect(force); libmimalloc_sys::mi_collect(force);
} }
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
{ {
let _ = force; let _ = force;
} }
...@@ -178,17 +178,17 @@ impl TurboMalloc { ...@@ -178,17 +178,17 @@ impl TurboMalloc {
/// Get the allocator for this platform that we should wrap with TurboMalloc. /// Get the allocator for this platform that we should wrap with TurboMalloc.
#[inline] #[inline]
fn base_alloc() -> &'static impl GlobalAlloc { fn base_alloc() -> &'static impl GlobalAlloc {
#[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] #[cfg(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri)))]
return &mimalloc::MiMalloc; return &mimalloc::MiMalloc;
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
return &std::alloc::System; return &std::alloc::System;
} }
#[allow(unused_variables)] #[allow(unused_variables)]
unsafe fn base_alloc_size(ptr: *const u8, layout: Layout) -> usize { unsafe fn base_alloc_size(ptr: *const u8, layout: Layout) -> usize {
#[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] #[cfg(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri)))]
return unsafe { mimalloc::MiMalloc.usable_size(ptr) }; return unsafe { mimalloc::MiMalloc.usable_size(ptr) };
#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"), not(miri))))]
return layout.size(); return layout.size();
} }
...@@ -271,7 +271,7 @@ mod tests { ...@@ -271,7 +271,7 @@ mod tests {
// On all supported platforms the value must be reported. // On all supported platforms the value must be reported.
#[cfg(any( #[cfg(any(
all(target_os = "linux", not(target_family = "wasm")), all(target_os = "linux", not(target_family = "wasm"), not(miri)),
target_os = "macos", target_os = "macos",
windows, windows,
))] ))]
...@@ -279,7 +279,7 @@ mod tests { ...@@ -279,7 +279,7 @@ mod tests {
// On unsupported platforms we expect None and have nothing further to assert. // On unsupported platforms we expect None and have nothing further to assert.
#[cfg(not(any( #[cfg(not(any(
all(target_os = "linux", not(target_family = "wasm")), all(target_os = "linux", not(target_family = "wasm"), not(miri)),
target_os = "macos", target_os = "macos",
windows, windows,
)))] )))]
......
...@@ -18,7 +18,7 @@ fn clamp_percent(value: f64) -> u8 { ...@@ -18,7 +18,7 @@ fn clamp_percent(value: f64) -> u8 {
value.round().clamp(0.0, 100.0) as u8 value.round().clamp(0.0, 100.0) as u8
} }
#[cfg(all(target_os = "linux", not(target_family = "wasm")))] #[cfg(all(target_os = "linux", not(target_family = "wasm"), not(miri)))]
mod platform { mod platform {
use super::clamp_percent; use super::clamp_percent;
...@@ -184,7 +184,7 @@ mod platform { ...@@ -184,7 +184,7 @@ mod platform {
} }
#[cfg(not(any( #[cfg(not(any(
all(target_os = "linux", not(target_family = "wasm")), all(target_os = "linux", not(target_family = "wasm"), not(miri)),
target_os = "macos", target_os = "macos",
windows, windows,
)))] )))]
......
...@@ -697,6 +697,7 @@ mod tests { ...@@ -697,6 +697,7 @@ mod tests {
assert_eq!(runner.total_queued(), 3); assert_eq!(runner.total_queued(), 3);
} }
#[cfg(not(miri))]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_cpu_bound_tasks() { async fn test_cpu_bound_tasks() {
struct ExecutorImpl; struct ExecutorImpl;
......
...@@ -916,6 +916,8 @@ mod tests { ...@@ -916,6 +916,8 @@ mod tests {
} }
/// Sustained work keeps workers alive rather than churning them: /// Sustained work keeps workers alive rather than churning them:
// This test is too slow to run under Miri.
#[cfg(not(miri))]
#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_unbounded_busy_queue_does_not_churn_workers() { async fn test_unbounded_busy_queue_does_not_churn_workers() {
const ITEMS: usize = 20_000; const ITEMS: usize = 20_000;
......
...@@ -880,6 +880,8 @@ mod tests { ...@@ -880,6 +880,8 @@ mod tests {
assert_ne!(lint_lightningcss(code), vec![], "lightningcss: {code}"); assert_ne!(lint_lightningcss(code), vec![], "lightningcss: {code}");
} }
// Lightning CSS currently triggers a Miri Stacked Borrows violation in its string parser.
#[cfg(not(miri))]
#[test] #[test]
fn css_module_pure_lint() { fn css_module_pure_lint() {
assert_lint_success( assert_lint_success(
...@@ -1008,6 +1010,8 @@ mod tests { ...@@ -1008,6 +1010,8 @@ mod tests {
); );
} }
// Lightning CSS currently triggers a Miri Stacked Borrows violation in its string parser.
#[cfg(not(miri))]
#[test] #[test]
fn strip_bom_lets_lightningcss_parse() { fn strip_bom_lets_lightningcss_parse() {
let with_bom = "\u{feff}@layer a {}"; let with_bom = "\u{feff}@layer a {}";
......
...@@ -481,108 +481,172 @@ mod tests { ...@@ -481,108 +481,172 @@ mod tests {
use crate::analyzer::{Bump, ConstantValue, JsValue, ThreadLocal, graph::EvalContext}; use crate::analyzer::{Bump, ConstantValue, JsValue, ThreadLocal, graph::EvalContext};
// A leaked arena for building test `JsValue`s with a `'static` lifetime. Tests are #[derive(Clone, Copy)]
// short-lived processes, so the leak is inconsequential. enum TestAtom {
fn test_arena() -> &'static Bump { Num(f64),
Box::leak(Box::new(Bump::new())) Str(&'static str),
True,
False,
Null,
Undefined,
}
impl TestAtom {
fn build<'a>(self, _arena: &'a Bump) -> JsValue<'a> {
match self {
TestAtom::Num(value) => JsValue::from(value),
TestAtom::Str(value) => JsValue::from(value),
TestAtom::True => ConstantValue::True.into(),
TestAtom::False => ConstantValue::False.into(),
TestAtom::Null => ConstantValue::Null.into(),
TestAtom::Undefined => ConstantValue::Undefined.into(),
}
}
} }
// `construct_test_ternary(cons, alt)` builds a ternary with an unknown test condition. #[derive(Clone, Copy)]
fn construct_test_ternary(cons: JsValue<'static>, alt: JsValue<'static>) -> JsValue<'static> { enum TestValue {
JsValue::tenary( Atom(TestAtom),
test_arena(), PromiseNull,
JsValue::unknown_empty(false, rcstr!("test")), Ternary(TestAtom, TestAtom),
cons, }
alt,
) impl TestValue {
fn build(self, arena: &Bump) -> JsValue<'_> {
match self {
TestValue::Atom(value) => value.build(arena),
TestValue::PromiseNull => JsValue::promise(arena, ConstantValue::Null.into()),
TestValue::Ternary(consequent, alternate) => JsValue::tenary(
arena,
JsValue::unknown_empty(false, rcstr!("test")),
consequent.build(arena),
alternate.build(arena),
),
}
}
} }
use TestAtom::*;
use TestValue::*;
#[rstest] #[rstest]
#[case(JsValue::from(1.0))] #[case(Atom(Num(1.0)))]
#[case(JsValue::from("hi"))] #[case(Atom(Str("hi")))]
#[case(ConstantValue::True.into())] #[case(Atom(True))]
#[case(JsValue::promise(test_arena(), ConstantValue::Null.into()))] #[case(PromiseNull)]
#[case(construct_test_ternary(JsValue::from(1.0), JsValue::from("hi")))] #[case(Ternary(Num(1.0), Str("hi")))]
fn is_truthy_positive(#[case] v: JsValue<'static>) { fn is_truthy_positive(#[case] value: TestValue) {
assert_eq!(v.is_truthy(), Some(true), "expected '{v}' to be truthy"); let arena = Bump::new();
let value = value.build(&arena);
assert_eq!(
value.is_truthy(),
Some(true),
"expected '{value}' to be truthy"
);
} }
#[rstest] #[rstest]
#[case(JsValue::from(0.0))] #[case(Atom(Num(0.0)))]
#[case(JsValue::from(""))] #[case(Atom(Str("")))]
#[case(ConstantValue::False.into())] #[case(Atom(False))]
#[case(ConstantValue::Null.into())] #[case(Atom(Null))]
#[case(ConstantValue::Undefined.into())] #[case(Atom(Undefined))]
#[case(construct_test_ternary(JsValue::from(0.0), JsValue::from("")))] #[case(Ternary(Num(0.0), Str("")))]
fn is_truthy_negative(#[case] v: JsValue<'static>) { fn is_truthy_negative(#[case] value: TestValue) {
assert_eq!(v.is_truthy(), Some(false), "expected '{v}' to be falsy"); let arena = Bump::new();
let value = value.build(&arena);
assert_eq!(
value.is_truthy(),
Some(false),
"expected '{value}' to be falsy"
);
} }
#[rstest] #[rstest]
#[case(ConstantValue::Null.into())] #[case(Atom(Null))]
#[case(ConstantValue::Undefined.into())] #[case(Atom(Undefined))]
#[case(construct_test_ternary(ConstantValue::Null.into(), ConstantValue::Undefined.into()))] #[case(Ternary(Null, Undefined))]
fn is_nullish_positive(#[case] v: JsValue<'static>) { fn is_nullish_positive(#[case] value: TestValue) {
assert_eq!(v.is_nullish(), Some(true), "expected '{v}' to be nullish"); let arena = Bump::new();
let value = value.build(&arena);
assert_eq!(
value.is_nullish(),
Some(true),
"expected '{value}' to be nullish"
);
} }
#[rstest] #[rstest]
#[case(JsValue::from(0.0))] #[case(Atom(Num(0.0)))]
#[case(JsValue::from(""))] #[case(Atom(Str("")))]
#[case(JsValue::from("hi"))] #[case(Atom(Str("hi")))]
#[case(ConstantValue::True.into())] #[case(Atom(True))]
#[case(JsValue::promise(test_arena(), ConstantValue::Null.into()))] #[case(PromiseNull)]
#[case(construct_test_ternary(JsValue::from(0.0), JsValue::from("hi")))] #[case(Ternary(Num(0.0), Str("hi")))]
fn is_nullish_negative(#[case] v: JsValue<'static>) { fn is_nullish_negative(#[case] value: TestValue) {
let arena = Bump::new();
let value = value.build(&arena);
assert_eq!( assert_eq!(
v.is_nullish(), value.is_nullish(),
Some(false), Some(false),
"expected '{v}' not to be nullish" "expected '{value}' not to be nullish"
); );
} }
#[rstest] #[rstest]
#[case(JsValue::from("hi"))] #[case(Atom(Str("hi")))]
#[case(JsValue::from(""))] #[case(Atom(Str("")))]
#[case(construct_test_ternary(JsValue::from("a"), JsValue::from("b")))] #[case(Ternary(Str("a"), Str("b")))]
fn is_string_positive(#[case] v: JsValue<'static>) { fn is_string_positive(#[case] value: TestValue) {
assert_eq!(v.is_string(), Some(true), "expected '{v}' to be a string"); let arena = Bump::new();
let value = value.build(&arena);
assert_eq!(
value.is_string(),
Some(true),
"expected '{value}' to be a string"
);
} }
#[rstest] #[rstest]
#[case(JsValue::from(1.0))] #[case(Atom(Num(1.0)))]
#[case(ConstantValue::True.into())] #[case(Atom(True))]
#[case(ConstantValue::Null.into())] #[case(Atom(Null))]
#[case(construct_test_ternary(JsValue::from(1.0), JsValue::from(2.0)))] #[case(Ternary(Num(1.0), Num(2.0)))]
fn is_string_negative(#[case] v: JsValue<'static>) { fn is_string_negative(#[case] value: TestValue) {
let arena = Bump::new();
let value = value.build(&arena);
assert_eq!( assert_eq!(
v.is_string(), value.is_string(),
Some(false), Some(false),
"expected '{v}' not to be a string" "expected '{value}' not to be a string"
); );
} }
#[rstest] #[rstest]
#[case(JsValue::from(""))] #[case(Atom(Str("")))]
#[case(construct_test_ternary(JsValue::from(""), JsValue::from("")))] #[case(Ternary(Str(""), Str("")))]
fn is_empty_string_positive(#[case] v: JsValue<'static>) { fn is_empty_string_positive(#[case] value: TestValue) {
let arena = Bump::new();
let value = value.build(&arena);
assert_eq!( assert_eq!(
v.is_empty_string(), value.is_empty_string(),
Some(true), Some(true),
"expected '{v}' to be an empty string" "expected '{value}' to be an empty string"
); );
} }
#[rstest] #[rstest]
#[case(JsValue::from("hi"))] #[case(Atom(Str("hi")))]
#[case(JsValue::from(1.0))] #[case(Atom(Num(1.0)))]
#[case(ConstantValue::True.into())] #[case(Atom(True))]
#[case(construct_test_ternary(JsValue::from("a"), JsValue::from("b")))] #[case(Ternary(Str("a"), Str("b")))]
fn is_empty_string_negative(#[case] v: JsValue<'static>) { fn is_empty_string_negative(#[case] value: TestValue) {
let arena = Bump::new();
let value = value.build(&arena);
assert_eq!( assert_eq!(
v.is_empty_string(), value.is_empty_string(),
Some(false), Some(false),
"expected '{v}' not to be an empty string" "expected '{value}' not to be an empty string"
); );
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment