Unverified Commit 4a4715e7 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

feat(turbopack): Print a warning about performance when starting with an invalidated cache (#80631)

We want to show a warning to the user when we start up with an invalidated cache, letting them know why `next dev` might be a little slower than usual.

This is done by writing a JSON-encoded `reason_code` value to the invalidation marker file that's then read when starting back up.

I'm using @Cy-Tek's compilation event logging to log this from the JS side, so that log styling matches other Next.js warnings. This is a little tricky because of a chicken-and-egg dependency on turbo-tasks. We must invalidate the cache before constructing the turbo-tasks backend, and then emit the event after turbo-tasks is constructed.

## Testing

Reset the cache using the web UI:

<img src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/HAZVitxRNnZz8QMiPn4a/ac47b80d-d09c-48a8-bdcf-6f6f1bfd8288.png" width="500">

And see a warning logged to the terminal:

<img src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/HAZVitxRNnZz8QMiPn4a/e1860d1a-defd-46c2-9a85-a0c64e26f634.png" width="700">

This user-requested use-case is a bit contrived, we mostly care about automatic invalidation upon panic, as those are the ones that may be surprising to the user.
parent e830e85f
......@@ -35,6 +35,7 @@ use turbo_tasks::{
message_queue::{CompilationEvent, Severity, TimingEvent},
trace::TraceRawVcs,
};
use turbo_tasks_backend::db_invalidation::invalidation_reasons;
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystem, FileSystemPath, get_relative_path_to,
util::uri_from_file,
......@@ -571,9 +572,15 @@ pub async fn project_update(
pub async fn project_invalidate_persistent_cache(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
) -> napi::Result<()> {
tokio::task::spawn_blocking(move || project.turbo_tasks.invalidate_persistent_cache())
.await
.context("panicked while invalidating persistent cache")??;
tokio::task::spawn_blocking(move || {
// TODO: Let the JS caller specify a reason? We need to limit the reasons to ones we know
// how to generate a message for on the Rust side of the FFI.
project
.turbo_tasks
.invalidate_persistent_cache(invalidation_reasons::USER_REQUEST)
})
.await
.context("panicked while invalidating persistent cache")??;
Ok(())
}
......
......@@ -11,12 +11,14 @@ use serde::Serialize;
use tokio::sync::mpsc::Receiver;
use turbo_tasks::{
Effects, OperationVc, ReadRef, TaskId, TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo,
Vc, VcValueType, get_effects, message_queue::CompilationEvent,
task_statistics::TaskStatisticsApi, trace::TraceRawVcs,
Vc, VcValueType, get_effects,
message_queue::{CompilationEvent, Severity},
task_statistics::TaskStatisticsApi,
trace::TraceRawVcs,
};
use turbo_tasks_backend::{
DefaultBackingStorage, GitVersionInfo, NoopBackingStorage, default_backing_storage,
noop_backing_storage,
BackingStorage, DefaultBackingStorage, GitVersionInfo, NoopBackingStorage, StartupCacheState,
db_invalidation::invalidation_reasons, default_backing_storage, noop_backing_storage,
};
use turbo_tasks_fs::FileContent;
use turbopack_core::{
......@@ -150,17 +152,51 @@ impl NextTurboTasks {
}
}
pub fn invalidate_persistent_cache(&self) -> Result<()> {
pub fn invalidate_persistent_cache(&self, reason_code: &str) -> Result<()> {
match self {
NextTurboTasks::Memory(_) => {}
NextTurboTasks::PersistentCaching(turbo_tasks) => {
turbo_tasks.backend().invalidate_storage()?
}
NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks
.backend()
.backing_storage()
.invalidate(reason_code)?,
}
Ok(())
}
}
#[derive(Serialize)]
struct StartupCacheInvalidationEvent {
reason_code: Option<String>,
}
impl CompilationEvent for StartupCacheInvalidationEvent {
fn type_name(&self) -> &'static str {
"StartupCacheInvalidationEvent"
}
fn severity(&self) -> Severity {
Severity::Warning
}
fn message(&self) -> String {
let reason_msg = match self.reason_code.as_deref() {
Some(invalidation_reasons::PANIC) => {
" because we previously detected an internal error in Turbopack"
}
Some(invalidation_reasons::USER_REQUEST) => " as the result of a user request",
_ => "", // ignore unknown reasons
};
format!(
"Turbopack's persistent cache has been deleted{reason_msg}. Builds or page loads may \
be slower as a result."
)
}
fn to_json(&self) -> String {
serde_json::to_string(self).unwrap()
}
}
pub fn create_turbo_tasks(
output_path: PathBuf,
persistent_caching: bool,
......@@ -174,24 +210,24 @@ pub fn create_turbo_tasks(
dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true",
};
NextTurboTasks::PersistentCaching(TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new(
turbo_tasks_backend::BackendOptions {
storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
turbo_tasks_backend::StorageMode::ReadOnly
} else {
turbo_tasks_backend::StorageMode::ReadWrite
}),
dependency_tracking,
..Default::default()
},
default_backing_storage(
&output_path.join("cache/turbopack"),
&version_info,
is_ci,
)?,
),
))
let (backing_storage, cache_state) =
default_backing_storage(&output_path.join("cache/turbopack"), &version_info, is_ci)?;
let tt = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(
turbo_tasks_backend::BackendOptions {
storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() {
turbo_tasks_backend::StorageMode::ReadOnly
} else {
turbo_tasks_backend::StorageMode::ReadWrite
}),
dependency_tracking,
..Default::default()
},
backing_storage,
));
if let StartupCacheState::Invalidated { reason_code } = cache_state {
tt.send_compilation_event(Arc::new(StartupCacheInvalidationEvent { reason_code }));
}
NextTurboTasks::PersistentCaching(tt)
} else {
NextTurboTasks::Memory(TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new(
......
......@@ -731,13 +731,14 @@ function bindingToApi(
)
}
compilationEventsSubscribe() {
compilationEventsSubscribe(eventTypes?: string[]) {
return subscribe<TurbopackResult<CompilationEvent>>(
true,
async (callback) => {
binding.projectCompilationEventsSubscribe(
this._nativeProject,
callback
callback,
eventTypes
)
}
)
......
......@@ -238,9 +238,9 @@ export interface Project {
aggregationMs: number
): AsyncIterableIterator<TurbopackResult<UpdateMessage>>
compilationEventsSubscribe(): AsyncIterableIterator<
TurbopackResult<CompilationEvent>
>
compilationEventsSubscribe(
eventTypes?: string[]
): AsyncIterableIterator<TurbopackResult<CompilationEvent>>
invalidatePersistentCache(): Promise<void>
......
......@@ -19,8 +19,8 @@ import loadConfig from '../../server/config'
import { hasCustomExportOutput } from '../../export/utils'
import { Telemetry } from '../../telemetry/storage'
import { setGlobal } from '../../trace'
import * as Log from '../output/log'
import { isCI } from '../../server/ci-info'
import { backgroundLogCompilationEvents } from '../../shared/lib/turbopack/compilation-events'
export async function turbopackBuild(): Promise<{
duration: number
......@@ -91,32 +91,7 @@ export async function turbopackBuild(): Promise<{
}
)
try {
;(async function logCompilationEvents() {
for await (const event of project.compilationEventsSubscribe()) {
switch (event.severity) {
case 'EVENT':
Log.event(event.message)
break
case 'TRACE':
Log.trace(event.message)
break
case 'INFO':
Log.info(event.message)
break
case 'WARNING':
Log.warn(event.message)
break
case 'ERROR':
Log.error(event.message)
break
case 'FATAL':
Log.error(event.message)
break
default:
break
}
}
})()
backgroundLogCompilationEvents(project)
// Write an empty file in a known location to signal this was built with Turbopack
await fs.writeFile(path.join(distDir, 'turbopack'), '')
......
......@@ -98,6 +98,7 @@ import { getDevOverlayFontMiddleware } from '../../next-devtools/server/font/get
import { devIndicatorServerState } from './dev-indicator-server-state'
import { getDisableDevIndicatorMiddleware } from '../../next-devtools/server/dev-indicator-middleware'
import { getRestartDevServerMiddleware } from '../../next-devtools/server/restart-dev-server-middleware'
import { backgroundLogCompilationEvents } from '../../shared/lib/turbopack/compilation-events'
// import { getSupportedBrowsers } from '../../build/utils'
const wsServer = new ws.Server({ noServer: true })
......@@ -248,6 +249,9 @@ export async function createHotReloaderTurbopack(
memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit,
}
)
backgroundLogCompilationEvents(project, {
eventTypes: ['StartupCacheInvalidationEvent'],
})
setBundlerFindSourceMapImplementation(
getSourceMapFromTurbopack.bind(null, project, projectPath)
)
......
import type { Project } from '../../../build/swc/types'
import * as Log from '../../../build/output/log'
/**
* Subscribes to compilation events for `project` and prints them using the
* `Log` library.
*
* The `signal` argument is partially implemented. The abort may not happen until the next
* compilation event arrives.
*/
export function backgroundLogCompilationEvents(
project: Project,
{ eventTypes, signal }: { eventTypes?: string[]; signal?: AbortSignal } = {}
) {
;(async function () {
for await (const event of project.compilationEventsSubscribe(eventTypes)) {
if (signal?.aborted) {
return
}
switch (event.severity) {
case 'EVENT':
Log.event(event.message)
break
case 'TRACE':
Log.trace(event.message)
break
case 'INFO':
Log.info(event.message)
break
case 'WARNING':
Log.warn(event.message)
break
case 'ERROR':
Log.error(event.message)
break
case 'FATAL':
Log.error(event.message)
break
default:
break
}
}
})()
}
......@@ -41,6 +41,7 @@ rand = { workspace = true }
rayon = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_path_to_error = { workspace = true }
smallvec = { workspace = true }
tokio = { workspace = true }
......@@ -54,7 +55,6 @@ turbo-tasks-testing = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] }
regex = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
rstest = { workspace = true }
......
......@@ -210,8 +210,8 @@ impl<B: BackingStorage> TurboTasksBackend<B> {
)))
}
pub fn invalidate_storage(&self) -> Result<()> {
self.0.backing_storage.invalidate()
pub fn backing_storage(&self) -> &B {
&self.0.backing_storage
}
}
......
......@@ -10,7 +10,36 @@ use crate::{
utils::chunked_vec::ChunkedVec,
};
pub trait BackingStorage: 'static + Send + Sync {
/// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by
/// [`default_backing_storage`] or [`noop_backing_storage`].
///
/// This trait is [sealed]. External crates are not allowed to implement it.
///
/// [`default_backing_storage`]: crate::default_backing_storage
/// [`noop_backing_storage`]: crate::noop_backing_storage
/// [`TurboTasksBackend::new`]: crate::TurboTasksBackend::new
/// [sealed]: https://predr.ag/blog/definitive-guide-to-sealed-traits-in-rust/
pub trait BackingStorage: BackingStorageSealed {
/// Called when the database should be invalidated upon re-initialization.
///
/// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh
/// database. If this happens, there's no point in writing anything else to disk, or flushing
/// during [`KeyValueDatabase::shutdown`].
///
/// This can be implemented by calling [`invalidate_db`] with
/// the database's non-versioned base path.
///
/// [`KeyValueDatabase::shutdown`]: crate::database::key_value_database::KeyValueDatabase::shutdown
/// [`invalidate_db`]: crate::database::db_invalidation::invalidate_db
fn invalidate(&self, reason_code: &str) -> Result<()>;
}
/// Private methods used by [`BackingStorage`]. This trait is `pub` (because of the sealed-trait
/// pattern), but should not be exported outside of the crate.
///
/// [`BackingStorage`] is exported for documentation reasons and to expose the public
/// [`BackingStorage::invalidate`] method.
pub trait BackingStorageSealed: 'static + Send + Sync {
type ReadTransaction<'l>;
fn lower_read_transaction<'l: 'i + 'r, 'i: 'r, 'r>(
tx: &'r Self::ReadTransaction<'l>,
......@@ -63,16 +92,6 @@ pub trait BackingStorage: 'static + Send + Sync {
category: TaskDataCategory,
) -> Result<Vec<CachedDataItem>>;
/// Called when the database should be invalidated upon re-initialization.
///
/// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh
/// database. If this happens, there's no point in writing anything else to disk, or flushing
/// during [`KeyValueDatabase::shutdown`].
///
/// This can be implemented by calling [`crate::database::db_invalidation::invalidate_db`] with
/// the database's non-versioned base path.
fn invalidate(&self) -> Result<()>;
fn shutdown(&self) -> Result<()> {
Ok(())
}
......
use std::{
fs::{self, read_dir},
io::{self, ErrorKind},
borrow::Cow,
fs::{self, File, read_dir},
io::{self, BufReader, BufWriter, ErrorKind, Write},
path::Path,
};
use anyhow::Context;
use serde::{Deserialize, Serialize};
const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db";
/// Atomically write an invalidation marker.
const EXPLANATION: &str = "The cache database has been invalidated. The existence of this file \
will cause the cache directory to be cleaned up the next time \
Turbopack starts up.";
const EASTER_EGG: &str =
"you just wrote me, and this is crazy, but if you see me, delete everything maybe?";
/// The data written to the file at [`INVALIDATION_MARKER`].
#[derive(Serialize, Deserialize)]
struct InvalidationFile<'a> {
#[serde(skip_deserializing)]
_explanation: Option<&'static str>,
#[serde(skip_deserializing)]
_easter_egg: Option<&'static str>,
/// See [`StartupCacheState::Invalidated::reason_code`].
reason_code: Cow<'a, str>,
}
/// Information about if there's was a pre-existing cache or if the cache was detected as
/// invalidated during startup.
///
/// If the cache was invalidated, the application may choose to show a warning to the user or log it
/// to telemetry.
///
/// This value is returned by [`crate::turbo_backing_storage`] and
/// [`crate::default_backing_storage`].
pub enum StartupCacheState {
NoCache,
Cached,
Invalidated {
/// A short code passed to [`BackingStorage::invalidate`]. This value is
/// application-specific.
///
/// If the value is `None` or doesn't match an expected value, the application should just
/// treat this reason as unknown. The invalidation file may have been corrupted or
/// modified by an external tool.
///
/// See [`invalidation_reasons`] for some common reason codes.
///
/// [`BackingStorage::invalidate`]: crate::BackingStorage::invalidate
reason_code: Option<String>,
},
}
/// Common invalidation reason codes. The application or libraries it uses may choose to use these
/// reasons, or it may define it's own reasons.
pub mod invalidation_reasons {
/// This invalidation reason is used by [`crate::turbo_backing_storage`] when the database was
/// invalidated by a panic.
pub const PANIC: &str = concat!(module_path!(), "::PANIC");
/// Indicates that the user explicitly clicked a button or ran a command that invalidates the
/// cache.
pub const USER_REQUEST: &str = concat!(module_path!(), "::USER_REQUEST");
}
/// Atomically create an invalidation marker.
///
/// Makes a best-effort attempt to write `reason_code` to the file, but ignores any failure with
/// writing to the file.
///
/// Because attempting to delete currently open database files could cause issues, actual deletion
/// of files is deferred until the next start-up (in [`check_db_invalidation_and_cleanup`]).
......@@ -18,9 +77,29 @@ const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db";
///
/// This should be run with the base (non-versioned) path, as that likely aligns closest with user
/// expectations (e.g. if they're clearing the cache for disk space reasons).
pub fn invalidate_db(base_path: &Path) -> anyhow::Result<()> {
match fs::write(base_path.join(INVALIDATION_MARKER), [0u8; 0]) {
Ok(_) => Ok(()),
///
/// In most cases, you should prefer a higher-level API like [`crate::BackingStorage::invalidate`]
/// to this one.
pub(crate) fn invalidate_db(base_path: &Path, reason_code: &str) -> anyhow::Result<()> {
match File::create_new(base_path.join(INVALIDATION_MARKER)) {
Ok(file) => {
let mut writer = BufWriter::new(file);
// ignore errors: We've already successfully invalidated the cache just by creating the
// marker file, writing the reason_code is best-effort.
let _ = serde_json::to_writer_pretty(
&mut writer,
&InvalidationFile {
_explanation: Some(EXPLANATION),
_easter_egg: Some(EASTER_EGG),
reason_code: Cow::Borrowed(reason_code),
},
);
let _ = writer.flush();
Ok(())
}
// the database was already invalidated, avoid overwriting that reason or risking concurrent
// writes to the same file.
Err(err) if err.kind() == ErrorKind::AlreadyExists => Ok(()),
// just ignore if the cache directory doesn't exist at all
Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
Err(err) => Err(err).context("Failed to invalidate database"),
......@@ -31,13 +110,37 @@ pub fn invalidate_db(base_path: &Path) -> anyhow::Result<()> {
/// delete any invalidated database files.
///
/// This should be run with the base (non-versioned) path.
pub fn check_db_invalidation_and_cleanup(base_path: &Path) -> anyhow::Result<()> {
if fs::exists(base_path.join(INVALIDATION_MARKER))? {
// if this cleanup fails, we might try to open an invalid database later, so it's best to
// just propagate the error here.
cleanup_db(base_path)?;
};
Ok(())
///
/// In most cases, you should prefer a higher-level API like
/// [`crate::KeyValueDatabaseBackingStorage::open_versioned_on_disk`] to this one.
pub(crate) fn check_db_invalidation_and_cleanup(
base_path: &Path,
) -> anyhow::Result<StartupCacheState> {
match File::open(base_path.join(INVALIDATION_MARKER)) {
Ok(file) => {
// Best-effort: Try to read the reason_code from the file, if the file format is
// corrupted (or anything else) just use `None`.
let reason_code = serde_json::from_reader::<_, InvalidationFile>(BufReader::new(file))
.ok()
.map(|contents| contents.reason_code.into_owned());
// `file` is dropped at this point: That's important for Windows where we can't delete
// open files.
// if this cleanup fails, we might try to open an invalid database later, so it's best
// to just propagate the error here.
cleanup_db(base_path)?;
Ok(StartupCacheState::Invalidated { reason_code })
}
Err(err) if err.kind() == ErrorKind::NotFound => {
if fs::exists(base_path)? {
Ok(StartupCacheState::Cached)
} else {
Ok(StartupCacheState::NoCache)
}
}
Err(err) => Err(err)
.with_context(|| format!("Failed to check for {INVALIDATION_MARKER} in {base_path:?}")),
}
}
/// Helper for [`check_db_invalidation_and_cleanup`]. You can call this to explicitly clean up a
......@@ -45,7 +148,7 @@ pub fn check_db_invalidation_and_cleanup(base_path: &Path) -> anyhow::Result<()>
///
/// You should not run this if the database has not yet been invalidated, as this operation is not
/// atomic and could result in a partially-deleted and corrupted database.
pub fn cleanup_db(base_path: &Path) -> anyhow::Result<()> {
pub(crate) fn cleanup_db(base_path: &Path) -> anyhow::Result<()> {
cleanup_db_inner(base_path).with_context(|| {
format!(
"Unable to remove invalid database. If this issue persists you can work around by \
......
......@@ -21,10 +21,10 @@ use turbo_tasks::{
use crate::{
GitVersionInfo,
backend::{AnyOperation, TaskDataCategory},
backing_storage::BackingStorage,
backing_storage::{BackingStorage, BackingStorageSealed},
data::CachedDataItem,
database::{
db_invalidation::{check_db_invalidation_and_cleanup, invalidate_db},
db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db},
db_versioning::handle_db_versioning,
key_value_database::{KeySpace, KeyValueDatabase},
write_batch::{
......@@ -32,6 +32,7 @@ use crate::{
WriteBuffer,
},
},
db_invalidation::invalidation_reasons,
utils::chunked_vec::ChunkedVec,
};
......@@ -120,15 +121,21 @@ pub struct KeyValueDatabaseBackingStorageInner<T: KeyValueDatabase> {
base_path: Option<PathBuf>,
/// Used to skip calling [`invalidate_db`] when the database has already been invalidated.
invalidated: Mutex<bool>,
/// We configure a panic hook to invalidate the cache. This guard cleans up our panic hook upon
/// drop.
_panic_hook_guard: Option<PanicHookGuard>,
}
pub struct KeyValueDatabaseBackingStorage<T: KeyValueDatabase> {
// wrapped so that `register_panic_hook` can hold a weak reference to `inner`.
inner: Arc<KeyValueDatabaseBackingStorageInner<T>>,
}
/// A wrapper type used by [`crate::turbo_backing_storage`] and [`crate::noop_backing_storage`].
///
/// Wraps a low-level key-value database into a higher-level [`BackingStorage`] type.
impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorage<T> {
pub fn new_in_memory(database: T) -> Self {
pub(crate) fn new_in_memory(database: T) -> Self {
Self {
inner: Arc::new(KeyValueDatabaseBackingStorageInner {
database,
......@@ -139,19 +146,30 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorage<T> {
}
}
pub fn open_versioned_on_disk(
/// Handles boilerplate logic for an on-disk persisted database with versioning.
///
/// - Creates a directory per version, with a maximum number of old versions and performs
/// automatic cleanup of old versions.
/// - Checks for a database invalidation marker file, and cleans up the database as needed.
/// - [Registers a dynamic panic hook][turbo_tasks::panic_hooks] to invalidate the database upon
/// a panic. This invalidates the database using [`invalidation_reasons::PANIC`].
///
/// Along with returning a [`KeyValueDatabaseBackingStorage`], this returns a
/// [`StartupCacheState`], which can be used by the application for logging information to the
/// user or telemetry about the cache.
pub(crate) fn open_versioned_on_disk(
base_path: PathBuf,
version_info: &GitVersionInfo,
is_ci: bool,
database: impl FnOnce(PathBuf) -> Result<T>,
) -> Result<Self>
) -> Result<(Self, StartupCacheState)>
where
T: Send + Sync + 'static,
{
check_db_invalidation_and_cleanup(&base_path)?;
let startup_cache_state = check_db_invalidation_and_cleanup(&base_path)?;
let versioned_path = handle_db_versioning(&base_path, version_info, is_ci)?;
let database = (database)(versioned_path)?;
Ok(Self {
let backing_storage = Self {
inner: Arc::new_cyclic(
move |weak_inner: &Weak<KeyValueDatabaseBackingStorageInner<T>>| {
let panic_hook_guard = if should_invalidate_on_panic() {
......@@ -164,7 +182,7 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorage<T> {
// or turbo-tasks failed, and it may be hard to recover. We don't want
// the cache to stick around, as that may persist bugs. Make a
// best-effort attempt to invalidate the database (ignoring failures).
let _ = inner.invalidate();
let _ = inner.invalidate(invalidation_reasons::PANIC);
})))
} else {
None
......@@ -177,7 +195,8 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorage<T> {
}
},
),
})
};
Ok((backing_storage, startup_cache_state))
}
}
......@@ -197,7 +216,7 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorageInner<T> {
}
}
fn invalidate(&self) -> Result<()> {
fn invalidate(&self, reason_code: &str) -> Result<()> {
// `base_path` can be `None` for a `NoopKvDb`
if let Some(base_path) = &self.base_path {
// Invalidation could happen frequently if there's a bunch of panics. We only need to
......@@ -212,7 +231,7 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorageInner<T> {
// Invalidate first, as it's a very fast atomic operation. `prevent_writes` is allowed
// to be slower (e.g. wait for a lock) and is allowed to corrupt the database with
// partial writes.
invalidate_db(base_path)?;
invalidate_db(base_path, reason_code)?;
self.database.prevent_writes();
// Avoid redundant invalidations from future panics
*invalidated_guard = true;
......@@ -232,6 +251,14 @@ impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorageInner<T> {
impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
for KeyValueDatabaseBackingStorage<T>
{
fn invalidate(&self, reason_code: &str) -> Result<()> {
self.inner.invalidate(reason_code)
}
}
impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
for KeyValueDatabaseBackingStorage<T>
{
type ReadTransaction<'l> = T::ReadTransaction<'l>;
......@@ -567,10 +594,6 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
.with_context(|| format!("Looking up data for {task_id} from database failed"))
}
fn invalidate(&self) -> Result<()> {
self.inner.invalidate()
}
fn shutdown(&self) -> Result<()> {
self.inner.database.shutdown()
}
......
......@@ -18,7 +18,10 @@ use anyhow::Result;
use crate::database::{noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase};
pub use crate::{
backend::{BackendOptions, StorageMode, TurboTasksBackend},
database::db_versioning::GitVersionInfo,
backing_storage::BackingStorage,
database::{
db_invalidation, db_invalidation::StartupCacheState, db_versioning::GitVersionInfo,
},
kv_backing_storage::KeyValueDatabaseBackingStorage,
};
......@@ -33,12 +36,20 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
>,
>;
/// Creates an [`lmdb`]-based `BackingStorage` to be passed to [`TurboTasksBackend::new`].
///
/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
///
/// This is backend is slower than [`turbo_backing_storage`], but it's a known-good database that
/// can be used when reproducing user-reported issues to isolate bugs.
///
/// When the `lmdb` cargo feature is enabled, [`default_backing_storage`] will return this value.
#[cfg(feature = "lmdb")]
pub fn lmdb_backing_storage(
base_path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<LmdbBackingStorage> {
) -> Result<(LmdbBackingStorage, StartupCacheState)> {
use crate::database::{
fresh_db_optimization::{FreshDbOptimization, is_fresh},
read_transaction_cache::ReadTransactionCache,
......@@ -62,11 +73,17 @@ pub fn lmdb_backing_storage(
pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
///
/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
///
/// This is the fastest most-tested implementation of `BackingStorage`, and is normally returned by
/// [`default_backing_storage`].
pub fn turbo_backing_storage(
base_path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<TurboBackingStorage> {
) -> Result<(TurboBackingStorage, StartupCacheState)> {
KeyValueDatabaseBackingStorage::open_versioned_on_disk(
base_path.to_owned(),
version_info,
......@@ -77,6 +94,7 @@ pub fn turbo_backing_storage(
pub type NoopBackingStorage = KeyValueDatabaseBackingStorage<NoopKvDb>;
/// Creates an no-op in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`].
pub fn noop_backing_storage() -> NoopBackingStorage {
KeyValueDatabaseBackingStorage::new_in_memory(NoopKvDb)
}
......@@ -84,23 +102,22 @@ pub fn noop_backing_storage() -> NoopBackingStorage {
#[cfg(feature = "lmdb")]
pub type DefaultBackingStorage = LmdbBackingStorage;
#[cfg(feature = "lmdb")]
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<DefaultBackingStorage> {
lmdb_backing_storage(path, version_info, is_ci)
}
#[cfg(not(feature = "lmdb"))]
pub type DefaultBackingStorage = TurboBackingStorage;
#[cfg(not(feature = "lmdb"))]
/// Calls [`turbo_backing_storage`] (recommended) or `lmdb_backing_storage`, depending on if the
/// `lmdb` cargo feature is enabled.
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<DefaultBackingStorage> {
turbo_backing_storage(path, version_info, is_ci)
) -> Result<(DefaultBackingStorage, StartupCacheState)> {
#[cfg(feature = "lmdb")]
{
lmdb_backing_storage(path, version_info, is_ci)
}
#[cfg(not(feature = "lmdb"))]
{
turbo_backing_storage(path, version_info, is_ci)
}
}
......@@ -17,7 +17,7 @@
dirty: false,
},
false
).unwrap()
).unwrap().0
)
)
}
......@@ -17,7 +17,7 @@
dirty: false,
},
false
).unwrap()
).unwrap().0
)
)
}
......@@ -284,7 +284,8 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
},
false,
)
.unwrap(),
.unwrap()
.0,
))
});
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment