Unverified Commit 7c6c06de authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: show timing event for database compaction (#82369)

### What?

- show timing event for database compaction
- show timing event for persistent caching in dev too
parent 9934b378
...@@ -258,7 +258,7 @@ export async function createHotReloaderTurbopack( ...@@ -258,7 +258,7 @@ export async function createHotReloaderTurbopack(
} }
) )
backgroundLogCompilationEvents(project, { backgroundLogCompilationEvents(project, {
eventTypes: ['StartupCacheInvalidationEvent'], eventTypes: ['StartupCacheInvalidationEvent', 'TimingEvent'],
}) })
setBundlerFindSourceMapImplementation( setBundlerFindSourceMapImplementation(
getSourceMapFromTurbopack.bind(null, project, projectPath) getSourceMapFromTurbopack.bind(null, project, projectPath)
......
...@@ -650,7 +650,7 @@ impl TurboPersistence { ...@@ -650,7 +650,7 @@ impl TurboPersistence {
/// files is above the given threshold. The coverage is the average number of SST files that /// files is above the given threshold. The coverage is the average number of SST files that
/// need to be read to find a key. It also limits the maximum number of SST files that are /// need to be read to find a key. It also limits the maximum number of SST files that are
/// merged at once, which is the main factor for the runtime of the compaction. /// merged at once, which is the main factor for the runtime of the compaction.
pub fn compact(&self, compact_config: &CompactConfig) -> Result<()> { pub fn compact(&self, compact_config: &CompactConfig) -> Result<bool> {
if self.read_only { if self.read_only {
bail!("Compaction is not allowed on a read only database"); bail!("Compaction is not allowed on a read only database");
} }
...@@ -689,7 +689,8 @@ impl TurboPersistence { ...@@ -689,7 +689,8 @@ impl TurboPersistence {
.context("Failed to compact database")?; .context("Failed to compact database")?;
} }
if !new_meta_files.is_empty() { let has_changes = !new_meta_files.is_empty();
if has_changes {
self.commit(CommitOptions { self.commit(CommitOptions {
new_meta_files, new_meta_files,
new_sst_files, new_sst_files,
...@@ -704,7 +705,7 @@ impl TurboPersistence { ...@@ -704,7 +705,7 @@ impl TurboPersistence {
self.active_write_operation.store(false, Ordering::Release); self.active_write_operation.store(false, Ordering::Release);
Ok(()) Ok(has_changes)
} }
/// Internal function to perform a compaction. /// Internal function to perform a compaction.
......
use std::{ use std::{cmp::max, path::PathBuf, sync::Arc, thread::available_parallelism, time::Instant};
cmp::max,
path::PathBuf,
sync::Arc,
thread::{JoinHandle, available_parallelism, spawn},
};
use anyhow::{Ok, Result}; use anyhow::{Ok, Result};
use parking_lot::Mutex; use parking_lot::Mutex;
use turbo_persistence::{ use turbo_persistence::{
ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer, ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer,
}; };
use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks};
use crate::database::{ use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase}, key_value_database::{KeySpace, KeyValueDatabase},
...@@ -84,7 +80,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -84,7 +80,7 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> { ) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>> {
// Wait for the compaction to finish // Wait for the compaction to finish
if let Some(join_handle) = self.compact_join_handle.lock().take() { if let Some(join_handle) = self.compact_join_handle.lock().take() {
join_handle.join().unwrap()?; join_handle.join()?;
} }
// Start a new write batch // Start a new write batch
Ok(WriteBatch::concurrent(TurboWriteBatch { Ok(WriteBatch::concurrent(TurboWriteBatch {
...@@ -100,23 +96,44 @@ impl KeyValueDatabase for TurboKeyValueDatabase { ...@@ -100,23 +96,44 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
fn shutdown(&self) -> Result<()> { fn shutdown(&self) -> Result<()> {
// Wait for the compaction to finish // Wait for the compaction to finish
if let Some(join_handle) = self.compact_join_handle.lock().take() { if let Some(join_handle) = self.compact_join_handle.lock().take() {
join_handle.join().unwrap()?; join_handle.join()?;
} }
// Compact the database on shutdown // Compact the database on shutdown
self.db.compact(&CompactConfig { if self.is_ci {
max_merge_segment_count: if self.is_ci { // Fully compact in CI to reduce cache size
// Fully compact in CI to reduce cache size do_compact(&self.db, "Finished database compaction", usize::MAX)?;
usize::MAX } else {
} else { // Compact with a reasonable limit in non-CI environments
available_parallelism().map_or(4, |c| max(4, c.get())) do_compact(
}, &self.db,
..COMPACT_CONFIG "Finished database compaction",
})?; available_parallelism().map_or(4, |c| max(4, c.get())),
)?;
}
// Shutdown the database // Shutdown the database
self.db.shutdown() self.db.shutdown()
} }
} }
fn do_compact(
db: &TurboPersistence,
message: &'static str,
max_merge_segment_count: usize,
) -> Result<()> {
let start = Instant::now();
// Compact the database with the given max merge segment count
let ran = db.compact(&CompactConfig {
max_merge_segment_count,
..COMPACT_CONFIG
})?;
if ran {
let elapsed = start.elapsed();
turbo_tasks()
.send_compilation_event(Arc::new(TimingEvent::new(message.to_string(), elapsed)));
}
Ok(())
}
pub struct TurboWriteBatch<'a> { pub struct TurboWriteBatch<'a> {
batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>, batch: turbo_persistence::WriteBatch<WriteBuffer<'static>, 5>,
db: &'a Arc<TurboPersistence>, db: &'a Arc<TurboPersistence>,
...@@ -144,12 +161,12 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> { ...@@ -144,12 +161,12 @@ impl<'a> BaseWriteBatch<'a> for TurboWriteBatch<'a> {
if let Some(compact_join_handle) = self.compact_join_handle { if let Some(compact_join_handle) = self.compact_join_handle {
// Start a new compaction in the background // Start a new compaction in the background
let db = self.db.clone(); let db = self.db.clone();
let handle = spawn(move || { let handle = spawn(async move {
db.compact(&CompactConfig { do_compact(
max_merge_segment_count: available_parallelism() &db,
.map_or(4, |c| max(4, c.get() / 2)), "Finished database compaction",
..COMPACT_CONFIG available_parallelism().map_or(4, |c| max(4, c.get() / 2)),
}) )
}); });
compact_join_handle.lock().replace(handle); compact_join_handle.lock().replace(handle);
} }
......
...@@ -118,7 +118,7 @@ use rustc_hash::FxHasher; ...@@ -118,7 +118,7 @@ use rustc_hash::FxHasher;
pub use scope::scope; pub use scope::scope;
pub use serialization_invalidation::SerializationInvalidator; pub use serialization_invalidation::SerializationInvalidator;
pub use shrink_to_fit::ShrinkToFit; pub use shrink_to_fit::ShrinkToFit;
pub use spawn::{JoinHandle, spawn, spawn_blocking, spawn_thread}; pub use spawn::{JoinHandle, block_for_future, spawn, spawn_blocking, spawn_thread};
pub use state::{State, TransientState}; pub use state::{State, TransientState};
pub use task::{SharedReference, TypedSharedReference, task_input::TaskInput}; pub use task::{SharedReference, TypedSharedReference, task_input::TaskInput};
pub use task_execution_reason::TaskExecutionReason; pub use task_execution_reason::TaskExecutionReason;
......
...@@ -1058,27 +1058,30 @@ impl<B: Backend + 'static> TurboTasks<B> { ...@@ -1058,27 +1058,30 @@ impl<B: Backend + 'static> TurboTasks<B> {
} }
pub async fn stop_and_wait(&self) { pub async fn stop_and_wait(&self) {
self.backend.stopping(self); turbo_tasks_future_scope(self.pin(), async move {
self.stopped.store(true, Ordering::Release); self.backend.stopping(self);
{ self.stopped.store(true, Ordering::Release);
let listener = self {
.event let listener = self
.listen_with_note(|| || "wait for stop".to_string()); .event
if self.currently_scheduled_tasks.load(Ordering::Acquire) != 0 { .listen_with_note(|| || "wait for stop".to_string());
listener.await; if self.currently_scheduled_tasks.load(Ordering::Acquire) != 0 {
listener.await;
}
} }
}
{
let listener = self.event_background.listen();
if self
.currently_scheduled_background_jobs
.load(Ordering::Acquire)
!= 0
{ {
listener.await; let listener = self.event_background.listen();
if self
.currently_scheduled_background_jobs
.load(Ordering::Acquire)
!= 0
{
listener.await;
}
} }
} self.backend.stop(self);
self.backend.stop(self); })
.await;
} }
#[track_caller] #[track_caller]
......
...@@ -8,7 +8,7 @@ use std::{ ...@@ -8,7 +8,7 @@ use std::{
use anyhow::Result; use anyhow::Result;
use futures::{FutureExt, ready}; use futures::{FutureExt, ready};
use tokio::runtime::Handle; use tokio::{runtime::Handle, task::block_in_place};
use tracing::{Instrument, Span, info_span}; use tracing::{Instrument, Span, info_span};
use turbo_tasks_malloc::{AllocationInfo, TurboMalloc}; use turbo_tasks_malloc::{AllocationInfo, TurboMalloc};
...@@ -23,6 +23,12 @@ pub struct JoinHandle<T> { ...@@ -23,6 +23,12 @@ pub struct JoinHandle<T> {
join_handle: tokio::task::JoinHandle<(Result<T, TurboTasksPanic>, Duration, AllocationInfo)>, join_handle: tokio::task::JoinHandle<(Result<T, TurboTasksPanic>, Duration, AllocationInfo)>,
} }
impl<T: Send + 'static> JoinHandle<T> {
pub fn join(self) -> T {
block_for_future(self)
}
}
impl<T> Future for JoinHandle<T> { impl<T> Future for JoinHandle<T> {
type Output = T; type Output = T;
...@@ -87,3 +93,14 @@ pub fn spawn_thread(func: impl FnOnce() + Send + 'static) { ...@@ -87,3 +93,14 @@ pub fn spawn_thread(func: impl FnOnce() + Send + 'static) {
}) })
}); });
} }
/// Blocking waits for a future to complete. This blocks the current thread potentially staling
/// other concurrent futures (but not other concurrent tasks). Try to avoid this method infavor of
/// awaiting the future instead.
pub fn block_for_future<T: Send>(future: impl Future<Output = T> + Send + 'static) -> T {
let handle = Handle::current();
block_in_place(|| {
let _span = info_span!("blocking").entered();
handle.block_on(future)
})
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment