Commit 94948b0b authored by Tobias Koppers's avatar Tobias Koppers

Per function stats

parent 2f7873f8
...@@ -4418,6 +4418,7 @@ dependencies = [ ...@@ -4418,6 +4418,7 @@ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
"next-api", "next-api",
"pot",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"turbo-persistence", "turbo-persistence",
"twox-hash 2.1.0", "twox-hash 2.1.0",
......
...@@ -286,6 +286,7 @@ debug = true ...@@ -286,6 +286,7 @@ debug = true
[workspace.dependencies] [workspace.dependencies]
# Workspace crates # Workspace crates
next-swc-napi = { path = "crates/napi" }
next-api = { path = "crates/next-api" } next-api = { path = "crates/next-api" }
next-build = { path = "crates/next-build" } next-build = { path = "crates/next-build" }
next-core = { path = "crates/next-core" } next-core = { path = "crates/next-core" }
......
...@@ -18,6 +18,7 @@ workspace = true ...@@ -18,6 +18,7 @@ workspace = true
anyhow = { workspace = true } anyhow = { workspace = true }
clap = { workspace = true, features = ["derive"] } clap = { workspace = true, features = ["derive"] }
next-api = { workspace = true } next-api = { workspace = true }
pot = "3.0.0"
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
turbo-persistence = { workspace = true } turbo-persistence = { workspace = true }
twox-hash = { workspace = true } twox-hash = { workspace = true }
This diff is collapsed.
...@@ -285,6 +285,10 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -285,6 +285,10 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
for entry in entries { for entry in entries {
let entry = entry?; let entry = entry?;
let path = entry.path(); let path = entry.path();
let file_name = path.file_name();
if file_name.is_some_and(|s| s.as_encoded_bytes().starts_with(b".")) {
continue;
}
if let Some(ext) = path.extension().and_then(|s| s.to_str()) { if let Some(ext) = path.extension().and_then(|s| s.to_str()) {
let seq: u32 = path let seq: u32 = path
.file_stem() .file_stem()
...@@ -331,17 +335,12 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -331,17 +335,12 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
// ignore blobs and sst, they are read when needed // ignore blobs and sst, they are read when needed
} }
_ => { _ => {
if !path bail!("Unexpected file in persistence directory: {:?}", path);
.file_name()
.is_some_and(|s| s.as_encoded_bytes().starts_with(b"."))
{
bail!("Unexpected file in persistence directory: {:?}", path);
}
} }
} }
} }
} else { } else {
match path.file_stem().and_then(|s| s.to_str()) { match file_name.and_then(|s| s.to_str()) {
Some("CURRENT") => { Some("CURRENT") => {
// Already read // Already read
} }
...@@ -349,12 +348,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -349,12 +348,7 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
// Ignored, write-only // Ignored, write-only
} }
_ => { _ => {
if !path bail!("Unexpected file in persistence directory: {:?}", path);
.file_name()
.is_some_and(|s| s.as_encoded_bytes().starts_with(b"."))
{
bail!("Unexpected file in persistence directory: {:?}", path);
}
} }
} }
} }
...@@ -405,6 +399,11 @@ impl<S: ParallelScheduler> TurboPersistence<S> { ...@@ -405,6 +399,11 @@ impl<S: ParallelScheduler> TurboPersistence<S> {
self.inner.read().meta_files.is_empty() self.inner.read().meta_files.is_empty()
} }
/// Returns true if the database is opened in read-only mode.
pub fn is_read_only(&self) -> bool {
self.read_only
}
/// Starts a new WriteBatch for the database. Only a single write operation is allowed at a /// Starts a new WriteBatch for the database. Only a single write operation is allowed at a
/// time. The WriteBatch need to be committed with [`TurboPersistence::commit_write_batch`]. /// time. The WriteBatch need to be committed with [`TurboPersistence::commit_write_batch`].
/// Note that the WriteBatch might start writing data to disk while it's filled up with data. /// Note that the WriteBatch might start writing data to disk while it's filled up with data.
......
use std::{ use std::{
cmp::max, cmp::max,
fs::OpenOptions,
io::BufWriter,
path::PathBuf, path::PathBuf,
sync::Arc, sync::Arc,
thread::available_parallelism, thread::available_parallelism,
...@@ -11,7 +13,9 @@ use parking_lot::Mutex; ...@@ -11,7 +13,9 @@ use parking_lot::Mutex;
use turbo_persistence::{ use turbo_persistence::{
ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer, ArcSlice, CompactConfig, KeyBase, StoreKey, TurboPersistence, ValueBuffer,
}; };
use turbo_tasks::{JoinHandle, message_queue::TimingEvent, spawn, turbo_tasks}; use turbo_tasks::{
JoinHandle, message_queue::TimingEvent, registry::iter_functions, spawn, turbo_tasks,
};
use crate::database::{ use crate::database::{
key_value_database::{KeySpace, KeyValueDatabase}, key_value_database::{KeySpace, KeyValueDatabase},
...@@ -41,7 +45,19 @@ pub struct TurboKeyValueDatabase { ...@@ -41,7 +45,19 @@ pub struct TurboKeyValueDatabase {
impl TurboKeyValueDatabase { impl TurboKeyValueDatabase {
pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> { pub fn new(versioned_path: PathBuf, is_ci: bool, is_short_session: bool) -> Result<Self> {
let mappings_path = versioned_path.join(".functions.map");
let db = Arc::new(TurboPersistence::open(versioned_path)?); let db = Arc::new(TurboPersistence::open(versioned_path)?);
if db.is_empty() && !db.is_read_only() {
let mappings_file = OpenOptions::new()
.create(true)
.append(true)
.open(mappings_path)?;
let mut mappings_file = BufWriter::new(mappings_file);
for (id, func) in iter_functions() {
use std::io::Write;
writeln!(mappings_file, "{} {}", *id as u32, func.global_name())?;
}
}
Ok(Self { Ok(Self {
db: db.clone(), db: db.clone(),
compact_join_handle: Mutex::new(None), compact_join_handle: Mutex::new(None),
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
mod backend; mod backend;
mod backing_storage; mod backing_storage;
mod data; pub mod data;
mod data_storage; mod data_storage;
mod database; mod database;
mod kv_backing_storage; mod kv_backing_storage;
......
...@@ -237,6 +237,10 @@ impl NativeFunction { ...@@ -237,6 +237,10 @@ impl NativeFunction {
} }
} }
pub fn global_name(&self) -> &'static str {
self.global_name
}
pub fn span(&'static self, persistence: TaskPersistence, reason: TaskExecutionReason) -> Span { pub fn span(&'static self, persistence: TaskPersistence, reason: TaskExecutionReason) -> Span {
let flags = match persistence { let flags = match persistence {
TaskPersistence::Persistent => "", TaskPersistence::Persistent => "",
......
...@@ -117,6 +117,13 @@ impl<T: RegistryItem> Registry<T> { ...@@ -117,6 +117,13 @@ impl<T: RegistryItem> Registry<T> {
)) ))
} }
} }
pub fn iter(&self) -> impl Iterator<Item = (T::Id, &'static T)> + '_ {
self.id_to_item
.iter()
.enumerate()
.map(|(i, &item)| (T::Id::from(NonZeroU16::new((i + 1) as u16).unwrap()), item))
}
} }
static FUNCTIONS: Lazy<Registry<NativeFunction>> = Lazy::new(|| { static FUNCTIONS: Lazy<Registry<NativeFunction>> = Lazy::new(|| {
...@@ -139,6 +146,10 @@ pub fn validate_function_id(id: FunctionId) -> Option<Error> { ...@@ -139,6 +146,10 @@ pub fn validate_function_id(id: FunctionId) -> Option<Error> {
FUNCTIONS.validate_id(id) FUNCTIONS.validate_id(id)
} }
pub fn iter_functions() -> impl Iterator<Item = (FunctionId, &'static NativeFunction)> {
FUNCTIONS.iter()
}
static VALUES: Lazy<Registry<ValueType>> = Lazy::new(|| { static VALUES: Lazy<Registry<ValueType>> = Lazy::new(|| {
// Inventory does not guarantee an order. So we sort by the global name to get a stable order // Inventory does not guarantee an order. So we sort by the global name to get a stable order
// This ensures that assigned ids are also stable which is important since they are serialized. // This ensures that assigned ids are also stable which is important since they are serialized.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment