Commit e77be5de authored by Tobias Koppers's avatar Tobias Koppers

add more stats

parent 94948b0b
...@@ -149,56 +149,106 @@ impl CachedDataItemStats { ...@@ -149,56 +149,106 @@ impl CachedDataItemStats {
/// Statistics for ReverseTaskCache function IDs /// Statistics for ReverseTaskCache function IDs
#[derive(Default, Debug)] #[derive(Default, Debug)]
struct FunctionIdStats { struct FunctionIdStats {
/// Maps function_id to count /// Maps function_id to (count, total_size)
function_counts: FxHashMap<u64, u64>, function_counts: FxHashMap<u64, (u64, u64)>,
} }
impl FunctionIdStats { impl FunctionIdStats {
fn add_function(&mut self, function_id: u64) { fn add_function(&mut self, function_id: u64, size: u64) {
*self.function_counts.entry(function_id).or_insert(0) += 1; self.function_counts
.entry(function_id)
.and_modify(|(count, total_size)| {
*count += 1;
*total_size += size;
})
.or_insert((1, size));
} }
fn total(&self) -> u64 { fn total(&self) -> u64 {
self.function_counts.values().sum() self.function_counts.values().map(|(count, _)| count).sum()
} }
fn top_n(&self, n: usize) -> Vec<(&u64, &u64)> { fn total_size(&self) -> u64 {
self.function_counts.values().map(|(_, size)| size).sum()
}
fn top_n_by_count(&self, n: usize) -> Vec<(&u64, &(u64, u64))> {
let mut entries: Vec<_> = self.function_counts.iter().collect(); let mut entries: Vec<_> = self.function_counts.iter().collect();
entries.sort_by(|a, b| b.1.cmp(a.1).then_with(|| a.0.cmp(b.0))); entries.sort_by(|a, b| b.1.0.cmp(&a.1.0).then_with(|| a.0.cmp(b.0)));
entries.into_iter().take(n).collect()
}
fn top_n_by_size(&self, n: usize) -> Vec<(&u64, &(u64, u64))> {
let mut entries: Vec<_> = self.function_counts.iter().collect();
entries.sort_by(|a, b| b.1.1.cmp(&a.1.1).then_with(|| a.0.cmp(b.0)));
entries.into_iter().take(n).collect() entries.into_iter().take(n).collect()
} }
} }
/// Per-function statistics including variant breakdown
#[derive(Default, Debug)]
struct FunctionStats {
total_size: u64,
count: u64,
/// Maps variant name to (count, total_size)
variant_breakdown: FxHashMap<String, (u64, u64)>,
}
impl FunctionStats {
fn add_entry(&mut self, size: u64) {
self.total_size += size;
self.count += 1;
}
fn add_variant(&mut self, variant_name: String, size: u64) {
self.variant_breakdown
.entry(variant_name)
.and_modify(|(count, total_size)| {
*count += 1;
*total_size += size;
})
.or_insert((1, size));
}
}
/// Statistics for TaskMeta/TaskData sizes by function ID /// Statistics for TaskMeta/TaskData sizes by function ID
#[derive(Default, Debug)] #[derive(Default, Debug)]
struct SizeByFunctionStats { struct SizeByFunctionStats {
/// Maps function_id to (total_size, count) /// Maps function_id to FunctionStats
function_stats: FxHashMap<u64, (u64, u64)>, function_stats: FxHashMap<u64, FunctionStats>,
} }
impl SizeByFunctionStats { impl SizeByFunctionStats {
fn add_size(&mut self, function_id: u64, size: u64) { fn add_size(&mut self, function_id: u64, size: u64) {
self.function_stats self.function_stats
.entry(function_id) .entry(function_id)
.and_modify(|(total_size, count)| { .or_default()
*total_size += size; .add_entry(size);
*count += 1; }
})
.or_insert((size, 1)); fn add_variant(&mut self, function_id: u64, variant_name: String, size: u64) {
self.function_stats
.entry(function_id)
.or_default()
.add_variant(variant_name, size);
} }
fn total_size(&self) -> u64 { fn total_size(&self) -> u64 {
self.function_stats.values().map(|(size, _)| size).sum() self.function_stats.values().map(|s| s.total_size).sum()
} }
fn total_count(&self) -> u64 { fn total_count(&self) -> u64 {
self.function_stats.values().map(|(_, count)| count).sum() self.function_stats.values().map(|s| s.count).sum()
} }
fn top_n(&self, n: usize) -> Vec<(&u64, &(u64, u64))> { fn top_n(&self, n: usize) -> Vec<(&u64, &FunctionStats)> {
let mut entries: Vec<_> = self.function_stats.iter().collect(); let mut entries: Vec<_> = self.function_stats.iter().collect();
// Sort by size (descending) // Sort by size (descending)
entries.sort_by(|a, b| b.1.0.cmp(&a.1.0).then_with(|| a.0.cmp(b.0))); entries.sort_by(|a, b| {
b.1.total_size
.cmp(&a.1.total_size)
.then_with(|| a.0.cmp(b.0))
});
entries.into_iter().take(n).collect() entries.into_iter().take(n).collect()
} }
} }
...@@ -275,7 +325,11 @@ impl FamilyStats { ...@@ -275,7 +325,11 @@ impl FamilyStats {
); );
} }
fn process_cached_data_items(&mut self, entry: &OwnedLookupEntry) -> Result<()> { fn process_cached_data_items(
&mut self,
entry: &OwnedLookupEntry,
task_to_function_map: &FxHashMap<Vec<u8>, u64>,
) -> Result<()> {
// Deserialize the value as pot::Value to get a Vec of items // Deserialize the value as pot::Value to get a Vec of items
let pot_value = deserialize_as_pot_value(&entry.value)?; let pot_value = deserialize_as_pot_value(&entry.value)?;
...@@ -288,13 +342,24 @@ impl FamilyStats { ...@@ -288,13 +342,24 @@ impl FamilyStats {
// Ensure we have stats tracking initialized // Ensure we have stats tracking initialized
let stats = self.cached_data_stats.get_or_insert_with(Default::default); let stats = self.cached_data_stats.get_or_insert_with(Default::default);
// Look up the function ID for this task ID (key)
let function_id_opt = task_to_function_map.get(entry.key.as_ref()).copied();
// Process each item // Process each item
for item in items { for item in items {
let variant_name = extract_variant_name(&item)?; let variant_name = extract_variant_name(&item)?;
// Serialize the item to get its size and hash // Serialize the item to get its size and hash
let (bytes, value_hash) = serialize_and_hash_pot_value(&item)?; let (bytes, value_hash) = serialize_and_hash_pot_value(&item)?;
let size = bytes.len() as u64; let size = bytes.len() as u64;
stats.add_variant(variant_name, value_hash, size); stats.add_variant(variant_name.clone(), value_hash, size);
// If we have a function ID, also track variant stats per function
if let Some(function_id) = function_id_opt {
let size_stats = self
.size_by_function_stats
.get_or_insert_with(Default::default);
size_stats.add_variant(function_id, variant_name, size);
}
} }
Ok(()) Ok(())
...@@ -307,9 +372,12 @@ impl FamilyStats { ...@@ -307,9 +372,12 @@ impl FamilyStats {
// Extract the function ID // Extract the function ID
let function_id = extract_function_id(&pot_value)?; let function_id = extract_function_id(&pot_value)?;
// Get the size of the entry
let size = entry.value.len() as u64;
// Ensure we have stats tracking initialized // Ensure we have stats tracking initialized
let stats = self.function_id_stats.get_or_insert_with(Default::default); let stats = self.function_id_stats.get_or_insert_with(Default::default);
stats.add_function(function_id); stats.add_function(function_id, size);
Ok(()) Ok(())
} }
...@@ -475,16 +543,19 @@ fn main() -> Result<()> { ...@@ -475,16 +543,19 @@ fn main() -> Result<()> {
stats.add_entry(&entry); stats.add_entry(&entry);
// For TaskMeta (1) and TaskData (2), also deserialize and analyze // For TaskMeta (1) and TaskData (2), also deserialize and analyze
// CachedDataItems and track size by function ID // CachedDataItems and track size by function ID (including variant
// breakdown)
if family == 1 || family == 2 { if family == 1 || family == 2 {
if let Err(e) = stats.process_cached_data_items(&entry) { if let Err(e) =
stats.process_cached_data_items(&entry, &task_to_function_map)
{
stats.deserialization_failures += 1; stats.deserialization_failures += 1;
// Capture first 10 errors // Capture first 10 errors
if stats.deserialization_errors.len() < 10 { if stats.deserialization_errors.len() < 10 {
stats.deserialization_errors.push(format!("{:#}", e)); stats.deserialization_errors.push(format!("{:#}", e));
} }
} }
// Track size by function ID // Track total size by function ID
stats.track_size_by_function(&entry, &task_to_function_map); stats.track_size_by_function(&entry, &task_to_function_map);
} }
...@@ -768,15 +839,34 @@ fn main() -> Result<()> { ...@@ -768,15 +839,34 @@ fn main() -> Result<()> {
); );
println!(" {}", separator); println!(" {}", separator);
for (function_id, (size, count)) in top_functions { for (function_id, func_stats) in top_functions {
let func_name = format_function_name(*function_id, &function_names); let func_name = format_function_name(*function_id, &function_names);
println!( println!(
" {:<width$} {:>15} {:>15}", " {:<width$} {:>15} {:>15}",
func_name, func_name,
count, func_stats.count,
format_size(*size), format_size(func_stats.total_size),
width = max_func_len width = max_func_len
); );
// Display variant breakdown for this function
if !func_stats.variant_breakdown.is_empty() {
let mut variants: Vec<_> =
func_stats.variant_breakdown.iter().collect();
// Sort by size descending
variants
.sort_by(|a, b| b.1.1.cmp(&a.1.1).then_with(|| a.0.cmp(b.0)));
for (variant_name, (var_count, var_size)) in variants {
println!(
" └─ {:<width$} {:>15} {:>15}",
variant_name,
var_count,
format_size(*var_size),
width = max_func_len.saturating_sub(3)
);
}
}
} }
println!(" {}", separator); println!(" {}", separator);
...@@ -801,15 +891,61 @@ fn main() -> Result<()> { ...@@ -801,15 +891,61 @@ fn main() -> Result<()> {
// Display Function ID statistics for ReverseTaskCache // Display Function ID statistics for ReverseTaskCache
if family == 4 { if family == 4 {
if let Some(func_stats) = &stats.function_id_stats { if let Some(func_stats) = &stats.function_id_stats {
println!();
println!(" Top 20 Functions:");
let total_funcs = func_stats.total(); let total_funcs = func_stats.total();
let total_size = func_stats.total_size();
if total_funcs > 0 { if total_funcs > 0 {
let top_functions = func_stats.top_n(20); // First table: Top 20 by Size
println!();
println!(" Top 20 Functions by Size:");
let top_by_size = func_stats.top_n_by_size(20);
// Measure the maximum function name length in top 20 // Measure the maximum function name length in top 20
let max_func_len = top_functions let max_func_len = top_by_size
.iter()
.map(|(function_id, _)| {
format_function_name(**function_id, &function_names).len()
})
.max()
.unwrap_or(50)
.max(20); // Minimum width of 20
// Calculate separator length
let separator_len = max_func_len + 4 + 15 + 15 + 6; // padding + two columns + spaces
let separator = "─".repeat(separator_len);
println!(" {}", separator);
println!(
" {:<width$} {:>15} {:>15}",
"Function",
"Count",
"Total Size",
width = max_func_len
);
println!(" {}", separator);
for (function_id, (count, size)) in top_by_size {
let func_name = format_function_name(*function_id, &function_names);
println!(
" {:<width$} {:>15} {:>15}",
func_name,
count,
format_size(*size),
width = max_func_len
);
}
println!(" {}", separator);
// Second table: Top 20 by Count
println!();
println!(" Top 20 Functions by Count:");
let top_by_count = func_stats.top_n_by_count(20);
// Measure the maximum function name length in top 20
let max_func_len = top_by_count
.iter() .iter()
.map(|(function_id, _)| { .map(|(function_id, _)| {
format_function_name(**function_id, &function_names).len() format_function_name(**function_id, &function_names).len()
...@@ -819,39 +955,43 @@ fn main() -> Result<()> { ...@@ -819,39 +955,43 @@ fn main() -> Result<()> {
.max(20); // Minimum width of 20 .max(20); // Minimum width of 20
// Calculate separator length // Calculate separator length
let separator_len = max_func_len + 4 + 15 + 3; // padding + one column + spaces let separator_len = max_func_len + 4 + 15 + 15 + 6; // padding + two columns + spaces
let separator = "─".repeat(separator_len); let separator = "─".repeat(separator_len);
println!(" {}", separator); println!(" {}", separator);
println!( println!(
" {:<width$} {:>15}", " {:<width$} {:>15} {:>15}",
"Function", "Function",
"Count", "Count",
"Total Size",
width = max_func_len width = max_func_len
); );
println!(" {}", separator); println!(" {}", separator);
for (function_id, count) in top_functions { for (function_id, (count, size)) in top_by_count {
let func_name = format_function_name(*function_id, &function_names); let func_name = format_function_name(*function_id, &function_names);
println!( println!(
" {:<width$} {:>15}", " {:<width$} {:>15} {:>15}",
func_name, func_name,
count, count,
format_size(*size),
width = max_func_len width = max_func_len
); );
} }
println!(" {}", separator); println!(" {}", separator);
println!( println!(
" {:<width$} {:>15}", " {:<width$} {:>15} {:>15}",
"Total Functions:", "Total Functions:",
func_stats.function_counts.len(), func_stats.function_counts.len(),
"",
width = max_func_len width = max_func_len
); );
println!( println!(
" {:<width$} {:>15}", " {:<width$} {:>15} {:>15}",
"Total Entries:", "Total:",
total_funcs, total_funcs,
format_size(total_size),
width = max_func_len width = max_func_len
); );
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment