Unverified Commit e4579de5 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Precompute analyzer SCCs with petgraph

Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent f57896de
...@@ -4149,6 +4149,7 @@ dependencies = [ ...@@ -4149,6 +4149,7 @@ dependencies = [
"indexmap 2.14.0", "indexmap 2.14.0",
"itertools 0.10.5", "itertools 0.10.5",
"next-core", "next-core",
"petgraph 0.8.3",
"regex", "regex",
"roaring", "roaring",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
......
...@@ -27,6 +27,7 @@ futures = { workspace = true } ...@@ -27,6 +27,7 @@ futures = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
next-core = { workspace = true } next-core = { workspace = true }
petgraph = { workspace = true }
regex = { workspace = true } regex = { workspace = true }
roaring = { workspace = true } roaring = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
......
...@@ -4,6 +4,7 @@ use anyhow::Result; ...@@ -4,6 +4,7 @@ use anyhow::Result;
use byteorder::{BE, WriteBytesExt}; use byteorder::{BE, WriteBytesExt};
use either::Either; use either::Either;
use next_core::app_structure::FileSystemPathVec; use next_core::app_structure::FileSystemPathVec;
use petgraph::{Directed, Graph, algo::kosaraju_scc};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use serde::Serialize; use serde::Serialize;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
...@@ -126,6 +127,8 @@ struct AnalyzeDataHeader { ...@@ -126,6 +127,8 @@ struct AnalyzeDataHeader {
#[derive(Serialize)] #[derive(Serialize)]
struct ModulesDataHeader { struct ModulesDataHeader {
pub modules: Vec<AnalyzeModule>, pub modules: Vec<AnalyzeModule>,
/// Deterministic petgraph SCC ID for each module, using synchronous edges only.
pub sync_scc_ids: Vec<u32>,
/// Edges from modules to modules /// Edges from modules to modules
pub module_dependents: EdgesDataReference, pub module_dependents: EdgesDataReference,
/// Edges from modules to modules /// Edges from modules to modules
...@@ -297,6 +300,110 @@ impl AnalyzeDataBuilder { ...@@ -297,6 +300,110 @@ impl AnalyzeDataBuilder {
} }
} }
fn compute_sync_scc_ids(module_idents: &[RcStr], dependencies: &[Vec<u32>]) -> Vec<u32> {
debug_assert_eq!(module_idents.len(), dependencies.len());
let mut graph = Graph::<u32, (), Directed, u32>::with_capacity(
module_idents.len(),
dependencies.iter().map(Vec::len).sum(),
);
let nodes: Vec<_> = (0..module_idents.len())
.map(|index| graph.add_node(index as u32))
.collect();
for (from, targets) in dependencies.iter().enumerate() {
for &to in targets {
graph.add_edge(nodes[from], nodes[to as usize], ());
}
}
let mut components: Vec<Vec<u32>> = kosaraju_scc(&graph)
.into_iter()
.map(|component| {
let mut members: Vec<_> = component.into_iter().map(|node| graph[node]).collect();
members.sort_unstable_by(|&a, &b| {
module_idents[a as usize].cmp(&module_idents[b as usize])
});
members
})
.collect();
components
.sort_unstable_by(|a, b| module_idents[a[0] as usize].cmp(&module_idents[b[0] as usize]));
let mut ids = vec![0; module_idents.len()];
for (component_id, members) in components.into_iter().enumerate() {
for module_index in members {
ids[module_index as usize] = component_id as u32;
}
}
ids
}
#[cfg(test)]
mod tests {
use turbo_rcstr::RcStr;
use super::{ModulesDataBuilder, compute_sync_scc_ids};
fn id_map(idents: &[RcStr], ids: &[u32]) -> Vec<(RcStr, u32)> {
let mut result: Vec<_> = idents.iter().cloned().zip(ids.iter().copied()).collect();
result.sort_unstable_by(|a, b| a.0.cmp(&b.0));
result
}
#[test]
fn sync_scc_ids_are_deterministic_across_insertion_order() {
let first_idents: Vec<RcStr> = ["z", "a", "b"].into_iter().map(Into::into).collect();
let first = compute_sync_scc_ids(&first_idents, &[vec![1], vec![0], vec![]]);
let second_idents: Vec<RcStr> = ["b", "z", "a"].into_iter().map(Into::into).collect();
let second = compute_sync_scc_ids(&second_idents, &[vec![], vec![2], vec![1]]);
assert_eq!(
id_map(&first_idents, &first),
id_map(&second_idents, &second)
);
assert_eq!(
id_map(&first_idents, &first),
vec![("a".into(), 0), ("b".into(), 1), ("z".into(), 0)]
);
}
#[test]
fn sync_scc_ids_ignore_async_and_traced_edges() {
let mut builder = ModulesDataBuilder::new();
builder.ensure_module("entry", "entry");
builder.ensure_module("dependency", "dependency");
builder.modules[0].dependencies.insert(1);
builder.modules[1].async_dependencies.insert(0);
builder.modules[1].traced_dependencies.insert(0);
let without_sync_back_edge = builder.sync_scc_ids();
assert_ne!(without_sync_back_edge[0], without_sync_back_edge[1]);
builder.modules[1].dependencies.insert(0);
let with_sync_back_edge = builder.sync_scc_ids();
assert_eq!(with_sync_back_edge[0], with_sync_back_edge[1]);
}
#[test]
fn sync_scc_ids_keep_cycles_and_singletons_separate() {
let idents: Vec<RcStr> = ["entry", "left", "right", "unused"]
.into_iter()
.map(Into::into)
.collect();
let ids = compute_sync_scc_ids(&idents, &[vec![1], vec![2], vec![1], vec![]]);
assert_ne!(ids[0], ids[1]);
assert_eq!(ids[1], ids[2]);
assert_ne!(ids[2], ids[3]);
assert_eq!(
ids.iter()
.copied()
.collect::<std::collections::BTreeSet<_>>(),
std::collections::BTreeSet::from([0_u32, 1, 2])
);
}
}
impl ModulesDataBuilder { impl ModulesDataBuilder {
fn new() -> Self { fn new() -> Self {
Self { Self {
...@@ -305,6 +412,21 @@ impl ModulesDataBuilder { ...@@ -305,6 +412,21 @@ impl ModulesDataBuilder {
} }
} }
fn sync_scc_ids(&self) -> Vec<u32> {
compute_sync_scc_ids(
&self
.modules
.iter()
.map(|module| module.module.ident.clone())
.collect::<Vec<_>>(),
&self
.modules
.iter()
.map(|module| module.dependencies.iter().copied().collect())
.collect::<Vec<_>>(),
)
}
fn get_module(&mut self, ident: &str) -> (&mut AnalyzeModuleBuilder, u32) { fn get_module(&mut self, ident: &str) -> (&mut AnalyzeModuleBuilder, u32) {
if let Some(&index) = self.module_index_map.get(ident) { if let Some(&index) = self.module_index_map.get(ident) {
return (&mut self.modules[index as usize], index); return (&mut self.modules[index as usize], index);
...@@ -364,6 +486,7 @@ impl ModulesDataBuilder { ...@@ -364,6 +486,7 @@ impl ModulesDataBuilder {
.map(|s| s.traced_dependents.iter().copied().collect()) .map(|s| s.traced_dependents.iter().copied().collect())
.collect(); .collect();
let sync_scc_ids = self.sync_scc_ids();
let module_dependencies = EdgesData::from_iterator(&module_dependencies_vecs); let module_dependencies = EdgesData::from_iterator(&module_dependencies_vecs);
let async_module_dependencies = EdgesData::from_iterator(&async_module_dependencies_vecs); let async_module_dependencies = EdgesData::from_iterator(&async_module_dependencies_vecs);
let traced_module_dependencies = EdgesData::from_iterator(&traced_module_dependencies_vecs); let traced_module_dependencies = EdgesData::from_iterator(&traced_module_dependencies_vecs);
...@@ -375,6 +498,7 @@ impl ModulesDataBuilder { ...@@ -375,6 +498,7 @@ impl ModulesDataBuilder {
let header = ModulesDataHeader { let header = ModulesDataHeader {
modules: self.modules.into_iter().map(|s| s.module).collect(), modules: self.modules.into_iter().map(|s| s.module).collect(),
sync_scc_ids,
module_dependents: binary_section.add_edges(&module_dependents), module_dependents: binary_section.add_edges(&module_dependents),
async_module_dependents: binary_section.add_edges(&async_module_dependents), async_module_dependents: binary_section.add_edges(&async_module_dependents),
traced_module_dependents: binary_section.add_edges(&traced_module_dependents), traced_module_dependents: binary_section.add_edges(&traced_module_dependents),
......
...@@ -34,7 +34,10 @@ function frame(header: object, binary: Uint8Array[]): ArrayBuffer { ...@@ -34,7 +34,10 @@ function frame(header: object, binary: Uint8Array[]): ArrayBuffer {
return result.buffer return result.buffer
} }
function modulesBuffer(): ArrayBuffer { function modulesBuffer(
exact = true,
syncSccIds: number[] = [0, 1]
): ArrayBuffer {
const sections = [ const sections = [
edges([[1], []]), edges([[1], []]),
edges([[], [0]]), edges([[], [0]]),
...@@ -55,6 +58,7 @@ function modulesBuffer(): ArrayBuffer { ...@@ -55,6 +58,7 @@ function modulesBuffer(): ArrayBuffer {
{ ident: 'first', path: '[project]/src/a.ts' }, { ident: 'first', path: '[project]/src/a.ts' },
{ ident: 'second', path: '[project]/src/a.ts' }, { ident: 'second', path: '[project]/src/a.ts' },
], ],
...(exact ? { sync_scc_ids: syncSccIds } : {}),
module_dependents: references[0], module_dependents: references[0],
async_module_dependents: references[1], async_module_dependents: references[1],
traced_module_dependents: references[2], traced_module_dependents: references[2],
...@@ -135,12 +139,29 @@ describe('analyzer data parser', () => { ...@@ -135,12 +139,29 @@ describe('analyzer data parser', () => {
expect(modules.asyncModuleDependents(1)).toEqual([0]) expect(modules.asyncModuleDependents(1)).toEqual([0])
expect(modules.moduleDependencies(1)).toEqual([0]) expect(modules.moduleDependencies(1)).toEqual([0])
expect(modules.getModuleIndexFromIdent('second')).toBe(1) expect(modules.getModuleIndexFromIdent('second')).toBe(1)
expect(modules.hasExactSyncSccs()).toBe(true)
expect(modules.syncSccId(0)).toBe(0)
expect(modules.syncSccId(1)).toBe(1)
}) })
it('keeps old artifacts readable with explicit unavailable evidence', () => { it('keeps old artifacts readable with explicit unavailable evidence', () => {
const analyze = new AnalyzeData(analyzeBuffer(false)) const analyze = new AnalyzeData(analyzeBuffer(false))
expect(analyze.hasExactRouteEntries()).toBe(false) expect(analyze.hasExactRouteEntries()).toBe(false)
expect(analyze.routeEntries()).toEqual([]) expect(analyze.routeEntries()).toEqual([])
const modules = new ModulesData(modulesBuffer(false))
expect(modules.hasExactSyncSccs()).toBe(false)
expect(modules.syncSccId(0)).toBeUndefined()
})
it.each([
[[0], 'wrong cardinality'],
[[0, -1], 'negative ID'],
[[0, 2], 'gapped ID'],
[[0, 0.5], 'non-integer ID'],
])('rejects invalid sync SCC IDs: %s (%s)', (ids) => {
expect(() => new ModulesData(modulesBuffer(true, ids as number[]))).toThrow(
'Invalid modules.data sync SCC IDs'
)
}) })
it('rejects duplicate exact route entry IDs', () => { it('rejects duplicate exact route entry IDs', () => {
......
...@@ -55,6 +55,8 @@ interface AnalyzeDataHeader { ...@@ -55,6 +55,8 @@ interface AnalyzeDataHeader {
interface ModulesDataHeader { interface ModulesDataHeader {
modules: AnalyzeModule[] modules: AnalyzeModule[]
/** Absent in analyzer artifacts produced before petgraph SCC support. */
sync_scc_ids?: number[]
module_dependents: EdgesDataReference module_dependents: EdgesDataReference
async_module_dependents: EdgesDataReference async_module_dependents: EdgesDataReference
traced_module_dependents: EdgesDataReference traced_module_dependents: EdgesDataReference
...@@ -143,8 +145,26 @@ export class ModulesData { ...@@ -143,8 +145,26 @@ export class ModulesData {
;[this.modulesHeader, this.modulesBinaryData] = ;[this.modulesHeader, this.modulesBinaryData] =
parseHeader<ModulesDataHeader>(modulesArrayBuffer, 'modules.data') parseHeader<ModulesDataHeader>(modulesArrayBuffer, 'modules.data')
requireArray(this.modulesHeader.modules, 'modules.data modules') requireArray(this.modulesHeader.modules, 'modules.data modules')
if (this.modulesHeader.sync_scc_ids !== undefined) {
requireArray(this.modulesHeader.sync_scc_ids, 'modules.data sync SCC IDs')
if (
this.modulesHeader.sync_scc_ids.length !==
this.modulesHeader.modules.length ||
this.modulesHeader.sync_scc_ids.some(
(id) => !Number.isInteger(id) || id < 0
)
) {
throw new Error('Invalid modules.data sync SCC IDs')
}
const distinct = [...new Set(this.modulesHeader.sync_scc_ids)].sort(
(a, b) => a - b
)
if (distinct.some((id, index) => id !== index)) {
throw new Error('Invalid modules.data sync SCC IDs')
}
}
for (const [name, reference] of Object.entries(this.modulesHeader)) { for (const [name, reference] of Object.entries(this.modulesHeader)) {
if (name === 'modules') continue if (name === 'modules' || name === 'sync_scc_ids') continue
validateEdges( validateEdges(
this.modulesBinaryData, this.modulesBinaryData,
reference as EdgesDataReference, reference as EdgesDataReference,
...@@ -193,6 +213,14 @@ export class ModulesData { ...@@ -193,6 +213,14 @@ export class ModulesData {
return this.identToModuleIndex.get(ident) return this.identToModuleIndex.get(ident)
} }
hasExactSyncSccs(): boolean {
return this.modulesHeader.sync_scc_ids !== undefined
}
syncSccId(index: ModuleIndex): number | undefined {
return this.modulesHeader.sync_scc_ids?.[index]
}
private readEdgesDataAtIndex( private readEdgesDataAtIndex(
reference: EdgesDataReference, reference: EdgesDataReference,
index: ModuleIndex index: ModuleIndex
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment