Unverified Commit f57896de authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Add exact analyzer route entries

Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent e3333ef1
...@@ -86,6 +86,20 @@ pub struct AnalyzeOutputFile { ...@@ -86,6 +86,20 @@ pub struct AnalyzeOutputFile {
pub filename: RcStr, pub filename: RcStr,
} }
/// Exact module-graph entry associated with one route-specific analyzer dataset.
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug, Serialize)]
pub struct AnalyzeRouteEntry {
pub route_entry_id: RcStr,
pub module_ident: RcStr,
pub module_path: RcStr,
pub role: RcStr,
pub runtime: Option<RcStr>,
}
#[turbo_tasks::value(transparent)]
pub struct AnalyzeRouteEntries(Vec<AnalyzeRouteEntry>);
#[derive(Serialize)] #[derive(Serialize)]
struct EdgesDataReference { struct EdgesDataReference {
pub offset: u32, pub offset: u32,
...@@ -97,7 +111,9 @@ struct AnalyzeDataHeader { ...@@ -97,7 +111,9 @@ struct AnalyzeDataHeader {
pub sources: Vec<AnalyzeSource>, pub sources: Vec<AnalyzeSource>,
pub chunk_parts: Vec<AnalyzeChunkPart>, pub chunk_parts: Vec<AnalyzeChunkPart>,
pub output_files: Vec<AnalyzeOutputFile>, pub output_files: Vec<AnalyzeOutputFile>,
/// Edges from chunks to chunk parts /// Exact module-graph entries for this route dataset.
pub route_entries: Vec<AnalyzeRouteEntry>,
/// Edges from output files to chunk parts
pub output_file_chunk_parts: EdgesDataReference, pub output_file_chunk_parts: EdgesDataReference,
/// Edges from sources to chunk parts /// Edges from sources to chunk parts
pub source_chunk_parts: EdgesDataReference, pub source_chunk_parts: EdgesDataReference,
...@@ -150,6 +166,7 @@ struct AnalyzeDataBuilder { ...@@ -150,6 +166,7 @@ struct AnalyzeDataBuilder {
source_index_map: FxHashMap<RcStr, u32>, source_index_map: FxHashMap<RcStr, u32>,
chunk_parts: Vec<AnalyzeChunkPart>, chunk_parts: Vec<AnalyzeChunkPart>,
output_files: Vec<AnalyzeOutputFileBuilder>, output_files: Vec<AnalyzeOutputFileBuilder>,
route_entries: Vec<AnalyzeRouteEntry>,
} }
struct ModulesDataBuilder { struct ModulesDataBuilder {
...@@ -175,12 +192,13 @@ impl EdgesDataSectionBuilder { ...@@ -175,12 +192,13 @@ impl EdgesDataSectionBuilder {
} }
impl AnalyzeDataBuilder { impl AnalyzeDataBuilder {
fn new() -> Self { fn new(route_entries: Vec<AnalyzeRouteEntry>) -> Self {
Self { Self {
sources: vec![], sources: vec![],
source_index_map: FxHashMap::default(), source_index_map: FxHashMap::default(),
chunk_parts: vec![], chunk_parts: vec![],
output_files: vec![], output_files: vec![],
route_entries,
} }
} }
...@@ -251,7 +269,6 @@ impl AnalyzeDataBuilder { ...@@ -251,7 +269,6 @@ impl AnalyzeDataBuilder {
let output_file_chunk_parts = let output_file_chunk_parts =
EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices)); EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices));
let mut binary_section = EdgesDataSectionBuilder::new(); let mut binary_section = EdgesDataSectionBuilder::new();
let header = AnalyzeDataHeader { let header = AnalyzeDataHeader {
...@@ -262,6 +279,7 @@ impl AnalyzeDataBuilder { ...@@ -262,6 +279,7 @@ impl AnalyzeDataBuilder {
.into_iter() .into_iter()
.map(|of| of.output_file) .map(|of| of.output_file)
.collect(), .collect(),
route_entries: self.route_entries,
output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts), output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts),
source_chunk_parts: binary_section.add_edges(&source_chunk_parts), source_chunk_parts: binary_section.add_edges(&source_chunk_parts),
source_children: binary_section.add_edges(&source_children), source_children: binary_section.add_edges(&source_children),
...@@ -405,17 +423,18 @@ pub async fn combine_traced_files( ...@@ -405,17 +423,18 @@ pub async fn combine_traced_files(
pub async fn analyze_output_assets( pub async fn analyze_output_assets(
output_assets: Vc<OutputAssets>, output_assets: Vc<OutputAssets>,
traced_files: Vc<FileSystemPathVec>, traced_files: Vc<FileSystemPathVec>,
route_entries: Vc<AnalyzeRouteEntries>,
) -> Result<Vc<FileContent>> { ) -> Result<Vc<FileContent>> {
let output_assets = all_assets_from_entries(output_assets); let output_assets = all_assets_from_entries(output_assets).await?;
let route_entries = route_entries.await?.iter().cloned().collect();
let mut builder = AnalyzeDataBuilder::new(); let mut builder = AnalyzeDataBuilder::new(route_entries);
let prefix = format!("{SOURCE_URL_PROTOCOL}///"); let prefix = format!("{SOURCE_URL_PROTOCOL}///");
// Process the output assets and extract chunk parts. // Process the output assets and extract chunk parts.
// Also creates sources for the chunk parts. // Also creates sources for the chunk parts.
for asset in output_assets for asset in output_assets
.await?
.iter() .iter()
.copied() .copied()
.map(Either::Left) .map(Either::Left)
...@@ -637,6 +656,7 @@ pub struct AnalyzeDataOutputAsset { ...@@ -637,6 +656,7 @@ pub struct AnalyzeDataOutputAsset {
pub path: FileSystemPath, pub path: FileSystemPath,
pub output_assets: ResolvedVc<OutputAssets>, pub output_assets: ResolvedVc<OutputAssets>,
pub traced_files: ResolvedVc<FileSystemPathVec>, pub traced_files: ResolvedVc<FileSystemPathVec>,
pub route_entries: ResolvedVc<AnalyzeRouteEntries>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -646,11 +666,13 @@ impl AnalyzeDataOutputAsset { ...@@ -646,11 +666,13 @@ impl AnalyzeDataOutputAsset {
path: FileSystemPath, path: FileSystemPath,
output_assets: ResolvedVc<OutputAssets>, output_assets: ResolvedVc<OutputAssets>,
traced_files: ResolvedVc<FileSystemPathVec>, traced_files: ResolvedVc<FileSystemPathVec>,
route_entries: ResolvedVc<AnalyzeRouteEntries>,
) -> Result<Vc<Self>> { ) -> Result<Vc<Self>> {
Ok(Self { Ok(Self {
path, path,
output_assets, output_assets,
traced_files, traced_files,
route_entries,
} }
.cell()) .cell())
} }
...@@ -660,7 +682,8 @@ impl AnalyzeDataOutputAsset { ...@@ -660,7 +682,8 @@ impl AnalyzeDataOutputAsset {
impl Asset for AnalyzeDataOutputAsset { impl Asset for AnalyzeDataOutputAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
fn content(&self) -> Vc<AssetContent> { fn content(&self) -> Vc<AssetContent> {
let file_content = analyze_output_assets(*self.output_assets, *self.traced_files); let file_content =
analyze_output_assets(*self.output_assets, *self.traced_files, *self.route_entries);
AssetContent::file(file_content) AssetContent::file(file_content)
} }
} }
......
...@@ -3,15 +3,19 @@ use std::{iter::once, sync::Arc}; ...@@ -3,15 +3,19 @@ use std::{iter::once, sync::Arc};
use anyhow::Result; use anyhow::Result;
use next_api::{ use next_api::{
analyze::{ analyze::{
AnalyzeDataOutputAsset, ModulesDataOutputAsset, combine_output_assets, combine_traced_files, AnalyzeDataOutputAsset, AnalyzeRouteEntries, AnalyzeRouteEntry, ModulesDataOutputAsset,
combine_output_assets, combine_traced_files,
}, },
project::ProjectContainer, project::ProjectContainer,
route::EndpointGroupKey, route::{Endpoint, EndpointGroup, EndpointGroupKey},
};
use turbo_tasks::{
Effects, FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, ValueToStringRef, Vc,
}; };
use turbo_tasks::{Effects, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
issue::PlainIssue, issue::PlainIssue,
module::Module,
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
}; };
...@@ -53,6 +57,43 @@ async fn write_analyze_data_with_issues_operation_inner( ...@@ -53,6 +57,43 @@ async fn write_analyze_data_with_issues_operation_inner(
Ok(()) Ok(())
} }
async fn route_entries(
key: &EndpointGroupKey,
endpoint_group: &EndpointGroup,
role: &str,
) -> Result<Vec<AnalyzeRouteEntry>> {
let mut seen = FxIndexSet::default();
let mut result = vec![];
for (endpoint_index, endpoint) in endpoint_group.primary.iter().enumerate() {
let entries = endpoint.endpoint.entries().await?;
for module in entries.all_modules() {
let module_ident = module.ident().to_string().owned().await?;
if !seen.insert(module_ident.clone()) {
continue;
}
let module_path = module.ident().await?.path.to_string_ref().await?;
let sub_name = endpoint.sub_name.as_deref().unwrap_or("");
result.push(AnalyzeRouteEntry {
route_entry_id: format!(
"{}|{}|{}|{}|{}",
key.as_str(),
role,
endpoint_index,
sub_name,
module_ident
)
.into(),
module_ident,
module_path,
role: role.into(),
runtime: None,
});
}
}
result.sort_by(|a, b| a.route_entry_id.cmp(&b.route_entry_id));
Ok(result)
}
#[turbo_tasks::function(operation)] #[turbo_tasks::function(operation)]
async fn get_analyze_data_operation( async fn get_analyze_data_operation(
container: ResolvedVc<ProjectContainer>, container: ResolvedVc<ProjectContainer>,
...@@ -87,6 +128,15 @@ async fn get_analyze_data_operation( ...@@ -87,6 +128,15 @@ async fn get_analyze_data_operation(
let has_combined = !combined_output_assets.is_empty(); let has_combined = !combined_output_assets.is_empty();
let combined_assets_vc = Vc::cell(combined_output_assets); let combined_assets_vc = Vc::cell(combined_output_assets);
let combined_traced_vc = Vc::cell(combined_traced_files); let combined_traced_vc = Vc::cell(combined_traced_files);
let mut shared_route_entries = vec![];
for (key, endpoint_group) in endpoint_groups.iter() {
if matches!(
key,
EndpointGroupKey::PagesApp | EndpointGroupKey::PagesDocument
) {
shared_route_entries.extend(route_entries(key, endpoint_group, "shared").await?);
}
}
let analyze_data = endpoint_groups let analyze_data = endpoint_groups
.iter() .iter()
...@@ -113,12 +163,25 @@ async fn get_analyze_data_operation( ...@@ -113,12 +163,25 @@ async fn get_analyze_data_operation(
} else { } else {
endpoint_group.traced_files() endpoint_group.traced_files()
}; };
let mut entries = route_entries(key, endpoint_group, "route").await?;
if has_combined
&& !matches!(
key,
EndpointGroupKey::PagesApp | EndpointGroupKey::PagesDocument
)
{
entries.extend(shared_route_entries.iter().cloned());
entries.sort_by(|a, b| a.route_entry_id.cmp(&b.route_entry_id));
entries.dedup_by(|a, b| a.route_entry_id == b.route_entry_id);
}
let route_entries: Vc<AnalyzeRouteEntries> = Vc::cell(entries);
let analyze_data = AnalyzeDataOutputAsset::new( let analyze_data = AnalyzeDataOutputAsset::new(
analyze_output_root analyze_output_root
.join(&key.to_string())? .join(&key.to_string())?
.join("analyze.data")?, .join("analyze.data")?,
output_assets, output_assets,
traced_files, traced_files,
route_entries,
) )
.to_resolved() .to_resolved()
.await?; .await?;
......
...@@ -66,8 +66,8 @@ function modulesBuffer(): ArrayBuffer { ...@@ -66,8 +66,8 @@ function modulesBuffer(): ArrayBuffer {
) )
} }
function analyzeBuffer(): ArrayBuffer { function analyzeBuffer(exact = true): ArrayBuffer {
const sections = [edges([[0]]), edges([[], [0]]), edges([[1], []])] const sections = [edges([[0], []]), edges([[], [0]]), edges([[1], []])]
let offset = 0 let offset = 0
const references = sections.map((section) => { const references = sections.map((section) => {
const reference = { offset, length: section.byteLength } const reference = { offset, length: section.byteLength }
...@@ -88,7 +88,23 @@ function analyzeBuffer(): ArrayBuffer { ...@@ -88,7 +88,23 @@ function analyzeBuffer(): ArrayBuffer {
compressed_size: 40, compressed_size: 40,
}, },
], ],
output_files: [{ filename: '[client-fs]/app.js' }], output_files: [
{ filename: '[client-fs]/app.js' },
{ filename: '[client-fs]/font.woff2' },
],
...(exact
? {
route_entries: [
{
route_entry_id: 'route|client|first',
module_ident: 'first',
module_path: '[project]/src/a.ts',
role: 'route',
runtime: 'client',
},
],
}
: {}),
output_file_chunk_parts: references[0], output_file_chunk_parts: references[0],
source_chunk_parts: references[1], source_chunk_parts: references[1],
source_children: references[2], source_children: references[2],
...@@ -107,7 +123,10 @@ describe('analyzer data parser', () => { ...@@ -107,7 +123,10 @@ describe('analyzer data parser', () => {
expect(analyze.getSourceFlags(1).client).toBe(true) expect(analyze.getSourceFlags(1).client).toBe(true)
expect(analyze.sourceChunks(1)).toEqual(['[client-fs]/app.js']) expect(analyze.sourceChunks(1)).toEqual(['[client-fs]/app.js'])
expect(analyze.sourceChildren(0)).toEqual([1]) expect(analyze.sourceChildren(0)).toEqual([1])
expect(analyze.hasExactRouteEntries()).toBe(true)
expect(analyze.routeEntries()).toEqual([
expect.objectContaining({ route_entry_id: 'route|client|first' }),
])
const modules = new ModulesData(modulesBuffer()) const modules = new ModulesData(modulesBuffer())
expect( expect(
modules.getModuleIndiciesFromPath('[project]/src/a.ts') modules.getModuleIndiciesFromPath('[project]/src/a.ts')
...@@ -115,6 +134,26 @@ describe('analyzer data parser', () => { ...@@ -115,6 +134,26 @@ describe('analyzer data parser', () => {
expect(modules.moduleDependents(0)).toEqual([1]) expect(modules.moduleDependents(0)).toEqual([1])
expect(modules.asyncModuleDependents(1)).toEqual([0]) expect(modules.asyncModuleDependents(1)).toEqual([0])
expect(modules.moduleDependencies(1)).toEqual([0]) expect(modules.moduleDependencies(1)).toEqual([0])
expect(modules.getModuleIndexFromIdent('second')).toBe(1)
})
it('keeps old artifacts readable with explicit unavailable evidence', () => {
const analyze = new AnalyzeData(analyzeBuffer(false))
expect(analyze.hasExactRouteEntries()).toBe(false)
expect(analyze.routeEntries()).toEqual([])
})
it('rejects duplicate exact route entry IDs', () => {
const valid = new Uint8Array(analyzeBuffer())
const jsonLength = new DataView(valid.buffer).getUint32(0, false)
const header = JSON.parse(
new TextDecoder().decode(valid.slice(4, 4 + jsonLength))
)
header.route_entries.push({ ...header.route_entries[0] })
const binary = valid.slice(4 + jsonLength)
expect(() => new AnalyzeData(frame(header, [binary]))).toThrow(
'Invalid analyze.data route entry'
)
}) })
it('rejects truncated and malformed framing', () => { it('rejects truncated and malformed framing', () => {
......
...@@ -24,6 +24,14 @@ export interface AnalyzeOutputFile { ...@@ -24,6 +24,14 @@ export interface AnalyzeOutputFile {
filename: string filename: string
} }
export interface AnalyzeRouteEntry {
route_entry_id: string
module_ident: string
module_path: string
role: 'route' | 'shared'
runtime: string | null
}
export interface AnalyzeLayer { export interface AnalyzeLayer {
name: string name: string
} }
...@@ -37,6 +45,8 @@ interface AnalyzeDataHeader { ...@@ -37,6 +45,8 @@ interface AnalyzeDataHeader {
sources: AnalyzeSource[] sources: AnalyzeSource[]
chunk_parts: AnalyzeChunkPart[] chunk_parts: AnalyzeChunkPart[]
output_files: AnalyzeOutputFile[] output_files: AnalyzeOutputFile[]
/** Absent in analyzer artifacts produced before exact route-entry support. */
route_entries?: AnalyzeRouteEntry[]
output_file_chunk_parts: EdgesDataReference output_file_chunk_parts: EdgesDataReference
source_chunk_parts: EdgesDataReference source_chunk_parts: EdgesDataReference
source_children: EdgesDataReference source_children: EdgesDataReference
...@@ -127,6 +137,7 @@ export class ModulesData { ...@@ -127,6 +137,7 @@ export class ModulesData {
private modulesHeader: ModulesDataHeader private modulesHeader: ModulesDataHeader
private modulesBinaryData: DataView private modulesBinaryData: DataView
private pathToModuleIndex: Map<string, ModuleIndex[]> private pathToModuleIndex: Map<string, ModuleIndex[]>
private identToModuleIndex: Map<string, ModuleIndex>
constructor(modulesArrayBuffer: ArrayBuffer) { constructor(modulesArrayBuffer: ArrayBuffer) {
;[this.modulesHeader, this.modulesBinaryData] = ;[this.modulesHeader, this.modulesBinaryData] =
...@@ -144,6 +155,7 @@ export class ModulesData { ...@@ -144,6 +155,7 @@ export class ModulesData {
} }
this.pathToModuleIndex = new Map() this.pathToModuleIndex = new Map()
this.identToModuleIndex = new Map()
for (let i = 0; i < this.modulesHeader.modules.length; i++) { for (let i = 0; i < this.modulesHeader.modules.length; i++) {
const module = this.modulesHeader.modules[i] const module = this.modulesHeader.modules[i]
if ( if (
...@@ -152,6 +164,10 @@ export class ModulesData { ...@@ -152,6 +164,10 @@ export class ModulesData {
) { ) {
throw new Error('Invalid modules.data module') throw new Error('Invalid modules.data module')
} }
if (this.identToModuleIndex.has(module.ident)) {
throw new Error('Invalid modules.data duplicate module identity')
}
this.identToModuleIndex.set(module.ident, i)
const existing = this.pathToModuleIndex.get(module.path) const existing = this.pathToModuleIndex.get(module.path)
if (existing) { if (existing) {
existing.push(i) existing.push(i)
...@@ -173,6 +189,10 @@ export class ModulesData { ...@@ -173,6 +189,10 @@ export class ModulesData {
return this.pathToModuleIndex.get(path) ?? [] return this.pathToModuleIndex.get(path) ?? []
} }
getModuleIndexFromIdent(ident: string): ModuleIndex | undefined {
return this.identToModuleIndex.get(ident)
}
private readEdgesDataAtIndex( private readEdgesDataAtIndex(
reference: EdgesDataReference, reference: EdgesDataReference,
index: ModuleIndex index: ModuleIndex
...@@ -281,6 +301,28 @@ export class AnalyzeData { ...@@ -281,6 +301,28 @@ export class AnalyzeData {
requireArray(parts, 'analyze.data chunk parts') requireArray(parts, 'analyze.data chunk parts')
requireArray(outputs, 'analyze.data output files') requireArray(outputs, 'analyze.data output files')
requireArray(this.analyzeHeader.source_roots, 'analyze.data source roots') requireArray(this.analyzeHeader.source_roots, 'analyze.data source roots')
if (this.analyzeHeader.route_entries !== undefined) {
requireArray(
this.analyzeHeader.route_entries,
'analyze.data route entries'
)
const ids = new Set<string>()
for (const entry of this.analyzeHeader.route_entries) {
if (
typeof entry?.route_entry_id !== 'string' ||
!entry.route_entry_id ||
typeof entry.module_ident !== 'string' ||
!entry.module_ident ||
typeof entry.module_path !== 'string' ||
(entry.role !== 'route' && entry.role !== 'shared') ||
(entry.runtime !== null && typeof entry.runtime !== 'string') ||
ids.has(entry.route_entry_id)
) {
throw new Error('Invalid analyze.data route entry')
}
ids.add(entry.route_entry_id)
}
}
validateEdges( validateEdges(
this.analyzeBinaryData, this.analyzeBinaryData,
this.analyzeHeader.output_file_chunk_parts, this.analyzeHeader.output_file_chunk_parts,
...@@ -431,6 +473,14 @@ export class AnalyzeData { ...@@ -431,6 +473,14 @@ export class AnalyzeData {
return edges return edges
} }
routeEntries(): AnalyzeRouteEntry[] {
return this.analyzeHeader.route_entries ?? []
}
hasExactRouteEntries(): boolean {
return this.analyzeHeader.route_entries !== undefined
}
outputFileChunkParts(index: number): number[] { outputFileChunkParts(index: number): number[] {
return this.readEdgesDataAtIndex( return this.readEdgesDataAtIndex(
this.analyzeHeader.output_file_chunk_parts, this.analyzeHeader.output_file_chunk_parts,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment