Unverified Commit f57896de authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Add exact analyzer route entries

Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent e3333ef1
......@@ -86,6 +86,20 @@ pub struct AnalyzeOutputFile {
pub filename: RcStr,
}
/// Exact module-graph entry associated with one route-specific analyzer dataset.
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug, Serialize)]
pub struct AnalyzeRouteEntry {
pub route_entry_id: RcStr,
pub module_ident: RcStr,
pub module_path: RcStr,
pub role: RcStr,
pub runtime: Option<RcStr>,
}
#[turbo_tasks::value(transparent)]
pub struct AnalyzeRouteEntries(Vec<AnalyzeRouteEntry>);
#[derive(Serialize)]
struct EdgesDataReference {
pub offset: u32,
......@@ -97,7 +111,9 @@ struct AnalyzeDataHeader {
pub sources: Vec<AnalyzeSource>,
pub chunk_parts: Vec<AnalyzeChunkPart>,
pub output_files: Vec<AnalyzeOutputFile>,
/// Edges from chunks to chunk parts
/// Exact module-graph entries for this route dataset.
pub route_entries: Vec<AnalyzeRouteEntry>,
/// Edges from output files to chunk parts
pub output_file_chunk_parts: EdgesDataReference,
/// Edges from sources to chunk parts
pub source_chunk_parts: EdgesDataReference,
......@@ -150,6 +166,7 @@ struct AnalyzeDataBuilder {
source_index_map: FxHashMap<RcStr, u32>,
chunk_parts: Vec<AnalyzeChunkPart>,
output_files: Vec<AnalyzeOutputFileBuilder>,
route_entries: Vec<AnalyzeRouteEntry>,
}
struct ModulesDataBuilder {
......@@ -175,12 +192,13 @@ impl EdgesDataSectionBuilder {
}
impl AnalyzeDataBuilder {
fn new() -> Self {
fn new(route_entries: Vec<AnalyzeRouteEntry>) -> Self {
Self {
sources: vec![],
source_index_map: FxHashMap::default(),
chunk_parts: vec![],
output_files: vec![],
route_entries,
}
}
......@@ -251,7 +269,6 @@ impl AnalyzeDataBuilder {
let output_file_chunk_parts =
EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices));
let mut binary_section = EdgesDataSectionBuilder::new();
let header = AnalyzeDataHeader {
......@@ -262,6 +279,7 @@ impl AnalyzeDataBuilder {
.into_iter()
.map(|of| of.output_file)
.collect(),
route_entries: self.route_entries,
output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts),
source_chunk_parts: binary_section.add_edges(&source_chunk_parts),
source_children: binary_section.add_edges(&source_children),
......@@ -405,17 +423,18 @@ pub async fn combine_traced_files(
pub async fn analyze_output_assets(
output_assets: Vc<OutputAssets>,
traced_files: Vc<FileSystemPathVec>,
route_entries: Vc<AnalyzeRouteEntries>,
) -> Result<Vc<FileContent>> {
let output_assets = all_assets_from_entries(output_assets);
let output_assets = all_assets_from_entries(output_assets).await?;
let route_entries = route_entries.await?.iter().cloned().collect();
let mut builder = AnalyzeDataBuilder::new();
let mut builder = AnalyzeDataBuilder::new(route_entries);
let prefix = format!("{SOURCE_URL_PROTOCOL}///");
// Process the output assets and extract chunk parts.
// Also creates sources for the chunk parts.
for asset in output_assets
.await?
.iter()
.copied()
.map(Either::Left)
......@@ -637,6 +656,7 @@ pub struct AnalyzeDataOutputAsset {
pub path: FileSystemPath,
pub output_assets: ResolvedVc<OutputAssets>,
pub traced_files: ResolvedVc<FileSystemPathVec>,
pub route_entries: ResolvedVc<AnalyzeRouteEntries>,
}
#[turbo_tasks::value_impl]
......@@ -646,11 +666,13 @@ impl AnalyzeDataOutputAsset {
path: FileSystemPath,
output_assets: ResolvedVc<OutputAssets>,
traced_files: ResolvedVc<FileSystemPathVec>,
route_entries: ResolvedVc<AnalyzeRouteEntries>,
) -> Result<Vc<Self>> {
Ok(Self {
path,
output_assets,
traced_files,
route_entries,
}
.cell())
}
......@@ -660,7 +682,8 @@ impl AnalyzeDataOutputAsset {
impl Asset for AnalyzeDataOutputAsset {
#[turbo_tasks::function]
fn content(&self) -> Vc<AssetContent> {
let file_content = analyze_output_assets(*self.output_assets, *self.traced_files);
let file_content =
analyze_output_assets(*self.output_assets, *self.traced_files, *self.route_entries);
AssetContent::file(file_content)
}
}
......
......@@ -3,15 +3,19 @@ use std::{iter::once, sync::Arc};
use anyhow::Result;
use next_api::{
analyze::{
AnalyzeDataOutputAsset, ModulesDataOutputAsset, combine_output_assets, combine_traced_files,
AnalyzeDataOutputAsset, AnalyzeRouteEntries, AnalyzeRouteEntry, ModulesDataOutputAsset,
combine_output_assets, combine_traced_files,
},
project::ProjectContainer,
route::EndpointGroupKey,
route::{Endpoint, EndpointGroup, EndpointGroupKey},
};
use turbo_tasks::{
Effects, FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, ValueToStringRef, Vc,
};
use turbo_tasks::{Effects, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
issue::PlainIssue,
module::Module,
output::{OutputAsset, OutputAssets},
};
......@@ -53,6 +57,43 @@ async fn write_analyze_data_with_issues_operation_inner(
Ok(())
}
async fn route_entries(
key: &EndpointGroupKey,
endpoint_group: &EndpointGroup,
role: &str,
) -> Result<Vec<AnalyzeRouteEntry>> {
let mut seen = FxIndexSet::default();
let mut result = vec![];
for (endpoint_index, endpoint) in endpoint_group.primary.iter().enumerate() {
let entries = endpoint.endpoint.entries().await?;
for module in entries.all_modules() {
let module_ident = module.ident().to_string().owned().await?;
if !seen.insert(module_ident.clone()) {
continue;
}
let module_path = module.ident().await?.path.to_string_ref().await?;
let sub_name = endpoint.sub_name.as_deref().unwrap_or("");
result.push(AnalyzeRouteEntry {
route_entry_id: format!(
"{}|{}|{}|{}|{}",
key.as_str(),
role,
endpoint_index,
sub_name,
module_ident
)
.into(),
module_ident,
module_path,
role: role.into(),
runtime: None,
});
}
}
result.sort_by(|a, b| a.route_entry_id.cmp(&b.route_entry_id));
Ok(result)
}
#[turbo_tasks::function(operation)]
async fn get_analyze_data_operation(
container: ResolvedVc<ProjectContainer>,
......@@ -87,6 +128,15 @@ async fn get_analyze_data_operation(
let has_combined = !combined_output_assets.is_empty();
let combined_assets_vc = Vc::cell(combined_output_assets);
let combined_traced_vc = Vc::cell(combined_traced_files);
let mut shared_route_entries = vec![];
for (key, endpoint_group) in endpoint_groups.iter() {
if matches!(
key,
EndpointGroupKey::PagesApp | EndpointGroupKey::PagesDocument
) {
shared_route_entries.extend(route_entries(key, endpoint_group, "shared").await?);
}
}
let analyze_data = endpoint_groups
.iter()
......@@ -113,12 +163,25 @@ async fn get_analyze_data_operation(
} else {
endpoint_group.traced_files()
};
let mut entries = route_entries(key, endpoint_group, "route").await?;
if has_combined
&& !matches!(
key,
EndpointGroupKey::PagesApp | EndpointGroupKey::PagesDocument
)
{
entries.extend(shared_route_entries.iter().cloned());
entries.sort_by(|a, b| a.route_entry_id.cmp(&b.route_entry_id));
entries.dedup_by(|a, b| a.route_entry_id == b.route_entry_id);
}
let route_entries: Vc<AnalyzeRouteEntries> = Vc::cell(entries);
let analyze_data = AnalyzeDataOutputAsset::new(
analyze_output_root
.join(&key.to_string())?
.join("analyze.data")?,
output_assets,
traced_files,
route_entries,
)
.to_resolved()
.await?;
......
......@@ -66,8 +66,8 @@ function modulesBuffer(): ArrayBuffer {
)
}
function analyzeBuffer(): ArrayBuffer {
const sections = [edges([[0]]), edges([[], [0]]), edges([[1], []])]
function analyzeBuffer(exact = true): ArrayBuffer {
const sections = [edges([[0], []]), edges([[], [0]]), edges([[1], []])]
let offset = 0
const references = sections.map((section) => {
const reference = { offset, length: section.byteLength }
......@@ -88,7 +88,23 @@ function analyzeBuffer(): ArrayBuffer {
compressed_size: 40,
},
],
output_files: [{ filename: '[client-fs]/app.js' }],
output_files: [
{ filename: '[client-fs]/app.js' },
{ filename: '[client-fs]/font.woff2' },
],
...(exact
? {
route_entries: [
{
route_entry_id: 'route|client|first',
module_ident: 'first',
module_path: '[project]/src/a.ts',
role: 'route',
runtime: 'client',
},
],
}
: {}),
output_file_chunk_parts: references[0],
source_chunk_parts: references[1],
source_children: references[2],
......@@ -107,7 +123,10 @@ describe('analyzer data parser', () => {
expect(analyze.getSourceFlags(1).client).toBe(true)
expect(analyze.sourceChunks(1)).toEqual(['[client-fs]/app.js'])
expect(analyze.sourceChildren(0)).toEqual([1])
expect(analyze.hasExactRouteEntries()).toBe(true)
expect(analyze.routeEntries()).toEqual([
expect.objectContaining({ route_entry_id: 'route|client|first' }),
])
const modules = new ModulesData(modulesBuffer())
expect(
modules.getModuleIndiciesFromPath('[project]/src/a.ts')
......@@ -115,6 +134,26 @@ describe('analyzer data parser', () => {
expect(modules.moduleDependents(0)).toEqual([1])
expect(modules.asyncModuleDependents(1)).toEqual([0])
expect(modules.moduleDependencies(1)).toEqual([0])
expect(modules.getModuleIndexFromIdent('second')).toBe(1)
})
it('keeps old artifacts readable with explicit unavailable evidence', () => {
const analyze = new AnalyzeData(analyzeBuffer(false))
expect(analyze.hasExactRouteEntries()).toBe(false)
expect(analyze.routeEntries()).toEqual([])
})
it('rejects duplicate exact route entry IDs', () => {
const valid = new Uint8Array(analyzeBuffer())
const jsonLength = new DataView(valid.buffer).getUint32(0, false)
const header = JSON.parse(
new TextDecoder().decode(valid.slice(4, 4 + jsonLength))
)
header.route_entries.push({ ...header.route_entries[0] })
const binary = valid.slice(4 + jsonLength)
expect(() => new AnalyzeData(frame(header, [binary]))).toThrow(
'Invalid analyze.data route entry'
)
})
it('rejects truncated and malformed framing', () => {
......
......@@ -24,6 +24,14 @@ export interface AnalyzeOutputFile {
filename: string
}
export interface AnalyzeRouteEntry {
route_entry_id: string
module_ident: string
module_path: string
role: 'route' | 'shared'
runtime: string | null
}
export interface AnalyzeLayer {
name: string
}
......@@ -37,6 +45,8 @@ interface AnalyzeDataHeader {
sources: AnalyzeSource[]
chunk_parts: AnalyzeChunkPart[]
output_files: AnalyzeOutputFile[]
/** Absent in analyzer artifacts produced before exact route-entry support. */
route_entries?: AnalyzeRouteEntry[]
output_file_chunk_parts: EdgesDataReference
source_chunk_parts: EdgesDataReference
source_children: EdgesDataReference
......@@ -127,6 +137,7 @@ export class ModulesData {
private modulesHeader: ModulesDataHeader
private modulesBinaryData: DataView
private pathToModuleIndex: Map<string, ModuleIndex[]>
private identToModuleIndex: Map<string, ModuleIndex>
constructor(modulesArrayBuffer: ArrayBuffer) {
;[this.modulesHeader, this.modulesBinaryData] =
......@@ -144,6 +155,7 @@ export class ModulesData {
}
this.pathToModuleIndex = new Map()
this.identToModuleIndex = new Map()
for (let i = 0; i < this.modulesHeader.modules.length; i++) {
const module = this.modulesHeader.modules[i]
if (
......@@ -152,6 +164,10 @@ export class ModulesData {
) {
throw new Error('Invalid modules.data module')
}
if (this.identToModuleIndex.has(module.ident)) {
throw new Error('Invalid modules.data duplicate module identity')
}
this.identToModuleIndex.set(module.ident, i)
const existing = this.pathToModuleIndex.get(module.path)
if (existing) {
existing.push(i)
......@@ -173,6 +189,10 @@ export class ModulesData {
return this.pathToModuleIndex.get(path) ?? []
}
getModuleIndexFromIdent(ident: string): ModuleIndex | undefined {
return this.identToModuleIndex.get(ident)
}
private readEdgesDataAtIndex(
reference: EdgesDataReference,
index: ModuleIndex
......@@ -281,6 +301,28 @@ export class AnalyzeData {
requireArray(parts, 'analyze.data chunk parts')
requireArray(outputs, 'analyze.data output files')
requireArray(this.analyzeHeader.source_roots, 'analyze.data source roots')
if (this.analyzeHeader.route_entries !== undefined) {
requireArray(
this.analyzeHeader.route_entries,
'analyze.data route entries'
)
const ids = new Set<string>()
for (const entry of this.analyzeHeader.route_entries) {
if (
typeof entry?.route_entry_id !== 'string' ||
!entry.route_entry_id ||
typeof entry.module_ident !== 'string' ||
!entry.module_ident ||
typeof entry.module_path !== 'string' ||
(entry.role !== 'route' && entry.role !== 'shared') ||
(entry.runtime !== null && typeof entry.runtime !== 'string') ||
ids.has(entry.route_entry_id)
) {
throw new Error('Invalid analyze.data route entry')
}
ids.add(entry.route_entry_id)
}
}
validateEdges(
this.analyzeBinaryData,
this.analyzeHeader.output_file_chunk_parts,
......@@ -431,6 +473,14 @@ export class AnalyzeData {
return edges
}
routeEntries(): AnalyzeRouteEntry[] {
return this.analyzeHeader.route_entries ?? []
}
hasExactRouteEntries(): boolean {
return this.analyzeHeader.route_entries !== undefined
}
outputFileChunkParts(index: number): number[] {
return this.readEdgesDataAtIndex(
this.analyzeHeader.output_file_chunk_parts,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment