Unverified Commit a8d92d4f authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Add emitted output analyzer queries

Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent 0d832730
...@@ -114,6 +114,8 @@ struct AnalyzeDataHeader { ...@@ -114,6 +114,8 @@ struct AnalyzeDataHeader {
pub output_files: Vec<AnalyzeOutputFile>, pub output_files: Vec<AnalyzeOutputFile>,
/// Exact module-graph entries for this route dataset. /// Exact module-graph entries for this route dataset.
pub route_entries: Vec<AnalyzeRouteEntry>, pub route_entries: Vec<AnalyzeRouteEntry>,
/// Edges from output files to directly referenced output files.
pub output_file_references: EdgesDataReference,
/// Edges from output files to chunk parts /// Edges from output files to chunk parts
pub output_file_chunk_parts: EdgesDataReference, pub output_file_chunk_parts: EdgesDataReference,
/// Edges from sources to chunk parts /// Edges from sources to chunk parts
...@@ -146,6 +148,7 @@ struct ModulesDataHeader { ...@@ -146,6 +148,7 @@ struct ModulesDataHeader {
struct AnalyzeOutputFileBuilder { struct AnalyzeOutputFileBuilder {
output_file: AnalyzeOutputFile, output_file: AnalyzeOutputFile,
chunk_part_indices: Vec<u32>, chunk_part_indices: Vec<u32>,
referenced_output_file_indices: Vec<u32>,
} }
struct AnalyzeSourceBuilder { struct AnalyzeSourceBuilder {
...@@ -234,6 +237,7 @@ impl AnalyzeDataBuilder { ...@@ -234,6 +237,7 @@ impl AnalyzeDataBuilder {
self.output_files.push(AnalyzeOutputFileBuilder { self.output_files.push(AnalyzeOutputFileBuilder {
output_file, output_file,
chunk_part_indices: vec![], chunk_part_indices: vec![],
referenced_output_file_indices: vec![],
}); });
i i
} }
...@@ -244,6 +248,12 @@ impl AnalyzeDataBuilder { ...@@ -244,6 +248,12 @@ impl AnalyzeDataBuilder {
.push(chunk_part_index); .push(chunk_part_index);
} }
fn add_output_file_reference(&mut self, output_file_index: u32, referenced_index: u32) {
self.output_files[output_file_index as usize]
.referenced_output_file_indices
.push(referenced_index);
}
fn add_chunk_part_to_source(&mut self, source_index: u32, chunk_part_index: u32) { fn add_chunk_part_to_source(&mut self, source_index: u32, chunk_part_index: u32) {
self.sources[source_index as usize] self.sources[source_index as usize]
.chunk_part_indices .chunk_part_indices
...@@ -272,6 +282,12 @@ impl AnalyzeDataBuilder { ...@@ -272,6 +282,12 @@ impl AnalyzeDataBuilder {
let output_file_chunk_parts = let output_file_chunk_parts =
EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices)); EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices));
let output_file_references = EdgesData::from_iterator(
self.output_files
.iter()
.map(|of| &of.referenced_output_file_indices),
);
let mut binary_section = EdgesDataSectionBuilder::new(); let mut binary_section = EdgesDataSectionBuilder::new();
let header = AnalyzeDataHeader { let header = AnalyzeDataHeader {
...@@ -283,6 +299,7 @@ impl AnalyzeDataBuilder { ...@@ -283,6 +299,7 @@ impl AnalyzeDataBuilder {
.map(|of| of.output_file) .map(|of| of.output_file)
.collect(), .collect(),
route_entries: self.route_entries, route_entries: self.route_entries,
output_file_references: binary_section.add_edges(&output_file_references),
output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts), output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts),
source_chunk_parts: binary_section.add_edges(&source_chunk_parts), source_chunk_parts: binary_section.add_edges(&source_chunk_parts),
source_children: binary_section.add_edges(&source_children), source_children: binary_section.add_edges(&source_children),
...@@ -553,6 +570,7 @@ pub async fn analyze_output_assets( ...@@ -553,6 +570,7 @@ pub async fn analyze_output_assets(
let route_entries = route_entries.await?.iter().cloned().collect(); let route_entries = route_entries.await?.iter().cloned().collect();
let mut builder = AnalyzeDataBuilder::new(route_entries); let mut builder = AnalyzeDataBuilder::new(route_entries);
let mut output_asset_indices = FxHashMap::default();
let prefix = format!("{SOURCE_URL_PROTOCOL}///"); let prefix = format!("{SOURCE_URL_PROTOCOL}///");
...@@ -585,6 +603,9 @@ pub async fn analyze_output_assets( ...@@ -585,6 +603,9 @@ pub async fn analyze_output_assets(
let output_file_index = builder.add_output_file(AnalyzeOutputFile { let output_file_index = builder.add_output_file(AnalyzeOutputFile {
filename: filename.clone(), filename: filename.clone(),
}); });
if let Either::Left(asset) = &asset {
output_asset_indices.insert(*asset, output_file_index);
}
let chunk_parts = match asset { let chunk_parts = match asset {
Either::Left(asset) => split_output_asset_into_parts(*asset).await?, Either::Left(asset) => split_output_asset_into_parts(*asset).await?,
Either::Right(path) => split_traced_file_into_parts(path).await?, Either::Right(path) => split_traced_file_into_parts(path).await?,
...@@ -614,6 +635,17 @@ pub async fn analyze_output_assets( ...@@ -614,6 +635,17 @@ pub async fn analyze_output_assets(
} }
} }
// Preserve direct output-asset references before the flattened asset list loses
// their relationship. Queries use these facts for CSS-to-asset evidence.
for (asset, output_file_index) in &output_asset_indices {
let references = asset.references().all_assets().await?;
for referenced in references.iter() {
if let Some(referenced_index) = output_asset_indices.get(referenced) {
builder.add_output_file_reference(*output_file_index, *referenced_index);
}
}
}
// Build a directory structure for the sources. // Build a directory structure for the sources.
let mut i: u32 = 0; let mut i: u32 = 0;
while i < builder.sources.len().try_into().unwrap() { while i < builder.sources.len().try_into().unwrap() {
......
...@@ -63,6 +63,27 @@ interface SourceChunksArgs { ...@@ -63,6 +63,27 @@ interface SourceChunksArgs {
limit?: number limit?: number
} }
interface RouteOutputsArgs {
route: string
snapshot?: string
search?: string
kinds?: OutputKind[]
environment?: Environment
groupBy?: 'file' | 'kind' | 'environment'
metric?: Metric
offset?: number
limit?: number
}
interface CssAssetsArgs {
route: string
snapshot?: string
cssSearch?: string
assetKinds?: Array<'font' | 'image' | 'media'>
offset?: number
limit?: number
}
export class AnalyzeQueryError extends Error { export class AnalyzeQueryError extends Error {
constructor(readonly output: Record<string, unknown>) { constructor(readonly output: Record<string, unknown>) {
super( super(
...@@ -195,6 +216,18 @@ function outputKind(filename: string): OutputKind { ...@@ -195,6 +216,18 @@ function outputKind(filename: string): OutputKind {
return 'other' return 'other'
} }
function outputSizes(data: AnalyzeData, outputIndex: number) {
let rawSize = 0
let compressedSize = 0
for (const partIndex of data.outputFileChunkParts(outputIndex)) {
const part = data.chunkPart(partIndex)
if (!part) continue
rawSize += part.size
compressedSize += part.compressed_size
}
return { rawSize, compressedSize }
}
function packageNameFromPath(sourcePath: string): string | undefined { function packageNameFromPath(sourcePath: string): string | undefined {
const marker = 'node_modules/' const marker = 'node_modules/'
const start = sourcePath.lastIndexOf(marker) const start = sourcePath.lastIndexOf(marker)
...@@ -472,5 +505,197 @@ export function createAnalyzeQueryRegistry(repository: AnalyzeRepository) { ...@@ -472,5 +505,197 @@ export function createAnalyzeQueryRegistry(repository: AnalyzeRepository) {
}) })
) )
registerQuery(
'get_route_outputs',
{
route: z.string().max(4096),
snapshot: snapshotSchema,
search: z.string().max(1000).optional(),
kinds: z
.array(
z.enum([
'js',
'css',
'json',
'font',
'image',
'media',
'wasm',
'other',
])
)
.max(8)
.optional(),
environment: environmentSchema,
groupBy: z.enum(['file', 'kind', 'environment']).optional(),
metric: metricSchema,
...pagingSchema,
},
safeQuery(async (args: RouteOutputsArgs) => {
const data = await repository.loadRoute(args.snapshot, args.route)
const environment = args.environment ?? 'total'
const groupBy = args.groupBy ?? 'file'
const metric = args.metric ?? 'raw'
const search = args.search?.toLocaleLowerCase()
const files = Array.from(
{ length: data.outputFileCount() },
(_, index) => {
const filename = data.outputFile(index)!.filename
return {
key: filename,
filename,
kind: outputKind(filename),
environment: outputEnvironment(filename),
...outputSizes(data, index),
outputCount: 1,
emissionEvidence: 'emitted' as const,
requestEvidence: 'unknown' as const,
}
}
).filter(
(row) =>
(!search || row.filename.toLocaleLowerCase().includes(search)) &&
(!args.kinds?.length || args.kinds.includes(row.kind)) &&
(environment === 'total' || row.environment === environment)
)
const grouped = new Map<string, (typeof files)[number]>()
for (const row of files) {
const key =
groupBy === 'kind'
? row.kind
: groupBy === 'environment'
? row.environment
: row.filename
const existing = grouped.get(key)
if (existing) {
existing.rawSize += row.rawSize
existing.compressedSize += row.compressedSize
existing.outputCount += 1
} else {
grouped.set(key, { ...row, key })
}
}
const rows = [...grouped.values()].sort(
(a, b) =>
metricValue(b, metric) - metricValue(a, metric) ||
compareText(a.key, b.key)
)
const paged = page(rows, args.offset ?? 0, args.limit ?? DEFAULT_LIMIT)
return {
route: args.route,
snapshot: provenance(
(await repository.getSnapshot(args.snapshot)).metadata
),
environment,
groupBy,
metric,
outputs: paged.values,
totals: files.reduce(
(total, row) => ({
rawSize: total.rawSize + row.rawSize,
compressedSize: total.compressedSize + row.compressedSize,
outputCount: total.outputCount + 1,
}),
{ rawSize: 0, compressedSize: 0, outputCount: 0 }
),
pagination: paged.pagination,
caveats: [
COMPRESSED_CAVEAT,
'Emitted outputs are not observed browser requests.',
],
}
})
)
registerQuery(
'get_css_assets',
{
route: z.string().max(4096),
snapshot: snapshotSchema,
cssSearch: z.string().max(1000).optional(),
assetKinds: z
.array(z.enum(['font', 'image', 'media']))
.max(3)
.optional(),
...pagingSchema,
},
safeQuery(async (args: CssAssetsArgs) => {
const data = await repository.loadRoute(args.snapshot, args.route)
const available = data.hasOutputFileReferences()
const search = args.cssSearch?.toLocaleLowerCase()
const kinds = args.assetKinds ?? ['font', 'image', 'media']
const rows: Array<{
key: string
cssFilename: string
assetFilename: string
assetKind: 'font' | 'image' | 'media'
rawSize: number
compressedSize: number
relationshipEvidence: 'output-reference'
emissionEvidence: 'emitted'
requestEvidence: 'unknown'
}> = []
if (available) {
for (let cssIndex = 0; cssIndex < data.outputFileCount(); cssIndex++) {
const css = data.outputFile(cssIndex)!
if (
outputKind(css.filename) !== 'css' ||
(search && !css.filename.toLocaleLowerCase().includes(search))
) {
continue
}
for (const assetIndex of data.outputFileReferences(cssIndex)) {
const asset = data.outputFile(assetIndex)
if (!asset) continue
const kind = outputKind(asset.filename)
if (kind !== 'font' && kind !== 'image' && kind !== 'media')
continue
if (!kinds.includes(kind)) continue
rows.push({
key: `${css.filename}\0${asset.filename}`,
cssFilename: css.filename,
assetFilename: asset.filename,
assetKind: kind,
...outputSizes(data, assetIndex),
relationshipEvidence: 'output-reference',
emissionEvidence: 'emitted',
requestEvidence: 'unknown',
})
}
}
}
rows.sort((a, b) => compareText(a.key, b.key))
const paged = page(rows, args.offset ?? 0, args.limit ?? DEFAULT_LIMIT)
return {
route: args.route,
snapshot: provenance(
(await repository.getSnapshot(args.snapshot)).metadata
),
relationshipEvidence: available ? 'output-reference' : 'unavailable',
assets: paged.values,
totals: rows.reduce(
(totals, row) => {
totals.rawSize += row.rawSize
totals.compressedSize += row.compressedSize
totals.assetCount++
totals.byKind[row.assetKind]++
return totals
},
{
rawSize: 0,
compressedSize: 0,
assetCount: 0,
byKind: { font: 0, image: 0, media: 0 },
}
),
pagination: paged.pagination,
caveats: [
COMPRESSED_CAVEAT,
'Output references prove emitted relationships, not browser requests or timing.',
],
}
})
)
return new AnalyzeQueryRegistry(queries) return new AnalyzeQueryRegistry(queries)
} }
...@@ -71,7 +71,12 @@ function modulesBuffer( ...@@ -71,7 +71,12 @@ function modulesBuffer(
} }
function analyzeBuffer(exact = true): ArrayBuffer { function analyzeBuffer(exact = true): ArrayBuffer {
const sections = [edges([[0], []]), edges([[], [0]]), edges([[1], []])] const sections = [
edges(exact ? [[1], []] : []),
edges([[0], []]),
edges([[], [0]]),
edges([[1], []]),
]
let offset = 0 let offset = 0
const references = sections.map((section) => { const references = sections.map((section) => {
const reference = { offset, length: section.byteLength } const reference = { offset, length: section.byteLength }
...@@ -107,11 +112,12 @@ function analyzeBuffer(exact = true): ArrayBuffer { ...@@ -107,11 +112,12 @@ function analyzeBuffer(exact = true): ArrayBuffer {
runtime: 'client', runtime: 'client',
}, },
], ],
output_file_references: references[0],
} }
: {}), : {}),
output_file_chunk_parts: references[0], output_file_chunk_parts: references[1],
source_chunk_parts: references[1], source_chunk_parts: references[2],
source_children: references[2], source_children: references[3],
source_roots: [0], source_roots: [0],
}, },
sections sections
...@@ -131,6 +137,8 @@ describe('analyzer data parser', () => { ...@@ -131,6 +137,8 @@ describe('analyzer data parser', () => {
expect(analyze.routeEntries()).toEqual([ expect(analyze.routeEntries()).toEqual([
expect.objectContaining({ route_entry_id: 'route|client|first' }), expect.objectContaining({ route_entry_id: 'route|client|first' }),
]) ])
expect(analyze.hasOutputFileReferences()).toBe(true)
expect(analyze.outputFileReferences(0)).toEqual([1])
const modules = new ModulesData(modulesBuffer()) const modules = new ModulesData(modulesBuffer())
expect( expect(
modules.getModuleIndiciesFromPath('[project]/src/a.ts') modules.getModuleIndiciesFromPath('[project]/src/a.ts')
...@@ -148,6 +156,8 @@ describe('analyzer data parser', () => { ...@@ -148,6 +156,8 @@ describe('analyzer data parser', () => {
const analyze = new AnalyzeData(analyzeBuffer(false)) const analyze = new AnalyzeData(analyzeBuffer(false))
expect(analyze.hasExactRouteEntries()).toBe(false) expect(analyze.hasExactRouteEntries()).toBe(false)
expect(analyze.routeEntries()).toEqual([]) expect(analyze.routeEntries()).toEqual([])
expect(analyze.hasOutputFileReferences()).toBe(false)
expect(analyze.outputFileReferences(0)).toEqual([])
const modules = new ModulesData(modulesBuffer(false)) const modules = new ModulesData(modulesBuffer(false))
expect(modules.hasExactSyncSccs()).toBe(false) expect(modules.hasExactSyncSccs()).toBe(false)
expect(modules.syncSccId(0)).toBeUndefined() expect(modules.syncSccId(0)).toBeUndefined()
......
...@@ -47,6 +47,8 @@ interface AnalyzeDataHeader { ...@@ -47,6 +47,8 @@ interface AnalyzeDataHeader {
output_files: AnalyzeOutputFile[] output_files: AnalyzeOutputFile[]
/** Absent in analyzer artifacts produced before exact route-entry support. */ /** Absent in analyzer artifacts produced before exact route-entry support. */
route_entries?: AnalyzeRouteEntry[] route_entries?: AnalyzeRouteEntry[]
/** Absent in analyzer artifacts that flattened output references. */
output_file_references?: EdgesDataReference
output_file_chunk_parts: EdgesDataReference output_file_chunk_parts: EdgesDataReference
source_chunk_parts: EdgesDataReference source_chunk_parts: EdgesDataReference
source_children: EdgesDataReference source_children: EdgesDataReference
...@@ -351,6 +353,15 @@ export class AnalyzeData { ...@@ -351,6 +353,15 @@ export class AnalyzeData {
ids.add(entry.route_entry_id) ids.add(entry.route_entry_id)
} }
} }
if (this.analyzeHeader.output_file_references !== undefined) {
validateEdges(
this.analyzeBinaryData,
this.analyzeHeader.output_file_references,
outputs.length,
outputs.length,
'analyze.data output references'
)
}
validateEdges( validateEdges(
this.analyzeBinaryData, this.analyzeBinaryData,
this.analyzeHeader.output_file_chunk_parts, this.analyzeHeader.output_file_chunk_parts,
...@@ -509,6 +520,15 @@ export class AnalyzeData { ...@@ -509,6 +520,15 @@ export class AnalyzeData {
return this.analyzeHeader.route_entries !== undefined return this.analyzeHeader.route_entries !== undefined
} }
outputFileReferences(index: number): number[] {
const reference = this.analyzeHeader.output_file_references
return reference ? this.readEdgesDataAtIndex(reference, index) : []
}
hasOutputFileReferences(): boolean {
return this.analyzeHeader.output_file_references !== undefined
}
outputFileChunkParts(index: number): number[] { outputFileChunkParts(index: number): number[] {
return this.readEdgesDataAtIndex( return this.readEdgesDataAtIndex(
this.analyzeHeader.output_file_chunk_parts, this.analyzeHeader.output_file_chunk_parts,
......
analyzer-font-fixture
\ No newline at end of file
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1 1"><path d="M0 0h1v1H0z"/></svg>
import './styles.css'
export default function Page() { export default function Page() {
return <div>Hello World</div> return <div>Hello World</div>
} }
@font-face {
font-family: 'AnalyzerFixture';
src: url('./fixture-font.woff2') format('woff2');
}
button {
font-family: 'AnalyzerFixture', sans-serif;
background-image: url('./fixture-image.svg');
}
...@@ -124,7 +124,47 @@ describe('next experimental-analyze', () => { ...@@ -124,7 +124,47 @@ describe('next experimental-analyze', () => {
expect(sources.exitCode).toBe(0) expect(sources.exitCode).toBe(0)
expect(JSON.parse(sources.stdout)).toMatchObject({ expect(JSON.parse(sources.stdout)).toMatchObject({
route: '/', route: '/',
sources: [{ sourcePath: expect.any(String) }], sources: [
{
sourcePath: expect.any(String),
chunkCount: expect.any(Number),
},
],
})
expect(JSON.parse(sources.stdout).sources[0]).not.toHaveProperty(
'chunks'
)
const outputs = await next.runCommand([
'experimental-analyze',
'query',
'get_route_outputs',
'--input',
'{"route":"/","kinds":["css","font","image"]}',
])
expect(outputs.exitCode).toBe(0)
expect(JSON.parse(outputs.stdout)).toMatchObject({
outputs: expect.arrayContaining([
expect.objectContaining({ kind: 'css' }),
expect.objectContaining({ kind: 'font' }),
expect.objectContaining({ kind: 'image' }),
]),
})
const cssAssets = await next.runCommand([
'experimental-analyze',
'query',
'get_css_assets',
'--input',
'{"route":"/"}',
])
expect(cssAssets.exitCode).toBe(0)
expect(JSON.parse(cssAssets.stdout)).toMatchObject({
relationshipEvidence: 'output-reference',
assets: expect.arrayContaining([
expect.objectContaining({ assetKind: 'font' }),
expect.objectContaining({ assetKind: 'image' }),
]),
}) })
}) })
}) })
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment