Unverified Commit 08954138 authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add next internal upload-trace (#92191)

### What?

Adds `next internal upload-trace` CLI command for uploading CPU profiles
and Turbopack traces to Vercel Blob storage.

### Why?

When investigating performance issues, the Next.js team often needs CPU
profiles and Turbopack traces from contributors and bug reporters.
Currently there's no streamlined way to collect these files. This
command provides a single step to upload all profiling output after a
build.

### How?

- **`.next-profiles` directory**: CPU profiles
(`--experimental-cpu-prof`) and Turbopack traces (`--internal-trace`)
now write to `.next-profiles/` instead of separate locations,
consolidating all profiling output.
- **`next internal upload-trace [directory]`**: New CLI command that
uploads `.cpuprofile` files and `trace-turbopack` from `.next-profiles/`
to Vercel Blob via a secure handshake with an external API endpoint.
- Validates file headers before upload (JSON for `.cpuprofile`, binary
for `trace-turbopack`)
  - Skips empty files
  - Shows progress bar in TTY environments
  - Groups all files under a single server-assigned session ID
  - Only logs the session ID to the user (no internal blob URLs exposed)
- **`@vercel/blob` compiled**: Vendored into
`packages/next/src/compiled/@vercel/blob/` via ncc.
- **`--internal-trace` CLI flag**: Added to `next build` and `next dev`
to enable Turbopack tracing to `.next-profiles/trace-turbopack`.
- **Security**: The server controls the storage path (client only sends
filename), session continuity uses HMAC-signed tokens (stateless across
serverless instances), `allowOverwrite: false` prevents replacing
existing uploads, and filenames are validated against an allowlist.
- **Tests**: Added production test suite with a mock HTTP server.

Usage:

```bash
next build --experimental-cpu-prof --internal-trace
next internal upload-trace
```
parent 2800d2a7
......@@ -219,6 +219,7 @@
"@types/webpack-sources1": "npm:@types/webpack-sources@0.1.5",
"@types/ws": "8.2.0",
"@vercel/ncc": "0.34.0",
"@vercel/blob": "2.3.2",
"@vercel/nft": "0.27.1",
"@vercel/routing-utils": "5.2.0",
"@vercel/turbopack-ecmascript-runtime": "*",
......
......@@ -627,4 +627,22 @@ internal
})
.usage('[directory] [options]')
internal
.command('upload-trace')
.description(
'Upload CPU profiles from .next-profiles/ to Vercel Blob storage.'
)
.argument(
'[directory]',
`The project directory containing .next-profiles/. ${italic(
'If no directory is provided, the current directory will be used.'
)}`
)
.action((directory: string) => {
return import('../cli/internal/upload-trace.js').then((mod) =>
mod.uploadTraceToBlob({ directory })
)
})
.usage('[directory] [options]')
program.parse(process.argv)
import path from 'path'
import fs from 'fs/promises'
const UPLOAD_TRACE_URL = 'https://api.nextjs.org/api/upload-trace'
// V8 CPU profiles are JSON objects starting with {"nodes":
const CPUPROFILE_HEADER = Buffer.from('{"nodes":')
// Turbopack trace files start with this magic header (written by trace_writer.rs)
const TURBOPACK_TRACE_HEADER = Buffer.from('TRACEv0')
const PROGRESS_CHUNK_SIZE = 64 * 1024 // 64 KB
export interface UploadTraceOptions {
directory?: string
}
function getUploadUrl(): string {
return process.env.__NEXT_UPLOAD_TRACE_URL_OVERRIDE || UPLOAD_TRACE_URL
}
function formatBytes(bytes: number): string {
if (bytes < 1024) return `${bytes} B`
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
}
function renderProgressBar(current: number, total: number): string {
const width = 30
const ratio = Math.min(current / total, 1)
const filled = Math.round(width * ratio)
const bar = '\u2588'.repeat(filled) + '\u2591'.repeat(width - filled)
const percent = (ratio * 100).toFixed(0).padStart(3)
return ` [${bar}] ${percent}% ${formatBytes(current)}/${formatBytes(total)}`
}
function createProgressStream(
content: Buffer,
onProgress: (bytesRead: number) => void
): ReadableStream<Uint8Array> {
let offset = 0
return new ReadableStream({
pull(controller) {
if (offset >= content.length) {
controller.close()
return
}
const end = Math.min(offset + PROGRESS_CHUNK_SIZE, content.length)
controller.enqueue(content.subarray(offset, end))
offset = end
onProgress(offset)
},
})
}
function validateCpuProfile(header: Buffer, file: string): void {
if (
header.length < CPUPROFILE_HEADER.length ||
!header.subarray(0, CPUPROFILE_HEADER.length).equals(CPUPROFILE_HEADER)
) {
console.error(
`Error: ${file} does not appear to be a valid V8 CPU profile.`
)
process.exit(1)
}
}
function validateTurbopackTrace(header: Buffer, file: string): void {
if (
header.length < TURBOPACK_TRACE_HEADER.length ||
!header
.subarray(0, TURBOPACK_TRACE_HEADER.length)
.equals(TURBOPACK_TRACE_HEADER)
) {
console.error(
`Error: ${file} does not appear to be a valid Turbopack trace (missing TRACEv0 header).`
)
process.exit(1)
}
}
export async function uploadTraceToBlob(
options: UploadTraceOptions
): Promise<void> {
const dir = options.directory || process.cwd()
const profilesDir = path.join(dir, '.next-profiles')
let entries: string[]
try {
entries = await fs.readdir(profilesDir)
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
console.error(
`Error: Profiles directory not found at ${profilesDir}. Run "next build --experimental-cpu-prof" or "next build --internal-trace" first.`
)
process.exit(1)
}
throw err
}
const uploadableFiles = entries.filter(
(f) => f.endsWith('.cpuprofile') || f.endsWith('trace-turbopack')
)
if (uploadableFiles.length === 0) {
console.error(`Error: No profile or trace files found in ${profilesDir}.`)
process.exit(1)
}
const uploadUrl = getUploadUrl()
console.log(`Found ${uploadableFiles.length} file(s) in ${profilesDir}.`)
console.log(`Uploading to the Next.js team...`)
const { put } =
require('next/dist/compiled/@vercel/blob') as typeof import('next/dist/compiled/@vercel/blob')
let sessionId: string | undefined
let sessionToken: string | undefined
let uploadedCount = 0
for (const file of uploadableFiles) {
const filePath = path.join(profilesDir, file)
const stat = await fs.stat(filePath)
if (stat.size === 0) {
console.warn(`Skipping ${file}: file is empty.`)
continue
}
const fd = await fs.open(filePath, 'r')
const headerBuf = Buffer.alloc(16)
await fd.read(headerBuf, 0, 16, 0)
await fd.close()
if (file.endsWith('.cpuprofile')) {
validateCpuProfile(headerBuf, file)
} else if (file.endsWith('trace-turbopack')) {
validateTurbopackTrace(headerBuf, file)
}
const content = await fs.readFile(filePath)
const tokenRes = await fetch(uploadUrl, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
filename: file,
...(sessionId && sessionToken ? { sessionId, sessionToken } : {}),
}),
})
if (!tokenRes.ok) {
console.error(
`Error: Failed to get upload token for ${file} (${tokenRes.status} ${tokenRes.statusText})`
)
process.exit(1)
}
const tokenBody = (await tokenRes.json()) as {
clientToken: string
pathname: string
sessionId: string
sessionToken: string
}
if (!tokenBody.clientToken || !tokenBody.pathname) {
console.error('Error: Invalid response from the upload endpoint.')
process.exit(1)
}
if (!sessionId) {
sessionId = tokenBody.sessionId
sessionToken = tokenBody.sessionToken
}
const totalSize = content.length
if (process.stdout.isTTY) {
const stream = createProgressStream(content, (bytesRead) => {
process.stdout.write(`\r${renderProgressBar(bytesRead, totalSize)}`)
})
await put(tokenBody.pathname, stream, {
access: 'private',
token: tokenBody.clientToken,
})
process.stdout.write('\r' + ' '.repeat(80) + '\r')
} else {
await put(tokenBody.pathname, content, {
access: 'private',
token: tokenBody.clientToken,
})
}
uploadedCount++
console.log(`Uploaded ${file} (${formatBytes(totalSize)})`)
}
if (uploadedCount === 0) {
console.error('Error: No files were uploaded (all candidates were empty).')
process.exit(1)
}
if (sessionId) {
console.log(`\nUpload session: ${sessionId}`)
}
console.log('All files uploaded successfully.')
}
This diff is collapsed.
This diff is collapsed.
{"name":"@vercel/blob","main":"index.cjs","license":"Apache-2.0"}
......@@ -1887,6 +1887,14 @@ export async function ncc_strip_ansi(task, opts) {
.ncc({ packageName: 'strip-ansi', externals })
.target('src/compiled/strip-ansi')
}
externals['@vercel/blob'] = 'next/dist/compiled/@vercel/blob'
export async function ncc_vercel_blob(task, opts) {
await task
.source(relative(__dirname, require.resolve('@vercel/blob')))
.ncc({ packageName: '@vercel/blob', externals })
.target('src/compiled/@vercel/blob')
}
externals['@vercel/nft'] = 'next/dist/compiled/@vercel/nft'
export async function ncc_nft(task, opts) {
await task
......@@ -2300,6 +2308,7 @@ export async function ncc(task, opts) {
'ncc_superstruct',
'ncc_zod',
'ncc_zod_validation_error',
'ncc_vercel_blob',
'ncc_nft',
'ncc_tar',
'ncc_terser',
......
......@@ -758,6 +758,10 @@ declare module 'next/dist/compiled/strip-ansi' {
import m from 'strip-ansi'
export = m
}
declare module 'next/dist/compiled/@vercel/blob' {
import m from '@vercel/blob'
export = m
}
declare module 'next/dist/compiled/@vercel/nft' {
import m from '@vercel/nft'
export = m
......
......@@ -1376,6 +1376,9 @@ importers:
'@types/ws':
specifier: 8.2.0
version: 8.2.0
'@vercel/blob':
specifier: 2.3.2
version: 2.3.2
'@vercel/ncc':
specifier: 0.34.0
version: 0.34.0
......@@ -7260,6 +7263,10 @@ packages:
engines: {node: '>=18.0.0'}
hasBin: true
'@vercel/blob@2.3.2':
resolution: {integrity: sha512-FjlTJ5ffGj0RssrU0dJ90Zgr83AdhpIs2y9f3x386w+q9sM0P8m8Ouxsm3fqsjr4IyqTTo7Ioc2PJqxxOKNhjA==}
engines: {node: '>=20.0.0'}
'@vercel/kv@3.0.0':
resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==}
engines: {node: '>=14.6'}
......@@ -12009,6 +12016,10 @@ packages:
resolution: {integrity: sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==}
engines: {node: '>=4'}
is-buffer@2.0.5:
resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==}
engines: {node: '>=4'}
is-builtin-module@3.2.1:
resolution: {integrity: sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==}
engines: {node: '>=6'}
......@@ -17851,6 +17862,10 @@ packages:
third-party-capital@1.0.20:
resolution: {integrity: sha512-oB7yIimd8SuGptespDAZnNkzIz+NWaJCu2RMsbs4Wmp9zSDUM8Nhi3s2OOcqYuv3mN4hitXc8DVx+LyUmbUDiA==}
throttleit@2.1.0:
resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}
engines: {node: '>=18'}
through2@0.4.2:
resolution: {integrity: sha512-45Llu+EwHKtAZYTPPVn3XZHBgakWMN3rokhEv5hu596XP+cNgplMg+Gj+1nmAvj+L0K7+N49zBKx5rah5u0QIQ==}
......@@ -18269,6 +18284,10 @@ packages:
resolution: {integrity: sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==}
engines: {node: '>=14.0'}
undici@6.24.1:
resolution: {integrity: sha512-sC+b0tB1whOCzbtlx20fx3WgCXwkW627p4EA9uM+/tNNPkSS+eSEld6pAs9nDv7WbY1UUljBMYPtu9BCOrCWKA==}
engines: {node: '>=18.17'}
undici@7.22.0:
resolution: {integrity: sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==}
engines: {node: '>=20.18.1'}
......@@ -24983,6 +25002,14 @@ snapshots:
- react-native-b4a
- supports-color
'@vercel/blob@2.3.2':
dependencies:
async-retry: 1.3.3
is-buffer: 2.0.5
is-node-process: 1.2.0
throttleit: 2.1.0
undici: 6.24.1
'@vercel/kv@3.0.0':
dependencies:
'@upstash/redis': 1.35.3
......@@ -30999,6 +31026,8 @@ snapshots:
is-buffer@2.0.4: {}
is-buffer@2.0.5: {}
is-builtin-module@3.2.1:
dependencies:
builtin-modules: 3.3.0
......@@ -38543,6 +38572,8 @@ snapshots:
third-party-capital@1.0.20: {}
throttleit@2.1.0: {}
through2@0.4.2:
dependencies:
readable-stream: 1.0.34
......@@ -38914,6 +38945,8 @@ snapshots:
dependencies:
'@fastify/busboy': 2.0.0
undici@6.24.1: {}
undici@7.22.0: {}
unfetch@4.2.0: {}
import { ReactNode } from 'react'
export default function Root({ children }: { children: ReactNode }) {
return (
<html>
<body>{children}</body>
</html>
)
}
export default function Page() {
return <p>hello world</p>
}
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {}
module.exports = nextConfig
import { nextTestSetup, isNextStart } from 'e2e-utils'
import { runNextCommand } from 'next-test-utils'
import http from 'http'
import path from 'path'
import fs from 'fs'
describe('upload-trace', () => {
if (!isNextStart) {
it('skipped for non-start mode', () => {})
return
}
const { next, isTurbopack, skipped } = nextTestSetup({
files: __dirname,
skipStart: true,
skipDeployment: true,
buildCommand: 'pnpm next build --experimental-cpu-prof --internal-trace',
})
if (skipped) return
it('should upload profiles and trace to the mock endpoint after build', async () => {
const buildResult = await next.build()
expect(buildResult.exitCode).toBe(0)
const profilesDir = path.join(next.testDir, '.next-profiles')
expect(fs.existsSync(profilesDir)).toBe(true)
const allFiles = fs.readdirSync(profilesDir)
const cpuProfiles = allFiles.filter((f: string) =>
f.endsWith('.cpuprofile')
)
expect(cpuProfiles.length).toBeGreaterThan(0)
if (isTurbopack) {
expect(allFiles).toContain('trace-turbopack')
}
const uploadableFiles = allFiles.filter(
(f: string) => f.endsWith('.cpuprofile') || f === 'trace-turbopack'
)
const expectedUploadCount = uploadableFiles.length
const handshakeRequests: any[] = []
const mockServer = http.createServer(async (req, res) => {
const chunks: Buffer[] = []
for await (const chunk of req) {
chunks.push(chunk as Buffer)
}
const body = Buffer.concat(chunks).toString('utf-8')
let parsed: any = null
try {
parsed = JSON.parse(body)
} catch {
// Not JSON — binary blob upload
}
// Handle the upload-trace token handshake (sends { filename })
if (parsed && parsed.filename) {
handshakeRequests.push(parsed)
res.writeHead(200, { 'content-type': 'application/json' })
res.end(
JSON.stringify({
clientToken: 'vercel_blob_client_TESTSTOREID_dGVzdC5wYXlsb2Fk',
pathname: `profiles/${parsed.filename}`,
sessionId: 'test-session-id',
sessionToken: 'test-session-token',
})
)
return
}
// Handle @vercel/blob put() — the actual blob upload
res.writeHead(200, { 'content-type': 'application/json' })
res.end(
JSON.stringify({
url: 'https://test.blob.vercel-storage.com/profiles/test',
downloadUrl:
'https://test.blob.vercel-storage.com/profiles/test?download=1',
pathname: 'profiles/test',
contentType: 'application/octet-stream',
contentDisposition: 'attachment; filename="test"',
})
)
})
await new Promise<void>((resolve) => {
mockServer.listen(0, '127.0.0.1', resolve)
})
const address = mockServer.address() as { port: number }
const mockUrl = `http://127.0.0.1:${address.port}`
try {
const result = await runNextCommand(['internal', 'upload-trace'], {
cwd: next.testDir,
stderr: true,
stdout: true,
env: {
__NEXT_UPLOAD_TRACE_URL_OVERRIDE: mockUrl,
VERCEL_BLOB_API_URL: mockUrl,
},
})
if (result.code !== 0) {
console.log('upload-trace stdout:', result.stdout)
console.log('upload-trace stderr:', result.stderr)
}
expect(result.code).toBe(0)
expect(handshakeRequests.length).toBe(expectedUploadCount)
for (const req of handshakeRequests) {
expect(req.filename).toBeTruthy()
}
expect(result.stdout + result.stderr).toContain(
'All files uploaded successfully'
)
} finally {
mockServer.close()
}
})
it('should fail gracefully when no profiles directory exists', async () => {
const emptyDir = path.join(next.testDir, 'empty-project')
fs.mkdirSync(emptyDir, { recursive: true })
const result = await runNextCommand(
['internal', 'upload-trace', emptyDir],
{
cwd: next.testDir,
stderr: true,
stdout: true,
env: {
__NEXT_UPLOAD_TRACE_URL_OVERRIDE: 'http://127.0.0.1:1',
},
}
)
expect(result.code).toBe(1)
expect(result.stdout + result.stderr).toContain(
'Profiles directory not found'
)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment