Unverified Commit b7b8340b authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

upload-trace: accept gzip and zstd compressed Turbopack traces

`next internal upload-trace` only accepted raw traces. gzip and zstd compressed traces are now validated by decompressing the start of the file and checking the TRACEv1 header, and uploaded unchanged. On Node.js versions without zstd support, zstd files are accepted by their magic bytes.
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 422c890f
import zlib from 'zlib'
import { checkTurbopackTrace } from './upload-trace'
const FILE = 'trace-turbopack.bin'
const NOT_A_TRACE = `Error: ${FILE} does not appear to be a valid Turbopack trace (missing TRACEv1 header).`
const unsupportedVersion = (found: string) =>
`Error: ${FILE} has an unsupported Turbopack trace version (expected TRACEv1, found ${found}). Capture the trace again with this version of Next.js.`
/** A trace with the given header followed by compressible (but not trivial) row data. */
function trace(header: string, size = 1024 * 1024): Buffer {
const rows = Buffer.alloc(size)
for (let i = 0; i < size; i++) {
rows[i] = (i * 7919) % 251 > 200 ? i % 13 : 0
}
return Buffer.concat([Buffer.from(header), rows])
}
const zstd = zlib as {
zstdCompressSync?: (data: Buffer) => Buffer
createZstdDecompress?: () => import('stream').Transform
}
describe('checkTurbopackTrace', () => {
it('checks the header of raw traces', async () => {
expect(await checkTurbopackTrace(trace('TRACEv1', 100), FILE)).toEqual({
valid: true,
})
expect(await checkTurbopackTrace(trace('TRACEv0', 100), FILE)).toEqual({
valid: false,
error: unsupportedVersion('TRACEv0'),
})
expect(await checkTurbopackTrace(Buffer.from('hello world'), FILE)).toEqual(
{ valid: false, error: NOT_A_TRACE }
)
// A header that ends before the version
expect(await checkTurbopackTrace(Buffer.from('TRACEv'), FILE)).toEqual({
valid: false,
error: NOT_A_TRACE,
})
})
it('checks the header of gzip compressed traces', async () => {
const v1 = zlib.gzipSync(trace('TRACEv1'))
expect(await checkTurbopackTrace(v1, FILE)).toEqual({ valid: true })
// Only the start of the file is passed in
expect(await checkTurbopackTrace(v1.subarray(0, 4096), FILE)).toEqual({
valid: true,
})
expect(
await checkTurbopackTrace(zlib.gzipSync(trace('TRACEv0')), FILE)
).toEqual({ valid: false, error: unsupportedVersion('TRACEv0') })
expect(
await checkTurbopackTrace(zlib.gzipSync(Buffer.from('nope')), FILE)
).toEqual({ valid: false, error: NOT_A_TRACE })
})
it('rejects gzip data that ends or is invalid before the header', async () => {
const v1 = zlib.gzipSync(trace('TRACEv1'))
// Only the gzip header, no compressed data
expect(await checkTurbopackTrace(v1.subarray(0, 10), FILE)).toEqual({
valid: false,
error: NOT_A_TRACE,
})
const corrupt = Buffer.concat([
Buffer.from([0x1f, 0x8b]),
Buffer.from('definitely not deflate data'),
])
expect(await checkTurbopackTrace(corrupt, FILE)).toEqual({
valid: false,
error: NOT_A_TRACE,
})
})
it('only decompresses the start of highly compressed data', async () => {
// 64 MB of zeros, about 64 KB compressed
const bomb = zlib.gzipSync(Buffer.alloc(64 * 1024 * 1024))
expect(await checkTurbopackTrace(bomb, FILE)).toEqual({
valid: false,
error: NOT_A_TRACE,
})
})
it('accepts zstd compressed traces by their magic bytes when Node.js has no zstd support', async () => {
const data = Buffer.concat([
Buffer.from([0x28, 0xb5, 0x2f, 0xfd]),
Buffer.from('anything'),
])
expect(await checkTurbopackTrace(data, FILE, null)).toEqual({
valid: true,
warning: `Warning: Can't check the trace version of ${FILE}, since this version of Node.js doesn't support zstd.`,
})
})
it('checks the header of zstd compressed traces', async () => {
const { zstdCompressSync, createZstdDecompress } = zstd
if (!zstdCompressSync || !createZstdDecompress) {
// This Node.js version has no zstd support, so zstd traces are accepted by their magic
// bytes alone
const data = Buffer.concat([
Buffer.from([0x28, 0xb5, 0x2f, 0xfd]),
Buffer.from('anything'),
])
expect(await checkTurbopackTrace(data, FILE)).toMatchObject({
valid: true,
})
return
}
const v1 = zstdCompressSync(trace('TRACEv1'))
expect(await checkTurbopackTrace(v1, FILE)).toEqual({ valid: true })
expect(
await checkTurbopackTrace(zstdCompressSync(trace('TRACEv0')), FILE)
).toEqual({ valid: false, error: unsupportedVersion('TRACEv0') })
// Ends before the first block is complete
expect(await checkTurbopackTrace(v1.subarray(0, 12), FILE)).toEqual({
valid: false,
error: NOT_A_TRACE,
})
const corrupt = Buffer.concat([
Buffer.from([0x28, 0xb5, 0x2f, 0xfd]),
Buffer.from('definitely not zstd data'),
])
expect(await checkTurbopackTrace(corrupt, FILE)).toEqual({
valid: false,
error: NOT_A_TRACE,
})
})
})
import path from 'path'
import fs from 'fs/promises'
import zlib from 'zlib'
import type { Transform } from 'stream'
const UPLOAD_TRACE_URL = 'https://nextjs.org/api/upload-trace'
......@@ -11,6 +13,20 @@ const CPUPROFILE_HEADER = Buffer.from('{"nodes":')
const TURBOPACK_TRACE_HEADER = Buffer.from('TRACEv1')
const TURBOPACK_TRACE_HEADER_PREFIX = Buffer.from('TRACEv')
// Turbopack traces can be compressed with gzip or zstd (`NEXT_TURBOPACK_TRACING=1,gz` or
// `NEXT_TURBOPACK_TRACING=1,zstd`). They are uploaded as they are, but the trace header is
// checked after decompressing the start of the file.
const GZIP_MAGIC = Buffer.from([0x1f, 0x8b])
const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
/**
* How much of a Turbopack trace is read to check its header. For compressed traces this needs to
* cover the first compressed block (up to 128 KB of uncompressed data for zstd).
*/
const TRACE_HEADER_CHECK_READ_SIZE = 512 * 1024
/** How much of a CPU profile is read to check its header. */
const CPUPROFILE_HEADER_CHECK_READ_SIZE = 16
const PROGRESS_CHUNK_SIZE = 64 * 1024 // 64 KB
export interface UploadTraceOptions {
......@@ -67,25 +83,129 @@ function validateCpuProfile(header: Buffer, file: string): void {
}
}
function validateTurbopackTrace(header: Buffer, file: string): void {
/** Creates a zstd decompression stream. */
export type CreateZstdDecompress = () => Transform
/**
* Returns Node.js's zstd decompression, or `null` when this Node.js version doesn't support zstd
* (it was added in Node.js 22.15 and 23.8).
*/
function getCreateZstdDecompress(): CreateZstdDecompress | null {
const { createZstdDecompress } = zlib as {
createZstdDecompress?: CreateZstdDecompress
}
return typeof createZstdDecompress === 'function'
? () => createZstdDecompress()
: null
}
/**
* Decompresses `data` with `decompress` until at least `length` bytes are available and returns
* them. Returns fewer bytes when the data ends or is invalid before that. Only as much is
* decompressed as needed, so a small, highly compressed input doesn't use a lot of memory.
*/
function decompressStart(
decompress: Transform,
data: Buffer,
length: number
): Promise<Buffer> {
return new Promise((resolve) => {
const chunks: Buffer[] = []
let size = 0
let done = false
const finish = () => {
if (done) return
done = true
decompress.destroy()
resolve(Buffer.concat(chunks).subarray(0, length))
}
decompress.on('data', (chunk: Buffer) => {
chunks.push(chunk)
size += chunk.length
if (size >= length) finish()
})
// `data` is usually only the start of the file, so the stream ends with an error
decompress.on('error', finish)
decompress.on('end', finish)
decompress.end(data)
})
}
export type TurbopackTraceCheck =
| { valid: true; warning?: string }
| { valid: false; error: string }
/**
* Checks that a Turbopack trace file has the supported `TRACEv1` header. `start` is the start
* of the file (`TRACE_HEADER_CHECK_READ_SIZE` bytes, or the whole file if it is smaller). gzip
* and zstd compressed traces are decompressed to check the header. When Node.js doesn't support
* zstd (`createZstdDecompress` is `null`), zstd compressed files are accepted without checking
* the header.
*/
export async function checkTurbopackTrace(
start: Buffer,
file: string,
createZstdDecompress: CreateZstdDecompress | null = getCreateZstdDecompress()
): Promise<TurbopackTraceCheck> {
let header = start
if (startsWith(start, GZIP_MAGIC)) {
header = await decompressStart(
zlib.createGunzip(),
start,
TURBOPACK_TRACE_HEADER.length
)
} else if (startsWith(start, ZSTD_MAGIC)) {
if (!createZstdDecompress) {
return {
valid: true,
warning: `Warning: Can't check the trace version of ${file}, since this version of Node.js doesn't support zstd.`,
}
}
header = await decompressStart(
createZstdDecompress(),
start,
TURBOPACK_TRACE_HEADER.length
)
}
const actual = header.subarray(0, TURBOPACK_TRACE_HEADER.length)
if (actual.equals(TURBOPACK_TRACE_HEADER)) {
return
return { valid: true }
}
if (
actual
.subarray(0, TURBOPACK_TRACE_HEADER_PREFIX.length)
.equals(TURBOPACK_TRACE_HEADER_PREFIX)
actual.length === TURBOPACK_TRACE_HEADER.length &&
startsWith(actual, TURBOPACK_TRACE_HEADER_PREFIX)
) {
console.error(
`Error: ${file} has an unsupported Turbopack trace version (expected ${TURBOPACK_TRACE_HEADER}, found ${actual}). Capture the trace again with this version of Next.js.`
)
} else {
console.error(
`Error: ${file} does not appear to be a valid Turbopack trace (missing ${TURBOPACK_TRACE_HEADER} header).`
)
return {
valid: false,
error: `Error: ${file} has an unsupported Turbopack trace version (expected ${TURBOPACK_TRACE_HEADER}, found ${actual}). Capture the trace again with this version of Next.js.`,
}
}
return {
valid: false,
error: `Error: ${file} does not appear to be a valid Turbopack trace (missing ${TURBOPACK_TRACE_HEADER} header).`,
}
}
function startsWith(data: Buffer, prefix: Buffer): boolean {
return (
data.length >= prefix.length &&
data.subarray(0, prefix.length).equals(prefix)
)
}
async function validateTurbopackTrace(
start: Buffer,
file: string
): Promise<void> {
const result = await checkTurbopackTrace(start, file)
if (!result.valid) {
console.error(result.error)
process.exit(1)
}
if (result.warning) {
console.warn(result.warning)
}
process.exit(1)
}
export async function uploadTraceToBlob(
......@@ -138,14 +258,21 @@ export async function uploadTraceToBlob(
}
const fd = await fs.open(filePath, 'r')
const headerBuf = Buffer.alloc(16)
await fd.read(headerBuf, 0, 16, 0)
await fd.close()
const readSize = file.endsWith('.cpuprofile')
? CPUPROFILE_HEADER_CHECK_READ_SIZE
: TRACE_HEADER_CHECK_READ_SIZE
let headerBuf = Buffer.alloc(Math.min(stat.size, readSize))
try {
const { bytesRead } = await fd.read(headerBuf, 0, headerBuf.length, 0)
headerBuf = headerBuf.subarray(0, bytesRead)
} finally {
await fd.close()
}
if (file.endsWith('.cpuprofile')) {
validateCpuProfile(headerBuf, file)
} else if (file.endsWith('trace-turbopack.bin')) {
validateTurbopackTrace(headerBuf, file)
await validateTurbopackTrace(headerBuf, file)
}
const content = await fs.readFile(filePath)
......
......@@ -2,6 +2,87 @@ import { nextTestSetup, isNextStart } from 'e2e-utils'
import http from 'http'
import path from 'path'
import fs from 'fs'
import zlib from 'zlib'
/**
* Starts a server that mocks the upload-trace token endpoint and the blob store. Records the
* token requests and the uploaded blobs.
*/
async function startMockServer() {
const handshakeRequests: any[] = []
const uploads: Buffer[] = []
const mockServer = http.createServer(async (req, res) => {
const chunks: Buffer[] = []
for await (const chunk of req) {
chunks.push(chunk as Buffer)
}
const raw = Buffer.concat(chunks)
const body = raw.toString('utf-8')
let parsed: any = null
try {
parsed = JSON.parse(body)
} catch {
// Not JSON — binary blob upload
}
// Handle the upload-trace token handshake (sends { filename })
if (parsed && parsed.filename) {
handshakeRequests.push(parsed)
res.writeHead(200, { 'content-type': 'application/json' })
res.end(
JSON.stringify({
clientToken: 'vercel_blob_client_TESTSTOREID_dGVzdC5wYXlsb2Fk',
pathname: `profiles/${parsed.filename}`,
sessionId: 'test-session-id',
sessionToken: 'test-session-token',
})
)
return
}
// Handle @vercel/blob put() — the actual blob upload
uploads.push(raw)
res.writeHead(200, { 'content-type': 'application/json' })
res.end(
JSON.stringify({
url: 'https://test.blob.vercel-storage.com/profiles/test',
downloadUrl:
'https://test.blob.vercel-storage.com/profiles/test?download=1',
pathname: 'profiles/test',
contentType: 'application/octet-stream',
contentDisposition: 'attachment; filename="test"',
})
)
})
await new Promise<void>((resolve) => {
mockServer.listen(0, '127.0.0.1', resolve)
})
const address = mockServer.address() as { port: number }
return {
url: `http://127.0.0.1:${address.port}`,
handshakeRequests,
uploads,
close: () => mockServer.close(),
}
}
/** A Turbopack trace with the given header, followed by some row data. */
function trace(header: string): Buffer {
const rows = Buffer.alloc(256 * 1024)
for (let i = 0; i < rows.length; i++) {
rows[i] = (i * 7919) % 251 > 200 ? i % 13 : 0
}
return Buffer.concat([Buffer.from(header), rows])
}
const { zstdCompressSync } = zlib as {
zstdCompressSync?: (data: Buffer) => Buffer
}
// This suite controls the local build lifecycle directly, which deployment tests cannot reproduce.
// @force-gate !deploy
......@@ -39,64 +120,13 @@ describe('upload-trace', () => {
)
const expectedUploadCount = uploadableFiles.length
const handshakeRequests: any[] = []
const mockServer = http.createServer(async (req, res) => {
const chunks: Buffer[] = []
for await (const chunk of req) {
chunks.push(chunk as Buffer)
}
const body = Buffer.concat(chunks).toString('utf-8')
let parsed: any = null
try {
parsed = JSON.parse(body)
} catch {
// Not JSON — binary blob upload
}
// Handle the upload-trace token handshake (sends { filename })
if (parsed && parsed.filename) {
handshakeRequests.push(parsed)
res.writeHead(200, { 'content-type': 'application/json' })
res.end(
JSON.stringify({
clientToken: 'vercel_blob_client_TESTSTOREID_dGVzdC5wYXlsb2Fk',
pathname: `profiles/${parsed.filename}`,
sessionId: 'test-session-id',
sessionToken: 'test-session-token',
})
)
return
}
// Handle @vercel/blob put() — the actual blob upload
res.writeHead(200, { 'content-type': 'application/json' })
res.end(
JSON.stringify({
url: 'https://test.blob.vercel-storage.com/profiles/test',
downloadUrl:
'https://test.blob.vercel-storage.com/profiles/test?download=1',
pathname: 'profiles/test',
contentType: 'application/octet-stream',
contentDisposition: 'attachment; filename="test"',
})
)
})
await new Promise<void>((resolve) => {
mockServer.listen(0, '127.0.0.1', resolve)
})
const address = mockServer.address() as { port: number }
const mockUrl = `http://127.0.0.1:${address.port}`
const mockServer = await startMockServer()
try {
const result = await next.runCommand(['internal', 'upload-trace'], {
env: {
__NEXT_UPLOAD_TRACE_URL_OVERRIDE: mockUrl,
VERCEL_BLOB_API_URL: mockUrl,
__NEXT_UPLOAD_TRACE_URL_OVERRIDE: mockServer.url,
VERCEL_BLOB_API_URL: mockServer.url,
},
})
......@@ -106,8 +136,8 @@ describe('upload-trace', () => {
}
expect(result.exitCode).toBe(0)
expect(handshakeRequests.length).toBe(expectedUploadCount)
for (const req of handshakeRequests) {
expect(mockServer.handshakeRequests.length).toBe(expectedUploadCount)
for (const req of mockServer.handshakeRequests) {
expect(req.filename).toBeTruthy()
}
expect(result.cliOutput).toContain('All files uploaded successfully')
......@@ -116,6 +146,79 @@ describe('upload-trace', () => {
}
})
/** Creates a project directory with only a Turbopack trace file in `.next-profiles`. */
function createProjectWithTrace(name: string, content: Buffer): string {
const projectDir = path.join(next.testDir, name)
fs.mkdirSync(path.join(projectDir, '.next-profiles'), { recursive: true })
fs.writeFileSync(
path.join(projectDir, '.next-profiles', 'trace-turbopack.bin'),
content
)
return projectDir
}
async function uploadTrace(projectDir: string) {
const mockServer = await startMockServer()
try {
const result = await next.runCommand(
['internal', 'upload-trace', projectDir],
{
env: {
__NEXT_UPLOAD_TRACE_URL_OVERRIDE: mockServer.url,
VERCEL_BLOB_API_URL: mockServer.url,
},
}
)
return { result, mockServer }
} finally {
mockServer.close()
}
}
it('should upload a gzip compressed Turbopack trace unchanged', async () => {
const content = zlib.gzipSync(trace('TRACEv1'))
const { result, mockServer } = await uploadTrace(
createProjectWithTrace('gzip-trace', content)
)
expect(result.exitCode).toBe(0)
expect(mockServer.handshakeRequests.length).toBe(1)
expect(mockServer.uploads.length).toBe(1)
expect(mockServer.uploads[0].equals(content)).toBe(true)
expect(result.cliOutput).toContain('All files uploaded successfully')
})
// zstd is only available in Node.js 22.15+ and 23.8+. Without it, zstd traces are accepted by
// their magic bytes alone, which is covered by the unit tests of `checkTurbopackTrace`.
;(zstdCompressSync ? describe : describe.skip)('with zstd support', () => {
it('should upload a zstd compressed Turbopack trace unchanged', async () => {
const content = zstdCompressSync!(trace('TRACEv1'))
const { result, mockServer } = await uploadTrace(
createProjectWithTrace('zstd-trace', content)
)
expect(result.exitCode).toBe(0)
expect(mockServer.handshakeRequests.length).toBe(1)
expect(mockServer.uploads.length).toBe(1)
expect(mockServer.uploads[0].equals(content)).toBe(true)
expect(result.cliOutput).not.toContain("Can't check the trace version")
expect(result.cliOutput).toContain('All files uploaded successfully')
})
})
it('should reject a compressed Turbopack trace with an unsupported version', async () => {
const { result, mockServer } = await uploadTrace(
createProjectWithTrace('gzip-trace-v0', zlib.gzipSync(trace('TRACEv0')))
)
expect(result.exitCode).toBe(1)
expect(result.cliOutput).toContain(
'has an unsupported Turbopack trace version (expected TRACEv1, found TRACEv0)'
)
expect(mockServer.handshakeRequests.length).toBe(0)
expect(mockServer.uploads.length).toBe(0)
})
it('should fail gracefully when no profiles directory exists', async () => {
const emptyDir = path.join(next.testDir, 'empty-project')
fs.mkdirSync(emptyDir, { recursive: true })
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment