Unverified Commit 04de9f4c authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbopack-node: extract PacketReader and add unit tests

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 22ae8a3b
/* eslint-env jest */
import { PacketReader } from '../../turbopack/crates/turbopack-node/js/src/child_process/packet-reader'
function frame(payload: string): Buffer {
const body = Buffer.from(payload, 'utf8')
const header = Buffer.alloc(4)
header.writeUInt32BE(body.length, 0)
return Buffer.concat([header, body])
}
const messages: unknown[] = [
{ a: 1 },
'é'.repeat(1000),
[1, 2, 3],
{ big: 'x'.repeat(70000) },
null,
'z',
]
const stream = Buffer.concat(messages.map((m) => frame(JSON.stringify(m))))
function splitEvery(buffer: Buffer, size: number): Buffer[] {
const chunks: Buffer[] = []
for (let i = 0; i < buffer.length; i += size) {
chunks.push(buffer.subarray(i, i + size))
}
return chunks
}
function splitRandomly(buffer: Buffer, seed: number): Buffer[] {
const chunks: Buffer[] = []
for (let i = 0; i < buffer.length; ) {
seed = (seed * 1103515245 + 12345) & 0x7fffffff
const size = 1 + (seed % 9000)
chunks.push(buffer.subarray(i, i + size))
i += size
}
return chunks
}
async function receiveAll(
reader: PacketReader<unknown>,
count: number
): Promise<unknown[]> {
const received: unknown[] = []
for (let i = 0; i < count; i++) {
received.push(await reader.recv())
}
return received
}
describe('turbopack-node PacketReader', () => {
it.each([
['in one chunk', () => [stream]],
['byte by byte', () => splitEvery(stream, 1)],
['in 3 byte chunks', () => splitEvery(stream, 3)],
['in random chunks', () => splitRandomly(stream, 42)],
])('decodes frames delivered %s', async (_name, split) => {
const reader = new PacketReader<unknown>()
for (const chunk of split()) {
reader.push(chunk)
}
expect(await receiveAll(reader, messages.length)).toEqual(messages)
})
it('handles a chunk that ends one frame and starts the next header', async () => {
const first = frame('"first"')
const second = frame('"second"')
const reader = new PacketReader<unknown>()
reader.push(first.subarray(0, 5))
reader.push(Buffer.concat([first.subarray(5), second.subarray(0, 2)]))
reader.push(second.subarray(2))
expect(await receiveAll(reader, 2)).toEqual(['first', 'second'])
})
it('resolves recv() calls that were pending before data arrived in order', async () => {
const reader = new PacketReader<unknown>()
const pending = messages.map(() => reader.recv())
for (const chunk of splitRandomly(stream, 7)) {
reader.push(chunk)
}
expect(await Promise.all(pending)).toEqual(messages)
})
it('keeps framing after a queued zero-length frame', async () => {
const reader = new PacketReader<unknown>()
reader.push(Buffer.concat([frame(''), frame('"after"')]))
await expect(reader.recv()).rejects.toThrow(SyntaxError)
expect(await reader.recv()).toBe('after')
})
it('throws from push() when a zero-length frame completes a pending recv()', () => {
const reader = new PacketReader<unknown>()
void reader.recv()
expect(() => reader.push(frame(''))).toThrow(SyntaxError)
})
it('does not slice chunks that are consumed whole', async () => {
const payload = Buffer.from('{"exact":true}', 'utf8')
const header = Buffer.alloc(4)
header.writeUInt32BE(payload.length, 0)
const reader = new PacketReader<unknown>()
const subarray = jest.spyOn(Buffer.prototype, 'subarray')
try {
reader.push(header)
reader.push(payload)
expect(subarray).not.toHaveBeenCalled()
} finally {
subarray.mockRestore()
}
// The payload chunk itself (not a copy or view of it) is what gets decoded.
const toString = jest.spyOn(payload, 'toString')
try {
expect(await reader.recv()).toEqual({ exact: true })
expect(toString).toHaveBeenCalledWith('utf8')
} finally {
toString.mockRestore()
}
})
it('copies each byte of a fragmented frame at most once', async () => {
const size = 4 * 1024 * 1024
const payload = Buffer.alloc(size, 0x78)
payload[0] = 0x22
payload[size - 1] = 0x22
const header = Buffer.alloc(4)
header.writeUInt32BE(size, 0)
const chunks = splitEvery(Buffer.concat([header, payload]), 64 * 1024)
let copied = 0
const originalCopy = Buffer.prototype.copy
const originalConcat = Buffer.concat
const copy = jest
.spyOn(Buffer.prototype, 'copy')
.mockImplementation(function (this: Buffer, ...args) {
const count = originalCopy.apply(this, args)
copied += count
return count
})
const concat = jest
.spyOn(Buffer, 'concat')
.mockImplementation((...args) => {
const result = originalConcat(...args)
copied += result.length
return result
})
const reader = new PacketReader<string>()
try {
for (const chunk of chunks) {
reader.push(chunk)
}
} finally {
copy.mockRestore()
concat.mockRestore()
}
expect(copied).toBeLessThanOrEqual(size + 4)
expect((await reader.recv()).length).toBe(size - 2)
})
})
import { createConnection } from 'node:net' import { createConnection } from 'node:net'
import { Writable } from 'node:stream' import { Writable } from 'node:stream'
import { structuredError } from '../error' import { structuredError } from '../error'
import { PacketReader } from './packet-reader'
type State =
| {
type: 'waiting'
}
| {
type: 'packet'
length: number
}
export type Ipc<TIncoming, TOutgoing> = { export type Ipc<TIncoming, TOutgoing> = {
recv(): Promise<TIncoming> recv(): Promise<TIncoming>
...@@ -45,95 +37,12 @@ function createIpc<TIncoming, TOutgoing>( ...@@ -45,95 +37,12 @@ function createIpc<TIncoming, TOutgoing>(
}, },
}) })
const packetQueue: Buffer[] = [] const reader = new PacketReader<TIncoming>()
const recvPromiseResolveQueue: Array<(message: TIncoming) => void> = []
function pushPacket(packet: Buffer) {
const recvPromiseResolve = recvPromiseResolveQueue.shift()
if (recvPromiseResolve != null) {
recvPromiseResolve(JSON.parse(packet.toString('utf8')) as TIncoming)
} else {
packetQueue.push(packet)
}
}
let state: State = { type: 'waiting' }
// Received chunks that have not been consumed yet. We keep them as a list
// (instead of concatenating on every `data` event) so that assembling a
// packet that arrives in many chunks copies each byte only once.
const chunks: Buffer[] = []
let bufferedLength = 0
/**
* Removes the first `length` bytes from `chunks` and returns them.
* The caller must ensure that `bufferedLength >= length`.
*/
function takeBytes(length: number): Buffer {
if (length === 0) {
return Buffer.alloc(0)
}
bufferedLength -= length
const first = chunks[0]
if (first.length >= length) {
// Fast path: the bytes are contained in the first chunk, no copy needed.
if (first.length === length) {
chunks.shift()
} else {
chunks[0] = first.subarray(length)
}
return first.subarray(0, length)
}
const result = Buffer.allocUnsafe(length)
let offset = 0
let consumed = 0
while (offset < length) {
const chunk = chunks[consumed]
const remaining = length - offset
if (chunk.length <= remaining) {
chunk.copy(result, offset)
offset += chunk.length
consumed++
} else {
chunk.copy(result, offset, 0, remaining)
chunks[consumed] = chunk.subarray(remaining)
offset += remaining
}
}
chunks.splice(0, consumed)
return result
}
socket.once('connect', () => { socket.once('connect', () => {
socket.setNoDelay(true) socket.setNoDelay(true)
socket.on('data', (chunk) => { socket.on('data', (chunk) => {
chunks.push(chunk) reader.push(chunk)
bufferedLength += chunk.length
loop: while (true) {
switch (state.type) {
case 'waiting': {
if (bufferedLength >= 4) {
const length = takeBytes(4).readUInt32BE(0)
state = { type: 'packet', length }
} else {
break loop
}
break
}
case 'packet': {
if (bufferedLength >= state.length) {
const packet = takeBytes(state.length)
state = { type: 'waiting' }
pushPacket(packet)
} else {
break loop
}
break
}
default:
invariant(state, (state) => `Unknown state type: ${state?.type}`)
}
}
}) })
}) })
// When the socket is closed, this process is no longer needed. // When the socket is closed, this process is no longer needed.
...@@ -170,19 +79,8 @@ function createIpc<TIncoming, TOutgoing>( ...@@ -170,19 +79,8 @@ function createIpc<TIncoming, TOutgoing>(
} }
return { return {
async recv() { recv() {
const packet = packetQueue.shift() return reader.recv()
if (packet != null) {
return JSON.parse(packet.toString('utf8')) as TIncoming
}
const result = await new Promise<TIncoming>((resolve) => {
recvPromiseResolveQueue.push((result) => {
resolve(result)
})
})
return result
}, },
send(message: TOutgoing) { send(message: TOutgoing) {
...@@ -256,10 +154,3 @@ improveConsole('timeEnd', 'stdout', true) ...@@ -256,10 +154,3 @@ improveConsole('timeEnd', 'stdout', true)
improveConsole('timeLog', 'stdout', true) improveConsole('timeLog', 'stdout', true)
improveConsole('timeStamp', 'stdout', true) improveConsole('timeStamp', 'stdout', true)
improveConsole('assert', 'stderr', true) improveConsole('assert', 'stderr', true)
/**
* Utility function to ensure all variants of an enum are handled.
*/
function invariant(never: never, computeMessage: (arg: any) => string): never {
throw new Error(`Invariant: ${computeMessage(never)}`)
}
type State =
| {
type: 'waiting'
}
| {
type: 'packet'
length: number
}
/**
* Decodes length-prefixed packets (a u32 big-endian byte length followed by a
* UTF-8 JSON payload) from a stream of chunks, and hands the decoded messages
* to `recv()` callers in order.
*/
export class PacketReader<TIncoming> {
private state: State = { type: 'waiting' }
// Received chunks that have not been consumed yet. We keep them as a list
// (instead of concatenating on every chunk) so that assembling a packet that
// arrives in many chunks copies each byte only once.
private readonly chunks: Buffer[] = []
private bufferedLength = 0
// Complete packets that no `recv()` call has consumed yet.
private readonly packetQueue: Buffer[] = []
// `recv()` calls waiting for the next packet.
private readonly recvPromiseResolveQueue: Array<
(message: TIncoming) => void
> = []
/**
* Feeds a chunk of received bytes into the reader. Each packet it completes
* is decoded for the oldest waiting `recv()` call, or queued until the next
* `recv()` call if none is waiting.
*/
push(chunk: Buffer): void {
this.chunks.push(chunk)
this.bufferedLength += chunk.length
loop: while (true) {
switch (this.state.type) {
case 'waiting': {
if (this.bufferedLength >= 4) {
const length = this.takeBytes(4).readUInt32BE(0)
this.state = { type: 'packet', length }
} else {
break loop
}
break
}
case 'packet': {
if (this.bufferedLength >= this.state.length) {
const packet = this.takeBytes(this.state.length)
this.state = { type: 'waiting' }
this.pushPacket(packet)
} else {
break loop
}
break
}
default:
invariant(this.state, (state) => `Unknown state type: ${state?.type}`)
}
}
}
/**
* Resolves with the next decoded message.
*/
async recv(): Promise<TIncoming> {
const packet = this.packetQueue.shift()
if (packet != null) {
return JSON.parse(packet.toString('utf8')) as TIncoming
}
return new Promise<TIncoming>((resolve) => {
this.recvPromiseResolveQueue.push(resolve)
})
}
private pushPacket(packet: Buffer): void {
const recvPromiseResolve = this.recvPromiseResolveQueue.shift()
if (recvPromiseResolve != null) {
recvPromiseResolve(JSON.parse(packet.toString('utf8')) as TIncoming)
} else {
this.packetQueue.push(packet)
}
}
/**
* Removes the first `length` bytes from `chunks` and returns them.
* The caller must ensure that `bufferedLength >= length`.
*/
private takeBytes(length: number): Buffer {
if (length === 0) {
return Buffer.alloc(0)
}
this.bufferedLength -= length
const chunks = this.chunks
const first = chunks[0]
if (first.length === length) {
// Fast path: the first chunk is exactly the requested bytes.
chunks.shift()
return first
}
if (first.length > length) {
// The bytes are contained in the first chunk, no copy needed.
chunks[0] = first.subarray(length)
return first.subarray(0, length)
}
const result = Buffer.allocUnsafe(length)
let offset = 0
let consumed = 0
while (offset < length) {
const chunk = chunks[consumed]
const remaining = length - offset
if (chunk.length <= remaining) {
chunk.copy(result, offset)
offset += chunk.length
consumed++
} else {
chunk.copy(result, offset, 0, remaining)
chunks[consumed] = chunk.subarray(remaining)
offset += remaining
}
}
chunks.splice(0, consumed)
return result
}
}
/**
* Utility function to ensure all variants of an enum are handled.
*/
function invariant(never: never, computeMessage: (arg: any) => string): never {
throw new Error(`Invariant: ${computeMessage(never)}`)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment