Unverified Commit 1c87d7dd authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

turbopack-node: decode IPC payloads to strings while reading

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 04de9f4c
...@@ -94,30 +94,104 @@ describe('turbopack-node PacketReader', () => { ...@@ -94,30 +94,104 @@ describe('turbopack-node PacketReader', () => {
expect(() => reader.push(frame(''))).toThrow(SyntaxError) expect(() => reader.push(frame(''))).toThrow(SyntaxError)
}) })
it('does not slice chunks that are consumed whole', async () => { it('decodes chunks that are consumed whole without slicing them', async () => {
const payload = Buffer.from('{"exact":true}', 'utf8') const payload = Buffer.from('{"exact":true}', 'utf8')
const header = Buffer.alloc(4) const header = Buffer.alloc(4)
header.writeUInt32BE(payload.length, 0) header.writeUInt32BE(payload.length, 0)
const reader = new PacketReader<unknown>() const reader = new PacketReader<unknown>()
const subarray = jest.spyOn(Buffer.prototype, 'subarray') const subarray = jest.spyOn(Buffer.prototype, 'subarray')
// The payload chunk itself (not a copy or view of it) is what gets decoded.
const toString = jest.spyOn(payload, 'toString')
try { try {
reader.push(header) reader.push(header)
reader.push(payload) reader.push(payload)
expect(subarray).not.toHaveBeenCalled() expect(subarray).not.toHaveBeenCalled()
expect(toString.mock.calls).toEqual([['utf8', 0, payload.length]])
} finally { } finally {
subarray.mockRestore() subarray.mockRestore()
}
// The payload chunk itself (not a copy or view of it) is what gets decoded.
const toString = jest.spyOn(payload, 'toString')
try {
expect(await reader.recv()).toEqual({ exact: true })
expect(toString).toHaveBeenCalledWith('utf8')
} finally {
toString.mockRestore() toString.mockRestore()
} }
expect(await reader.recv()).toEqual({ exact: true })
})
describe('matches Buffer#toString decoding for every split point', () => {
// Captures the decoded strings by bypassing JSON.parse, so invalid JSON
// (e.g. a leading byte order mark) can be compared too. All payloads are
// sent as consecutive frames, so decoder state must not leak between them.
async function check(...payloads: Buffer[]) {
const stream = Buffer.concat(
payloads.map((payload) => {
const header = Buffer.alloc(4)
header.writeUInt32BE(payload.length, 0)
return Buffer.concat([header, payload])
})
)
const expected = payloads.map((payload) => payload.toString('utf8'))
for (let split = 1; split < stream.length; split++) {
const reader = new PacketReader<unknown>()
reader.push(stream.subarray(0, split))
reader.push(stream.subarray(split))
const parse = jest.spyOn(JSON, 'parse').mockImplementation((s) => s)
let decoded: unknown[]
try {
decoded = await receiveAll(reader, payloads.length)
} finally {
parse.mockRestore()
}
expect({ split, decoded }).toEqual({ split, decoded: expected })
}
}
it('multi-byte characters', async () => {
await check(Buffer.from(JSON.stringify('aé€😀bé€😀'), 'utf8'))
})
it('invalid UTF-8 sequences', async () => {
await check(
Buffer.from([
0x22, 0xff, 0x41, 0xe2, 0x82, 0x41, 0xf0, 0x9f, 0x98, 0x41, 0xc0,
0xaf, 0xed, 0xa0, 0x80, 0xf4, 0x90, 0x80, 0x80, 0x22,
])
)
})
it('a leading byte order mark', async () => {
await check(Buffer.from([0xef, 0xbb, 0xbf, 0x22, 0x61, 0x22]))
})
it.each([
['2-byte', [0xc3]],
['3-byte', [0xe2, 0x82]],
['4-byte', [0xf0, 0x9f, 0x98]],
])(
'a truncated %s sequence at the end of a frame',
async (_name, truncated) => {
await check(
Buffer.from([0x22, 0x61, ...truncated]),
Buffer.from('"b"', 'utf8')
)
}
)
})
it('does not retain received chunks in queued packets', async () => {
const first = frame('"first"')
const second = frame('"second"')
const reader = new PacketReader<unknown>()
// One frame decoded from a single chunk, one spanning two chunks.
const chunks = [first, second.subarray(0, 6), second.subarray(6)].map(
(chunk) => Buffer.from(chunk)
)
for (const chunk of chunks) {
reader.push(chunk)
}
for (const chunk of chunks) {
chunk.fill(0x20)
}
expect(await receiveAll(reader, 2)).toEqual(['first', 'second'])
}) })
it('copies each byte of a fragmented frame at most once', async () => { it('does not assemble fragmented payloads into a Buffer', async () => {
const size = 4 * 1024 * 1024 const size = 4 * 1024 * 1024
const payload = Buffer.alloc(size, 0x78) const payload = Buffer.alloc(size, 0x78)
payload[0] = 0x22 payload[0] = 0x22
...@@ -126,33 +200,22 @@ describe('turbopack-node PacketReader', () => { ...@@ -126,33 +200,22 @@ describe('turbopack-node PacketReader', () => {
header.writeUInt32BE(size, 0) header.writeUInt32BE(size, 0)
const chunks = splitEvery(Buffer.concat([header, payload]), 64 * 1024) const chunks = splitEvery(Buffer.concat([header, payload]), 64 * 1024)
let copied = 0 const copy = jest.spyOn(Buffer.prototype, 'copy')
const originalCopy = Buffer.prototype.copy const concat = jest.spyOn(Buffer, 'concat')
const originalConcat = Buffer.concat const allocUnsafe = jest.spyOn(Buffer, 'allocUnsafe')
const copy = jest
.spyOn(Buffer.prototype, 'copy')
.mockImplementation(function (this: Buffer, ...args) {
const count = originalCopy.apply(this, args)
copied += count
return count
})
const concat = jest
.spyOn(Buffer, 'concat')
.mockImplementation((...args) => {
const result = originalConcat(...args)
copied += result.length
return result
})
const reader = new PacketReader<string>() const reader = new PacketReader<string>()
try { try {
for (const chunk of chunks) { for (const chunk of chunks) {
reader.push(chunk) reader.push(chunk)
} }
expect(copy).not.toHaveBeenCalled()
expect(concat).not.toHaveBeenCalled()
expect(allocUnsafe).not.toHaveBeenCalled()
} finally { } finally {
copy.mockRestore() copy.mockRestore()
concat.mockRestore() concat.mockRestore()
allocUnsafe.mockRestore()
} }
expect(copied).toBeLessThanOrEqual(size + 4) expect(await reader.recv()).toBe('x'.repeat(size - 2))
expect((await reader.recv()).length).toBe(size - 2)
}) })
}) })
...@@ -41,9 +41,7 @@ function createIpc<TIncoming, TOutgoing>( ...@@ -41,9 +41,7 @@ function createIpc<TIncoming, TOutgoing>(
socket.once('connect', () => { socket.once('connect', () => {
socket.setNoDelay(true) socket.setNoDelay(true)
socket.on('data', (chunk) => { socket.on('data', (chunk) => reader.push(chunk))
reader.push(chunk)
})
}) })
// When the socket is closed, this process is no longer needed. // When the socket is closed, this process is no longer needed.
// This might happen e. g. when parent process is killed or // This might happen e. g. when parent process is killed or
......
import { TextDecoder } from 'node:util'
type State = type State =
| { | {
type: 'waiting' type: 'waiting'
...@@ -15,12 +17,14 @@ type State = ...@@ -15,12 +17,14 @@ type State =
export class PacketReader<TIncoming> { export class PacketReader<TIncoming> {
private state: State = { type: 'waiting' } private state: State = { type: 'waiting' }
// Received chunks that have not been consumed yet. We keep them as a list // Received chunks that have not been consumed yet. We keep them as a list
// (instead of concatenating on every chunk) so that assembling a packet that // (instead of concatenating on every chunk) so that a packet that arrives in
// arrives in many chunks copies each byte only once. // many chunks can be decoded straight from them.
private readonly chunks: Buffer[] = [] private readonly chunks: Buffer[] = []
private bufferedLength = 0 private bufferedLength = 0
// Complete packets that no `recv()` call has consumed yet. // Decoded packets that no `recv()` call has consumed yet. Packets are
private readonly packetQueue: Buffer[] = [] // decoded as soon as they are complete so that queued packets don't retain
// the received chunks (which may be views into larger Buffers).
private readonly packetQueue: string[] = []
// `recv()` calls waiting for the next packet. // `recv()` calls waiting for the next packet.
private readonly recvPromiseResolveQueue: Array< private readonly recvPromiseResolveQueue: Array<
(message: TIncoming) => void (message: TIncoming) => void
...@@ -48,7 +52,7 @@ export class PacketReader<TIncoming> { ...@@ -48,7 +52,7 @@ export class PacketReader<TIncoming> {
} }
case 'packet': { case 'packet': {
if (this.bufferedLength >= this.state.length) { if (this.bufferedLength >= this.state.length) {
const packet = this.takeBytes(this.state.length) const packet = this.takeString(this.state.length)
this.state = { type: 'waiting' } this.state = { type: 'waiting' }
this.pushPacket(packet) this.pushPacket(packet)
} else { } else {
...@@ -68,7 +72,7 @@ export class PacketReader<TIncoming> { ...@@ -68,7 +72,7 @@ export class PacketReader<TIncoming> {
async recv(): Promise<TIncoming> { async recv(): Promise<TIncoming> {
const packet = this.packetQueue.shift() const packet = this.packetQueue.shift()
if (packet != null) { if (packet != null) {
return JSON.parse(packet.toString('utf8')) as TIncoming return JSON.parse(packet) as TIncoming
} }
return new Promise<TIncoming>((resolve) => { return new Promise<TIncoming>((resolve) => {
...@@ -76,15 +80,63 @@ export class PacketReader<TIncoming> { ...@@ -76,15 +80,63 @@ export class PacketReader<TIncoming> {
}) })
} }
private pushPacket(packet: Buffer): void { private pushPacket(packet: string): void {
const recvPromiseResolve = this.recvPromiseResolveQueue.shift() const recvPromiseResolve = this.recvPromiseResolveQueue.shift()
if (recvPromiseResolve != null) { if (recvPromiseResolve != null) {
recvPromiseResolve(JSON.parse(packet.toString('utf8')) as TIncoming) recvPromiseResolve(JSON.parse(packet) as TIncoming)
} else { } else {
this.packetQueue.push(packet) this.packetQueue.push(packet)
} }
} }
/**
* Removes the first `length` bytes from `chunks` and returns them decoded as
* UTF-8, without first assembling them into a single Buffer.
* The caller must ensure that `bufferedLength >= length`.
*/
private takeString(length: number): string {
if (length === 0) {
return ''
}
this.bufferedLength -= length
const chunks = this.chunks
const first = chunks[0]
if (first.length >= length) {
// Fast path: the bytes are contained in the first chunk, decode them in
// place.
const result = first.toString('utf8', 0, length)
if (first.length === length) {
chunks.shift()
} else {
chunks[0] = first.subarray(length)
}
return result
}
// The bytes span multiple chunks. A streaming decoder carries multi-byte
// characters that are split across chunk boundaries. `ignoreBOM` keeps a
// leading byte order mark, matching `Buffer#toString`.
const decoder = new TextDecoder('utf-8', { ignoreBOM: true })
let result = ''
let remaining = length
let consumed = 0
while (remaining > 0) {
const chunk = chunks[consumed]
if (chunk.length <= remaining) {
result += decoder.decode(chunk, { stream: true })
remaining -= chunk.length
consumed++
} else {
result += decoder.decode(chunk.subarray(0, remaining), {
stream: true,
})
chunks[consumed] = chunk.subarray(remaining)
remaining = 0
}
}
chunks.splice(0, consumed)
return result + decoder.decode()
}
/** /**
* Removes the first `length` bytes from `chunks` and returns them. * Removes the first `length` bytes from `chunks` and returns them.
* The caller must ensure that `bufferedLength >= length`. * The caller must ensure that `bufferedLength >= length`.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment