Unverified Commit 42987701 authored by Hendrik Liebau's avatar Hendrik Liebau Committed by GitHub

Reland the debug channel on native Node streams, fixed for React 18 (#94462)

This relands #94433, which was reverted in #94459 after it broke app
rendering in dev on React 18. The first commit reapplies the original
change verbatim, and the second commit fixes the regression that
prompted the revert.

The regression came from `connectReactDebugChannel` reaching into
`app-render/stream-ops.node` for its Node byte batcher transform. That
render-layer module carries unconditional `react-dom/server` and
`react-dom/static` value imports, and the dev server's hot-reloaders run
in the precompiled server runtime where `react-dom/*` is not aliased to
the vendored React. The `require` therefore resolved `react-dom/static`
against the app's installed React, which on React 18 has no `./static`
subpath export and threw `ERR_PACKAGE_PATH_NOT_EXPORTED`.

The fix extracts the batcher into its own React-free module,
`stream-utils/node-buffered-transform-stream`, which depends only on
`node:stream`. Both `stream-ops.node` and the debug channel import it
from there, so the dev server can batch Node stream chunks without
dragging the render layer's React imports into an unaliased runtime. The
debug channel's runtime branch on the stream type is unchanged and the
web path already used a React-free batcher, so no `.node`/`.web` split
is needed.
parent 539efa8c
...@@ -3593,7 +3593,7 @@ async function renderToStream( ...@@ -3593,7 +3593,7 @@ async function renderToStream(
) { ) {
if (process.env.__NEXT_USE_NODE_STREAMS) { if (process.env.__NEXT_USE_NODE_STREAMS) {
// MARK: nodeStreams dev CacheComponents RSC // MARK: nodeStreams dev CacheComponents RSC
let debugChannel: DebugChannelPair | undefined let debugChannelClientStream: ReplayableNodeStream | undefined
// eslint-disable-next-line @typescript-eslint/no-shadow // eslint-disable-next-line @typescript-eslint/no-shadow
const getPayload = async (requestStore: RequestStore) => { const getPayload = async (requestStore: RequestStore) => {
...@@ -3652,11 +3652,11 @@ async function renderToStream( ...@@ -3652,11 +3652,11 @@ async function renderToStream(
let validationDebugChannelClient: AnyStream | undefined = undefined let validationDebugChannelClient: AnyStream | undefined = undefined
if (returnedDebugChannel) { if (returnedDebugChannel) {
const [t1, t2] = teeStream( debugChannelClientStream = new ReplayableNodeStream(
returnedDebugChannel.clientSide.readable returnedDebugChannel.clientSide.readable
) )
returnedDebugChannel.clientSide.readable = t1 validationDebugChannelClient =
validationDebugChannelClient = t2 debugChannelClientStream.createReplayStream()
} }
consoleAsyncStorage.run( consoleAsyncStorage.run(
...@@ -3676,14 +3676,14 @@ async function renderToStream( ...@@ -3676,14 +3676,14 @@ async function renderToStream(
reactServerResult = new ReactServerResult(serverStream) reactServerResult = new ReactServerResult(serverStream)
requestStore = finalRequestStore requestStore = finalRequestStore
debugChannel = returnedDebugChannel
} else { } else {
logValidationSkipped(ctx) logValidationSkipped(ctx)
// We're either bypassing caches or we can't restart the render. // We're either bypassing caches or we can't restart the render.
// Do a dynamic render, but with (basic) environment labels. // Do a dynamic render, but with (basic) environment labels.
debugChannel = setReactDebugChannel && createNodeDebugChannel() const debugChannel =
setReactDebugChannel && createNodeDebugChannel()
const serverStream = await stagedRenderWithoutCachesInDevNode( const serverStream = await stagedRenderWithoutCachesInDevNode(
ctx, ctx,
...@@ -3696,17 +3696,19 @@ async function renderToStream( ...@@ -3696,17 +3696,19 @@ async function renderToStream(
} }
) )
reactServerResult = new ReactServerResult(serverStream) reactServerResult = new ReactServerResult(serverStream)
}
if (debugChannel && setReactDebugChannel) { if (debugChannel) {
const [readableSsr, readableBrowser] = teeStream( debugChannelClientStream = new ReplayableNodeStream(
debugChannel.clientSide.readable debugChannel.clientSide.readable
) )
}
}
reactDebugStream = readableSsr if (debugChannelClientStream && setReactDebugChannel) {
reactDebugStream = debugChannelClientStream.createReplayStream()
setReactDebugChannel( setReactDebugChannel(
{ readable: readableBrowser }, { readable: debugChannelClientStream.createReplayStream() },
htmlRequestId, htmlRequestId,
requestId requestId
) )
......
...@@ -3,17 +3,27 @@ ...@@ -3,17 +3,27 @@
* Loaded by debug-channel-server.ts when __NEXT_USE_NODE_STREAMS is enabled. * Loaded by debug-channel-server.ts when __NEXT_USE_NODE_STREAMS is enabled.
*/ */
import { PassThrough, Writable } from 'node:stream' import { PassThrough, Writable, type Readable } from 'node:stream'
import type { DebugChannelPair } from './debug-channel-server.web' import type { DebugChannelServer } from './debug-channel-server.web'
export { createWebDebugChannel } from './debug-channel-server.web' export { createWebDebugChannel } from './debug-channel-server.web'
/**
* Node variant: identical to `DebugChannelPair` except the client-side readable
* is narrowed to a Node `Readable`, so node call sites can consume it (e.g.
* `new ReplayableNodeStream(...)`) without casting `AnyStream` down.
*/
export type NodeDebugChannelPair = {
serverSide: DebugChannelServer
clientSide: { readable: Readable }
}
/** /**
* Creates a debug channel using Node.js streams. * Creates a debug channel using Node.js streams.
* Use with renderToNodeFlightStream (React's renderToPipeableStream), * Use with renderToNodeFlightStream (React's renderToPipeableStream),
* which expects debugChannel to be a Node.js stream with a .write() method. * which expects debugChannel to be a Node.js stream with a .write() method.
*/ */
export function createNodeDebugChannel(): DebugChannelPair { export function createNodeDebugChannel(): NodeDebugChannelPair {
// The readable side is a PassThrough that the client reads from. The // The readable side is a PassThrough that the client reads from. The
// server-side write target is a separate, write-only Writable that forwards // server-side write target is a separate, write-only Writable that forwards
// into it rather than the PassThrough itself: React's renderToPipeableStream // into it rather than the PassThrough itself: React's renderToPipeableStream
...@@ -23,12 +33,11 @@ export function createNodeDebugChannel(): DebugChannelPair { ...@@ -23,12 +33,11 @@ export function createNodeDebugChannel(): DebugChannelPair {
// The forwarding must use `passthrough.write()` / `passthrough.end()`, not // The forwarding must use `passthrough.write()` / `passthrough.end()`, not
// `passthrough.push()` / `passthrough.push(null)`. A PassThrough is a Duplex; // `passthrough.push()` / `passthrough.push(null)`. A PassThrough is a Duplex;
// pushing `null` ends only its readable half and leaves the writable half // pushing `null` ends only its readable half and leaves the writable half
// open (`writableEnded` stays false). The readable is later consumed via // open (`writableEnded` stays false). If the readable is consumed via
// `Readable.toWeb()` (in `connectReactDebugChannel`, and inside `teeStream`), // `Readable.toWeb()`, that web stream never closes while the PassThrough's
// and `Readable.toWeb()` never closes the resulting web stream while the // writable half is still open — so the debug channel would never close on the
// PassThrough's writable half is still open — so the debug channel never // client. Ending it via `passthrough.end()` closes both halves and the close
// closes on the client. Ending it via `passthrough.end()` closes both halves // propagates.
// and the close propagates.
const passthrough = new PassThrough() const passthrough = new PassThrough()
const writable = new Writable({ const writable = new Writable({
......
...@@ -8,11 +8,13 @@ export type { ...@@ -8,11 +8,13 @@ export type {
DebugChannelPair, DebugChannelPair,
DebugChannelServer, DebugChannelServer,
} from './debug-channel-server.web' } from './debug-channel-server.web'
export type { NodeDebugChannelPair } from './debug-channel-server.node'
import type { DebugChannelPair } from './debug-channel-server.web' import type { DebugChannelPair } from './debug-channel-server.web'
import type { NodeDebugChannelPair } from './debug-channel-server.node'
type DebugChannelMod = { type DebugChannelMod = {
createWebDebugChannel: typeof import('./debug-channel-server.web').createWebDebugChannel createWebDebugChannel: typeof import('./debug-channel-server.web').createWebDebugChannel
createNodeDebugChannel: typeof import('./debug-channel-server.web').createNodeDebugChannel createNodeDebugChannel: typeof import('./debug-channel-server.node').createNodeDebugChannel
} }
let _m: DebugChannelMod let _m: DebugChannelMod
...@@ -32,7 +34,7 @@ export function createWebDebugChannel(): DebugChannelPair | undefined { ...@@ -32,7 +34,7 @@ export function createWebDebugChannel(): DebugChannelPair | undefined {
return _m.createWebDebugChannel() return _m.createWebDebugChannel()
} }
export function createNodeDebugChannel(): DebugChannelPair | undefined { export function createNodeDebugChannel(): NodeDebugChannelPair | undefined {
if (process.env.NODE_ENV === 'production') { if (process.env.NODE_ENV === 'production') {
return undefined return undefined
} }
......
...@@ -53,6 +53,6 @@ export function createWebDebugChannel(): DebugChannelPair { ...@@ -53,6 +53,6 @@ export function createWebDebugChannel(): DebugChannelPair {
* Use with renderToNodeFlightStream (React's renderToPipeableStream), * Use with renderToNodeFlightStream (React's renderToPipeableStream),
* which expects debugChannel to be a Node.js stream with a .write() method. * which expects debugChannel to be a Node.js stream with a .write() method.
*/ */
export function createNodeDebugChannel(): DebugChannelPair { export function createNodeDebugChannel(): never {
throw new Error('not implemented') throw new Error('not implemented')
} }
...@@ -30,13 +30,17 @@ import { ...@@ -30,13 +30,17 @@ import {
} from '../stream-utils/node-web-streams-helper' } from '../stream-utils/node-web-streams-helper'
import { indexOfUint8Array } from '../stream-utils/uint8array-helpers' import { indexOfUint8Array } from '../stream-utils/uint8array-helpers'
import { ENCODED_TAGS } from '../stream-utils/encoded-tags' import { ENCODED_TAGS } from '../stream-utils/encoded-tags'
import { createNodeBufferedTransformStream } from '../stream-utils/node-buffered-transform-stream'
import { MISSING_ROOT_TAGS_ERROR } from '../../shared/lib/errors/constants' import { MISSING_ROOT_TAGS_ERROR } from '../../shared/lib/errors/constants'
import { import {
htmlEscapeAttributeString, htmlEscapeAttributeString,
htmlEscapeJsonString, htmlEscapeJsonString,
} from '../../shared/lib/htmlescape' } from '../../shared/lib/htmlescape'
import { createInlinedDataReadableStream } from './use-flight-response' import { createInlinedDataReadableStream } from './use-flight-response'
import type { AnyStream as AnyStreamType } from './app-render-prerender-utils' import {
ReplayableNodeStream,
type AnyStream as AnyStreamType,
} from './app-render-prerender-utils'
import { DetachedPromise } from '../../lib/detached-promise' import { DetachedPromise } from '../../lib/detached-promise'
import { getTracer } from '../lib/trace/tracer' import { getTracer } from '../lib/trace/tracer'
import { AppRenderSpan } from '../lib/trace/constants' import { AppRenderSpan } from '../lib/trace/constants'
...@@ -116,52 +120,6 @@ function webToReadable( ...@@ -116,52 +120,6 @@ function webToReadable(
return Readable.fromWeb(stream as WebReadableStream) return Readable.fromWeb(stream as WebReadableStream)
} }
// ---------------------------------------------------------------------------
// Buffered transform – Node.js Transform that coalesces chunks written in the
// same microtask into a single Uint8Array before pushing downstream.
// ---------------------------------------------------------------------------
function createBufferedTransformStream(): Transform {
let bufferedChunks: Array<Uint8Array> = []
let bufferByteLength = 0
let flushScheduled = false
function flushBuffered(stream: Transform): void {
if (bufferedChunks.length === 0) return
const merged = new Uint8Array(bufferByteLength)
let copiedBytes = 0
for (let i = 0; i < bufferedChunks.length; i++) {
const bufferedChunk = bufferedChunks[i]
merged.set(bufferedChunk, copiedBytes)
copiedBytes += bufferedChunk.byteLength
}
bufferedChunks.length = 0
bufferByteLength = 0
stream.push(merged)
}
return new Transform({
transform(chunk, _encoding, callback) {
bufferedChunks.push(chunk)
bufferByteLength += chunk.byteLength
if (!flushScheduled) {
flushScheduled = true
queueMicrotask(() => {
flushScheduled = false
flushBuffered(this)
})
}
callback()
},
flush(callback) {
flushBuffered(this)
callback()
},
})
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Flight data injection – Node.js Transform that passes HTML chunks through // Flight data injection – Node.js Transform that passes HTML chunks through
// while pulling from a separate data stream and interleaving its chunks. // while pulling from a separate data stream and interleaving its chunks.
...@@ -725,7 +683,7 @@ export async function continueFizzStream( ...@@ -725,7 +683,7 @@ export async function continueFizzStream(
// 1. Buffer – coalesces chunks written in the same microtask into one Uint8Array // 1. Buffer – coalesces chunks written in the same microtask into one Uint8Array
// 2. Flight data injection – interleaves RSC data chunks with the HTML stream // 2. Flight data injection – interleaves RSC data chunks with the HTML stream
// 3. Head insertion – inserts server-generated HTML before </head> // 3. Head insertion – inserts server-generated HTML before </head>
const buffered = createBufferedTransformStream() const buffered = createNodeBufferedTransformStream()
webToReadable(renderStream).pipe(buffered) webToReadable(renderStream).pipe(buffered)
let source: Readable = buffered let source: Readable = buffered
...@@ -836,7 +794,7 @@ export async function continueDynamicHTMLResumeNode( ...@@ -836,7 +794,7 @@ export async function continueDynamicHTMLResumeNode(
): Promise<AnyStream> { ): Promise<AnyStream> {
await waitAtLeastOneReactRenderTask() await waitAtLeastOneReactRenderTask()
const buffered = createBufferedTransformStream() const buffered = createNodeBufferedTransformStream()
webToReadable(renderStream).pipe(buffered) webToReadable(renderStream).pipe(buffered)
let source: Readable = buffered let source: Readable = buffered
...@@ -1098,7 +1056,12 @@ export function getServerPrerender(ComponentMod: { ...@@ -1098,7 +1056,12 @@ export function getServerPrerender(ComponentMod: {
export const getClientPrerender: typeof import('react-dom/static').prerender = export const getClientPrerender: typeof import('react-dom/static').prerender =
prerender prerender
// Node counterpart of the web `teeStream`. Like the web version it assumes the
// stream type matching its build — here a Node `Readable` — and fans out
// through `ReplayableNodeStream`. Need three or more consumers from one source?
// Use `ReplayableNodeStream` directly (N `createReplayStream()` calls) to avoid
// nesting tees.
export function teeStream(stream: AnyStream): [AnyStream, AnyStream] { export function teeStream(stream: AnyStream): [AnyStream, AnyStream] {
const [s1, s2] = nodeReadableToWebReadableStream(stream).tee() const replayable = new ReplayableNodeStream(stream as Readable)
return [webToReadable(s1), webToReadable(s2)] return [replayable.createReplayStream(), replayable.createReplayStream()]
} }
import type { Readable } from 'node:stream' import type { Readable } from 'node:stream'
import { createBufferedTransformStream } from '../stream-utils/node-web-streams-helper' import { createBufferedTransformStream } from '../stream-utils/node-web-streams-helper'
import { createNodeBufferedTransformStream } from '../stream-utils/node-buffered-transform-stream'
import { import {
HMR_MESSAGE_SENT_TO_BROWSER, HMR_MESSAGE_SENT_TO_BROWSER,
type HmrMessageSentToBrowser, type HmrMessageSentToBrowser,
} from './hot-reloader-types' } from './hot-reloader-types'
import type { AnyStream } from '../app-render/stream-ops' import type { AnyStream } from '../app-render/stream-ops'
function toWebReadableStream(stream: AnyStream): ReadableStream<Uint8Array> { // Chunks are sent to the browser in batches to reduce overhead, flushing
if (stream instanceof ReadableStream) { // synchronously once this many bytes have accumulated.
return stream const MAX_DEBUG_CHANNEL_BATCH_BYTES = 128 * 1024
}
const { Readable: ReadableClass } =
require('node:stream') as typeof import('node:stream')
return ReadableClass.toWeb(stream as Readable) as ReadableStream<Uint8Array>
}
export interface ReactDebugChannelForBrowser { export interface ReactDebugChannelForBrowser {
readonly readable: AnyStream readonly readable: AnyStream
...@@ -24,19 +20,23 @@ const reactDebugChannelsByHtmlRequestId = new Map< ...@@ -24,19 +20,23 @@ const reactDebugChannelsByHtmlRequestId = new Map<
ReactDebugChannelForBrowser ReactDebugChannelForBrowser
>() >()
/**
* Reads the React debug channel and forwards its chunks to the browser through
* the websocket. Branches on the stream type so that Node streams stay
* node-native — batched with a Node `Transform` and consumed via events.
*/
export function connectReactDebugChannel( export function connectReactDebugChannel(
requestId: string, requestId: string,
debugChannel: ReactDebugChannelForBrowser, debugChannel: ReactDebugChannelForBrowser,
sendToClient: (message: HmrMessageSentToBrowser) => void sendToClient: (message: HmrMessageSentToBrowser) => void
) { ) {
const reader = toWebReadableStream(debugChannel.readable) let finished = false
.pipeThrough(
// We're sending the chunks in batches to reduce overhead in the browser.
createBufferedTransformStream({ maxBufferByteLength: 128 * 1024 })
)
.getReader()
const stop = () => { const stop = () => {
if (finished) {
return
}
finished = true
sendToClient({ sendToClient({
type: HMR_MESSAGE_SENT_TO_BROWSER.REACT_DEBUG_CHUNK, type: HMR_MESSAGE_SENT_TO_BROWSER.REACT_DEBUG_CHUNK,
requestId, requestId,
...@@ -45,25 +45,55 @@ export function connectReactDebugChannel( ...@@ -45,25 +45,55 @@ export function connectReactDebugChannel(
} }
const onError = (err: unknown) => { const onError = (err: unknown) => {
console.error(new Error('React debug channel stream error', { cause: err })) if (!finished) {
console.error(
new Error('React debug channel stream error', { cause: err })
)
}
stop() stop()
} }
const progress = (entry: ReadableStreamReadResult<Uint8Array>) => { const sendChunk = (chunk: Uint8Array) => {
if (entry.done) { sendToClient({
stop() type: HMR_MESSAGE_SENT_TO_BROWSER.REACT_DEBUG_CHUNK,
} else { requestId,
sendToClient({ chunk,
type: HMR_MESSAGE_SENT_TO_BROWSER.REACT_DEBUG_CHUNK, })
requestId,
chunk: entry.value,
})
reader.read().then(progress, onError)
}
} }
reader.read().then(progress, onError) const { readable } = debugChannel
if (readable instanceof ReadableStream) {
const reader = readable
.pipeThrough(
createBufferedTransformStream({
maxBufferByteLength: MAX_DEBUG_CHANNEL_BATCH_BYTES,
})
)
.getReader()
const progress = (entry: ReadableStreamReadResult<Uint8Array>) => {
if (entry.done) {
stop()
} else {
sendChunk(entry.value)
reader.read().then(progress, onError)
}
}
reader.read().then(progress, onError)
} else {
const source = readable as Readable
// `pipe` does not forward source errors to the destination, so handle them
// on the source directly.
source.on('error', onError)
const batched = source.pipe(
createNodeBufferedTransformStream(MAX_DEBUG_CHANNEL_BATCH_BYTES)
)
batched.on('data', sendChunk)
batched.on('end', stop)
batched.on('error', onError)
}
} }
export function connectReactDebugChannelForHtmlRequest( export function connectReactDebugChannelForHtmlRequest(
......
import { Transform } from 'node:stream'
/**
* Node.js `Transform` that coalesces chunks written in the same microtask into
* a single `Uint8Array` before pushing downstream, flushing synchronously once
* the buffer reaches `maxBufferByteLength` (default: never, i.e.
* microtask-only).
*
* This lives in its own module, separate from `stream-ops.node`, so that
* consumers outside the render layer (e.g. the dev server's React debug
* channel) can batch Node streams without pulling in `stream-ops.node`'s
* `react-dom/server` and `react-dom/static` imports. Those resolve to the
* vendored React only inside aliased app bundles; loaded from the unaliased
* dev-server runtime they resolve to the app's installed React, which on React
* 18 has no `react-dom/static` export.
*/
export function createNodeBufferedTransformStream(
maxBufferByteLength: number = Infinity
): Transform {
let bufferedChunks: Array<Uint8Array> = []
let bufferByteLength = 0
let flushScheduled = false
function flushBuffered(stream: Transform): void {
if (bufferedChunks.length === 0) return
const merged = new Uint8Array(bufferByteLength)
let copiedBytes = 0
for (let i = 0; i < bufferedChunks.length; i++) {
const bufferedChunk = bufferedChunks[i]
merged.set(bufferedChunk, copiedBytes)
copiedBytes += bufferedChunk.byteLength
}
bufferedChunks.length = 0
bufferByteLength = 0
stream.push(merged)
}
return new Transform({
transform(chunk, _encoding, callback) {
bufferedChunks.push(chunk)
bufferByteLength += chunk.byteLength
if (bufferByteLength >= maxBufferByteLength) {
flushBuffered(this)
} else if (!flushScheduled) {
flushScheduled = true
queueMicrotask(() => {
flushScheduled = false
flushBuffered(this)
})
}
callback()
},
flush(callback) {
flushBuffered(this)
callback()
},
})
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment