Commit 74077c65 authored by Jimmy Lai's avatar Jimmy Lai Committed by Tim Neutkens

feat(node-streams): add node stream primitives, build infra, and config flag

Add the building blocks for native Node.js stream rendering:

- Config flag: experimental.useNodeStreams with __NEXT_USE_NODE_STREAMS env var
- Node stream primitives: node-stream-helpers, pipeable-stream-wrappers,
  node-stream-tee, pipe-readable, chain-node-streams
- Stream ops: stream-ops.node.ts with compile-time switcher in stream-ops.ts
- Debug channel: debug-channel-server.node.ts with 3-way conditional switcher
- Build: taskfile.js bundle tasks, webpack config routing, module.compiled.js
- Flight response: createInlinedDataNodeStream for Node Transform encoding
- Entry base: conditional exports for renderToPipeableStream/prerenderToNodeStream
- Tests: pipe-readable, flight response node stream, env precedence e2e
parent cbc4888b
This diff is collapsed.
...@@ -1069,5 +1069,23 @@ ...@@ -1069,5 +1069,23 @@
"1068": "Expected workStore to be initialized", "1068": "Expected workStore to be initialized",
"1069": "Invariant: cache entry \"%s\" not found in dir \"%s\"", "1069": "Invariant: cache entry \"%s\" not found in dir \"%s\"",
"1070": "image of size %s could not be tracked by lru cache", "1070": "image of size %s could not be tracked by lru cache",
"1071": "toNodeDebugChannel cannot be used in edge/web runtime, this is a bug in the Next.js codebase" "1071": "toNodeDebugChannel cannot be used in edge/web runtime, this is a bug in the Next.js codebase",
"1072": "failed to pipe node readable to response",
"1073": "Node.js Readable responses should use the pipeToNodeResponse path directly",
"1074": "Node.js Readable responses cannot be coerced to web ReadableStream arrays",
"1075": "createRuntimePrefetchNodeTransform is not supported in edge runtime",
"1076": "createInlinedDataNodeStream is not supported in edge runtime",
"1077": "NODE_STREAMS_PATH_REACHED",
"1078": "processNodePrelude is not supported in edge runtime",
"1079": "createUnclosingNodeStream is not supported in edge runtime",
"1080": "createClosingNodeStream is not supported in edge runtime",
"1081": "Cannot tee a Node.js stream in the edge runtime",
"1082": "toNodeDebugChannel can only be used in Node.js environments, this is a bug in the Next.js codebase",
"1083": "teeNodeReadable is not supported in edge runtime",
"1084": "Node stream tee buffered too much pending data while waiting for drain",
"1085": "Debug channel readable is not a ReadableStream but node streams are not enabled, this is a bug in the Next.js codebase",
"1086": "createNodeDebugChannel requires Node.js streams support",
"1087": "debug-channel-server.node.ts must not be loaded in edge runtime. Import from debug-channel-server.compiled instead.",
"1088": "debug-channel-server.node.ts must not be loaded in edge runtime. Import from debug-channel-server instead.",
"1089": "Invariant: Node.js stream chaining is only available in node streams mode"
} }
...@@ -119,10 +119,18 @@ const bundleTypes = { ...@@ -119,10 +119,18 @@ const bundleTypes = {
* @param {boolean} options.turbo * @param {boolean} options.turbo
* @param {keyof typeof bundleTypes} options.bundleType * @param {keyof typeof bundleTypes} options.bundleType
* @param {boolean} options.experimental * @param {boolean} options.experimental
* @param {boolean} options.nodeStreams
* @param {Partial<webpack.Configuration>} options.rest * @param {Partial<webpack.Configuration>} options.rest
* @returns {webpack.Configuration} * @returns {webpack.Configuration}
*/ */
module.exports = ({ dev, turbo, bundleType, experimental, ...rest }) => { module.exports = ({
dev,
turbo,
bundleType,
experimental,
nodeStreams,
...rest
}) => {
const externalHandler = ({ context, request, getResolve }, callback) => { const externalHandler = ({ context, request, getResolve }, callback) => {
;(async () => { ;(async () => {
if ( if (
...@@ -182,7 +190,7 @@ module.exports = ({ dev, turbo, bundleType, experimental, ...rest }) => { ...@@ -182,7 +190,7 @@ module.exports = ({ dev, turbo, bundleType, experimental, ...rest }) => {
path: path.join(__dirname, 'dist/compiled/next-server'), path: path.join(__dirname, 'dist/compiled/next-server'),
filename: `[name]${turbo ? '-turbo' : ''}${ filename: `[name]${turbo ? '-turbo' : ''}${
experimental ? '-experimental' : '' experimental ? '-experimental' : ''
}.runtime.${dev ? 'dev' : 'prod'}.js`, }${nodeStreams ? '-nodestreams' : ''}.runtime.${dev ? 'dev' : 'prod'}.js`,
libraryTarget: 'commonjs2', libraryTarget: 'commonjs2',
}, },
devtool: 'source-map', devtool: 'source-map',
...@@ -220,6 +228,15 @@ module.exports = ({ dev, turbo, bundleType, experimental, ...rest }) => { ...@@ -220,6 +228,15 @@ module.exports = ({ dev, turbo, bundleType, experimental, ...rest }) => {
'process.env.__NEXT_EXPERIMENTAL_REACT': JSON.stringify( 'process.env.__NEXT_EXPERIMENTAL_REACT': JSON.stringify(
experimental ? true : false experimental ? true : false
), ),
// Only define for app bundles where it controls render code paths.
// The server bundle sets this env var at runtime from config.
...(bundleType === 'app'
? {
'process.env.__NEXT_USE_NODE_STREAMS': JSON.stringify(
nodeStreams ? true : false
),
}
: {}),
'process.env.NEXT_RUNTIME': JSON.stringify('nodejs'), 'process.env.NEXT_RUNTIME': JSON.stringify('nodejs'),
'process.turbopack': JSON.stringify(turbo), 'process.turbopack': JSON.stringify(turbo),
'process.env.TURBOPACK': JSON.stringify(turbo), 'process.env.TURBOPACK': JSON.stringify(turbo),
......
...@@ -123,6 +123,7 @@ export function getDefineEnv({ ...@@ -123,6 +123,7 @@ export function getDefineEnv({
const isPPREnabled = checkIsAppPPREnabled(config.experimental.ppr) const isPPREnabled = checkIsAppPPREnabled(config.experimental.ppr)
const isCacheComponentsEnabled = !!config.cacheComponents const isCacheComponentsEnabled = !!config.cacheComponents
const isUseCacheEnabled = !!config.experimental.useCache const isUseCacheEnabled = !!config.experimental.useCache
const isUseNodeStreamsEnabled = !!config.experimental.useNodeStreams
const defineEnv: DefineEnv = { const defineEnv: DefineEnv = {
// internal field to identify the plugin config // internal field to identify the plugin config
...@@ -175,6 +176,9 @@ export function getDefineEnv({ ...@@ -175,6 +176,9 @@ export function getDefineEnv({
'process.env.__NEXT_INSTANT_NAV_TOGGLE': 'process.env.__NEXT_INSTANT_NAV_TOGGLE':
!!config.experimental.instantNavigationDevToolsToggle, !!config.experimental.instantNavigationDevToolsToggle,
'process.env.__NEXT_USE_CACHE': isUseCacheEnabled, 'process.env.__NEXT_USE_CACHE': isUseCacheEnabled,
'process.env.__NEXT_USE_NODE_STREAMS': isEdgeServer
? false
: isUseNodeStreamsEnabled,
...(config.experimental?.useSkewCookie || !config.deploymentId ...(config.experimental?.useSkewCookie || !config.deploymentId
? { ? {
......
...@@ -27,3 +27,9 @@ exports.renderToStaticMarkup = error ...@@ -27,3 +27,9 @@ exports.renderToStaticMarkup = error
if (b.resume) { if (b.resume) {
exports.resume = b.resume exports.resume = b.resume
} }
if (b.renderToPipeableStream) {
exports.renderToPipeableStream = b.renderToPipeableStream
}
if (b.resumeToPipeableStream) {
exports.resumeToPipeableStream = b.resumeToPipeableStream
}
...@@ -27,3 +27,9 @@ exports.renderToStaticMarkup = error ...@@ -27,3 +27,9 @@ exports.renderToStaticMarkup = error
if (b.resume) { if (b.resume) {
exports.resume = b.resume exports.resume = b.resume
} }
if (b.renderToPipeableStream) {
exports.renderToPipeableStream = b.renderToPipeableStream
}
if (b.resumeToPipeableStream) {
exports.resumeToPipeableStream = b.resumeToPipeableStream
}
...@@ -58,6 +58,7 @@ export interface ExportPageInput { ...@@ -58,6 +58,7 @@ export interface ExportPageInput {
debugOutput?: boolean debugOutput?: boolean
nextConfigOutput?: NextConfigComplete['output'] nextConfigOutput?: NextConfigComplete['output']
enableExperimentalReact?: boolean enableExperimentalReact?: boolean
enableNodeStreams?: boolean
sriEnabled: boolean sriEnabled: boolean
renderResumeDataCache: RenderResumeDataCache | undefined renderResumeDataCache: RenderResumeDataCache | undefined
} }
......
...@@ -80,6 +80,7 @@ async function exportPageImpl( ...@@ -80,6 +80,7 @@ async function exportPageImpl(
disableOptimizedLoading, disableOptimizedLoading,
debugOutput = false, debugOutput = false,
enableExperimentalReact, enableExperimentalReact,
enableNodeStreams,
trailingSlash, trailingSlash,
sriEnabled, sriEnabled,
renderOpts: commonRenderOpts, renderOpts: commonRenderOpts,
...@@ -92,6 +93,9 @@ async function exportPageImpl( ...@@ -92,6 +93,9 @@ async function exportPageImpl(
if (enableExperimentalReact) { if (enableExperimentalReact) {
process.env.__NEXT_EXPERIMENTAL_REACT = 'true' process.env.__NEXT_EXPERIMENTAL_REACT = 'true'
} }
if (enableNodeStreams) {
process.env.__NEXT_USE_NODE_STREAMS = 'true'
}
const { const {
path, path,
...@@ -415,6 +419,7 @@ export async function exportPages( ...@@ -415,6 +419,7 @@ export async function exportPages(
httpAgentOptions: nextConfig.httpAgentOptions, httpAgentOptions: nextConfig.httpAgentOptions,
debugOutput: options.debugOutput, debugOutput: options.debugOutput,
enableExperimentalReact: needsExperimentalReact(nextConfig), enableExperimentalReact: needsExperimentalReact(nextConfig),
enableNodeStreams: !!nextConfig.experimental.useNodeStreams,
sriEnabled: Boolean(nextConfig.experimental.sri?.algorithm), sriEnabled: Boolean(nextConfig.experimental.sri?.algorithm),
buildId: input.buildId, buildId: input.buildId,
deploymentId: input.deploymentId, deploymentId: input.deploymentId,
......
import type { Readable as NodeReadable } from 'node:stream'
import { InvariantError } from '../../shared/lib/invariant-error' import { InvariantError } from '../../shared/lib/invariant-error'
// React's RSC prerender function will emit an incomplete flight stream when using `prerender`. If the connection // React's RSC prerender function will emit an incomplete flight stream when using `prerender`. If the connection
...@@ -160,3 +161,34 @@ export async function processPrelude( ...@@ -160,3 +161,34 @@ export async function processPrelude(
return { prelude, preludeIsEmpty } return { prelude, preludeIsEmpty }
} }
export async function processNodePrelude(unprocessedPrelude: NodeReadable) {
if (process.env.NEXT_RUNTIME === 'edge') {
throw new Error('processNodePrelude is not supported in edge runtime')
} else {
const { teeNodeReadable } =
require('./node-stream-tee') as typeof import('./node-stream-tee')
const [prelude, peek] = teeNodeReadable(unprocessedPrelude)
// Read the first chunk from the peek stream to check if it's empty
const firstChunk = await new Promise<Uint8Array | null>(
(resolve, reject) => {
peek.once('data', (chunk) => {
peek.destroy()
resolve(chunk)
})
peek.once('end', () => {
resolve(null)
})
peek.once('error', (err) => {
if (!prelude.destroyed) prelude.destroy(err)
reject(err)
})
}
)
const preludeIsEmpty = firstChunk === null
return { prelude, preludeIsEmpty }
}
}
import { Readable, Writable } from 'node:stream'
// ---------------------------------------------------------------------------
// Shared types (imported by debug-channel-server.web.ts via `import type`)
// ---------------------------------------------------------------------------
export type DebugChannelPair = {
// serverSide is the write end. When using Node streams, this is a
// PassThrough (Node Writable). React's renderToPipeableStream checks
// typeof debugChannel.write === 'function' and writes directly.
// When using web streams, this is { writable: WritableStream }.
serverSide: DebugChannelServer | Writable
clientSide: DebugChannelClient
}
// Web shape: React's renderToReadableStream reads .writable from it.
// For renderToPipeableStream (Node), use toNodeDebugChannel() to convert.
export type DebugChannelServer = {
readable?: ReadableStream<Uint8Array>
writable: WritableStream<Uint8Array>
}
export type DebugChannelClient = {
readable: ReadableStream<Uint8Array> | Readable
writable?: WritableStream<Uint8Array>
}
// Runtime invariant: this module must never be loaded in edge runtime.
// The .js switcher should prevent this, but this guard catches
// direct imports or switcher bugs.
if (process.env.NEXT_RUNTIME === 'edge') {
throw new Error(
'debug-channel-server.node.ts must not be loaded in edge runtime. ' +
'Import from debug-channel-server instead.'
)
}
export function createDebugChannel(): DebugChannelPair | undefined {
if (process.env.NODE_ENV === 'production') {
return undefined
}
return createNodeDebugChannel()
}
export function createNodeDebugChannel(): DebugChannelPair {
// The read side is a plain Readable (NOT a PassThrough/Duplex).
// PassThrough is a Transform (Duplex) which has both readable and writable
// sides. Node's `for await...of` uses endOfStream() internally, which checks
// both sides of a Duplex. Ending the writable side triggers a 'close' event
// that endOfStream interprets as ERR_STREAM_PREMATURE_CLOSE before the
// async iterator finishes consuming the readable side. A plain Readable has
// no writable side, so endOfStream only checks readable completion via
// push(null), which works cleanly with async iteration.
// autoDestroy: false is critical. By default (Node 16+), a push-mode Readable
// that receives push(null) gets auto-destroyed if nobody is consuming it yet.
// If the render completes and the event loop ticks before the tee is set up,
// the Readable is destroyed and won't emit 'data' events for buffered data.
const reader = new Readable({ read() {}, autoDestroy: false })
// The write side must be a plain Writable (NOT a Duplex/PassThrough).
// React's renderToPipeableStream checks `typeof debugChannel.read === 'function'`
// to detect a bidirectional channel. If it finds .read(), it reads from the
// same object expecting debug commands back from the client, which causes
// "Unknown command" errors when it reads its own output.
//
// autoDestroy: false is critical. Without it, the Writable auto-destroys
// after end(), triggering its destroy() handler which explicitly destroys
// the reader. The reader may not be consumed yet (tee setup happens later
// after an async gap), so destroying it would discard buffered debug data.
const writer = new Writable({
autoDestroy: false,
write(
chunk: Buffer | Uint8Array,
_encoding: string,
callback: (error?: Error | null) => void
) {
reader.push(chunk)
callback()
},
final(callback: (error?: Error | null) => void) {
reader.push(null)
callback()
},
destroy(err, callback) {
reader.destroy(err ?? undefined)
callback(err)
},
})
return {
// Plain Writable: has .write() but NOT .read(), so React uses it
// write-only. The isNodeWritable check in pipeable-stream-wrappers.ts
// also passes, skipping toNodeDebugChannel().
serverSide: writer,
// Plain Readable: available for tee or direct consumption.
// No writable side means for-await-of works without ERR_STREAM_PREMATURE_CLOSE.
clientSide: { readable: reader },
}
}
/**
* Converts a web-shaped debug channel to a Node Writable for use with
* React's renderToPipeableStream, which checks typeof debugChannel.write
* to find its debug destination.
*/
export function toNodeDebugChannel(
webDebugChannel: DebugChannelServer
): Writable {
const writerStream = webDebugChannel.writable.getWriter()
return new Writable({
write(
chunk: Buffer | Uint8Array,
_encoding: string,
callback: (error?: Error | null) => void
) {
writerStream.write(chunk).then(() => callback(), callback)
},
final(callback: (error?: Error | null) => void) {
writerStream.close().then(() => callback(), callback)
},
destroy(_err, callback) {
writerStream.abort(_err).then(
() => callback(_err),
() => callback(_err)
)
},
})
}
/** import type {
* Compile-time switcher for debug channel operations.
*
* Simple re-export from the web implementation.
* A future change will add a conditional branch for node streams.
*/
export type {
DebugChannelPair, DebugChannelPair,
DebugChannelServer, DebugChannelServer,
DebugChannelClient,
} from './debug-channel-server.web' } from './debug-channel-server.web'
export { type DebugChannelRuntimeModule = {
createDebugChannel, createDebugChannel: () => DebugChannelPair | undefined
toNodeDebugChannel, toNodeDebugChannel: (
} from './debug-channel-server.web' webDebugChannel: DebugChannelServer
) => import('node:stream').Writable
}
let debugChannelRuntimeModule: DebugChannelRuntimeModule
if (process.env.NEXT_RUNTIME === 'edge') {
debugChannelRuntimeModule =
require('./debug-channel-server.web') as typeof import('./debug-channel-server.web')
} else if (process.env.__NEXT_USE_NODE_STREAMS) {
debugChannelRuntimeModule =
require('./debug-channel-server.node') as typeof import('./debug-channel-server.node') as unknown as DebugChannelRuntimeModule
} else {
debugChannelRuntimeModule =
require('./debug-channel-server.web') as typeof import('./debug-channel-server.web')
}
export const createDebugChannel = debugChannelRuntimeModule.createDebugChannel
export const toNodeDebugChannel = debugChannelRuntimeModule.toNodeDebugChannel
export type { DebugChannelPair, DebugChannelServer, DebugChannelClient }
...@@ -14,7 +14,7 @@ export type DebugChannelServer = { ...@@ -14,7 +14,7 @@ export type DebugChannelServer = {
writable: WritableStream<Uint8Array> writable: WritableStream<Uint8Array>
} }
type DebugChannelClient = { export type DebugChannelClient = {
readable: ReadableStream<Uint8Array> readable: ReadableStream<Uint8Array>
writable?: WritableStream<Uint8Array> writable?: WritableStream<Uint8Array>
} }
......
...@@ -12,6 +12,34 @@ export { ...@@ -12,6 +12,34 @@ export {
// eslint-disable-next-line import/no-extraneous-dependencies // eslint-disable-next-line import/no-extraneous-dependencies
export { prerender } from 'react-server-dom-webpack/static' export { prerender } from 'react-server-dom-webpack/static'
// Node.js-specific Flight APIs, needed by stream-ops.node.ts via ComponentMod.
// These must be exported from entry-base (react-server layer) because direct
// imports from react-server-dom-webpack/* fail outside this layer.
type FlightRenderToPipeableStream = (...args: any[]) => {
pipe<Writable extends NodeJS.WritableStream>(destination: Writable): Writable
abort: (reason?: unknown) => void
}
type FlightPrerenderToNodeStream = (...args: any[]) => Promise<{
prelude: import('node:stream').Readable
}>
/* eslint-disable import/no-extraneous-dependencies */
export let renderToPipeableStream: FlightRenderToPipeableStream | undefined
export let prerenderToNodeStream: FlightPrerenderToNodeStream | undefined
if (process.env.__NEXT_USE_NODE_STREAMS) {
renderToPipeableStream = (
require('react-server-dom-webpack/server.node') as typeof import('react-server-dom-webpack/server.node')
).renderToPipeableStream
prerenderToNodeStream = (
require('react-server-dom-webpack/static') as typeof import('react-server-dom-webpack/static')
).prerenderToNodeStream
} else {
renderToPipeableStream = undefined
prerenderToNodeStream = undefined
}
/* eslint-enable import/no-extraneous-dependencies */
// TODO: Just re-export `* as ReactServer` // TODO: Just re-export `* as ReactServer`
export { captureOwnerStack, createElement, Fragment } from 'react' export { captureOwnerStack, createElement, Fragment } from 'react'
...@@ -45,7 +73,8 @@ export { taintObjectReference } from './rsc/taint' ...@@ -45,7 +73,8 @@ export { taintObjectReference } from './rsc/taint'
export { collectSegmentData } from './collect-segment-data' export { collectSegmentData } from './collect-segment-data'
export const InstantValidation = export const InstantValidation =
process.env.NODE_ENV === 'development' && process.env.NEXT_RUNTIME !== 'edge' process.env.NODE_ENV === 'development' &&
process.env.NEXT_RUNTIME === 'nodejs'
? (require('./instant-validation/instant-validation') as typeof import('./instant-validation/instant-validation')) ? (require('./instant-validation/instant-validation') as typeof import('./instant-validation/instant-validation'))
: undefined : undefined
......
import type { Readable as NodeReadable } from 'node:stream'
const MAX_NODE_TEE_PENDING_BYTES = 8 * 1024 * 1024
type NodeTeeChunk = Buffer | Uint8Array
type NodeTeeState = {
closed: boolean
pending: Array<NodeTeeChunk>
pendingStartIndex: number
pendingBytes: number
waitingDrain: boolean
stream: import('node:stream').PassThrough
}
function chunkByteLength(chunk: NodeTeeChunk): number {
return Buffer.isBuffer(chunk) ? chunk.length : chunk.byteLength
}
function pendingChunkCount(state: NodeTeeState): number {
return state.pending.length - state.pendingStartIndex
}
function dequeuePendingChunk(state: NodeTeeState): NodeTeeChunk | undefined {
if (pendingChunkCount(state) === 0) return undefined
const chunk = state.pending[state.pendingStartIndex]
state.pendingStartIndex++
// Compact occasionally so stale entries do not accumulate.
if (
state.pendingStartIndex > 1024 &&
state.pendingStartIndex * 2 >= state.pending.length
) {
state.pending = state.pending.slice(state.pendingStartIndex)
state.pendingStartIndex = 0
} else if (state.pendingStartIndex === state.pending.length) {
state.pending.length = 0
state.pendingStartIndex = 0
}
return chunk
}
/**
* Tees a Node Readable into two Readables without coupling backpressure
* across branches.
*/
export function teeNodeReadable(
source: NodeReadable,
runInContext: <T>(fn: () => T) => T = (fn) => fn()
): [NodeReadable, NodeReadable] {
if (process.env.NEXT_RUNTIME === 'edge') {
throw new Error('teeNodeReadable is not supported in edge runtime')
} else {
const { PassThrough } =
require('node:stream') as typeof import('node:stream')
const left = new PassThrough()
const right = new PassThrough()
const states: [NodeTeeState, NodeTeeState] = [
{
stream: left,
closed: false,
pending: [],
pendingStartIndex: 0,
pendingBytes: 0,
waitingDrain: false,
},
{
stream: right,
closed: false,
pending: [],
pendingStartIndex: 0,
pendingBytes: 0,
waitingDrain: false,
},
]
let sourceEnded = false
let failed = false
function totalPendingBytes(): number {
return states[0].pendingBytes + states[1].pendingBytes
}
function areAllOpenBranchesBlocked(): boolean {
for (const state of states) {
if (state.closed) continue
if (!state.waitingDrain && pendingChunkCount(state) === 0) {
return false
}
}
return true
}
function maybePauseSource() {
if (
!source.isPaused() &&
areAllOpenBranchesBlocked() &&
totalPendingBytes() >= MAX_NODE_TEE_PENDING_BYTES
) {
source.pause()
}
}
function maybeResumeSource() {
if (
source.isPaused() &&
!sourceEnded &&
(totalPendingBytes() < MAX_NODE_TEE_PENDING_BYTES / 2 ||
!areAllOpenBranchesBlocked())
) {
source.resume()
}
}
function closeBranch(state: NodeTeeState) {
if (state.closed) return
state.closed = true
state.pending.length = 0
state.pendingStartIndex = 0
state.pendingBytes = 0
state.waitingDrain = false
maybeResumeSource()
if (states[0].closed && states[1].closed) {
cleanup()
if (!source.destroyed) {
source.destroy()
}
}
}
function fail(error: Error) {
if (failed) return
failed = true
cleanup()
for (const state of states) {
state.pending.length = 0
state.pendingStartIndex = 0
state.pendingBytes = 0
if (!state.closed && !state.stream.destroyed) {
state.stream.destroy(error)
}
}
if (!source.destroyed) {
source.destroy(error)
}
}
function maybeCleanupAfterSourceEnd() {
if (!sourceEnded || failed) return
for (const state of states) {
if (
state.closed ||
state.stream.destroyed ||
state.stream.writableEnded
) {
continue
}
return
}
cleanup()
}
function maybeEndBranch(state: NodeTeeState) {
if (
failed ||
state.closed ||
state.stream.destroyed ||
state.stream.writableEnded
) {
return
}
if (!sourceEnded) return
if (pendingChunkCount(state) > 0) return
// Do not wait for drain here. A write() that returned false has already
// enqueued its chunk into the stream's internal buffer. Waiting for
// drain before ending can deadlock when this branch is intentionally
// consumed later than its sibling.
state.stream.end()
maybeCleanupAfterSourceEnd()
}
function enqueue(state: NodeTeeState, chunk: NodeTeeChunk) {
if (state.closed) return
if (
state.pendingBytes + chunkByteLength(chunk) >
MAX_NODE_TEE_PENDING_BYTES
) {
fail(
new Error(
'Node stream tee buffered too much pending data while waiting for drain'
)
)
return
}
state.pending.push(chunk)
state.pendingBytes += chunkByteLength(chunk)
maybePauseSource()
}
function writeOrQueue(state: NodeTeeState, chunk: NodeTeeChunk) {
if (state.closed) return
if (state.waitingDrain || pendingChunkCount(state) > 0) {
enqueue(state, chunk)
return
}
if (!state.stream.write(chunk)) {
state.waitingDrain = true
}
}
function flushPending(state: NodeTeeState) {
if (state.closed || failed) return
while (pendingChunkCount(state) > 0) {
if (!sourceEnded && state.waitingDrain) {
break
}
const chunk = dequeuePendingChunk(state)!
state.pendingBytes -= chunkByteLength(chunk)
if (!state.stream.write(chunk)) {
state.waitingDrain = true
if (!sourceEnded) {
break
}
}
}
if (sourceEnded) {
maybeEndBranch(state)
return
}
maybeResumeSource()
}
const onSourceData = (chunk: NodeTeeChunk) => {
runInContext(() => {
if (failed) return
writeOrQueue(states[0], chunk)
writeOrQueue(states[1], chunk)
maybePauseSource()
})
}
const onSourceEnd = () => {
runInContext(() => {
sourceEnded = true
flushPending(states[0])
flushPending(states[1])
maybeEndBranch(states[0])
maybeEndBranch(states[1])
maybeCleanupAfterSourceEnd()
})
}
const onSourceError = (error: Error) => {
runInContext(() => {
fail(error)
})
}
const onLeftDrain = () => {
runInContext(() => {
states[0].waitingDrain = false
flushPending(states[0])
})
}
const onRightDrain = () => {
runInContext(() => {
states[1].waitingDrain = false
flushPending(states[1])
})
}
const onLeftClose = () => {
runInContext(() => {
closeBranch(states[0])
})
}
const onRightClose = () => {
runInContext(() => {
closeBranch(states[1])
})
}
function cleanup() {
source.off('data', onSourceData)
source.off('end', onSourceEnd)
source.off('error', onSourceError)
states[0].stream.off('drain', onLeftDrain)
states[1].stream.off('drain', onRightDrain)
states[0].stream.off('close', onLeftClose)
states[1].stream.off('close', onRightClose)
}
source.on('data', onSourceData)
source.on('end', onSourceEnd)
source.on('error', onSourceError)
states[0].stream.on('drain', onLeftDrain)
states[1].stream.on('drain', onRightDrain)
states[0].stream.on('close', onLeftClose)
states[1].stream.on('close', onRightClose)
// Handle the case where source already ended before we attached listeners.
// Node 'end' events fire once, so if the source ended before our listener
// was attached (e.g. debug channel completed before tee setup), we'd never
// get the event and the branches would hang.
//
// When readableEnded is true, any data pushed into the Readable sits in the
// internal buffer. Adding a 'data' listener switches to flowing mode and
// schedules async draining, but we need the data NOW before signaling end.
// Manually pull buffered data with source.read() to ensure branches receive
// all chunks before being ended.
if (source.readableEnded) {
let chunk: NodeTeeChunk | null
while ((chunk = source.read() as NodeTeeChunk | null) !== null) {
onSourceData(chunk)
}
onSourceEnd()
}
return [left, right]
}
}
import type { Readable } from 'node:stream'
import type { PostponedState } from 'react-dom/static'
// Lazy require to avoid webpack trying to resolve node:stream at parse time.
// When __NEXT_USE_NODE_STREAMS is false, DCE removes all call sites so this
// require() is never reached at runtime.
function getNodeStream(): typeof import('node:stream') {
return require('node:stream') as typeof import('node:stream')
}
import type {
RenderToPipeableStreamOptions,
PipeableStream,
} from 'react-dom/server'
import { DetachedPromise } from '../../lib/detached-promise'
import { getTracer } from '../lib/trace/tracer'
import { AppRenderSpan } from '../lib/trace/constants'
export interface FizzPipeableStreamResult {
stream: Readable
allReady: Promise<void>
abort: (reason?: unknown) => void
}
type FlightRenderToPipeableStream =
typeof import('react-server-dom-webpack/server.node').renderToPipeableStream
type FlightRenderToReadableStream =
typeof import('react-server-dom-webpack/server.edge').renderToReadableStream
type FlightModel = Parameters<FlightRenderToReadableStream>[0]
type FlightWebpackMap = Parameters<FlightRenderToReadableStream>[1]
type FlightRenderOptions = Parameters<FlightRenderToReadableStream>[2]
type ResumeToPipeableOptions = Parameters<
typeof import('react-dom/server').resumeToPipeableStream
>[2] & {
onHeaders?: NonNullable<RenderToPipeableStreamOptions['onHeaders']>
onAllReady?: () => void
}
function wrapOnHeaders(
onHeaders: RenderToPipeableStreamOptions['onHeaders'] | undefined
): RenderToPipeableStreamOptions['onHeaders'] | undefined {
if (!onHeaders) return undefined
return (headersDescriptor: Headers | HeadersInit) => {
onHeaders(new Headers(headersDescriptor))
}
}
/**
* Wraps react-dom/server renderToPipeableStream in an async/await interface
* that matches the existing renderToReadableStream usage patterns.
*
* The returned promise resolves when the shell is ready (onShellReady),
* equivalent to when renderToReadableStream's promise resolves.
*/
export function renderToFizzPipeableStream(
renderToPipeableStream: (
children: React.ReactElement,
options?: RenderToPipeableStreamOptions
) => PipeableStream,
element: React.ReactElement,
options?: RenderToPipeableStreamOptions
): Promise<FizzPipeableStreamResult> {
return getTracer().trace(AppRenderSpan.renderToReadableStream, async () => {
const shellReady = new DetachedPromise<FizzPipeableStreamResult>()
const allReady = new DetachedPromise<void>()
const { PassThrough } = getNodeStream()
const passthrough = new PassThrough()
const originalOnShellReady = options?.onShellReady
const originalOnAllReady = options?.onAllReady
const originalOnShellError = options?.onShellError
// renderToPipeableStream calls onHeaders with a plain object descriptor
// (e.g. { Link: "..." }), while renderToReadableStream wraps it in a
// Headers instance. We need to do the same wrapping here so that
// callers can use headers.forEach().
const wrappedOnHeaders = wrapOnHeaders(options?.onHeaders)
const { pipe, abort } = renderToPipeableStream(element, {
...options,
onHeaders: wrappedOnHeaders,
onShellReady() {
pipe(passthrough)
originalOnShellReady?.()
shellReady.resolve({
stream: passthrough,
allReady: allReady.promise,
abort,
})
},
onAllReady() {
originalOnAllReady?.()
allReady.resolve()
},
onShellError(error: unknown) {
originalOnShellError?.(error)
shellReady.reject(error)
},
})
return shellReady.promise
})
}
/**
* Wraps react-dom/server resumeToPipeableStream in an async/await interface.
*/
export function resumeToFizzPipeableStream(
resumeToPipeableStreamFn: (
children: React.ReactElement,
postponedState: PostponedState,
options?: ResumeToPipeableOptions
) => Promise<PipeableStream>,
element: React.ReactElement,
postponedState: PostponedState,
options?: ResumeToPipeableOptions
): Promise<FizzPipeableStreamResult> {
return getTracer().trace(AppRenderSpan.renderToReadableStream, async () => {
const allReady = new DetachedPromise<void>()
const { PassThrough } = getNodeStream()
const passthrough = new PassThrough()
const originalOnAllReady = options?.onAllReady
// Same onHeaders wrapping as renderToFizzPipeableStream
const wrappedOnHeaders = wrapOnHeaders(options?.onHeaders)
const { pipe, abort } = await resumeToPipeableStreamFn(
element,
postponedState,
{
...options,
onHeaders: wrappedOnHeaders,
onAllReady() {
originalOnAllReady?.()
allReady.resolve()
},
}
)
pipe(passthrough)
return {
stream: passthrough,
allReady: allReady.promise,
abort,
}
})
}
/**
* Wraps react-server-dom-webpack/server.node renderToPipeableStream
* for Flight (RSC) rendering. Since Flight rendering starts immediately
* (no shell concept), we pipe to a PassThrough right away.
*/
export function renderToFlightPipeableStream(
renderToPipeableStreamFn: FlightRenderToPipeableStream,
model: FlightModel,
webpackMap: FlightWebpackMap,
options?: FlightRenderOptions,
runInContext?: <T>(fn: () => T) => T
): Readable {
const { PassThrough } = getNodeStream()
const passthrough = new PassThrough()
// React's renderToPipeableStream checks typeof debugChannel.write to find
// its debug destination. The web-shaped { writable: WritableStream } doesn't
// have .write() on the object, so convert to a Node Writable.
if (options?.debugChannel) {
const { toNodeDebugChannel } =
require('./debug-channel-server') as typeof import('./debug-channel-server')
const debugChannel = options.debugChannel
const isNodeWritable =
typeof debugChannel === 'object' &&
debugChannel !== null &&
'write' in debugChannel &&
typeof debugChannel.write === 'function'
options = {
...options,
debugChannel: isNodeWritable
? debugChannel
: toNodeDebugChannel(
debugChannel as import('./debug-channel-server').DebugChannelServer
),
}
}
const run: <T>(fn: () => T) => T = runInContext ?? ((fn) => fn())
const { pipe } = run(() =>
renderToPipeableStreamFn(
model,
webpackMap,
options as Parameters<FlightRenderToPipeableStream>[2]
)
)
run(() => pipe(passthrough))
return passthrough
}
/**
* Node.js stream operations for the rendering pipeline.
* Loaded by stream-ops.js when __NEXT_USE_NODE_STREAMS is true.
*/
import type { Readable } from 'node:stream'
import { Readable as NodeReadable } from 'node:stream'
import type { PostponedState, PrerenderOptions } from 'react-dom/static'
import {
resumeAndPrerenderToNodeStream,
prerenderToNodeStream,
} from 'react-dom/static'
import * as ReactDOMServer from 'react-dom/server'
import {
renderToFlightPipeableStream,
renderToFizzPipeableStream,
resumeToFizzPipeableStream,
} from './pipeable-stream-wrappers'
import {
nodeStreamToString,
safePipe,
createRuntimePrefetchNodeTransform,
} from '../stream-utils/node-stream-helpers'
import { createInlinedDataNodeStream } from './use-flight-response'
// ---------------------------------------------------------------------------
// Shared types (imported by stream-ops.web.ts via `import type`)
// ---------------------------------------------------------------------------
export type AnyStream = ReadableStream<Uint8Array> | Readable
export type ContinueStreamSharedOptions = {
deploymentId: string | undefined
getServerInsertedHTML: () => Promise<string>
getServerInsertedMetadata: () => Promise<string>
}
export type ContinueFizzStreamOptions = ContinueStreamSharedOptions & {
inlinedDataStream: AnyStream | undefined
isStaticGeneration: boolean
allReady?: Promise<void>
validateRootLayout?: boolean
suffix?: string
}
export type ContinueStaticPrerenderOptions = ContinueStreamSharedOptions & {
inlinedDataStream: AnyStream
}
export type ContinueDynamicHTMLResumeOptions = ContinueStreamSharedOptions & {
inlinedDataStream: AnyStream
delayDataUntilFirstHtmlChunk: boolean
}
type FlightRenderToReadableStream =
typeof import('react-server-dom-webpack/server.edge').renderToReadableStream
type FlightRenderToPipeableStream =
typeof import('react-server-dom-webpack/server.node').renderToPipeableStream
type FlightPrerender =
typeof import('react-server-dom-webpack/static').prerender
type FlightPrerenderToNodeStream =
typeof import('react-server-dom-webpack/static').prerenderToNodeStream
export type FlightComponentMod = {
renderToReadableStream: FlightRenderToReadableStream
renderToPipeableStream?: FlightRenderToPipeableStream
}
export type ServerPrerenderComponentMod = {
prerender: FlightPrerender
prerenderToNodeStream?: FlightPrerenderToNodeStream
}
export type FlightPayload = Parameters<FlightRenderToReadableStream>[0]
export type FlightClientModules = Parameters<FlightRenderToReadableStream>[1]
export type FlightRenderOptions = Parameters<FlightRenderToReadableStream>[2]
type ResumeAndPrerenderOptions = Parameters<
typeof import('react-dom/static').resumeAndPrerenderToNodeStream
>[2]
// ---------------------------------------------------------------------------
// Continue functions
// ---------------------------------------------------------------------------
export {
continueFizzStreamNode as continueFizzStream,
continueStaticPrerenderNode as continueStaticPrerender,
continueDynamicPrerenderNode as continueDynamicPrerender,
continueStaticFallbackPrerenderNode as continueStaticFallbackPrerender,
continueDynamicHTMLResumeNode as continueDynamicHTMLResume,
nodeStreamToBuffer as streamToBuffer,
chainNodeStreams as chainStreams,
createDocumentClosingNodeStream as createDocumentClosingStream,
nodeReadableToWeb,
} from '../stream-utils/node-stream-helpers'
export { processNodePrelude as processPrelude } from './app-render-prerender-utils'
// ---------------------------------------------------------------------------
// Composed helpers
// ---------------------------------------------------------------------------
export function createInlinedDataStream(
source: AnyStream,
nonce: string | undefined,
formState: unknown | null
): AnyStream {
return safePipe(
source as Readable,
createInlinedDataNodeStream(nonce, formState)
)
}
export function createPendingStream(): AnyStream {
return new NodeReadable({ read() {} })
}
export function createOnHeadersCallback(
appendHeader: (key: string, value: string) => void
): NonNullable<PrerenderOptions['onHeaders']> {
return (headersDescriptor: Headers) => {
const headers = new Headers(headersDescriptor)
headers.forEach((value: string, key: string) => {
appendHeader(key, value)
})
}
}
export async function resumeAndAbort(
element: React.ReactElement,
postponed: PostponedState | null,
opts: ResumeAndPrerenderOptions
): Promise<AnyStream> {
const { prelude } = await resumeAndPrerenderToNodeStream(
element,
postponed,
opts
)
return prelude as Readable
}
export function renderToFlightStream(
ComponentMod: FlightComponentMod,
payload: FlightPayload,
clientModules: FlightClientModules,
opts: FlightRenderOptions,
runInContext?: <T>(fn: () => T) => T
): AnyStream {
return renderToFlightPipeableStream(
ComponentMod.renderToPipeableStream!,
payload,
clientModules,
opts,
runInContext
)
}
export async function streamToString(stream: AnyStream): Promise<string> {
return nodeStreamToString(stream as Readable)
}
export type FizzStreamResult = {
stream: AnyStream
allReady: Promise<void>
abort?: (reason?: unknown) => void
}
export async function renderToFizzStream(
element: React.ReactElement,
streamOptions: any,
runInContext?: <T>(fn: () => T) => T
): Promise<FizzStreamResult> {
const run: <T>(fn: () => T) => T = runInContext ?? ((fn) => fn())
const renderFn = (...args: any[]) =>
run(() => (ReactDOMServer as any).renderToPipeableStream(...args))
return renderToFizzPipeableStream(renderFn, element, streamOptions)
}
export async function resumeToFizzStream(
element: React.ReactElement,
postponedState: PostponedState,
streamOptions: any,
runInContext?: <T>(fn: () => T) => T
): Promise<FizzStreamResult> {
const run: <T>(fn: () => T) => T = runInContext ?? ((fn) => fn())
const resumeFn = (...args: any[]) =>
run(() => (ReactDOMServer as any).resumeToPipeableStream(...args))
return resumeToFizzPipeableStream(
resumeFn,
element,
postponedState,
streamOptions
)
}
export function getServerPrerender(
ComponentMod: ServerPrerenderComponentMod
): (...args: any[]) => any {
return ComponentMod.prerenderToNodeStream!
}
// Keep the same exported signature as stream-ops.web.ts. In node mode the
// underlying API returns a Node prelude stream instead of a web stream.
export const getClientPrerender: typeof import('react-dom/static').prerender =
prerenderToNodeStream as unknown as typeof getClientPrerender
export function pipeRuntimePrefetchTransform(
stream: AnyStream,
sentinel: number,
isPartial: boolean,
staleTime: number
): AnyStream {
return safePipe(
stream as Readable,
createRuntimePrefetchNodeTransform(sentinel, isPartial, staleTime)
)
}
/** import type { PrerenderOptions, PostponedState } from 'react-dom/static'
* Compile-time switcher for stream operations. import type {
*
* PR2: Simple re-export from the web implementation.
* A future change will add a conditional branch for node streams.
*/
export type {
AnyStream, AnyStream,
ContinueFizzStreamOptions, ContinueFizzStreamOptions,
ContinueStaticPrerenderOptions, ContinueStaticPrerenderOptions,
...@@ -15,29 +10,139 @@ export type { ...@@ -15,29 +10,139 @@ export type {
FlightPayload, FlightPayload,
FlightClientModules, FlightClientModules,
FlightRenderOptions, FlightRenderOptions,
FizzStreamResult,
} from './stream-ops.web' } from './stream-ops.web'
export { export type {
continueFizzStream, AnyStream,
continueStaticPrerender, ContinueFizzStreamOptions,
continueDynamicPrerender, ContinueStaticPrerenderOptions,
continueStaticFallbackPrerender, ContinueStreamSharedOptions,
continueDynamicHTMLResume, ContinueDynamicHTMLResumeOptions,
streamToBuffer, FlightComponentMod,
chainStreams, ServerPrerenderComponentMod,
createDocumentClosingStream, FlightPayload,
processPrelude, FlightClientModules,
nodeReadableToWeb, FlightRenderOptions,
createInlinedDataStream,
createPendingStream,
createOnHeadersCallback,
resumeAndAbort,
renderToFlightStream,
streamToString,
renderToFizzStream,
resumeToFizzStream,
getServerPrerender,
getClientPrerender,
pipeRuntimePrefetchTransform,
} from './stream-ops.web' } from './stream-ops.web'
export type FizzStreamResult = {
stream: AnyStream
allReady: Promise<void>
abort?: (reason?: unknown) => void
}
type StreamOpsRuntimeModule = {
continueFizzStream: (
stream: AnyStream,
opts: ContinueFizzStreamOptions
) => Promise<AnyStream>
continueStaticPrerender: (
stream: AnyStream,
opts: ContinueStaticPrerenderOptions
) => Promise<AnyStream>
continueDynamicPrerender: (
stream: AnyStream,
opts: ContinueStreamSharedOptions
) => Promise<AnyStream>
continueStaticFallbackPrerender: (
stream: AnyStream,
opts: ContinueStaticPrerenderOptions
) => Promise<AnyStream>
continueDynamicHTMLResume: (
stream: AnyStream,
opts: ContinueDynamicHTMLResumeOptions
) => Promise<AnyStream>
streamToBuffer: (stream: AnyStream) => Promise<Buffer>
chainStreams: (...streams: Array<AnyStream>) => AnyStream
processPrelude: (unprocessedPrelude: AnyStream) => Promise<{
prelude: AnyStream
preludeIsEmpty: boolean
}>
createDocumentClosingStream: () => AnyStream
createInlinedDataStream: (
source: AnyStream,
nonce: string | undefined,
formState: unknown | null
) => AnyStream
createPendingStream: () => AnyStream
createOnHeadersCallback: (
appendHeader: (key: string, value: string) => void
) => NonNullable<PrerenderOptions['onHeaders']>
resumeAndAbort: (
element: React.ReactElement,
postponed: PostponedState | null,
opts: Record<string, any>
) => Promise<AnyStream>
renderToFlightStream: (
ComponentMod: FlightComponentMod,
payload: FlightPayload,
clientModules: FlightClientModules,
opts: FlightRenderOptions,
runInContext?: <T>(fn: () => T) => T
) => AnyStream
streamToString: (stream: AnyStream) => Promise<string>
renderToFizzStream: (
element: React.ReactElement,
streamOptions: any,
runInContext?: <T>(fn: () => T) => T
) => Promise<FizzStreamResult>
resumeToFizzStream: (
element: React.ReactElement,
postponedState: PostponedState,
streamOptions: any,
runInContext?: <T>(fn: () => T) => T
) => Promise<FizzStreamResult>
getServerPrerender: (
ComponentMod: ServerPrerenderComponentMod
) => (...args: any[]) => any
getClientPrerender: typeof import('react-dom/static').prerender
pipeRuntimePrefetchTransform: (
stream: AnyStream,
sentinel: number,
isPartial: boolean,
staleTime: number
) => AnyStream
nodeReadableToWeb:
| ((readable: import('node:stream').Readable) => ReadableStream<Uint8Array>)
| undefined
}
let streamOpsRuntimeModule: StreamOpsRuntimeModule
if (process.env.__NEXT_USE_NODE_STREAMS) {
streamOpsRuntimeModule =
require('./stream-ops.node') as typeof import('./stream-ops.node') as unknown as StreamOpsRuntimeModule
} else {
streamOpsRuntimeModule =
require('./stream-ops.web') as typeof import('./stream-ops.web') as unknown as StreamOpsRuntimeModule
}
export const continueFizzStream = streamOpsRuntimeModule.continueFizzStream
export const continueStaticPrerender =
streamOpsRuntimeModule.continueStaticPrerender
export const continueDynamicPrerender =
streamOpsRuntimeModule.continueDynamicPrerender
export const continueStaticFallbackPrerender =
streamOpsRuntimeModule.continueStaticFallbackPrerender
export const continueDynamicHTMLResume =
streamOpsRuntimeModule.continueDynamicHTMLResume
export const streamToBuffer = streamOpsRuntimeModule.streamToBuffer
export const chainStreams = streamOpsRuntimeModule.chainStreams
export const processPrelude = streamOpsRuntimeModule.processPrelude
export const createDocumentClosingStream =
streamOpsRuntimeModule.createDocumentClosingStream
export const createInlinedDataStream =
streamOpsRuntimeModule.createInlinedDataStream
export const createPendingStream = streamOpsRuntimeModule.createPendingStream
export const createOnHeadersCallback =
streamOpsRuntimeModule.createOnHeadersCallback
export const resumeAndAbort = streamOpsRuntimeModule.resumeAndAbort
export const renderToFlightStream = streamOpsRuntimeModule.renderToFlightStream
export const streamToString = streamOpsRuntimeModule.streamToString
export const renderToFizzStream = streamOpsRuntimeModule.renderToFizzStream
export const resumeToFizzStream = streamOpsRuntimeModule.resumeToFizzStream
export const getServerPrerender = streamOpsRuntimeModule.getServerPrerender
export const getClientPrerender = streamOpsRuntimeModule.getClientPrerender
export const pipeRuntimePrefetchTransform =
streamOpsRuntimeModule.pipeRuntimePrefetchTransform
export const nodeReadableToWeb = streamOpsRuntimeModule.nodeReadableToWeb
import type { BinaryStreamOf } from './app-render' import type { BinaryStreamOf } from './app-render'
import type { Readable } from 'node:stream' import type { Readable, Transform } from 'node:stream'
import { htmlEscapeJsonString } from '../htmlescape' import { htmlEscapeJsonString } from '../htmlescape'
import { workUnitAsyncStorage } from './work-unit-async-storage.external' import { workUnitAsyncStorage } from './work-unit-async-storage.external'
...@@ -14,10 +14,18 @@ const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2 ...@@ -14,10 +14,18 @@ const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2
const INLINE_FLIGHT_PAYLOAD_BINARY = 3 const INLINE_FLIGHT_PAYLOAD_BINARY = 3
const flightResponses = new WeakMap< const flightResponses = new WeakMap<
Readable | BinaryStreamOf<any>, Readable | BinaryStreamOf<unknown>,
Promise<any> Promise<unknown>
>() >()
const encoder = new TextEncoder() const encoder = new TextEncoder()
const INLINE_FLIGHT_PAYLOAD_SUFFIX = ')</script>'
const INLINE_FLIGHT_PAYLOAD_BINARY_PREFIX = `[${INLINE_FLIGHT_PAYLOAD_BINARY},"`
const INLINE_FLIGHT_PAYLOAD_BINARY_SUFFIX = '"]'
const INLINE_FLIGHT_PAYLOAD_BINARY_PREFIX_LENGTH =
INLINE_FLIGHT_PAYLOAD_BINARY_PREFIX.length
const INLINE_FLIGHT_PAYLOAD_BINARY_SUFFIX_LENGTH =
INLINE_FLIGHT_PAYLOAD_BINARY_SUFFIX.length
const BINARY_TO_BASE64_CHUNK_SIZE = 0x8000
const findSourceMapURL = const findSourceMapURL =
process.env.NODE_ENV !== 'production' process.env.NODE_ENV !== 'production'
...@@ -38,7 +46,7 @@ export function getFlightStream<T>( ...@@ -38,7 +46,7 @@ export function getFlightStream<T>(
const response = flightResponses.get(flightStream) const response = flightResponses.get(flightStream)
if (response) { if (response) {
return response return response as Promise<T>
} }
const { moduleLoading, edgeSSRModuleMapping, ssrModuleMapping } = const { moduleLoading, edgeSSRModuleMapping, ssrModuleMapping } =
...@@ -76,9 +84,20 @@ export function getFlightStream<T>( ...@@ -76,9 +84,20 @@ export function getFlightStream<T>(
const { Readable } = const { Readable } =
require('node:stream') as typeof import('node:stream') require('node:stream') as typeof import('node:stream')
// The types of flightStream and debugStream should match. // When using native Node debug channels, the debug stream arrives
if (debugStream && !(debugStream instanceof Readable)) { // as a Node Readable directly. Fall back to Readable.fromWeb() if
throw new InvariantError('Expected debug stream to be a Readable') // a web ReadableStream is passed (e.g. non-Node-stream path).
let nodeDebugStream: import('node:stream').Readable | undefined
if (debugStream) {
if (debugStream instanceof Readable) {
nodeDebugStream = debugStream
} else {
// Node's fromWeb() overload expects stream/web.ReadableStream.
// Convert from the global ReadableStream type to satisfy that overload.
nodeDebugStream = Readable.fromWeb(
debugStream as import('stream/web').ReadableStream<Uint8Array>
)
}
} }
// react-server-dom-webpack/client.edge must not be hoisted for require cache clearing to work correctly // react-server-dom-webpack/client.edge must not be hoisted for require cache clearing to work correctly
...@@ -96,7 +115,7 @@ export function getFlightStream<T>( ...@@ -96,7 +115,7 @@ export function getFlightStream<T>(
{ {
findSourceMapURL, findSourceMapURL,
nonce, nonce,
debugChannel: debugStream, debugChannel: nodeDebugStream,
endTime: debugEndTime, endTime: debugEndTime,
} }
) )
...@@ -127,6 +146,7 @@ export function getFlightStream<T>( ...@@ -127,6 +146,7 @@ export function getFlightStream<T>(
case 'prerender-ppr': case 'prerender-ppr':
case 'prerender-legacy': case 'prerender-legacy':
case 'request': case 'request':
break
case 'cache': case 'cache':
case 'private-cache': case 'private-cache':
case 'unstable-cache': case 'unstable-cache':
...@@ -212,44 +232,182 @@ function writeInitialInstructions( ...@@ -212,44 +232,182 @@ function writeInitialInstructions(
scriptStart: string, scriptStart: string,
formState: unknown | null formState: unknown | null
) { ) {
let scriptContents = `(self.__next_f=self.__next_f||[]).push(${htmlEscapeJsonString( controller.enqueue(encodeInitialFlightData(scriptStart, formState))
JSON.stringify([INLINE_FLIGHT_PAYLOAD_BOOTSTRAP]) }
)})`
if (formState != null) { function encodeBase64Chunk(chunk: Uint8Array): string {
scriptContents += `;self.__next_f.push(${htmlEscapeJsonString( if (typeof Buffer !== 'undefined') {
JSON.stringify([INLINE_FLIGHT_PAYLOAD_FORM_STATE, formState]) if (Buffer.isBuffer(chunk)) {
)})` return chunk.toString('base64')
}
return Buffer.from(
chunk.buffer,
chunk.byteOffset,
chunk.byteLength
).toString('base64')
} }
controller.enqueue(encoder.encode(`${scriptStart}${scriptContents}</script>`)) const parts: string[] = []
for (let i = 0; i < chunk.length; i += BINARY_TO_BASE64_CHUNK_SIZE) {
const slice = chunk.subarray(i, i + BINARY_TO_BASE64_CHUNK_SIZE)
parts.push(String.fromCharCode(...slice))
}
return btoa(parts.join(''))
} }
function writeFlightDataInstruction( function encodeFlightPayload(chunk: string | Uint8Array): string {
controller: ReadableStreamDefaultController, if (typeof chunk === 'string') {
scriptStart: string, return htmlEscapeJsonString(
chunk: string | Uint8Array `[${INLINE_FLIGHT_PAYLOAD_DATA},${JSON.stringify(chunk)}]`
) { )
let htmlInlinedData: string }
// Binary payloads are base64 and cannot include script-breaking tokens.
const base64 = encodeBase64Chunk(chunk)
return `${INLINE_FLIGHT_PAYLOAD_BINARY_PREFIX}${base64}${INLINE_FLIGHT_PAYLOAD_BINARY_SUFFIX}`
}
function encodeFlightDataChunkNode(
scriptPrefix: Buffer,
scriptSuffix: Buffer,
chunk: string | Uint8Array
): Buffer {
if (typeof chunk === 'string') { if (typeof chunk === 'string') {
htmlInlinedData = htmlEscapeJsonString( const payload = htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunk]) `[${INLINE_FLIGHT_PAYLOAD_DATA},${JSON.stringify(chunk)}]`
) )
} else { const payloadByteLength = Buffer.byteLength(payload)
// The chunk cannot be embedded as a UTF-8 string in the script tag. const chunkBuffer = Buffer.allocUnsafe(
// Instead let's inline it in base64. scriptPrefix.length + payloadByteLength + scriptSuffix.length
// Credits to Devon Govett (devongovett) for the technique.
// https://github.com/devongovett/rsc-html-stream
const base64 = btoa(String.fromCodePoint(...chunk))
htmlInlinedData = htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_BINARY, base64])
) )
let offset = scriptPrefix.copy(chunkBuffer, 0)
offset += chunkBuffer.write(payload, offset, payloadByteLength, 'utf8')
scriptSuffix.copy(chunkBuffer, offset)
return chunkBuffer
} }
const base64 = encodeBase64Chunk(chunk)
const chunkBuffer = Buffer.allocUnsafe(
scriptPrefix.length +
INLINE_FLIGHT_PAYLOAD_BINARY_PREFIX_LENGTH +
base64.length +
INLINE_FLIGHT_PAYLOAD_BINARY_SUFFIX_LENGTH +
scriptSuffix.length
)
let offset = scriptPrefix.copy(chunkBuffer, 0)
offset += chunkBuffer.write(
INLINE_FLIGHT_PAYLOAD_BINARY_PREFIX,
offset,
'ascii'
)
offset += chunkBuffer.write(base64, offset, 'ascii')
offset += chunkBuffer.write(
INLINE_FLIGHT_PAYLOAD_BINARY_SUFFIX,
offset,
'ascii'
)
scriptSuffix.copy(chunkBuffer, offset)
return chunkBuffer
}
function writeFlightDataInstruction(
controller: ReadableStreamDefaultController,
scriptStart: string,
chunk: string | Uint8Array
) {
const htmlInlinedData = encodeFlightPayload(chunk)
controller.enqueue( controller.enqueue(
encoder.encode( encoder.encode(
`${scriptStart}self.__next_f.push(${htmlInlinedData})</script>` `${scriptStart}self.__next_f.push(${htmlInlinedData}${INLINE_FLIGHT_PAYLOAD_SUFFIX}`
) )
) )
} }
function encodeInitialFlightData(
scriptStart: string,
formState: unknown | null
): Uint8Array {
let scriptContents = `(self.__next_f=self.__next_f||[]).push(${htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_BOOTSTRAP])
)})`
if (formState != null) {
scriptContents += `;self.__next_f.push(${htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_FORM_STATE, formState])
)})`
}
return encoder.encode(`${scriptStart}${scriptContents}</script>`)
}
/**
* Creates a Node.js Readable that provides inline script tag chunks for writing
* hydration data to the client outside the React render itself.
*
* This is the Node.js stream equivalent of createInlinedDataReadableStream.
*/
export function createInlinedDataNodeStream(
nonce: string | undefined,
formState: unknown | null
): Transform {
if (process.env.NEXT_RUNTIME === 'edge') {
throw new Error(
'createInlinedDataNodeStream is not supported in edge runtime'
)
} else {
const startScriptTag = nonce
? `<script nonce=${JSON.stringify(nonce)}>`
: '<script>'
const scriptPrefix = Buffer.from(`${startScriptTag}self.__next_f.push(`)
const scriptSuffix = Buffer.from(INLINE_FLIGHT_PAYLOAD_SUFFIX)
const decoder = new TextDecoder('utf-8', { fatal: true })
let bootstrapWritten = false
const { Transform: NodeTransform } =
require('node:stream') as typeof import('node:stream')
return new NodeTransform({
transform(
chunk: Uint8Array,
_encoding: string,
callback: (error?: Error) => void
) {
try {
if (!bootstrapWritten) {
bootstrapWritten = true
this.push(encodeInitialFlightData(startScriptTag, formState))
}
try {
const decodedString = decoder.decode(chunk, { stream: true })
this.push(
encodeFlightDataChunkNode(
scriptPrefix,
scriptSuffix,
decodedString
)
)
} catch {
this.push(
encodeFlightDataChunkNode(scriptPrefix, scriptSuffix, chunk)
)
}
callback()
} catch (error) {
callback(error as Error)
}
},
flush(callback: (error?: Error) => void) {
// If no data was ever received, still write the bootstrap
if (!bootstrapWritten) {
this.push(encodeInitialFlightData(startScriptTag, formState))
}
callback()
},
})
}
}
...@@ -336,10 +336,18 @@ export const experimentalSchema = { ...@@ -336,10 +336,18 @@ export const experimentalSchema = {
turbopackClientSideNestedAsyncChunking: z.boolean().optional(), turbopackClientSideNestedAsyncChunking: z.boolean().optional(),
turbopackServerSideNestedAsyncChunking: z.boolean().optional(), turbopackServerSideNestedAsyncChunking: z.boolean().optional(),
turbopackImportTypeBytes: z.boolean().optional(), turbopackImportTypeBytes: z.boolean().optional(),
turbopackImportTypeText: z.boolean().optional(),
turbopackUseBuiltinBabel: z.boolean().optional(), turbopackUseBuiltinBabel: z.boolean().optional(),
turbopackUseBuiltinSass: z.boolean().optional(), turbopackUseBuiltinSass: z.boolean().optional(),
turbopackModuleIds: z.enum(['named', 'deterministic']).optional(), turbopackModuleIds: z.enum(['named', 'deterministic']).optional(),
turbopackIgnoreIssue: z
.array(
z.object({
path: z.union([z.string(), z.instanceof(RegExp)]),
title: z.union([z.string(), z.instanceof(RegExp)]).optional(),
description: z.union([z.string(), z.instanceof(RegExp)]).optional(),
})
)
.optional(),
turbopackInferModuleSideEffects: z.boolean().optional(), turbopackInferModuleSideEffects: z.boolean().optional(),
optimizePackageImports: z.array(z.string()).optional(), optimizePackageImports: z.array(z.string()).optional(),
optimizeServerReact: z.boolean().optional(), optimizeServerReact: z.boolean().optional(),
...@@ -361,6 +369,7 @@ export const experimentalSchema = { ...@@ -361,6 +369,7 @@ export const experimentalSchema = {
serverComponentsHmrCache: z.boolean().optional(), serverComponentsHmrCache: z.boolean().optional(),
authInterrupts: z.boolean().optional(), authInterrupts: z.boolean().optional(),
useCache: z.boolean().optional(), useCache: z.boolean().optional(),
useNodeStreams: z.boolean().optional(),
slowModuleDetection: z slowModuleDetection: z
.object({ .object({
buildTimeThresholdMs: z.number().int(), buildTimeThresholdMs: z.number().int(),
......
...@@ -148,7 +148,6 @@ export type TurbopackModuleType = ...@@ -148,7 +148,6 @@ export type TurbopackModuleType =
| 'raw' | 'raw'
| 'node' | 'node'
| 'bytes' | 'bytes'
| 'text'
export type TurbopackRuleConfigItem = { export type TurbopackRuleConfigItem = {
/** Loaders to apply to matched files. */ /** Loaders to apply to matched files. */
...@@ -507,11 +506,6 @@ export interface ExperimentalConfig { ...@@ -507,11 +506,6 @@ export interface ExperimentalConfig {
*/ */
turbopackImportTypeBytes?: boolean turbopackImportTypeBytes?: boolean
/**
* Enable support for `with {type: "text"}` for ESM imports.
*/
turbopackImportTypeText?: boolean
/** /**
* Enable scope hoisting. Defaults to true in build mode. Always disabled in development mode. * Enable scope hoisting. Defaults to true in build mode. Always disabled in development mode.
*/ */
...@@ -601,6 +595,19 @@ export interface ExperimentalConfig { ...@@ -601,6 +595,19 @@ export interface ExperimentalConfig {
*/ */
turbopackModuleIds?: 'named' | 'deterministic' turbopackModuleIds?: 'named' | 'deterministic'
/**
* Filter out specific Turbopack errors and warnings so they do not appear
* in the CLI output or the error overlay. String values for `path` are
* glob patterns; string values for `title`/`description` are exact matches.
* RegExp values match anywhere within the string (use `^` and `$` anchors
* for full-string matching).
*/
turbopackIgnoreIssue?: Array<{
path: string | RegExp
title?: string | RegExp
description?: string | RegExp
}>
/** /**
* For use with `@next/mdx`. Compile MDX files using the new Rust compiler. * For use with `@next/mdx`. Compile MDX files using the new Rust compiler.
* @see https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs * @see https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs
...@@ -836,6 +843,13 @@ export interface ExperimentalConfig { ...@@ -836,6 +843,13 @@ export interface ExperimentalConfig {
*/ */
useCache?: boolean useCache?: boolean
/**
* Use Node.js native streams instead of web streams for the App Router
* rendering pipeline on the Node.js runtime. This can improve performance
* by avoiding the overhead of web stream wrappers.
*/
useNodeStreams?: boolean
/** /**
* Enables detection and reporting of slow modules during development builds. * Enables detection and reporting of slow modules during development builds.
* Enabling this may impact build performance to ensure accurate measurements. * Enabling this may impact build performance to ensure accurate measurements.
...@@ -1811,6 +1825,7 @@ export interface NextConfigRuntime { ...@@ -1811,6 +1825,7 @@ export interface NextConfigRuntime {
| 'maxPostponedStateSize' | 'maxPostponedStateSize'
| 'devCacheControlNoCache' | 'devCacheControlNoCache'
| 'exposeTestingApiInProductionBuild' | 'exposeTestingApiInProductionBuild'
| 'useNodeStreams'
> & { > & {
// Pick on @internal fields generates invalid .d.ts files // Pick on @internal fields generates invalid .d.ts files
/** @internal */ /** @internal */
...@@ -1875,6 +1890,7 @@ export function getNextConfigRuntime( ...@@ -1875,6 +1890,7 @@ export function getNextConfigRuntime(
maxPostponedStateSize: ex.maxPostponedStateSize, maxPostponedStateSize: ex.maxPostponedStateSize,
devCacheControlNoCache: ex.devCacheControlNoCache, devCacheControlNoCache: ex.devCacheControlNoCache,
exposeTestingApiInProductionBuild: ex.exposeTestingApiInProductionBuild, exposeTestingApiInProductionBuild: ex.exposeTestingApiInProductionBuild,
useNodeStreams: ex.useNodeStreams,
trustHostHeader: ex.trustHostHeader, trustHostHeader: ex.trustHostHeader,
isExperimentalCompile: ex.isExperimentalCompile, isExperimentalCompile: ex.isExperimentalCompile,
......
...@@ -1932,6 +1932,25 @@ function enforceExperimentalFeatures( ...@@ -1932,6 +1932,25 @@ function enforceExperimentalFeatures(
} }
} }
// Enable node streams via env var (for CI testing).
if (
process.env.__NEXT_USE_NODE_STREAMS === 'true' &&
(config.experimental.useNodeStreams === undefined ||
(isDefaultConfig && !config.experimental.useNodeStreams))
) {
config.experimental.useNodeStreams = true
}
// Keep runtime bundle selection env in sync with the resolved config.
// Explicit user config (e.g. useNodeStreams: false) should win over an
// inherited shell env var to avoid selecting nodestream runtime bundles
// while define-env compiled user bundles with node streams disabled.
if (config.experimental.useNodeStreams) {
process.env.__NEXT_USE_NODE_STREAMS = 'true'
} else {
delete process.env.__NEXT_USE_NODE_STREAMS
}
// TODO: Remove this once using the debug channel is the default. // TODO: Remove this once using the debug channel is the default.
if ( if (
process.env.__NEXT_EXPERIMENTAL_DEBUG_CHANNEL === 'true' && process.env.__NEXT_EXPERIMENTAL_DEBUG_CHANNEL === 'true' &&
......
...@@ -216,6 +216,9 @@ export default class NextNodeServer extends BaseServer< ...@@ -216,6 +216,9 @@ export default class NextNodeServer extends BaseServer<
if (this.renderOpts.nextScriptWorkers) { if (this.renderOpts.nextScriptWorkers) {
process.env.__NEXT_SCRIPT_WORKERS = JSON.stringify(true) process.env.__NEXT_SCRIPT_WORKERS = JSON.stringify(true)
} }
if (this.nextConfig.experimental.useNodeStreams) {
process.env.__NEXT_USE_NODE_STREAMS = 'true'
}
if (!this.minimalMode) { if (!this.minimalMode) {
this.imageResponseCache = new ResponseCache(this.minimalMode) this.imageResponseCache = new ResponseCache(this.minimalMode)
......
import { Readable } from 'node:stream'
import { EventEmitter } from 'node:events'
import type { ServerResponse } from 'node:http'
// Enable the node streams code path. pipeNodeReadableToResponse is gated
// behind this env var and is a no-op without it.
process.env.__NEXT_USE_NODE_STREAMS = 'true'
import { pipeNodeReadableToResponse } from './pipe-readable'
/**
* Creates a minimal mock ServerResponse backed by EventEmitter so tests can
* manually emit 'drain' and 'close' events to simulate backpressure and
* client disconnects.
*
* @param opts.backpressureAfter - After this many write() calls, write()
* starts returning false to simulate kernel buffer full / TCP backpressure.
* Callers must emit 'drain' on the mock to unblock the pipe.
*/
function createMockResponse(opts: { backpressureAfter: number }) {
const ee = new EventEmitter()
const { backpressureAfter } = opts
let writeCount = 0
let _destroyed = false
const mock = Object.assign(ee, {
write(_chunk: any): boolean {
writeCount++
// Returning false tells the caller to stop writing until 'drain' fires.
// This mirrors real ServerResponse behavior when the socket buffer is full.
return writeCount < backpressureAfter
},
end() {},
destroy(_err?: Error) {
_destroyed = true
mock.emit('close')
},
flushHeaders() {},
flush() {},
get errored() {
return null
},
get destroyed() {
return _destroyed
},
get writableFinished() {
return false
},
})
return mock as typeof mock & ServerResponse
}
describe('pipeNodeReadableToResponse', () => {
// Regression test for the backpressure + disconnect hang.
//
// Reproduction sequence:
// 1. Readable pushes a chunk, res.write() returns false (backpressure)
// 2. The writable registers res.once('drain', callback) to resume
// 3. Client disconnects: res emits 'close' but never emits 'drain'
// 4. Without the fix, the orphaned drain callback means the writable's
// destroy() handler never fires, so finished.resolve() is never
// called, and `await finished.promise` hangs forever.
//
// The fix: onClose destroys BOTH readable and writable. Destroying the
// writable triggers its destroy() handler which calls finished.resolve().
it('resolves when client disconnects during backpressure (no hang)', async () => {
const readable = Readable.from([
Buffer.from('chunk1'),
Buffer.from('chunk2'),
Buffer.from('chunk3'),
])
// write() returns false on the very first call to immediately trigger backpressure
const res = createMockResponse({ backpressureAfter: 1 })
const pipePromise = pipeNodeReadableToResponse(readable, res as any)
// Let the pipe start and hit backpressure (waiting on 'drain')
await new Promise((r) => setTimeout(r, 50))
// Client disconnects without ever emitting 'drain'
res.emit('close')
// Race against a 2s timeout. If the bug is present, pipePromise never
// resolves and we get 'timeout' instead of 'resolved'.
const timeout = new Promise<'timeout'>((r) =>
setTimeout(() => r('timeout'), 2000)
)
const result = await Promise.race([
pipePromise.then(() => 'resolved' as const),
timeout,
])
expect(result).toBe('resolved')
})
})
...@@ -144,3 +144,157 @@ export async function pipeToNodeResponse( ...@@ -144,3 +144,157 @@ export async function pipeToNodeResponse(
throw new Error('failed to pipe response', { cause: err }) throw new Error('failed to pipe response', { cause: err })
} }
} }
/**
* Pipes a Node.js Readable stream directly to a ServerResponse.
* This avoids the overhead of wrapping in web WritableStream.
*/
export async function pipeNodeReadableToResponse(
readable: import('node:stream').Readable,
res: ServerResponse,
waitUntilForEnd?: Promise<unknown>
) {
// Guard so webpack can DCE node:stream require when the node-streams
// runtime path is disabled.
if (process.env.__NEXT_USE_NODE_STREAMS) {
try {
const { errored, destroyed } = res
if (errored || destroyed) return
let started = false
const finished = new DetachedPromise<void>()
const { Writable: NodeWritable } =
require('node:stream') as typeof import('node:stream')
const writable = new NodeWritable({
write(chunk: Uint8Array, _encoding, callback) {
if (!started) {
started = true
if (
'performance' in globalThis &&
process.env.NEXT_OTEL_PERFORMANCE_PREFIX
) {
const metrics = getClientComponentLoaderMetrics()
if (metrics) {
performance.measure(
`${process.env.NEXT_OTEL_PERFORMANCE_PREFIX}:next-client-component-loading`,
{
start: metrics.clientComponentLoadStart,
end:
metrics.clientComponentLoadStart +
metrics.clientComponentLoadTimes,
}
)
}
}
res.flushHeaders()
getTracer().trace(
NextNodeServerSpan.startResponse,
{
spanName: 'start response',
},
() => undefined
)
}
try {
const ok = res.write(chunk)
// Added by the `compression` middleware, this is a function that will
// flush the partially-compressed response to the client.
if ('flush' in res && typeof res.flush === 'function') {
res.flush()
}
if (!ok) {
res.once('drain', callback)
} else {
callback()
}
} catch (err) {
if (!res.writableFinished) {
res.end()
}
// Destroy the readable to stop it from pushing more data
if (!readable.destroyed) {
readable.destroy()
}
callback(
new Error('failed to write chunk to response', { cause: err })
)
}
},
final(callback) {
if (waitUntilForEnd) {
const finalize = () => {
if (!res.writableFinished) {
res.end()
}
callback()
finished.resolve()
}
waitUntilForEnd.then(finalize, finalize)
} else {
if (!res.writableFinished) {
res.end()
}
callback()
finished.resolve()
}
},
destroy(err, callback) {
if (err && !res.writableFinished) {
res.destroy(err)
}
callback(err)
finished.resolve()
},
})
// Handle client disconnect.
//
// When the client disconnects during backpressure, the failure chain is:
// 1. res.write(chunk) returned false (backpressure)
// 2. res.once('drain', callback) was registered to resume writing
// 3. Client disconnects, res emits 'close'
// 4. The 'drain' event will never fire on the closed response
//
// Node.js .pipe() does NOT propagate source destruction to the
// destination, so destroying the readable alone is not enough:
// the writable would stay alive waiting for a drain that never comes,
// and finished.promise (awaited below) would never resolve.
//
// Destroying the writable triggers its destroy() handler above,
// which calls finished.resolve() and unblocks the await.
const onClose = () => {
if (!readable.destroyed) {
readable.destroy()
}
if (!writable.destroyed) {
writable.destroy()
}
}
res.once('close', onClose)
// Forward errors since Node.js .pipe() does not propagate them.
readable.on('error', (err) => {
if (!writable.destroyed) {
writable.destroy(err)
}
})
readable.pipe(writable)
await finished.promise
res.off('close', onClose)
} catch (err: any) {
if (isAbortError(err)) return
throw new Error('failed to pipe node readable to response', {
cause: err,
})
}
}
}
if (process.env.NEXT_RUNTIME === 'edge') { if (process.env.NEXT_RUNTIME === 'edge') {
module.exports = require('next/dist/server/route-modules/app-page/module.js') module.exports = require('next/dist/server/route-modules/app-page/module.js')
} else { } else {
// Keep this as explicit nested conditionals with branch-local `require()` calls.
// Collapsing this into a map/object with eager requires pulls all runtime
// variants into the module graph and breaks env-based dead-code elimination.
if (process.env.__NEXT_EXPERIMENTAL_REACT) { if (process.env.__NEXT_EXPERIMENTAL_REACT) {
if (process.env.__NEXT_USE_NODE_STREAMS) {
if (process.env.NODE_ENV === 'development') {
if (process.env.TURBOPACK) {
module.exports = require('next/dist/compiled/next-server/app-page-turbo-experimental-nodestreams.runtime.dev.js')
} else {
module.exports = require('next/dist/compiled/next-server/app-page-experimental-nodestreams.runtime.dev.js')
}
} else {
if (process.env.TURBOPACK) {
module.exports = require('next/dist/compiled/next-server/app-page-turbo-experimental-nodestreams.runtime.prod.js')
} else {
module.exports = require('next/dist/compiled/next-server/app-page-experimental-nodestreams.runtime.prod.js')
}
}
} else {
if (process.env.NODE_ENV === 'development') {
if (process.env.TURBOPACK) {
module.exports = require('next/dist/compiled/next-server/app-page-turbo-experimental.runtime.dev.js')
} else {
module.exports = require('next/dist/compiled/next-server/app-page-experimental.runtime.dev.js')
}
} else {
if (process.env.TURBOPACK) {
module.exports = require('next/dist/compiled/next-server/app-page-turbo-experimental.runtime.prod.js')
} else {
module.exports = require('next/dist/compiled/next-server/app-page-experimental.runtime.prod.js')
}
}
}
} else if (process.env.__NEXT_USE_NODE_STREAMS) {
if (process.env.NODE_ENV === 'development') { if (process.env.NODE_ENV === 'development') {
if (process.env.TURBOPACK) { if (process.env.TURBOPACK) {
module.exports = require('next/dist/compiled/next-server/app-page-turbo-experimental.runtime.dev.js') module.exports = require('next/dist/compiled/next-server/app-page-turbo-nodestreams.runtime.dev.js')
} else { } else {
module.exports = require('next/dist/compiled/next-server/app-page-experimental.runtime.dev.js') module.exports = require('next/dist/compiled/next-server/app-page-nodestreams.runtime.dev.js')
} }
} else { } else {
if (process.env.TURBOPACK) { if (process.env.TURBOPACK) {
module.exports = require('next/dist/compiled/next-server/app-page-turbo-experimental.runtime.prod.js') module.exports = require('next/dist/compiled/next-server/app-page-turbo-nodestreams.runtime.prod.js')
} else { } else {
module.exports = require('next/dist/compiled/next-server/app-page-experimental.runtime.prod.js') module.exports = require('next/dist/compiled/next-server/app-page-nodestreams.runtime.prod.js')
} }
} }
} else { } else {
......
import type { Readable as NodeReadable } from 'node:stream'
export function chainNodeStreams(...streams: NodeReadable[]): NodeReadable {
if (!process.env.__NEXT_USE_NODE_STREAMS) {
throw new Error(
'Invariant: Node.js stream chaining is only available in node streams mode'
)
} else {
const { chainNodeStreams: chainNodeStreamsImpl } =
require('./node-stream-helpers') as typeof import('./node-stream-helpers')
return chainNodeStreamsImpl(...streams)
}
}
This diff is collapsed.
...@@ -2957,6 +2957,119 @@ export async function next_bundle_app_dev_experimental(task, opts) { ...@@ -2957,6 +2957,119 @@ export async function next_bundle_app_dev_experimental(task, opts) {
}) })
} }
// Experimental + node streams variants
export async function next_bundle_app_prod_turbo_experimental_nodestreams(
task,
opts
) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
turbo: true,
bundleType: 'app',
experimental: true,
nodeStreams: true,
}),
name: 'next-bundle-app-prod-turbo-experimental-nodestreams',
})
}
export async function next_bundle_app_prod_experimental_nodestreams(
task,
opts
) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
dev: false,
bundleType: 'app',
experimental: true,
nodeStreams: true,
}),
name: 'next-bundle-app-prod-experimental-nodestreams',
})
}
export async function next_bundle_app_dev_turbo_experimental_nodestreams(
task,
opts
) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
dev: true,
turbo: true,
bundleType: 'app',
experimental: true,
nodeStreams: true,
}),
name: 'next-bundle-app-dev-turbo-experimental-nodestreams',
})
}
export async function next_bundle_app_dev_experimental_nodestreams(task, opts) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
dev: true,
bundleType: 'app',
experimental: true,
nodeStreams: true,
}),
name: 'next-bundle-app-dev-experimental-nodestreams',
})
}
// Node streams variants
export async function next_bundle_app_prod_turbo_nodestreams(task, opts) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
turbo: true,
bundleType: 'app',
nodeStreams: true,
}),
name: 'next-bundle-app-prod-turbo-nodestreams',
})
}
export async function next_bundle_app_prod_nodestreams(task, opts) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
dev: false,
bundleType: 'app',
nodeStreams: true,
}),
name: 'next-bundle-app-prod-nodestreams',
})
}
export async function next_bundle_app_dev_turbo_nodestreams(task, opts) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
dev: true,
turbo: true,
bundleType: 'app',
nodeStreams: true,
}),
name: 'next-bundle-app-dev-turbo-nodestreams',
})
}
export async function next_bundle_app_dev_nodestreams(task, opts) {
await task.source('dist').webpack({
watch: opts.dev,
config: require('./next-runtime.webpack-config')({
dev: true,
bundleType: 'app',
nodeStreams: true,
}),
name: 'next-bundle-app-dev-nodestreams',
})
}
export async function next_bundle_pages_prod(task, opts) { export async function next_bundle_pages_prod(task, opts) {
await task.source('dist').webpack({ await task.source('dist').webpack({
watch: opts.dev, watch: opts.dev,
...@@ -3036,6 +3149,16 @@ export async function next_bundle(task, opts) { ...@@ -3036,6 +3149,16 @@ export async function next_bundle(task, opts) {
'next_bundle_app_prod_experimental', 'next_bundle_app_prod_experimental',
'next_bundle_app_dev_turbo_experimental', 'next_bundle_app_dev_turbo_experimental',
'next_bundle_app_dev_experimental', 'next_bundle_app_dev_experimental',
// builds the app (route/page) bundles with experimental react + node streams
'next_bundle_app_prod_turbo_experimental_nodestreams',
'next_bundle_app_prod_experimental_nodestreams',
'next_bundle_app_dev_turbo_experimental_nodestreams',
'next_bundle_app_dev_experimental_nodestreams',
// builds the app (route/page) bundles with node streams
'next_bundle_app_prod_turbo_nodestreams',
'next_bundle_app_prod_nodestreams',
'next_bundle_app_dev_turbo_nodestreams',
'next_bundle_app_dev_nodestreams',
// builds the pages (page/api) bundles // builds the pages (page/api) bundles
'next_bundle_pages_prod', 'next_bundle_pages_prod',
'next_bundle_pages_dev', 'next_bundle_pages_dev',
......
...@@ -114,7 +114,9 @@ declare module 'react-server-dom-webpack/client.browser' { ...@@ -114,7 +114,9 @@ declare module 'react-server-dom-webpack/client.browser' {
findSourceMapURL: FindSourceMapURLCallback | undefined findSourceMapURL: FindSourceMapURLCallback | undefined
replayConsoleLogs?: boolean replayConsoleLogs?: boolean
temporaryReferences?: TemporaryReferenceSet temporaryReferences?: TemporaryReferenceSet
debugChannel?: { readable?: ReadableStream; writable?: WritableStream } debugChannel?:
| { readable?: ReadableStream; writable?: WritableStream }
| import('node:stream').Writable
startTime?: number startTime?: number
endTime?: number endTime?: number
unstable_allowPartialStream?: boolean unstable_allowPartialStream?: boolean
...@@ -169,7 +171,9 @@ declare module 'react-server-dom-webpack/server.edge' { ...@@ -169,7 +171,9 @@ declare module 'react-server-dom-webpack/server.edge' {
| undefined | undefined
onError?: (error: unknown) => void onError?: (error: unknown) => void
signal?: AbortSignal signal?: AbortSignal
debugChannel?: { readable?: ReadableStream; writable?: WritableStream } debugChannel?:
| { readable?: ReadableStream; writable?: WritableStream }
| import('node:stream').Writable
startTime?: number startTime?: number
} }
): ReadableStream<Uint8Array> ): ReadableStream<Uint8Array>
...@@ -225,6 +229,35 @@ declare module 'react-server-dom-webpack/server.node' { ...@@ -225,6 +229,35 @@ declare module 'react-server-dom-webpack/server.node' {
renderToReadableStream, renderToReadableStream,
} from 'react-server-dom-webpack/server.edge' } from 'react-server-dom-webpack/server.edge'
export function renderToPipeableStream(
model: any,
webpackMap: import('react-server-dom-webpack/server.edge').ClientManifest,
options?: {
temporaryReferences?: import('react-server-dom-webpack/server.edge').TemporaryReferenceSet
environmentName?: string | (() => string)
filterStackFrame:
| ((
url: string,
functionName: string,
lineNumber: number,
columnNumber: number
) => boolean)
| undefined
onError?: (error: unknown) => void
signal?: AbortSignal
// React's Node API expects debugChannel to be a Node.js Writable
// (has .write()), Duplex (has .read()), or WebSocket (has .send()).
// This differs from the web API which expects { readable?, writable? }.
debugChannel?: import('node:stream').Writable
startTime?: number
}
): {
pipe<Writable extends NodeJS.WritableStream>(
destination: Writable
): Writable
abort(reason?: unknown): void
}
export type TemporaryReferenceSet = WeakMap<any, string> export type TemporaryReferenceSet = WeakMap<any, string>
export type ImportManifestEntry = { export type ImportManifestEntry = {
...@@ -307,6 +340,35 @@ declare module 'react-server-dom-webpack/static' { ...@@ -307,6 +340,35 @@ declare module 'react-server-dom-webpack/static' {
): Promise<{ ): Promise<{
prelude: ReadableStream<Uint8Array> prelude: ReadableStream<Uint8Array>
}> }>
export function prerenderToNodeStream(
children: any,
webpackMap: {
readonly [id: string]: {
readonly id: string | number
readonly chunks: ReadonlyArray<string>
readonly name: string
readonly async?: boolean
}
},
options?: {
environmentName?: string | (() => string)
filterStackFrame:
| ((
url: string,
functionName: string,
lineNumber: number,
columnNumber: number
) => boolean)
| undefined
identifierPrefix?: string
signal?: AbortSignal
temporaryReferences?: TemporaryReferenceSet
onError?: (error: unknown) => void
}
): Promise<{
prelude: import('node:stream').Readable
}>
} }
declare module 'react-server-dom-webpack/client.edge' { declare module 'react-server-dom-webpack/client.edge' {
export interface Options { export interface Options {
......
export default function RootLayout({ children }) {
return (
<html>
<body>{children}</body>
</html>
)
}
export default function Page() {
return <p>env precedence fixture</p>
}
/** @type {import('next').NextConfig} */
const nextConfig = {
experimental: {
useNodeStreams: false,
},
}
module.exports = nextConfig
import { nextTestSetup } from 'e2e-utils'
describe('use-node-streams env precedence', () => {
const { next, isNextDev, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
skipStart: true,
env: {
__NEXT_USE_NODE_STREAMS: 'true',
},
})
if (isNextDev || skipped) {
it.skip('only testable in start mode', () => {})
return
}
it('should respect explicit useNodeStreams=false even when env flag is true', async () => {
const { exitCode, cliOutput } = await next.build()
expect(exitCode).toBe(0)
expect(cliOutput).toContain('useNodeStreams')
}, 30_000)
})
import { Readable } from 'node:stream'
import { createInlinedDataNodeStream } from 'next/dist/server/app-render/use-flight-response'
async function collectToString(stream: Readable): Promise<string> {
const chunks: Buffer[] = []
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
}
return Buffer.concat(chunks).toString('utf8')
}
describe('createInlinedDataNodeStream', () => {
it('emits bootstrap script even when no chunks are provided', async () => {
const source = Readable.from([])
const transformed = source.pipe(
createInlinedDataNodeStream(undefined, null)
)
const output = await collectToString(transformed)
expect(output).toContain('(self.__next_f=self.__next_f||[]).push([0])')
})
it('encodes UTF-8 chunks as data payloads', async () => {
const source = Readable.from([Buffer.from('hello')])
const transformed = source.pipe(
createInlinedDataNodeStream(undefined, null)
)
const output = await collectToString(transformed)
expect(output).toContain('self.__next_f.push([1,"hello"])')
expect(output).not.toContain('self.__next_f.push([3,"')
})
it('encodes invalid UTF-8 chunks as binary payloads', async () => {
const source = Readable.from([Buffer.from([0xff, 0xfe, 0xfd])])
const transformed = source.pipe(
createInlinedDataNodeStream(undefined, null)
)
const output = await collectToString(transformed)
expect(output).toContain('self.__next_f.push([3,"//79"])')
})
it('handles multi-byte UTF-8 split across chunks', async () => {
const source = Readable.from([
Buffer.from([0xe2]),
Buffer.from([0x9c, 0x93]),
])
const transformed = source.pipe(
createInlinedDataNodeStream(undefined, null)
)
const output = await collectToString(transformed)
expect(output).toContain('self.__next_f.push([1,')
expect(output).toContain('✓')
expect(output).not.toContain('self.__next_f.push([3,"')
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment