Unverified Commit e6478e14 authored by Jan Kassens's avatar Jan Kassens Committed by GitHub

[testmode] Fix compressed HTTP responses with test proxy (#100805)

## Summary

Fix compressed responses received by Node HTTP clients when
`experimental.testProxy` is enabled. Requests outside a test context and
proxy `continue` responses now use native HTTP passthrough, preserving
the original compressed bytes and headers.

Normalize stale `Content-Encoding` and `Content-Length` headers when
serializing fetched responses with supported compression encodings,
including cloned fetch responses. Preserve explicitly constructed mock
responses as supplied, including deliberately compressed bytes. Handlers
that construct a response with a modified body must update its headers
themselves. Preserve bodyless responses such as HEAD so their
representation metadata remains available.

Add regression coverage for chunked responses and compressed bodies
whose decoded length is shorter or longer than the upstream content
length, across native passthrough, proxy continue, fetched responses,
and modified responses. Also cover explicitly compressed mocks,
fetched-response clones, unsupported encodings, and HEAD metadata.

## Verification

- All 36 testmode integration tests passed in development mode with
webpack, using the published native binding because the local native
build has an incompatible lockfile API.
- Confirmed the standalone gzip reproduction fails against the unchanged
build and passes with native passthrough.

<!-- NEXT_JS_LLM -->
parent c550b32e
......@@ -94,14 +94,21 @@ function passthroughFetch(
return originalFetch(request)
}
export async function handleFetch(
/**
* Returns the response supplied by the test proxy, or `null` when there is no
* test context or the proxy asks to continue the request. A `null` result means
* the caller must send the request using its original transport, preserving
* that transport's response handling (such as raw compressed HTTP bodies).
* Throws if the proxy aborts the request or leaves it unhandled.
*/
export async function handleProxyFetch(
originalFetch: Fetch,
request: Request
): Promise<Response> {
): Promise<Response | null> {
const testInfo = getTestReqInfo(request, reader)
if (!testInfo) {
// Passthrough non-test requests.
return passthroughFetch(originalFetch, request)
// Let the caller pass through using its original transport.
return null
}
const { testData, proxyPort } = testInfo
......@@ -131,7 +138,7 @@ export async function handleFetch(
const { api } = proxyResponse
switch (api) {
case 'continue':
return passthroughFetch(originalFetch, request)
return null
case 'abort':
case 'unhandled':
throw new Error(
......@@ -145,7 +152,7 @@ export async function handleFetch(
}
export function interceptFetch(originalFetch: Fetch) {
global.fetch = function testFetch(
global.fetch = async function testFetch(
input: FetchInputArg,
init?: FetchInitArg
): Promise<Response> {
......@@ -154,7 +161,9 @@ export function interceptFetch(originalFetch: Fetch) {
if (init?.next?.internal) {
return originalFetch(input, init)
}
return handleFetch(originalFetch, new Request(input, init))
const request = new Request(input, init)
const response = await handleProxyFetch(originalFetch, request)
return response ?? passthroughFetch(originalFetch, request)
}
return () => {
global.fetch = originalFetch
......
import { ClientRequestInterceptor } from 'next/dist/compiled/@mswjs/interceptors/ClientRequest'
import { handleFetch } from './fetch'
import { handleProxyFetch } from './fetch'
type Fetch = typeof fetch
......@@ -8,13 +8,17 @@ export function interceptHttpGet(originalFetch: Fetch): () => void {
clientRequestInterceptor.on('request', async ({ request, controller }) => {
if (request.headers.get('next-test-internal') === '1') {
// A request that's part of the test proxy protocol itself, sent by
// `handleFetch` from within this listener. Not responding lets the
// `handleProxyFetch` from within this listener. Not responding lets the
// interceptor perform it against the real server. Handling it here
// instead would recurse indefinitely.
return
}
const response = await handleFetch(originalFetch, request)
controller.respondWith(response)
const response = await handleProxyFetch(originalFetch, request)
// Native passthrough preserves compressed bytes, headers, and redirects.
// Fetch would decode the body while retaining the upstream headers.
if (response !== null) {
controller.respondWith(response)
}
})
clientRequestInterceptor.apply()
......
......@@ -23,6 +23,12 @@ function buildRequest(req: ProxyFetchRequest): Request {
})
}
/**
* Serializes a handler result, normalizing compression headers on fetched
* responses. Constructed responses are forwarded as supplied, including
* compressed bodies. Handlers that replace a fetched body must also update
* its headers.
*/
async function buildResponse(
response: FetchHandlerResult
): Promise<ProxyResponse> {
......@@ -36,7 +42,26 @@ async function buildResponse(
return CONTINUE
}
const { status, headers, body } = response
const { status, body } = response
const headers = new Headers(response.headers)
const contentEncoding = headers.get('content-encoding')
// Fetch responses (including clones) have a URL; constructed mocks do not.
// Normalize only encodings fetch decodes, preserving unknown encodings and
// bodyless responses such as HEAD, whose headers describe the representation.
if (
response.url &&
body &&
contentEncoding &&
contentEncoding
.toLowerCase()
.split(',')
.every((coding) =>
['gzip', 'x-gzip', 'deflate', 'br'].includes(coding.trim())
)
) {
headers.delete('content-encoding')
headers.delete('content-length')
}
return {
api: 'fetch',
response: {
......
import { get } from 'node:http'
export async function GET(request: Request) {
const params = new URL(request.url).searchParams
const url = params.get('url')!
const result = await new Promise((resolve, reject) => {
const req = get(
url,
{
method: params.get('method') ?? 'GET',
headers: { 'accept-encoding': 'gzip' },
},
(res) => {
const chunks: Buffer[] = []
res.on('data', (chunk) => chunks.push(chunk))
res.on('error', reject)
res.on('end', () => {
resolve({
headers: res.headers,
body: Buffer.concat(chunks).toString('base64'),
})
})
}
)
req.on('error', reject)
req.setTimeout(3000, () => req.destroy(new Error('HTTP request timed out')))
})
return Response.json(result)
}
import { nextTestSetup } from 'e2e-utils'
import { createServer, type Server } from 'node:http'
import { gzipSync, gunzipSync } from 'node:zlib'
import { createProxyServer } from 'next/experimental/testmode/proxy'
const PNG_1X1 = Buffer.from(
......@@ -43,6 +45,43 @@ describe('testmode', () => {
) {
return new Response(`middleware-${testData}`)
}
if (new URL(request.url).pathname === '/compressed') {
if (testData === 'head-response') {
return new Response(null, {
headers: {
'content-type': 'text/plain',
'content-encoding': 'gzip',
'content-length': '1234',
},
})
}
if (testData === 'fetch-response') {
return fetch(request)
}
if (testData === 'fetch-response-clone') {
return (await fetch(request)).clone()
}
if (testData === 'compressed-response') {
const body = gzipSync('explicitly compressed mock')
return new Response(new Uint8Array(body), {
headers: {
'content-type': 'text/plain',
'content-encoding': 'gzip',
'content-length': String(body.length),
},
})
}
if (testData === 'modified-response') {
const response = await fetch(request)
const headers = new Headers(response.headers)
headers.delete('content-encoding')
headers.delete('content-length')
return new Response(`modified:${await response.text()}`, {
headers,
})
}
return 'continue'
}
return undefined
},
})
......@@ -61,6 +100,151 @@ describe('testmode', () => {
})
}
describe('compressed HTTP responses', () => {
let upstream: Server
let upstreamUrl: string
const cases = [
{ name: 'chunked', plain: '{"ok":true}', withLength: false },
{ name: 'shorter decoded body', plain: '{"ok":true}', withLength: true },
{
name: 'longer decoded body',
plain: 'x'.repeat(2000),
withLength: true,
},
]
beforeAll(async () => {
upstream = createServer((req, res) => {
const params = new URL(req.url!, 'http://localhost').searchParams
const index = Number(params.get('case'))
const { plain, withLength } = cases[index]
const body = gzipSync(plain)
res.setHeader('content-type', 'text/plain')
res.setHeader('content-encoding', params.get('encoding') ?? 'gzip')
if (withLength) res.setHeader('content-length', body.length)
res.writeHead(200)
res.end(body)
})
await new Promise<void>((resolve) =>
upstream.listen(0, '127.0.0.1', resolve)
)
const address = upstream.address()
if (!address || typeof address === 'string')
throw new Error('Missing upstream address')
upstreamUrl = `http://127.0.0.1:${address.port}/compressed`
})
afterAll(async () => {
upstream.closeAllConnections()
await new Promise<void>((resolve, reject) =>
upstream.close((error) => (error ? reject(error) : resolve()))
)
})
it.each(['fetch-response', 'head-response'])(
'should preserve representation headers on a HEAD %s',
async (testData) => {
const url = `/api/compressed-http?method=HEAD&url=${encodeURIComponent(`${upstreamUrl}?case=1`)}`
const response = await fetchForTest(url, testData)
expect(response.status).toBe(200)
const result = await response.json()
expect(result.body).toBe('')
expect(result.headers['content-type']).toBe('text/plain')
expect(result.headers['content-encoding']).toBe('gzip')
expect(result.headers['content-length']).toBe(
testData === 'head-response'
? '1234'
: String(gzipSync(cases[1].plain).length)
)
}
)
it('should preserve explicitly compressed mock bytes and headers', async () => {
const url = `/api/compressed-http?url=${encodeURIComponent(`${upstreamUrl}?case=1`)}`
const response = await fetchForTest(url, 'compressed-response')
expect(response.status).toBe(200)
const result = await response.json()
const body = Buffer.from(result.body, 'base64')
expect(body).toEqual(gzipSync('explicitly compressed mock'))
expect(gunzipSync(body).toString()).toBe('explicitly compressed mock')
expect(result.headers['content-type']).toBe('text/plain')
expect(result.headers['content-encoding']).toBe('gzip')
expect(result.headers['content-length']).toBe(String(body.length))
})
it.each(['custom', 'gzip, custom'])(
'should preserve fetched bytes with unsupported encoding %s',
async (encoding) => {
const upstream = `${upstreamUrl}?case=1&encoding=${encodeURIComponent(encoding)}`
const url = `/api/compressed-http?url=${encodeURIComponent(upstream)}`
const response = await fetchForTest(url, 'fetch-response')
expect(response.status).toBe(200)
const result = await response.json()
const body = Buffer.from(result.body, 'base64')
expect(body).toEqual(gzipSync(cases[1].plain))
expect(result.headers['content-encoding']).toBe(encoding)
expect(result.headers['content-length']).toBe(String(body.length))
}
)
describe.each(['without test context', 'with proxy continue'])(
'%s',
(context) => {
it.each(cases)('preserves $name', async ({ plain, withLength }) => {
const index = cases.findIndex(
(entry) => entry.plain === plain && entry.withLength === withLength
)
const url = `/api/compressed-http?url=${encodeURIComponent(`${upstreamUrl}?case=${index}`)}`
const response =
context === 'without test context'
? await next.fetch(url)
: await fetchForTest(url)
expect(response.status).toBe(200)
const result = await response.json()
const body = Buffer.from(result.body, 'base64')
expect(result.headers['content-encoding']).toBe('gzip')
expect(result.headers['content-length']).toBe(
withLength ? String(gzipSync(plain).length) : undefined
)
expect(body).toEqual(gzipSync(plain))
expect(gunzipSync(body).toString()).toBe(plain)
})
}
)
describe.each([
'fetch-response',
'fetch-response-clone',
'modified-response',
])('with proxy %s', (testData) => {
it.each(cases)(
'should forward $name with a decoded body and matching headers',
async ({ plain, withLength }) => {
const index = cases.findIndex(
(entry) => entry.plain === plain && entry.withLength === withLength
)
const url = `/api/compressed-http?url=${encodeURIComponent(`${upstreamUrl}?case=${index}`)}`
const response = await fetchForTest(url, testData)
expect(response.status).toBe(200)
const result = await response.json()
const body = Buffer.from(result.body, 'base64')
const expected =
testData === 'modified-response' ? `modified:${plain}` : plain
expect(body.toString()).toBe(expected)
expect(result.headers['content-type']).toBe('text/plain')
expect(result.headers['content-encoding']).toBeUndefined()
// The interceptor may omit the length, but must not retain a stale one.
if (result.headers['content-length'] !== undefined) {
expect(result.headers['content-length']).toBe(
String(Buffer.byteLength(expected))
)
}
}
)
})
})
describe('app router', () => {
it('should fetch real data when Next-Test-* headers are not present', async () => {
const html = await (await next.fetch('/app/rsc-fetch')).text()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment