Commit 1eebab7c authored by Jiachi Liu's avatar Jiachi Liu

fix non ascii encoded middleware header

parent 2b1c4ad6
...@@ -111,7 +111,7 @@ import { ...@@ -111,7 +111,7 @@ import {
import { BaseServerSpan } from './lib/trace/constants' import { BaseServerSpan } from './lib/trace/constants'
import { I18NProvider } from './lib/i18n-provider' import { I18NProvider } from './lib/i18n-provider'
import { sendResponse } from './send-response' import { sendResponse } from './send-response'
import { normalizeNextQueryParam } from './web/utils' import { normalizeNextQueryParam, decodeHeaderValue } from './web/utils'
import { import {
HTML_CONTENT_TYPE_HEADER, HTML_CONTENT_TYPE_HEADER,
JSON_CONTENT_TYPE_HEADER, JSON_CONTENT_TYPE_HEADER,
...@@ -943,6 +943,26 @@ export default abstract class Server< ...@@ -943,6 +943,26 @@ export default abstract class Server<
isNodeNextResponse(res) ? res.originalResponse : res isNodeNextResponse(res) ? res.originalResponse : res
) )
// In minimal mode (Vercel), decode headers that were percent-encoded by middleware
// to preserve non-ASCII characters during transport
if (this.minimalMode) {
const encodedHeadersList = req.headers['x-middleware-request-encoded']
if (typeof encodedHeadersList === 'string') {
const encodedHeaders = encodedHeadersList.split(',')
for (const headerName of encodedHeaders) {
const trimmedName = headerName.trim()
const headerValue = req.headers[trimmedName]
if (typeof headerValue === 'string') {
req.headers[trimmedName] = decodeHeaderValue(headerValue)
} else if (Array.isArray(headerValue)) {
req.headers[trimmedName] = headerValue.map(decodeHeaderValue)
}
}
// Clean up the marker header
delete req.headers['x-middleware-request-encoded']
}
}
const urlParts = (req.url || '').split('?', 1) const urlParts = (req.url || '').split('?', 1)
const urlNoQuery = urlParts[0] const urlNoQuery = urlParts[0]
......
...@@ -12,3 +12,37 @@ export function fixMojibake(input: string): string { ...@@ -12,3 +12,37 @@ export function fixMojibake(input: string): string {
const decoder = new TextDecoder('utf-8') const decoder = new TextDecoder('utf-8')
return decoder.decode(bytes) return decoder.decode(bytes)
} }
/**
* Attempts to recover a header value that may have been corrupted by
* UTF-8 bytes being interpreted as Latin-1 characters (Mojibake).
*
* For example, "Montréal" sent as UTF-8 (bytes: 4D 6F 6E 74 72 C3 A9 61 6C)
* might arrive as "Montréal" when those UTF-8 bytes are interpreted as Latin-1.
*
* This function detects this pattern and recovers the original UTF-8 string.
* Unlike fixMojibake, this function is safe to call on any string - it will
* return the original value if decoding fails or if the value isn't Mojibake.
*/
export function recoverMojibake(value: string): string {
// Check if value contains non-ASCII characters that could be Mojibake
if (!/[\x80-\xFF]/.test(value)) {
return value
}
try {
// Convert the string to Latin-1 bytes, then decode as UTF-8
const bytes = new Uint8Array(value.length)
for (let i = 0; i < value.length; i++) {
bytes[i] = value.charCodeAt(i)
}
const decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
// If decoding succeeded and produced a different string, we recovered from Mojibake
if (decoded !== value) {
return decoded
}
} catch {
// If UTF-8 decoding fails, the value wasn't Mojibake - return original
}
return value
}
...@@ -14,7 +14,7 @@ import { getCloneableBody } from '../../body-streams' ...@@ -14,7 +14,7 @@ import { getCloneableBody } from '../../body-streams'
import { filterReqHeaders, ipcForbiddenHeaders } from '../server-ipc/utils' import { filterReqHeaders, ipcForbiddenHeaders } from '../server-ipc/utils'
import { stringifyQuery } from '../../server-route-utils' import { stringifyQuery } from '../../server-route-utils'
import { formatHostname } from '../format-hostname' import { formatHostname } from '../format-hostname'
import { toNodeOutgoingHttpHeaders } from '../../web/utils' import { toNodeOutgoingHttpHeaders, decodeHeaderValue } from '../../web/utils'
import { isAbortError } from '../../pipe-readable' import { isAbortError } from '../../pipe-readable'
import { getHostname } from '../../../shared/lib/get-hostname' import { getHostname } from '../../../shared/lib/get-hostname'
import { import {
...@@ -611,14 +611,23 @@ export function getResolveRoutes( ...@@ -611,14 +611,23 @@ export function getResolveRoutes(
// Update or add headers. // Update or add headers.
for (const key of overriddenHeaders.keys()) { for (const key of overriddenHeaders.keys()) {
const valueKey = 'x-middleware-request-' + key const valueKey = 'x-middleware-request-' + key
const newValue = middlewareHeaders[valueKey] const encodedValue = middlewareHeaders[valueKey]
const newValue =
encodedValue == null
? undefined
: typeof encodedValue === 'string'
? decodeHeaderValue(encodedValue)
: encodedValue.map(decodeHeaderValue)
const oldValue = req.headers[key] const oldValue = req.headers[key]
if (oldValue !== newValue) { if (oldValue !== newValue) {
req.headers[key] = newValue === null ? undefined : newValue req.headers[key] = newValue
} }
delete middlewareHeaders[valueKey] delete middlewareHeaders[valueKey]
} }
// Clean up the encoded headers marker
delete middlewareHeaders['x-middleware-request-encoded']
} }
if ( if (
......
...@@ -45,6 +45,7 @@ const INTERNAL_HEADERS = [ ...@@ -45,6 +45,7 @@ const INTERNAL_HEADERS = [
'x-middleware-set-cookie', 'x-middleware-set-cookie',
'x-middleware-skip', 'x-middleware-skip',
'x-middleware-override-headers', 'x-middleware-override-headers',
'x-middleware-request-encoded',
'x-middleware-next', 'x-middleware-next',
'x-now-route-matches', 'x-now-route-matches',
'x-matched-path', 'x-matched-path',
......
import { stringifyCookie } from '../../web/spec-extension/cookies' import { stringifyCookie } from '../../web/spec-extension/cookies'
import type { I18NConfig } from '../../config-shared' import type { I18NConfig } from '../../config-shared'
import { NextURL } from '../next-url' import { NextURL } from '../next-url'
import { toNodeOutgoingHttpHeaders, validateURL } from '../utils' import {
toNodeOutgoingHttpHeaders,
validateURL,
encodeHeaderValue,
decodeNodeHeaderValue,
} from '../utils'
import { ReflectAdapter } from './adapters/reflect' import { ReflectAdapter } from './adapters/reflect'
import { ResponseCookies } from './cookies' import { ResponseCookies } from './cookies'
...@@ -19,12 +24,26 @@ function handleMiddlewareField( ...@@ -19,12 +24,26 @@ function handleMiddlewareField(
} }
const keys = [] const keys = []
const encodedKeys = []
for (const [key, value] of init.request.headers) { for (const [key, value] of init.request.headers) {
headers.set('x-middleware-request-' + key, value) // First decode any values that were percent-encoded by fromNodeOutgoingHttpHeaders
// to preserve non-ASCII characters, then re-encode for the middleware header
const decodedValue = decodeNodeHeaderValue(value)
const encodedValue = encodeHeaderValue(decodedValue)
headers.set('x-middleware-request-' + key, encodedValue)
keys.push(key) keys.push(key)
// Track headers that were encoded (value changed after encoding)
if (encodedValue !== decodedValue) {
encodedKeys.push(key)
}
} }
headers.set('x-middleware-override-headers', keys.join(',')) headers.set('x-middleware-override-headers', keys.join(','))
// Add a marker header listing which headers have percent-encoded values
// This allows the server to decode them in minimal mode (Vercel)
if (encodedKeys.length > 0) {
headers.set('x-middleware-request-encoded', encodedKeys.join(','))
}
} }
} }
......
import { toNodeOutgoingHttpHeaders } from './utils' import {
toNodeOutgoingHttpHeaders,
encodeHeaderValue,
decodeHeaderValue,
} from './utils'
describe('toNodeHeaders', () => { describe('toNodeHeaders', () => {
it('should handle multiple set-cookie headers correctly', () => { it('should handle multiple set-cookie headers correctly', () => {
...@@ -49,3 +53,76 @@ describe('toNodeHeaders', () => { ...@@ -49,3 +53,76 @@ describe('toNodeHeaders', () => {
}) })
}) })
}) })
describe('encodeHeaderValue', () => {
it('should return ASCII values unchanged', () => {
expect(encodeHeaderValue('hello-world')).toBe('hello-world')
expect(encodeHeaderValue('test_value')).toBe('test_value')
expect(encodeHeaderValue('123')).toBe('123')
})
it('should encode non-ASCII characters', () => {
expect(encodeHeaderValue('Montréal')).toBe('Montr%C3%A9al')
expect(encodeHeaderValue('café')).toBe('caf%C3%A9')
expect(encodeHeaderValue('北京')).toBe('%E5%8C%97%E4%BA%AC')
expect(encodeHeaderValue('русский')).toBe(
'%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9'
)
})
it('should preserve spaces', () => {
expect(encodeHeaderValue('hello world')).toBe('hello world')
expect(encodeHeaderValue('Montréal city')).toBe('Montr%C3%A9al city')
})
it('should handle mixed ASCII and non-ASCII', () => {
expect(encodeHeaderValue('City: Montréal')).toBe('City: Montr%C3%A9al')
})
})
describe('decodeHeaderValue', () => {
it('should return unencoded values unchanged', () => {
expect(decodeHeaderValue('hello-world')).toBe('hello-world')
expect(decodeHeaderValue('test_value')).toBe('test_value')
expect(decodeHeaderValue('123')).toBe('123')
})
it('should decode percent-encoded values', () => {
expect(decodeHeaderValue('Montr%C3%A9al')).toBe('Montréal')
expect(decodeHeaderValue('caf%C3%A9')).toBe('café')
expect(decodeHeaderValue('%E5%8C%97%E4%BA%AC')).toBe('北京')
expect(
decodeHeaderValue('%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9')
).toBe('русский')
})
it('should handle spaces correctly', () => {
expect(decodeHeaderValue('hello world')).toBe('hello world')
expect(decodeHeaderValue('Montr%C3%A9al city')).toBe('Montréal city')
})
it('should handle mixed encoded and unencoded values', () => {
expect(decodeHeaderValue('City: Montr%C3%A9al')).toBe('City: Montréal')
})
it('should return original value if decoding fails', () => {
expect(decodeHeaderValue('invalid%')).toBe('invalid%')
expect(decodeHeaderValue('invalid%2')).toBe('invalid%2')
expect(decodeHeaderValue('%XX')).toBe('%XX')
})
it('should round-trip encode/decode correctly', () => {
const testValues = [
'Montréal',
'café',
'北京',
'русский',
'hello world',
'mixed: Montréal café 北京',
]
for (const value of testValues) {
expect(decodeHeaderValue(encodeHeaderValue(value))).toBe(value)
}
})
})
...@@ -3,6 +3,7 @@ import { ...@@ -3,6 +3,7 @@ import {
NEXT_INTERCEPTION_MARKER_PREFIX, NEXT_INTERCEPTION_MARKER_PREFIX,
NEXT_QUERY_PARAM_PREFIX, NEXT_QUERY_PARAM_PREFIX,
} from '../../lib/constants' } from '../../lib/constants'
import { recoverMojibake } from '../lib/fix-mojibake'
/** /**
* Converts a Node.js IncomingHttpHeaders object to a Headers object. Any * Converts a Node.js IncomingHttpHeaders object to a Headers object. Any
...@@ -25,12 +26,38 @@ export function fromNodeOutgoingHttpHeaders( ...@@ -25,12 +26,38 @@ export function fromNodeOutgoingHttpHeaders(
v = v.toString() v = v.toString()
} }
// Try to recover from Mojibake (UTF-8 bytes interpreted as Latin-1)
v = recoverMojibake(v)
// Encode non-ASCII characters to prevent corruption by the Headers API.
// The Headers API in edge runtime doesn't properly preserve Latin-1 characters.
// eslint-disable-next-line no-control-regex
if (/[^\u0000-\u007F]/.test(v)) {
// Encode non-ASCII as percent-encoded UTF-8
v = encodeURIComponent(v)
}
headers.append(key, v) headers.append(key, v)
} }
} }
return headers return headers
} }
/**
* Decodes a header value that may have been percent-encoded by fromNodeOutgoingHttpHeaders.
*/
export function decodeNodeHeaderValue(value: string): string {
// Check if value contains percent-encoded sequences
if (/%[0-9A-Fa-f]{2}/.test(value)) {
try {
return decodeURIComponent(value)
} catch {
return value
}
}
return value
}
/* /*
Set-Cookie header field-values are sometimes comma joined in one string. This splits them without choking on commas Set-Cookie header field-values are sometimes comma joined in one string. This splits them without choking on commas
that are within a single set-cookie field-value, such as in the Expires portion. that are within a single set-cookie field-value, such as in the Expires portion.
...@@ -164,3 +191,45 @@ export function normalizeNextQueryParam(key: string): null | string { ...@@ -164,3 +191,45 @@ export function normalizeNextQueryParam(key: string): null | string {
} }
return null return null
} }
/**
* Encodes header values containing non-ASCII characters for safe HTTP transmission.
* HTTP headers must be ASCII-safe, so non-ASCII characters are percent-encoded.
*
* @param value - The header value to encode
* @returns The encoded header value
*/
export function encodeHeaderValue(value: string): string {
// Check if the value contains non-ASCII characters
// eslint-disable-next-line no-control-regex
if (/[^\u0000-\u007F]/.test(value)) {
// Use encodeURIComponent but preserve spaces and common header-safe characters
return encodeURIComponent(value)
.replace(/%20/g, ' ') // Preserve spaces
.replace(/%3A/g, ':') // Preserve colons (common in headers)
.replace(/%2D/g, '-') // Preserve hyphens
.replace(/%2E/g, '.') // Preserve dots
.replace(/%5F/g, '_') // Preserve underscores
}
return value
}
/**
* Decodes header values that were encoded with encodeHeaderValue.
*
* @param value - The header value to decode
* @returns The decoded header value
*/
export function decodeHeaderValue(value: string): string {
try {
// decodeURIComponent will throw if the value is not properly encoded
// Only attempt to decode if the value contains percent-encoded sequences
if (/%[0-9A-Fa-f]{2}/.test(value)) {
return decodeURIComponent(value.replace(/ /g, '%20'))
}
return value
} catch {
// If decoding fails, return the original value
return value
}
}
...@@ -12,9 +12,8 @@ import { NextResponse } from 'next/server' ...@@ -12,9 +12,8 @@ import { NextResponse } from 'next/server'
* @param {import('next/server').NextRequest} request * @param {import('next/server').NextRequest} request
*/ */
export async function middleware(request) { export async function middleware(request) {
// Simply forward all existing headers through NextResponse.next() // This is the typical middleware pattern - create a new Headers object
// This is the common pattern that triggers the issue when incoming // to allow modifications
// headers contain non-ASCII values (e.g., from Cloudflare geolocation)
const headers = new Headers(request.headers) const headers = new Headers(request.headers)
// Add a marker header to prove middleware ran // Add a marker header to prove middleware ran
......
...@@ -10,13 +10,29 @@ import { createNext, FileRef } from 'e2e-utils' ...@@ -10,13 +10,29 @@ import { createNext, FileRef } from 'e2e-utils'
* cf-ipcity: "Montréal"), the request fails because x-middleware-request-* headers * cf-ipcity: "Montréal"), the request fails because x-middleware-request-* headers
* must be ASCII-safe per HTTP spec. * must be ASCII-safe per HTTP spec.
* *
* Error: "Header 'x-middleware-request-cf-ipcity: Montréal' contains non-ASCII characters" * IMPORTANT: In real-world scenarios (Cloudflare, browsers), UTF-8 header values are
* transmitted as bytes and interpreted by Node.js as Latin-1, causing "Mojibake".
* For example, "Montréal" (UTF-8: 4D 6F 6E 74 72 C3 A9 61 6C) arrives as "Montréal"
* when those UTF-8 bytes are interpreted as Latin-1 characters.
* *
* Note: HTTP headers can only contain ISO-8859-1 (Latin-1) characters (char codes 0-255). * node-fetch (used in tests) handles headers differently - it converts non-ASCII to
* Characters outside this range cannot be sent via HTTP headers at all. * replacement characters. So these tests send the Mojibake directly to simulate
* The issue specifically affects Latin-1 characters like French accents (é, è, ê), * what the server actually receives in production.
* German umlauts (ü, ö, ä), and other Western European characters.
*/ */
/**
* Converts a UTF-8 string to its Mojibake representation.
* This simulates what happens when UTF-8 bytes are interpreted as Latin-1.
*/
function toMojibake(str: string): string {
const encoder = new TextEncoder() // UTF-8
const bytes = encoder.encode(str)
// Interpret UTF-8 bytes as Latin-1 characters
return Array.from(bytes)
.map((b) => String.fromCharCode(b))
.join('')
}
describe('Middleware Non-ASCII Headers', () => { describe('Middleware Non-ASCII Headers', () => {
let next: NextInstance let next: NextInstance
...@@ -32,21 +48,23 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -32,21 +48,23 @@ describe('Middleware Non-ASCII Headers', () => {
}) })
it('should handle headers with French accented characters (Montréal)', async () => { it('should handle headers with French accented characters (Montréal)', async () => {
// This is the exact scenario from issue #85631 - Cloudflare's cf-ipcity header // Simulate what the server receives when Cloudflare sends "Montréal" as UTF-8
// The UTF-8 bytes get interpreted as Latin-1, producing "Montréal"
const res = await fetchViaHTTP( const res = await fetchViaHTTP(
next.url, next.url,
'/api/headers', '/api/headers',
{}, {},
{ {
headers: { headers: {
'x-city': 'Montréal', 'x-city': toMojibake('Montréal'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-city']).toBe('Montréal') // The middleware should recover the original UTF-8 string
expect(headers['x-city']).toBe('Montréal')
}) })
it('should handle headers with German umlauts (München)', async () => { it('should handle headers with German umlauts (München)', async () => {
...@@ -56,14 +74,14 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -56,14 +74,14 @@ describe('Middleware Non-ASCII Headers', () => {
{}, {},
{ {
headers: { headers: {
'x-city': 'München', 'x-city': toMojibake('München'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-city']).toBe('München') expect(headers['x-city']).toBe('München')
}) })
it('should handle headers with Spanish characters (España)', async () => { it('should handle headers with Spanish characters (España)', async () => {
...@@ -73,14 +91,14 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -73,14 +91,14 @@ describe('Middleware Non-ASCII Headers', () => {
{}, {},
{ {
headers: { headers: {
'x-country': 'España', 'x-country': toMojibake('España'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-country']).toBe('España') expect(headers['x-country']).toBe('España')
}) })
it('should handle headers with Nordic characters (Malmö, København)', async () => { it('should handle headers with Nordic characters (Malmö, København)', async () => {
...@@ -90,15 +108,15 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -90,15 +108,15 @@ describe('Middleware Non-ASCII Headers', () => {
{}, {},
{ {
headers: { headers: {
'x-city': 'Malmö', 'x-city': toMojibake('Malmö'),
'x-region': 'København', 'x-region': toMojibake('København'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-city']).toBe('Malmö') expect(headers['x-city']).toBe('Malmö')
expect(headers['x-region']).toBe('København') expect(headers['x-region']).toBe('København')
}) })
...@@ -109,14 +127,14 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -109,14 +127,14 @@ describe('Middleware Non-ASCII Headers', () => {
{}, {},
{ {
headers: { headers: {
'x-region': 'Île-de-France', 'x-region': toMojibake('Île-de-France'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-region']).toBe('Île-de-France') expect(headers['x-region']).toBe('Île-de-France')
}) })
it('should handle headers with Austrian characters (Österreich)', async () => { it('should handle headers with Austrian characters (Österreich)', async () => {
...@@ -126,34 +144,34 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -126,34 +144,34 @@ describe('Middleware Non-ASCII Headers', () => {
{}, {},
{ {
headers: { headers: {
'x-country': 'Österreich', 'x-country': toMojibake('Österreich'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-country']).toBe('Österreich') expect(headers['x-country']).toBe('Österreich')
}) })
it('should handle multiple headers with various Latin-1 characters', async () => { it('should handle multiple headers with various non-ASCII characters', async () => {
const res = await fetchViaHTTP( const res = await fetchViaHTTP(
next.url, next.url,
'/api/headers', '/api/headers',
{}, {},
{ {
headers: { headers: {
'x-city': 'Montréal', 'x-city': toMojibake('Montréal'),
'x-region': 'Île-de-France', 'x-region': toMojibake('Île-de-France'),
'x-country': 'Österreich', 'x-country': toMojibake('Österreich'),
'x-greeting': 'Grüß Gott', 'x-greeting': toMojibake('Grüß Gott'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-city']).toBe('Montréal') expect(headers['x-city']).toBe('Montréal')
expect(headers['x-region']).toBe('Île-de-France') expect(headers['x-region']).toBe('Île-de-France')
expect(headers['x-country']).toBe('Österreich') expect(headers['x-country']).toBe('Österreich')
expect(headers['x-greeting']).toBe('Grüß Gott') expect(headers['x-greeting']).toBe('Grüß Gott')
...@@ -166,13 +184,13 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -166,13 +184,13 @@ describe('Middleware Non-ASCII Headers', () => {
{}, {},
{ {
headers: { headers: {
'x-city': 'São Paulo', 'x-city': toMojibake('São Paulo'),
}, },
} }
) )
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
expect(headers['x-city']).toBe('São Paulo') expect(headers['x-city']).toBe('São Paulo')
}) })
}) })
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment