Commit a6e9da93 authored by Jiachi Liu's avatar Jiachi Liu

refactor utils

parent 1eebab7c
...@@ -146,7 +146,7 @@ import { SegmentPrefixRSCPathnameNormalizer } from './normalizers/request/segmen ...@@ -146,7 +146,7 @@ import { SegmentPrefixRSCPathnameNormalizer } from './normalizers/request/segmen
import { shouldServeStreamingMetadata } from './lib/streaming-metadata' import { shouldServeStreamingMetadata } from './lib/streaming-metadata'
import { decodeQueryPathParameter } from './lib/decode-query-path-parameter' import { decodeQueryPathParameter } from './lib/decode-query-path-parameter'
import { NoFallbackError } from '../shared/lib/no-fallback-error.external' import { NoFallbackError } from '../shared/lib/no-fallback-error.external'
import { fixMojibake } from './lib/fix-mojibake' import { decodeMojibake } from './lib/mojibake'
import { computeCacheBustingSearchParam } from '../shared/lib/router/utils/cache-busting-search-param' import { computeCacheBustingSearchParam } from '../shared/lib/router/utils/cache-busting-search-param'
import { setCacheBustingSearchParamWithHash } from '../client/components/router-reducer/set-cache-busting-search-param' import { setCacheBustingSearchParamWithHash } from '../client/components/router-reducer/set-cache-busting-search-param'
import type { CacheControl } from './lib/cache-control' import type { CacheControl } from './lib/cache-control'
...@@ -1057,7 +1057,7 @@ export default abstract class Server< ...@@ -1057,7 +1057,7 @@ export default abstract class Server<
// x-matched-path is the source of truth, it tells what page // x-matched-path is the source of truth, it tells what page
// should be rendered because we don't process rewrites in minimalMode // should be rendered because we don't process rewrites in minimalMode
let { pathname: matchedPath } = new URL( let { pathname: matchedPath } = new URL(
fixMojibake(req.headers[MATCHED_PATH_HEADER] as string), decodeMojibake(req.headers[MATCHED_PATH_HEADER] as string),
'http://localhost' 'http://localhost'
) )
......
import { fixMojibake } from './fix-mojibake'
describe('Mojibake handling', () => {
const validValues = [
'hello-world',
'hello world',
encodeURIComponent('こんにちは'),
]
it.each(validValues)(
'should maintain value when encoding is correct $1',
(testValue) => {
expect(fixMojibake(testValue)).toBe(testValue)
}
)
it('should fix invalid encoding', () => {
expect(fixMojibake('/blog/ã\x81\x93ã\x82\x93ã\x81«ã\x81¡ã\x81¯')).toBe(
'/blog/こんにちは'
)
})
})
// x-matched-path header can be decoded incorrectly
// and should only be utf8 characters so this fixes
// incorrectly encoded values
export function fixMojibake(input: string): string {
// Convert each character's char code to a byte
const bytes = new Uint8Array(input.length)
for (let i = 0; i < input.length; i++) {
bytes[i] = input.charCodeAt(i)
}
// Decode the bytes as proper UTF-8
const decoder = new TextDecoder('utf-8')
return decoder.decode(bytes)
}
/**
* Attempts to recover a header value that may have been corrupted by
* UTF-8 bytes being interpreted as Latin-1 characters (Mojibake).
*
* For example, "Montréal" sent as UTF-8 (bytes: 4D 6F 6E 74 72 C3 A9 61 6C)
* might arrive as "Montréal" when those UTF-8 bytes are interpreted as Latin-1.
*
* This function detects this pattern and recovers the original UTF-8 string.
* Unlike fixMojibake, this function is safe to call on any string - it will
* return the original value if decoding fails or if the value isn't Mojibake.
*/
export function recoverMojibake(value: string): string {
// Check if value contains non-ASCII characters that could be Mojibake
if (!/[\x80-\xFF]/.test(value)) {
return value
}
try {
// Convert the string to Latin-1 bytes, then decode as UTF-8
const bytes = new Uint8Array(value.length)
for (let i = 0; i < value.length; i++) {
bytes[i] = value.charCodeAt(i)
}
const decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
// If decoding succeeded and produced a different string, we recovered from Mojibake
if (decoded !== value) {
return decoded
}
} catch {
// If UTF-8 decoding fails, the value wasn't Mojibake - return original
}
return value
}
import { encodeMojibake, decodeMojibake } from './mojibake'
describe('Mojibake encoding/decoding', () => {
describe('encodeMojibake', () => {
it('should convert UTF-8 to Mojibake representation', () => {
// "Montréal" UTF-8 bytes interpreted as Latin-1
expect(encodeMojibake('Montréal')).toBe('Montréal')
expect(encodeMojibake('café')).toBe('café')
expect(encodeMojibake('München')).toBe('München')
})
it('should return ASCII strings unchanged', () => {
expect(encodeMojibake('hello')).toBe('hello')
expect(encodeMojibake('test-123')).toBe('test-123')
})
it('should handle multi-byte UTF-8 characters', () => {
// Japanese "こんにちは" becomes longer when each UTF-8 byte is a Latin-1 char
const encoded = encodeMojibake('こんにちは')
expect(encoded.length).toBeGreaterThan('こんにちは'.length)
})
})
describe('decodeMojibake', () => {
it('should recover UTF-8 from Mojibake', () => {
expect(decodeMojibake('Montréal')).toBe('Montréal')
expect(decodeMojibake('café')).toBe('café')
expect(decodeMojibake('München')).toBe('München')
})
it('should return ASCII strings unchanged', () => {
expect(decodeMojibake('hello')).toBe('hello')
expect(decodeMojibake('test-123')).toBe('test-123')
})
it('should return original if not valid Mojibake', () => {
// Invalid UTF-8 sequence should return original
expect(decodeMojibake('\xFF\xFE')).toBe('\xFF\xFE')
})
it('should handle already-valid UTF-8 strings', () => {
// If the string is already valid UTF-8 and not Mojibake, return as-is
expect(decodeMojibake('Montréal')).toBe('Montréal')
})
})
describe('round-trip', () => {
const testCases = [
'Montréal',
'café',
'München',
'España',
'Île-de-France',
'Österreich',
'São Paulo',
'Malmö',
'København',
'Grüß Gott',
'こんにちは',
'北京',
'Москва',
]
it.each(testCases)('should round-trip encode/decode: %s', (original) => {
const encoded = encodeMojibake(original)
const decoded = decodeMojibake(encoded)
expect(decoded).toBe(original)
})
})
})
/**
* Mojibake utilities for handling UTF-8 bytes incorrectly interpreted as Latin-1.
*
* "Mojibake" occurs when UTF-8 encoded text is incorrectly decoded as Latin-1 (ISO-8859-1).
* For example, "Montréal" encoded as UTF-8 (bytes: 4D 6F 6E 74 72 C3 A9 61 6C)
* becomes "Montréal" when those bytes are interpreted as Latin-1 characters.
*
* This commonly happens with HTTP headers from CDNs that send
* UTF-8 values (e.g., x-city: "Montréal") which get mangled when Node.js
* interprets them as Latin-1.
*/
/**
* Encodes a UTF-8 string to its Mojibake representation.
* This simulates what happens when UTF-8 bytes are interpreted as Latin-1.
*
* @param input - The UTF-8 string to encode
* @returns The Mojibake representation of the string
*/
export function encodeMojibake(input: string): string {
const encoder = new TextEncoder() // UTF-8
const bytes = encoder.encode(input)
// Interpret UTF-8 bytes as Latin-1 characters
return Array.from(bytes)
.map((b) => String.fromCharCode(b))
.join('')
}
/**
* Decodes a Mojibake string back to proper UTF-8.
* This is safe to call on any string - it will return the original value
* if decoding fails or if the value isn't Mojibake.
*
* @param input - The potentially Mojibake-encoded string
* @returns The decoded UTF-8 string, or the original if not Mojibake
*/
export function decodeMojibake(input: string): string {
// Check if value contains non-ASCII characters that could be Mojibake
if (!/[\x80-\xFF]/.test(input)) {
return input
}
try {
// Convert the string to Latin-1 bytes, then decode as UTF-8
const bytes = new Uint8Array(input.length)
for (let i = 0; i < input.length; i++) {
bytes[i] = input.charCodeAt(i)
}
const decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
// If decoding succeeded and produced a different string, we recovered from Mojibake
if (decoded !== input) {
return decoded
}
} catch {
// If UTF-8 decoding fails, the value wasn't Mojibake - return original
}
return input
}
...@@ -70,13 +70,13 @@ describe('encodeHeaderValue', () => { ...@@ -70,13 +70,13 @@ describe('encodeHeaderValue', () => {
) )
}) })
it('should preserve spaces', () => { it('should encode spaces when value contains non-ASCII', () => {
expect(encodeHeaderValue('hello world')).toBe('hello world') expect(encodeHeaderValue('hello world')).toBe('hello world') // ASCII only, no encoding
expect(encodeHeaderValue('Montréal city')).toBe('Montr%C3%A9al city') expect(encodeHeaderValue('Montréal city')).toBe('Montr%C3%A9al%20city')
}) })
it('should handle mixed ASCII and non-ASCII', () => { it('should handle mixed ASCII and non-ASCII', () => {
expect(encodeHeaderValue('City: Montréal')).toBe('City: Montr%C3%A9al') expect(encodeHeaderValue('City: Montréal')).toBe('City%3A%20Montr%C3%A9al')
}) })
}) })
...@@ -98,11 +98,11 @@ describe('decodeHeaderValue', () => { ...@@ -98,11 +98,11 @@ describe('decodeHeaderValue', () => {
it('should handle spaces correctly', () => { it('should handle spaces correctly', () => {
expect(decodeHeaderValue('hello world')).toBe('hello world') expect(decodeHeaderValue('hello world')).toBe('hello world')
expect(decodeHeaderValue('Montr%C3%A9al city')).toBe('Montréal city') expect(decodeHeaderValue('Montr%C3%A9al%20city')).toBe('Montréal city')
}) })
it('should handle mixed encoded and unencoded values', () => { it('should handle mixed encoded and unencoded values', () => {
expect(decodeHeaderValue('City: Montr%C3%A9al')).toBe('City: Montréal') expect(decodeHeaderValue('City%3A%20Montr%C3%A9al')).toBe('City: Montréal')
}) })
it('should return original value if decoding fails', () => { it('should return original value if decoding fails', () => {
......
...@@ -3,7 +3,7 @@ import { ...@@ -3,7 +3,7 @@ import {
NEXT_INTERCEPTION_MARKER_PREFIX, NEXT_INTERCEPTION_MARKER_PREFIX,
NEXT_QUERY_PARAM_PREFIX, NEXT_QUERY_PARAM_PREFIX,
} from '../../lib/constants' } from '../../lib/constants'
import { recoverMojibake } from '../lib/fix-mojibake' import { decodeMojibake } from '../lib/mojibake'
/** /**
* Converts a Node.js IncomingHttpHeaders object to a Headers object. Any * Converts a Node.js IncomingHttpHeaders object to a Headers object. Any
...@@ -27,7 +27,7 @@ export function fromNodeOutgoingHttpHeaders( ...@@ -27,7 +27,7 @@ export function fromNodeOutgoingHttpHeaders(
} }
// Try to recover from Mojibake (UTF-8 bytes interpreted as Latin-1) // Try to recover from Mojibake (UTF-8 bytes interpreted as Latin-1)
v = recoverMojibake(v) v = decodeMojibake(v)
// Encode non-ASCII characters to prevent corruption by the Headers API. // Encode non-ASCII characters to prevent corruption by the Headers API.
// The Headers API in edge runtime doesn't properly preserve Latin-1 characters. // The Headers API in edge runtime doesn't properly preserve Latin-1 characters.
...@@ -203,33 +203,7 @@ export function encodeHeaderValue(value: string): string { ...@@ -203,33 +203,7 @@ export function encodeHeaderValue(value: string): string {
// Check if the value contains non-ASCII characters // Check if the value contains non-ASCII characters
// eslint-disable-next-line no-control-regex // eslint-disable-next-line no-control-regex
if (/[^\u0000-\u007F]/.test(value)) { if (/[^\u0000-\u007F]/.test(value)) {
// Use encodeURIComponent but preserve spaces and common header-safe characters
return encodeURIComponent(value) return encodeURIComponent(value)
.replace(/%20/g, ' ') // Preserve spaces
.replace(/%3A/g, ':') // Preserve colons (common in headers)
.replace(/%2D/g, '-') // Preserve hyphens
.replace(/%2E/g, '.') // Preserve dots
.replace(/%5F/g, '_') // Preserve underscores
} }
return value return value
} }
/**
* Decodes header values that were encoded with encodeHeaderValue.
*
* @param value - The header value to decode
* @returns The decoded header value
*/
export function decodeHeaderValue(value: string): string {
try {
// decodeURIComponent will throw if the value is not properly encoded
// Only attempt to decode if the value contains percent-encoded sequences
if (/%[0-9A-Fa-f]{2}/.test(value)) {
return decodeURIComponent(value.replace(/ /g, '%20'))
}
return value
} catch {
// If decoding fails, return the original value
return value
}
}
...@@ -2,12 +2,12 @@ import { NextResponse } from 'next/server' ...@@ -2,12 +2,12 @@ import { NextResponse } from 'next/server'
/** /**
* Middleware that forwards request headers including non-ASCII values. * Middleware that forwards request headers including non-ASCII values.
* This simulates how Cloudflare or other CDNs might add headers with * This simulates how CDNs might add headers with non-ASCII characters
* non-ASCII characters (e.g., cf-ipcity: Montréal). * (e.g., x-city: Montréal).
* *
* The issue (#85631) is that when middleware forwards headers containing * The issue (#85631) is that when middleware forwards headers containing
* non-ASCII characters, they get passed through x-middleware-request-* headers * non-ASCII characters, they get passed through x-middleware-request-* headers
* which must be ASCII-safe. This causes errors on Vercel's runtime. * which must be ASCII-safe.
* *
* @param {import('next/server').NextRequest} request * @param {import('next/server').NextRequest} request
*/ */
......
...@@ -4,20 +4,14 @@ import { fetchViaHTTP } from 'next-test-utils' ...@@ -4,20 +4,14 @@ import { fetchViaHTTP } from 'next-test-utils'
import { createNext, FileRef } from 'e2e-utils' import { createNext, FileRef } from 'e2e-utils'
/** /**
* Test for issue #85631: Middleware crashes when processing headers with non-ASCII characters.
* *
* When middleware forwards headers containing non-ASCII values (e.g., Cloudflare's * When middleware forwards headers containing non-ASCII values
* cf-ipcity: "Montréal"), the request fails because x-middleware-request-* headers * (e.g. x-city: "Montréal"), the request fails because x-middleware-request-* headers
* must be ASCII-safe per HTTP spec. * must be ASCII-safe per HTTP spec.
* *
* IMPORTANT: In real-world scenarios (Cloudflare, browsers), UTF-8 header values are * IMPORTANT: In real-world scenarios, UTF-8 header values are
* transmitted as bytes and interpreted by Node.js as Latin-1, causing "Mojibake". * transmitted as bytes and interpreted by Node.js as Latin-1, causing "Mojibake".
* For example, "Montréal" (UTF-8: 4D 6F 6E 74 72 C3 A9 61 6C) arrives as "Montréal" * For example, "Montréal" (UTF-8: 4D 6F 6E 74 72 C3 A9 61 6C) arrives as "Montréal" when those UTF-8 bytes are interpreted as Latin-1 characters.
* when those UTF-8 bytes are interpreted as Latin-1 characters.
*
* node-fetch (used in tests) handles headers differently - it converts non-ASCII to
* replacement characters. So these tests send the Mojibake directly to simulate
* what the server actually receives in production.
*/ */
/** /**
...@@ -48,7 +42,7 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -48,7 +42,7 @@ describe('Middleware Non-ASCII Headers', () => {
}) })
it('should handle headers with French accented characters (Montréal)', async () => { it('should handle headers with French accented characters (Montréal)', async () => {
// Simulate what the server receives when Cloudflare sends "Montréal" as UTF-8 // Simulate what the server receives when sends "Montréal" as UTF-8
// The UTF-8 bytes get interpreted as Latin-1, producing "Montréal" // The UTF-8 bytes get interpreted as Latin-1, producing "Montréal"
const res = await fetchViaHTTP( const res = await fetchViaHTTP(
next.url, next.url,
...@@ -63,7 +57,6 @@ describe('Middleware Non-ASCII Headers', () => { ...@@ -63,7 +57,6 @@ describe('Middleware Non-ASCII Headers', () => {
expect(res.status).toBe(200) expect(res.status).toBe(200)
const headers = await res.json() const headers = await res.json()
// The middleware should recover the original UTF-8 string
expect(headers['x-city']).toBe('Montréal') expect(headers['x-city']).toBe('Montréal')
}) })
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment