Unverified Commit e2262334 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Implement improved deobfuscation for free calls and module identifiers. (#85060)

Improve deobfuscation of 'imported module identifier' and attempt to deobfuscate error messages logged by the server.

Before
```
 ⨯ TypeError: (0 , __TURBOPACK__imported__module__$5b$project$5d2f$examples$2f$with$2d$turbopack$2f$app$2f$foo$2e$ts__$5b$app$2d$rsc$5d$__$28$ecmascript$29$__.foo) is not a function
    at Page (app/page.tsx:5:6)
  3 | export default function Page() {
  4 |   /** @ts-ignore */
> 5 |   foo();
    |      ^
  6 |   return <h1>Hello, Next.js!</h1>;
  7 | }
  8 | {
```
after 
```
 ⨯ TypeError: {imported module ./examples/with-turbopack/app/foo.ts}.foo is not a function
    at Page (app/page.tsx:5:6)
  3 | export default function Page() {
  4 |   /** @ts-ignore */
> 5 |   foo();
    |      ^
  6 |   return <h1>Hello, Next.js!</h1>;
  7 | }
  8 | {
```

Similarly in devtools
before:

![image.png](https://app.graphite.dev/user-attachments/assets/fd50d94a-d34a-4026-862e-0ca8ee6658ba.png)

after

![image.png](https://app.graphite.dev/user-attachments/assets/d9080ed7-29c8-428a-831a-9bbcd818cc3e.png)


This is definitely an improvement but i am not sure i put the deobfuscation logic in the best place for the console case.  Happy for feedback
parent f50c769f
......@@ -21,6 +21,12 @@
"strictNullChecks": true,
"target": "ES2017"
},
"include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"],
"include": [
"next-env.d.ts",
".next/types/**/*.ts",
"**/*.ts",
"**/*.tsx",
".next/dev/types/**/*.ts"
],
"exclude": ["node_modules", "**/*.test.ts", "**/*.test.tsx"]
}
import { foo } from "./foo";
export default function Page() {
/** @ts-ignore */
foo();
return <h1>Hello, Next.js!</h1>;
}
......@@ -20,6 +20,12 @@
}
]
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx",
".next/types/**/*.ts",
".next/dev/types/**/*.ts"
],
"exclude": ["node_modules"]
}
import React from 'react'
import {
decodeMagicIdentifier,
MAGIC_IDENTIFIER_REGEX,
} from '../../../../shared/lib/magic-identifier'
import { deobfuscateTextParts } from '../../../../shared/lib/magic-identifier'
const linkRegex = /https?:\/\/[^\s/$.?#].[^\s)'"]*/i
const splitRegexp = new RegExp(`(${MAGIC_IDENTIFIER_REGEX.source}|\\s+)`)
export const HotlinkedText: React.FC<{
text: string
matcher?: (text: string) => boolean
}> = function HotlinkedText(props) {
const { text, matcher } = props
const wordsAndWhitespaces = text.split(splitRegexp)
// Deobfuscate the entire text first
const deobfuscatedParts = deobfuscateTextParts(text)
return (
<>
{wordsAndWhitespaces.map((word, index) => {
if (linkRegex.test(word)) {
const link = linkRegex.exec(word)!
const href = link[0]
// If link matcher is present but the link doesn't match, don't turn it into a link
if (typeof matcher === 'function' && !matcher(href)) {
return word
}
return (
<React.Fragment key={`link-${index}`}>
<a href={href} target="_blank" rel="noreferrer noopener">
{word}
</a>
</React.Fragment>
)
}
try {
const decodedWord = decodeMagicIdentifier(word)
if (decodedWord !== word) {
return (
<i key={`ident-${index}`}>
{'{'}
{decodedWord}
{'}'}
</i>
)
}
} catch (e) {
{deobfuscatedParts.map(([type, part], outerIndex) => {
if (type === 'raw') {
return (
<i key={`ident-${index}`}>
{'{'}
{word} (decoding failed: {'' + e}){'}'}
</i>
part
// Split on whitespace and links
.split(/(\s+|https?:\/\/[^\s/$.?#].[^\s)'"]*)/)
.map((rawPart, index) => {
if (linkRegex.test(rawPart)) {
const link = linkRegex.exec(rawPart)!
const href = link[0]
// If link matcher is present but the link doesn't match, don't turn it into a link
if (typeof matcher === 'function' && !matcher(href)) {
return (
<React.Fragment key={`link-${outerIndex}-${index}`}>
{rawPart}
</React.Fragment>
)
}
return (
<React.Fragment key={`link-${outerIndex}-${index}`}>
<a href={href} target="_blank" rel="noreferrer noopener">
{rawPart}
</a>
</React.Fragment>
)
} else {
return (
<React.Fragment key={`text-${outerIndex}-${index}`}>
{rawPart}
</React.Fragment>
)
}
})
)
} else if (type === 'deobfuscated') {
// italicize the deobfuscated part
return <i key={`ident-${outerIndex}`}>{part}</i>
} else {
throw new Error(`Unknown text part type: ${type}`)
}
return <React.Fragment key={`text-${index}`}>{word}</React.Fragment>
})}
</>
)
......
......@@ -88,6 +88,7 @@ import {
import { isParallelRouteSegment } from '../../../shared/lib/segment'
import { ensureLeadingSlash } from '../../../shared/lib/page-path/ensure-leading-slash'
import { Lockfile } from '../../../build/lockfile'
import { deobfuscateText } from '../../../shared/lib/magic-identifier'
export type SetupOpts = {
renderServer: LazyRenderServerInstance
......@@ -1224,6 +1225,9 @@ async function startWatcher(
err: unknown,
type?: 'unhandledRejection' | 'uncaughtException' | 'warning' | 'app-dir'
) {
if (err instanceof Error) {
err.message = deobfuscateText(err.message)
}
if (err instanceof ModuleBuildError) {
// Errors that may come from issues from the user's code
Log.error(err.message)
......
import {
decodeMagicIdentifier,
MAGIC_IDENTIFIER_REGEX,
deobfuscateModuleId,
removeFreeCallWrapper,
deobfuscateText,
deobfuscateTextParts,
} from './magic-identifier'
describe('decodeMagicIdentifier', () => {
// Basic decoding tests (ported from Rust)
test('decodes module evaluation', () => {
expect(decodeMagicIdentifier('__TURBOPACK__module__evaluation__')).toBe(
'module evaluation'
)
})
test('decodes path with slashes', () => {
expect(decodeMagicIdentifier('__TURBOPACK__Hello$2f$World__')).toBe(
'Hello/World'
)
})
test('decodes emoji', () => {
expect(decodeMagicIdentifier('__TURBOPACK__Hello$_1f600$World__')).toBe(
'Hello😀World'
)
})
test('returns unchanged if not a magic identifier', () => {
expect(decodeMagicIdentifier('regular_identifier')).toBe(
'regular_identifier'
)
})
})
describe('MAGIC_IDENTIFIER_REGEX', () => {
test('matches magic identifiers globally', () => {
const text =
'Hello __TURBOPACK__Hello__World__ and __TURBOPACK__foo$2f$bar__'
const matches = text.match(MAGIC_IDENTIFIER_REGEX)
expect(matches).toHaveLength(2)
})
})
describe('deobfuscateModuleId', () => {
test('replaces [project] with .', () => {
expect(
deobfuscateModuleId('[project]/examples/with-turbopack/app/foo.ts')
).toBe('./examples/with-turbopack/app/foo.ts')
})
test('removes content in square brackets', () => {
expect(
deobfuscateModuleId('./examples/with-turbopack/app/foo.ts [app-rsc]')
).toBe('./examples/with-turbopack/app/foo.ts')
})
test('removes content in parentheses', () => {
expect(
deobfuscateModuleId('./examples/with-turbopack/app/foo.ts (ecmascript)')
).toBe('./examples/with-turbopack/app/foo.ts')
})
test('removes content in angle brackets', () => {
expect(
deobfuscateModuleId('./examples/with-turbopack/app/foo.ts <locals>')
).toBe('./examples/with-turbopack/app/foo.ts')
})
test('handles combined cleanup', () => {
expect(
deobfuscateModuleId(
'[project]/examples/with-turbopack/app/foo.ts [app-rsc] (ecmascript)'
)
).toBe('./examples/with-turbopack/app/foo.ts')
})
test('handles parenthesis in path', () => {
expect(
deobfuscateModuleId(
'[project]/examples/(group)/with-turbopack/app/foo.ts [app-rsc] (ecmascript)'
)
).toBe('./examples/(group)/with-turbopack/app/foo.ts')
})
})
describe('removeFreeCallWrapper', () => {
test('removes (0, ) wrapper', () => {
expect(removeFreeCallWrapper('(0, __TURBOPACK__foo__.bar)')).toBe(
'__TURBOPACK__foo__.bar'
)
})
test('removes (0 , ) wrapper with spaces', () => {
expect(removeFreeCallWrapper('(0 , __TURBOPACK__foo__.bar)')).toBe(
'__TURBOPACK__foo__.bar'
)
})
test('leaves non-free-call expressions unchanged', () => {
expect(removeFreeCallWrapper('(foo, bar)')).toBe('(foo, bar)')
expect(removeFreeCallWrapper('foo()')).toBe('foo()')
})
})
describe('deobfuscateText', () => {
test('deobfuscates complete error message with imported module', () => {
const input =
'(0 , __TURBOPACK__imported__module__$5b$project$5d2f$examples$2f$with$2d$turbopack$2f$app$2f$foo$2e$ts__$5b$app$2d$rsc$5d$__$28$ecmascript$29$__.foo) is not a function'
const output = deobfuscateText(input)
expect(output).toBe(
'{imported module ./examples/with-turbopack/app/foo.ts}.foo is not a function'
)
})
test('handles multiple magic identifiers', () => {
const input =
'__TURBOPACK__module__evaluation__ called __TURBOPACK__foo$2f$bar__'
const output = deobfuscateText(input)
expect(output).toBe('{module evaluation} called {foo/bar}')
})
test('leaves regular text unchanged', () => {
const input = 'This is a regular error message'
expect(deobfuscateText(input)).toBe(input)
})
})
describe('deobfuscateTextParts', () => {
test('returns discriminated parts with raw and deobfuscated text', () => {
const input = 'Error in __TURBOPACK__module__evaluation__ at line 10'
const output = deobfuscateTextParts(input)
expect(output).toEqual([
['raw', 'Error in '],
['deobfuscated', '{module evaluation}'],
['raw', ' at line 10'],
])
})
test('handles multiple magic identifiers with interleaved raw text', () => {
const input =
'__TURBOPACK__module__evaluation__ called __TURBOPACK__foo$2f$bar__'
const output = deobfuscateTextParts(input)
expect(output).toEqual([
['deobfuscated', '{module evaluation}'],
['raw', ' called '],
['deobfuscated', '{foo/bar}'],
])
})
test('returns single raw part for text without magic identifiers', () => {
const input = 'This is a regular error message'
const output = deobfuscateTextParts(input)
expect(output).toEqual([['raw', 'This is a regular error message']])
})
test('handles imported module with free call wrapper', () => {
const input =
'(0 , __TURBOPACK__imported__module__$5b$project$5d2f$examples$2f$with$2d$turbopack$2f$app$2f$foo$2e$ts__$5b$app$2d$rsc$5d$__$28$ecmascript$29$__.foo) is not a function'
const output = deobfuscateTextParts(input)
expect(output).toEqual([
[
'deobfuscated',
'{imported module ./examples/with-turbopack/app/foo.ts}',
],
['raw', '.foo is not a function'],
])
})
test('produces same result as deobfuscateText when joined', () => {
const input =
'Error in __TURBOPACK__module__evaluation__ at __TURBOPACK__foo$2f$bar__'
const parts = deobfuscateTextParts(input)
const joined = parts.map((part) => part[1]).join('')
expect(joined).toBe(deobfuscateText(input))
})
})
......@@ -93,3 +93,122 @@ export function decodeMagicIdentifier(identifier: string): string {
}
export const MAGIC_IDENTIFIER_REGEX = /__TURBOPACK__[a-zA-Z0-9_$]+__/g
/**
* Cleans up module IDs by removing implementation details.
* - Replaces [project] with .
* - Removes content in brackets [], parentheses (), and angle brackets <>
*/
export function deobfuscateModuleId(moduleId: string): string {
return (
moduleId
// Replace [project] with .
.replace(/\[project\]/g, '.')
// Remove content in square brackets (e.g. [app-rsc])
.replace(/\s\[([^\]]*)\]/g, '')
// Remove content in parentheses (e.g. (ecmascript))
.replace(/\s\(([^)]*)\)/g, '')
// Remove content in angle brackets (e.g. <locals>)
.replace(/\s<([^>]*)>/g, '')
// Clean up any extra whitespace
.trim()
)
}
/**
* Removes the free call wrapper pattern (0, expr) from expressions.
* This is a JavaScript pattern to call a function without binding 'this',
* but it's noise for developers reading error messages.
*/
export function removeFreeCallWrapper(text: string): string {
// Match (0, <ident>.<ident>) patterns anywhere in the text the beginning
// Use Unicode property escapes (\p{ID_Start}, \p{ID_Continue}) for full JS identifier support
// Requires the 'u' (unicode) flag in the regex
return text.replace(
/\(0\s*,\s*(__TURBOPACK__[a-zA-Z0-9_$]+__\.[\p{ID_Start}_$][\p{ID_Continue}$]*)\)/u,
'$1'
)
}
export type TextPartType = 'raw' | 'deobfuscated'
/**
* Deobfuscates text and returns an array of discriminated parts.
* Each part is a tuple of [type, string] where type is either 'raw' (unchanged text)
* or 'deobfuscated' (a magic identifier that was decoded).
*
* This is useful when you need to process or display deobfuscated and raw text differently.
*/
export function deobfuscateTextParts(
text: string
): Array<[TextPartType, string]> {
// First, remove free call wrappers
const withoutFreeCall = removeFreeCallWrapper(text)
const parts: Array<[TextPartType, string]> = []
let lastIndex = 0
// Create a new regex instance for global matching
const regex = new RegExp(MAGIC_IDENTIFIER_REGEX.source, 'g')
for (
let match = regex.exec(withoutFreeCall);
match !== null;
match = regex.exec(withoutFreeCall)
) {
const matchStart = match.index
const matchEnd = regex.lastIndex
const ident = match[0]
// Add raw text before this match (if any)
if (matchStart > lastIndex) {
const rawText = withoutFreeCall.substring(lastIndex, matchStart)
parts.push(['raw', rawText])
}
// Process and add the deobfuscated part
try {
const decoded = decodeMagicIdentifier(ident)
// If it was a magic identifier, clean up the module ID
if (decoded !== ident) {
// Check if this is an "imported module" reference
const importedModuleMatch = decoded.match(/^imported module (.+)$/)
if (importedModuleMatch) {
// Clean the entire module path (which includes [app-rsc], etc.)
const modulePathWithMetadata = importedModuleMatch[1]
const cleaned = deobfuscateModuleId(modulePathWithMetadata)
parts.push(['deobfuscated', `{imported module ${cleaned}}`])
} else {
const cleaned = deobfuscateModuleId(decoded)
parts.push(['deobfuscated', `{${cleaned}}`])
}
} else {
// Not actually a magic identifier, treat as raw
parts.push(['raw', ident])
}
} catch (e) {
parts.push(['deobfuscated', `{${ident} (decoding failed: ${e})}`])
}
lastIndex = matchEnd
}
// Add any remaining raw text after the last match
if (lastIndex < withoutFreeCall.length) {
const rawText = withoutFreeCall.substring(lastIndex)
parts.push(['raw', rawText])
}
return parts
}
/**
* Deobfuscates text by:
* 1. Decoding magic identifiers
* 2. Cleaning up module IDs
* 3. Removing free call wrappers
*/
export function deobfuscateText(text: string): string {
const parts = deobfuscateTextParts(text)
return parts.map((part) => part[1]).join('')
}
......@@ -7,10 +7,7 @@ import type {
import { bold, green, magenta, red } from '../../../lib/picocolors'
import isInternal from '../is-internal'
import {
decodeMagicIdentifier,
MAGIC_IDENTIFIER_REGEX,
} from '../magic-identifier'
import { deobfuscateText } from '../magic-identifier'
import type { EntryKey } from './entry-key'
import * as Log from '../../../build/output/log'
import type { NextConfigComplete } from '../../../server/config-shared'
......@@ -288,23 +285,20 @@ function isNodeModulesIssue(issue: Issue): boolean {
}
export function renderStyledStringToErrorAnsi(string: StyledString): string {
function decodeMagicIdentifiers(str: string): string {
return str.replaceAll(MAGIC_IDENTIFIER_REGEX, (ident) => {
try {
return magenta(`{${decodeMagicIdentifier(ident)}}`)
} catch (e) {
return magenta(`{${ident} (decoding failed: ${e})}`)
}
})
function applyDeobfuscation(str: string): string {
// Use shared deobfuscate function and apply magenta color to identifiers
const deobfuscated = deobfuscateText(str)
// Color any {...} wrapped identifiers with magenta
return deobfuscated.replace(/\{([^}]+)\}/g, (match) => magenta(match))
}
switch (string.type) {
case 'text':
return decodeMagicIdentifiers(string.value)
return applyDeobfuscation(string.value)
case 'strong':
return bold(red(decodeMagicIdentifiers(string.value)))
return bold(red(applyDeobfuscation(string.value)))
case 'code':
return green(decodeMagicIdentifiers(string.value))
return green(applyDeobfuscation(string.value))
case 'line':
return string.value.map(renderStyledStringToErrorAnsi).join('')
case 'stack':
......
......@@ -1222,7 +1222,7 @@ describe('Cache Components Errors', () => {
],
},
{
"description": "(0 , <turbopack-module-id>.cookies)(...).get is not a function",
"description": "<turbopack-module-id>.cookies(...).get is not a function",
"environmentLabel": "Prerender",
"label": "Runtime TypeError",
"source": "app/sync-cookies/page.tsx (18:36) @ CookiesReadingComponent
......@@ -1404,7 +1404,7 @@ describe('Cache Components Errors', () => {
],
},
{
"description": "(0 , <turbopack-module-id>.cookies)(...).get is not a function",
"description": "<turbopack-module-id>.cookies(...).get is not a function",
"environmentLabel": "Server",
"label": "Runtime TypeError",
"source": "app/sync-cookies-runtime/page.tsx (24:36) @ CookiesReadingComponent
......@@ -1573,7 +1573,7 @@ describe('Cache Components Errors', () => {
],
},
{
"description": "(0 , <turbopack-module-id>.headers)(...).get is not a function",
"description": "<turbopack-module-id>.headers(...).get is not a function",
"environmentLabel": "Prerender",
"label": "Runtime TypeError",
"source": "app/sync-headers/page.tsx (18:40) @ HeadersReadingComponent
......@@ -1755,7 +1755,7 @@ describe('Cache Components Errors', () => {
],
},
{
"description": "(0 , <turbopack-module-id>.headers)(...).get is not a function",
"description": "<turbopack-module-id>.headers(...).get is not a function",
"environmentLabel": "Server",
"label": "Runtime TypeError",
"source": "app/sync-headers-runtime/page.tsx (24:40) @ HeadersReadingComponent
......
......@@ -268,6 +268,8 @@ impl ReferencedAsset {
chunking_context: Vc<Box<dyn ChunkingContext>>,
) -> Result<String> {
let id = asset.chunk_item_id(chunking_context).await?;
// There are a number of places in `next` that match on this prefix.
// See `packages/next/src/shared/lib/magic-identifier.ts`
Ok(magic_identifier::mangle(&format!("imported module {id}")))
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment