Commit f7d4dbd1 authored by Jiachi Liu's avatar Jiachi Liu Committed by Janka Uryga

Support esm externals in app router (#65041)

Support `esmExternals` working in app router

`esmExternals` was disabled for app router that most of the packages are
picking up the CJS bundles for externals. This PR enables to resolve the
ESM bundle for external packages.

We have two issues discovered while enabling the flag, any esm external
packages will fail in client page SSR and server action. We fixed them
by changing the below bundling logics:

* When a client page having a async dependency, we can await the page
during in rendering
* When a server action having a async dependency, we changed the server
action entry creation with webpack along with the server client entry
creation together, then webpack can handle the modules async propagation
properly.

Fixes #60756
Closes NEXT-2435
Closes NEXT-2472
Closes NEXT-3225

---------
Co-authored-by: 's avatarTobias Koppers <tobias.koppers@googlemail.com>
parent 435780a8
import { WEBPACK_LAYERS } from '../lib/constants'
import type { WebpackLayerName } from '../lib/constants' import type { WebpackLayerName } from '../lib/constants'
import type { NextConfigComplete } from '../server/config-shared'
import type { ResolveOptions } from 'webpack'
import { defaultOverrides } from '../server/require-hook' import { defaultOverrides } from '../server/require-hook'
import { BARREL_OPTIMIZATION_PREFIX } from '../shared/lib/constants' import { BARREL_OPTIMIZATION_PREFIX } from '../shared/lib/constants'
import path from '../shared/lib/isomorphic/path' import path from '../shared/lib/isomorphic/path'
...@@ -10,7 +11,6 @@ import { ...@@ -10,7 +11,6 @@ import {
NODE_RESOLVE_OPTIONS, NODE_RESOLVE_OPTIONS,
} from './webpack-config' } from './webpack-config'
import { isWebpackAppLayer, isWebpackServerOnlyLayer } from './utils' import { isWebpackAppLayer, isWebpackServerOnlyLayer } from './utils'
import type { NextConfigComplete } from '../server/config-shared'
import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep' import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep'
const reactPackagesRegex = /^(react|react-dom|react-server-dom-webpack)($|\/)/ const reactPackagesRegex = /^(react|react-dom|react-server-dom-webpack)($|\/)/
...@@ -25,15 +25,6 @@ const externalPattern = new RegExp( ...@@ -25,15 +25,6 @@ const externalPattern = new RegExp(
const nodeModulesRegex = /node_modules[/\\].*\.[mc]?js$/ const nodeModulesRegex = /node_modules[/\\].*\.[mc]?js$/
function containsImportInPackages(
request: string,
packages: string[]
): boolean {
return packages.some(
(pkg) => request === pkg || request.startsWith(pkg + '/')
)
}
export function isResourceInPackages( export function isResourceInPackages(
resource: string, resource: string,
packageNames?: string[], packageNames?: string[],
...@@ -57,9 +48,9 @@ export async function resolveExternal( ...@@ -57,9 +48,9 @@ export async function resolveExternal(
context: string, context: string,
request: string, request: string,
isEsmRequested: boolean, isEsmRequested: boolean,
optOutBundlingPackages: string[], _optOutBundlingPackages: string[],
getResolve: ( getResolve: (
options: any options: ResolveOptions
) => ( ) => (
resolveContext: string, resolveContext: string,
resolveRequest: string resolveRequest: string
...@@ -78,19 +69,12 @@ export async function resolveExternal( ...@@ -78,19 +69,12 @@ export async function resolveExternal(
let isEsm: boolean = false let isEsm: boolean = false
const preferEsmOptions = const preferEsmOptions =
esmExternals && esmExternals && isEsmRequested ? [true, false] : [false]
isEsmRequested &&
// For package that marked as externals that should be not bundled,
// we don't resolve them as ESM since it could be resolved as async module,
// such as `import(external package)` in the bundle, valued as a `Promise`.
!containsImportInPackages(request, optOutBundlingPackages)
? [true, false]
: [false]
for (const preferEsm of preferEsmOptions) { for (const preferEsm of preferEsmOptions) {
const resolve = getResolve( const resolveOptions = preferEsm ? esmResolveOptions : nodeResolveOptions
preferEsm ? esmResolveOptions : nodeResolveOptions
) const resolve = getResolve(resolveOptions)
// Resolve the import with the webpack provided context, this // Resolve the import with the webpack provided context, this
// ensures we're resolving the correct version when multiple // ensures we're resolving the correct version when multiple
...@@ -273,23 +257,6 @@ export function makeExternalHandler({ ...@@ -273,23 +257,6 @@ export function makeExternalHandler({
return resolveNextExternal(request) return resolveNextExternal(request)
} }
// Early return if the request needs to be bundled, such as in the client layer.
// Treat react packages and next internals as external for SSR layer,
// also map react to builtin ones with require-hook.
// Otherwise keep continue the process to resolve the externals.
if (layer === WEBPACK_LAYERS.serverSideRendering) {
const isRelative = request.startsWith('.')
const fullRequest = isRelative
? normalizePathSep(path.join(context, request))
: request
// Check if it's opt out bundling package first
if (containsImportInPackages(fullRequest, optOutBundlingPackages)) {
return fullRequest
}
return resolveNextExternal(fullRequest)
}
// TODO-APP: Let's avoid this resolve call as much as possible, and eventually get rid of it. // TODO-APP: Let's avoid this resolve call as much as possible, and eventually get rid of it.
const resolveResult = await resolveExternal( const resolveResult = await resolveExternal(
dir, dir,
...@@ -320,6 +287,13 @@ export function makeExternalHandler({ ...@@ -320,6 +287,13 @@ export function makeExternalHandler({
return return
} }
const isOptOutBundling = optOutBundlingPackageRegex.test(res)
// Apply bundling rules to all app layers.
// Since handleExternals only handle the server layers, we don't need to exclude client here
if (!isOptOutBundling && isAppLayer) {
return
}
// ESM externals can only be imported (and not required). // ESM externals can only be imported (and not required).
// Make an exception in loose mode. // Make an exception in loose mode.
if (!isEsmRequested && isEsm && !looseEsmExternals && !isLocal) { if (!isEsmRequested && isEsm && !looseEsmExternals && !isLocal) {
...@@ -370,13 +344,11 @@ export function makeExternalHandler({ ...@@ -370,13 +344,11 @@ export function makeExternalHandler({
const resolvedBundlingOptOutRes = resolveBundlingOptOutPackages({ const resolvedBundlingOptOutRes = resolveBundlingOptOutPackages({
resolvedRes: res, resolvedRes: res,
optOutBundlingPackageRegex,
config, config,
resolvedExternalPackageDirs, resolvedExternalPackageDirs,
isEsm,
isAppLayer, isAppLayer,
layer,
externalType, externalType,
isOptOutBundling,
request, request,
}) })
if (resolvedBundlingOptOutRes) { if (resolvedBundlingOptOutRes) {
...@@ -390,41 +362,34 @@ export function makeExternalHandler({ ...@@ -390,41 +362,34 @@ export function makeExternalHandler({
function resolveBundlingOptOutPackages({ function resolveBundlingOptOutPackages({
resolvedRes, resolvedRes,
optOutBundlingPackageRegex,
config, config,
resolvedExternalPackageDirs, resolvedExternalPackageDirs,
isEsm,
isAppLayer, isAppLayer,
layer,
externalType, externalType,
isOptOutBundling,
request, request,
}: { }: {
resolvedRes: string resolvedRes: string
optOutBundlingPackageRegex: RegExp
config: NextConfigComplete config: NextConfigComplete
resolvedExternalPackageDirs: Map<string, string> resolvedExternalPackageDirs: Map<string, string>
isEsm: boolean
isAppLayer: boolean isAppLayer: boolean
layer: WebpackLayerName | null
externalType: string externalType: string
isOptOutBundling: boolean
request: string request: string
}) { }) {
const shouldBeBundled =
isResourceInPackages(
resolvedRes,
config.transpilePackages,
resolvedExternalPackageDirs
) ||
(isEsm && isAppLayer) ||
(!isAppLayer && config.experimental.bundlePagesExternals)
if (nodeModulesRegex.test(resolvedRes)) { if (nodeModulesRegex.test(resolvedRes)) {
const isOptOutBundling = optOutBundlingPackageRegex.test(resolvedRes) const shouldBundlePages =
if (isWebpackServerOnlyLayer(layer)) { !isAppLayer &&
if (isOptOutBundling) { config.experimental.bundlePagesExternals &&
return `${externalType} ${request}` // Externalize if opted out !isOptOutBundling
} const shouldBeBundled =
} else if (!shouldBeBundled || isOptOutBundling) { shouldBundlePages ||
isResourceInPackages(
resolvedRes,
config.transpilePackages,
resolvedExternalPackageDirs
)
if (!shouldBeBundled) {
return `${externalType} ${request}` // Externalize if not bundled or opted out return `${externalType} ${request}` // Externalize if not bundled or opted out
} }
} }
......
...@@ -430,75 +430,6 @@ export class FlightClientEntryPlugin { ...@@ -430,75 +430,6 @@ export class FlightClientEntryPlugin {
) )
} }
compilation.hooks.finishModules.tapPromise(PLUGIN_NAME, async () => {
const addedClientActionEntryList: Promise<any>[] = []
const actionMapsPerClientEntry: Record<string, Map<string, string[]>> = {}
// We need to create extra action entries that are created from the
// client layer.
// Start from each entry's created SSR dependency from our previous step.
for (const [name, ssrEntryDependencies] of Object.entries(
createdSSRDependenciesForEntry
)) {
// Collect from all entries, e.g. layout.js, page.js, loading.js, ...
// add aggregate them.
const actionEntryImports = this.collectClientActionsFromDependencies({
compilation,
dependencies: ssrEntryDependencies,
})
if (actionEntryImports.size > 0) {
if (!actionMapsPerClientEntry[name]) {
actionMapsPerClientEntry[name] = new Map()
}
actionMapsPerClientEntry[name] = new Map([
...actionMapsPerClientEntry[name],
...actionEntryImports,
])
}
}
for (const [name, actionEntryImports] of Object.entries(
actionMapsPerClientEntry
)) {
// If an action method is already created in the server layer, we don't
// need to create it again in the action layer.
// This is to avoid duplicate action instances and make sure the module
// state is shared.
let remainingClientImportedActions = false
const remainingActionEntryImports = new Map<string, string[]>()
for (const [dep, actionNames] of actionEntryImports) {
const remainingActionNames = []
for (const actionName of actionNames) {
const id = name + '@' + dep + '@' + actionName
if (!createdActions.has(id)) {
remainingActionNames.push(actionName)
}
}
if (remainingActionNames.length > 0) {
remainingActionEntryImports.set(dep, remainingActionNames)
remainingClientImportedActions = true
}
}
if (remainingClientImportedActions) {
addedClientActionEntryList.push(
this.injectActionEntry({
compiler,
compilation,
actions: remainingActionEntryImports,
entryName: name,
bundlePath: name,
fromClient: true,
})
)
}
}
await Promise.all(addedClientActionEntryList)
return
})
// Invalidate in development to trigger recompilation // Invalidate in development to trigger recompilation
const invalidator = getInvalidator(compiler.outputPath) const invalidator = getInvalidator(compiler.outputPath)
// Check if any of the entry injections need an invalidation // Check if any of the entry injections need an invalidation
...@@ -521,6 +452,72 @@ export class FlightClientEntryPlugin { ...@@ -521,6 +452,72 @@ export class FlightClientEntryPlugin {
// Wait for action entries to be added. // Wait for action entries to be added.
await Promise.all(addActionEntryList) await Promise.all(addActionEntryList)
const addedClientActionEntryList: Promise<any>[] = []
const actionMapsPerClientEntry: Record<string, Map<string, string[]>> = {}
// We need to create extra action entries that are created from the
// client layer.
// Start from each entry's created SSR dependency from our previous step.
for (const [name, ssrEntryDependencies] of Object.entries(
createdSSRDependenciesForEntry
)) {
// Collect from all entries, e.g. layout.js, page.js, loading.js, ...
// add aggregate them.
const actionEntryImports = this.collectClientActionsFromDependencies({
compilation,
dependencies: ssrEntryDependencies,
})
if (actionEntryImports.size > 0) {
if (!actionMapsPerClientEntry[name]) {
actionMapsPerClientEntry[name] = new Map()
}
actionMapsPerClientEntry[name] = new Map([
...actionMapsPerClientEntry[name],
...actionEntryImports,
])
}
}
for (const [name, actionEntryImports] of Object.entries(
actionMapsPerClientEntry
)) {
// If an action method is already created in the server layer, we don't
// need to create it again in the action layer.
// This is to avoid duplicate action instances and make sure the module
// state is shared.
let remainingClientImportedActions = false
const remainingActionEntryImports = new Map<string, string[]>()
for (const [dep, actionNames] of actionEntryImports) {
const remainingActionNames = []
for (const actionName of actionNames) {
const id = name + '@' + dep + '@' + actionName
if (!createdActions.has(id)) {
remainingActionNames.push(actionName)
}
}
if (remainingActionNames.length > 0) {
remainingActionEntryImports.set(dep, remainingActionNames)
remainingClientImportedActions = true
}
}
if (remainingClientImportedActions) {
addedClientActionEntryList.push(
this.injectActionEntry({
compiler,
compilation,
actions: remainingActionEntryImports,
entryName: name,
bundlePath: name,
fromClient: true,
})
)
}
}
await Promise.all(addedClientActionEntryList)
} }
collectClientActionsFromDependencies({ collectClientActionsFromDependencies({
......
...@@ -260,7 +260,7 @@ async function createComponentTreeInternal({ ...@@ -260,7 +260,7 @@ async function createComponentTreeInternal({
} }
const LayoutOrPage: React.ComponentType<any> | undefined = layoutOrPageMod const LayoutOrPage: React.ComponentType<any> | undefined = layoutOrPageMod
? interopDefault(layoutOrPageMod) ? await interopDefault(layoutOrPageMod)
: undefined : undefined
/** /**
......
...@@ -35,7 +35,7 @@ createNextDescribe( ...@@ -35,7 +35,7 @@ createNextDescribe(
buildCommand: 'pnpm build', buildCommand: 'pnpm build',
skipDeployment: true, skipDeployment: true,
}, },
({ next }) => { ({ next, isTurbopack }) => {
it('should be able to opt-out 3rd party packages being bundled in server components', async () => { it('should be able to opt-out 3rd party packages being bundled in server components', async () => {
await next.fetch('/react-server/optout').then(async (response) => { await next.fetch('/react-server/optout').then(async (response) => {
const result = await resolveStreamResponse(response) const result = await resolveStreamResponse(response)
...@@ -284,14 +284,17 @@ createNextDescribe( ...@@ -284,14 +284,17 @@ createNextDescribe(
}) })
describe('server actions', () => { describe('server actions', () => {
it('should not prefer to resolve esm over cjs for bundling optout packages', async () => { it('should prefer to resolve esm over cjs for bundling optout packages', async () => {
const browser = await next.browser('/optout/action') const browser = await next.browser('/optout/action')
expect(await browser.elementByCss('#dual-pkg-outout p').text()).toBe('') expect(await browser.elementByCss('#dual-pkg-outout p').text()).toBe('')
browser.elementByCss('#dual-pkg-outout button').click() browser.elementByCss('#dual-pkg-outout button').click()
await check(async () => { await check(async () => {
const text = await browser.elementByCss('#dual-pkg-outout p').text() const text = await browser.elementByCss('#dual-pkg-outout p').text()
expect(text).toBe('dual-pkg-optout:cjs') // TODO: enable esm externals for app router in turbopack for actions
expect(text).toBe(
isTurbopack ? 'dual-pkg-optout:cjs' : 'dual-pkg-optout:mjs'
)
return 'success' return 'success'
}, /success/) }, /success/)
}) })
......
...@@ -13,13 +13,15 @@ describe('esm-externals', () => { ...@@ -13,13 +13,15 @@ describe('esm-externals', () => {
const urls = ['/static', '/ssr', '/ssg'] const urls = ['/static', '/ssr', '/ssg']
for (const url of urls) { for (const url of urls) {
// TODO fix webpack behavior // For invalid esm packages that have "import" pointing to a non-esm-flagged module
// webpack is using the CJS version instead but Turbopack is opting out of
// externalizing and bundling the non-esm-flagged module.
const expectedHtml = isTurbopack const expectedHtml = isTurbopack
? /Hello World\+World\+World\+World\+World\+World/ ? /Hello World\+World\+World\+World\+World\+World/
: url === '/static' : url === '/static'
? /Hello World\+World\+Alternative\+World\+World\+World/ ? /Hello World\+World\+Alternative\+World\+World\+World/
: /Hello World\+World\+Alternative\+World\+World\+Alternative/ : /Hello World\+World\+Alternative\+World\+World\+Alternative/
// TODO fix webpack behavior // On client side, webpack always bundlings so it uses the non-esm-flagged module too.
const expectedText = const expectedText =
isTurbopack || url === '/static' isTurbopack || url === '/static'
? /Hello World\+World\+World\+World\+World\+World/ ? /Hello World\+World\+World\+World\+World\+World/
...@@ -40,18 +42,21 @@ describe('esm-externals', () => { ...@@ -40,18 +42,21 @@ describe('esm-externals', () => {
// App dir // App dir
{ {
// TODO App Dir doesn't use esmExternals: true correctly for webpack and Turbopack // TODO App Dir doesn't use esmExternals: true correctly for Turbopack
// so we only verify that the page doesn't crash, but ignore the actual content // so we only verify that the page doesn't crash, but ignore the actual content
// const expectedHtml = /Hello World\+World\+World/ // const expectedHtml = isTurbopack ? /Hello World\+World\+World/ : /Hello World\+World\+Alternative/
const expectedHtml = /Hello Wrong\+Wrong\+Alternative/ const expectedHtml = isTurbopack
// const expectedText = /Hello World\+World\+World/ ? /Hello Wrong\+Wrong\+Alternative/
: /Hello World\+World\+Alternative/
const urls = ['/server', '/client'] const urls = ['/server', '/client']
for (const url of urls) { for (const url of urls) {
const expectedText = const expectedText =
url === '/server' url !== '/server'
? /Hello World\+World\+World/
: isTurbopack
? /Hello Wrong\+Wrong\+Alternative/ ? /Hello Wrong\+Wrong\+Alternative/
: /Hello World\+World\+World/ : /Hello World\+World\+Alternative/
it(`should return the correct SSR HTML for ${url}`, async () => { it(`should return the correct SSR HTML for ${url}`, async () => {
const res = await next.fetch(url) const res = await next.fetch(url)
const html = await res.text() const html = await res.text()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment