Unverified Commit bf89bee3 authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

Support esm externals in app router (#65041)

### What

Support `esmExternals` working in app router

### Why

`esmExternals` was disabled for app router that most of the packages are
picking up the CJS bundles for externals. This PR enables to resolve the
ESM bundle for external packages.

We have two issues discovered while enabling the flag, any esm external
packages will fail in client page SSR and server action. We fixed them
by changing the below bundling logics:

* When a client page having a async dependency, we can await the page
during in rendering
* When a server action having a async dependency, we changed the server
action entry creation with webpack along with the server client entry
creation together, then webpack can handle the modules async propagation
properly.


Fixes #60756 
Closes NEXT-2435
Closes NEXT-2472
Closes NEXT-3225

---------
Co-authored-by: 's avatarTobias Koppers <tobias.koppers@googlemail.com>
parent b91019df
import { WEBPACK_LAYERS } from '../lib/constants'
import type { WebpackLayerName } from '../lib/constants' import type { WebpackLayerName } from '../lib/constants'
import type { NextConfigComplete } from '../server/config-shared'
import type { ResolveOptions } from 'webpack'
import { defaultOverrides } from '../server/require-hook' import { defaultOverrides } from '../server/require-hook'
import { BARREL_OPTIMIZATION_PREFIX } from '../shared/lib/constants' import { BARREL_OPTIMIZATION_PREFIX } from '../shared/lib/constants'
import path from '../shared/lib/isomorphic/path' import path from '../shared/lib/isomorphic/path'
...@@ -10,7 +11,6 @@ import { ...@@ -10,7 +11,6 @@ import {
NODE_RESOLVE_OPTIONS, NODE_RESOLVE_OPTIONS,
} from './webpack-config' } from './webpack-config'
import { isWebpackAppLayer, isWebpackServerOnlyLayer } from './utils' import { isWebpackAppLayer, isWebpackServerOnlyLayer } from './utils'
import type { NextConfigComplete } from '../server/config-shared'
import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep' import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep'
const reactPackagesRegex = /^(react|react-dom|react-server-dom-webpack)($|\/)/ const reactPackagesRegex = /^(react|react-dom|react-server-dom-webpack)($|\/)/
...@@ -25,15 +25,6 @@ const externalPattern = new RegExp( ...@@ -25,15 +25,6 @@ const externalPattern = new RegExp(
const nodeModulesRegex = /node_modules[/\\].*\.[mc]?js$/ const nodeModulesRegex = /node_modules[/\\].*\.[mc]?js$/
function containsImportInPackages(
request: string,
packages: string[]
): boolean {
return packages.some(
(pkg) => request === pkg || request.startsWith(pkg + '/')
)
}
export function isResourceInPackages( export function isResourceInPackages(
resource: string, resource: string,
packageNames?: string[], packageNames?: string[],
...@@ -57,9 +48,9 @@ export async function resolveExternal( ...@@ -57,9 +48,9 @@ export async function resolveExternal(
context: string, context: string,
request: string, request: string,
isEsmRequested: boolean, isEsmRequested: boolean,
optOutBundlingPackages: string[], _optOutBundlingPackages: string[],
getResolve: ( getResolve: (
options: any options: ResolveOptions
) => ( ) => (
resolveContext: string, resolveContext: string,
resolveRequest: string resolveRequest: string
...@@ -78,19 +69,12 @@ export async function resolveExternal( ...@@ -78,19 +69,12 @@ export async function resolveExternal(
let isEsm: boolean = false let isEsm: boolean = false
const preferEsmOptions = const preferEsmOptions =
esmExternals && esmExternals && isEsmRequested ? [true, false] : [false]
isEsmRequested &&
// For package that marked as externals that should be not bundled,
// we don't resolve them as ESM since it could be resolved as async module,
// such as `import(external package)` in the bundle, valued as a `Promise`.
!containsImportInPackages(request, optOutBundlingPackages)
? [true, false]
: [false]
for (const preferEsm of preferEsmOptions) { for (const preferEsm of preferEsmOptions) {
const resolve = getResolve( const resolveOptions = preferEsm ? esmResolveOptions : nodeResolveOptions
preferEsm ? esmResolveOptions : nodeResolveOptions
) const resolve = getResolve(resolveOptions)
// Resolve the import with the webpack provided context, this // Resolve the import with the webpack provided context, this
// ensures we're resolving the correct version when multiple // ensures we're resolving the correct version when multiple
...@@ -273,23 +257,6 @@ export function makeExternalHandler({ ...@@ -273,23 +257,6 @@ export function makeExternalHandler({
return resolveNextExternal(request) return resolveNextExternal(request)
} }
// Early return if the request needs to be bundled, such as in the client layer.
// Treat react packages and next internals as external for SSR layer,
// also map react to builtin ones with require-hook.
// Otherwise keep continue the process to resolve the externals.
if (layer === WEBPACK_LAYERS.serverSideRendering) {
const isRelative = request.startsWith('.')
const fullRequest = isRelative
? normalizePathSep(path.join(context, request))
: request
// Check if it's opt out bundling package first
if (containsImportInPackages(fullRequest, optOutBundlingPackages)) {
return fullRequest
}
return resolveNextExternal(fullRequest)
}
// TODO-APP: Let's avoid this resolve call as much as possible, and eventually get rid of it. // TODO-APP: Let's avoid this resolve call as much as possible, and eventually get rid of it.
const resolveResult = await resolveExternal( const resolveResult = await resolveExternal(
dir, dir,
...@@ -320,6 +287,13 @@ export function makeExternalHandler({ ...@@ -320,6 +287,13 @@ export function makeExternalHandler({
return return
} }
const isOptOutBundling = optOutBundlingPackageRegex.test(res)
// Apply bundling rules to all app layers.
// Since handleExternals only handle the server layers, we don't need to exclude client here
if (!isOptOutBundling && isAppLayer) {
return
}
// ESM externals can only be imported (and not required). // ESM externals can only be imported (and not required).
// Make an exception in loose mode. // Make an exception in loose mode.
if (!isEsmRequested && isEsm && !looseEsmExternals && !isLocal) { if (!isEsmRequested && isEsm && !looseEsmExternals && !isLocal) {
...@@ -370,13 +344,11 @@ export function makeExternalHandler({ ...@@ -370,13 +344,11 @@ export function makeExternalHandler({
const resolvedBundlingOptOutRes = resolveBundlingOptOutPackages({ const resolvedBundlingOptOutRes = resolveBundlingOptOutPackages({
resolvedRes: res, resolvedRes: res,
optOutBundlingPackageRegex,
config, config,
resolvedExternalPackageDirs, resolvedExternalPackageDirs,
isEsm,
isAppLayer, isAppLayer,
layer,
externalType, externalType,
isOptOutBundling,
request, request,
}) })
if (resolvedBundlingOptOutRes) { if (resolvedBundlingOptOutRes) {
...@@ -390,41 +362,32 @@ export function makeExternalHandler({ ...@@ -390,41 +362,32 @@ export function makeExternalHandler({
function resolveBundlingOptOutPackages({ function resolveBundlingOptOutPackages({
resolvedRes, resolvedRes,
optOutBundlingPackageRegex,
config, config,
resolvedExternalPackageDirs, resolvedExternalPackageDirs,
isEsm,
isAppLayer, isAppLayer,
layer,
externalType, externalType,
isOptOutBundling,
request, request,
}: { }: {
resolvedRes: string resolvedRes: string
optOutBundlingPackageRegex: RegExp
config: NextConfigComplete config: NextConfigComplete
resolvedExternalPackageDirs: Map<string, string> resolvedExternalPackageDirs: Map<string, string>
isEsm: boolean
isAppLayer: boolean isAppLayer: boolean
layer: WebpackLayerName | null
externalType: string externalType: string
isOptOutBundling: boolean
request: string request: string
}) { }) {
if (nodeModulesRegex.test(resolvedRes)) {
const shouldBundlePages =
!isAppLayer && config.bundlePagesRouterDependencies && !isOptOutBundling
const shouldBeBundled = const shouldBeBundled =
shouldBundlePages ||
isResourceInPackages( isResourceInPackages(
resolvedRes, resolvedRes,
config.transpilePackages, config.transpilePackages,
resolvedExternalPackageDirs resolvedExternalPackageDirs
) || )
(isEsm && isAppLayer) || if (!shouldBeBundled) {
(!isAppLayer && config.bundlePagesRouterDependencies)
if (nodeModulesRegex.test(resolvedRes)) {
const isOptOutBundling = optOutBundlingPackageRegex.test(resolvedRes)
if (isWebpackServerOnlyLayer(layer)) {
if (isOptOutBundling) {
return `${externalType} ${request}` // Externalize if opted out
}
} else if (!shouldBeBundled || isOptOutBundling) {
return `${externalType} ${request}` // Externalize if not bundled or opted out return `${externalType} ${request}` // Externalize if not bundled or opted out
} }
} }
......
...@@ -430,7 +430,29 @@ export class FlightClientEntryPlugin { ...@@ -430,7 +430,29 @@ export class FlightClientEntryPlugin {
) )
} }
compilation.hooks.finishModules.tapPromise(PLUGIN_NAME, async () => { // Invalidate in development to trigger recompilation
const invalidator = getInvalidator(compiler.outputPath)
// Check if any of the entry injections need an invalidation
if (
invalidator &&
addClientEntryAndSSRModulesList.some(
([shouldInvalidate]) => shouldInvalidate === true
)
) {
invalidator.invalidate([COMPILER_NAMES.client])
}
// Client compiler is invalidated before awaiting the compilation of the SSR client component entries
// so that the client compiler is running in parallel to the server compiler.
await Promise.all(
addClientEntryAndSSRModulesList.map(
(addClientEntryAndSSRModules) => addClientEntryAndSSRModules[1]
)
)
// Wait for action entries to be added.
await Promise.all(addActionEntryList)
const addedClientActionEntryList: Promise<any>[] = [] const addedClientActionEntryList: Promise<any>[] = []
const actionMapsPerClientEntry: Record<string, Map<string, string[]>> = {} const actionMapsPerClientEntry: Record<string, Map<string, string[]>> = {}
...@@ -496,31 +518,6 @@ export class FlightClientEntryPlugin { ...@@ -496,31 +518,6 @@ export class FlightClientEntryPlugin {
} }
await Promise.all(addedClientActionEntryList) await Promise.all(addedClientActionEntryList)
return
})
// Invalidate in development to trigger recompilation
const invalidator = getInvalidator(compiler.outputPath)
// Check if any of the entry injections need an invalidation
if (
invalidator &&
addClientEntryAndSSRModulesList.some(
([shouldInvalidate]) => shouldInvalidate === true
)
) {
invalidator.invalidate([COMPILER_NAMES.client])
}
// Client compiler is invalidated before awaiting the compilation of the SSR client component entries
// so that the client compiler is running in parallel to the server compiler.
await Promise.all(
addClientEntryAndSSRModulesList.map(
(addClientEntryAndSSRModules) => addClientEntryAndSSRModules[1]
)
)
// Wait for action entries to be added.
await Promise.all(addActionEntryList)
} }
collectClientActionsFromDependencies({ collectClientActionsFromDependencies({
......
...@@ -265,7 +265,7 @@ async function createComponentTreeInternal({ ...@@ -265,7 +265,7 @@ async function createComponentTreeInternal({
} }
const LayoutOrPage: React.ComponentType<any> | undefined = layoutOrPageMod const LayoutOrPage: React.ComponentType<any> | undefined = layoutOrPageMod
? interopDefault(layoutOrPageMod) ? await interopDefault(layoutOrPageMod)
: undefined : undefined
/** /**
......
...@@ -16,7 +16,7 @@ async function resolveStreamResponse(response: any, onData?: any) { ...@@ -16,7 +16,7 @@ async function resolveStreamResponse(response: any, onData?: any) {
} }
describe('app dir - external dependency', () => { describe('app dir - external dependency', () => {
const { next, skipped } = nextTestSetup({ const { next, skipped, isTurbopack } = nextTestSetup({
files: __dirname, files: __dirname,
dependencies: { dependencies: {
swr: 'latest', swr: 'latest',
...@@ -279,14 +279,17 @@ describe('app dir - external dependency', () => { ...@@ -279,14 +279,17 @@ describe('app dir - external dependency', () => {
}) })
describe('server actions', () => { describe('server actions', () => {
it('should not prefer to resolve esm over cjs for bundling optout packages', async () => { it('should prefer to resolve esm over cjs for bundling optout packages', async () => {
const browser = await next.browser('/optout/action') const browser = await next.browser('/optout/action')
expect(await browser.elementByCss('#dual-pkg-outout p').text()).toBe('') expect(await browser.elementByCss('#dual-pkg-outout p').text()).toBe('')
browser.elementByCss('#dual-pkg-outout button').click() browser.elementByCss('#dual-pkg-outout button').click()
await check(async () => { await check(async () => {
const text = await browser.elementByCss('#dual-pkg-outout p').text() const text = await browser.elementByCss('#dual-pkg-outout p').text()
expect(text).toBe('dual-pkg-optout:cjs') // TODO: enable esm externals for app router in turbopack for actions
expect(text).toBe(
isTurbopack ? 'dual-pkg-optout:cjs' : 'dual-pkg-optout:mjs'
)
return 'success' return 'success'
}, /success/) }, /success/)
}) })
......
...@@ -13,13 +13,15 @@ describe('esm-externals', () => { ...@@ -13,13 +13,15 @@ describe('esm-externals', () => {
const urls = ['/static', '/ssr', '/ssg'] const urls = ['/static', '/ssr', '/ssg']
for (const url of urls) { for (const url of urls) {
// TODO fix webpack behavior // For invalid esm packages that have "import" pointing to a non-esm-flagged module
// webpack is using the CJS version instead but Turbopack is opting out of
// externalizing and bundling the non-esm-flagged module.
const expectedHtml = isTurbopack const expectedHtml = isTurbopack
? /Hello World\+World\+World\+World\+World\+World/ ? /Hello World\+World\+World\+World\+World\+World/
: url === '/static' : url === '/static'
? /Hello World\+World\+Alternative\+World\+World\+World/ ? /Hello World\+World\+Alternative\+World\+World\+World/
: /Hello World\+World\+Alternative\+World\+World\+Alternative/ : /Hello World\+World\+Alternative\+World\+World\+Alternative/
// TODO fix webpack behavior // On client side, webpack always bundlings so it uses the non-esm-flagged module too.
const expectedText = const expectedText =
isTurbopack || url === '/static' isTurbopack || url === '/static'
? /Hello World\+World\+World\+World\+World\+World/ ? /Hello World\+World\+World\+World\+World\+World/
...@@ -40,18 +42,21 @@ describe('esm-externals', () => { ...@@ -40,18 +42,21 @@ describe('esm-externals', () => {
// App dir // App dir
{ {
// TODO App Dir doesn't use esmExternals: true correctly for webpack and Turbopack // TODO App Dir doesn't use esmExternals: true correctly for Turbopack
// so we only verify that the page doesn't crash, but ignore the actual content // so we only verify that the page doesn't crash, but ignore the actual content
// const expectedHtml = /Hello World\+World\+World/ // const expectedHtml = isTurbopack ? /Hello World\+World\+World/ : /Hello World\+World\+Alternative/
const expectedHtml = /Hello Wrong\+Wrong\+Alternative/ const expectedHtml = isTurbopack
// const expectedText = /Hello World\+World\+World/ ? /Hello Wrong\+Wrong\+Alternative/
: /Hello World\+World\+Alternative/
const urls = ['/server', '/client'] const urls = ['/server', '/client']
for (const url of urls) { for (const url of urls) {
const expectedText = const expectedText =
url === '/server' url !== '/server'
? /Hello World\+World\+World/
: isTurbopack
? /Hello Wrong\+Wrong\+Alternative/ ? /Hello Wrong\+Wrong\+Alternative/
: /Hello World\+World\+World/ : /Hello World\+World\+Alternative/
it(`should return the correct SSR HTML for ${url}`, async () => { it(`should return the correct SSR HTML for ${url}`, async () => {
const res = await next.fetch(url) const res = await next.fetch(url)
const html = await res.text() const html = await res.text()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment