Unverified Commit 9b671046 authored by Shu Ding's avatar Shu Ding Committed by GitHub

Flatten recursive wildcard exports in barrel optimization (#56489)

This PR flattens the recursive optimization logic of our barrel optimization loader. So now if there're any recursive `export * from ...`, they won't be created as multiple individual Webpack modules, but optimized in one module.

With this change, we are running SWC transform to get the export map directly inside the barrel loader (instead of a separate loader rule). And that map is recursively calculated and cached in memory.

I also published https://unpkg.com/browse/recursive-barrel@1.0.0/ to give this a test. It contains 4 levels of 10 `export *` expressions, which is 10,000 modules in total. Without this change, it takes ~30s to compile and with this change, it goes down to ~7s.
parent c6f5089e
...@@ -321,7 +321,6 @@ export function getLoaderSWCOptions({ ...@@ -321,7 +321,6 @@ export function getLoaderSWCOptions({
hasServerComponents, hasServerComponents,
isServerLayer, isServerLayer,
isServerActionsEnabled, isServerActionsEnabled,
optimizeBarrelExports,
bundleTarget, bundleTarget,
}: // This is not passed yet as "paths" resolving is handled by webpack currently. }: // This is not passed yet as "paths" resolving is handled by webpack currently.
// resolvedBaseUrl, // resolvedBaseUrl,
...@@ -348,9 +347,6 @@ export function getLoaderSWCOptions({ ...@@ -348,9 +347,6 @@ export function getLoaderSWCOptions({
hasServerComponents?: boolean hasServerComponents?: boolean
isServerLayer: boolean isServerLayer: boolean
isServerActionsEnabled?: boolean isServerActionsEnabled?: boolean
optimizeBarrelExports?: {
wildcard: boolean
}
}) { }) {
let baseOptions: any = getBaseSWCOptions({ let baseOptions: any = getBaseSWCOptions({
filename, filename,
...@@ -405,9 +401,6 @@ export function getLoaderSWCOptions({ ...@@ -405,9 +401,6 @@ export function getLoaderSWCOptions({
packages: optimizePackageImports, packages: optimizePackageImports,
} }
} }
if (optimizeBarrelExports) {
baseOptions.optimizeBarrelExports = optimizeBarrelExports
}
const isNextDist = nextDistPath.test(filename) const isNextDist = nextDistPath.test(filename)
......
...@@ -1412,27 +1412,6 @@ export default async function getBaseWebpackConfig( ...@@ -1412,27 +1412,6 @@ export default async function getBaseWebpackConfig(
}, },
module: { module: {
rules: [ rules: [
{
// This loader rule passes the resource to the SWC loader with
// `optimizeBarrelExports` enabled. This option makes the SWC to
// transform the original code to be a JSON of its export map, so
// the barrel loader can analyze it and only keep the needed ones.
test: /__barrel_transform__/,
use: ({ resourceQuery }: { resourceQuery: string }) => {
const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery)
return [
getSwcLoader({
isServerLayer: false,
bundleTarget: 'client',
hasServerComponents: false,
optimizeBarrelExports: {
wildcard: isFromWildcardExport,
},
}),
]
},
},
{ {
// This loader rule works like a bridge between user's import and // This loader rule works like a bridge between user's import and
// the target module behind a package's barrel file. It reads SWC's // the target module behind a package's barrel file. It reads SWC's
...@@ -1443,14 +1422,18 @@ export default async function getBaseWebpackConfig( ...@@ -1443,14 +1422,18 @@ export default async function getBaseWebpackConfig(
const names = ( const names = (
resourceQuery.match(/\?names=([^&]+)/)?.[1] || '' resourceQuery.match(/\?names=([^&]+)/)?.[1] || ''
).split(',') ).split(',')
const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery)
return [ return [
{ {
loader: 'next-barrel-loader', loader: 'next-barrel-loader',
options: { options: {
names, names,
wildcard: isFromWildcardExport, swcCacheDir: path.join(
dir,
config?.distDir ?? '.next',
'cache',
'swc'
),
}, },
// This is part of the request value to serve as the module key. // This is part of the request value to serve as the module key.
// The barrel loader are no-op re-exported modules keyed by // The barrel loader are no-op re-exported modules keyed by
......
...@@ -86,35 +86,177 @@ ...@@ -86,35 +86,177 @@
import type webpack from 'webpack' import type webpack from 'webpack'
import path from 'path'
import { transform } from '../../swc'
// This is a in-memory cache for the mapping of barrel exports. This only applies
// to the packages that we optimize. It will never change (e.g. upgrading packages)
// during the lifetime of the server so we can safely cache it.
// There is also no need to collect the cache for the same reason.
const barrelTransformMappingCache = new Map<
string,
{
prefix: string
exportList: [string, string, string][]
wildcardExports: string[]
} | null
>()
async function getBarrelMapping(
resourcePath: string,
swcCacheDir: string,
resolve: (context: string, request: string) => Promise<string>,
fs: {
readFile: (
path: string,
callback: (err: any, data: string | Buffer | undefined) => void
) => void
}
) {
if (barrelTransformMappingCache.has(resourcePath)) {
return barrelTransformMappingCache.get(resourcePath)!
}
// This is a SWC transform specifically for `optimizeBarrelExports`. We don't
// care about other things but the export map only.
async function transpileSource(
filename: string,
source: string,
isWildcard: boolean
) {
const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx')
return new Promise<string>((res) =>
transform(source, {
filename,
inputSourceMap: undefined,
sourceFileName: filename,
optimizeBarrelExports: {
wildcard: isWildcard,
},
jsc: {
parser: {
syntax: isTypeScript ? 'typescript' : 'ecmascript',
[isTypeScript ? 'tsx' : 'jsx']: true,
},
experimental: {
cacheRoot: swcCacheDir,
},
},
}).then((output) => {
res(output.code)
})
)
}
// Avoid circular `export *` dependencies
const visited = new Set<string>()
async function getMatches(file: string, isWildcard: boolean) {
if (visited.has(file)) {
return null
}
visited.add(file)
const source = await new Promise<string>((res, rej) => {
fs.readFile(file, (err, data) => {
if (err || data === undefined) {
rej(err)
} else {
res(data.toString())
}
})
})
const output = await transpileSource(file, source, isWildcard)
const matches = output.match(
/^([^]*)export (const|var) __next_private_export_map__ = ('[^']+'|"[^"]+")/
)
if (!matches) {
return null
}
const prefix = matches[1]
let exportList = JSON.parse(matches[3].slice(1, -1)) as [
string,
string,
string
][]
const wildcardExports = [
...output.matchAll(/export \* from "([^"]+)"/g),
].map((match) => match[1])
// In the wildcard case, if the value is exported from another file, we
// redirect to that file (decl[0]). Otherwise, export from the current
// file itself.
if (isWildcard) {
for (const decl of exportList) {
decl[1] = file
decl[2] = decl[0]
}
}
// This recursively handles the wildcard exports (e.g. `export * from './a'`)
if (wildcardExports.length) {
await Promise.all(
wildcardExports.map(async (req) => {
const targetPath = await resolve(
path.dirname(file),
req.replace('__barrel_optimize__?names=__PLACEHOLDER__!=!', '')
)
const targetMatches = await getMatches(targetPath, true)
if (targetMatches) {
// Merge the export list
exportList = exportList.concat(targetMatches.exportList)
}
})
)
}
return {
prefix,
exportList,
wildcardExports,
}
}
const res = await getMatches(resourcePath, false)
barrelTransformMappingCache.set(resourcePath, res)
return res
}
const NextBarrelLoader = async function ( const NextBarrelLoader = async function (
this: webpack.LoaderContext<{ this: webpack.LoaderContext<{
names: string[] names: string[]
wildcard: boolean swcCacheDir: string
}> }>
) { ) {
this.async() this.async()
const { names, wildcard } = this.getOptions() this.cacheable(true)
const source = await new Promise<string>((resolve, reject) => { const { names, swcCacheDir } = this.getOptions()
this.loadModule(
`__barrel_transform__${wildcard ? '?wildcard' : ''}!=!${ // For barrel optimizations, we always prefer the "module" field over the
this.resourcePath // "main" field because ESM handling is more robust with better tree-shaking.
}`, const resolve = this.getResolve({
(err, src) => { mainFields: ['module', 'main'],
if (err) {
reject(err)
} else {
resolve(src)
}
}
)
}) })
const matches = source.match( const mapping = await getBarrelMapping(
/^([^]*)export const __next_private_export_map__ = ('[^']+'|"[^"]+")/ this.resourcePath,
swcCacheDir,
resolve,
this.fs
) )
if (!matches) { // `resolve` adds all sub-paths to the dependency graph. However, we already
// cached the mapping and we assume them to not change. So, we can safely
// clear the dependencies here to avoid unnecessary watchers which turned out
// to be very expensive.
this.clearDependencies()
if (!mapping) {
// This file isn't a barrel and we can't apply any optimizations. Let's re-export everything. // This file isn't a barrel and we can't apply any optimizations. Let's re-export everything.
// Since this loader accepts `names` and the request is keyed with `names`, we can't simply // Since this loader accepts `names` and the request is keyed with `names`, we can't simply
// return the original source here. That will create these imports with different names as // return the original source here. That will create these imports with different names as
...@@ -123,19 +265,12 @@ const NextBarrelLoader = async function ( ...@@ -123,19 +265,12 @@ const NextBarrelLoader = async function (
return return
} }
const wildcardExports = [...source.matchAll(/export \* from "([^"]+)"/g)]
// It needs to keep the prefix for comments and directives like "use client". // It needs to keep the prefix for comments and directives like "use client".
const prefix = matches[1] const prefix = mapping.prefix
const exportList = mapping.exportList
const exportList = JSON.parse(matches[2].slice(1, -1)) as [
string,
string,
string
][]
const exportMap = new Map<string, [string, string]>() const exportMap = new Map<string, [string, string]>()
for (const [name, path, orig] of exportList) { for (const [name, filePath, orig] of exportList) {
exportMap.set(name, [path, orig]) exportMap.set(name, [filePath, orig])
} }
let output = prefix let output = prefix
...@@ -145,15 +280,6 @@ const NextBarrelLoader = async function ( ...@@ -145,15 +280,6 @@ const NextBarrelLoader = async function (
if (exportMap.has(name)) { if (exportMap.has(name)) {
const decl = exportMap.get(name)! const decl = exportMap.get(name)!
// In the wildcard case, if the value is exported from another file, we
// redirect to that file (decl[0]). Otherwise, export from the current
// file itself (this.resourcePath).
if (wildcard && !decl[0]) {
// E.g. the file contains `export const a = 1`
decl[0] = this.resourcePath
decl[1] = name
}
if (decl[1] === '*') { if (decl[1] === '*') {
output += `\nexport * as ${name} from ${JSON.stringify(decl[0])}` output += `\nexport * as ${name} from ${JSON.stringify(decl[0])}`
} else if (decl[1] === 'default') { } else if (decl[1] === 'default') {
...@@ -174,11 +300,9 @@ const NextBarrelLoader = async function ( ...@@ -174,11 +300,9 @@ const NextBarrelLoader = async function (
// These are from wildcard exports. // These are from wildcard exports.
if (missedNames.length > 0) { if (missedNames.length > 0) {
for (const match of wildcardExports) { for (const req of mapping.wildcardExports) {
const path = match[1]
output += `\nexport * from ${JSON.stringify( output += `\nexport * from ${JSON.stringify(
path.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard') req.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard')
)}` )}`
} }
} }
......
...@@ -45,9 +45,6 @@ export interface SWCLoaderOptions { ...@@ -45,9 +45,6 @@ export interface SWCLoaderOptions {
bundleTarget: BundleType bundleTarget: BundleType
hasServerComponents?: boolean hasServerComponents?: boolean
isServerLayer: boolean isServerLayer: boolean
optimizeBarrelExports?: {
wildcard: boolean
}
} }
async function loaderTransform( async function loaderTransform(
...@@ -73,19 +70,11 @@ async function loaderTransform( ...@@ -73,19 +70,11 @@ async function loaderTransform(
swcCacheDir, swcCacheDir,
hasServerComponents, hasServerComponents,
isServerLayer, isServerLayer,
optimizeBarrelExports,
bundleTarget, bundleTarget,
} = loaderOptions } = loaderOptions
const isPageFile = filename.startsWith(pagesDir) const isPageFile = filename.startsWith(pagesDir)
const relativeFilePathFromRoot = path.relative(rootDir, filename) const relativeFilePathFromRoot = path.relative(rootDir, filename)
// For testing purposes
if (process.env.NEXT_TEST_MODE) {
if (loaderOptions.optimizeBarrelExports) {
console.log('optimizeBarrelExports:', filename)
}
}
const swcOptions = getLoaderSWCOptions({ const swcOptions = getLoaderSWCOptions({
pagesDir, pagesDir,
appDir, appDir,
...@@ -106,7 +95,6 @@ async function loaderTransform( ...@@ -106,7 +95,6 @@ async function loaderTransform(
hasServerComponents, hasServerComponents,
isServerActionsEnabled: nextConfig?.experimental?.serverActions, isServerActionsEnabled: nextConfig?.experimental?.serverActions,
isServerLayer, isServerLayer,
optimizeBarrelExports,
bundleTarget, bundleTarget,
}) })
......
...@@ -42,6 +42,7 @@ describe('optimizePackageImports', () => { ...@@ -42,6 +42,7 @@ describe('optimizePackageImports', () => {
'@headlessui/react': '1.7.17', '@headlessui/react': '1.7.17',
'@heroicons/react': '2.0.18', '@heroicons/react': '2.0.18',
'@visx/visx': '3.3.0', '@visx/visx': '3.3.0',
'recursive-barrel': '1.0.0',
}, },
}) })
}) })
...@@ -97,24 +98,40 @@ describe('optimizePackageImports', () => { ...@@ -97,24 +98,40 @@ describe('optimizePackageImports', () => {
} }
}) })
it('should reuse the transformed barrel meta file from SWC', async () => { it('app - should optimize recursive wildcard export mapping', async () => {
let logs = '' let logs = ''
next.on('stdout', (log) => { next.on('stdout', (log) => {
logs += log logs += log
}) })
const html = await next.render('/dedupe') await next.render('/recursive-barrel-app')
// Ensure the icons are rendered const modules = [...logs.matchAll(/\((\d+) modules\)/g)]
expect(html).toContain('<svg xmlns="http://www.w3.org/2000/svg"')
const swcOptimizeBarrelExports = [ expect(modules.length).toBeGreaterThanOrEqual(1)
...logs.matchAll( for (const [, moduleCount] of modules) {
/optimizeBarrelExports: .+\/dist\/esm\/lucide-react\.js/g // Ensure that the number of modules is less than 1000 - otherwise we're
), // importing the entire library.
] expect(parseInt(moduleCount)).toBeLessThan(1000)
}
})
expect(swcOptimizeBarrelExports.length).toBe(1) it('pages - should optimize recursive wildcard export mapping', async () => {
let logs = ''
next.on('stdout', (log) => {
logs += log
})
await next.render('/recursive-barrel')
const modules = [...logs.matchAll(/\((\d+) modules\)/g)]
expect(modules.length).toBeGreaterThanOrEqual(1)
for (const [, moduleCount] of modules) {
// Ensure that the number of modules is less than 1000 - otherwise we're
// importing the entire library.
expect(parseInt(moduleCount)).toBeLessThan(1000)
}
}) })
it('should handle recursive wildcard exports', async () => { it('should handle recursive wildcard exports', async () => {
......
'use client'
import { b_8_7_6_4 } from 'recursive-barrel'
export default function Page() {
return <h1>{b_8_7_6_4}</h1>
}
module.exports = { module.exports = {
experimental: { experimental: {
optimizePackageImports: ['my-lib'], optimizePackageImports: ['my-lib', 'recursive-barrel'],
}, },
} }
import { b_8_3_1_1 as v } from 'recursive-barrel'
export default function Page() {
return <h1>{v}</h1>
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment