Unverified Commit 9b671046 authored by Shu Ding's avatar Shu Ding Committed by GitHub

Flatten recursive wildcard exports in barrel optimization (#56489)

This PR flattens the recursive optimization logic of our barrel optimization loader. So now if there're any recursive `export * from ...`, they won't be created as multiple individual Webpack modules, but optimized in one module.

With this change, we are running SWC transform to get the export map directly inside the barrel loader (instead of a separate loader rule). And that map is recursively calculated and cached in memory.

I also published https://unpkg.com/browse/recursive-barrel@1.0.0/ to give this a test. It contains 4 levels of 10 `export *` expressions, which is 10,000 modules in total. Without this change, it takes ~30s to compile and with this change, it goes down to ~7s.
parent c6f5089e
......@@ -321,7 +321,6 @@ export function getLoaderSWCOptions({
hasServerComponents,
isServerLayer,
isServerActionsEnabled,
optimizeBarrelExports,
bundleTarget,
}: // This is not passed yet as "paths" resolving is handled by webpack currently.
// resolvedBaseUrl,
......@@ -348,9 +347,6 @@ export function getLoaderSWCOptions({
hasServerComponents?: boolean
isServerLayer: boolean
isServerActionsEnabled?: boolean
optimizeBarrelExports?: {
wildcard: boolean
}
}) {
let baseOptions: any = getBaseSWCOptions({
filename,
......@@ -405,9 +401,6 @@ export function getLoaderSWCOptions({
packages: optimizePackageImports,
}
}
if (optimizeBarrelExports) {
baseOptions.optimizeBarrelExports = optimizeBarrelExports
}
const isNextDist = nextDistPath.test(filename)
......
......@@ -1412,27 +1412,6 @@ export default async function getBaseWebpackConfig(
},
module: {
rules: [
{
// This loader rule passes the resource to the SWC loader with
// `optimizeBarrelExports` enabled. This option makes the SWC to
// transform the original code to be a JSON of its export map, so
// the barrel loader can analyze it and only keep the needed ones.
test: /__barrel_transform__/,
use: ({ resourceQuery }: { resourceQuery: string }) => {
const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery)
return [
getSwcLoader({
isServerLayer: false,
bundleTarget: 'client',
hasServerComponents: false,
optimizeBarrelExports: {
wildcard: isFromWildcardExport,
},
}),
]
},
},
{
// This loader rule works like a bridge between user's import and
// the target module behind a package's barrel file. It reads SWC's
......@@ -1443,14 +1422,18 @@ export default async function getBaseWebpackConfig(
const names = (
resourceQuery.match(/\?names=([^&]+)/)?.[1] || ''
).split(',')
const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery)
return [
{
loader: 'next-barrel-loader',
options: {
names,
wildcard: isFromWildcardExport,
swcCacheDir: path.join(
dir,
config?.distDir ?? '.next',
'cache',
'swc'
),
},
// This is part of the request value to serve as the module key.
// The barrel loader are no-op re-exported modules keyed by
......
......@@ -86,35 +86,177 @@
import type webpack from 'webpack'
import path from 'path'
import { transform } from '../../swc'
// This is a in-memory cache for the mapping of barrel exports. This only applies
// to the packages that we optimize. It will never change (e.g. upgrading packages)
// during the lifetime of the server so we can safely cache it.
// There is also no need to collect the cache for the same reason.
const barrelTransformMappingCache = new Map<
string,
{
prefix: string
exportList: [string, string, string][]
wildcardExports: string[]
} | null
>()
async function getBarrelMapping(
resourcePath: string,
swcCacheDir: string,
resolve: (context: string, request: string) => Promise<string>,
fs: {
readFile: (
path: string,
callback: (err: any, data: string | Buffer | undefined) => void
) => void
}
) {
if (barrelTransformMappingCache.has(resourcePath)) {
return barrelTransformMappingCache.get(resourcePath)!
}
// This is a SWC transform specifically for `optimizeBarrelExports`. We don't
// care about other things but the export map only.
async function transpileSource(
filename: string,
source: string,
isWildcard: boolean
) {
const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx')
return new Promise<string>((res) =>
transform(source, {
filename,
inputSourceMap: undefined,
sourceFileName: filename,
optimizeBarrelExports: {
wildcard: isWildcard,
},
jsc: {
parser: {
syntax: isTypeScript ? 'typescript' : 'ecmascript',
[isTypeScript ? 'tsx' : 'jsx']: true,
},
experimental: {
cacheRoot: swcCacheDir,
},
},
}).then((output) => {
res(output.code)
})
)
}
// Avoid circular `export *` dependencies
const visited = new Set<string>()
async function getMatches(file: string, isWildcard: boolean) {
if (visited.has(file)) {
return null
}
visited.add(file)
const source = await new Promise<string>((res, rej) => {
fs.readFile(file, (err, data) => {
if (err || data === undefined) {
rej(err)
} else {
res(data.toString())
}
})
})
const output = await transpileSource(file, source, isWildcard)
const matches = output.match(
/^([^]*)export (const|var) __next_private_export_map__ = ('[^']+'|"[^"]+")/
)
if (!matches) {
return null
}
const prefix = matches[1]
let exportList = JSON.parse(matches[3].slice(1, -1)) as [
string,
string,
string
][]
const wildcardExports = [
...output.matchAll(/export \* from "([^"]+)"/g),
].map((match) => match[1])
// In the wildcard case, if the value is exported from another file, we
// redirect to that file (decl[0]). Otherwise, export from the current
// file itself.
if (isWildcard) {
for (const decl of exportList) {
decl[1] = file
decl[2] = decl[0]
}
}
// This recursively handles the wildcard exports (e.g. `export * from './a'`)
if (wildcardExports.length) {
await Promise.all(
wildcardExports.map(async (req) => {
const targetPath = await resolve(
path.dirname(file),
req.replace('__barrel_optimize__?names=__PLACEHOLDER__!=!', '')
)
const targetMatches = await getMatches(targetPath, true)
if (targetMatches) {
// Merge the export list
exportList = exportList.concat(targetMatches.exportList)
}
})
)
}
return {
prefix,
exportList,
wildcardExports,
}
}
const res = await getMatches(resourcePath, false)
barrelTransformMappingCache.set(resourcePath, res)
return res
}
const NextBarrelLoader = async function (
this: webpack.LoaderContext<{
names: string[]
wildcard: boolean
swcCacheDir: string
}>
) {
this.async()
const { names, wildcard } = this.getOptions()
const source = await new Promise<string>((resolve, reject) => {
this.loadModule(
`__barrel_transform__${wildcard ? '?wildcard' : ''}!=!${
this.resourcePath
}`,
(err, src) => {
if (err) {
reject(err)
} else {
resolve(src)
}
}
)
this.cacheable(true)
const { names, swcCacheDir } = this.getOptions()
// For barrel optimizations, we always prefer the "module" field over the
// "main" field because ESM handling is more robust with better tree-shaking.
const resolve = this.getResolve({
mainFields: ['module', 'main'],
})
const matches = source.match(
/^([^]*)export const __next_private_export_map__ = ('[^']+'|"[^"]+")/
const mapping = await getBarrelMapping(
this.resourcePath,
swcCacheDir,
resolve,
this.fs
)
if (!matches) {
// `resolve` adds all sub-paths to the dependency graph. However, we already
// cached the mapping and we assume them to not change. So, we can safely
// clear the dependencies here to avoid unnecessary watchers which turned out
// to be very expensive.
this.clearDependencies()
if (!mapping) {
// This file isn't a barrel and we can't apply any optimizations. Let's re-export everything.
// Since this loader accepts `names` and the request is keyed with `names`, we can't simply
// return the original source here. That will create these imports with different names as
......@@ -123,19 +265,12 @@ const NextBarrelLoader = async function (
return
}
const wildcardExports = [...source.matchAll(/export \* from "([^"]+)"/g)]
// It needs to keep the prefix for comments and directives like "use client".
const prefix = matches[1]
const exportList = JSON.parse(matches[2].slice(1, -1)) as [
string,
string,
string
][]
const prefix = mapping.prefix
const exportList = mapping.exportList
const exportMap = new Map<string, [string, string]>()
for (const [name, path, orig] of exportList) {
exportMap.set(name, [path, orig])
for (const [name, filePath, orig] of exportList) {
exportMap.set(name, [filePath, orig])
}
let output = prefix
......@@ -145,15 +280,6 @@ const NextBarrelLoader = async function (
if (exportMap.has(name)) {
const decl = exportMap.get(name)!
// In the wildcard case, if the value is exported from another file, we
// redirect to that file (decl[0]). Otherwise, export from the current
// file itself (this.resourcePath).
if (wildcard && !decl[0]) {
// E.g. the file contains `export const a = 1`
decl[0] = this.resourcePath
decl[1] = name
}
if (decl[1] === '*') {
output += `\nexport * as ${name} from ${JSON.stringify(decl[0])}`
} else if (decl[1] === 'default') {
......@@ -174,11 +300,9 @@ const NextBarrelLoader = async function (
// These are from wildcard exports.
if (missedNames.length > 0) {
for (const match of wildcardExports) {
const path = match[1]
for (const req of mapping.wildcardExports) {
output += `\nexport * from ${JSON.stringify(
path.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard')
req.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard')
)}`
}
}
......
......@@ -45,9 +45,6 @@ export interface SWCLoaderOptions {
bundleTarget: BundleType
hasServerComponents?: boolean
isServerLayer: boolean
optimizeBarrelExports?: {
wildcard: boolean
}
}
async function loaderTransform(
......@@ -73,19 +70,11 @@ async function loaderTransform(
swcCacheDir,
hasServerComponents,
isServerLayer,
optimizeBarrelExports,
bundleTarget,
} = loaderOptions
const isPageFile = filename.startsWith(pagesDir)
const relativeFilePathFromRoot = path.relative(rootDir, filename)
// For testing purposes
if (process.env.NEXT_TEST_MODE) {
if (loaderOptions.optimizeBarrelExports) {
console.log('optimizeBarrelExports:', filename)
}
}
const swcOptions = getLoaderSWCOptions({
pagesDir,
appDir,
......@@ -106,7 +95,6 @@ async function loaderTransform(
hasServerComponents,
isServerActionsEnabled: nextConfig?.experimental?.serverActions,
isServerLayer,
optimizeBarrelExports,
bundleTarget,
})
......
......@@ -42,6 +42,7 @@ describe('optimizePackageImports', () => {
'@headlessui/react': '1.7.17',
'@heroicons/react': '2.0.18',
'@visx/visx': '3.3.0',
'recursive-barrel': '1.0.0',
},
})
})
......@@ -97,24 +98,40 @@ describe('optimizePackageImports', () => {
}
})
it('should reuse the transformed barrel meta file from SWC', async () => {
it('app - should optimize recursive wildcard export mapping', async () => {
let logs = ''
next.on('stdout', (log) => {
logs += log
})
const html = await next.render('/dedupe')
await next.render('/recursive-barrel-app')
// Ensure the icons are rendered
expect(html).toContain('<svg xmlns="http://www.w3.org/2000/svg"')
const modules = [...logs.matchAll(/\((\d+) modules\)/g)]
const swcOptimizeBarrelExports = [
...logs.matchAll(
/optimizeBarrelExports: .+\/dist\/esm\/lucide-react\.js/g
),
]
expect(modules.length).toBeGreaterThanOrEqual(1)
for (const [, moduleCount] of modules) {
// Ensure that the number of modules is less than 1000 - otherwise we're
// importing the entire library.
expect(parseInt(moduleCount)).toBeLessThan(1000)
}
})
expect(swcOptimizeBarrelExports.length).toBe(1)
it('pages - should optimize recursive wildcard export mapping', async () => {
let logs = ''
next.on('stdout', (log) => {
logs += log
})
await next.render('/recursive-barrel')
const modules = [...logs.matchAll(/\((\d+) modules\)/g)]
expect(modules.length).toBeGreaterThanOrEqual(1)
for (const [, moduleCount] of modules) {
// Ensure that the number of modules is less than 1000 - otherwise we're
// importing the entire library.
expect(parseInt(moduleCount)).toBeLessThan(1000)
}
})
it('should handle recursive wildcard exports', async () => {
......
'use client'
import { b_8_7_6_4 } from 'recursive-barrel'
export default function Page() {
return <h1>{b_8_7_6_4}</h1>
}
module.exports = {
experimental: {
optimizePackageImports: ['my-lib'],
optimizePackageImports: ['my-lib', 'recursive-barrel'],
},
}
import { b_8_3_1_1 as v } from 'recursive-barrel'
export default function Page() {
return <h1>{v}</h1>
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment