Unverified Commit 0b5f7d9b authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Update split chunk handling for edge/node (#62205)

While investigating OOMs noticed that our split chunks handling for edge
runtime isn't optimized properly causing a lot of duplicate
transpiling/minification which causes exponential memory/cache usage.

On a minimal app before this change the memory usage and cache size were
over `2GB`s after this change they are under `200MB`.

**Before**

![CleanShot 2024-02-17 at 16 08
46@2x](https://github.com/vercel/next.js/assets/22380829/5286e5fb-5e48-4296-a6be-d2ed1455eb95)


**After**

![CleanShot 2024-02-18 at 08 58
51@2x](https://github.com/vercel/next.js/assets/22380829/f813a185-51a7-40e0-b44b-e1ab95649194)



Closes NEXT-2521
parent 7bf82574
......@@ -715,11 +715,15 @@ export default async function getBaseWebpackConfig(
// Packages which will be split into the 'framework' chunk.
// Only top-level packages are included, e.g. nested copies like
// 'node_modules/meow/node_modules/object-assign' are not included.
const nextFrameworkPaths: string[] = []
const topLevelFrameworkPaths: string[] = []
const visitedFrameworkPackages = new Set<string>()
// Adds package-paths of dependencies recursively
const addPackagePath = (packageName: string, relativeToPath: string) => {
const addPackagePath = (
packageName: string,
relativeToPath: string,
paths: string[]
) => {
try {
if (visitedFrameworkPackages.has(packageName)) {
return
......@@ -738,11 +742,11 @@ export default async function getBaseWebpackConfig(
const directory = path.join(packageJsonPath, '../')
// Returning from the function in case the directory has already been added and traversed
if (topLevelFrameworkPaths.includes(directory)) return
topLevelFrameworkPaths.push(directory)
if (paths.includes(directory)) return
paths.push(directory)
const dependencies = require(packageJsonPath).dependencies || {}
for (const name of Object.keys(dependencies)) {
addPackagePath(name, directory)
addPackagePath(name, directory, paths)
}
} catch (_) {
// don't error on failing to resolve framework packages
......@@ -759,8 +763,9 @@ export default async function getBaseWebpackConfig(
]
: []),
]) {
addPackagePath(packageName, dir)
addPackagePath(packageName, dir, topLevelFrameworkPaths)
}
addPackagePath('next', dir, nextFrameworkPaths)
const crossOrigin = config.crossOrigin
......@@ -913,6 +918,7 @@ export default async function getBaseWebpackConfig(
splitChunks: (():
| Required<webpack.Configuration>['optimization']['splitChunks']
| false => {
// server chunking
if (dev) {
if (isNodeServer) {
/*
......@@ -963,21 +969,6 @@ export default async function getBaseWebpackConfig(
return false
}
if (isNodeServer) {
return {
filename: '[name].js',
chunks: 'all',
minChunks: 2,
}
}
if (isEdgeServer) {
return {
filename: 'edge-chunks/[name].js',
minChunks: 2,
}
}
const frameworkCacheGroup = {
chunks: 'all' as const,
name: 'framework',
......@@ -996,6 +987,22 @@ export default async function getBaseWebpackConfig(
// becoming a part of the commons chunk)
enforce: true,
}
const nextRuntimeCacheGroup = {
chunks: 'all' as const,
name: 'next-runtime',
test(module: any) {
const resource = module.nameForCondition?.()
return resource
? nextFrameworkPaths.some((pkgPath) =>
resource.startsWith(pkgPath)
)
: false
},
priority: 30,
enforce: true,
}
const libCacheGroup = {
test(module: {
size: Function
......@@ -1037,6 +1044,20 @@ export default async function getBaseWebpackConfig(
minChunks: 1,
reuseExistingChunk: true,
}
if (isNodeServer || isEdgeServer) {
return {
filename: `${isEdgeServer ? 'edge-chunks/' : ''}[name].js`,
cacheGroups: {
nextRuntime: nextRuntimeCacheGroup,
framework: frameworkCacheGroup,
lib: libCacheGroup,
},
minChunks: 2,
}
}
// client chunking
const cssCacheGroup = {
test: /\.(css|sass|scss)$/i,
chunks: 'all' as const,
......
......@@ -98,16 +98,6 @@ export class TerserPlugin {
return false
}
// don't minify _middleware as it can break in some cases
// and doesn't provide too much of a benefit as it's server-side
if (
name.match(
/(edge-runtime-webpack\.js|edge-chunks|middleware\.js$)/
)
) {
return false
}
const { info } = res
// Skip double minimize assets from child compilation
......
......@@ -111,11 +111,11 @@ describe('Middleware Runtime trailing slash', () => {
)
expect(manifest.middleware).toEqual({
'/': {
files: [
files: expect.arrayContaining([
'prerender-manifest.js',
'server/edge-runtime-webpack.js',
'server/middleware.js',
],
]),
name: 'middleware',
page: '/',
matchers: [{ regexp: '^/.*$', originalSource: '/:path*' }],
......
......@@ -187,10 +187,10 @@ describe('Switchable runtime', () => {
expect(manifest).toMatchObject({
functions: {
'/api/hello': {
files: [
files: expect.arrayContaining([
'server/edge-runtime-webpack.js',
'server/pages/api/hello.js',
],
]),
name: 'pages/api/hello',
page: '/api/hello',
matchers: [
......@@ -199,10 +199,10 @@ describe('Switchable runtime', () => {
wasm: [],
},
'/api/edge': {
files: [
files: expect.arrayContaining([
'server/edge-runtime-webpack.js',
'server/pages/api/edge.js',
],
]),
name: 'pages/api/edge',
page: '/api/edge',
matchers: [
......@@ -621,11 +621,11 @@ describe('Switchable runtime', () => {
expect(manifest).toMatchObject({
functions: {
'/api/hello': {
files: [
files: expect.arrayContaining([
'prerender-manifest.js',
'server/edge-runtime-webpack.js',
'server/pages/api/hello.js',
],
]),
name: 'pages/api/hello',
page: '/api/hello',
matchers: [
......@@ -634,11 +634,11 @@ describe('Switchable runtime', () => {
wasm: [],
},
'/api/edge': {
files: [
files: expect.arrayContaining([
'prerender-manifest.js',
'server/edge-runtime-webpack.js',
'server/pages/api/edge.js',
],
]),
name: 'pages/api/edge',
page: '/api/edge',
matchers: [
......
......@@ -50,11 +50,6 @@ describe('build trace with extra entries', () => {
expect(appTrace.files.some((file) => file.endsWith('hello.json'))).toBe(
true
)
expect(
appTrace.files.filter(
(file) => file.includes('chunks') && file.endsWith('.js')
).length
).toBe(0)
expect(
indexTrace.files.filter(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment