Unverified Commit 8e1df952 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: add minServerChunkSize and maxMergeServerChunkSize options (#100712)

### What?

Adds two experimental options under `experimental.turbopackChunking`:

- `minServerChunkSize` — the minimum chunk size used by Turbopack's
production chunker for Node.js server-side JavaScript.
- `maxMergeServerChunkSize` — the size above which a Node.js server
chunk is never merged with other chunks.

Both apply only to the Node.js server chunking context (production
builds only). Edge chunking is unchanged and not configurable.

### Why?

`minChunkSize` / `maxMergeChunkSize` only affect client-side chunking.
The Node.js server context used hardcoded thresholds, so there was no
way to tune server chunking (e.g. to trade fewer, larger server chunks
against duplication).

Edge is intentionally left out: Edge functions have a much smaller size
limit, so its existing hardcoded thresholds (min `20000`, no max merge
limit) are kept as-is.

### How?

- The options follow the same path as the existing client thresholds:
config type + zod schema → `TurbopackChunkingConfig` /
`TurbopackChunking` in `next-core` → `Project` →
`ServerChunkingContextOptions`, used by both the regular and the
`_with_client_assets` Node.js variants.
- When unset, the previously hardcoded Node.js values are used (min
`4000`, max merge `100000`), so default output is unchanged.
- Only the ECMAScript chunk type is affected; client chunking, CSS
chunking and the edge chunking context are untouched.
- Docs: new "Server Chunks" section in the `turbopackChunking` reference
(noting that Edge isn't affected), and the client thresholds are now
explicitly described as client-side.
- Tests: the existing `test/production/app-dir/turbopack-chunking` suite
gains Node.js server cases that change each server option in isolation
(relative to a merge-everything baseline) and assert the server chunk
count changes.

### Verification

- `pnpm test-start-turbo
test/production/app-dir/turbopack-chunking/turbopack-chunking.test.ts`
(local native build), plain and with the Cache Components CI env
(`__NEXT_CACHE_COMPONENTS=true
__NEXT_EXPERIMENTAL_CACHED_NAVIGATIONS=true
__NEXT_EXPERIMENTAL_SERVER_COMPONENTS_HMR_CANCELLATION=true`)
- One-off check: an edge build emits exactly the same chunks with the
options unset, set to `0` / `1`, or set to very large values

<!-- NEXT_JS_LLM -->


<!-- fleet 504cbf6c-3b96-4c51-aced-526e81854446 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent b0e1bf79
...@@ -1804,6 +1804,7 @@ impl Project { ...@@ -1804,6 +1804,7 @@ impl Project {
supports_lazy_dynamic_imports: bool, supports_lazy_dynamic_imports: bool,
) -> Result<Vc<NodeJsChunkingContext>> { ) -> Result<Vc<NodeJsChunkingContext>> {
let css_url_suffix = self.next_config().asset_suffix_path(); let css_url_suffix = self.next_config().asset_suffix_path();
let turbopack_chunking = self.next_config().turbopack_chunking().await?;
let options = ServerChunkingContextOptions { let options = ServerChunkingContextOptions {
mode: self.next_mode(), mode: self.next_mode(),
root_path: self.project_root_path().owned().await?, root_path: self.project_root_path().owned().await?,
...@@ -1837,6 +1838,8 @@ impl Project { ...@@ -1837,6 +1838,8 @@ impl Project {
.next_config() .next_config()
.turbopack_lazy_dynamic_imports_ssr(*self.next_mode().await?) .turbopack_lazy_dynamic_imports_ssr(*self.next_mode().await?)
.await?, .await?,
chunking_min_chunk_size: turbopack_chunking.min_server_chunk_size,
chunking_max_merge_chunk_size: turbopack_chunking.max_merge_server_chunk_size,
}; };
Ok(if client_assets { Ok(if client_assets {
get_server_chunking_context_with_client_assets(options) get_server_chunking_context_with_client_assets(options)
......
...@@ -965,6 +965,12 @@ pub struct TurbopackChunkingConfig { ...@@ -965,6 +965,12 @@ pub struct TurbopackChunkingConfig {
/// Minimum size, in bytes (default `20000`), for a component chunk to be emitted on its /// Minimum size, in bytes (default `20000`), for a component chunk to be emitted on its
/// own when `generate_component_chunks` is enabled. /// own when `generate_component_chunks` is enabled.
min_component_chunk_size: Option<usize>, min_component_chunk_size: Option<usize>,
/// Avoid creating more than one Node.js server chunk smaller than this size, in bytes
/// (default `4000`). Doesn't apply to edge chunks.
min_server_chunk_size: Option<usize>,
/// Don't merge Node.js server chunks bigger than this size, in bytes, with other chunks
/// (default `100000`). Doesn't apply to edge chunks.
max_merge_server_chunk_size: Option<usize>,
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -988,6 +994,10 @@ pub struct TurbopackChunking { ...@@ -988,6 +994,10 @@ pub struct TurbopackChunking {
pub max_merge_chunk_size: Option<usize>, pub max_merge_chunk_size: Option<usize>,
/// Override for the client JS `min_component_chunk_size`, or `None` to use the default. /// Override for the client JS `min_component_chunk_size`, or `None` to use the default.
pub min_component_chunk_size: Option<usize>, pub min_component_chunk_size: Option<usize>,
/// Override for the Node.js server JS `min_chunk_size`, or `None` to use the default.
pub min_server_chunk_size: Option<usize>,
/// Override for the Node.js server JS `max_merge_chunk_size`, or `None` to use the default.
pub max_merge_server_chunk_size: Option<usize>,
/// Whether to emit component chunks alongside merged chunks (default `false`). /// Whether to emit component chunks alongside merged chunks (default `false`).
pub generate_component_chunks: bool, pub generate_component_chunks: bool,
} }
...@@ -1974,6 +1984,8 @@ impl NextConfig { ...@@ -1974,6 +1984,8 @@ impl NextConfig {
max_chunk_count_per_group: config.and_then(|c| c.max_chunk_count_per_group), max_chunk_count_per_group: config.and_then(|c| c.max_chunk_count_per_group),
max_merge_chunk_size: config.and_then(|c| c.max_merge_chunk_size), max_merge_chunk_size: config.and_then(|c| c.max_merge_chunk_size),
min_component_chunk_size: config.and_then(|c| c.min_component_chunk_size), min_component_chunk_size: config.and_then(|c| c.min_component_chunk_size),
min_server_chunk_size: config.and_then(|c| c.min_server_chunk_size),
max_merge_server_chunk_size: config.and_then(|c| c.max_merge_server_chunk_size),
generate_component_chunks: config generate_component_chunks: config
.and_then(|c| c.generate_component_chunks) .and_then(|c| c.generate_component_chunks)
.unwrap_or(false), .unwrap_or(false),
......
...@@ -1035,6 +1035,10 @@ pub struct ServerChunkingContextOptions { ...@@ -1035,6 +1035,10 @@ pub struct ServerChunkingContextOptions {
pub style_groups_algorithm: StyleGroupsAlgorithm, pub style_groups_algorithm: StyleGroupsAlgorithm,
pub per_page_module_graph: Vc<bool>, pub per_page_module_graph: Vc<bool>,
pub lazy_dynamic_imports: bool, pub lazy_dynamic_imports: bool,
/// Override for the production JS `min_chunk_size` (default `4_000`).
pub chunking_min_chunk_size: Option<usize>,
/// Override for the production JS `max_merge_chunk_size` (default `100_000`).
pub chunking_max_merge_chunk_size: Option<usize>,
} }
/// Like `get_server_chunking_context` but all assets are emitted as client assets (so `/_next`) /// Like `get_server_chunking_context` but all assets are emitted as client assets (so `/_next`)
...@@ -1065,6 +1069,8 @@ pub async fn get_server_chunking_context_with_client_assets( ...@@ -1065,6 +1069,8 @@ pub async fn get_server_chunking_context_with_client_assets(
style_groups_algorithm, style_groups_algorithm,
per_page_module_graph, per_page_module_graph,
lazy_dynamic_imports, lazy_dynamic_imports,
chunking_min_chunk_size,
chunking_max_merge_chunk_size,
} = options; } = options;
let css_url_suffix = css_url_suffix.to_resolved().await?; let css_url_suffix = css_url_suffix.to_resolved().await?;
...@@ -1128,9 +1134,9 @@ pub async fn get_server_chunking_context_with_client_assets( ...@@ -1128,9 +1134,9 @@ pub async fn get_server_chunking_context_with_client_assets(
Vc::<EcmascriptChunkType>::default().to_resolved().await?, Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig { ChunkingConfig {
// Server chunks are loaded with require(), not fetched over the network. // Server chunks are loaded with require(), not fetched over the network.
min_chunk_size: 4_000, min_chunk_size: chunking_min_chunk_size.unwrap_or(4_000),
max_chunk_count_per_group: 0, max_chunk_count_per_group: 0,
max_merge_chunk_size: 100_000, max_merge_chunk_size: chunking_max_merge_chunk_size.unwrap_or(100_000),
request_cost: Some(10_000), request_cost: Some(10_000),
..Default::default() ..Default::default()
}, },
...@@ -1177,6 +1183,8 @@ pub async fn get_server_chunking_context( ...@@ -1177,6 +1183,8 @@ pub async fn get_server_chunking_context(
style_groups_algorithm, style_groups_algorithm,
per_page_module_graph, per_page_module_graph,
lazy_dynamic_imports, lazy_dynamic_imports,
chunking_min_chunk_size,
chunking_max_merge_chunk_size,
} = options; } = options;
let css_url_suffix = css_url_suffix.to_resolved().await?; let css_url_suffix = css_url_suffix.to_resolved().await?;
let next_mode = mode.await?; let next_mode = mode.await?;
...@@ -1241,9 +1249,9 @@ pub async fn get_server_chunking_context( ...@@ -1241,9 +1249,9 @@ pub async fn get_server_chunking_context(
Vc::<EcmascriptChunkType>::default().to_resolved().await?, Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig { ChunkingConfig {
// Server chunks are loaded with require(), not fetched over the network. // Server chunks are loaded with require(), not fetched over the network.
min_chunk_size: 4_000, min_chunk_size: chunking_min_chunk_size.unwrap_or(4_000),
max_chunk_count_per_group: 0, max_chunk_count_per_group: 0,
max_merge_chunk_size: 100_000, max_merge_chunk_size: chunking_max_merge_chunk_size.unwrap_or(100_000),
request_cost: Some(10_000), request_cost: Some(10_000),
..Default::default() ..Default::default()
}, },
......
--- ---
title: turbopackChunking title: turbopackChunking
description: Configure how Turbopack splits your client-side JavaScript into chunks in production. description: Configure how Turbopack splits your client-side and server-side JavaScript into chunks in production.
version: experimental version: experimental
--- ---
...@@ -101,8 +101,8 @@ module.exports = nextConfig ...@@ -101,8 +101,8 @@ module.exports = nextConfig
## Size Thresholds ## Size Thresholds
The following options control how aggressively Turbopack merges chunks and how large a chunk is The following options control how aggressively Turbopack merges client-side chunks and how large
allowed to grow. Sizes are in bytes of uncompressed, unminified code (roughly 5x a chunk is allowed to grow. Sizes are in bytes of uncompressed, unminified code (roughly 5x
the size of the compressed, minified output). the size of the compressed, minified output).
- **`minChunkSize`** (default `50000`): Turbopack will avoid creating more - **`minChunkSize`** (default `50000`): Turbopack will avoid creating more
...@@ -122,6 +122,32 @@ When merging chunks, the trade-off being made is an improvement to performance o ...@@ -122,6 +122,32 @@ When merging chunks, the trade-off being made is an improvement to performance o
loads at the cost of navigation performance. This is because each additional network request loads at the cost of navigation performance. This is because each additional network request
being made is costly, however, smaller chunks are more likely to be re-usable across pages. being made is costly, however, smaller chunks are more likely to be re-usable across pages.
## Server Chunks
The size thresholds above only apply to client-side JavaScript. Server-side JavaScript for the
Node.js runtime uses separate thresholds:
- **`minServerChunkSize`** (default `4000`): Turbopack will avoid creating more than one server
chunk smaller than this size by merging small chunks into larger ones.
- **`maxMergeServerChunkSize`** (default `100000`): Turbopack never merges a server chunk larger
than this size with other chunks.
These options don't apply to the Edge runtime, whose chunking is not configurable.
```js filename="next.config.js"
/** @type {import('next').NextConfig} */
const nextConfig = {
experimental: {
turbopackChunking: {
minServerChunkSize: 10000,
maxMergeServerChunkSize: 200000,
},
},
}
module.exports = nextConfig
```
<AppOnly> <AppOnly>
## Component Chunks ## Component Chunks
......
...@@ -443,6 +443,8 @@ export const experimentalSchema = { ...@@ -443,6 +443,8 @@ export const experimentalSchema = {
maxMergeChunkSize: z.number().min(0).optional(), maxMergeChunkSize: z.number().min(0).optional(),
minComponentChunkSize: z.number().min(0).optional(), minComponentChunkSize: z.number().min(0).optional(),
generateComponentChunks: z.boolean().optional(), generateComponentChunks: z.boolean().optional(),
minServerChunkSize: z.number().min(0).optional(),
maxMergeServerChunkSize: z.number().min(0).optional(),
}) })
.optional(), .optional(),
turbopackWorkerAssetPrefix: z.string().optional(), turbopackWorkerAssetPrefix: z.string().optional(),
......
...@@ -937,7 +937,7 @@ export interface ExperimentalConfig { ...@@ -937,7 +937,7 @@ export interface ExperimentalConfig {
*/ */
requestCost?: number requestCost?: number
/** /**
* Avoid creating more than one chunk smaller than this size, in bytes. Smaller * Avoid creating more than one client-side chunk smaller than this size, in bytes. Smaller
* chunks are merged into bigger ones to avoid that. Defaults to `50000` (50 KB). * chunks are merged into bigger ones to avoid that. Defaults to `50000` (50 KB).
*/ */
minChunkSize?: number minChunkSize?: number
...@@ -947,8 +947,9 @@ export interface ExperimentalConfig { ...@@ -947,8 +947,9 @@ export interface ExperimentalConfig {
*/ */
maxChunkCountPerGroup?: number maxChunkCountPerGroup?: number
/** /**
* Never merge chunks bigger than this size, in bytes, with other chunks. This keeps code * Never merge client-side chunks bigger than this size, in bytes, with other chunks. This
* in big chunks from being duplicated across multiple chunks. Defaults to `200000` (200 KB). * keeps code in big chunks from being duplicated across multiple chunks. Defaults to `200000`
* (200 KB).
*/ */
maxMergeChunkSize?: number maxMergeChunkSize?: number
/** /**
...@@ -962,6 +963,18 @@ export interface ExperimentalConfig { ...@@ -962,6 +963,18 @@ export interface ExperimentalConfig {
* single remainder chunk. Defaults to `20000` (20 KB). * single remainder chunk. Defaults to `20000` (20 KB).
*/ */
minComponentChunkSize?: number minComponentChunkSize?: number
/**
* Avoid creating more than one Node.js server chunk smaller than this size, in bytes.
* Smaller chunks are merged into bigger ones to avoid that. Defaults to `4000` (4 KB). Doesn't
* apply to Edge runtime chunks.
*/
minServerChunkSize?: number
/**
* Never merge Node.js server chunks bigger than this size, in bytes, with other chunks. This
* keeps code in big chunks from being duplicated across multiple chunks. Defaults to `100000`
* (100 KB). Doesn't apply to Edge runtime chunks.
*/
maxMergeServerChunkSize?: number
} }
/** /**
......
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { listClientChunks } from 'next-test-utils' import { listClientChunks } from 'next-test-utils'
import { join } from 'path' import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
import { join, relative } from 'path'
// `experimental.turbopackChunking` only affects Turbopack production builds, so // `experimental.turbopackChunking` only affects Turbopack production builds, so
// this suite is Turbopack-only. Webpack uses a different chunker and ignores it. // this suite is Turbopack-only. Webpack uses a different chunker and ignores it.
...@@ -12,7 +13,7 @@ import { join } from 'path' ...@@ -12,7 +13,7 @@ import { join } from 'path'
skipStart: true, skipStart: true,
}) })
async function buildWith(turbopackChunking: object | undefined) { async function build(turbopackChunking: object | undefined) {
const experimental = turbopackChunking ? { turbopackChunking } : {} const experimental = turbopackChunking ? { turbopackChunking } : {}
await next.patchFile( await next.patchFile(
'next.config.js', 'next.config.js',
...@@ -20,12 +21,25 @@ import { join } from 'path' ...@@ -20,12 +21,25 @@ import { join } from 'path'
) )
const { exitCode } = await next.build() const { exitCode } = await next.build()
expect(exitCode).toBe(0) expect(exitCode).toBe(0)
}
async function buildWith(turbopackChunking: object | undefined) {
await build(turbopackChunking)
const chunks = await listClientChunks(join(next.testDir, '.next')) const chunks = await listClientChunks(join(next.testDir, '.next'))
return chunks.filter( return chunks.filter(
(name) => name.includes('chunks') && name.endsWith('.js') (name) => name.includes('chunks') && name.endsWith('.js')
) )
} }
// Lists the Node.js server JS chunks of the last build. The server size
// thresholds don't apply to edge chunks.
async function listServerChunks() {
const serverDir = join(next.testDir, '.next', 'server')
return (await recursiveReadDir(serverDir, { relativePathnames: false }))
.map((f) => relative(serverDir, f).replace(/\\/g, '/'))
.filter((name) => name.startsWith('chunks/') && name.endsWith('.js'))
}
// Force the chunker to merge as much as possible into as few chunks as // Force the chunker to merge as much as possible into as few chunks as
// possible. // possible.
const mergeAll = { const mergeAll = {
...@@ -41,6 +55,12 @@ import { join } from 'path' ...@@ -41,6 +55,12 @@ import { join } from 'path'
maxMergeChunkSize: 0, maxMergeChunkSize: 0,
} }
// Server-side equivalent of `mergeAll`.
const serverMergeAll = {
minServerChunkSize: 100_000_000,
maxMergeServerChunkSize: 100_000_000,
}
let mergeAllChunks: string[] let mergeAllChunks: string[]
let noMergeChunks: string[] let noMergeChunks: string[]
let mergeAllWithComponentChunks: string[] let mergeAllWithComponentChunks: string[]
...@@ -61,6 +81,46 @@ import { join } from 'path' ...@@ -61,6 +81,46 @@ import { join } from 'path'
expect(noMergeChunks.length).toBeGreaterThan(mergeAllChunks.length) expect(noMergeChunks.length).toBeGreaterThan(mergeAllChunks.length)
}) })
describe('server size thresholds', () => {
let serverMergeAllChunks: string[]
beforeAll(async () => {
await build(serverMergeAll)
serverMergeAllChunks = await listServerChunks()
})
// Each comparison changes a single option relative to `serverMergeAll`,
// so it proves that specific option reaches the Node.js server chunking
// context.
describe('minServerChunkSize', () => {
let chunks: string[]
beforeAll(async () => {
// A minimum of `0` disables merging entirely.
await build({ ...serverMergeAll, minServerChunkSize: 0 })
chunks = await listServerChunks()
})
it('changes the Node.js server chunk output', () => {
expect(chunks.length).toBeGreaterThan(serverMergeAllChunks.length)
})
})
describe('maxMergeServerChunkSize', () => {
let chunks: string[]
beforeAll(async () => {
// Every chunk is bigger than 1 byte, so none of them can be merged.
await build({ ...serverMergeAll, maxMergeServerChunkSize: 1 })
chunks = await listServerChunks()
})
it('changes the Node.js server chunk output', () => {
expect(chunks.length).toBeGreaterThan(serverMergeAllChunks.length)
})
})
})
it('emits additional component chunks when generateComponentChunks is enabled', () => { it('emits additional component chunks when generateComponentChunks is enabled', () => {
// With the same merge settings, enabling component chunks emits extra // With the same merge settings, enabling component chunks emits extra
// per-component chunks alongside the merged chunks. // per-component chunks alongside the merged chunks.
...@@ -70,15 +130,7 @@ import { join } from 'path' ...@@ -70,15 +130,7 @@ import { join } from 'path'
}) })
it('renders and navigates correctly with custom chunking', async () => { it('renders and navigates correctly with custom chunking', async () => {
await next.patchFile( await build({ ...mergeAll, ...serverMergeAll })
'next.config.js',
`module.exports = ${JSON.stringify(
{ experimental: { turbopackChunking: mergeAll } },
null,
2
)}`
)
await next.build()
await next.start() await next.start()
try { try {
const browser = await next.browser('/') const browser = await next.browser('/')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment