Unverified Commit aa8a243e authored by Cong-Cong Pan's avatar Cong-Cong Pan Committed by GitHub

feat: use Rspack persistent cache by default (#81399)

Rspack now enables persistent caching by default.

## Performance Comparison

### Pages Router

I benchmarked performance using this repo:
https://github.com/SyMind/chakra-ui-docs/tree/next-rspack to test the
Next.js pages router.

I tested the performance with the following steps:

1. Execute `pnpm run dev`
2. Wait for the server to be ready (indicated by the 'Ready' message)
3. Run curl on the root endpoint (/)

Each build was run 5 times, and the shortest time to reach "Compiled
successfully" was recorded.

Test environment: Apple M1 Pro CPU

| Tool | Build without cache | Build with cache | Dev without cache |
Dev with cache |

|-------------|---------------------|------------------|---------------------------------|----------------|
| Rspack | 3.8s | 2.6s | 1.7s | 3ms |
| Webpack | 14.0s | 4.0s | 7.8s | 3.2s |

### App Router

I benchmarked performance using this repo:
https://github.com/SyMind/shadcn-ui/tree/next-rspack to test the Next.js
app router.

I tested the performance with the following steps:

1. Execute `pnpm run dev` or `pnpm run build`
2. Wait for the server to be ready (indicated by the 'Ready' message)
3. Run curl on the root endpoint (/)

Each build was run 5 times, and the shortest time to reach "Compiled
successfully" was recorded.

Test environment: Apple M1 Pro CPU

| Bundler | Build (No Cache) | Build (Cache) | Dev (No Cache) | Dev
(Cache) |

|------------|----------------------|-------------------|--------------------|-----------------|
| Rspack | 12.3s | 5.9s | 7.1s | 1941ms |
| Webpack | 27.0s | 13.0s | 11s | 9.6s |

## About Rspack Persistent Cache Strategy

> packages/next/src/server/dev/hot-reloader-rspack.ts

Rspack's persistent caching differs from Webpack in how it manages
module graphs. While Webpack incrementally updates modules, Rspack
operates on complete module graph snapshots for cache restoration.

Problem:
- Next.js dev server starts with no page modules in the initial entry
points
- When Rspack restores from persistent cache, it finds no modules and
purges the entire module graph
- Later page requests find no cached module information, preventing
cache reuse

Solution:
- Track successfully built page entries after each compilation
- Restore these entries on dev server restart to maintain module graph
continuity
- This ensures previously compiled pages can leverage persistent cache
for faster builds

## Note

I have updated the test case configuration in
`test/integration/telemetry/next.config.use-cache` to disable persistent
cache.

This is because, whether using webpack or Rspack, when persistent
caching is enabled, modules are no longer recompiled by loaders, which
prevents the Telemetry plugin from collecting information.

Please note that this issue also exists with webpack. You can reproduce
it locally by running `pnpm run test
test/integration/telemetry/test/config.test.js` twice.
parent 754db28e
......@@ -2361,7 +2361,7 @@ export default async function getBaseWebpackConfig(
serverReferenceHashSalt: encryptionKey,
})
const cache: any = {
const cache: webpack.Configuration['cache'] = {
type: 'filesystem',
// Disable memory cache in development in favor of our own MemoryWithGcCachePlugin.
maxMemoryGenerations: dev ? 0 : Infinity, // Infinity is default value for production in webpack currently.
......@@ -2407,6 +2407,30 @@ export default async function getBaseWebpackConfig(
webpack5Config.cache = cache
if (isRspack) {
const buildDependencies: string[] = []
if (config.configFile) {
buildDependencies.push(config.configFile)
}
if (babelConfigFile) {
buildDependencies.push(babelConfigFile)
}
if (jsConfigPath) {
buildDependencies.push(jsConfigPath)
}
// @ts-ignore
webpack5Config.experiments.cache = {
type: 'persistent',
buildDependencies,
storage: {
type: 'filesystem',
directory: cache.cacheDirectory,
},
version: `${__dirname}|${process.env.__NEXT_VERSION}|${configVars}`,
}
}
if (process.env.NEXT_WEBPACK_LOGGING) {
const infra = process.env.NEXT_WEBPACK_LOGGING.includes('infrastructure')
const profileClient =
......
import path from 'path'
import fs from 'fs/promises'
import { createHash } from 'crypto'
import HotReloaderWebpack from './hot-reloader-webpack'
import { BUILT, EntryTypes, getEntries } from './on-demand-entry-handler'
import type { __ApiPreviewProps } from '../api-utils'
import type { RouteDefinition } from '../route-definitions/route-definition'
import type { MultiCompiler } from 'webpack'
import { COMPILER_NAMES } from '../../shared/lib/constants'
/**
* Rspack Persistent Cache Strategy for Next.js Development
*
* Rspack's persistent caching differs from Webpack in how it manages module graphs.
* While Webpack incrementally updates modules, Rspack operates on complete module
* graph snapshots for cache restoration.
*
* Problem:
* - Next.js dev server starts with no page modules in the initial entry points
* - When Rspack restores from persistent cache, it finds no modules and purges
* the entire module graph
* - Later page requests find no cached module information, preventing cache reuse
*
* Solution:
* - Track successfully built page entries after each compilation
* - Restore these entries on dev server restart to maintain module graph continuity
* - This ensures previously compiled pages can leverage persistent cache for faster builds
*/
export default class HotReloaderRspack extends HotReloaderWebpack {
private builtEntriesCachePath?: string
private isClientCacheEnabled = false
private isServerCacheEnabled = false
private isEdgeServerCacheEnabled = false
public async afterCompile(multiCompiler: MultiCompiler): Promise<void> {
// Always initialize the fallback error watcher for Rspack.
// Rspack may restore/retain the previous build's error state, so without this
// a page that previously failed to build might not be rebuilt on the next request.
await super.buildFallbackError()
const rspackStartSpan = this.hotReloaderSpan.traceChild(
'rspack-after-compile'
)
await rspackStartSpan.traceAsyncFn(async () => {
const hash = createHash('sha1')
multiCompiler.compilers.forEach((compiler) => {
const cache = compiler.options.cache
if (typeof cache === 'object' && 'version' in cache && cache.version) {
hash.update(cache.version)
if (compiler.name === COMPILER_NAMES.client) {
this.isClientCacheEnabled = true
} else if (compiler.name === COMPILER_NAMES.server) {
this.isServerCacheEnabled = true
} else if (compiler.name === COMPILER_NAMES.edgeServer) {
this.isEdgeServerCacheEnabled = true
}
} else {
hash.update('-')
}
return undefined
})
this.builtEntriesCachePath = path.join(
this.distDir,
'cache',
'rspack',
hash.digest('hex').substring(0, 16),
'built-entries.json'
)
const hasBuiltEntriesCache = await fs
.access(this.builtEntriesCachePath)
.then(
() => true,
() => false
)
if (hasBuiltEntriesCache) {
try {
const builtEntries: ReturnType<typeof getEntries> = JSON.parse(
(await fs.readFile(this.builtEntriesCachePath, 'utf-8')) || '{}'
)
await Promise.all(
Object.keys(builtEntries).map(async (entryKey) => {
const entryData = builtEntries[entryKey]
const isEntry = entryData.type === EntryTypes.ENTRY
const isChildEntry = entryData.type === EntryTypes.CHILD_ENTRY
// Check if the page was removed or disposed and remove it
if (isEntry) {
const pageExists =
!entryData.dispose &&
(await fs.access(entryData.absolutePagePath).then(
() => true,
() => false
))
if (!pageExists) {
delete builtEntries[entryKey]
return
} else if (
!('hash' in builtEntries[entryKey]) ||
builtEntries[entryKey].hash !==
(await calculateFileHash(entryData.absolutePagePath))
) {
delete builtEntries[entryKey]
return
}
}
// For child entries, if it has an entry file and it's gone, remove it
if (isChildEntry) {
if (entryData.absoluteEntryFilePath) {
const pageExists =
!entryData.dispose &&
(await fs.access(entryData.absoluteEntryFilePath).then(
() => true,
() => false
))
if (!pageExists) {
delete builtEntries[entryKey]
return
} else {
if (
!('hash' in builtEntries[entryKey]) ||
builtEntries[entryKey].hash !==
(await calculateFileHash(
entryData.absoluteEntryFilePath
))
) {
delete builtEntries[entryKey]
return
}
}
}
}
})
)
Object.assign(getEntries(multiCompiler.outputPath), builtEntries)
} catch (error) {
console.error('Rspack failed to read built entries cache: ', error)
}
}
})
}
public async ensurePage({
page,
clientOnly,
appPaths,
definition,
isApp,
url,
}: {
page: string
clientOnly: boolean
appPaths?: ReadonlyArray<string> | null
isApp?: boolean
definition?: RouteDefinition
url?: string
}): Promise<void> {
await super.ensurePage({
page,
clientOnly,
appPaths,
definition,
isApp,
url,
})
const entries = getEntries(this.multiCompiler!.outputPath)
const builtEntries: { [entryName: string]: any } = {}
await Promise.all(
Object.keys(entries).map(async (entryName) => {
const entry = entries[entryName]
if (entry.status !== BUILT) return
const result =
/^(client|server|edge-server)@(app|pages|root)@(.*)/g.exec(entryName)
const [, key /* pageType */, ,] = result! // this match should always happen
if (key === 'client' && !this.isClientCacheEnabled) return
if (key === 'server' && !this.isServerCacheEnabled) return
if (key === 'edge-server' && !this.isEdgeServerCacheEnabled) return
// TODO: Rspack does not store middleware entries in persistent cache, causing
// test/integration/middleware-src/test/index.test.ts to fail. This is a temporary
// workaround to skip middleware entry caching until Rspack properly supports it.
if (page === '/middleware') {
return
}
let hash: string | undefined
if (entry.type === EntryTypes.ENTRY) {
hash = await calculateFileHash(entry.absolutePagePath)
} else if (entry.absoluteEntryFilePath) {
hash = await calculateFileHash(entry.absoluteEntryFilePath)
}
if (!hash) {
return
}
builtEntries[entryName] = entry
builtEntries[entryName].hash = hash
})
)
const hasBuitEntriesCache = await fs
.access(this.builtEntriesCachePath!)
.then(
() => true,
() => false
)
try {
if (!hasBuitEntriesCache) {
await fs.mkdir(path.dirname(this.builtEntriesCachePath!), {
recursive: true,
})
}
await fs.writeFile(
this.builtEntriesCachePath!,
JSON.stringify(builtEntries, null, 2)
)
} catch (error) {
console.error('Rspack failed to write built entries cache: ', error)
}
}
}
async function calculateFileHash(
filePath: string,
algorithm: string = 'sha256'
): Promise<string | undefined> {
if (
!(await fs.access(filePath).then(
() => true,
() => false
))
) {
return
}
const fileBuffer = await fs.readFile(filePath)
const hash = createHash(algorithm)
hash.update(fileBuffer)
return hash.digest('hex')
}
......@@ -226,18 +226,18 @@ function erroredPages(compilation: webpack.Compilation) {
export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
private hasAppRouterEntrypoints: boolean
private hasPagesRouterEntrypoints: boolean
private dir: string
private buildId: string
protected dir: string
protected buildId: string
private encryptionKey: string
private middlewares: ((
protected middlewares: ((
req: IncomingMessage,
res: ServerResponse,
next: () => void
) => Promise<void>)[]
private pagesDir?: string
private distDir: string
protected pagesDir?: string
protected distDir: string
private webpackHotMiddleware?: WebpackHotMiddleware
private config: NextConfigComplete
protected config: NextConfigComplete
private clientStats: webpack.Stats | null
private clientError: Error | null = null
private serverError: Error | null = null
......@@ -246,13 +246,13 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
private serverChunkNames?: Set<string>
private prevChunkNames?: Set<any>
private onDemandEntries?: ReturnType<typeof onDemandEntryHandler>
private previewProps: __ApiPreviewProps
protected previewProps: __ApiPreviewProps
private watcher: any
private rewrites: CustomRoutes['rewrites']
private fallbackWatcher: any
private hotReloaderSpan: Span
protected hotReloaderSpan: Span
private pagesMapping: { [key: string]: string } = {}
private appDir?: string
protected appDir?: string
private telemetry: Telemetry
private resetFetch: () => void
private lockfile: Lockfile | undefined
......@@ -1243,6 +1243,8 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
this.activeWebpackConfigs
) as unknown as webpack.MultiCompiler
await this.afterCompile(this.multiCompiler)
// Copy over the filesystem so that it is shared between all compilers.
const inputFileSystem = this.multiCompiler.compilers[0].inputFileSystem
for (const compiler of this.multiCompiler.compilers) {
......@@ -1643,7 +1645,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
}),
})
this.middlewares = [
this.middlewares.push(
getOverlayMiddleware({
rootDirectory: this.dir,
isSrcDir: this.isSrcDir,
......@@ -1694,9 +1696,8 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
getDevServerUrl: () => process.env.__NEXT_PRIVATE_ORIGIN,
}),
]
: []),
]
: [])
)
setStackFrameResolver(async (request) => {
return getOriginalStackFrames({
isServer: request.isServer,
......@@ -1711,6 +1712,8 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
})
}
protected async afterCompile(_multiCompiler: webpack.MultiCompiler) {}
public invalidate(
{ reloadAfterInvalidation }: { reloadAfterInvalidation: boolean } = {
reloadAfterInvalidation: false,
......
......@@ -8,7 +8,6 @@ import type {
} from '../../shared/lib/app-router-types'
import type { CompilerNameValues } from '../../shared/lib/constants'
import type { RouteDefinition } from '../route-definitions/route-definition'
import type HotReloaderWebpack from './hot-reloader-webpack'
import createDebug from 'next/dist/compiled/debug'
import { EventEmitter } from 'events'
......@@ -39,6 +38,7 @@ import { PAGE_SEGMENT_KEY } from '../../shared/lib/segment'
import {
HMR_MESSAGE_SENT_TO_BROWSER,
HMR_MESSAGE_SENT_TO_SERVER,
type NextJsHotReloaderInterface,
} from './hot-reloader-types'
import { isAppPageRouteDefinition } from '../route-definitions/app-page-route-definition'
import { scheduleOnNextTick } from '../../lib/scheduler'
......@@ -548,7 +548,7 @@ export function onDemandEntryHandler({
rootDir,
appDir,
}: {
hotReloader: HotReloaderWebpack
hotReloader: NextJsHotReloaderInterface
maxInactiveAge: number
multiCompiler: webpack.MultiCompiler
nextConfig: NextConfigComplete
......
......@@ -215,10 +215,14 @@ async function startWatcher(
)
})()
: await (async () => {
const HotReloaderWebpack = (
require('../../dev/hot-reloader-webpack') as typeof import('../../dev/hot-reloader-webpack')
).default
return new HotReloaderWebpack(opts.dir, {
const HotReloader = process.env.NEXT_RSPACK
? (
require('../../dev/hot-reloader-rspack') as typeof import('../../dev/hot-reloader-rspack')
).default
: (
require('../../dev/hot-reloader-webpack') as typeof import('../../dev/hot-reloader-webpack')
).default
return new HotReloader(opts.dir, {
isSrcDir: opts.isSrcDir,
appDir,
pagesDir,
......
/**
* @type {import('next').NextConfig}
*/
const nextConfig = {}
const nextConfig = {
webpack(config) {
if (process.env.NEXT_RSPACK) {
// Disable persistent cache when using Rspack.
// Rspack may reuse previously compiled pages from its persistent cache.
// In development, webpack typically compiles only the page being requested.
// Keeping Rspack's persistent cache enabled can cause tests to surface errors
// from previously compiled pages, making it hard to assert errors for the
// currently requested page only.
config.cache = false
}
return config
},
}
module.exports = nextConfig
......@@ -163,16 +163,16 @@ export function runErrorRecoveryHmrTest(nextConfig: {
if (process.env.IS_TURBOPACK_TEST) {
expect(source).toMatchInlineSnapshot(`
"./pages/hmr/about2.js (7:1)
Parsing ecmascript source code failed
5 | div
6 | )
> 7 | }
| ^
8 |
Unexpected token. Did you mean \`{'}'}\` or \`&rbrace;\`?"
`)
"./pages/hmr/about2.js (7:1)
Parsing ecmascript source code failed
5 | div
6 | )
> 7 | }
| ^
8 |
Unexpected token. Did you mean \`{'}'}\` or \`&rbrace;\`?"
`)
} else if (process.env.NEXT_RSPACK) {
expect(trimEndMultiline(source)).toMatchInlineSnapshot(`
"./pages/hmr/about2.js
......@@ -200,28 +200,28 @@ export function runErrorRecoveryHmrTest(nextConfig: {
`)
} else {
expect(source).toMatchInlineSnapshot(`
"./pages/hmr/about2.js
Error: x Unexpected token. Did you mean \`{'}'}\` or \`&rbrace;\`?
,-[7:1]
4 | <p>This is the about page.</p>
5 | div
6 | )
7 | }
: ^
\`----
x Expected '</', got '<eof>'
,-[7:3]
5 | div
6 | )
7 | }
\`----
Caused by:
Syntax Error
Import trace for requested module:
./pages/hmr/about2.js"
`)
"./pages/hmr/about2.js
Error: x Unexpected token. Did you mean \`{'}'}\` or \`&rbrace;\`?
,-[7:1]
4 | <p>This is the about page.</p>
5 | div
6 | )
7 | }
: ^
\`----
x Expected '</', got '<eof>'
,-[7:3]
5 | div
6 | )
7 | }
\`----
Caused by:
Syntax Error
Import trace for requested module:
./pages/hmr/about2.js"
`)
}
}
)
......@@ -442,25 +442,25 @@ export function runErrorRecoveryHmrTest(nextConfig: {
} else if (process.env.NEXT_RSPACK) {
expect(trimEndMultiline(await getRedboxSource(browser)))
.toMatchInlineSnapshot(`
"./components/parse-error.xyz
× Module parse failed:
╰─▶ × JavaScript parse error: Expression expected
╭─[3:0]
1 │ This
2 │ is
3 │ }}}
· ─
4 │ invalid
5 │ js
╰────
help:
You may need an appropriate loader to handle this file type.
Import trace for requested module:
./components/parse-error.xyz
./pages/hmr/about8.js"
`)
"./components/parse-error.xyz
× Module parse failed:
╰─▶ × JavaScript parse error: Expression expected
╭─[3:0]
1 │ This
2 │ is
3 │ }}}
· ─
4 │ invalid
5 │ js
╰────
help:
You may need an appropriate loader to handle this file type.
Import trace for requested module:
./components/parse-error.xyz
./pages/hmr/about8.js"
`)
} else {
expect(await getRedboxSource(browser)).toMatchInlineSnapshot(`
"./components/parse-error.xyz
......@@ -513,26 +513,26 @@ export function runErrorRecoveryHmrTest(nextConfig: {
if (process.env.IS_TURBOPACK_TEST) {
expect(next.normalizeTestDirContent(redboxSource))
.toMatchInlineSnapshot(`
"./components/parse-error.js (3:1)
Parsing ecmascript source code failed
1 | This
2 | is
> 3 | }}}
| ^
4 | invalid
5 | js
Expression expected
Import traces:
Browser:
./components/parse-error.js
./pages/hmr/about9.js
SSR:
./components/parse-error.js
./pages/hmr/about9.js"
`)
"./components/parse-error.js (3:1)
Parsing ecmascript source code failed
1 | This
2 | is
> 3 | }}}
| ^
4 | invalid
5 | js
Expression expected
Import traces:
Browser:
./components/parse-error.js
./pages/hmr/about9.js
SSR:
./components/parse-error.js
./pages/hmr/about9.js"
`)
} else if (process.env.NEXT_RSPACK) {
expect(trimEndMultiline(next.normalizeTestDirContent(redboxSource)))
.toMatchInlineSnapshot(`
......
module.exports = {}
module.exports = {
webpack(config) {
if (process.env.NEXT_RSPACK) {
// Disable persistent cache when using Rspack.
// Rspack may reuse previously compiled pages from its persistent cache.
// In development, webpack typically compiles only the page being requested.
// Keeping Rspack's persistent cache enabled can cause tests to surface errors
// from previously compiled pages, making it hard to assert errors for the
// currently requested page only.
config.cache = false
}
return config
},
}
......@@ -404,6 +404,13 @@ module.exports = {
.blue-text{color:#00f}",
]
`)
} else if (process.env.NEXT_RSPACK && useLightningcss) {
expect(cssContent).toMatchInlineSnapshot(`
[
"/_next/static/css/HASH.css:
.red-text{color:red;font-weight:bolder}.blue-text{color:#00f;font-weight:bolder}",
]
`)
} else if (useLightningcss) {
expect(cssContent).toMatchInlineSnapshot(`
[
......@@ -571,6 +578,13 @@ module.exports = {
.blue-text{color:#00f}",
]
`)
} else if (process.env.NEXT_RSPACK && useLightningcss) {
expect(cssContent).toMatchInlineSnapshot(`
[
"/_next/static/css/HASH.css:
.red-text{color:red;background-image:url(/_next/static/media/dark.6b01655b.svg),url(/_next/static/media/dark2.6b01655b.svg)}.blue-text{color:#00f;background-image:url(/_next/static/media/light.2da1d3d6.svg);font-weight:bolder}",
]
`)
} else if (useLightningcss) {
expect(cssContent).toMatchInlineSnapshot(`
[
......
......@@ -91,6 +91,10 @@ module.exports = {
expect(cssContentWithoutSourceMap).toMatchInlineSnapshot(
`"@media (min-width:480px) and (not (min-width:768px)){::placeholder{color:green}}.flex-parsing{flex:0 0 calc(50% - var(--vertical-gutter))}.transform-parsing{transform:translate3d(0px,0px)}.css-grid-shorthand{grid-column:span 2}.g-docs-sidenav .filter::-webkit-input-placeholder{opacity:.8}"`
)
} else if (process.env.NEXT_RSPACK && useLightningcss) {
expect(cssContentWithoutSourceMap).toMatchInlineSnapshot(
`"@media (min-width:480px) and (not (min-width:768px)){::placeholder{color:green}}.flex-parsing{flex:0 0 calc(50% - var(--vertical-gutter))}.transform-parsing{transform:translate3d(0px,0px)}.css-grid-shorthand{grid-column:span 2}.g-docs-sidenav .filter::-webkit-input-placeholder{opacity:.8}"`
)
} else if (useLightningcss) {
expect(cssContentWithoutSourceMap).toMatchInlineSnapshot(
`"@media (min-width:480px) and (not (min-width:768px)){::placeholder{color:green}}.flex-parsing{flex:0 0 calc(50% - var(--vertical-gutter))}.transform-parsing{transform:translate3d(0,0)}.css-grid-shorthand{grid-column:span 2}.g-docs-sidenav .filter::-webkit-input-placeholder{opacity:.8}"`
......@@ -147,6 +151,38 @@ module.exports = {
grid-column: span 2;
}
.g-docs-sidenav .filter::-webkit-input-placeholder {
opacity: 80%;
}
",
],
"version": 3,
}
`)
} else if (process.env.NEXT_RSPACK && !useLightningcss) {
expect(sourceMapContentParsed).toMatchInlineSnapshot(`
{
"mappings": "AAAA,+CACE,cACE,WACF,CACF,CAEA,cACE,2CACF,CAEA,mBACE,0BACF,CAEA,oBACE,kBACF,CAEA,mDACE,WACF",
"names": [],
"sourcesContent": [
"@media (480px <= width < 768px) {
::placeholder {
color: green;
}
}
.flex-parsing {
flex: 0 0 calc(50% - var(--vertical-gutter));
}
.transform-parsing {
transform: translate3d(0px, 0px);
}
.css-grid-shorthand {
grid-column: span 2;
}
.g-docs-sidenav .filter::-webkit-input-placeholder {
opacity: 80%;
}
......
......@@ -3,4 +3,13 @@ module.exports = {
cacheHandlers: {
custom: require.resolve('next/dist/server/lib/cache-handlers/default.external'),
},
webpack(config) {
if (process.env.NEXT_RSPACK) {
// Disable persistent cache.
// If enabled, test case modules will not be recompiled by loaders,
// which prevents the Telemetry plugin from collecting information.
config.cache = false
}
return config
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment