Unverified Commit 12bf495e authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Expand Turbopack dev cleanup (#97833)

## Summary

Builds on #97591. Webpack currently removes most of `.next` on devserver
startup. We can remove more as well.

This keeps the age-based removal introduced in #97591 and expands it to
`traces`, logs, etc. However, for any other directory that is not
`cache` or `lock`, completely remove it on startup. This includes types,
etc.

Apply age-based deletion to Turbopack-owned output that can be restored
from cache or recomputed, including route entries, chunks, assets,
service workers, middleware, and instrumentation. Completely remove
other dev output so logs, traces, generated types, diagnostics, and
future framework-owned output cannot be left as mixed generations.

Use exact-path exclusion sets in `recursiveDeleteSyncWithAsyncRetries`.
The complete pass retains `cache`, `lock`, and Turbopack-owned paths;
the age-based pass then sweeps those Turbopack paths. Both passes finish
before project creation.

## Verification

- `pnpm test-unit test/unit/recursive-delete.test.ts`
- `pnpm --filter=next types`

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent b882af08
...@@ -1282,7 +1282,7 @@ export default async function build( ...@@ -1282,7 +1282,7 @@ export default async function build(
.traceAsyncFn(() => .traceAsyncFn(() =>
recursiveDeleteSyncWithAsyncRetries( recursiveDeleteSyncWithAsyncRetries(
distDir, distDir,
/^(cache|dev|lock|trace)/ new Set(['cache', 'dev', 'lock', 'trace'])
) )
) )
} }
......
...@@ -94,7 +94,7 @@ export async function recursiveDeleteSyncWithAsyncRetries( ...@@ -94,7 +94,7 @@ export async function recursiveDeleteSyncWithAsyncRetries(
/** Directory to delete the contents of */ /** Directory to delete the contents of */
dir: string, dir: string,
/** Exclude based on relative file path */ /** Exclude based on relative file path */
exclude?: RegExp, exclude?: ReadonlySet<string>,
/** /**
* Only delete files whose mtime is at least this old. Directories are * Only delete files whose mtime is at least this old. Directories are
* removed once empty. * removed once empty.
...@@ -118,7 +118,7 @@ async function deleteContents( ...@@ -118,7 +118,7 @@ async function deleteContents(
futureMtimeThreshold, futureMtimeThreshold,
previousPath, previousPath,
}: { }: {
exclude: RegExp | undefined exclude: ReadonlySet<string> | undefined
staleBefore: number | undefined staleBefore: number | undefined
futureMtimeThreshold: number futureMtimeThreshold: number
/** Relative path to the directory being deleted, used for exclude */ /** Relative path to the directory being deleted, used for exclude */
...@@ -142,7 +142,7 @@ async function deleteContents( ...@@ -142,7 +142,7 @@ async function deleteContents(
const absolutePath = join(dir, part.name) const absolutePath = join(dir, part.name)
const pp = join(previousPath ?? '', part.name) const pp = join(previousPath ?? '', part.name)
if (exclude?.test(pp)) { if (exclude?.has(pp)) {
keptAnything = true keptAnything = true
return return
} }
......
...@@ -162,16 +162,26 @@ const sessionId = Math.floor(Number.MAX_SAFE_INTEGER * Math.random()) ...@@ -162,16 +162,26 @@ const sessionId = Math.floor(Number.MAX_SAFE_INTEGER * Math.random())
/** Output directory (relative to `distDir`) of server-HMR-managed chunks. */ /** Output directory (relative to `distDir`) of server-HMR-managed chunks. */
const SERVER_HMR_CHUNKS_DIR = join('server', 'chunks') const SERVER_HMR_CHUNKS_DIR = join('server', 'chunks')
const STALE_SWEPT_OUTPUT_DIRS = [ const TURBOPACK_OUTPUT_DIRS = [
join('static', 'chunks'), join('static', 'chunks'),
join('static', 'media'), join('static', 'media'),
join('static', 'service-worker'),
join('server', 'app'), join('server', 'app'),
join('server', 'pages'), join('server', 'pages'),
SERVER_HMR_CHUNKS_DIR, SERVER_HMR_CHUNKS_DIR,
join('server', 'assets'),
join('server', 'edge', 'chunks'), join('server', 'edge', 'chunks'),
join('server', 'edge', 'assets'), join('server', 'edge', 'assets'),
join('server', 'middleware'),
join('server', 'instrumentation'),
] ]
const RETAINED_OUTPUT_PATHS = new Set([
'cache',
'lock',
...TURBOPACK_OUTPUT_DIRS,
])
declare const __next__clear_chunk_cache__: (() => void) | null | undefined declare const __next__clear_chunk_cache__: (() => void) | null | undefined
declare const __turbopack_server_hmr_apply__: declare const __turbopack_server_hmr_apply__:
...@@ -401,6 +411,19 @@ export async function createHotReloaderTurbopack( ...@@ -401,6 +411,19 @@ export async function createHotReloaderTurbopack(
) )
} }
// This must finish before Turbopack records any writes. Once turbo-tasks has
// recorded a write effect, it dedups by hash without checking the file.
await recursiveDeleteSyncWithAsyncRetries(distDir, RETAINED_OUTPUT_PATHS)
await Promise.all(
TURBOPACK_OUTPUT_DIRS.map((subDir) =>
recursiveDeleteSyncWithAsyncRetries(
join(distDir, subDir),
undefined,
nextConfig.experimental.turbopackStaleOutputMaxAge
)
)
)
// For the debugging purpose, check if createNext or equivalent next instance setup in test cases // For the debugging purpose, check if createNext or equivalent next instance setup in test cases
// works correctly. Normally `run-test` hides output so only will be visible when `--debug` flag is used. // works correctly. Normally `run-test` hides output so only will be visible when `--debug` flag is used.
if (isTestMode) { if (isTestMode) {
...@@ -457,23 +480,6 @@ export async function createHotReloaderTurbopack( ...@@ -457,23 +480,6 @@ export async function createHotReloaderTurbopack(
}) })
} }
// Clean up any old output files from previous runs. This is safe as Turbopack
// will restore any missing chunks from persistent cache or recompute them.
//
// That only holds here, before the project exists: once turbo-tasks has
// recorded a write effect for a path, the write dedups on the recorded hash
// without checking the file, so a deleted output stays missing for the rest
// of the session.
await Promise.all(
STALE_SWEPT_OUTPUT_DIRS.map((subDir) =>
recursiveDeleteSyncWithAsyncRetries(
join(distDir, subDir),
undefined,
nextConfig.experimental.turbopackStaleOutputMaxAge!
)
)
)
const project = await bindings.turbo.createProject( const project = await bindings.turbo.createProject(
{ {
rootPath, rootPath,
......
...@@ -687,7 +687,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface { ...@@ -687,7 +687,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
.traceAsyncFn(() => .traceAsyncFn(() =>
recursiveDeleteSyncWithAsyncRetries( recursiveDeleteSyncWithAsyncRetries(
join(this.dir, this.config.distDir), join(this.dir, this.config.distDir),
/^(cache|lock)/ new Set(['cache', 'lock'])
) )
) )
} }
......
...@@ -37,11 +37,41 @@ describe('recursiveDeleteSyncWithAsyncRetries', () => { ...@@ -37,11 +37,41 @@ describe('recursiveDeleteSyncWithAsyncRetries', () => {
await recursiveCopy(resolveDataDir, testpreservefileDir, { await recursiveCopy(resolveDataDir, testpreservefileDir, {
overwrite: true, overwrite: true,
}) })
// preserve cache dir await recursiveDeleteSyncWithAsyncRetries(
await recursiveDeleteSyncWithAsyncRetries(testpreservefileDir, /^cache/) testpreservefileDir,
new Set(['cache'])
)
const result = await recursiveReadDir(testpreservefileDir) const result = await recursiveReadDir(testpreservefileDir)
expect(result.length).toBe(1) expect(result).toEqual(['/cache/test.txt'])
} finally {
// Ensure test cleanup
await recursiveDeleteSyncWithAsyncRetries(testpreservefileDir)
const cleanupResult = await recursiveReadDir(testpreservefileDir)
expect(cleanupResult.length).toBe(0)
}
})
it('should exclude a nested path', async () => {
expect.assertions(4)
try {
await recursiveCopy(resolveDataDir, testpreservefileDir, {
overwrite: true,
})
await recursiveDeleteSyncWithAsyncRetries(
testpreservefileDir,
new Set([join('aa', 'cache.js')])
)
const result = await recursiveReadDir(testpreservefileDir)
expect(result).toEqual(['/aa/cache.js'])
expect(
await fs.pathExists(join(testpreservefileDir, 'aa', 'cache.js'))
).toBe(true)
expect(
await fs.pathExists(join(testpreservefileDir, 'aa', 'index.js'))
).toBe(false)
} finally { } finally {
// Ensure test cleanup // Ensure test cleanup
await recursiveDeleteSyncWithAsyncRetries(testpreservefileDir) await recursiveDeleteSyncWithAsyncRetries(testpreservefileDir)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment