Unverified Commit 2cee98ab authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Sweep stale Turbopack output from distDir on dev startup (#97591)

## Summary

Turbopack dev never removes the content-hashed chunks it supersedes, so
`.next/dev` grows without bound. Webpack dev avoids this by wiping
`distDir` on every start.

Adds a `maxAgeMs` argument to `recursiveDeleteSyncWithAsyncRetries`:
delete files whose mtime is at least that old, remove directories once
empty. Turbopack dev passes one week.

## Verification

- `pnpm test-unit test/unit/recursive-delete.test.ts`
- ~Not run:~ dev-server behavior against a real `.next/dev`

<!-- NEXT_JS_LLM -->
parent 95b77e82
...@@ -59,6 +59,27 @@ function unlinkPath( ...@@ -59,6 +59,27 @@ function unlinkPath(
} }
} }
const FUTURE_MTIME_TOLERANCE_MS = 60 * 60 * 1000
/** Returns whether `p`'s own mtime falls outside the retained time range. */
function isStale(
p: string,
staleBefore: number,
futureMtimeThreshold: number
): boolean {
try {
// lstat, not stat: a symlink is deleted based on its own age, not its
// target's.
const mtimeMs = fs.lstatSync(p).mtimeMs
return mtimeMs <= staleBefore || mtimeMs > futureMtimeThreshold
} catch (e) {
if (isError(e) && e.code === 'ENOENT') {
return false
}
throw e
}
}
/** /**
* Recursively delete directory contents. * Recursively delete directory contents.
* *
...@@ -74,34 +95,83 @@ export async function recursiveDeleteSyncWithAsyncRetries( ...@@ -74,34 +95,83 @@ export async function recursiveDeleteSyncWithAsyncRetries(
dir: string, dir: string,
/** Exclude based on relative file path */ /** Exclude based on relative file path */
exclude?: RegExp, exclude?: RegExp,
/** Relative path to the directory being deleted, used for exclude */ /**
previousPath: string = '' * Only delete files whose mtime is at least this old. Directories are
* removed once empty.
*/
maxAgeMs?: number
): Promise<void> { ): Promise<void> {
const now = Date.now()
await deleteContents(dir, {
exclude,
staleBefore: maxAgeMs === undefined ? undefined : now - maxAgeMs,
futureMtimeThreshold: now + FUTURE_MTIME_TOLERANCE_MS,
})
}
/** @returns whether anything was kept. */
async function deleteContents(
dir: string,
{
exclude,
staleBefore,
futureMtimeThreshold,
previousPath,
}: {
exclude: RegExp | undefined
staleBefore: number | undefined
futureMtimeThreshold: number
/** Relative path to the directory being deleted, used for exclude */
previousPath?: string
}
): Promise<boolean> {
let result let result
try { try {
result = fs.readdirSync(dir, { withFileTypes: true }) result = fs.readdirSync(dir, { withFileTypes: true })
} catch (e) { } catch (e) {
if (isError(e) && e.code === 'ENOENT') { if (isError(e) && e.code === 'ENOENT') {
return return false
} }
throw e throw e
} }
let keptAnything = false
await Promise.all( await Promise.all(
result.map(async (part: Dirent) => { result.map(async (part: Dirent) => {
const absolutePath = join(dir, part.name) const absolutePath = join(dir, part.name)
const pp = join(previousPath, part.name) const pp = join(previousPath ?? '', part.name)
const isNotExcluded = !exclude || !exclude.test(pp)
if (exclude?.test(pp)) {
keptAnything = true
return
}
if (isNotExcluded) { // Note: readdir does not follow symbolic links, that's good: we want to
// Note: readdir does not follow symbolic links, that's good: we want to // delete the links and not the destination.
// delete the links and not the destination. const isDirectory = part.isDirectory()
let isDirectory = part.isDirectory() if (isDirectory) {
if (isDirectory) { if (
await recursiveDeleteSyncWithAsyncRetries(absolutePath, exclude, pp) await deleteContents(absolutePath, {
exclude,
staleBefore,
futureMtimeThreshold,
previousPath: pp,
})
) {
keptAnything = true
return
} }
return unlinkPath(absolutePath, isDirectory) } else if (
staleBefore !== undefined &&
!isStale(absolutePath, staleBefore, futureMtimeThreshold)
) {
keptAnything = true
return
} }
return unlinkPath(absolutePath, isDirectory)
}) })
) )
return keptAnything
} }
...@@ -390,6 +390,7 @@ export const experimentalSchema = { ...@@ -390,6 +390,7 @@ export const experimentalSchema = {
turbopackFileSystemCacheForDev: z.boolean().optional(), turbopackFileSystemCacheForDev: z.boolean().optional(),
turbopackFileSystemCacheForBuild: z.boolean().optional(), turbopackFileSystemCacheForBuild: z.boolean().optional(),
turbopackSeedCacheFromWorktree: z.boolean().optional(), turbopackSeedCacheFromWorktree: z.boolean().optional(),
turbopackStaleOutputMaxAge: z.number().min(0).finite().optional(),
turbopackSourceMaps: z.boolean().optional(), turbopackSourceMaps: z.boolean().optional(),
turbopackInputSourceMaps: z.boolean().optional(), turbopackInputSourceMaps: z.boolean().optional(),
turbopackModuleFragments: z.boolean().optional(), turbopackModuleFragments: z.boolean().optional(),
......
...@@ -958,6 +958,12 @@ export interface ExperimentalConfig { ...@@ -958,6 +958,12 @@ export interface ExperimentalConfig {
*/ */
turbopackSeedCacheFromWorktree?: boolean turbopackSeedCacheFromWorktree?: boolean
/**
* The maximum age, in milliseconds, of Turbopack development output kept
* between dev server sessions. Defaults to one week.
*/
turbopackStaleOutputMaxAge?: number
/** /**
* Enable source maps. Defaults to true. * Enable source maps. Defaults to true.
*/ */
...@@ -2338,6 +2344,7 @@ export const defaultConfig = Object.freeze({ ...@@ -2338,6 +2344,7 @@ export const defaultConfig = Object.freeze({
mcpServer: true, mcpServer: true,
turbopackFileSystemCacheForDev: true, turbopackFileSystemCacheForDev: true,
turbopackFileSystemCacheForBuild: true, turbopackFileSystemCacheForBuild: true,
turbopackStaleOutputMaxAge: 7 * 24 * 60 * 60 * 1000, // One week
turbopackInferModuleSideEffects: true, turbopackInferModuleSideEffects: true,
turbopackPluginRuntimeStrategy: 'childProcesses', turbopackPluginRuntimeStrategy: 'childProcesses',
turbopackSharedRuntime: !isStableBuild(), turbopackSharedRuntime: !isStableBuild(),
......
...@@ -18,6 +18,7 @@ import type { ...@@ -18,6 +18,7 @@ import type {
TurbopackConnectedMessage, TurbopackConnectedMessage,
} from './hot-reloader-types' } from './hot-reloader-types'
import { HMR_MESSAGE_SENT_TO_BROWSER } from './hot-reloader-types' import { HMR_MESSAGE_SENT_TO_BROWSER } from './hot-reloader-types'
import { recursiveDeleteSyncWithAsyncRetries } from '../../lib/recursive-delete'
import type { import type {
Update as TurbopackUpdate, Update as TurbopackUpdate,
Endpoint, Endpoint,
...@@ -161,6 +162,16 @@ const sessionId = Math.floor(Number.MAX_SAFE_INTEGER * Math.random()) ...@@ -161,6 +162,16 @@ const sessionId = Math.floor(Number.MAX_SAFE_INTEGER * Math.random())
/** Output directory (relative to `distDir`) of server-HMR-managed chunks. */ /** Output directory (relative to `distDir`) of server-HMR-managed chunks. */
const SERVER_HMR_CHUNKS_DIR = join('server', 'chunks') const SERVER_HMR_CHUNKS_DIR = join('server', 'chunks')
const STALE_SWEPT_OUTPUT_DIRS = [
join('static', 'chunks'),
join('static', 'media'),
join('server', 'app'),
join('server', 'pages'),
SERVER_HMR_CHUNKS_DIR,
join('server', 'edge', 'chunks'),
join('server', 'edge', 'assets'),
]
declare const __next__clear_chunk_cache__: (() => void) | null | undefined declare const __next__clear_chunk_cache__: (() => void) | null | undefined
declare const __turbopack_server_hmr_apply__: declare const __turbopack_server_hmr_apply__:
...@@ -446,6 +457,23 @@ export async function createHotReloaderTurbopack( ...@@ -446,6 +457,23 @@ export async function createHotReloaderTurbopack(
}) })
} }
// Clean up any old output files from previous runs. This is safe as Turbopack
// will restore any missing chunks from persistent cache or recompute them.
//
// That only holds here, before the project exists: once turbo-tasks has
// recorded a write effect for a path, the write dedups on the recorded hash
// without checking the file, so a deleted output stays missing for the rest
// of the session.
await Promise.all(
STALE_SWEPT_OUTPUT_DIRS.map((subDir) =>
recursiveDeleteSyncWithAsyncRetries(
join(distDir, subDir),
undefined,
nextConfig.experimental.turbopackStaleOutputMaxAge!
)
)
)
const project = await bindings.turbo.createProject( const project = await bindings.turbo.createProject(
{ {
rootPath, rootPath,
......
...@@ -50,6 +50,35 @@ describe('recursiveDeleteSyncWithAsyncRetries', () => { ...@@ -50,6 +50,35 @@ describe('recursiveDeleteSyncWithAsyncRetries', () => {
expect(cleanupResult.length).toBe(0) expect(cleanupResult.length).toBe(0)
} }
}) })
it('should only delete files older than maxAgeMs, and empty out dirs', async () => {
const dir = join(__dirname, 'isolated', 'ttl')
try {
await fs.outputFile(join(dir, 'stale', 'old.js'), 'old')
await fs.outputFile(join(dir, 'fresh', 'new.js'), 'new')
await fs.outputFile(join(dir, 'mixed', 'old.js'), 'old')
await fs.outputFile(join(dir, 'mixed', 'new.js'), 'new')
await fs.outputFile(join(dir, 'future.js'), 'future')
const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000)
for (const p of ['stale/old.js', 'mixed/old.js']) {
await fs.utimes(join(dir, p), weekAgo, weekAgo)
}
const twoHoursFromNow = new Date(Date.now() + 2 * 60 * 60 * 1000)
await fs.utimes(join(dir, 'future.js'), twoHoursFromNow, twoHoursFromNow)
await recursiveDeleteSyncWithAsyncRetries(dir, undefined, 60 * 60 * 1000)
expect((await recursiveReadDir(dir)).sort()).toEqual([
'/fresh/new.js',
'/mixed/new.js',
])
// `stale` held nothing but stale files, so the directory goes too
expect(await fs.pathExists(join(dir, 'stale'))).toBe(false)
} finally {
await recursiveDeleteSyncWithAsyncRetries(dir)
}
})
}) })
describe('calcBackoffMs', () => { describe('calcBackoffMs', () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment