Unverified Commit 2cee98ab authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Sweep stale Turbopack output from distDir on dev startup (#97591)

## Summary

Turbopack dev never removes the content-hashed chunks it supersedes, so
`.next/dev` grows without bound. Webpack dev avoids this by wiping
`distDir` on every start.

Adds a `maxAgeMs` argument to `recursiveDeleteSyncWithAsyncRetries`:
delete files whose mtime is at least that old, remove directories once
empty. Turbopack dev passes one week.

## Verification

- `pnpm test-unit test/unit/recursive-delete.test.ts`
- ~Not run:~ dev-server behavior against a real `.next/dev`

<!-- NEXT_JS_LLM -->
parent 95b77e82
......@@ -59,6 +59,27 @@ function unlinkPath(
}
}
const FUTURE_MTIME_TOLERANCE_MS = 60 * 60 * 1000
/** Returns whether `p`'s own mtime falls outside the retained time range. */
function isStale(
p: string,
staleBefore: number,
futureMtimeThreshold: number
): boolean {
try {
// lstat, not stat: a symlink is deleted based on its own age, not its
// target's.
const mtimeMs = fs.lstatSync(p).mtimeMs
return mtimeMs <= staleBefore || mtimeMs > futureMtimeThreshold
} catch (e) {
if (isError(e) && e.code === 'ENOENT') {
return false
}
throw e
}
}
/**
* Recursively delete directory contents.
*
......@@ -74,34 +95,83 @@ export async function recursiveDeleteSyncWithAsyncRetries(
dir: string,
/** Exclude based on relative file path */
exclude?: RegExp,
/** Relative path to the directory being deleted, used for exclude */
previousPath: string = ''
/**
* Only delete files whose mtime is at least this old. Directories are
* removed once empty.
*/
maxAgeMs?: number
): Promise<void> {
const now = Date.now()
await deleteContents(dir, {
exclude,
staleBefore: maxAgeMs === undefined ? undefined : now - maxAgeMs,
futureMtimeThreshold: now + FUTURE_MTIME_TOLERANCE_MS,
})
}
/** @returns whether anything was kept. */
async function deleteContents(
dir: string,
{
exclude,
staleBefore,
futureMtimeThreshold,
previousPath,
}: {
exclude: RegExp | undefined
staleBefore: number | undefined
futureMtimeThreshold: number
/** Relative path to the directory being deleted, used for exclude */
previousPath?: string
}
): Promise<boolean> {
let result
try {
result = fs.readdirSync(dir, { withFileTypes: true })
} catch (e) {
if (isError(e) && e.code === 'ENOENT') {
return
return false
}
throw e
}
let keptAnything = false
await Promise.all(
result.map(async (part: Dirent) => {
const absolutePath = join(dir, part.name)
const pp = join(previousPath, part.name)
const isNotExcluded = !exclude || !exclude.test(pp)
const pp = join(previousPath ?? '', part.name)
if (exclude?.test(pp)) {
keptAnything = true
return
}
if (isNotExcluded) {
// Note: readdir does not follow symbolic links, that's good: we want to
// delete the links and not the destination.
let isDirectory = part.isDirectory()
if (isDirectory) {
await recursiveDeleteSyncWithAsyncRetries(absolutePath, exclude, pp)
// Note: readdir does not follow symbolic links, that's good: we want to
// delete the links and not the destination.
const isDirectory = part.isDirectory()
if (isDirectory) {
if (
await deleteContents(absolutePath, {
exclude,
staleBefore,
futureMtimeThreshold,
previousPath: pp,
})
) {
keptAnything = true
return
}
return unlinkPath(absolutePath, isDirectory)
} else if (
staleBefore !== undefined &&
!isStale(absolutePath, staleBefore, futureMtimeThreshold)
) {
keptAnything = true
return
}
return unlinkPath(absolutePath, isDirectory)
})
)
return keptAnything
}
......@@ -390,6 +390,7 @@ export const experimentalSchema = {
turbopackFileSystemCacheForDev: z.boolean().optional(),
turbopackFileSystemCacheForBuild: z.boolean().optional(),
turbopackSeedCacheFromWorktree: z.boolean().optional(),
turbopackStaleOutputMaxAge: z.number().min(0).finite().optional(),
turbopackSourceMaps: z.boolean().optional(),
turbopackInputSourceMaps: z.boolean().optional(),
turbopackModuleFragments: z.boolean().optional(),
......
......@@ -958,6 +958,12 @@ export interface ExperimentalConfig {
*/
turbopackSeedCacheFromWorktree?: boolean
/**
* The maximum age, in milliseconds, of Turbopack development output kept
* between dev server sessions. Defaults to one week.
*/
turbopackStaleOutputMaxAge?: number
/**
* Enable source maps. Defaults to true.
*/
......@@ -2338,6 +2344,7 @@ export const defaultConfig = Object.freeze({
mcpServer: true,
turbopackFileSystemCacheForDev: true,
turbopackFileSystemCacheForBuild: true,
turbopackStaleOutputMaxAge: 7 * 24 * 60 * 60 * 1000, // One week
turbopackInferModuleSideEffects: true,
turbopackPluginRuntimeStrategy: 'childProcesses',
turbopackSharedRuntime: !isStableBuild(),
......
......@@ -18,6 +18,7 @@ import type {
TurbopackConnectedMessage,
} from './hot-reloader-types'
import { HMR_MESSAGE_SENT_TO_BROWSER } from './hot-reloader-types'
import { recursiveDeleteSyncWithAsyncRetries } from '../../lib/recursive-delete'
import type {
Update as TurbopackUpdate,
Endpoint,
......@@ -161,6 +162,16 @@ const sessionId = Math.floor(Number.MAX_SAFE_INTEGER * Math.random())
/** Output directory (relative to `distDir`) of server-HMR-managed chunks. */
const SERVER_HMR_CHUNKS_DIR = join('server', 'chunks')
const STALE_SWEPT_OUTPUT_DIRS = [
join('static', 'chunks'),
join('static', 'media'),
join('server', 'app'),
join('server', 'pages'),
SERVER_HMR_CHUNKS_DIR,
join('server', 'edge', 'chunks'),
join('server', 'edge', 'assets'),
]
declare const __next__clear_chunk_cache__: (() => void) | null | undefined
declare const __turbopack_server_hmr_apply__:
......@@ -446,6 +457,23 @@ export async function createHotReloaderTurbopack(
})
}
// Clean up any old output files from previous runs. This is safe as Turbopack
// will restore any missing chunks from persistent cache or recompute them.
//
// That only holds here, before the project exists: once turbo-tasks has
// recorded a write effect for a path, the write dedups on the recorded hash
// without checking the file, so a deleted output stays missing for the rest
// of the session.
await Promise.all(
STALE_SWEPT_OUTPUT_DIRS.map((subDir) =>
recursiveDeleteSyncWithAsyncRetries(
join(distDir, subDir),
undefined,
nextConfig.experimental.turbopackStaleOutputMaxAge!
)
)
)
const project = await bindings.turbo.createProject(
{
rootPath,
......
......@@ -50,6 +50,35 @@ describe('recursiveDeleteSyncWithAsyncRetries', () => {
expect(cleanupResult.length).toBe(0)
}
})
it('should only delete files older than maxAgeMs, and empty out dirs', async () => {
const dir = join(__dirname, 'isolated', 'ttl')
try {
await fs.outputFile(join(dir, 'stale', 'old.js'), 'old')
await fs.outputFile(join(dir, 'fresh', 'new.js'), 'new')
await fs.outputFile(join(dir, 'mixed', 'old.js'), 'old')
await fs.outputFile(join(dir, 'mixed', 'new.js'), 'new')
await fs.outputFile(join(dir, 'future.js'), 'future')
const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000)
for (const p of ['stale/old.js', 'mixed/old.js']) {
await fs.utimes(join(dir, p), weekAgo, weekAgo)
}
const twoHoursFromNow = new Date(Date.now() + 2 * 60 * 60 * 1000)
await fs.utimes(join(dir, 'future.js'), twoHoursFromNow, twoHoursFromNow)
await recursiveDeleteSyncWithAsyncRetries(dir, undefined, 60 * 60 * 1000)
expect((await recursiveReadDir(dir)).sort()).toEqual([
'/fresh/new.js',
'/mixed/new.js',
])
// `stale` held nothing but stale files, so the directory goes too
expect(await fs.pathExists(join(dir, 'stale'))).toBe(false)
} finally {
await recursiveDeleteSyncWithAsyncRetries(dir)
}
})
})
describe('calcBackoffMs', () => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment