Unverified Commit 2b052512 authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

[backport] feat(next/image): add lru disk cache and `images.maximumDiskCacheSize` (#91660)

Backports:
- #89963

---------
Co-authored-by: 's avatarSteven <steven@ceriously.com>
parent f88cee96
...@@ -774,6 +774,36 @@ module.exports = { ...@@ -774,6 +774,36 @@ module.exports = {
} }
``` ```
#### `maximumDiskCacheSize`
The default image optimization loader will write optimized images to disk so subsequent requests can be served faster from the disk cache.
You can configure the maximum disk cache size in bytes, for example 500 MB:
```js filename="next.config.js"
module.exports = {
images: {
maximumDiskCacheSize: 500_000_000,
},
}
```
You can also disable the disk cache entirely by setting the value to `0`.
```js filename="next.config.js"
module.exports = {
images: {
maximumDiskCacheSize: 0,
},
}
```
If no value is configured, the default behavior is to check the current available disk space once during startup and use 50%.
When the disk cache exceeds the configured size, the least recently used optimized images will be deleted until the cache is under the limit again.
Alternatively, you can implement your own cache handler using [`cacheHandler`](/docs/app/api-reference/config/next-config-js/incrementalCacheHandlerPath) which will ignore the `maximumDiskCacheSize` configuration.
#### `maximumResponseBody` #### `maximumResponseBody`
The default image optimization loader will fetch source images up to 50 MB in size. The default image optimization loader will fetch source images up to 50 MB in size.
...@@ -915,7 +945,6 @@ This `next/image` component uses browser native [lazy loading](https://caniuse.c ...@@ -915,7 +945,6 @@ This `next/image` component uses browser native [lazy loading](https://caniuse.c
- Use CSS `@supports (font: -apple-system-body) and (-webkit-appearance: none) { img[loading="lazy"] { clip-path: inset(0.6px) } }` - Use CSS `@supports (font: -apple-system-body) and (-webkit-appearance: none) { img[loading="lazy"] { clip-path: inset(0.6px) } }`
- Use [`priority`](#priority) if the image is above the fold - Use [`priority`](#priority) if the image is above the fold
- [Firefox 67+](https://bugzilla.mozilla.org/show_bug.cgi?id=1556156) displays a white background while loading. Possible solutions: - [Firefox 67+](https://bugzilla.mozilla.org/show_bug.cgi?id=1556156) displays a white background while loading. Possible solutions:
- Enable [AVIF `formats`](#formats) - Enable [AVIF `formats`](#formats)
- Use [`placeholder`](#placeholder) - Use [`placeholder`](#placeholder)
...@@ -1275,6 +1304,7 @@ export default function Home() { ...@@ -1275,6 +1304,7 @@ export default function Home() {
| Version | Changes | | Version | Changes |
| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `v15.5.14` | `maximumDiskCacheSize` configuration added. |
| `v15.3.0` | `remotePatterns` added support for array of `URL` objects. | | `v15.3.0` | `remotePatterns` added support for array of `URL` objects. |
| `v15.0.0` | `contentDispositionType` configuration default changed to `attachment`. | | `v15.0.0` | `contentDispositionType` configuration default changed to `attachment`. |
| `v14.2.23` | `qualities` configuration added. | | `v14.2.23` | `qualities` configuration added. |
......
...@@ -787,5 +787,7 @@ ...@@ -787,5 +787,7 @@
"786": "Server Actions are not enabled for this application. This request might be from an older or newer deployment.\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action", "786": "Server Actions are not enabled for this application. This request might be from an older or newer deployment.\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action",
"787": "Failed to find Server Action. This request might be from an older or newer deployment.\\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action", "787": "Failed to find Server Action. This request might be from an older or newer deployment.\\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action",
"788": "Failed to find Server Action%s. This request might be from an older or newer deployment.\\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action", "788": "Failed to find Server Action%s. This request might be from an older or newer deployment.\\nRead more: https://nextjs.org/docs/messages/failed-to-find-server-action",
"789": "LRUCache: calculateSize returned %s, but size must be > 0. Items with size 0 would never be evicted, causing unbounded cache growth." "789": "LRUCache: calculateSize returned %s, but size must be > 0. Items with size 0 would never be evicted, causing unbounded cache growth.",
"790": "Invariant: cache entry \"%s\" not found in dir \"%s\"",
"791": "image of size %s could not be tracked by lru cache"
} }
...@@ -654,6 +654,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() => ...@@ -654,6 +654,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
.optional(), .optional(),
loader: z.enum(VALID_LOADERS).optional(), loader: z.enum(VALID_LOADERS).optional(),
loaderFile: z.string().optional(), loaderFile: z.string().optional(),
maximumDiskCacheSize: z.number().int().min(0).optional(),
maximumResponseBody: z maximumResponseBody: z
.number() .number()
.int() .int()
......
...@@ -27,6 +27,7 @@ import { sendEtagResponse } from './send-payload' ...@@ -27,6 +27,7 @@ import { sendEtagResponse } from './send-payload'
import { getContentType, getExtension } from './serve-static' import { getContentType, getExtension } from './serve-static'
import * as Log from '../build/output/log' import * as Log from '../build/output/log'
import isError from '../lib/is-error' import isError from '../lib/is-error'
import { getOrInitDiskLRU } from './lib/disk-lru-cache.external'
import { parseUrl } from '../lib/url' import { parseUrl } from '../lib/url'
import type { CacheControl } from './lib/cache-control' import type { CacheControl } from './lib/cache-control'
import { InvariantError } from '../shared/lib/invariant-error' import { InvariantError } from '../shared/lib/invariant-error'
...@@ -55,6 +56,29 @@ const BLUR_QUALITY = 70 // should match `next-image-loader` ...@@ -55,6 +56,29 @@ const BLUR_QUALITY = 70 // should match `next-image-loader`
let _sharp: typeof import('sharp') let _sharp: typeof import('sharp')
async function initCacheEntries(
cacheDir: string
): Promise<Array<{ key: string; size: number; expireAt: number }>> {
const cacheKeys = await promises.readdir(cacheDir).catch(() => [])
const entries: Array<{ key: string; size: number; expireAt: number }> = []
for (const cacheKey of cacheKeys) {
try {
const { expireAt, buffer } = await readFromCacheDir(cacheDir, cacheKey)
entries.push({
key: cacheKey,
size: buffer.byteLength,
expireAt,
})
} catch {
// Skip entries that can't be read from disk
}
}
// Sort oldest-first so we can replay them chronologically into LRU
return entries.sort((a, b) => a.expireAt - b.expireAt)
}
export function getSharp(concurrency: number | null | undefined) { export function getSharp(concurrency: number | null | undefined) {
if (_sharp) { if (_sharp) {
return _sharp return _sharp
...@@ -133,7 +157,8 @@ export function getImageEtag(image: Buffer) { ...@@ -133,7 +157,8 @@ export function getImageEtag(image: Buffer) {
} }
async function writeToCacheDir( async function writeToCacheDir(
dir: string, cacheDir: string,
cacheKey: string,
extension: string, extension: string,
maxAge: number, maxAge: number,
expireAt: number, expireAt: number,
...@@ -141,6 +166,7 @@ async function writeToCacheDir( ...@@ -141,6 +166,7 @@ async function writeToCacheDir(
etag: string, etag: string,
upstreamEtag: string upstreamEtag: string
) { ) {
const dir = join(/* turbopackIgnore: true */ cacheDir, cacheKey)
const filename = join( const filename = join(
dir, dir,
`${maxAge}.${expireAt}.${etag}.${upstreamEtag}.${extension}` `${maxAge}.${expireAt}.${etag}.${upstreamEtag}.${extension}`
...@@ -152,6 +178,37 @@ async function writeToCacheDir( ...@@ -152,6 +178,37 @@ async function writeToCacheDir(
await promises.writeFile(filename, buffer) await promises.writeFile(filename, buffer)
} }
async function readFromCacheDir(cacheDir: string, cacheKey: string) {
const dir = join(/* turbopackIgnore: true */ cacheDir, cacheKey)
const files = await promises.readdir(dir)
const file = files[0]
if (!file) {
throw new Error(
`Invariant: cache entry "${cacheKey}" not found in dir "${cacheDir}"`
)
}
const [maxAgeSt, expireAtSt, etag, upstreamEtag, extension] = file.split(
'.',
5
)
const filePath = join(/* turbopackIgnore: true */ dir, file)
const buffer = await promises.readFile(/* turbopackIgnore: true */ filePath)
const expireAt = Number(expireAtSt)
const maxAge = Number(maxAgeSt)
return { maxAge, expireAt, etag, upstreamEtag, buffer, extension }
}
async function deleteFromCacheDir(cacheDir: string, cacheKey: string) {
return promises
.rm(join(/* turbopackIgnore: true */ cacheDir, cacheKey), {
recursive: true,
force: true,
})
.catch((err) => {
Log.error(`Failed to delete cache key ${cacheKey}`, err)
})
}
/** /**
* Inspects the first few bytes of a buffer to determine if * Inspects the first few bytes of a buffer to determine if
* it matches the "magic number" of known file signatures. * it matches the "magic number" of known file signatures.
...@@ -310,6 +367,8 @@ export async function detectContentType( ...@@ -310,6 +367,8 @@ export async function detectContentType(
export class ImageOptimizerCache { export class ImageOptimizerCache {
private cacheDir: string private cacheDir: string
private nextConfig: NextConfigComplete private nextConfig: NextConfigComplete
private cacheDiskLRU?: ReturnType<typeof getOrInitDiskLRU>
private isDiskCacheEnabled?: boolean
static validateParams( static validateParams(
req: IncomingMessage, req: IncomingMessage,
...@@ -496,35 +555,51 @@ export class ImageOptimizerCache { ...@@ -496,35 +555,51 @@ export class ImageOptimizerCache {
}) { }) {
this.cacheDir = join(distDir, 'cache', 'images') this.cacheDir = join(distDir, 'cache', 'images')
this.nextConfig = nextConfig this.nextConfig = nextConfig
// Eagerly start LRU initialization for filesystem cache
if (
nextConfig.images.maximumDiskCacheSize !== 0 &&
nextConfig.experimental.isrFlushToDisk
) {
this.isDiskCacheEnabled = true
this.cacheDiskLRU = getOrInitDiskLRU(
this.cacheDir,
nextConfig.images.maximumDiskCacheSize,
initCacheEntries,
deleteFromCacheDir
)
}
} }
async get(cacheKey: string): Promise<IncrementalResponseCacheEntry | null> { async get(cacheKey: string): Promise<IncrementalResponseCacheEntry | null> {
// If the filesystem cache is disabled, return early
if (!this.isDiskCacheEnabled) {
return null
}
// Fall back to filesystem cache
try { try {
const cacheDir = join(this.cacheDir, cacheKey)
const files = await promises.readdir(cacheDir)
const now = Date.now() const now = Date.now()
const { maxAge, expireAt, etag, upstreamEtag, buffer, extension } =
await readFromCacheDir(this.cacheDir, cacheKey)
for (const file of files) { // Promote entry in LRU (mark as recently used)
const [maxAgeSt, expireAtSt, etag, upstreamEtag, extension] = const lru = await this.cacheDiskLRU
file.split('.', 5) lru?.get(cacheKey)
const buffer = await promises.readFile(join(cacheDir, file))
const expireAt = Number(expireAtSt)
const maxAge = Number(maxAgeSt)
return { return {
value: { value: {
kind: CachedRouteKind.IMAGE, kind: CachedRouteKind.IMAGE,
etag, etag,
buffer, buffer,
extension, extension,
upstreamEtag, upstreamEtag,
}, },
revalidateAfter: revalidateAfter:
Math.max(maxAge, this.nextConfig.images.minimumCacheTTL) * 1000 + Math.max(maxAge, this.nextConfig.images.minimumCacheTTL) * 1000 +
Date.now(), Date.now(),
cacheControl: { revalidate: maxAge, expire: undefined }, cacheControl: { revalidate: maxAge, expire: undefined },
isStale: now > expireAt, isStale: now > expireAt,
}
} }
} catch (_) { } catch (_) {
// failed to read from cache dir, treat as cache miss // failed to read from cache dir, treat as cache miss
...@@ -554,13 +629,28 @@ export class ImageOptimizerCache { ...@@ -554,13 +629,28 @@ export class ImageOptimizerCache {
throw new InvariantError('revalidate must be a number for image-cache') throw new InvariantError('revalidate must be a number for image-cache')
} }
// If the filesystem cache is disabled, return early
if (!this.isDiskCacheEnabled) {
return
}
// Fall back to filesystem cache
const expireAt = const expireAt =
Math.max(revalidate, this.nextConfig.images.minimumCacheTTL) * 1000 + Math.max(revalidate, this.nextConfig.images.minimumCacheTTL) * 1000 +
Date.now() Date.now()
try { try {
const lru = await this.cacheDiskLRU
const success = lru?.set(cacheKey, value.buffer.byteLength)
if (success === false) {
throw new Error(
`image of size ${value.buffer.byteLength} could not be tracked by lru cache`
)
}
await writeToCacheDir( await writeToCacheDir(
join(this.cacheDir, cacheKey), this.cacheDir,
cacheKey,
value.extension, value.extension,
revalidate, revalidate,
expireAt, expireAt,
......
import { promises } from 'fs'
import { LRUCache } from './lru-cache'
/**
* Module-level LRU singleton for disk cache eviction.
* Initialized once on first `set()`, shared across all consumers.
* Once resolved, the promise stays resolved — subsequent calls just await the cached result.
*/
let _diskLRUPromise: Promise<LRUCache<number>> | null = null
/**
* Initialize or return the module-level LRU for disk cache eviction.
* Concurrent calls are deduplicated via the shared promise.
*
* @param cacheDir - The directory where cached files are stored
* @param maxDiskSize - Maximum disk cache size in bytes
* @param readEntries - Callback to scan existing cache entries (format-agnostic)
*/
export async function getOrInitDiskLRU(
cacheDir: string,
maxDiskSize: number | undefined,
readEntries: (
cacheDir: string
) => Promise<Array<{ key: string; size: number; expireAt: number }>>,
evictEntry: (cacheDir: string, cacheKey: string) => Promise<void>
): Promise<LRUCache<number>> {
if (!_diskLRUPromise) {
_diskLRUPromise = (async () => {
let maxSize = maxDiskSize
if (typeof maxSize === 'undefined') {
// Ensure cacheDir exists before checking disk space
await promises.mkdir(cacheDir, { recursive: true })
// Since config was not provided, default to 50% of available disk space
const { bavail, bsize } = await promises.statfs(cacheDir)
maxSize = Math.floor((bavail * bsize) / 2)
}
const lru = new LRUCache<number>(
maxSize,
(size) => size,
(cacheKey) => evictEntry(cacheDir, cacheKey)
)
const entries = await readEntries(cacheDir)
for (const entry of entries) {
lru.set(entry.key, entry.size)
}
return lru
})()
}
return _diskLRUPromise
}
/**
* Reset the module-level LRU singleton. Exported for testing only.
*/
export function resetDiskLRU(): void {
_diskLRUPromise = null
}
...@@ -9,7 +9,7 @@ describe('LRUCache', () => { ...@@ -9,7 +9,7 @@ describe('LRUCache', () => {
}) })
it('should set and get values', () => { it('should set and get values', () => {
cache.set('key1', 'value1') expect(cache.set('key1', 'value1')).toBe(true)
expect(cache.get('key1')).toBe('value1') expect(cache.get('key1')).toBe('value1')
}) })
...@@ -105,11 +105,11 @@ describe('LRUCache', () => { ...@@ -105,11 +105,11 @@ describe('LRUCache', () => {
expect(cache.currentSize).toBe(8) // 5 + 2 + 1 expect(cache.currentSize).toBe(8) // 5 + 2 + 1
}) })
it('should handle items larger than max size', () => { it('should prevent adding item larger than max size when lru is empty', () => {
const consoleSpy = jest.spyOn(console, 'warn').mockImplementation() const consoleSpy = jest.spyOn(console, 'warn').mockImplementation()
const cache = new LRUCache<string>(5, (value) => value.length) const cache = new LRUCache<string>(5, (value) => value.length)
cache.set('key1', 'toolarge') // size 8 > maxSize 5 expect(cache.set('key1', 'toolarge')).toBe(false) // size 8 > maxSize 5
expect(cache.has('key1')).toBe(false) expect(cache.has('key1')).toBe(false)
expect(cache.size).toBe(0) expect(cache.size).toBe(0)
...@@ -120,6 +120,27 @@ describe('LRUCache', () => { ...@@ -120,6 +120,27 @@ describe('LRUCache', () => {
consoleSpy.mockRestore() consoleSpy.mockRestore()
}) })
it('should prevent adding item larger than max size when lru is not empty', () => {
const consoleSpy = jest.spyOn(console, 'warn').mockImplementation()
const cache = new LRUCache<string>(5, (value) => value.length)
expect(cache.set('key1', 'ab')).toBe(true) // size 2
expect(cache.set('key2', 'cd')).toBe(true) // size 2, total = 4
expect(cache.set('key3', 'toolarge')).toBe(false) // size 8 > maxSize 5, should be rejected
expect(cache.has('key1')).toBe(true)
expect(cache.has('key2')).toBe(true)
expect(cache.has('key3')).toBe(false)
expect(cache.size).toBe(2)
expect(cache.currentSize).toBe(4)
expect(consoleSpy).toHaveBeenCalledWith(
'Single item size exceeds maxSize'
)
consoleSpy.mockRestore()
})
it('should update size when overwriting existing keys', () => { it('should update size when overwriting existing keys', () => {
const cache = new LRUCache<string>(10, (value) => value.length) const cache = new LRUCache<string>(10, (value) => value.length)
...@@ -184,7 +205,7 @@ describe('LRUCache', () => { ...@@ -184,7 +205,7 @@ describe('LRUCache', () => {
describe('Edge Cases', () => { describe('Edge Cases', () => {
it('should handle zero max size', () => { it('should handle zero max size', () => {
const cache = new LRUCache<string>(0) const cache = new LRUCache<string>(0)
cache.set('key1', 'value1') expect(cache.set('key1', 'value1')).toBe(false)
expect(cache.has('key1')).toBe(false) expect(cache.has('key1')).toBe(false)
expect(cache.size).toBe(0) expect(cache.size).toBe(0)
}) })
......
...@@ -123,7 +123,7 @@ export class LRUCache<T> { ...@@ -123,7 +123,7 @@ export class LRUCache<T> {
* - O(1) for uniform item sizes * - O(1) for uniform item sizes
* - O(k) where k is the number of items evicted (can be O(N) for variable sizes) * - O(k) where k is the number of items evicted (can be O(N) for variable sizes)
*/ */
public set(key: string, value: T): void { public set(key: string, value: T): boolean {
const size = this.calculateSize?.(value) ?? 1 const size = this.calculateSize?.(value) ?? 1
if (size <= 0) { if (size <= 0) {
throw new Error( throw new Error(
...@@ -133,7 +133,7 @@ export class LRUCache<T> { ...@@ -133,7 +133,7 @@ export class LRUCache<T> {
} }
if (size > this.maxSize) { if (size > this.maxSize) {
console.warn('Single item size exceeds maxSize') console.warn('Single item size exceeds maxSize')
return return false
} }
const existing = this.cache.get(key) const existing = this.cache.get(key)
...@@ -158,6 +158,8 @@ export class LRUCache<T> { ...@@ -158,6 +158,8 @@ export class LRUCache<T> {
this.totalSize -= tail.size this.totalSize -= tail.size
this.onEvict?.(tail.key, tail.data) this.onEvict?.(tail.key, tail.data)
} }
return true
} }
/** /**
......
...@@ -103,6 +103,9 @@ export type ImageConfigComplete = { ...@@ -103,6 +103,9 @@ export type ImageConfigComplete = {
/** @see [Acceptable formats](https://nextjs.org/docs/api-reference/next/image#acceptable-formats) */ /** @see [Acceptable formats](https://nextjs.org/docs/api-reference/next/image#acceptable-formats) */
formats: ImageFormat[] formats: ImageFormat[]
/** @see [Maximum Disk Cache Size (in bytes)](https://nextjs.org/docs/api-reference/next/image#maximumdiskcachesize) */
maximumDiskCacheSize: number | undefined
/** @see [Maximum Response Body](https://nextjs.org/docs/api-reference/next/image#maximumresponsebody) */ /** @see [Maximum Response Body](https://nextjs.org/docs/api-reference/next/image#maximumresponsebody) */
maximumResponseBody: number maximumResponseBody: number
...@@ -140,6 +143,7 @@ export const imageConfigDefault: ImageConfigComplete = { ...@@ -140,6 +143,7 @@ export const imageConfigDefault: ImageConfigComplete = {
disableStaticImages: false, disableStaticImages: false,
minimumCacheTTL: 60, minimumCacheTTL: 60,
formats: ['image/webp'], formats: ['image/webp'],
maximumDiskCacheSize: undefined, // auto-detect by default
maximumResponseBody: 50_000_000, // 50 MB maximumResponseBody: 50_000_000, // 50 MB
dangerouslyAllowSVG: false, dangerouslyAllowSVG: false,
contentSecurityPolicy: `script-src 'none'; frame-src 'none'; sandbox;`, contentSecurityPolicy: `script-src 'none'; frame-src 'none'; sandbox;`,
......
import { join } from 'path'
import { setupTests } from './util'
const appDir = join(__dirname, '../app')
describe('with maximumDiskCacheSize 85KB config', () => {
setupTests({
appDir,
nextConfigImages: {
maximumDiskCacheSize: 85_000,
},
})
})
import { join } from 'path'
import { setupTests } from './util'
const appDir = join(__dirname, '../app')
describe('with maximumDiskCacheSize zero config', () => {
setupTests({
appDir,
nextConfigImages: {
maximumDiskCacheSize: 0,
},
})
})
...@@ -12,6 +12,7 @@ import { ...@@ -12,6 +12,7 @@ import {
launchApp, launchApp,
nextBuild, nextBuild,
nextStart, nextStart,
retry,
waitFor, waitFor,
} from 'next-test-utils' } from 'next-test-utils'
import isAnimated from 'next/dist/compiled/is-animated' import isAnimated from 'next/dist/compiled/is-animated'
...@@ -20,7 +21,7 @@ import type { NextConfig } from 'next' ...@@ -20,7 +21,7 @@ import type { NextConfig } from 'next'
type SetupTestsCtx = { type SetupTestsCtx = {
appDir: string appDir: string
imagesDir: string imagesDir?: string
nextConfigImages?: Partial<import('next').NextConfig['images']> nextConfigImages?: Partial<import('next').NextConfig['images']>
nextConfigExperimental?: Partial<import('next').NextConfig['experimental']> nextConfigExperimental?: Partial<import('next').NextConfig['experimental']>
isDev?: boolean isDev?: boolean
...@@ -28,6 +29,8 @@ type SetupTestsCtx = { ...@@ -28,6 +29,8 @@ type SetupTestsCtx = {
type RunTestsCtx = SetupTestsCtx & { type RunTestsCtx = SetupTestsCtx & {
w: number w: number
q: number
imagesDir: string
app?: import('child_process').ChildProcess app?: import('child_process').ChildProcess
appDir?: string appDir?: string
appPort?: number appPort?: number
...@@ -102,6 +105,22 @@ export const cleanImagesDir = async (ctx: { imagesDir: string }) => { ...@@ -102,6 +105,22 @@ export const cleanImagesDir = async (ctx: { imagesDir: string }) => {
await fs.remove(ctx.imagesDir) await fs.remove(ctx.imagesDir)
} }
async function getDirSize(dir: string): Promise<number> {
let totalSize = 0
const entries = await fs.readdir(dir).catch(() => [] as string[])
for (const entry of entries) {
const entryPath = join(dir, entry)
const stat = await fs.stat(entryPath).catch(() => null)
if (!stat) continue
if (stat.isDirectory()) {
totalSize += await getDirSize(entryPath)
} else {
totalSize += stat.size
}
}
return totalSize
}
async function expectAvifSmallerThanWebp( async function expectAvifSmallerThanWebp(
w: number, w: number,
q: number, q: number,
...@@ -917,7 +936,10 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -917,7 +936,10 @@ export function runTests(ctx: RunTestsCtx) {
}) })
it('should use cache and stale-while-revalidate when query is the same for external image', async () => { it('should use cache and stale-while-revalidate when query is the same for external image', async () => {
if (ctx.nextConfigExperimental?.isrFlushToDisk === false) { if (
ctx.nextConfigExperimental?.isrFlushToDisk === false ||
ctx.nextConfigImages?.maximumDiskCacheSize === 0
) {
return // this test is not applicable when we don't write the cache return // this test is not applicable when we don't write the cache
} }
await cleanImagesDir(ctx) await cleanImagesDir(ctx)
...@@ -1141,7 +1163,10 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1141,7 +1163,10 @@ export function runTests(ctx: RunTestsCtx) {
} }
it('should use cache and stale-while-revalidate when query is the same for internal image', async () => { it('should use cache and stale-while-revalidate when query is the same for internal image', async () => {
if (ctx.nextConfigExperimental?.isrFlushToDisk === false) { if (
ctx.nextConfigExperimental?.isrFlushToDisk === false ||
ctx.nextConfigImages?.maximumDiskCacheSize === 0
) {
return // this test is not applicable when we don't write the cache return // this test is not applicable when we don't write the cache
} }
await cleanImagesDir(ctx) await cleanImagesDir(ctx)
...@@ -1294,7 +1319,10 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1294,7 +1319,10 @@ export function runTests(ctx: RunTestsCtx) {
} }
it('should use cached image file when parameters are the same for animated gif', async () => { it('should use cached image file when parameters are the same for animated gif', async () => {
if (ctx.nextConfigExperimental?.isrFlushToDisk === false) { if (
ctx.nextConfigExperimental?.isrFlushToDisk === false ||
ctx.nextConfigImages?.maximumDiskCacheSize === 0
) {
return // this test is not applicable when we don't write the cache return // this test is not applicable when we don't write the cache
} }
await cleanImagesDir(ctx) await cleanImagesDir(ctx)
...@@ -1393,7 +1421,10 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1393,7 +1421,10 @@ export function runTests(ctx: RunTestsCtx) {
`${contentDispositionType}; filename="test.bmp"` `${contentDispositionType}; filename="test.bmp"`
) )
if (ctx.nextConfigExperimental?.isrFlushToDisk === false) { if (
ctx.nextConfigExperimental?.isrFlushToDisk === false ||
ctx.nextConfigImages?.maximumDiskCacheSize === 0
) {
expect(json1).toEqual({}) expect(json1).toEqual({})
expect(await fsToJson(ctx.imagesDir)).toEqual({}) expect(await fsToJson(ctx.imagesDir)).toEqual({})
} else { } else {
...@@ -1518,7 +1549,10 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1518,7 +1549,10 @@ export function runTests(ctx: RunTestsCtx) {
await expectWidth(res3, ctx.w) await expectWidth(res3, ctx.w)
const length = const length =
ctx.nextConfigExperimental?.isrFlushToDisk === false ? 0 : 1 ctx.nextConfigExperimental?.isrFlushToDisk === false ||
ctx.nextConfigImages?.maximumDiskCacheSize === 0
? 0
: 1
await check(async () => { await check(async () => {
const json1 = await fsToJson(ctx.imagesDir) const json1 = await fsToJson(ctx.imagesDir)
...@@ -1535,10 +1569,75 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1535,10 +1569,75 @@ export function runTests(ctx: RunTestsCtx) {
expect(xCache).toEqual(['MISS', 'MISS', 'MISS']) expect(xCache).toEqual(['MISS', 'MISS', 'MISS'])
}) })
} }
if (typeof ctx.nextConfigImages?.maximumDiskCacheSize !== 'undefined') {
const { maximumDiskCacheSize } = ctx.nextConfigImages
it(`should handle maximumDiskCacheSize ${maximumDiskCacheSize}`, async () => {
const opts = { headers: { accept: 'image/webp' } }
const requests = [
{ url: '/test.png', w: largeSize },
{ url: '/test.jpg', w: largeSize },
{ url: '/test.gif', w: largeSize },
{ url: '/test.bmp', w: largeSize },
{ url: '/test.webp', w: largeSize },
{ url: '/test.avif', w: largeSize },
{ url: '/test.tiff', w: largeSize },
{ url: '/test.ico', w: largeSize },
{ url: '/animated.gif', w: largeSize },
{ url: '/animated.png', w: largeSize },
{ url: '/animated2.png', w: largeSize },
]
await cleanImagesDir(ctx)
const json1 = await fsToJson(ctx.imagesDir)
expect(Object.keys(json1).length).toEqual(0)
for (const { url, w } of requests) {
const query = { url, w, q: ctx.q }
const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
await res.buffer() // consume response body
await retry(async () => {
const size = await getDirSize(ctx.imagesDir)
expect(size).toBeLessThanOrEqual(maximumDiskCacheSize)
})
}
const json2 = await fsToJson(ctx.imagesDir)
const json2Length = Object.keys(json2).length
if (maximumDiskCacheSize === 0) {
expect(json2Length).toEqual(0)
} else {
expect(json2Length).toBeGreaterThan(0)
}
const res = await fetchViaHTTP(
ctx.appPort,
'/_next/image',
{ url: '/mountains.jpg', w: ctx.w, q: ctx.q },
opts
)
expect(res.status).toBe(200)
await retry(async () => {
const json3 = await fsToJson(ctx.imagesDir)
const json3Length = Object.keys(json3).length
if (maximumDiskCacheSize === 0) {
expect(json3Length).toEqual(0)
} else {
expect(json3Length).toBeGreaterThan(0)
expect(json3).not.toStrictEqual(json2)
}
const size = await getDirSize(ctx.imagesDir)
expect(size).toBeLessThanOrEqual(maximumDiskCacheSize)
})
})
}
} }
export const setupTests = (ctx: SetupTestsCtx) => { export const setupTests = (ctx: SetupTestsCtx) => {
const nextConfig = new File(join(ctx.appDir, 'next.config.js')) const nextConfig = new File(join(ctx.appDir, 'next.config.js'))
// Compute imagesDir if not provided
const imagesDir =
ctx.imagesDir || join(ctx.appDir, '.next', 'cache', 'images')
describe('dev support w/o next.config.js', () => { describe('dev support w/o next.config.js', () => {
if (ctx.nextConfigImages) { if (ctx.nextConfigImages) {
...@@ -1548,7 +1647,9 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1548,7 +1647,9 @@ export const setupTests = (ctx: SetupTestsCtx) => {
const size = 384 // defaults defined in server/config.ts const size = 384 // defaults defined in server/config.ts
const curCtx: RunTestsCtx = { const curCtx: RunTestsCtx = {
...ctx, ...ctx,
imagesDir,
w: size, w: size,
q: 75,
isDev: true, isDev: true,
} }
...@@ -1567,7 +1668,7 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1567,7 +1668,7 @@ export const setupTests = (ctx: SetupTestsCtx) => {
}, },
cwd: curCtx.appDir, cwd: curCtx.appDir,
}) })
await cleanImagesDir(ctx) await cleanImagesDir({ imagesDir })
}) })
afterAll(async () => { afterAll(async () => {
nextConfig.restore() nextConfig.restore()
...@@ -1581,7 +1682,9 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1581,7 +1682,9 @@ export const setupTests = (ctx: SetupTestsCtx) => {
const size = 400 const size = 400
const curCtx: RunTestsCtx = { const curCtx: RunTestsCtx = {
...ctx, ...ctx,
imagesDir,
w: size, w: size,
q: 75,
isDev: true, isDev: true,
nextConfigImages: { nextConfigImages: {
domains: [ domains: [
...@@ -1606,7 +1709,7 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1606,7 +1709,7 @@ export const setupTests = (ctx: SetupTestsCtx) => {
} satisfies NextConfig) } satisfies NextConfig)
curCtx.nextOutput = '' curCtx.nextOutput = ''
nextConfig.replace('{ /* replaceme */ }', json) nextConfig.replace('{ /* replaceme */ }', json)
await cleanImagesDir(ctx) await cleanImagesDir({ imagesDir })
curCtx.appPort = await findPort() curCtx.appPort = await findPort()
curCtx.app = await launchApp(curCtx.appDir, curCtx.appPort, { curCtx.app = await launchApp(curCtx.appDir, curCtx.appPort, {
onStderr(msg) { onStderr(msg) {
...@@ -1632,7 +1735,9 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1632,7 +1735,9 @@ export const setupTests = (ctx: SetupTestsCtx) => {
const size = 384 // defaults defined in server/config.ts const size = 384 // defaults defined in server/config.ts
const curCtx: RunTestsCtx = { const curCtx: RunTestsCtx = {
...ctx, ...ctx,
imagesDir,
w: size, w: size,
q: 75,
isDev: false, isDev: false,
} }
beforeAll(async () => { beforeAll(async () => {
...@@ -1644,7 +1749,7 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1644,7 +1749,7 @@ export const setupTests = (ctx: SetupTestsCtx) => {
nextConfig.replace('{ /* replaceme */ }', json) nextConfig.replace('{ /* replaceme */ }', json)
curCtx.nextOutput = '' curCtx.nextOutput = ''
await nextBuild(curCtx.appDir) await nextBuild(curCtx.appDir)
await cleanImagesDir(ctx) await cleanImagesDir({ imagesDir })
curCtx.appPort = await findPort() curCtx.appPort = await findPort()
curCtx.app = await nextStart(curCtx.appDir, curCtx.appPort, { curCtx.app = await nextStart(curCtx.appDir, curCtx.appPort, {
onStderr(msg) { onStderr(msg) {
...@@ -1666,7 +1771,9 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1666,7 +1771,9 @@ export const setupTests = (ctx: SetupTestsCtx) => {
const size = 399 const size = 399
const curCtx: RunTestsCtx = { const curCtx: RunTestsCtx = {
...ctx, ...ctx,
imagesDir,
w: size, w: size,
q: 75,
isDev: false, isDev: false,
nextConfigImages: { nextConfigImages: {
domains: [ domains: [
...@@ -1691,7 +1798,7 @@ export const setupTests = (ctx: SetupTestsCtx) => { ...@@ -1691,7 +1798,7 @@ export const setupTests = (ctx: SetupTestsCtx) => {
curCtx.nextOutput = '' curCtx.nextOutput = ''
nextConfig.replace('{ /* replaceme */ }', json) nextConfig.replace('{ /* replaceme */ }', json)
await nextBuild(curCtx.appDir) await nextBuild(curCtx.appDir)
await cleanImagesDir(ctx) await cleanImagesDir({ imagesDir })
curCtx.appPort = await findPort() curCtx.appPort = await findPort()
curCtx.app = await nextStart(curCtx.appDir, curCtx.appPort, { curCtx.app = await nextStart(curCtx.appDir, curCtx.appPort, {
onStderr(msg) { onStderr(msg) {
......
/* eslint-env jest */
import { join } from 'path'
import { promises } from 'fs'
import { tmpdir } from 'os'
import { setTimeout } from 'timers/promises'
import {
getOrInitDiskLRU,
resetDiskLRU,
} from 'next/dist/server/lib/disk-lru-cache.external'
async function writeEntry(
cacheDir: string,
key: string,
sizeInBytes: number,
expireAt: number = Date.now() + 60_000
) {
const dir = join(cacheDir, key)
const buffer = Buffer.alloc(sizeInBytes, 0x42) // Fill with dummy data
await promises.mkdir(dir, { recursive: true })
await promises.writeFile(join(dir, `${expireAt}.bin`), buffer)
}
async function readEntry(cacheDir: string, key: string) {
const dir = join(cacheDir, key)
const [file] = await promises.readdir(dir)
const buffer = await promises.readFile(join(dir, file))
const [expireAtStr] = file.split('.')
return { size: buffer.byteLength, expireAt: Number(expireAtStr) }
}
async function initEntries(
cacheDir: string
): Promise<Array<{ key: string; size: number; expireAt: number }>> {
const keys = await promises.readdir(cacheDir).catch(() => [])
const entries: Array<{ key: string; size: number; expireAt: number }> = []
for (const key of keys) {
const { size, expireAt } = await readEntry(cacheDir, key)
entries.push({ key, size, expireAt })
}
// Sort oldest-first so we can replay them chronologically into LRU
return entries.sort((a, b) => a.expireAt - b.expireAt)
}
async function rmEntry(cacheDir: string, cacheKey: string): Promise<void> {
await promises.rm(join(cacheDir, cacheKey), { recursive: true, force: true })
}
describe('LRU disk eviction', () => {
let cacheDir: string
beforeEach(async () => {
cacheDir = await promises.mkdtemp(join(tmpdir(), 'next-lru-test-'))
resetDiskLRU()
})
afterEach(async () => {
resetDiskLRU()
await promises.rm(cacheDir, { recursive: true, force: true })
})
it('should evict oldest entries on initialization', async () => {
const expireAt = Date.now() + 60_000
// Write 4 entries of 400 bytes each (total 1600)
await writeEntry(cacheDir, 'entry-a', 400, expireAt + 1)
await writeEntry(cacheDir, 'entry-b', 400, expireAt + 2)
await writeEntry(cacheDir, 'entry-c', 400, expireAt + 3)
await writeEntry(cacheDir, 'entry-d', 400, expireAt + 4)
// Init LRU with 1500 byte limit (less than 1600 current total)
const lru = await getOrInitDiskLRU(cacheDir, 1500, initEntries, rmEntry)
// entry-a should have been evicted (oldest)
expect(lru.has('entry-a')).toBe(false)
expect(lru.has('entry-b')).toBe(true)
expect(lru.has('entry-c')).toBe(true)
expect(lru.has('entry-d')).toBe(true)
// Verify disk eviction (fire-and-forget, so wait a tick)
await setTimeout(100)
const contents = await promises.readdir(cacheDir)
expect(contents).toEqual(['entry-b', 'entry-c', 'entry-d'])
})
it('should evict old entries when new entries are set', async () => {
const lru = await getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry)
// Add entries via LRU set (simulating what ImageOptimizerCache.set does)
await writeEntry(cacheDir, 'new-a', 400)
await writeEntry(cacheDir, 'new-b', 400)
lru.set('new-a', 400)
lru.set('new-b', 400)
// Both should exist
expect(lru.has('new-a')).toBe(true)
expect(lru.has('new-b')).toBe(true)
// Adding a third entry should evict the oldest (new-a)
await writeEntry(cacheDir, 'new-c', 400)
lru.set('new-c', 400)
expect(lru.has('new-a')).toBe(false)
expect(lru.has('new-b')).toBe(true)
expect(lru.has('new-c')).toBe(true)
// Verify disk eviction (fire-and-forget, wait a tick)
await setTimeout(100)
const contents = await promises.readdir(cacheDir)
expect(contents).toEqual(['new-b', 'new-c'])
})
it('should promote entries on get() to prevent eviction', async () => {
const lru = await getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry)
await writeEntry(cacheDir, 'x', 400)
await writeEntry(cacheDir, 'y', 400)
lru.set('x', 400)
lru.set('y', 400)
// Access 'x' to promote it (mark as recently used)
lru.get('x')
// Add 'z' - should evict 'y' (least recently used) instead of 'x'
await writeEntry(cacheDir, 'z', 400)
lru.set('z', 400)
expect(lru.has('x')).toBe(true)
expect(lru.has('y')).toBe(false)
expect(lru.has('z')).toBe(true)
})
it('should return the same LRU instance on subsequent calls', async () => {
const lru1 = await getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry)
const lru2 = await getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry)
expect(lru1 === lru2).toBeTrue()
})
it('should deduplicate concurrent init calls', async () => {
const [lru1, lru2] = await Promise.all([
getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry),
getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry),
])
expect(lru1 === lru2).toBeTrue()
})
it('should handle empty cache directory', async () => {
const lru = await getOrInitDiskLRU(cacheDir, 1000, initEntries, rmEntry)
expect(lru.size).toBe(0)
})
it('should handle non-existent cache directory', async () => {
const missing = join(cacheDir, 'this-does-not-exist')
const lru = await getOrInitDiskLRU(missing, 1000, initEntries, rmEntry)
expect(lru.size).toBe(0)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment