Unverified Commit 95734265 authored by Steven's avatar Steven Committed by GitHub

Add etag header to optimized image response (#18986)

Fixes #18563 by adding the etag header to the optimized image response.

This does _not_ change the expireAt (TTL) for cached files on the server, which still uses the max-age of the upstream response.

The new file format on disk for cached image files is the following:

```
.next/cache/images/<HASHED_QUERYSTRING>/<EXPIREAT>.<ETAG>.<EXT>
```
parent d5daa066
import { IncomingMessage, ServerResponse } from 'http'
import { parse } from 'next/dist/compiled/content-type'
import { CookieSerializeOptions } from 'next/dist/compiled/cookie'
import generateETag from 'etag'
import fresh from 'next/dist/compiled/fresh'
import getRawBody from 'raw-body'
import { PageConfig } from 'next/types'
import { Stream } from 'stream'
......@@ -10,6 +8,8 @@ import { isResSent, NextApiRequest, NextApiResponse } from '../lib/utils'
import { decryptWithSecret, encryptWithSecret } from './crypto-utils'
import { interopDefault } from './load-components'
import { Params } from './router'
import { sendEtagResponse } from './send-payload'
import generateETag from 'etag'
export type NextApiRequestCookies = { [key: string]: string }
export type NextApiRequestQuery = { [key: string]: string | string[] }
......@@ -216,23 +216,6 @@ export function redirect(
return res
}
function sendEtagResponse(
req: NextApiRequest,
res: NextApiResponse,
body: string | Buffer
): boolean {
const etag = generateETag(body)
if (fresh(req.headers, { etag })) {
res.statusCode = 304
res.end()
return true
}
res.setHeader('ETag', etag)
return false
}
/**
* Send `any` body to response
* @param req request object
......@@ -261,8 +244,8 @@ export function sendData(
const isJSONLike = ['object', 'number', 'boolean'].includes(typeof body)
const stringifiedBody = isJSONLike ? JSON.stringify(body) : body
if (sendEtagResponse(req, res, stringifiedBody)) {
const etag = generateETag(stringifiedBody)
if (sendEtagResponse(req, res, etag)) {
return
}
......
......@@ -10,6 +10,7 @@ import { fileExists } from '../../lib/file-exists'
// @ts-ignore no types for is-animated
import isAnimated from 'next/dist/compiled/is-animated'
import Stream from 'stream'
import { sendEtagResponse } from './send-payload'
let sharp: typeof import('sharp')
//const AVIF = 'image/avif'
......@@ -18,7 +19,7 @@ const PNG = 'image/png'
const JPEG = 'image/jpeg'
const GIF = 'image/gif'
const SVG = 'image/svg+xml'
const CACHE_VERSION = 1
const CACHE_VERSION = 2
const MODERN_TYPES = [/* AVIF, */ WEBP]
const ANIMATABLE_TYPES = [WEBP, PNG, GIF]
const VECTOR_TYPES = [SVG]
......@@ -143,17 +144,22 @@ export async function imageOptimizer(
if (await fileExists(hashDir, 'directory')) {
const files = await promises.readdir(hashDir)
for (let file of files) {
const [filename, extension] = file.split('.')
const expireAt = Number(filename)
const [prefix, etag, extension] = file.split('.')
const expireAt = Number(prefix)
const contentType = getContentType(extension)
const fsPath = join(hashDir, file)
if (now < expireAt) {
res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate')
if (sendEtagResponse(req, res, etag)) {
return { finished: true }
}
if (contentType) {
res.setHeader('Content-Type', contentType)
}
createReadStream(join(hashDir, file)).pipe(res)
createReadStream(fsPath).pipe(res)
return { finished: true }
} else {
await promises.unlink(join(hashDir, file))
await promises.unlink(fsPath)
}
}
}
......@@ -223,8 +229,7 @@ export async function imageOptimizer(
const animate =
ANIMATABLE_TYPES.includes(upstreamType) && isAnimated(upstreamBuffer)
if (vector || animate) {
res.setHeader('Content-Type', upstreamType)
res.end(upstreamBuffer)
sendResponse(req, res, upstreamType, upstreamBuffer)
return { finished: true }
}
}
......@@ -248,10 +253,8 @@ export async function imageOptimizer(
if (error.code === 'MODULE_NOT_FOUND') {
error.message += '\n\nLearn more: https://err.sh/next.js/install-sharp'
server.logError(error)
if (upstreamType) {
res.setHeader('Content-Type', upstreamType)
}
res.end(upstreamBuffer)
sendResponse(req, res, upstreamType, upstreamBuffer)
return { finished: true }
}
throw error
}
......@@ -281,30 +284,46 @@ export async function imageOptimizer(
const optimizedBuffer = await transformer.toBuffer()
await promises.mkdir(hashDir, { recursive: true })
const extension = getExtension(contentType)
const filename = join(hashDir, `${expireAt}.${extension}`)
const etag = getHash([optimizedBuffer])
const filename = join(hashDir, `${expireAt}.${etag}.${extension}`)
await promises.writeFile(filename, optimizedBuffer)
res.setHeader('Content-Type', contentType)
res.end(optimizedBuffer)
sendResponse(req, res, contentType, optimizedBuffer)
} catch (error) {
server.logError(error)
if (upstreamType) {
res.setHeader('Content-Type', upstreamType)
}
res.end(upstreamBuffer)
sendResponse(req, res, upstreamType, upstreamBuffer)
}
return { finished: true }
}
function sendResponse(
req: IncomingMessage,
res: ServerResponse,
contentType: string | null,
buffer: Buffer
) {
const etag = getHash([buffer])
res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate')
if (sendEtagResponse(req, res, etag)) {
return
}
if (contentType) {
res.setHeader('Content-Type', contentType)
}
res.end(buffer)
}
function getSupportedMimeType(options: string[], accept = ''): string {
const mimeType = mediaType(accept, options)
return accept.includes(mimeType) ? mimeType : ''
}
function getHash(items: (string | number | undefined)[]) {
function getHash(items: (string | number | Buffer)[]) {
const hash = createHash('sha256')
for (let item of items) {
hash.update(String(item))
if (typeof item === 'number') hash.update(String(item))
else {
hash.update(item)
}
}
// See https://en.wikipedia.org/wiki/Base64#Filenames
return hash.digest('base64').replace(/\//g, '-')
......
......@@ -26,17 +26,10 @@ export function sendPayload(
}
const etag = generateEtags ? generateETag(payload) : undefined
if (fresh(req.headers, { etag })) {
res.statusCode = 304
res.end()
if (sendEtagResponse(req, res, etag)) {
return
}
if (etag) {
res.setHeader('ETag', etag)
}
if (!res.getHeader('Content-Type')) {
res.setHeader(
'Content-Type',
......@@ -72,3 +65,27 @@ export function sendPayload(
}
res.end(req.method === 'HEAD' ? null : payload)
}
export function sendEtagResponse(
req: IncomingMessage,
res: ServerResponse,
etag: string | undefined
): boolean {
if (etag) {
/**
* The server generating a 304 response MUST generate any of the
* following header fields that would have been sent in a 200 (OK)
* response to the same request: Cache-Control, Content-Location, Date,
* ETag, Expires, and Vary. https://tools.ietf.org/html/rfc7232#section-4.1
*/
res.setHeader('ETag', etag)
}
if (fresh(req.headers, { etag })) {
res.statusCode = 304
res.end()
return true
}
return false
}
......@@ -55,6 +55,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, {})
expect(res.status).toBe(200)
expect(res.headers.get('content-type')).toContain('image/gif')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
expect(isAnimated(await res.buffer())).toBe(true)
})
......@@ -63,6 +67,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, {})
expect(res.status).toBe(200)
expect(res.headers.get('content-type')).toContain('image/png')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
expect(isAnimated(await res.buffer())).toBe(true)
})
......@@ -71,6 +79,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, {})
expect(res.status).toBe(200)
expect(res.headers.get('content-type')).toContain('image/webp')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
expect(isAnimated(await res.buffer())).toBe(true)
})
......@@ -80,6 +92,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toContain('image/svg+xml')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
const actual = await res.text()
const expected = await fs.readFile(
join(__dirname, '..', 'public', 'test.svg'),
......@@ -96,6 +112,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toContain('image/jpeg')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
})
it('should maintain png format for old Safari', async () => {
......@@ -106,6 +126,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toContain('image/png')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
})
it('should fail when url is missing', async () => {
......@@ -199,6 +223,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/webp')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
......@@ -208,6 +236,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/png')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
......@@ -217,6 +249,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/png')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
......@@ -226,6 +262,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/gif')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
......@@ -235,6 +275,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/tiff')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
......@@ -246,6 +290,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/webp')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
......@@ -257,6 +305,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/webp')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, w)
})
}
......@@ -310,6 +362,39 @@ function runTests({ w, isDev, domains }) {
expect(json2).toStrictEqual(json1)
})
it('should set 304 status without body when etag matches if-none-match', async () => {
const query = { url: '/test.jpg', w, q: 80 }
const opts1 = { headers: { accept: 'image/webp' } }
const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts1)
expect(res1.status).toBe(200)
expect(res1.headers.get('Content-Type')).toBe('image/webp')
expect(res1.headers.get('Cache-Control')).toBe(
'public, max-age=0, must-revalidate'
)
const etag = res1.headers.get('Etag')
expect(etag).toBeTruthy()
await expectWidth(res1, w)
const opts2 = { headers: { accept: 'image/webp', 'if-none-match': etag } }
const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts2)
expect(res2.status).toBe(304)
expect(res2.headers.get('Content-Type')).toBeFalsy()
expect(res2.headers.get('Etag')).toBe(etag)
expect((await res2.buffer()).length).toBe(0)
const query3 = { url: '/test.jpg', w, q: 25 }
const res3 = await fetchViaHTTP(appPort, '/_next/image', query3, opts2)
expect(res3.status).toBe(200)
expect(res3.headers.get('Content-Type')).toBe('image/webp')
expect(res3.headers.get('Cache-Control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res3.headers.get('Etag')).toBeTruthy()
expect(res3.headers.get('Etag')).not.toBe(etag)
await expectWidth(res3, w)
})
it('should proxy-pass unsupported image types and should not cache file', async () => {
const json1 = await fsToJson(imagesDir)
expect(json1).toBeTruthy()
......@@ -319,6 +404,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/bmp')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
const json2 = await fsToJson(imagesDir)
expect(json2).toStrictEqual(json1)
......@@ -330,6 +419,10 @@ function runTests({ w, isDev, domains }) {
const res = await fetchViaHTTP(appPort, '/_next/image', query, opts)
expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/webp')
expect(res.headers.get('cache-control')).toBe(
'public, max-age=0, must-revalidate'
)
expect(res.headers.get('etag')).toBeTruthy()
await expectWidth(res, 400)
})
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment