Unverified Commit d64c0d1e authored by Sebastian "Sebbie" Silbermann's avatar Sebastian "Sebbie" Silbermann Committed by GitHub

[15.5.x] Re-enable AVIF image optimization with a runtime libheif version guard (#97954)



Since 15.x supports a Node.js version not supported by a secure sharp
version, we have to conditionally block AVIF optimization based on the
used libheif version. I added a test that runs for each support major
sharp version to validate only the fixed version optimizes AVIF.
parent 7f6b4007
...@@ -130,7 +130,7 @@ ...@@ -130,7 +130,7 @@
} }
}, },
"optionalDependencies": { "optionalDependencies": {
"sharp": "^0.34.3 || ^0.35.3" "sharp": "^0.34.3 || ^0.35.4"
}, },
"devDependencies": { "devDependencies": {
"@ampproject/toolbox-optimizer": "2.8.3", "@ampproject/toolbox-optimizer": "2.8.3",
......
import isAnimated from 'next/dist/compiled/is-animated' import isAnimated from 'next/dist/compiled/is-animated'
import { optimizeImage } from '../../../../server/image-optimizer' import {
canDecodeAvif,
optimizeImage,
} from '../../../../server/image-optimizer'
const BLUR_IMG_SIZE = 8 const BLUR_IMG_SIZE = 8
const BLUR_QUALITY = 70 const BLUR_QUALITY = 70
...@@ -37,7 +40,13 @@ export async function getBlurImage( ...@@ -37,7 +40,13 @@ export async function getBlurImage(
let blurWidth: number = 0 let blurWidth: number = 0
let blurHeight: number = 0 let blurHeight: number = 0
if (VALID_BLUR_EXT.includes(extension) && !isAnimated(content)) { // AVIF blur placeholders decode the source image, so they are only
// generated when the installed sharp bundles a patched libheif.
const isValidBlurExtension =
VALID_BLUR_EXT.includes(extension) ||
(extension === 'avif' && canDecodeAvif(null))
if (isValidBlurExtension && !isAnimated(content)) {
// Shrink the image's largest dimension // Shrink the image's largest dimension
if (imageSize.width >= imageSize.height) { if (imageSize.width >= imageSize.height) {
blurWidth = BLUR_IMG_SIZE blurWidth = BLUR_IMG_SIZE
......
...@@ -50,7 +50,7 @@ const BMP = 'image/bmp' ...@@ -50,7 +50,7 @@ const BMP = 'image/bmp'
const PDF = 'application/pdf' const PDF = 'application/pdf'
const CACHE_VERSION = 4 const CACHE_VERSION = 4
const ANIMATABLE_TYPES = [WEBP, PNG, GIF] const ANIMATABLE_TYPES = [WEBP, PNG, GIF]
const BYPASS_TYPES = [SVG, ICO, ICNS, BMP, JXL, HEIC, AVIF] const BYPASS_TYPES = [SVG, ICO, ICNS, BMP, JXL, HEIC]
const BLUR_IMG_SIZE = 8 // should match `next-image-loader` const BLUR_IMG_SIZE = 8 // should match `next-image-loader`
const BLUR_QUALITY = 70 // should match `next-image-loader` const BLUR_QUALITY = 70 // should match `next-image-loader`
...@@ -103,6 +103,9 @@ export function getSharp(concurrency: number | null | undefined) { ...@@ -103,6 +103,9 @@ export function getSharp(concurrency: number | null | undefined) {
'VipsForeignLoadWebp', 'VipsForeignLoadWebp',
], ],
}) })
if (isAvifDecodeSafe(_sharp.versions?.heif ?? null)) {
_sharp.unblock({ operation: ['VipsForeignLoadHeif'] })
}
if (_sharp && _sharp.concurrency() > 1) { if (_sharp && _sharp.concurrency() > 1) {
// Reducing concurrency should reduce the memory usage too. // Reducing concurrency should reduce the memory usage too.
// We more aggressively reduce in dev but also reduce in prod. // We more aggressively reduce in dev but also reduce in prod.
...@@ -123,6 +126,47 @@ export function getSharp(concurrency: number | null | undefined) { ...@@ -123,6 +126,47 @@ export function getSharp(concurrency: number | null | undefined) {
return _sharp return _sharp
} }
// libheif before 1.23.2 has critical memory safety vulnerabilities in AVIF
// decoding (GHSA-g89c-p67h-r497 and GHSA-2jg2-4ch7-h545). sharp only bundles
// a patched libheif starting with 0.35.4, and Node 18 installs always resolve
// the vulnerable 0.34 line, so the decoder version is checked at runtime
// instead of relying on the sharp version.
// Exported for test integration only.
export function isAvifDecodeSafe(heifVersion: string | null): boolean {
if (heifVersion === null) {
return false
}
const [major = Number.NaN, minor = Number.NaN, patch = Number.NaN] =
heifVersion.split('.').map((part) => Number.parseInt(part, 10))
if (Number.isNaN(major) || Number.isNaN(minor) || Number.isNaN(patch)) {
return false
}
return (
major > 1 ||
(major === 1 && minor > 23) ||
(major === 1 && minor === 23 && patch >= 2)
)
}
let _avifDecodeSafe: boolean | null = null
export function canDecodeAvif(concurrency: number | null | undefined): boolean {
if (_avifDecodeSafe === null) {
let heifVersion: string | null = null
try {
// Custom libvips builds do not report dependency versions, so the
// absence of a heif version is treated as unsafe.
heifVersion = getSharp(concurrency).versions?.heif ?? null
} catch {
// Without sharp no AVIF can be decoded either, so treat it as unsafe
// and let the image pass through unoptimized.
heifVersion = null
}
_avifDecodeSafe = isAvifDecodeSafe(heifVersion)
}
return _avifDecodeSafe
}
export interface ImageParamsResult { export interface ImageParamsResult {
href: string href: string
isAbsolute: boolean isAbsolute: boolean
...@@ -993,7 +1037,11 @@ export async function imageOptimizer( ...@@ -993,7 +1037,11 @@ export async function imageOptimizer(
upstreamEtag, upstreamEtag,
} }
} }
if (BYPASS_TYPES.includes(upstreamType)) { if (
BYPASS_TYPES.includes(upstreamType) ||
(upstreamType === AVIF &&
!canDecodeAvif(nextConfig.experimental.imgOptConcurrency))
) {
return { return {
buffer: upstreamBuffer, buffer: upstreamBuffer,
contentType: upstreamType, contentType: upstreamType,
......
This diff is collapsed.
export default function Page() {
return <p>hello world</p>
}
import fs from 'fs-extra' import fs from 'fs-extra'
import { join } from 'path' import { join } from 'path'
import { optimizeImage } from 'next/dist/server/image-optimizer' import { nextTestSetup } from 'e2e-utils'
describe('with latest sharp', () => { const appDir = join(__dirname, 'app')
it('should block avif decoding below the optimizer bypass', async () => {
const buffer = await fs.readFile(join(__dirname, 'app/public/test.avif'))
await expect( describe.each([
optimizeImage({ // sharp 0.34.5 bundles libheif 1.20.2, which is affected by the AVIF
buffer, // decoding vulnerabilities fixed in libheif 1.23.2.
contentType: 'image/webp', {
quality: 75, sharp: '0.34.5',
width: 100, decodesAvif: false,
}) },
).rejects.toThrow() // sharp 0.35.4 is the first release bundling the patched libheif 1.23.2.
{
sharp: '0.35.4',
decodesAvif: true,
},
])('with sharp $sharp', ({ sharp, decodesAvif }) => {
const patchedSharpDeps = {
sharp,
[`@img/sharp-${process.platform}-${process.arch}`]: sharp,
}
const { isNextDeploy, next } = nextTestSetup({
files: appDir,
dependencies: patchedSharpDeps,
packageJson: {
// npm uses overrides
overrides: patchedSharpDeps,
pnpm: {
overrides: patchedSharpDeps,
},
},
})
it('only optimizes avif input when the bundled libheif is patched', async () => {
const query = new URLSearchParams({
url: '/test.avif',
w: '256',
q: '75',
})
const res = await next.fetch(`/_next/image?${query}`, {
headers: { accept: 'image/webp' },
})
expect(res.status).toBe(200)
if (
decodesAvif ||
// Vercel's Image Optimization pipeline uses a patched libheif i.e. doesn't use Next.js' pipeline.
isNextDeploy
) {
expect(res.headers.get('content-type')).toBe('image/webp')
} else {
expect(res.headers.get('content-type')).toBe('image/avif')
const source = await fs.readFile(join(appDir, 'public/test.avif'))
expect(Buffer.from(await res.arrayBuffer()).equals(source)).toBe(true)
}
})
it('still optimizes other formats', async () => {
const query = new URLSearchParams({
url: '/test.png',
w: '256',
q: '75',
})
const res = await next.fetch(`/_next/image?${query}`, {
headers: { accept: 'image/webp' },
})
expect(res.status).toBe(200)
expect(res.headers.get('content-type')).toBe('image/webp')
}) })
}) })
...@@ -16,6 +16,7 @@ import { ...@@ -16,6 +16,7 @@ import {
waitFor, waitFor,
} from 'next-test-utils' } from 'next-test-utils'
import isAnimated from 'next/dist/compiled/is-animated' import isAnimated from 'next/dist/compiled/is-animated'
import { getSharp, isAvifDecodeSafe } from 'next/dist/server/image-optimizer'
import type { RequestInit } from 'node-fetch' import type { RequestInit } from 'node-fetch'
import type { NextConfig } from 'next' import type { NextConfig } from 'next'
...@@ -109,6 +110,17 @@ async function expectUnoptimizedImage(res, filename: string) { ...@@ -109,6 +110,17 @@ async function expectUnoptimizedImage(res, filename: string) {
) )
} }
// The image optimizer only decodes AVIF input when the installed sharp
// bundles a patched libheif. The same check decides the expectations here so
// CI jobs pinned to the vulnerable 0.34 line (Node 18) assert the bypass.
function isAvifInputOptimizable(): boolean {
try {
return isAvifDecodeSafe(getSharp(null).versions?.heif ?? null)
} catch {
return false
}
}
export const cleanImagesDir = async (ctx: { imagesDir: string }) => { export const cleanImagesDir = async (ctx: { imagesDir: string }) => {
console.warn('Cleaning', ctx.imagesDir) console.warn('Cleaning', ctx.imagesDir)
await fs.remove(ctx.imagesDir) await fs.remove(ctx.imagesDir)
...@@ -532,23 +544,31 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -532,23 +544,31 @@ export function runTests(ctx: RunTestsCtx) {
await expectWidth(res, ctx.w) await expectWidth(res, ctx.w)
}) })
it('should bypass avif input for old Safari', async () => { it('should downlevel avif format to jpeg for old Safari', async () => {
const accept = const accept =
'image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5' 'image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5'
const query = { w: ctx.w, q: 74, url: '/test.avif' } const query = { w: ctx.w, q: 74, url: '/test.avif' }
const opts = { headers: { accept } } const opts = { headers: { accept } }
const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts)
expect(res.status).toBe(200) expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toContain('image/avif')
expect(res.headers.get('Cache-Control')).toBe( expect(res.headers.get('Cache-Control')).toBe(
`public, max-age=${isDev ? 0 : minimumCacheTTL}, must-revalidate` `public, max-age=${isDev ? 0 : minimumCacheTTL}, must-revalidate`
) )
expect(res.headers.get('Vary')).toBe('Accept') expect(res.headers.get('Vary')).toBe('Accept')
expect(res.headers.get('etag')).toBeTruthy() expect(res.headers.get('etag')).toBeTruthy()
expect(res.headers.get('Content-Disposition')).toBe( if (isAvifInputOptimizable()) {
`${contentDispositionType}; filename="test.avif"` expect(res.headers.get('Content-Type')).toContain('image/jpeg')
) expect(res.headers.get('Content-Disposition')).toBe(
await expectUnoptimizedImage(res, 'test.avif') `${contentDispositionType}; filename="test.jpeg"`
)
await expectWidth(res, ctx.w)
} else {
expect(res.headers.get('Content-Type')).toContain('image/avif')
expect(res.headers.get('Content-Disposition')).toBe(
`${contentDispositionType}; filename="test.avif"`
)
await expectUnoptimizedImage(res, 'test.avif')
}
}) })
it('should fail when url is missing', async () => { it('should fail when url is missing', async () => {
...@@ -814,21 +834,29 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -814,21 +834,29 @@ export function runTests(ctx: RunTestsCtx) {
await expectWidth(res, ctx.w) await expectWidth(res, ctx.w)
}) })
it('should bypass optimization for avif input', async () => { it('should resize avif', async () => {
const query = { url: '/test.avif', w: ctx.w, q: ctx.q } const query = { url: '/test.avif', w: ctx.w, q: ctx.q }
const opts = { headers: { accept: 'image/webp' } } const opts = { headers: { accept: 'image/webp' } }
const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts)
expect(res.status).toBe(200) expect(res.status).toBe(200)
expect(res.headers.get('Content-Type')).toBe('image/avif')
expect(res.headers.get('Cache-Control')).toBe( expect(res.headers.get('Cache-Control')).toBe(
`public, max-age=${isDev ? 0 : minimumCacheTTL}, must-revalidate` `public, max-age=${isDev ? 0 : minimumCacheTTL}, must-revalidate`
) )
expect(res.headers.get('Vary')).toBe('Accept') expect(res.headers.get('Vary')).toBe('Accept')
expect(res.headers.get('etag')).toBeTruthy() expect(res.headers.get('etag')).toBeTruthy()
expect(res.headers.get('Content-Disposition')).toBe( if (isAvifInputOptimizable()) {
`${contentDispositionType}; filename="test.avif"` expect(res.headers.get('Content-Type')).toBe('image/webp')
) expect(res.headers.get('Content-Disposition')).toBe(
await expectUnoptimizedImage(res, 'test.avif') `${contentDispositionType}; filename="test.webp"`
)
await expectWidth(res, ctx.w)
} else {
expect(res.headers.get('Content-Type')).toBe('image/avif')
expect(res.headers.get('Content-Disposition')).toBe(
`${contentDispositionType}; filename="test.avif"`
)
await expectUnoptimizedImage(res, 'test.avif')
}
}) })
it('should resize relative url and old Chrome accept header as webp', async () => { it('should resize relative url and old Chrome accept header as webp', async () => {
...@@ -872,7 +900,7 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -872,7 +900,7 @@ export function runTests(ctx: RunTestsCtx) {
await expectWidth(res, ctx.w) await expectWidth(res, ctx.w)
}) })
it('should bypass avif input when avif output is enabled', async () => { it('should resize avif and maintain format', async () => {
const query = { url: '/test.avif', w: ctx.w, q: ctx.q } const query = { url: '/test.avif', w: ctx.w, q: ctx.q }
const opts = { headers: { accept: 'image/avif' } } const opts = { headers: { accept: 'image/avif' } }
const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts) const res = await fetchViaHTTP(ctx.appPort, '/_next/image', query, opts)
...@@ -886,7 +914,11 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -886,7 +914,11 @@ export function runTests(ctx: RunTestsCtx) {
expect(res.headers.get('Content-Disposition')).toBe( expect(res.headers.get('Content-Disposition')).toBe(
`${contentDispositionType}; filename="test.avif"` `${contentDispositionType}; filename="test.avif"`
) )
await expectUnoptimizedImage(res, 'test.avif') if (isAvifInputOptimizable()) {
await expectWidth(res, ctx.w)
} else {
await expectUnoptimizedImage(res, 'test.avif')
}
}) })
it('should compress avif smaller than webp at q=100', async () => { it('should compress avif smaller than webp at q=100', async () => {
......
...@@ -35,7 +35,7 @@ describe('getBlurImage', () => { ...@@ -35,7 +35,7 @@ describe('getBlurImage', () => {
expect(result).toBeObject() expect(result).toBeObject()
expect(result.dataURL).toBeUndefined() expect(result.dataURL).toBeUndefined()
}) })
it('should not generate a blur placeholder for avif', async () => { it('should generate a blur placeholder for avif only when the decoder is patched', async () => {
const buffer = await getImage('../image-optimizer/images/test.avif') const buffer = await getImage('../image-optimizer/images/test.avif')
const result = await getBlurImage( const result = await getBlurImage(
buffer, buffer,
...@@ -47,10 +47,19 @@ describe('getBlurImage', () => { ...@@ -47,10 +47,19 @@ describe('getBlurImage', () => {
} }
) )
expect(result).toBeObject() expect(result).toBeObject()
expect(result).toStrictEqual({ // CI runs this on Node 18 with sharp 0.34.3 (vulnerable libheif, so AVIF
dataURL: undefined, // decoding is blocked) and on Node 20 with the lockfile's patched
width: 0, // libheif.
height: 0, if (process.version.startsWith('v18')) {
}) expect(result).toStrictEqual({
dataURL: undefined,
width: 0,
height: 0,
})
} else {
expect(result.dataURL).toBeString()
expect(result.width).toBe(8)
expect(result.height).toBe(8)
}
}) })
}) })
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment