Unverified Commit adb56ef2 authored by Jiachi Liu's avatar Jiachi Liu Committed by GitHub

Enable html post optimization for react 18 (#36837)

Follow up for #35888 to re-enable more test, and re-enable post processors after #36792 has better support for document.gIP with react 18. Apply post-pocessing when the the shell chunk is fully buffered.

re-enabled integration tests for react 18:
- amphtml
- amphtml-custom-optimizer
- app-document
- font-optimization

Fixes #35835


## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have helpful link attached, see `contributing.md`
parent b717b6eb
......@@ -86,18 +86,18 @@ export function decodeText(input?: Uint8Array, textDecoder?: TextDecoder) {
: new TextDecoder().decode(input)
}
export function createBufferedTransformStream(): TransformStream<
Uint8Array,
Uint8Array
> {
export function createBufferedTransformStream(
transform: (v: string) => string | Promise<string> = (v) => v
): TransformStream<Uint8Array, Uint8Array> {
let bufferedString = ''
let pendingFlush: Promise<void> | null = null
const flushBuffer = (controller: TransformStreamDefaultController) => {
if (!pendingFlush) {
pendingFlush = new Promise((resolve) => {
setTimeout(() => {
controller.enqueue(encodeText(bufferedString))
setTimeout(async () => {
const buffered = await transform(bufferedString)
controller.enqueue(encodeText(buffered))
bufferedString = ''
pendingFlush = null
resolve()
......
import type { RenderOpts } from './render'
import { parse, HTMLElement } from 'next/dist/compiled/node-html-parser'
import { OPTIMIZED_FONT_PROVIDERS } from './constants'
// const MIDDLEWARE_TIME_BUDGET = parseInt(process.env.__POST_PROCESS_MIDDLEWARE_TIME_BUDGET || '', 10) || 10
import { OPTIMIZED_FONT_PROVIDERS } from '../shared/lib/constants'
import { nonNullable } from '../lib/non-nullable'
let optimizeAmp: typeof import('./optimize-amp').default | undefined
let getFontDefinitionFromManifest:
| typeof import('./font-utils').getFontDefinitionFromManifest
| undefined
if (process.env.NEXT_RUNTIME !== 'edge') {
optimizeAmp = require('./optimize-amp').default
getFontDefinitionFromManifest =
require('./font-utils').getFontDefinitionFromManifest
}
type postProcessOptions = {
optimizeFonts: boolean
......@@ -165,6 +176,73 @@ class FontOptimizerMiddleware implements PostProcessMiddleware {
}
}
async function postProcessHTML(
pathname: string,
content: string,
renderOpts: RenderOpts,
{ inAmpMode, hybridAmp }: { inAmpMode: boolean; hybridAmp: boolean }
) {
const postProcessors: Array<(html: string) => Promise<string>> = [
process.env.NEXT_RUNTIME !== 'edge' && inAmpMode
? async (html: string) => {
html = await optimizeAmp!(html, renderOpts.ampOptimizerConfig)
if (!renderOpts.ampSkipValidation && renderOpts.ampValidator) {
await renderOpts.ampValidator(html, pathname)
}
return html
}
: null,
process.env.NEXT_RUNTIME !== 'edge' && process.env.__NEXT_OPTIMIZE_FONTS
? async (html: string) => {
const getFontDefinition = (url: string): string => {
if (renderOpts.fontManifest) {
return getFontDefinitionFromManifest!(
url,
renderOpts.fontManifest
)
}
return ''
}
return await processHTML(
html,
{ getFontDefinition },
{
optimizeFonts: renderOpts.optimizeFonts,
}
)
}
: null,
process.env.NEXT_RUNTIME !== 'edge' && renderOpts.optimizeCss
? async (html: string) => {
// eslint-disable-next-line import/no-extraneous-dependencies
const Critters = require('critters')
const cssOptimizer = new Critters({
ssrMode: true,
reduceInlineStyles: false,
path: renderOpts.distDir,
publicPath: `${renderOpts.assetPrefix}/_next/`,
preload: 'media',
fonts: false,
...renderOpts.optimizeCss,
})
return await cssOptimizer.process(html)
}
: null,
inAmpMode || hybridAmp
? async (html: string) => {
return html.replace(/&amp;amp=1/g, '&amp=1')
}
: null,
].filter(nonNullable)
for (const postProcessor of postProcessors) {
if (postProcessor) {
content = await postProcessor(content)
}
}
return content
}
// Initialization
registerPostProcessor(
'Inline-Fonts',
......@@ -174,4 +252,4 @@ registerPostProcessor(
(options) => options.optimizeFonts || process.env.__NEXT_OPTIMIZE_FONTS
)
export default processHTML
export { postProcessHTML }
......@@ -83,12 +83,10 @@ import { FlushEffectsContext } from '../shared/lib/flush-effects'
import { interopDefault } from '../lib/interop-default'
import stripAnsi from 'next/dist/compiled/strip-ansi'
import { urlQueryToSearchParams } from '../shared/lib/router/utils/querystring'
import { postProcessHTML } from './post-process'
let optimizeAmp: typeof import('./optimize-amp').default
let getFontDefinitionFromManifest: typeof import('./font-utils').getFontDefinitionFromManifest
let tryGetPreviewData: typeof import('./api-utils/node').tryGetPreviewData
let warn: typeof import('../build/output/log').warn
let postProcess: typeof import('../shared/lib/post-process').default
const DOCTYPE = '<!DOCTYPE html>'
const ReactDOMServer = process.env.__NEXT_REACT_ROOT
......@@ -97,12 +95,8 @@ const ReactDOMServer = process.env.__NEXT_REACT_ROOT
if (process.env.NEXT_RUNTIME !== 'edge') {
require('./node-polyfill-web-streams')
optimizeAmp = require('./optimize-amp').default
getFontDefinitionFromManifest =
require('./font-utils').getFontDefinitionFromManifest
tryGetPreviewData = require('./api-utils/node').tryGetPreviewData
warn = require('../build/output/log').warn
postProcess = require('../shared/lib/post-process').default
} else {
warn = console.warn.bind(console)
}
......@@ -471,7 +465,6 @@ export async function renderToHTML(
ampPath = '',
pageConfig = {},
buildManifest,
fontManifest,
reactLoadableManifest,
ErrorDebug,
getStaticProps,
......@@ -535,13 +528,6 @@ export async function renderToHTML(
})
}
const getFontDefinition = (url: string): string => {
if (fontManifest) {
return getFontDefinitionFromManifest(url, fontManifest)
}
return ''
}
let renderServerComponentData = isServerComponent
? query.__flight__ !== undefined
: false
......@@ -1487,65 +1473,67 @@ export async function renderToHTML(
})
}
const createBodyResult =
(initialStream: ReactReadableStream) => (suffix: string) => {
// this must be called inside bodyResult so appWrappers is
// up to date when `wrapApp` is called
const flushEffectHandler = (): string => {
const allFlushEffects = [
styledJsxFlushEffect,
...(flushEffects || []),
]
const flushed = ReactDOMServer.renderToString(
<>
{allFlushEffects.map((flushEffect, i) => (
<React.Fragment key={i}>{flushEffect()}</React.Fragment>
))}
</>
)
return flushed
}
const createBodyResult = (
initialStream: ReactReadableStream,
suffix?: string
) => {
// this must be called inside bodyResult so appWrappers is
// up to date when `wrapApp` is called
const flushEffectHandler = (): string => {
const allFlushEffects = [
styledJsxFlushEffect,
...(flushEffects || []),
]
const flushed = ReactDOMServer.renderToString(
<>
{allFlushEffects.map((flushEffect, i) => (
<React.Fragment key={i}>{flushEffect()}</React.Fragment>
))}
</>
)
return flushed
}
// Handle static data for server components.
async function generateStaticFlightDataIfNeeded() {
if (serverComponentsPageDataTransformStream) {
// If it's a server component with the Node.js runtime, we also
// statically generate the page data.
let data = ''
const readable = serverComponentsPageDataTransformStream.readable
const reader = readable.getReader()
const textDecoder = new TextDecoder()
while (true) {
const { done, value } = await reader.read()
if (done) {
break
}
data += decodeText(value, textDecoder)
// Handle static data for server components.
async function generateStaticFlightDataIfNeeded() {
if (serverComponentsPageDataTransformStream) {
// If it's a server component with the Node.js runtime, we also
// statically generate the page data.
let data = ''
const readable = serverComponentsPageDataTransformStream.readable
const reader = readable.getReader()
const textDecoder = new TextDecoder()
while (true) {
const { done, value } = await reader.read()
if (done) {
break
}
data += decodeText(value, textDecoder)
}
;(renderOpts as any).pageData = {
...(renderOpts as any).pageData,
__flight__: data,
}
return data
;(renderOpts as any).pageData = {
...(renderOpts as any).pageData,
__flight__: data,
}
return data
}
// @TODO: A potential improvement would be to reuse the inlined
// data stream, or pass a callback inside as this doesn't need to
// be streamed.
// Do not use `await` here.
generateStaticFlightDataIfNeeded()
return continueFromInitialStream(initialStream, {
suffix,
dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML,
flushEffectHandler,
})
}
// @TODO: A potential improvement would be to reuse the inlined
// data stream, or pass a callback inside as this doesn't need to
// be streamed.
// Do not use `await` here.
generateStaticFlightDataIfNeeded()
return continueFromInitialStream(initialStream, {
suffix,
dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML,
flushEffectHandler,
})
}
const hasDocumentGetInitialProps = !(
isServerComponent ||
process.env.NEXT_RUNTIME === 'edge' ||
......@@ -1563,10 +1551,12 @@ export async function renderToHTML(
documentInitialPropsRes = await loadDocumentInitialProps(renderShell)
if (documentInitialPropsRes === null) return null
const { docProps } = documentInitialPropsRes as any
bodyResult = createBodyResult(streamFromArray([docProps.html]))
// includes suffix in initial html stream
bodyResult = (suffix: string) =>
createBodyResult(streamFromArray([docProps.html, suffix]))
} else {
const stream = await renderShell(App, Component)
bodyResult = createBodyResult(stream)
bodyResult = (suffix: string) => createBodyResult(stream, suffix)
documentInitialPropsRes = {}
}
......@@ -1738,7 +1728,7 @@ export async function renderToHTML(
prefix.push('<!-- __NEXT_DATA__ -->')
}
let streams = [
const streams = [
streamFromArray(prefix),
await documentResult.bodyResult(renderTargetSuffix),
]
......@@ -1751,67 +1741,19 @@ export async function renderToHTML(
return RenderResult.fromStatic((renderOpts as any).pageData)
}
const postProcessors: Array<((html: string) => Promise<string>) | null> = (
generateStaticHTML
? [
inAmpMode
? async (html: string) => {
html = await optimizeAmp(html, renderOpts.ampOptimizerConfig)
if (!renderOpts.ampSkipValidation && renderOpts.ampValidator) {
await renderOpts.ampValidator(html, pathname)
}
return html
}
: null,
process.env.NEXT_RUNTIME !== 'edge' &&
process.env.__NEXT_OPTIMIZE_FONTS
? async (html: string) => {
return await postProcess(
html,
{ getFontDefinition },
{
optimizeFonts: renderOpts.optimizeFonts,
}
)
}
: null,
process.env.NEXT_RUNTIME !== 'edge' && renderOpts.optimizeCss
? async (html: string) => {
// eslint-disable-next-line import/no-extraneous-dependencies
const Critters = require('critters')
const cssOptimizer = new Critters({
ssrMode: true,
reduceInlineStyles: false,
path: renderOpts.distDir,
publicPath: `${renderOpts.assetPrefix}/_next/`,
preload: 'media',
fonts: false,
...renderOpts.optimizeCss,
})
return await cssOptimizer.process(html)
}
: null,
inAmpMode || hybridAmp
? async (html: string) => {
return html.replace(/&amp;amp=1/g, '&amp=1')
}
: null,
]
: []
).filter(Boolean)
if (generateStaticHTML || postProcessors.length > 0) {
let html = await streamToString(chainStreams(streams))
for (const postProcessor of postProcessors) {
if (postProcessor) {
html = await postProcessor(html)
}
}
return new RenderResult(html)
const postOptimize = (html: string) =>
postProcessHTML(pathname, html, renderOpts, { inAmpMode, hybridAmp })
if (generateStaticHTML) {
const html = await streamToString(chainStreams(streams))
const optimizedHtml = await postOptimize(html)
return new RenderResult(optimizedHtml)
}
return new RenderResult(
chainStreams(streams).pipeThrough(createBufferedTransformStream())
chainStreams(streams).pipeThrough(
createBufferedTransformStream(postOptimize)
)
)
}
......
const path = require('path')
module.exports = require(path.join(__dirname, '../../lib/with-react-17.js'))({
module.exports = {
experimental: {
amp: {
optimizer: {
......@@ -10,4 +9,4 @@ module.exports = require(path.join(__dirname, '../../lib/with-react-17.js'))({
skipValidation: true,
},
},
})
}
......@@ -12,19 +12,14 @@ import {
let app
let appPort
const appDir = join(__dirname, '../')
const nodeArgs = ['-r', join(appDir, '../../lib/react-17-require-hook.js')]
describe('AMP Custom Optimizer', () => {
it('should build and start for static page', async () => {
const { code } = await nextBuild(appDir, undefined, {
nodeArgs,
})
const { code } = await nextBuild(appDir)
expect(code).toBe(0)
appPort = await findPort()
app = await nextStart(appDir, appPort, {
nodeArgs,
})
app = await nextStart(appDir, appPort)
const html = await renderViaHTTP(appPort, '/')
await killApp(app)
......@@ -39,11 +34,11 @@ describe('AMP Custom Optimizer', () => {
})
it('should build and start for dynamic page', async () => {
const { code } = await nextBuild(appDir, undefined, { nodeArgs })
const { code } = await nextBuild(appDir)
expect(code).toBe(0)
appPort = await findPort()
app = await nextStart(appDir, appPort, { nodeArgs })
app = await nextStart(appDir, appPort)
const html = await renderViaHTTP(appPort, '/dynamic')
await killApp(app)
......
const path = require('path')
module.exports = require(path.join(__dirname, '../../lib/with-react-17.js'))({
module.exports = {
onDemandEntries: {
// Make sure entries are not getting disposed.
maxInactiveAge: 1000 * 60 * 60,
......@@ -8,4 +7,4 @@ module.exports = require(path.join(__dirname, '../../lib/with-react-17.js'))({
canonicalBase: 'http://localhost:1234',
},
// edit here
})
}
......@@ -18,7 +18,6 @@ import webdriver from 'next-webdriver'
import { join } from 'path'
const appDir = join(__dirname, '../')
const nodeArgs = ['-r', join(appDir, '../../lib/react-17-require-hook.js')]
let appPort
let app
......@@ -36,14 +35,11 @@ describe('AMP Usage', () => {
const result = await nextBuild(appDir, undefined, {
stdout: true,
stderr: true,
nodeArgs,
})
output = result.stdout + result.stderr
appPort = context.appPort = await findPort()
app = await nextStart(appDir, context.appPort, {
nodeArgs,
})
app = await nextStart(appDir, context.appPort)
})
afterAll(async () => {
await rename(
......@@ -276,7 +272,6 @@ describe('AMP Usage', () => {
onStderr(msg) {
inspectPayload += msg
},
nodeArgs,
})
await renderViaHTTP(dynamicAppPort, '/only-amp')
......@@ -301,7 +296,6 @@ describe('AMP Usage', () => {
onStderr(msg) {
output += msg
},
nodeArgs,
})
})
......@@ -549,7 +543,6 @@ describe('AMP Usage', () => {
onStderr(msg) {
inspectPayload += msg
},
nodeArgs,
})
await renderViaHTTP(dynamicAppPort, '/invalid-amp')
......
const path = require('path')
module.exports = require(path.join(__dirname, '../../lib/with-react-17.js'))({
crossOrigin: 'anonymous',
})
module.exports = { crossOrigin: 'anonymous' }
......@@ -38,7 +38,7 @@ export default class MyDocument extends Document {
}
}
const result = ctx.renderPage(options)
const result = await ctx.renderPage(options)
return {
...result,
......
......@@ -28,10 +28,6 @@ describe('Document and App', () => {
context.server = await launchApp(join(__dirname, '../'), context.appPort, {
onStdout: collectOutput,
onStderr: collectOutput,
nodeArgs: [
'-r',
join(__dirname, '../../../lib/react-17-require-hook.js'),
],
})
// pre-build all pages at the start
......
module.exports = { experimental: { optimizeCss: true } }
......@@ -14,7 +14,6 @@ import fs from 'fs-extra'
const glob = promisify(globOrigig)
const appDir = join(__dirname, '../')
const nextConfig = join(appDir, 'next.config.js')
const nodeArgs = ['-r', join(appDir, '../../lib/react-17-require-hook.js')]
let appPort
let app
......@@ -68,13 +67,9 @@ describe('CSS optimization for SSR apps', () => {
if (fs.pathExistsSync(join(appDir, '.next'))) {
await fs.remove(join(appDir, '.next'))
}
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
appPort = await findPort()
app = await nextStart(appDir, appPort, {
nodeArgs,
})
app = await nextStart(appDir, appPort)
})
afterAll(async () => {
await killApp(app)
......@@ -90,13 +85,9 @@ describe('Font optimization for emulated serverless apps', () => {
`module.exports = { target: 'experimental-serverless-trace', experimental: {optimizeCss: true} }`,
'utf8'
)
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
appPort = await findPort()
app = await nextStart(appDir, appPort, {
nodeArgs,
})
app = await nextStart(appDir, appPort)
})
afterAll(async () => {
await killApp(app)
......
const path = require('path')
module.exports = require(path.join(
__dirname,
'../../../../lib/with-react-17.js'
))({
cleanDistDir: false,
})
module.exports = { cleanDistDir: false }
const path = require('path')
module.exports = require(path.join(
__dirname,
'../../../../lib/with-react-17.js'
))({
module.exports = {
cleanDistDir: false,
})
}
......@@ -15,10 +15,6 @@ import {
import webdriver from 'next-webdriver'
const fixturesDir = join(__dirname, '..', 'fixtures')
const nodeArgs = [
'-r',
join(__dirname, '../../../lib/react-17-require-hook.js'),
]
const fsExists = (file) =>
fs
......@@ -229,9 +225,7 @@ describe('Font Optimization', () => {
// Re-run build to check if it works when build is cached
it('should work when build is cached', async () => {
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
const testJson = JSON.parse(
await fs.readFile(builtPage('font-manifest.json'), {
encoding: 'utf-8',
......@@ -246,13 +240,9 @@ describe('Font Optimization', () => {
if (fs.pathExistsSync(join(appDir, '.next'))) {
await fs.remove(join(appDir, '.next'))
}
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
appPort = await findPort()
app = await nextStart(appDir, appPort, {
nodeArgs,
})
app = await nextStart(appDir, appPort)
builtServerPagesDir = join(appDir, '.next', 'server')
builtPage = (file) => join(builtServerPagesDir, file)
})
......@@ -266,19 +256,12 @@ describe('Font Optimization', () => {
beforeAll(async () => {
await fs.writeFile(
nextConfig,
`
const path = require('path')
module.exports = require(path.join(__dirname, '../../../../lib/with-react-17.js'))({ target: 'serverless', cleanDistDir: false })
`,
`module.exports = ({ target: 'serverless', cleanDistDir: false })`,
'utf8'
)
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
appPort = await findPort()
app = await nextStart(appDir, appPort, {
nodeArgs,
})
app = await nextStart(appDir, appPort)
builtServerPagesDir = join(appDir, '.next', 'serverless')
builtPage = (file) => join(builtServerPagesDir, file)
})
......@@ -295,19 +278,12 @@ describe('Font Optimization', () => {
beforeAll(async () => {
await fs.writeFile(
nextConfig,
`
const path = require('path')
module.exports = require(path.join(__dirname, '../../../../lib/with-react-17.js'))({ target: 'experimental-serverless-trace', cleanDistDir: false })
`,
`module.exports = ({ target: 'experimental-serverless-trace', cleanDistDir: false })`,
'utf8'
)
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
appPort = await findPort()
app = await startServerlessEmulator(appDir, appPort, {
nodeArgs,
})
app = await startServerlessEmulator(appDir, appPort)
builtServerPagesDir = join(appDir, '.next', 'serverless')
builtPage = (file) => join(builtServerPagesDir, file)
})
......@@ -320,18 +296,14 @@ describe('Font Optimization', () => {
describe('Font optimization for unreachable font definitions.', () => {
beforeAll(async () => {
await nextBuild(appDir, undefined, {
nodeArgs,
})
await nextBuild(appDir)
await fs.writeFile(
join(appDir, '.next', 'server', 'font-manifest.json'),
'[]',
'utf8'
)
appPort = await findPort()
app = await nextStart(appDir, appPort, {
nodeArgs,
})
app = await nextStart(appDir, appPort)
builtServerPagesDir = join(appDir, '.next', 'serverless')
builtPage = (file) => join(builtServerPagesDir, file)
})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment