Unverified Commit 6344c5f0 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Add optimizing AMP pages (#6835)

* Added amp-toolbox-optimizer and added
optimizing AMP pages as dirty and clean

* Fix amp-optimizer breaking serverless build

* Exclude amp-toolbox-optimizer from serverless build

* Added check to make sure hybrid AMP pages
optimize cleanly during export

* Add check to make sure noDirtyAmp is applied
parent b1fdffec
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
] ]
}, },
"dependencies": { "dependencies": {
"amp-toolbox-optimizer": "0.5.2-beta.0",
"etag": "1.8.1", "etag": "1.8.1",
"find-up": "3.0.0", "find-up": "3.0.0",
"fresh": "0.5.2", "fresh": "0.5.2",
......
...@@ -22,6 +22,7 @@ const defaultConfig = { ...@@ -22,6 +22,7 @@ const defaultConfig = {
}, },
experimental: { experimental: {
amp: false, amp: false,
noDirtyAmp: false,
cpus: Math.max( cpus: Math.max(
1, 1,
(Number(process.env.CIRCLE_NODE_TOTAL) || (Number(process.env.CIRCLE_NODE_TOTAL) ||
......
...@@ -10,9 +10,10 @@ export type BuildManifest = { ...@@ -10,9 +10,10 @@ export type BuildManifest = {
export function getPageFiles(buildManifest: BuildManifest, page: string): string[] { export function getPageFiles(buildManifest: BuildManifest, page: string): string[] {
const normalizedPage = normalizePagePath(page) const normalizedPage = normalizePagePath(page)
const files = buildManifest.pages[normalizedPage] let files = buildManifest.pages[normalizedPage]
if (!files) { if (!files) {
page = tryAmp(buildManifest.pages, normalizedPage) page = tryAmp(buildManifest.pages, normalizedPage)
files = buildManifest.pages[page]
} }
if (!files) { if (!files) {
......
...@@ -38,6 +38,7 @@ export default class Server { ...@@ -38,6 +38,7 @@ export default class Server {
buildId: string buildId: string
renderOpts: { renderOpts: {
ampEnabled: boolean ampEnabled: boolean
noDirtyAmp: boolean
ampBindInitData: boolean ampBindInitData: boolean
staticMarkup: boolean staticMarkup: boolean
buildId: string buildId: string
...@@ -77,6 +78,7 @@ export default class Server { ...@@ -77,6 +78,7 @@ export default class Server {
this.buildId = this.readBuildId() this.buildId = this.readBuildId()
this.renderOpts = { this.renderOpts = {
ampEnabled: this.nextConfig.experimental.amp, ampEnabled: this.nextConfig.experimental.amp,
noDirtyAmp: this.nextConfig.experimental.noDirtyAmp,
ampBindInitData: this.nextConfig.experimental.ampBindInitData, ampBindInitData: this.nextConfig.experimental.ampBindInitData,
staticMarkup, staticMarkup,
buildId: this.buildId, buildId: this.buildId,
......
import { ParsedUrlQuery } from "querystring"
interface IOptimizerConfig {
transforms?: string[],
validAmp?: boolean,
verbose?: boolean,
}
interface IOptimizer {
transformHtml: (html: string, params?: object) => Promise<string>
setConfig: (config: IOptimizerConfig) => void
}
interface IOptimizeOptions {
amphtml?: boolean
noDirtyAmp?: boolean
query?: ParsedUrlQuery
}
export default async function optimize(html: string, { amphtml, noDirtyAmp, query }: IOptimizeOptions): Promise<string> {
let ampOptimizer: IOptimizer
try {
ampOptimizer = require('amp-toolbox-optimizer')
} catch (_) {
return html
}
const validAmp = Boolean(noDirtyAmp || (amphtml && query && query.amp))
// Examples below
//
// pages/foo.js
// pages/foo.amp.js
// pages/bar.amp.js
//
// /foo => traditional
// /foo?amp=1 => clean AMP
// /bar => dirty amp
// /bar?amp=1 => clean AMP
ampOptimizer.setConfig({ validAmp })
html = await ampOptimizer.transformHtml(html)
return html
}
...@@ -19,6 +19,7 @@ import { ...@@ -19,6 +19,7 @@ import {
} from './get-dynamic-import-bundles' } from './get-dynamic-import-bundles'
import { getPageFiles, BuildManifest } from './get-page-files' import { getPageFiles, BuildManifest } from './get-page-files'
import { IsAmpContext } from '../lib/amphtml-context' import { IsAmpContext } from '../lib/amphtml-context'
import optimizeAmp from './optimize-amp'
type Enhancer = (Component: React.ComponentType) => React.ComponentType type Enhancer = (Component: React.ComponentType) => React.ComponentType
type ComponentsEnhancer = type ComponentsEnhancer =
...@@ -111,6 +112,7 @@ function render( ...@@ -111,6 +112,7 @@ function render(
type RenderOpts = { type RenderOpts = {
ampEnabled: boolean ampEnabled: boolean
noDirtyAmp: boolean
ampBindInitData: boolean ampBindInitData: boolean
staticMarkup: boolean staticMarkup: boolean
buildId: string buildId: string
...@@ -216,6 +218,7 @@ export async function renderToHTML( ...@@ -216,6 +218,7 @@ export async function renderToHTML(
dev = false, dev = false,
ampBindInitData = false, ampBindInitData = false,
staticMarkup = false, staticMarkup = false,
noDirtyAmp = false,
amphtml = false, amphtml = false,
hasAmp = false, hasAmp = false,
ampPath = '', ampPath = '',
...@@ -399,7 +402,7 @@ export async function renderToHTML( ...@@ -399,7 +402,7 @@ export async function renderToHTML(
return dataManagerData return dataManagerData
} }
return renderDocument(Document, { let html = renderDocument(Document, {
...renderOpts, ...renderOpts,
dataManagerData, dataManagerData,
props, props,
...@@ -414,6 +417,11 @@ export async function renderToHTML( ...@@ -414,6 +417,11 @@ export async function renderToHTML(
files, files,
devFiles, devFiles,
}) })
if (amphtml && html) {
html = await optimizeAmp(html, { amphtml, noDirtyAmp, query })
}
return html
} }
function errorToJSON(err: Error): Error { function errorToJSON(err: Error): Error {
......
...@@ -127,6 +127,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, isServe ...@@ -127,6 +127,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, isServe
] : [ ] : [
// When the serverless target is used all node_modules will be compiled into the output bundles // When the serverless target is used all node_modules will be compiled into the output bundles
// So that the serverless bundles have 0 runtime dependencies // So that the serverless bundles have 0 runtime dependencies
'amp-toolbox-optimizer' // except this one
], ],
optimization: isServer ? { optimization: isServer ? {
splitChunks: false, splitChunks: false,
......
...@@ -59,11 +59,27 @@ export default async function (dir, options, configuration) { ...@@ -59,11 +59,27 @@ export default async function (dir, options, configuration) {
const isAmp = path.indexOf('.amp') > -1 const isAmp = path.indexOf('.amp') > -1
if (isAmp) { if (isAmp) {
defaultPathMap[path].query = { amp: 1 } defaultPathMap[path].query = { amphtml: true }
const nonAmp = cleanAmpPath(path).replace(/\/$/, '') || '/' const nonAmp = cleanAmpPath(path).replace(/\/$/, '') || '/'
if (!defaultPathMap[nonAmp]) { if (!defaultPathMap[nonAmp]) {
if (!nextConfig.experimental.noDirtyAmp) {
// dirty optimized
defaultPathMap[nonAmp] = {
...defaultPathMap[path],
query: { ...defaultPathMap[path].query }
}
defaultPathMap[nonAmp].query.ampOnly = true
defaultPathMap[nonAmp].query.ampPath = path
// clean optimized
defaultPathMap[path].query.amp = 1
} else {
// dirty optimizing is disabled
defaultPathMap[path].query.amp = 1
defaultPathMap[path].query.ampOnly = true defaultPathMap[path].query.ampOnly = true
} }
} else {
defaultPathMap[path].query.amp = 1
}
} else { } else {
const ampPath = tryAmp(defaultPathMap, path) const ampPath = tryAmp(defaultPathMap, path)
if (ampPath !== path) { if (ampPath !== path) {
......
...@@ -32,9 +32,12 @@ process.on( ...@@ -32,9 +32,12 @@ process.on(
const work = async path => { const work = async path => {
await sema.acquire() await sema.acquire()
const { page, query = {} } = exportPathMap[path] const { page, query = {} } = exportPathMap[path]
const ampOpts = { amphtml: Boolean(query.amp), hasAmp: query.hasAmp, ampPath: query.ampPath } const ampOpts = { amphtml: query.amphtml, hasAmp: query.hasAmp, ampPath: query.ampPath }
const ampOnly = query.ampOnly
delete query.ampOnly
delete query.hasAmp delete query.hasAmp
delete query.ampPath delete query.ampPath
delete query.amphtml
const req = { url: path } const req = { url: path }
const res = {} const res = {}
...@@ -43,9 +46,9 @@ process.on( ...@@ -43,9 +46,9 @@ process.on(
publicRuntimeConfig: renderOpts.runtimeConfig publicRuntimeConfig: renderOpts.runtimeConfig
}) })
if (query.ampOnly) { if (ampOnly) {
delete query.ampOnly
path = cleanAmpPath(path) path = cleanAmpPath(path)
ampOpts.ampPath = path + '.amp'
} }
// replace /docs/index.amp with /docs.amp // replace /docs/index.amp with /docs.amp
...@@ -69,7 +72,7 @@ process.on( ...@@ -69,7 +72,7 @@ process.on(
const components = await loadComponents(distDir, buildId, page) const components = await loadComponents(distDir, buildId, page)
const html = await renderToHTML(req, res, page, query, { ...components, ...renderOpts, ...ampOpts }) const html = await renderToHTML(req, res, page, query, { ...components, ...renderOpts, ...ampOpts })
if (ampOpts.amphtml) { if (ampOpts.amphtml && query.amp) {
const validator = await AmpHtmlValidator.getInstance() const validator = await AmpHtmlValidator.getInstance()
const result = validator.validateString(html) const result = validator.validateString(html)
const errors = result.errors.filter(e => e.severity === 'ERROR') const errors = result.errors.filter(e => e.severity === 'ERROR')
......
...@@ -158,6 +158,7 @@ export class Head extends Component { ...@@ -158,6 +158,7 @@ export class Head extends Component {
} = this.context._documentProps } = this.context._documentProps
const { _devOnlyInvalidateCacheQueryString } = this.context const { _devOnlyInvalidateCacheQueryString } = this.context
const { page, buildId } = __NEXT_DATA__ const { page, buildId } = __NEXT_DATA__
const isDirtyAmp = amphtml && !__NEXT_DATA__.query.amp
let { head } = this.context._documentProps let { head } = this.context._documentProps
let children = this.props.children let children = this.props.children
...@@ -211,6 +212,7 @@ export class Head extends Component { ...@@ -211,6 +212,7 @@ export class Head extends Component {
content="width=device-width,minimum-scale=1,initial-scale=1" content="width=device-width,minimum-scale=1,initial-scale=1"
/> />
<link rel="canonical" href={cleanAmpPath(page)} /> <link rel="canonical" href={cleanAmpPath(page)} />
{isDirtyAmp && <link rel="amphtml" href={ampPath ? ampPath : `${page}?amp=1`} />}
{/* https://www.ampproject.org/docs/fundamentals/optimize_amp#optimize-the-amp-runtime-loading */} {/* https://www.ampproject.org/docs/fundamentals/optimize_amp#optimize-the-amp-runtime-loading */}
<link <link
rel="preload" rel="preload"
......
export default () => (
<p>Hello AMP!</p>
)
export default () => (
<p>Hello again AMP!</p>
)
export default () => (
'I am a hybrid AMP page'
)
...@@ -3,20 +3,48 @@ ...@@ -3,20 +3,48 @@
import fs from 'fs' import fs from 'fs'
import { join } from 'path' import { join } from 'path'
import { promisify } from 'util' import { promisify } from 'util'
import { validateAMP } from 'amp-test-utils'
import { import {
File, File,
nextBuild, nextBuild,
nextExport,
runNextCommand runNextCommand
} from 'next-test-utils' } from 'next-test-utils'
jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5 jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
const access = promisify(fs.access) const access = promisify(fs.access)
const readFile = promisify(fs.readFile)
const appDir = join(__dirname, '../') const appDir = join(__dirname, '../')
const outDir = join(appDir, 'out') const outDir = join(appDir, 'out')
const nextConfig = new File(join(appDir, 'next.config.js')) const nextConfig = new File(join(appDir, 'next.config.js'))
describe('AMP Validation on Export', () => { describe('AMP Validation on Export', () => {
beforeAll(() => nextBuild(appDir)) beforeAll(async () => {
await nextBuild(appDir)
await nextExport(appDir, { outdir: outDir })
})
it('should export both clean and dirty optimized pages', async () => {
const toCheck = ['first', 'second', 'third']
await Promise.all(toCheck.map(async page => {
const dirty = await readFile(join(outDir, `${page}/index.html`))
const clean = await readFile(join(outDir, `${page}.amp/index.html`))
await validateAMP(clean.toString())
if (page !== 'third') await validateAMP(dirty.toString(), true)
}))
})
it('should disable dirty AMP with noDirtyAmp set', async () => {
nextConfig.replace(`amp: true`, `amp: true,noDirtyAmp: true`)
await nextExport(appDir, { outdir: outDir })
const ampOnly = ['first', 'second', 'third']
await Promise.all(ampOnly.map(async page => {
const html = await readFile(join(outDir, `${
page + (page === 'third' ? '.amp' : '')}/index.html`))
await validateAMP(html.toString())
}))
nextConfig.restore()
})
it('shows AMP warning without throwing error', async () => { it('shows AMP warning without throwing error', async () => {
nextConfig.replace('// exportPathMap', nextConfig.replace('// exportPathMap',
...@@ -26,15 +54,8 @@ describe('AMP Validation on Export', () => { ...@@ -26,15 +54,8 @@ describe('AMP Validation on Export', () => {
} }
},`) },`)
let stdout = ''
try { try {
await runNextCommand(['export', appDir], { const { stdout } = await runNextCommand(['export', appDir], { stdout: true })
instance: child => {
child.stdout.on('data', chunk => {
stdout += chunk.toString()
})
}
})
expect(stdout).toMatch(/warn.*The tag 'amp-video extension \.js script' is missing/) expect(stdout).toMatch(/warn.*The tag 'amp-video extension \.js script' is missing/)
await expect(access(join(outDir, 'cat/index.html'))).resolves.toBe(undefined) await expect(access(join(outDir, 'cat/index.html'))).resolves.toBe(undefined)
} finally { } finally {
...@@ -50,15 +71,8 @@ describe('AMP Validation on Export', () => { ...@@ -50,15 +71,8 @@ describe('AMP Validation on Export', () => {
} }
},`) },`)
let stdout = ''
try { try {
await runNextCommand(['export', appDir], { const { stdout } = await runNextCommand(['export', appDir], { stdout: true })
instance: child => {
child.stdout.on('data', chunk => {
stdout += chunk.toString()
})
}
})
expect(stdout).toMatch(/error.*The tag 'img' may only appear as a descendant of tag 'noscript'. Did you mean 'amp-img'\?/) expect(stdout).toMatch(/error.*The tag 'img' may only appear as a descendant of tag 'noscript'. Did you mean 'amp-img'\?/)
await expect(access(join(outDir, 'dog/index.html'))).resolves.toBe(undefined) await expect(access(join(outDir, 'dog/index.html'))).resolves.toBe(undefined)
} finally { } finally {
...@@ -74,15 +88,8 @@ describe('AMP Validation on Export', () => { ...@@ -74,15 +88,8 @@ describe('AMP Validation on Export', () => {
} }
},`) },`)
let stdout = ''
try { try {
await runNextCommand(['export', appDir], { const { stdout } = await runNextCommand(['export', appDir], { stdout: true })
instance: child => {
child.stdout.on('data', chunk => {
stdout += chunk.toString()
})
}
})
expect(stdout).toMatch(/warn.*The tag 'amp-video extension \.js script' is missing/) expect(stdout).toMatch(/warn.*The tag 'amp-video extension \.js script' is missing/)
expect(stdout).toMatch(/error.*The tag 'img' may only appear as a descendant of tag 'noscript'. Did you mean 'amp-img'\?/) expect(stdout).toMatch(/error.*The tag 'img' may only appear as a descendant of tag 'noscript'. Did you mean 'amp-img'\?/)
await expect(access(join(outDir, 'dog-cat/index.html'))).resolves.toBe(undefined) await expect(access(join(outDir, 'dog-cat/index.html'))).resolves.toBe(undefined)
......
/* eslint-env jest */ /* eslint-env jest */
/* global jasmine */ /* global jasmine */
import { join } from 'path' import { join } from 'path'
import cheerio from 'cheerio'
import webdriver from 'next-webdriver' import webdriver from 'next-webdriver'
import { validateAMP } from 'amp-test-utils'
import { readFileSync, writeFileSync } from 'fs' import { readFileSync, writeFileSync } from 'fs'
import { import {
nextServer, nextServer,
...@@ -15,8 +17,7 @@ import { ...@@ -15,8 +17,7 @@ import {
launchApp, launchApp,
killApp killApp
} from 'next-test-utils' } from 'next-test-utils'
import cheerio from 'cheerio'
import amphtmlValidator from 'amphtml-validator'
const appDir = join(__dirname, '../') const appDir = join(__dirname, '../')
let appPort let appPort
let server let server
...@@ -25,23 +26,6 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5 ...@@ -25,23 +26,6 @@ jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
const context = {} const context = {}
async function validateAMP (html) {
const validator = await amphtmlValidator.getInstance()
const result = validator.validateString(html)
if (result.status !== 'PASS') {
for (let ii = 0; ii < result.errors.length; ii++) {
const error = result.errors[ii]
let msg =
'line ' + error.line + ', col ' + error.col + ': ' + error.message
if (error.specUrl !== null) {
msg += ' (see ' + error.specUrl + ')'
}
;(error.severity === 'ERROR' ? console.error : console.warn)(msg)
}
}
expect(result.status).toBe('PASS')
}
describe('AMP Usage', () => { describe('AMP Usage', () => {
beforeAll(async () => { beforeAll(async () => {
await nextBuild(appDir) await nextBuild(appDir)
...@@ -114,6 +98,16 @@ describe('AMP Usage', () => { ...@@ -114,6 +98,16 @@ describe('AMP Usage', () => {
expect(html).not.toMatch(/src='\/im-not-allowed\.js'/) expect(html).not.toMatch(/src='\/im-not-allowed\.js'/)
expect(html).not.toMatch(/console\.log("I'm not either :p")'/) expect(html).not.toMatch(/console\.log("I'm not either :p")'/)
}) })
it('should optimize dirty when ?amp=1 is not specified', async () => {
const html = await renderViaHTTP(appPort, '/only-amp')
await validateAMP(html, true)
})
it('should optimize clean when ?amp=1 is specified', async () => {
const html = await renderViaHTTP(appPort, '/only-amp?amp=1')
await validateAMP(html)
})
}) })
describe('With AMP context', () => { describe('With AMP context', () => {
...@@ -165,7 +159,6 @@ describe('AMP Usage', () => { ...@@ -165,7 +159,6 @@ describe('AMP Usage', () => {
it('should render a canonical regardless of amp-only status (implicit)', async () => { it('should render a canonical regardless of amp-only status (implicit)', async () => {
const html = await renderViaHTTP(appPort, '/only-amp') const html = await renderViaHTTP(appPort, '/only-amp')
const $ = cheerio.load(html) const $ = cheerio.load(html)
await validateAMP(html)
expect( expect(
$('link[rel=canonical]') $('link[rel=canonical]')
.first() .first()
...@@ -194,6 +187,16 @@ describe('AMP Usage', () => { ...@@ -194,6 +187,16 @@ describe('AMP Usage', () => {
).not.toBeTruthy() ).not.toBeTruthy()
}) })
it('should render amphtml link tag with dirty AMP page', async () => {
const html = await renderViaHTTP(appPort, '/only-amp')
const $ = cheerio.load(html)
expect(
$('link[rel=amphtml]')
.first()
.attr('href')
).toBe('/only-amp?amp=1')
})
it('should remove conflicting amp tags', async () => { it('should remove conflicting amp tags', async () => {
const html = await renderViaHTTP(appPort, '/conflicting-tag?amp=1') const html = await renderViaHTTP(appPort, '/conflicting-tag?amp=1')
const $ = cheerio.load(html) const $ = cheerio.load(html)
......
/* eslint-env jest */
import amphtmlValidator from 'amphtml-validator'
export async function validateAMP (html, expectFail) {
const validator = await amphtmlValidator.getInstance()
const result = validator.validateString(html)
if (result.status !== 'PASS') {
for (let ii = 0; ii < result.errors.length; ii++) {
const error = result.errors[ii]
let msg =
'line ' + error.line + ', col ' + error.col + ': ' + error.message
if (error.specUrl !== null) {
msg += ' (see ' + error.specUrl + ')'
}
if (!expectFail) {
;(error.severity === 'ERROR' ? console.error : console.warn)(msg)
}
}
}
expect(result.status).toBe(expectFail ? 'FAIL' : 'PASS')
}
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment