Unverified Commit 2169c0ce authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Use buildManifest to load page JS (#13870)

Initial work to use chunkhashes instead of buildid for the page files in production. This does not change the calculation of the filename itself initially.
parent b4ec9928
......@@ -380,10 +380,6 @@ async function computeFromManifest(
let expected = 0
const files = new Map<string, number>()
Object.keys(manifest.pages).forEach((key) => {
if (key === '/_polyfills') {
return
}
if (pageInfos) {
const pageInfo = pageInfos.get(key)
// don't include AMP pages since they don't rely on shared bundles
......
import devalue from 'next/dist/compiled/devalue'
import { Compiler } from 'webpack'
import { Compiler, compilation as CompilationType } from 'webpack'
import { RawSource } from 'webpack-sources'
import {
BUILD_MANIFEST,
......@@ -7,22 +7,21 @@ import {
CLIENT_STATIC_FILES_RUNTIME_MAIN,
CLIENT_STATIC_FILES_RUNTIME_POLYFILLS,
CLIENT_STATIC_FILES_RUNTIME_REACT_REFRESH,
IS_BUNDLED_PAGE_REGEX,
} from '../../../next-server/lib/constants'
import { BuildManifest } from '../../../next-server/server/get-page-files'
import getRouteFromEntrypoint from '../../../next-server/server/get-route-from-entrypoint'
// This function takes the asset map generated in BuildManifestPlugin and creates a
// reduced version to send to the client.
const generateClientManifest = (
function generateClientManifest(
assetMap: BuildManifest,
isModern: boolean
): string => {
): string {
const clientManifest: { [s: string]: string[] } = {}
const appDependencies = new Set(assetMap.pages['/_app'])
Object.entries(assetMap.pages).forEach(([page, dependencies]) => {
if (page === '/_app' || page === '/_polyfills') return
if (page === '/_app') return
// Filter out dependencies in the _app entry, because those will have already
// been loaded by the client prior to a navigation event
const filteredDeps = dependencies.filter(
......@@ -54,7 +53,7 @@ export default class BuildManifestPlugin {
compiler.hooks.emit.tapAsync(
'NextJsBuildManifest',
(compilation, callback) => {
const { chunks } = compilation
const chunks: CompilationType.Chunk[] = compilation.chunks
const assetMap: BuildManifest = {
polyfillFiles: [],
devFiles: [],
......@@ -65,20 +64,22 @@ export default class BuildManifestPlugin {
const mainJsChunk = chunks.find(
(c) => c.name === CLIENT_STATIC_FILES_RUNTIME_MAIN
)
const mainJsFiles: string[] =
mainJsChunk && mainJsChunk.files.length > 0
? mainJsChunk.files.filter((file: string) => /\.js$/.test(file))
: []
mainJsChunk?.files.filter((file: string) => file.endsWith('.js')) ??
[]
const polyfillChunk = chunks.find(
(c) => c.name === CLIENT_STATIC_FILES_RUNTIME_POLYFILLS
)
const polyfillFiles: string[] = polyfillChunk ? polyfillChunk.files : []
// Create a separate entry for polyfills
assetMap.polyfillFiles = polyfillChunk?.files ?? []
const reactRefreshChunk = chunks.find(
(c) => c.name === CLIENT_STATIC_FILES_RUNTIME_REACT_REFRESH
)
assetMap.devFiles.push(...(reactRefreshChunk?.files ?? []))
assetMap.devFiles = reactRefreshChunk?.files ?? []
for (const entrypoint of compilation.entrypoints.values()) {
const pagePath = getRouteFromEntrypoint(entrypoint.name)
......@@ -91,17 +92,7 @@ export default class BuildManifestPlugin {
// getFiles() - helper function to read the files for an entrypoint from stats object
for (const file of entrypoint.getFiles()) {
if (/\.map$/.test(file) || /\.hot-update\.js$/.test(file)) {
continue
}
// Only `.js` and `.css` files are added for now. In the future we can also handle other file types.
if (!/\.js$/.test(file) && !/\.css$/.test(file)) {
continue
}
// The page bundles are manually added to _document.js as they need extra properties
if (IS_BUNDLED_PAGE_REGEX.exec(file)) {
if (!(file.endsWith('.js') || file.endsWith('.css'))) {
continue
}
......@@ -111,9 +102,6 @@ export default class BuildManifestPlugin {
assetMap.pages[pagePath] = [...filesForEntry, ...mainJsFiles]
}
// Create a separate entry for polyfills
assetMap.polyfillFiles = polyfillFiles
// Add the runtime build manifest file (generated later in this file)
// as a dependency for the app. If the flag is false, the file won't be
// downloaded by the client.
......
......@@ -50,7 +50,7 @@ envConfig.setConfig({
const asPath = getURL()
const pageLoader = new PageLoader(buildId, prefix)
const pageLoader = new PageLoader(buildId, prefix, page)
const register = ([r, f]) => pageLoader.registerPage(r, f)
if (window.__NEXT_P) {
// Defer page registration for another tick. This will increase the overall
......@@ -170,7 +170,7 @@ export default async ({ webpackHMR: passedWebpackHMR } = {}) => {
if (process.env.NODE_ENV === 'development') {
webpackHMR = passedWebpackHMR
}
const { page: app, mod } = await pageLoader.loadPageScript('/_app')
const { page: app, mod } = await pageLoader.loadPage('/_app')
CachedApp = app
if (mod && mod.reportWebVitals) {
......
......@@ -11,6 +11,12 @@ function hasRel(rel, link) {
} catch {}
}
function pageLoadError(route) {
const error = new Error(`Error loading ${route}`)
error.code = 'PAGE_LOAD_ERROR'
return error
}
const relPrefetch =
hasRel('preload') && !hasRel('prefetch')
? // https://caniuse.com/#feat=link-rel-preload
......@@ -56,13 +62,22 @@ function appendLink(href, rel, as) {
}
export default class PageLoader {
constructor(buildId, assetPrefix) {
constructor(buildId, assetPrefix, initialPage) {
this.buildId = buildId
this.assetPrefix = assetPrefix
this.pageCache = {}
this.pageRegisterEvents = mitt()
this.loadingRoutes = {}
this.loadingRoutes = {
// By default these 2 pages are being loaded in the initial html
'/_app': true,
}
// TODO: get rid of this limitation for rendering the error page
if (initialPage !== '/_error') {
this.loadingRoutes[initialPage] = true
}
if (process.env.NODE_ENV === 'production') {
this.promisedBuildManifest = new Promise((resolve) => {
if (window.__BUILD_MANIFEST) {
......@@ -88,14 +103,13 @@ export default class PageLoader {
// Returns a promise for the dependencies for a particular route
getDependencies(route) {
return this.promisedBuildManifest.then(
(man) =>
(man[route] &&
man[route].map(
(url) => `${this.assetPrefix}/_next/${encodeURI(url)}`
)) ||
[]
)
return this.promisedBuildManifest.then((m) => {
return m[route]
? m[route].map((url) => `${this.assetPrefix}/_next/${encodeURI(url)}`)
: this.pageRegisterEvents.emit(route, {
error: pageLoadError(route),
}) ?? []
})
}
/**
......@@ -182,13 +196,17 @@ export default class PageLoader {
}
loadPage(route) {
return this.loadPageScript(route)
}
loadPageScript(route) {
route = normalizeRoute(route)
return new Promise((resolve, reject) => {
// If there's a cached version of the page, let's use it.
const cachedPage = this.pageCache[route]
if (cachedPage) {
const { error, page, mod } = cachedPage
error ? reject(error) : resolve({ page, mod })
return
}
const fire = ({ error, page, mod }) => {
this.pageRegisterEvents.off(route, fire)
delete this.loadingRoutes[route]
......@@ -200,36 +218,22 @@ export default class PageLoader {
}
}
// If there's a cached version of the page, let's use it.
const cachedPage = this.pageCache[route]
if (cachedPage) {
const { error, page, mod } = cachedPage
error ? reject(error) : resolve({ page, mod })
return
}
// Register a listener to get the page
this.pageRegisterEvents.on(route, fire)
// If the page is loading via SSR, we need to wait for it
// rather downloading it again.
if (document.querySelector(`script[data-next-page="${route}"]`)) {
return
}
if (!this.loadingRoutes[route]) {
this.loadingRoutes[route] = true
if (process.env.NODE_ENV === 'production') {
this.getDependencies(route).then((deps) => {
deps.forEach((d) => {
if (
/\.js$/.test(d) &&
d.endsWith('.js') &&
!document.querySelector(`script[src^="${d}"]`)
) {
this.loadScript(d, route, false)
this.loadScript(d, route)
}
if (
/\.css$/.test(d) &&
d.endsWith('.css') &&
!document.querySelector(`link[rel=stylesheet][href^="${d}"]`)
) {
appendLink(d, 'stylesheet').catch(() => {
......@@ -238,39 +242,30 @@ export default class PageLoader {
})
}
})
this.loadRoute(route)
})
} else {
this.loadRoute(route)
// Development only. In production the page file is part of the build manifest
route = normalizeRoute(route)
let scriptRoute = getAssetPath(route)
const url = `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${encodeURI(scriptRoute)}.js`
this.loadScript(url, route)
}
}
})
}
loadRoute(route) {
route = normalizeRoute(route)
let scriptRoute = getAssetPath(route)
const url = `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${encodeURI(scriptRoute)}.js`
this.loadScript(url, route, true)
}
loadScript(url, route, isPage) {
loadScript(url, route) {
const script = document.createElement('script')
if (process.env.__NEXT_MODERN_BUILD && hasNoModule) {
script.type = 'module'
// Only page bundle scripts need to have .module added to url,
// dependencies already have it added during build manifest creation
if (isPage) url = url.replace(/\.js$/, '.module.js')
}
script.crossOrigin = process.env.__NEXT_CROSS_ORIGIN
script.src = url
script.onerror = () => {
const error = new Error(`Error loading script ${url}`)
error.code = 'PAGE_LOAD_ERROR'
this.pageRegisterEvents.emit(route, { error })
this.pageRegisterEvents.emit(route, { error: pageLoadError(url) })
}
document.body.appendChild(script)
}
......@@ -329,28 +324,29 @@ export default class PageLoader {
if (isDependency) {
url = route
} else {
route = normalizeRoute(route)
if (process.env.NODE_ENV !== 'production') {
route = normalizeRoute(route)
const scriptRoute = getAssetPath(route)
const ext =
process.env.__NEXT_MODERN_BUILD && hasNoModule ? '.module.js' : '.js'
const scriptRoute = getAssetPath(route)
const ext =
process.env.__NEXT_MODERN_BUILD && hasNoModule ? '.module.js' : '.js'
url = `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${encodeURI(scriptRoute)}${ext}`
url = `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${encodeURI(scriptRoute)}${ext}`
}
}
return Promise.all(
document.querySelector(
`link[rel="${relPrefetch}"][href^="${url}"], script[data-next-page="${route}"]`
)
document.querySelector(`link[rel="${relPrefetch}"][href^="${url}"]`)
? []
: [
appendLink(
url,
relPrefetch,
url.match(/\.css$/) ? 'style' : 'script'
),
url &&
appendLink(
url,
relPrefetch,
url.endsWith('.css') ? 'style' : 'script'
),
process.env.NODE_ENV === 'production' &&
!isDependency &&
this.getDependencies(route).then((urls) =>
......
......@@ -657,7 +657,7 @@ export async function renderToHTML(
const files = [
...new Set([
...getPageFiles(buildManifest, '/_app'),
...getPageFiles(buildManifest, pathname),
...(pathname !== '/_error' ? getPageFiles(buildManifest, pathname) : []),
]),
]
......
......@@ -159,7 +159,7 @@ export class Head extends Component<
const { assetPrefix, files } = this.context._documentProps
const { _devOnlyInvalidateCacheQueryString } = this.context
const cssFiles =
files && files.length ? files.filter((f) => /\.css$/.test(f)) : []
files && files.length ? files.filter((f) => f.endsWith('.css')) : []
const cssLinkElements: JSX.Element[] = []
cssFiles.forEach((file) => {
......@@ -264,7 +264,6 @@ export class Head extends Component<
styles,
ampPath,
inAmpMode,
assetPrefix,
hybridAmp,
canonicalBase,
__NEXT_DATA__,
......@@ -273,8 +272,6 @@ export class Head extends Component<
unstable_runtimeJS,
} = this.context._documentProps
const disableRuntimeJS = unstable_runtimeJS === false
const { _devOnlyInvalidateCacheQueryString } = this.context
const { page, buildId } = __NEXT_DATA__
let { head } = this.context._documentProps
let children = this.props.children
......@@ -453,42 +450,6 @@ export class Head extends Component<
/>
)}
{this.getCssLinks()}
{!disableRuntimeJS && (
<link
rel="preload"
href={
assetPrefix +
getOptionalModernScriptVariant(
encodeURI(`/_next/static/${buildId}/pages/_app.js`)
) +
_devOnlyInvalidateCacheQueryString
}
as="script"
nonce={this.props.nonce}
crossOrigin={
this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN
}
/>
)}
{!disableRuntimeJS && page !== '/_error' && (
<link
rel="preload"
href={
assetPrefix +
getOptionalModernScriptVariant(
encodeURI(
`/_next/static/${buildId}/pages${getPageFile(page)}`
)
) +
_devOnlyInvalidateCacheQueryString
}
as="script"
nonce={this.props.nonce}
crossOrigin={
this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN
}
/>
)}
{!disableRuntimeJS && this.getPreloadDynamicChunks()}
{!disableRuntimeJS && this.getPreloadMainLinks()}
{this.context._documentProps.isDevelopment && (
......@@ -506,16 +467,12 @@ export class Head extends Component<
}
}
export class Main extends Component {
static contextType = DocumentComponentContext
context!: React.ContextType<typeof DocumentComponentContext>
render() {
const { inAmpMode, html } = this.context._documentProps
if (inAmpMode) return AMP_RENDER_TARGET
return <div id="__next" dangerouslySetInnerHTML={{ __html: html }} />
}
export function Main() {
const { inAmpMode, html } = useContext(
DocumentComponentContext
)._documentProps
if (inAmpMode) return <>{AMP_RENDER_TARGET}</>
return <div id="__next" dangerouslySetInnerHTML={{ __html: html }} />
}
export class NextScript extends Component<OriginProps> {
......@@ -539,12 +496,13 @@ export class NextScript extends Component<OriginProps> {
return dedupe(dynamicImports).map((bundle: any) => {
let modernProps = {}
if (process.env.__NEXT_MODERN_BUILD) {
modernProps = /\.module\.js$/.test(bundle.file)
modernProps = bundle.file.endsWith('.module.js')
? { type: 'module' }
: { noModule: true }
}
if (!/\.js$/.test(bundle.file) || files.includes(bundle.file)) return null
if (!bundle.file.endsWith('.js') || files.includes(bundle.file))
return null
return (
<script
......@@ -605,7 +563,7 @@ export class NextScript extends Component<OriginProps> {
return buildManifest.polyfillFiles
.filter(
(polyfill) =>
polyfill.endsWith('.js') && !/\.module\.js$/.test(polyfill)
polyfill.endsWith('.js') && !polyfill.endsWith('.module.js')
)
.map((polyfill) => (
<script
......@@ -640,7 +598,6 @@ export class NextScript extends Component<OriginProps> {
assetPrefix,
inAmpMode,
buildManifest,
__NEXT_DATA__,
unstable_runtimeJS,
} = this.context._documentProps
const disableRuntimeJS = unstable_runtimeJS === false
......@@ -693,8 +650,6 @@ export class NextScript extends Component<OriginProps> {
)
}
const { page, buildId } = __NEXT_DATA__
if (process.env.NODE_ENV !== 'production') {
if (this.props.crossOrigin)
console.warn(
......@@ -702,92 +657,21 @@ export class NextScript extends Component<OriginProps> {
)
}
const pageScript = [
<script
async
data-next-page={page}
key={page}
src={
assetPrefix +
encodeURI(`/_next/static/${buildId}/pages${getPageFile(page)}`) +
_devOnlyInvalidateCacheQueryString
}
nonce={this.props.nonce}
crossOrigin={this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN}
{...(process.env.__NEXT_MODERN_BUILD ? { noModule: true } : {})}
/>,
process.env.__NEXT_MODERN_BUILD && (
<script
async
data-next-page={page}
key={`${page}-modern`}
src={
assetPrefix +
getOptionalModernScriptVariant(
encodeURI(`/_next/static/${buildId}/pages${getPageFile(page)}`)
) +
_devOnlyInvalidateCacheQueryString
}
nonce={this.props.nonce}
crossOrigin={
this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN
}
type="module"
/>
),
]
const appScript = [
<script
async
data-next-page="/_app"
src={
assetPrefix +
`/_next/static/${buildId}/pages/_app.js` +
_devOnlyInvalidateCacheQueryString
}
key="_app"
nonce={this.props.nonce}
crossOrigin={this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN}
{...(process.env.__NEXT_MODERN_BUILD ? { noModule: true } : {})}
/>,
process.env.__NEXT_MODERN_BUILD && (
<script
async
data-next-page="/_app"
src={
assetPrefix +
`/_next/static/${buildId}/pages/_app.module.js` +
_devOnlyInvalidateCacheQueryString
}
key="_app-modern"
nonce={this.props.nonce}
crossOrigin={
this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN
}
type="module"
/>
),
]
return (
<>
{!disableRuntimeJS && buildManifest.devFiles
? buildManifest.devFiles.map(
(file: string) =>
!file.match(/\.js\.map/) && (
<script
key={file}
src={`${assetPrefix}/_next/${encodeURI(
file
)}${_devOnlyInvalidateCacheQueryString}`}
nonce={this.props.nonce}
crossOrigin={
this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN
}
/>
)
)
? buildManifest.devFiles.map((file: string) => (
<script
key={file}
src={`${assetPrefix}/_next/${encodeURI(
file
)}${_devOnlyInvalidateCacheQueryString}`}
nonce={this.props.nonce}
crossOrigin={
this.props.crossOrigin || process.env.__NEXT_CROSS_ORIGIN
}
/>
))
: null}
{disableRuntimeJS ? null : (
<script
......@@ -817,8 +701,6 @@ export class NextScript extends Component<OriginProps> {
/>
) : null}
{!disableRuntimeJS && this.getPolyfillScripts()}
{!disableRuntimeJS && appScript}
{!disableRuntimeJS && page !== '/_error' && pageScript}
{disableRuntimeJS ? null : this.getDynamicChunks()}
{disableRuntimeJS ? null : this.getScripts()}
</>
......@@ -829,13 +711,3 @@ export class NextScript extends Component<OriginProps> {
function getAmpPath(ampPath: string, asPath: string): string {
return ampPath || `${asPath}${asPath.includes('?') ? '&' : '?'}amp=1`
}
function getPageFile(page: string, buildId?: string): string {
const startingUrl =
page === '/'
? '/index'
: /^\/index(\/|$)/.test(page)
? `/index${page}`
: page
return buildId ? `${startingUrl}.${buildId}.js` : `${startingUrl}.js`
}
module.exports = {
experimental: {
modern: true,
},
}
/* eslint-env jest */
import { join } from 'path'
import { readFile } from 'fs-extra'
import { nextBuild } from 'next-test-utils'
jest.setTimeout(1000 * 60 * 1)
const appDir = join(__dirname, '../')
let buildId
describe('Chunking (minimal)', () => {
beforeAll(async () => {
await nextBuild(appDir)
buildId = await readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
})
it('should have an empty client-manifest', async () => {
const manifest = await readFile(
join(appDir, '.next/static', buildId, '_buildManifest.js'),
'utf8'
)
expect(manifest).not.toMatch(/\.js/)
})
it('should have an empty modern client-manifest', async () => {
const manifest = await readFile(
join(appDir, '.next/static', buildId, '_buildManifest.module.js'),
'utf8'
)
expect(manifest).not.toMatch(/\.js/)
})
})
......@@ -562,16 +562,6 @@ describe('Production Usage', () => {
// Let the browser to prefetch the page and error it on the console.
await waitFor(3000)
const browserLogs = await browser.log('browser')
let foundLog = false
browserLogs.forEach((log) => {
if (
log.message.match(/\/no-such-page\.js - Failed to load resource/)
) {
foundLog = true
}
})
expect(foundLog).toBe(true)
// When we go to the 404 page, it'll do a hard reload.
// So, it's possible for the front proxy to load a page from another zone.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment