Commit 3e8b36e8 authored by Alex Castle's avatar Alex Castle Committed by Joe Haddad

Experimental: Granular build chunking (#7696)

* Refactor SplitChunksPlugin configs and add experimental chunking strategy

* Use typeDefs for SplitChunksConfig

* Modify build manifest plugin to create runtime build manifest

* Add support for granular chunks to page-loader

* Ensure normal behavior if experimental granularChunks flag is false

* Update client build manifest to remove iife & implicit global

* Factor out '/_next/' prepending into getDependencies

* Update packages/next/build/webpack-config.ts filepath regex
Co-Authored-By: 's avatarJason Miller <developit@users.noreply.github.com>

* Simplify dependency load ordering in page-loader.js

* Use SHA1 hash to shorten filenames for dependency modules

* Add scheduler to framework cacheGroup in webpack-config

* Update page loader to not duplicate script tags with query parameters

* Ensure no slashes end up in the file hashes

* Add prop-types to framework chunk

* Fix issue with mis-attributed events

* Increase modern build size budget--possibly decrement after consulting with @janicklasralph

* Use module.rawRequest for lib chunks
Co-Authored-By: 's avatarDaniel Stockman <daniel.stockman@gmail.com>

* Dasherize lib chunk names
Co-Authored-By: 's avatarDaniel Stockman <daniel.stockman@gmail.com>

* Fix typescript errors, reorganize lib name logic

* Dasherize rawRequest, short circuit name logic when rawRequest found

* Add `scheduler` package to test regex

* Fix a nit

* Adjust build manifest plugin

* Shorten key name

* Extract createPreloadLink helper

* Extract getDependencies helper

* Move method

* Minimize diff

* Minimize diff x2

* Fix Array.from polyfill

* Simplify page loader code

* Remove async=false for script tags

* Code golf `getDependencies` implementation

* Require lib chunks be in node_modules

* Update packages/next/build/webpack-config.ts
Co-Authored-By: 's avatarJoe Haddad <timer150@gmail.com>

* Replace remaining missed windows compat regex

* Trim client manifest

* Prevent duplicate link preload tags

* Revert size test changes

* Squash manifest size even further

* Add comment for clarity

* Code golfing 🏌️‍♂️

* Correctly select modern dependencies

* Ship separate modern client manifest when module/module enabled

* Update packages/next/build/webpack/plugins/build-manifest-plugin.ts
Co-Authored-By: 's avatarJoe Haddad <timer150@gmail.com>

* Remove unneccessary filter from page-loader

* Add lookbehind to file extension regex in page-loader

* v9.0.3

* Update examples for Apollo with AppTree (#8180)

* Update examples for Apollo with AppTree

* Fix apolloClient being overwritten when rendering AppTree

* Golf page-loader (#8190)

* Remove lookbehind for module replacement

* Wait for build manifest promise before page load or prefetch

* Updating modern-only chunks inside the right entry point

* Fixing ts errors

* Rename variable

* Revert "Wait for build manifest promise before page load or prefetch"

This reverts commit c370528c6888ba7fa71162a0854534ed280224ef.

* Use proper typedef for webpack chunk

* Re-enable promisified client build manifest

* Fix bug in getDependencies map

* Insert check for granularChunks in page-loader

* Increase size limit temporarily for granular chunks

* Add 50ms delay to flaky test

* Set env.__NEXT_GRANULAR_CHUNKS in webpack config

* Reset size limit to 187

* Set process.env.__NEXT_GRANULAR_CHUNKS to false if selectivePageBuilding

* Update test/integration/production/test/index.test.js
Co-Authored-By: 's avatarJoe Haddad <timer150@gmail.com>

* Do not create promise if not using chunking PR
parent fa5be497
...@@ -40,6 +40,7 @@ const defaultConfig: { [key: string]: any } = { ...@@ -40,6 +40,7 @@ const defaultConfig: { [key: string]: any } = {
flyingShuttle: false, flyingShuttle: false,
asyncToPromises: false, asyncToPromises: false,
documentMiddleware: false, documentMiddleware: false,
granularChunks: false,
publicDirectory: false, publicDirectory: false,
modern: false, modern: false,
}, },
......
...@@ -8,6 +8,7 @@ import { ...@@ -8,6 +8,7 @@ import {
} from 'next-server/constants' } from 'next-server/constants'
import resolve from 'next/dist/compiled/resolve/index.js' import resolve from 'next/dist/compiled/resolve/index.js'
import path from 'path' import path from 'path'
import crypto from 'crypto'
import webpack from 'webpack' import webpack from 'webpack'
import { import {
...@@ -219,6 +220,72 @@ export default async function getBaseWebpackConfig( ...@@ -219,6 +220,72 @@ export default async function getBaseWebpackConfig(
}, },
}, },
}, },
prodGranular: {
chunks: 'all',
cacheGroups: {
default: false,
vendors: false,
framework: {
name: 'framework',
chunks: 'all',
test: /[\\/]node_modules[\\/](react|react-dom|scheduler|prop-types)[\\/]/,
priority: 40,
},
lib: {
test(module: { size: Function; identifier: Function }): boolean {
return (
module.size() > 160000 &&
/node_modules[/\\]/.test(module.identifier())
)
},
name(module: { identifier: Function; rawRequest: string }): string {
const rawRequest =
module.rawRequest &&
module.rawRequest.replace(/^@(\w+)[/\\]/, '$1-')
if (rawRequest) return rawRequest
const identifier = module.identifier()
const trimmedIdentifier = /(?:^|[/\\])node_modules[/\\](.*)/.exec(
identifier
)
const processedIdentifier =
trimmedIdentifier &&
trimmedIdentifier[1].replace(/^@(\w+)[/\\]/, '$1-')
return processedIdentifier || identifier
},
priority: 30,
minChunks: 1,
reuseExistingChunk: true,
},
commons: {
name: 'commons',
chunks: 'all',
minChunks: totalPages,
priority: 20,
},
shared: {
name(module, chunks) {
return crypto
.createHash('sha1')
.update(
chunks.reduce(
(acc: string, chunk: webpack.compilation.Chunk) => {
return acc + chunk.name
},
''
)
)
.digest('base64')
.replace(/\//g, '')
},
priority: 10,
minChunks: 2,
reuseExistingChunk: true,
},
},
maxInitialRequests: 20,
},
} }
// Select appropriate SplitChunksPlugin config for this build // Select appropriate SplitChunksPlugin config for this build
...@@ -228,7 +295,9 @@ export default async function getBaseWebpackConfig( ...@@ -228,7 +295,9 @@ export default async function getBaseWebpackConfig(
} else if (selectivePageBuilding) { } else if (selectivePageBuilding) {
splitChunksConfig = splitChunksConfigs.selective splitChunksConfig = splitChunksConfigs.selective
} else { } else {
splitChunksConfig = splitChunksConfigs.prod splitChunksConfig = config.experimental.granularChunks
? splitChunksConfigs.prodGranular
: splitChunksConfigs.prod
} }
const crossOrigin = const crossOrigin =
...@@ -482,6 +551,8 @@ export default async function getBaseWebpackConfig( ...@@ -482,6 +551,8 @@ export default async function getBaseWebpackConfig(
config.exportTrailingSlash config.exportTrailingSlash
), ),
'process.env.__NEXT_MODERN_BUILD': config.experimental.modern && !dev, 'process.env.__NEXT_MODERN_BUILD': config.experimental.modern && !dev,
'process.env.__NEXT_GRANULAR_CHUNKS':
config.experimental.granularChunks && !selectivePageBuilding && !dev,
...(isServer ...(isServer
? { ? {
// Allow browser-only code to be eliminated // Allow browser-only code to be eliminated
...@@ -577,7 +648,12 @@ export default async function getBaseWebpackConfig( ...@@ -577,7 +648,12 @@ export default async function getBaseWebpackConfig(
isServer && isServer &&
new NextJsSSRModuleCachePlugin({ outputPath }), new NextJsSSRModuleCachePlugin({ outputPath }),
isServer && new NextJsSsrImportPlugin(), isServer && new NextJsSsrImportPlugin(),
!isServer && new BuildManifestPlugin(), !isServer &&
new BuildManifestPlugin({
buildId,
clientManifest: config.experimental.granularChunks,
modern: config.experimental.modern,
}),
config.experimental.profiling && config.experimental.profiling &&
new webpack.debug.ProfilingPlugin({ new webpack.debug.ProfilingPlugin({
outputPath: path.join( outputPath: path.join(
......
...@@ -4,21 +4,65 @@ import { ...@@ -4,21 +4,65 @@ import {
BUILD_MANIFEST, BUILD_MANIFEST,
ROUTE_NAME_REGEX, ROUTE_NAME_REGEX,
IS_BUNDLED_PAGE_REGEX, IS_BUNDLED_PAGE_REGEX,
CLIENT_STATIC_FILES_PATH,
CLIENT_STATIC_FILES_RUNTIME_MAIN, CLIENT_STATIC_FILES_RUNTIME_MAIN,
} from 'next-server/constants' } from 'next-server/constants'
interface AssetMap {
devFiles: string[]
pages: {
'/_app': string[]
[s: string]: string[]
}
}
// This function takes the asset map generated in BuildManifestPlugin and creates a
// reduced version to send to the client.
const generateClientManifest = (
assetMap: AssetMap,
isModern: boolean
): string => {
const clientManifest: { [s: string]: string[] } = {}
const appDependencies = new Set(assetMap.pages['/_app'])
delete assetMap.pages['/_app']
Object.entries(assetMap.pages).forEach(([page, dependencies]) => {
// Filter out dependencies in the _app entry, because those will have already
// been loaded by the client prior to a navigation event
const filteredDeps = dependencies.filter(
dep => !appDependencies.has(dep) && /\.module\.js$/.test(dep) === isModern
)
// The manifest can omit the page if it has no requirements
if (filteredDeps.length) {
clientManifest[page] = filteredDeps
}
})
return JSON.stringify(clientManifest)
}
// This plugin creates a build-manifest.json for all assets that are being output // This plugin creates a build-manifest.json for all assets that are being output
// It has a mapping of "entry" filename to real filename. Because the real filename can be hashed in production // It has a mapping of "entry" filename to real filename. Because the real filename can be hashed in production
export default class BuildManifestPlugin { export default class BuildManifestPlugin {
private buildId: string
private clientManifest: boolean
private modern: boolean
constructor(options: {
buildId: string
clientManifest: boolean
modern: boolean
}) {
this.buildId = options.buildId
this.clientManifest = options.clientManifest
this.modern = options.modern
}
apply(compiler: Compiler) { apply(compiler: Compiler) {
compiler.hooks.emit.tapAsync( compiler.hooks.emit.tapAsync(
'NextJsBuildManifest', 'NextJsBuildManifest',
(compilation, callback) => { (compilation, callback) => {
const { chunks } = compilation const { chunks } = compilation
const assetMap: { const assetMap: AssetMap = { devFiles: [], pages: { '/_app': [] } }
devFiles: string[]
pages: { [page: string]: string[] }
} = { devFiles: [], pages: {} }
const mainJsChunk = chunks.find( const mainJsChunk = chunks.find(
c => c.name === CLIENT_STATIC_FILES_RUNTIME_MAIN c => c.name === CLIENT_STATIC_FILES_RUNTIME_MAIN
...@@ -84,6 +128,22 @@ export default class BuildManifestPlugin { ...@@ -84,6 +128,22 @@ export default class BuildManifestPlugin {
assetMap.pages['/'] = assetMap.pages['/index'] assetMap.pages['/'] = assetMap.pages['/index']
} }
// Add the runtime build manifest file (generated later in this file)
// as a dependency for the app. If the flag is false, the file won't be
// downloaded by the client.
if (this.clientManifest) {
assetMap.pages['/_app'].push(
`${CLIENT_STATIC_FILES_PATH}/${this.buildId}/_buildManifest.js`
)
if (this.modern) {
assetMap.pages['/_app'].push(
`${CLIENT_STATIC_FILES_PATH}/${
this.buildId
}/_buildManifest.module.js`
)
}
}
assetMap.pages = Object.keys(assetMap.pages) assetMap.pages = Object.keys(assetMap.pages)
.sort() .sort()
// eslint-disable-next-line // eslint-disable-next-line
...@@ -92,6 +152,34 @@ export default class BuildManifestPlugin { ...@@ -92,6 +152,34 @@ export default class BuildManifestPlugin {
compilation.assets[BUILD_MANIFEST] = new RawSource( compilation.assets[BUILD_MANIFEST] = new RawSource(
JSON.stringify(assetMap, null, 2) JSON.stringify(assetMap, null, 2)
) )
if (this.clientManifest) {
const clientManifestPath = `${CLIENT_STATIC_FILES_PATH}/${
this.buildId
}/_buildManifest.js`
compilation.assets[clientManifestPath] = new RawSource(
`self.__BUILD_MANIFEST = JSON.parse('${generateClientManifest(
assetMap,
false
)}')
self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()`
)
if (this.modern) {
const modernClientManifestPath = `${CLIENT_STATIC_FILES_PATH}/${
this.buildId
}/_buildManifest.module.js`
compilation.assets[modernClientManifestPath] = new RawSource(
`self.__BUILD_MANIFEST = JSON.parse('${generateClientManifest(
assetMap,
true
)}')
self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB()`
)
}
}
callback() callback()
} }
) )
......
...@@ -130,22 +130,41 @@ export default class NextEsmPlugin implements Plugin { ...@@ -130,22 +130,41 @@ export default class NextEsmPlugin implements Plugin {
) )
const childChunkFileMap = childCompilation.chunks.reduce( const childChunkFileMap = childCompilation.chunks.reduce(
(chunkMap: { [key: string]: any[] }, chunk: compilation.Chunk) => { (
chunkMap[chunk.name] = chunk.files chunkMap: { [key: string]: compilation.Chunk },
chunk: compilation.Chunk
) => {
chunkMap[chunk.name] = chunk
return chunkMap return chunkMap
}, },
{} {}
) )
// Merge files from similar chunks
compilation.chunks.forEach((chunk: compilation.Chunk) => { compilation.chunks.forEach((chunk: compilation.Chunk) => {
const childChunkFiles = childChunkFileMap[chunk.name] const childChunk = childChunkFileMap[chunk.name]
if (childChunkFiles) { if (childChunk && childChunk.files) {
delete childChunkFileMap[chunk.name]
chunk.files.push( chunk.files.push(
...childChunkFiles.filter((v: any) => !chunk.files.includes(v)) ...childChunk.files.filter((v: any) => !chunk.files.includes(v))
) )
} }
}) })
// Add modern only chunks
compilation.chunks.push(...Object.values(childChunkFileMap))
// Place modern only chunk inside the right entry point
compilation.entrypoints.forEach((entryPoint, entryPointName) => {
const childEntryPoint = childCompilation.entrypoints.get(entryPointName)
childEntryPoint.chunks.forEach((chunk: compilation.Chunk) => {
if (childChunkFileMap.hasOwnProperty(chunk.name)) {
entryPoint.chunks.push(chunk)
}
})
})
} }
async runBuild(compiler: Compiler, compilation: compilation.Compilation) { async runBuild(compiler: Compiler, compilation: compilation.Compilation) {
......
/* global document */ /* global document, window */
import mitt from 'next-server/dist/lib/mitt' import mitt from 'next-server/dist/lib/mitt'
import unfetch from 'unfetch' import unfetch from 'unfetch'
...@@ -27,10 +27,27 @@ export default class PageLoader { ...@@ -27,10 +27,27 @@ export default class PageLoader {
this.assetPrefix = assetPrefix this.assetPrefix = assetPrefix
this.pageCache = {} this.pageCache = {}
this.prefetchCache = new Set()
this.pageRegisterEvents = mitt() this.pageRegisterEvents = mitt()
this.loadingRoutes = {} this.loadingRoutes = {}
this.promisedBuildId = Promise.resolve() this.promisedBuildId = Promise.resolve()
if (process.env.__NEXT_GRANULAR_CHUNKS) {
this.promisedBuildManifest = new Promise(resolve => {
if (window.__BUILD_MANIFEST) {
resolve(window.__BUILD_MANIFEST)
} else {
window.__BUILD_MANIFEST_CB = () => {
resolve(window.__BUILD_MANIFEST)
}
}
})
}
}
// Returns a promise for the dependencies for a particular route
getDependencies (route) {
return this.promisedBuildManifest.then(
man => (man[route] && man[route].map(url => `/_next/${url}`)) || []
)
} }
normalizeRoute (route) { normalizeRoute (route) {
...@@ -75,10 +92,21 @@ export default class PageLoader { ...@@ -75,10 +92,21 @@ export default class PageLoader {
return return
} }
// Load the script if not asked to load yet.
if (!this.loadingRoutes[route]) { if (!this.loadingRoutes[route]) {
this.loadScript(route) if (process.env.__NEXT_GRANULAR_CHUNKS) {
this.loadingRoutes[route] = true this.getDependencies(route).then(deps => {
deps.forEach(d => {
if (!document.querySelector(`script[src^="${d}"]`)) {
this.loadScript(d, route, false)
}
})
this.loadRoute(route)
this.loadingRoutes[route] = true
})
} else {
this.loadRoute(route)
this.loadingRoutes[route] = true
}
} }
}) })
} }
...@@ -112,22 +140,26 @@ export default class PageLoader { ...@@ -112,22 +140,26 @@ export default class PageLoader {
}) })
} }
async loadScript (route) { async loadRoute (route) {
await this.promisedBuildId await this.promisedBuildId
route = this.normalizeRoute(route) route = this.normalizeRoute(route)
let scriptRoute = route === '/' ? '/index.js' : `${route}.js` let scriptRoute = route === '/' ? '/index.js' : `${route}.js`
const script = document.createElement('script') const url = `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${scriptRoute}`
this.loadScript(url, route, true)
}
loadScript (url, route, isPage) {
const script = document.createElement('script')
if (process.env.__NEXT_MODERN_BUILD && 'noModule' in script) { if (process.env.__NEXT_MODERN_BUILD && 'noModule' in script) {
script.type = 'module' script.type = 'module'
scriptRoute = scriptRoute.replace(/\.js$/, '.module.js') // Only page bundle scripts need to have .module added to url,
// dependencies already have it added during build manifest creation
if (isPage) url = url.replace(/\.js$/, '.module.js')
} }
const url = `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${scriptRoute}`
script.crossOrigin = process.crossOrigin script.crossOrigin = process.crossOrigin
script.src = url script.src = url
script.onerror = () => { script.onerror = () => {
...@@ -135,7 +167,6 @@ export default class PageLoader { ...@@ -135,7 +167,6 @@ export default class PageLoader {
error.code = 'PAGE_LOAD_ERROR' error.code = 'PAGE_LOAD_ERROR'
this.pageRegisterEvents.emit(route, { error }) this.pageRegisterEvents.emit(route, { error })
} }
document.body.appendChild(script) document.body.appendChild(script)
} }
...@@ -174,7 +205,9 @@ export default class PageLoader { ...@@ -174,7 +205,9 @@ export default class PageLoader {
register() register()
} }
async prefetch (route) { async prefetch (route, isDependency) {
await this.promisedBuildId
route = this.normalizeRoute(route) route = this.normalizeRoute(route)
let scriptRoute = `${route === '/' ? '/index' : route}.js` let scriptRoute = `${route === '/' ? '/index' : route}.js`
...@@ -184,14 +217,20 @@ export default class PageLoader { ...@@ -184,14 +217,20 @@ export default class PageLoader {
) { ) {
scriptRoute = scriptRoute.replace(/\.js$/, '.module.js') scriptRoute = scriptRoute.replace(/\.js$/, '.module.js')
} }
const url = isDependency
? route
: `${this.assetPrefix}/_next/static/${encodeURIComponent(
this.buildId
)}/pages${scriptRoute}`
// n.b. If preload is not supported, we fall back to `loadPage` which has
// its own deduping mechanism.
if ( if (
this.prefetchCache.has(scriptRoute) || document.querySelector(`link[rel="preload"][href^="${url}"]`) ||
document.getElementById(`__NEXT_PAGE__${route}`) document.getElementById(`__NEXT_PAGE__${route}`)
) { ) {
return return
} }
this.prefetchCache.add(scriptRoute)
// Inspired by quicklink, license: https://github.com/GoogleChromeLabs/quicklink/blob/master/LICENSE // Inspired by quicklink, license: https://github.com/GoogleChromeLabs/quicklink/blob/master/LICENSE
let cn let cn
...@@ -202,17 +241,23 @@ export default class PageLoader { ...@@ -202,17 +241,23 @@ export default class PageLoader {
} }
} }
if (process.env.__NEXT_GRANULAR_CHUNKS && !isDependency) {
;(await this.getDependencies(route)).forEach(url => {
this.prefetch(url, true)
})
}
// Feature detection is used to see if preload is supported // Feature detection is used to see if preload is supported
// If not fall back to loading script tags before the page is loaded // If not fall back to loading script tags before the page is loaded
// https://caniuse.com/#feat=link-rel-preload // https://caniuse.com/#feat=link-rel-preload
if (hasPreload) { if (hasPreload) {
await this.promisedBuildId preloadScript(url)
return
}
preloadScript( if (isDependency) {
`${this.assetPrefix}/_next/static/${encodeURIComponent( // loadPage will automatically handle depencies, so no need to
this.buildId // preload them manually
)}/pages${scriptRoute}`
)
return return
} }
......
...@@ -417,7 +417,9 @@ describe('Production Usage', () => { ...@@ -417,7 +417,9 @@ describe('Production Usage', () => {
if (browserName === 'chrome') { if (browserName === 'chrome') {
it('should add preload tags when Link prefetch prop is used', async () => { it('should add preload tags when Link prefetch prop is used', async () => {
const browser = await webdriver(appPort, '/prefetch') const browser = await webdriver(appPort, '/prefetch')
await waitFor(2000)
const elements = await browser.elementsByCss('link[rel=preload]') const elements = await browser.elementsByCss('link[rel=preload]')
expect(elements.length).toBe(9) expect(elements.length).toBe(9)
await Promise.all( await Promise.all(
elements.map(async element => { elements.map(async element => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment