Commit 8cd21a63 authored by Joe Haddad's avatar Joe Haddad Committed by Tim Neutkens

Server file inclusive Flying Shuttle (#7128)

* Add Flying Shuttle plugin to server compilation

* Ignore build artifacts in flying shuttle manifest

* Add comments explaining what's going on

* Emit Shuttle manifest after both compilations
parent 3fa04620
import chalk from 'chalk' import chalk from 'chalk'
import { PHASE_PRODUCTION_BUILD, BLOCKED_PAGES } from 'next-server/constants' import {
CHUNK_GRAPH_MANIFEST,
PHASE_PRODUCTION_BUILD,
} from 'next-server/constants'
import loadConfig from 'next-server/next-config' import loadConfig from 'next-server/next-config'
import nanoid from 'next/dist/compiled/nanoid/index.js' import nanoid from 'next/dist/compiled/nanoid/index.js'
import path from 'path' import path from 'path'
import fs from 'fs'
import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages' import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages'
import { recursiveDelete } from '../lib/recursive-delete' import { recursiveDelete } from '../lib/recursive-delete'
...@@ -20,6 +22,7 @@ import { ...@@ -20,6 +22,7 @@ import {
printTreeView, printTreeView,
} from './utils' } from './utils'
import getBaseWebpackConfig from './webpack-config' import getBaseWebpackConfig from './webpack-config'
import { exportManifest } from './webpack/plugins/chunk-graph-plugin'
import { writeBuildId } from './write-build-id' import { writeBuildId } from './write-build-id'
export default async function build(dir: string, conf = null): Promise<void> { export default async function build(dir: string, conf = null): Promise<void> {
...@@ -149,7 +152,6 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -149,7 +152,6 @@ export default async function build(dir: string, conf = null): Promise<void> {
isServer: false, isServer: false,
config, config,
target: config.target, target: config.target,
selectivePageBuildingCacheIdentifier,
entrypoints: entrypoints.client, entrypoints: entrypoints.client,
selectivePageBuilding, selectivePageBuilding,
}), }),
...@@ -159,7 +161,6 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -159,7 +161,6 @@ export default async function build(dir: string, conf = null): Promise<void> {
isServer: true, isServer: true,
config, config,
target: config.target, target: config.target,
selectivePageBuildingCacheIdentifier,
entrypoints: entrypoints.server, entrypoints: entrypoints.server,
selectivePageBuilding, selectivePageBuilding,
}), }),
...@@ -194,6 +195,12 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -194,6 +195,12 @@ export default async function build(dir: string, conf = null): Promise<void> {
if (isFlyingShuttle) { if (isFlyingShuttle) {
console.log() console.log()
exportManifest({
dir: dir,
fileName: path.join(distDir, CHUNK_GRAPH_MANIFEST),
selectivePageBuildingCacheIdentifier,
})
} }
if (result.errors.length > 0) { if (result.errors.length > 0) {
......
...@@ -7,7 +7,7 @@ import PagesManifestPlugin from './webpack/plugins/pages-manifest-plugin' ...@@ -7,7 +7,7 @@ import PagesManifestPlugin from './webpack/plugins/pages-manifest-plugin'
import BuildManifestPlugin from './webpack/plugins/build-manifest-plugin' import BuildManifestPlugin from './webpack/plugins/build-manifest-plugin'
import ChunkNamesPlugin from './webpack/plugins/chunk-names-plugin' import ChunkNamesPlugin from './webpack/plugins/chunk-names-plugin'
import { ReactLoadablePlugin } from './webpack/plugins/react-loadable-plugin' import { ReactLoadablePlugin } from './webpack/plugins/react-loadable-plugin'
import { SERVER_DIRECTORY, REACT_LOADABLE_MANIFEST, CHUNK_GRAPH_MANIFEST, CLIENT_STATIC_FILES_RUNTIME_WEBPACK, CLIENT_STATIC_FILES_RUNTIME_MAIN } from 'next-server/constants' import { SERVER_DIRECTORY, REACT_LOADABLE_MANIFEST, CLIENT_STATIC_FILES_RUNTIME_WEBPACK, CLIENT_STATIC_FILES_RUNTIME_MAIN } from 'next-server/constants'
import { NEXT_PROJECT_ROOT, NEXT_PROJECT_ROOT_DIST_CLIENT, PAGES_DIR_ALIAS, DOT_NEXT_ALIAS } from '../lib/constants' import { NEXT_PROJECT_ROOT, NEXT_PROJECT_ROOT_DIST_CLIENT, PAGES_DIR_ALIAS, DOT_NEXT_ALIAS } from '../lib/constants'
import {TerserPlugin} from './webpack/plugins/terser-webpack-plugin/src/index' import {TerserPlugin} from './webpack/plugins/terser-webpack-plugin/src/index'
import { ServerlessPlugin } from './webpack/plugins/serverless-plugin' import { ServerlessPlugin } from './webpack/plugins/serverless-plugin'
...@@ -20,7 +20,7 @@ import { importAutoDllPlugin } from './webpack/plugins/dll-import' ...@@ -20,7 +20,7 @@ import { importAutoDllPlugin } from './webpack/plugins/dll-import'
import { WebpackEntrypoints } from './entries' import { WebpackEntrypoints } from './entries'
type ExcludesFalse = <T>(x: T | false) => x is T type ExcludesFalse = <T>(x: T | false) => x is T
export default async function getBaseWebpackConfig (dir: string, {dev = false, debug = false, isServer = false, buildId, config, target = 'server', entrypoints, selectivePageBuilding = false, selectivePageBuildingCacheIdentifier = ''}: {dev?: boolean, debug?: boolean, isServer?: boolean, buildId: string, config: any, target?: string, entrypoints: WebpackEntrypoints, selectivePageBuilding?: boolean, selectivePageBuildingCacheIdentifier?: string}): Promise<webpack.Configuration> { export default async function getBaseWebpackConfig (dir: string, {dev = false, debug = false, isServer = false, buildId, config, target = 'server', entrypoints, selectivePageBuilding = false}: {dev?: boolean, debug?: boolean, isServer?: boolean, buildId: string, config: any, target?: string, entrypoints: WebpackEntrypoints, selectivePageBuilding?: boolean}): Promise<webpack.Configuration> {
const distDir = path.join(dir, config.distDir) const distDir = path.join(dir, config.distDir)
const defaultLoaders = { const defaultLoaders = {
babel: { babel: {
...@@ -281,7 +281,9 @@ export default async function getBaseWebpackConfig (dir: string, {dev = false, d ...@@ -281,7 +281,9 @@ export default async function getBaseWebpackConfig (dir: string, {dev = false, d
!isServer && new ReactLoadablePlugin({ !isServer && new ReactLoadablePlugin({
filename: REACT_LOADABLE_MANIFEST filename: REACT_LOADABLE_MANIFEST
}), }),
!isServer && selectivePageBuilding && new ChunkGraphPlugin(buildId, path.resolve(dir), { filename: CHUNK_GRAPH_MANIFEST, selectivePageBuildingCacheIdentifier }), selectivePageBuilding && new ChunkGraphPlugin(buildId, {
dir, distDir, isServer
}),
!isServer && new DropClientPage(), !isServer && new DropClientPage(),
...(dev ? (() => { ...(dev ? (() => {
// Even though require.cache is server only we have to clear assets from both compilations // Even though require.cache is server only we have to clear assets from both compilations
......
...@@ -6,6 +6,56 @@ import path from 'path' ...@@ -6,6 +6,56 @@ import path from 'path'
import { parse } from 'querystring' import { parse } from 'querystring'
import { Compiler, Plugin } from 'webpack' import { Compiler, Plugin } from 'webpack'
type StringDictionary = { [pageName: string]: string[] }
const manifest: {
pages: StringDictionary
pageChunks: StringDictionary
chunks: StringDictionary
} = {
pages: {},
pageChunks: {},
chunks: {},
}
export function exportManifest({
dir,
fileName,
selectivePageBuildingCacheIdentifier,
}: {
dir: string
fileName: string
selectivePageBuildingCacheIdentifier: string
}) {
const finalManifest = {
...manifest,
hashes: {} as { [pageName: string]: string },
}
const allFiles = new Set<string>()
for (const page of Object.keys(finalManifest.pages)) {
finalManifest.pages[page].forEach(f => allFiles.add(f))
}
finalManifest.hashes = [...allFiles].sort().reduce(
(acc, cur) =>
Object.assign(
acc,
fs.existsSync(path.join(dir, cur))
? {
[cur]: createHash('sha1')
.update(selectivePageBuildingCacheIdentifier)
.update(fs.readFileSync(path.join(dir, cur)))
.digest('hex'),
}
: undefined
),
{}
)
const json = JSON.stringify(finalManifest, null, 2) + EOL
fs.writeFileSync(fileName, json)
}
function getFiles(dir: string, modules: any[]): string[] { function getFiles(dir: string, modules: any[]): string[] {
if (!(modules && modules.length)) { if (!(modules && modules.length)) {
return [] return []
...@@ -49,45 +99,34 @@ function getFiles(dir: string, modules: any[]): string[] { ...@@ -49,45 +99,34 @@ function getFiles(dir: string, modules: any[]): string[] {
export class ChunkGraphPlugin implements Plugin { export class ChunkGraphPlugin implements Plugin {
private buildId: string private buildId: string
private dir: string private dir: string
private filename: string private distDir: string
private selectivePageBuildingCacheIdentifier: string private isServer: boolean
constructor( constructor(
buildId: string, buildId: string,
dir: string,
{ {
filename, dir,
selectivePageBuildingCacheIdentifier, distDir,
}: { filename?: string; selectivePageBuildingCacheIdentifier?: string } = {} isServer,
}: {
dir: string
distDir: string
isServer: boolean
}
) { ) {
this.buildId = buildId this.buildId = buildId
this.dir = dir this.dir = dir
this.filename = filename || 'chunk-graph-manifest.json' this.distDir = distDir
this.selectivePageBuildingCacheIdentifier = this.isServer = isServer
selectivePageBuildingCacheIdentifier || ''
} }
apply(compiler: Compiler) { apply(compiler: Compiler) {
const { dir } = this const { dir } = this
compiler.hooks.emit.tap('ChunkGraphPlugin', compilation => { compiler.hooks.emit.tap('ChunkGraphPlugin', compilation => {
type StringDictionary = { [pageName: string]: string[] }
const manifest: {
pages: StringDictionary
pageChunks: StringDictionary
chunks: StringDictionary
hashes: { [pageName: string]: string }
} = {
pages: {},
pageChunks: {},
chunks: {},
hashes: {},
}
const sharedFiles = [] as string[] const sharedFiles = [] as string[]
const sharedChunks = [] as string[] const sharedChunks = [] as string[]
const pages: StringDictionary = {} const pages: StringDictionary = {}
const pageChunks: StringDictionary = {} const pageChunks: StringDictionary = {}
const allFiles = new Set()
compilation.chunks.forEach(chunk => { compilation.chunks.forEach(chunk => {
if (!chunk.hasEntryModule()) { if (!chunk.hasEntryModule()) {
...@@ -118,11 +157,15 @@ export class ChunkGraphPlugin implements Plugin { ...@@ -118,11 +157,15 @@ export class ChunkGraphPlugin implements Plugin {
const modules = [...chunkModules.values()] const modules = [...chunkModules.values()]
const files = getFiles(dir, modules) const files = getFiles(dir, modules)
// we don't care about node_modules (yet) because we invalidate the
// entirety of flying shuttle on package changes
.filter(val => !val.includes('node_modules')) .filter(val => !val.includes('node_modules'))
// build artifacts shouldn't be considered, so we ensure all paths
// are outside of this directory
.filter(val => path.relative(this.distDir, val).startsWith('..'))
// convert from absolute path to be portable across operating systems
// and directories
.map(f => path.relative(dir, f)) .map(f => path.relative(dir, f))
.sort()
files.forEach(f => allFiles.add(f))
let pageName: string | undefined let pageName: string | undefined
if (chunk.entryModule && chunk.entryModule.loaders) { if (chunk.entryModule && chunk.entryModule.loaders) {
...@@ -133,8 +176,7 @@ export class ChunkGraphPlugin implements Plugin { ...@@ -133,8 +176,7 @@ export class ChunkGraphPlugin implements Plugin {
}: { }: {
loader?: string | null loader?: string | null
options?: string | null options?: string | null
}) => }) => loader && loader.match(/next-(\w+-)+loader/) && options
loader && loader.includes('next-client-pages-loader') && options
) )
if (entryLoader) { if (entryLoader) {
const { page } = parse(entryLoader.options) const { page } = parse(entryLoader.options)
...@@ -161,7 +203,9 @@ export class ChunkGraphPlugin implements Plugin { ...@@ -161,7 +203,9 @@ export class ChunkGraphPlugin implements Plugin {
sharedFiles.push(...files) sharedFiles.push(...files)
sharedChunks.push(...involvedChunks) sharedChunks.push(...involvedChunks)
} else { } else {
manifest.chunks[chunk.name] = files manifest.chunks[chunk.name] = [
...new Set([...(manifest.chunks[chunk.name] || []), ...files]),
].sort()
} }
} }
}) })
...@@ -174,41 +218,26 @@ export class ChunkGraphPlugin implements Plugin { ...@@ -174,41 +218,26 @@ export class ChunkGraphPlugin implements Plugin {
: name : name
for (const page in pages) { for (const page in pages) {
manifest.pages[page] = [...pages[page], ...sharedFiles] manifest.pages[page] = [
manifest.pageChunks[page] = [
...new Set([ ...new Set([
...pageChunks[page], ...(manifest.pages[page] || []),
...pageChunks[page].map(getLambdaChunk), ...pages[page],
...sharedChunks, ...sharedFiles,
...sharedChunks.map(getLambdaChunk),
]), ]),
].sort() ].sort()
}
manifest.hashes = ([...allFiles] as string[]).sort().reduce( // There's no chunks to save from serverless bundles
(acc, cur) => if (!this.isServer) {
Object.assign( manifest.pageChunks[page] = [
acc, ...new Set([
fs.existsSync(path.join(dir, cur)) ...(manifest.pageChunks[page] || []),
? { ...pageChunks[page],
[cur]: createHash('sha1') ...pageChunks[page].map(getLambdaChunk),
.update(this.selectivePageBuildingCacheIdentifier) ...sharedChunks,
.update(fs.readFileSync(path.join(dir, cur))) ...sharedChunks.map(getLambdaChunk),
.digest('hex'), ]),
} ].sort()
: undefined }
),
{}
)
const json = JSON.stringify(manifest, null, 2) + EOL
compilation.assets[this.filename] = {
source() {
return json
},
size() {
return json.length
},
} }
}) })
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment