Unverified Commit c2d0a4ef authored by Joe Haddad's avatar Joe Haddad Committed by GitHub

Add Flying Shuttle v1 (#6991)

* Add Flying Shuttle v1

* Add serverless file restoration

* Add another message

* Update test

* Resolve page version last

* Print number of changed pages

* Remove old tests

* Re-add head build ID writing

* Change syntax appearance

* Start adding test components

* Fix react loadable manifest

* Complicate dynamic testing

* Fix first build missing head build id

* Remove flying shuttle tests
parent 65fcf351
import Sema from 'async-sema'
import crypto from 'crypto'
import findCacheDir from 'find-cache-dir'
import fs from 'fs'
import mkdirpModule from 'mkdirp'
import { CHUNK_GRAPH_MANIFEST } from 'next-server/constants'
import { EOL } from 'os'
import path from 'path'
import { promisify } from 'util'
import { recursiveDelete } from '../lib/recursive-delete'
import * as Log from './output/log'
const FILE_BUILD_ID = 'HEAD_BUILD_ID'
const DIR_FILES_NAME = 'files'
const mkdirp = promisify(mkdirpModule)
const fsExists = promisify(fs.exists)
const fsReadFile = promisify(fs.readFile)
const fsWriteFile = promisify(fs.writeFile)
const fsCopyFile = promisify(fs.copyFile)
type ChunkGraphManifest = {
pages: { [page: string]: string[] }
pageChunks: { [page: string]: string[] }
chunks: { [page: string]: string[] }
hashes: { [page: string]: string }
}
export class FlyingShuttle {
private shuttleDirectory: string
private buildId: string
private pagesDirectory: string
private distDirectory: string
private _shuttleBuildId: string | undefined
private _restoreSema = new Sema(1)
private _recalledManifest: ChunkGraphManifest = {
pages: {},
pageChunks: {},
chunks: {},
hashes: {},
}
constructor({
buildId,
pagesDirectory,
distDirectory,
}: {
buildId: string
pagesDirectory: string
distDirectory: string
}) {
this.shuttleDirectory = findCacheDir({
name: 'next-flying-shuttle',
create: true,
})!
this.buildId = buildId
this.pagesDirectory = pagesDirectory
this.distDirectory = distDirectory
}
hasShuttle = async () => {
const found =
this.shuttleBuildId &&
(await fsExists(path.join(this.shuttleDirectory, CHUNK_GRAPH_MANIFEST)))
if (found) {
Log.info('flying shuttle is docked')
} else {
Log.info('could not locate flying shuttle')
}
return found
}
get shuttleBuildId() {
if (this._shuttleBuildId) {
return this._shuttleBuildId
}
const headBuildIdPath = path.join(this.shuttleDirectory, FILE_BUILD_ID)
if (!fs.existsSync(headBuildIdPath)) {
return (this._shuttleBuildId = undefined)
}
const contents = fs.readFileSync(headBuildIdPath, 'utf8').trim()
return (this._shuttleBuildId = contents)
}
getUnchangedPages = async () => {
const manifestPath = path.join(this.shuttleDirectory, CHUNK_GRAPH_MANIFEST)
const manifest = require(manifestPath) as ChunkGraphManifest
const { pages: pageFileDictionary, hashes } = manifest
const pageNames = Object.keys(pageFileDictionary)
const allFiles = new Set()
pageNames.forEach(pageName =>
pageFileDictionary[pageName].forEach(file => allFiles.add(file))
)
const fileChanged = new Map()
await Promise.all(
[...allFiles].map(async file => {
const filePath = path.join(path.dirname(this.pagesDirectory), file)
const exists = await fsExists(filePath)
if (!exists) {
fileChanged.set(file, true)
return
}
// TODO: hash needs to include Next.js environment variables and packages
const hash = crypto
.createHash('sha1')
.update(await fsReadFile(filePath))
.digest('hex')
fileChanged.set(file, hash !== hashes[file])
})
)
const unchangedPages = pageNames
.filter(
p => !pageFileDictionary[p].map(f => fileChanged.get(f)).some(Boolean)
)
.filter(
pageName =>
pageName !== '/_app' &&
pageName !== '/_error' &&
pageName !== '/_document'
)
if (unchangedPages.length) {
const u = unchangedPages.length
const c = pageNames.length - u
Log.info(`found ${c} changed and ${u} unchanged page${u > 1 ? 's' : ''}`)
} else {
Log.warn(`flying shuttle had no pages we can reuse`)
}
return unchangedPages
}
restorePage = async (page: string): Promise<boolean> => {
await this._restoreSema.acquire()
try {
const manifestPath = path.join(
this.shuttleDirectory,
CHUNK_GRAPH_MANIFEST
)
const manifest = require(manifestPath) as ChunkGraphManifest
const { pages, pageChunks, hashes } = manifest
if (!(pages.hasOwnProperty(page) && pageChunks.hasOwnProperty(page))) {
Log.warn(`unable to find ${page} in shuttle`)
return false
}
const serverless = path.join(
'serverless/pages',
`${page === '/' ? 'index' : page}.js`
)
const files = [serverless, ...pageChunks[page]]
const filesExists = await Promise.all(
files
.map(f => path.join(this.shuttleDirectory, DIR_FILES_NAME, f))
.map(f => fsExists(f))
)
if (!filesExists.every(Boolean)) {
Log.warn(`unable to locate files for ${page} in shuttle`)
return false
}
const rewriteRegex = new RegExp(`${this.shuttleBuildId}[\\/\\\\]`)
const movedPageChunks: string[] = []
await Promise.all(
files.map(async recallFileName => {
if (!rewriteRegex.test(recallFileName)) {
const recallPath = path.join(this.distDirectory, recallFileName)
const recallPathExists = await fsExists(recallPath)
if (!recallPathExists) {
await mkdirp(path.dirname(recallPath))
await fsCopyFile(
path.join(
this.shuttleDirectory,
DIR_FILES_NAME,
recallFileName
),
recallPath
)
}
movedPageChunks.push(recallFileName)
return
}
const newFileName = recallFileName.replace(
rewriteRegex,
`${this.buildId}/`
)
const recallPath = path.join(this.distDirectory, newFileName)
const recallPathExists = await fsExists(recallPath)
if (!recallPathExists) {
await mkdirp(path.dirname(recallPath))
await fsCopyFile(
path.join(this.shuttleDirectory, DIR_FILES_NAME, recallFileName),
recallPath
)
}
movedPageChunks.push(newFileName)
})
)
this._recalledManifest.pages[page] = pages[page]
this._recalledManifest.pageChunks[page] = movedPageChunks.filter(
f => f !== serverless
)
this._recalledManifest.hashes = Object.assign(
{},
this._recalledManifest.hashes,
pages[page].reduce(
(acc, cur) => Object.assign(acc, { [cur]: hashes[cur] }),
{}
)
)
return true
} finally {
this._restoreSema.release()
}
}
save = async () => {
Log.wait('docking flying shuttle')
await recursiveDelete(this.shuttleDirectory)
await mkdirp(this.shuttleDirectory)
const nextManifestPath = path.join(this.distDirectory, CHUNK_GRAPH_MANIFEST)
if (!(await fsExists(nextManifestPath))) {
Log.warn('could not find shuttle payload :: shuttle will not be docked')
return
}
const nextManifest = JSON.parse(
await fsReadFile(nextManifestPath, 'utf8')
) as ChunkGraphManifest
if (nextManifest.chunks && Object.keys(nextManifest.chunks).length) {
Log.warn('build emitted assets that cannot fit in flying shuttle')
return
}
const storeManifest: ChunkGraphManifest = {
pages: Object.assign(
{},
this._recalledManifest.pages,
nextManifest.pages
),
pageChunks: Object.assign(
{},
this._recalledManifest.pageChunks,
nextManifest.pageChunks
),
chunks: Object.assign(
{},
this._recalledManifest.chunks,
nextManifest.chunks
),
hashes: Object.assign(
{},
this._recalledManifest.hashes,
nextManifest.hashes
),
}
await fsWriteFile(
path.join(this.shuttleDirectory, FILE_BUILD_ID),
this.buildId
)
const usedChunks = new Set()
const pages = Object.keys(storeManifest.pageChunks)
pages.forEach(page => {
storeManifest.pageChunks[page].forEach(file => usedChunks.add(file))
usedChunks.add(
path.join('serverless/pages', `${page === '/' ? 'index' : page}.js`)
)
})
await fsWriteFile(
path.join(this.shuttleDirectory, CHUNK_GRAPH_MANIFEST),
JSON.stringify(storeManifest, null, 2) + EOL
)
await Promise.all(
[...usedChunks].map(async usedChunk => {
const target = path.join(
this.shuttleDirectory,
DIR_FILES_NAME,
usedChunk
)
await mkdirp(path.dirname(target))
return fsCopyFile(path.join(this.distDirectory, usedChunk), target)
})
)
Log.info(`flying shuttle payload: ${usedChunks.size + 2} files`)
Log.ready('flying shuttle docked')
}
}
...@@ -5,11 +5,18 @@ import nanoid from 'next/dist/compiled/nanoid/index.js' ...@@ -5,11 +5,18 @@ import nanoid from 'next/dist/compiled/nanoid/index.js'
import path from 'path' import path from 'path'
import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages' import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages'
import { recursiveDelete } from '../lib/recursive-delete'
import { CompilerResult, runCompiler } from './compiler' import { CompilerResult, runCompiler } from './compiler'
import { createEntrypoints, createPagesMapping } from './entries' import { createEntrypoints, createPagesMapping } from './entries'
import { FlyingShuttle } from './flying-shuttle'
import { generateBuildId } from './generate-build-id' import { generateBuildId } from './generate-build-id'
import { isWriteable } from './is-writeable' import { isWriteable } from './is-writeable'
import { collectPages, getSpecifiedPages, printTreeView } from './utils' import {
collectPages,
getFileForPage,
getSpecifiedPages,
printTreeView,
} from './utils'
import getBaseWebpackConfig from './webpack-config' import getBaseWebpackConfig from './webpack-config'
import { writeBuildId } from './write-build-id' import { writeBuildId } from './write-build-id'
...@@ -54,7 +61,21 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -54,7 +61,21 @@ export default async function build(dir: string, conf = null): Promise<void> {
) )
} }
let pagePaths let flyingShuttle: FlyingShuttle | undefined
if (isFlyingShuttle) {
console.log(chalk.magenta('Building with Flying Shuttle enabled ...'))
console.log()
await recursiveDelete(distDir)
flyingShuttle = new FlyingShuttle({
buildId,
pagesDirectory: pagesDir,
distDirectory: distDir,
})
}
let pagePaths: string[]
if (process.env.__NEXT_BUILDER_EXPERIMENTAL_PAGE) { if (process.env.__NEXT_BUILDER_EXPERIMENTAL_PAGE) {
pagePaths = await getSpecifiedPages( pagePaths = await getSpecifiedPages(
dir, dir,
...@@ -64,6 +85,45 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -64,6 +85,45 @@ export default async function build(dir: string, conf = null): Promise<void> {
} else { } else {
pagePaths = await collectPages(pagesDir, config.pageExtensions) pagePaths = await collectPages(pagesDir, config.pageExtensions)
} }
if (flyingShuttle && (await flyingShuttle.hasShuttle())) {
const _unchangedPages = new Set(await flyingShuttle.getUnchangedPages())
for (const unchangedPage of _unchangedPages) {
const recalled = await flyingShuttle.restorePage(unchangedPage)
if (recalled) {
continue
}
_unchangedPages.delete(unchangedPage)
}
const unchangedPages = await Promise.all(
[..._unchangedPages].map(async page => {
const file = await getFileForPage({
page,
pagesDirectory: pagesDir,
pageExtensions: config.pageExtensions,
})
if (file) {
return file
}
return Promise.reject(
new Error(
`Failed to locate page file: ${page}. ` +
`Did pageExtensions change? We can't recover from this yet.`
)
)
})
)
const pageSet = new Set(pagePaths)
for (const unchangedPage of unchangedPages) {
pageSet.delete(unchangedPage)
}
pagePaths = [...pageSet]
}
const mappedPages = createPagesMapping(pagePaths, config.pageExtensions) const mappedPages = createPagesMapping(pagePaths, config.pageExtensions)
const entrypoints = createEntrypoints( const entrypoints = createEntrypoints(
mappedPages, mappedPages,
...@@ -120,6 +180,10 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -120,6 +180,10 @@ export default async function build(dir: string, conf = null): Promise<void> {
result = formatWebpackMessages(result) result = formatWebpackMessages(result)
if (isFlyingShuttle) {
console.log()
}
if (result.errors.length > 0) { if (result.errors.length > 0) {
// Only keep the first error. Others are often indicative // Only keep the first error. Others are often indicative
// of the same problem, but confuse the reader with noise. // of the same problem, but confuse the reader with noise.
...@@ -148,5 +212,9 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -148,5 +212,9 @@ export default async function build(dir: string, conf = null): Promise<void> {
printTreeView(Object.keys(mappedPages)) printTreeView(Object.keys(mappedPages))
if (flyingShuttle) {
await flyingShuttle.save()
}
await writeBuildId(distDir, buildId, selectivePageBuilding) await writeBuildId(distDir, buildId, selectivePageBuilding)
} }
...@@ -104,6 +104,7 @@ ...@@ -104,6 +104,7 @@
"@types/babel__traverse": "7.0.3", "@types/babel__traverse": "7.0.3",
"@types/cross-spawn": "6.0.0", "@types/cross-spawn": "6.0.0",
"@types/etag": "1.8.0", "@types/etag": "1.8.0",
"@types/find-cache-dir": "2.0.0",
"@types/fresh": "0.5.0", "@types/fresh": "0.5.0",
"@types/loader-utils": "1.1.3", "@types/loader-utils": "1.1.3",
"@types/mkdirp": "0.5.2", "@types/mkdirp": "0.5.2",
......
module.exports = {
target: 'serverless',
onDemandEntries: {
// Make sure entries are not getting disposed.
maxInactiveAge: 1000 * 60 * 60
},
experimental: {
flyingShuttle: true
}
}
export default () => (
<div>Hello World</div>
)
/* eslint-env jest */
/* global jasmine */
import { join, resolve, relative } from 'path'
import { existsSync, readFileSync } from 'fs'
import { CHUNK_GRAPH_MANIFEST } from 'next-server/constants'
import { nextBuild } from 'next-test-utils'
const appDir = join(__dirname, '../')
jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
describe('Flying Shuttle', () => {
beforeAll(async () => {
await nextBuild(appDir)
})
describe('Chunk Graph Module file', () => {
it('should emit a manifest file', () => {
const cgf = join(__dirname, `/../.next/${CHUNK_GRAPH_MANIFEST}`)
expect(existsSync(cgf)).toBeTruthy()
expect(
JSON.parse(readFileSync(cgf, 'utf8')).pages['/'].includes(
relative(appDir, resolve(__dirname, '..', 'pages', 'index.js'))
)
)
})
})
})
...@@ -1578,6 +1578,11 @@ ...@@ -1578,6 +1578,11 @@
dependencies: dependencies:
"@types/node" "*" "@types/node" "*"
"@types/find-cache-dir@2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/find-cache-dir/-/find-cache-dir-2.0.0.tgz#6ee79b947b8e51ce8c565fc8278822b2605609db"
integrity sha512-LHAReDNv7IVTE2Q+nPcRBgUZAUKPJIvR7efMrWgx69442KMoMK+QYjtTtK9WGUdaqUYVLkd/0cvCfb55LFWsVw==
"@types/fresh@0.5.0": "@types/fresh@0.5.0":
version "0.5.0" version "0.5.0"
resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f" resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment