Commit 61b8b732 authored by JJ Kasper's avatar JJ Kasper Committed by Joe Haddad

Update workers for `next export` (#8623)

* Add failing test

* Add jest-worker to export
parent 110c83fa
import { cpus } from 'os' import { cpus } from 'os'
import { fork } from 'child_process' import chalk from 'chalk'
import { recursiveCopy } from '../lib/recursive-copy' import Worker from 'jest-worker'
import { promisify } from 'util'
import mkdirpModule from 'mkdirp' import mkdirpModule from 'mkdirp'
import { resolve, join } from 'path' import { resolve, join } from 'path'
import { API_ROUTE } from '../lib/constants'
import { existsSync, readFileSync } from 'fs' import { existsSync, readFileSync } from 'fs'
import chalk from 'chalk' import createProgress from 'tty-aware-progress'
import { recursiveCopy } from '../lib/recursive-copy'
import { recursiveDelete } from '../lib/recursive-delete'
import { formatAmpMessages } from '../build/output/index'
import loadConfig, { import loadConfig, {
isTargetLikeServerless isTargetLikeServerless
} from '../next-server/server/config' } from '../next-server/server/config'
...@@ -17,11 +22,6 @@ import { ...@@ -17,11 +22,6 @@ import {
CLIENT_PUBLIC_FILES_PATH, CLIENT_PUBLIC_FILES_PATH,
CLIENT_STATIC_FILES_PATH CLIENT_STATIC_FILES_PATH
} from '../next-server/lib/constants' } from '../next-server/lib/constants'
import createProgress from 'tty-aware-progress'
import { promisify } from 'util'
import { recursiveDelete } from '../lib/recursive-delete'
import { API_ROUTE } from '../lib/constants'
import { formatAmpMessages } from '../build/output/index'
const mkdirp = promisify(mkdirpModule) const mkdirp = promisify(mkdirpModule)
...@@ -33,7 +33,6 @@ export default async function (dir, options, configuration) { ...@@ -33,7 +33,6 @@ export default async function (dir, options, configuration) {
dir = resolve(dir) dir = resolve(dir)
const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir) const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir)
const concurrency = options.concurrency || 10
const threads = options.threads || Math.max(cpus().length - 1, 1) const threads = options.threads || Math.max(cpus().length - 1, 1)
const distDir = join(dir, nextConfig.distDir) const distDir = join(dir, nextConfig.distDir)
const subFolders = nextConfig.exportTrailingSlash const subFolders = nextConfig.exportTrailingSlash
...@@ -130,9 +129,7 @@ export default async function (dir, options, configuration) { ...@@ -130,9 +129,7 @@ export default async function (dir, options, configuration) {
nextExport: true nextExport: true
} }
log( log(` launching ${threads} workers`)
` launching ${threads} threads with concurrency of ${concurrency} per thread`
)
const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, { const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, {
dev: false, dev: false,
dir, dir,
...@@ -163,20 +160,6 @@ export default async function (dir, options, configuration) { ...@@ -163,20 +160,6 @@ export default async function (dir, options, configuration) {
const progress = !options.silent && createProgress(filteredPaths.length) const progress = !options.silent && createProgress(filteredPaths.length)
const chunks = filteredPaths.reduce((result, route, i) => {
const worker = i % threads
if (!result[worker]) {
result[worker] = { paths: [], pathMap: {} }
}
result[worker].pathMap[route] = exportPathMap[route]
result[worker].paths.push(route)
if (options.sprPages && options.sprPages.has(route)) {
result[worker].pathMap[route].sprPage = true
}
return result
}, [])
const ampValidations = {} const ampValidations = {}
let hadValidationError = false let hadValidationError = false
...@@ -195,46 +178,45 @@ export default async function (dir, options, configuration) { ...@@ -195,46 +178,45 @@ export default async function (dir, options, configuration) {
} }
}) })
} }
const workers = new Set()
await Promise.all( const worker = new Worker(require.resolve('./worker'), {
chunks.map( maxRetries: 0,
chunk => numWorkers: threads,
new Promise((resolve, reject) => { enableWorkerThreads: true,
const worker = fork(require.resolve('./worker'), [], { exposedMethods: ['default']
env: process.env
}) })
workers.add(worker)
worker.send({ worker.getStdout().pipe(process.stdout)
worker.getStderr().pipe(process.stderr)
let renderError = false
await Promise.all(
filteredPaths.map(async path => {
const result = await worker.default({
path,
pathMap: exportPathMap[path],
distDir, distDir,
buildId, buildId,
exportPaths: chunk.paths,
exportPathMap: chunk.pathMap,
outDir, outDir,
renderOpts, renderOpts,
serverRuntimeConfig, serverRuntimeConfig,
concurrency,
subFolders, subFolders,
serverless: isTargetLikeServerless(nextConfig.target) serverless: isTargetLikeServerless(nextConfig.target)
}) })
worker.on('message', ({ type, payload }) => {
if (type === 'progress' && progress) { for (const validation of result.ampValidations || []) {
progress() const { page, result } = validation
} else if (type === 'error') { ampValidations[page] = result
reject(payload) hadValidationError |=
} else if (type === 'done') { Array.isArray(result && result.errors) && result.errors.length > 0
resolve()
} else if (type === 'amp-validation') {
ampValidations[payload.page] = payload.result
hadValidationError =
hadValidationError || payload.result.errors.length
} }
renderError |= result.error
if (progress) progress()
}) })
})
)
) )
workers.forEach(worker => worker.kill()) worker.end()
if (Object.keys(ampValidations).length) { if (Object.keys(ampValidations).length) {
console.log(formatAmpMessages(ampValidations)) console.log(formatAmpMessages(ampValidations))
...@@ -245,6 +227,9 @@ export default async function (dir, options, configuration) { ...@@ -245,6 +227,9 @@ export default async function (dir, options, configuration) {
) )
} }
if (renderError) {
throw new Error(`Export encountered errors`)
}
// Add an empty line to the console for the better readability. // Add an empty line to the console for the better readability.
log('') log('')
} }
...@@ -3,7 +3,6 @@ import { promisify } from 'util' ...@@ -3,7 +3,6 @@ import { promisify } from 'util'
import { extname, join, dirname, sep } from 'path' import { extname, join, dirname, sep } from 'path'
import { renderToHTML } from '../next-server/server/render' import { renderToHTML } from '../next-server/server/render'
import { writeFile, access } from 'fs' import { writeFile, access } from 'fs'
import { Sema } from 'async-sema'
import AmpHtmlValidator from 'amphtml-validator' import AmpHtmlValidator from 'amphtml-validator'
import { loadComponents } from '../next-server/server/load-components' import { loadComponents } from '../next-server/server/load-components'
import { isDynamicRoute } from '../next-server/lib/router/utils/is-dynamic' import { isDynamicRoute } from '../next-server/lib/router/utils/is-dynamic'
...@@ -11,34 +10,32 @@ import { getRouteMatcher } from '../next-server/lib/router/utils/route-matcher' ...@@ -11,34 +10,32 @@ import { getRouteMatcher } from '../next-server/lib/router/utils/route-matcher'
import { getRouteRegex } from '../next-server/lib/router/utils/route-regex' import { getRouteRegex } from '../next-server/lib/router/utils/route-regex'
const envConfig = require('../next-server/lib/runtime-config') const envConfig = require('../next-server/lib/runtime-config')
const mkdirp = promisify(mkdirpModule)
const writeFileP = promisify(writeFile) const writeFileP = promisify(writeFile)
const mkdirp = promisify(mkdirpModule)
const accessP = promisify(access) const accessP = promisify(access)
global.__NEXT_DATA__ = { global.__NEXT_DATA__ = {
nextExport: true nextExport: true
} }
process.on( export default async function ({
'message', path,
async ({ pathMap,
distDir, distDir,
buildId, buildId,
exportPaths,
exportPathMap,
outDir, outDir,
renderOpts, renderOpts,
serverRuntimeConfig, serverRuntimeConfig,
concurrency,
subFolders, subFolders,
serverless serverless
}) => { }) {
const sema = new Sema(concurrency, { capacity: exportPaths.length }) let results = {
ampValidations: []
}
try { try {
const work = async path => { let { query = {} } = pathMap
await sema.acquire() const { page, sprPage } = pathMap
let { query = {} } = exportPathMap[path]
const { page, sprPage } = exportPathMap[path]
const filePath = path === '/' ? '/index' : path const filePath = path === '/' ? '/index' : path
const ampPath = `${filePath}.amp` const ampPath = `${filePath}.amp`
...@@ -77,8 +74,7 @@ process.on( ...@@ -77,8 +74,7 @@ process.on(
if (sprPage && isDynamicRoute(page)) { if (sprPage && isDynamicRoute(page)) {
query._nextPreviewSkeleton = 1 query._nextPreviewSkeleton = 1
// pass via `req` to avoid adding code to serverless bundle // pass via `req` to avoid adding code to serverless bundle
req.url += req.url += (req.url.includes('?') ? '&' : '?') + '_nextPreviewSkeleton=1'
(req.url.includes('?') ? '&' : '?') + '_nextPreviewSkeleton=1'
} }
envConfig.setConfig({ envConfig.setConfig({
...@@ -140,15 +136,12 @@ process.on( ...@@ -140,15 +136,12 @@ process.on(
const warnings = result.errors.filter(e => e.severity !== 'ERROR') const warnings = result.errors.filter(e => e.severity !== 'ERROR')
if (warnings.length || errors.length) { if (warnings.length || errors.length) {
process.send({ results.ampValidations.push({
type: 'amp-validation',
payload: {
page, page,
result: { result: {
errors, errors,
warnings warnings
} }
}
}) })
} }
} }
...@@ -187,16 +180,10 @@ process.on( ...@@ -187,16 +180,10 @@ process.on(
await writeFileP(ampHtmlFilepath, ampHtml, 'utf8') await writeFileP(ampHtmlFilepath, ampHtml, 'utf8')
} }
} }
await writeFileP(htmlFilepath, html, 'utf8') await writeFileP(htmlFilepath, html, 'utf8')
process.send({ type: 'progress' }) return results
sema.release() } catch (error) {
} console.error(`\nError occurred prerendering ${path}:`, error)
await Promise.all(exportPaths.map(work)) return { ...results, error: true }
process.send({ type: 'done' })
} catch (err) {
console.error(err)
process.send({ type: 'error', payload: err })
} }
} }
)
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
// eslint-disable-next-line
export default () => hello.world
/* eslint-env jest */
/* global jasmine */
import path from 'path'
import { nextBuild } from 'next-test-utils'
jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
const appDir = path.join(__dirname, '..')
describe('Handles Errors During Export', () => {
it('Does not crash workers', async () => {
const { stdout, stderr } = await nextBuild(appDir, [], {
stdout: true,
stderr: true
})
expect(stdout + stderr).not.toMatch(/ERR_IPC_CHANNEL_CLOSED/)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment