Unverified Commit 74cb4f4b authored by Joe Haddad's avatar Joe Haddad Committed by GitHub

Switch to using `.next/cache` for flying shuttle and terser (#7013)

parent c1eba492
import Sema from 'async-sema' import Sema from 'async-sema'
import crypto from 'crypto' import crypto from 'crypto'
import findCacheDir from 'find-cache-dir'
import fs from 'fs' import fs from 'fs'
import mkdirpModule from 'mkdirp' import mkdirpModule from 'mkdirp'
import { CHUNK_GRAPH_MANIFEST } from 'next-server/constants' import { CHUNK_GRAPH_MANIFEST } from 'next-server/constants'
...@@ -55,10 +54,13 @@ export class FlyingShuttle { ...@@ -55,10 +54,13 @@ export class FlyingShuttle {
distDirectory: string distDirectory: string
cacheIdentifier: string cacheIdentifier: string
}) { }) {
this.shuttleDirectory = findCacheDir({ mkdirpModule.sync(
name: 'next-flying-shuttle', (this.shuttleDirectory = path.join(
create: true, distDirectory,
})! 'cache',
'next-flying-shuttle'
))
)
this.buildId = buildId this.buildId = buildId
this.pagesDirectory = pagesDirectory this.pagesDirectory = pagesDirectory
......
...@@ -74,7 +74,7 @@ export default async function build(dir: string, conf = null): Promise<void> { ...@@ -74,7 +74,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
console.log(chalk.magenta('Building with Flying Shuttle enabled ...')) console.log(chalk.magenta('Building with Flying Shuttle enabled ...'))
console.log() console.log()
await recursiveDelete(distDir) await recursiveDelete(distDir, /^(?!cache(?:[\/\\]|$)).*$/)
flyingShuttle = new FlyingShuttle({ flyingShuttle = new FlyingShuttle({
buildId, buildId,
......
...@@ -73,6 +73,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, debug = ...@@ -73,6 +73,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, debug =
sourceMap: false, sourceMap: false,
cache: true, cache: true,
cpus: config.experimental.cpus, cpus: config.experimental.cpus,
distDir: distDir
} }
let webpackConfig: webpack.Configuration = { let webpackConfig: webpack.Configuration = {
......
import { join } from 'path'; import { join } from 'path'
import minify from './minify'; import minify from './minify'
import { promisify } from 'util'; import { promisify } from 'util'
import workerFarm from 'worker-farm'; import workerFarm from 'worker-farm'
import { writeFile, readFile } from 'fs'; import { writeFile, readFile } from 'fs'
import findCacheDir from 'find-cache-dir'; import serialize from 'serialize-javascript'
import serialize from 'serialize-javascript'; import Sema from 'async-sema'
import Sema from 'async-sema'; import mkdirp from 'mkdirp'
const worker = require.resolve('./worker'); const worker = require.resolve('./worker')
const writeFileP = promisify(writeFile) const writeFileP = promisify(writeFile)
const readFileP = promisify(readFile) const readFileP = promisify(readFile)
export default class TaskRunner { export default class TaskRunner {
constructor(cpus) { constructor(distDir, cpus) {
this.cacheDir = findCacheDir({ name: 'next-minifier', create: true }) mkdirp.sync((this.cacheDir = join(distDir, 'cache', 'next-minifier')))
// In some cases cpus() returns undefined // In some cases cpus() returns undefined
// https://github.com/nodejs/node/issues/19022 // https://github.com/nodejs/node/issues/19022
this.maxConcurrentWorkers = cpus this.maxConcurrentWorkers = cpus
...@@ -23,8 +24,8 @@ export default class TaskRunner { ...@@ -23,8 +24,8 @@ export default class TaskRunner {
run(tasks, callback) { run(tasks, callback) {
/* istanbul ignore if */ /* istanbul ignore if */
if (!tasks.length) { if (!tasks.length) {
callback(null, []); callback(null, [])
return; return
} }
if (this.maxConcurrentWorkers > 1) { if (this.maxConcurrentWorkers > 1) {
...@@ -34,58 +35,55 @@ export default class TaskRunner { ...@@ -34,58 +35,55 @@ export default class TaskRunner {
maxConcurrentWorkers: this.maxConcurrentWorkers, maxConcurrentWorkers: this.maxConcurrentWorkers,
maxConcurrentCallsPerWorker: 1, maxConcurrentCallsPerWorker: 1,
} }
: { maxConcurrentWorkers: this.maxConcurrentWorkers }; : { maxConcurrentWorkers: this.maxConcurrentWorkers }
this.workers = workerFarm(workerOptions, worker); this.workers = workerFarm(workerOptions, worker)
this.boundWorkers = (options, cb) => { this.boundWorkers = (options, cb) => {
try { try {
this.workers(serialize(options), cb); this.workers(serialize(options), cb)
} catch (error) { } catch (error) {
// worker-farm can fail with ENOMEM or something else // worker-farm can fail with ENOMEM or something else
cb(error); cb(error)
} }
}; }
} else { } else {
this.boundWorkers = (options, cb) => { this.boundWorkers = (options, cb) => {
try { try {
cb(null, minify(options)); cb(null, minify(options))
} catch (error) { } catch (error) {
cb(error); cb(error)
} }
}; }
} }
let toRun = tasks.length; let toRun = tasks.length
const results = []; const results = []
const step = (index, data) => { const step = (index, data) => {
this.sema.release(); this.sema.release()
toRun -= 1; toRun -= 1
results[index] = data; results[index] = data
if (!toRun) { if (!toRun) {
callback(null, results); callback(null, results)
} }
}; }
tasks.forEach((task, index) => { tasks.forEach((task, index) => {
const cachePath = join( const cachePath = join(this.cacheDir, task.cacheKey)
this.cacheDir,
task.cacheKey
)
const enqueue = () => { const enqueue = () => {
this.boundWorkers(task, (error, data) => { this.boundWorkers(task, (error, data) => {
const result = error ? { error } : data; const result = error ? { error } : data
const done = () => step(index, result); const done = () => step(index, result)
if (this.cacheDir && !result.error) { if (this.cacheDir && !result.error) {
writeFileP(cachePath, JSON.stringify(data), 'utf8') writeFileP(cachePath, JSON.stringify(data), 'utf8')
.then(done) .then(done)
.catch(done) .catch(done)
} else { } else {
done(); done()
} }
}); })
}; }
this.sema.acquire().then(() => { this.sema.acquire().then(() => {
if (this.cacheDir) { if (this.cacheDir) {
...@@ -93,15 +91,15 @@ export default class TaskRunner { ...@@ -93,15 +91,15 @@ export default class TaskRunner {
.then(data => step(index, JSON.parse(data))) .then(data => step(index, JSON.parse(data)))
.catch(() => enqueue()) .catch(() => enqueue())
} else { } else {
enqueue(); enqueue()
} }
}) })
}); })
} }
exit() { exit() {
if (this.workers) { if (this.workers) {
workerFarm.end(this.workers); workerFarm.end(this.workers)
} }
} }
} }
...@@ -12,7 +12,13 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => { ...@@ -12,7 +12,13 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => {
try { try {
await unlink(p) await unlink(p)
} catch (e) { } catch (e) {
if ((e.code === 'EBUSY' || e.code === 'ENOTEMPTY' || e.code === 'EPERM' || e.code === 'EMFILE') && t < 3) { if (
(e.code === 'EBUSY' ||
e.code === 'ENOTEMPTY' ||
e.code === 'EPERM' ||
e.code === 'EMFILE') &&
t < 3
) {
await sleep(t * 100) await sleep(t * 100)
return unlinkFile(p, t++) return unlinkFile(p, t++)
} }
...@@ -28,34 +34,50 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => { ...@@ -28,34 +34,50 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => {
/** /**
* Recursively delete directory contents * Recursively delete directory contents
* @param {string} dir Directory to delete the contents of * @param {string} dir Directory to delete the contents of
* @param {RegExp} filter Filter for the file name, only the name part is considered, not the full path * @param {RegExp} [filter] Filter for the relative file path
* @param {boolean} ensure Esures that parameter dir exists, this is not passed recursively * @param {boolean} [ensure] Esures that parameter dir exists, this is not passed recursively
* @param {string} [previousPath] Esures that parameter dir exists, this is not passed recursively
* @returns Promise void * @returns Promise void
*/ */
export async function recursiveDelete(dir: string, filter?: RegExp, ensure?: boolean): Promise<void> { export async function recursiveDelete(
dir: string,
filter?: RegExp,
previousPath: string = '',
ensure?: boolean,
): Promise<void> {
let result let result
try { try {
result = await readdir(dir) result = await readdir(dir)
} catch (e) { } catch (e) {
if (e.code === 'ENOENT' && !ensure) return if (e.code === 'ENOENT' && !ensure) {
return
}
throw e throw e
} }
await Promise.all(result.map(async (part: string) => { await Promise.all(
const absolutePath = join(dir, part) result.map(async (part: string) => {
const pathStat = await stat(absolutePath).catch((e) => { const absolutePath = join(dir, part)
if (e.code !== 'ENOENT') throw e const pathStat = await stat(absolutePath).catch((e) => {
}) if (e.code !== 'ENOENT') throw e
if (!pathStat) return })
if (!pathStat) {
return
}
if (pathStat.isDirectory()) { if (pathStat.isDirectory()) {
await recursiveDelete(absolutePath, filter) const pp = join(previousPath, part)
return rmdir(absolutePath) await recursiveDelete(absolutePath, filter, pp)
}
if (!filter || filter.test(part)) { if (!filter || filter.test(pp)) {
return unlinkFile(absolutePath) return rmdir(absolutePath)
} }
})) return
}
if (!filter || filter.test(join(previousPath, part))) {
return unlinkFile(absolutePath)
}
}),
)
} }
...@@ -60,7 +60,6 @@ ...@@ -60,7 +60,6 @@
"babel-plugin-transform-async-to-promises": "0.8.9", "babel-plugin-transform-async-to-promises": "0.8.9",
"babel-plugin-transform-react-remove-prop-types": "0.4.15", "babel-plugin-transform-react-remove-prop-types": "0.4.15",
"chalk": "2.4.2", "chalk": "2.4.2",
"find-cache-dir": "2.0.0",
"find-up": "2.1.0", "find-up": "2.1.0",
"fresh": "0.5.2", "fresh": "0.5.2",
"launch-editor": "2.2.1", "launch-editor": "2.2.1",
...@@ -104,7 +103,6 @@ ...@@ -104,7 +103,6 @@
"@types/babel__traverse": "7.0.3", "@types/babel__traverse": "7.0.3",
"@types/cross-spawn": "6.0.0", "@types/cross-spawn": "6.0.0",
"@types/etag": "1.8.0", "@types/etag": "1.8.0",
"@types/find-cache-dir": "2.0.0",
"@types/find-up": "2.1.1", "@types/find-up": "2.1.1",
"@types/fresh": "0.5.0", "@types/fresh": "0.5.0",
"@types/loader-utils": "1.1.3", "@types/loader-utils": "1.1.3",
......
...@@ -1578,11 +1578,6 @@ ...@@ -1578,11 +1578,6 @@
dependencies: dependencies:
"@types/node" "*" "@types/node" "*"
"@types/find-cache-dir@2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/find-cache-dir/-/find-cache-dir-2.0.0.tgz#6ee79b947b8e51ce8c565fc8278822b2605609db"
integrity sha512-LHAReDNv7IVTE2Q+nPcRBgUZAUKPJIvR7efMrWgx69442KMoMK+QYjtTtK9WGUdaqUYVLkd/0cvCfb55LFWsVw==
"@types/find-up@2.1.1": "@types/find-up@2.1.1":
version "2.1.1" version "2.1.1"
resolved "https://registry.yarnpkg.com/@types/find-up/-/find-up-2.1.1.tgz#1cd2d240f1ad1f48d32346074724dc3107248a11" resolved "https://registry.yarnpkg.com/@types/find-up/-/find-up-2.1.1.tgz#1cd2d240f1ad1f48d32346074724dc3107248a11"
...@@ -5473,15 +5468,6 @@ finalhandler@1.1.1: ...@@ -5473,15 +5468,6 @@ finalhandler@1.1.1:
statuses "~1.4.0" statuses "~1.4.0"
unpipe "~1.0.0" unpipe "~1.0.0"
find-cache-dir@2.0.0, find-cache-dir@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.0.0.tgz#4c1faed59f45184530fb9d7fa123a4d04a98472d"
integrity sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==
dependencies:
commondir "^1.0.1"
make-dir "^1.0.0"
pkg-dir "^3.0.0"
find-cache-dir@^1.0.0: find-cache-dir@^1.0.0:
version "1.0.0" version "1.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-1.0.0.tgz#9288e3e9e3cc3748717d39eade17cf71fc30ee6f" resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-1.0.0.tgz#9288e3e9e3cc3748717d39eade17cf71fc30ee6f"
...@@ -5491,6 +5477,15 @@ find-cache-dir@^1.0.0: ...@@ -5491,6 +5477,15 @@ find-cache-dir@^1.0.0:
make-dir "^1.0.0" make-dir "^1.0.0"
pkg-dir "^2.0.0" pkg-dir "^2.0.0"
find-cache-dir@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.0.0.tgz#4c1faed59f45184530fb9d7fa123a4d04a98472d"
integrity sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==
dependencies:
commondir "^1.0.1"
make-dir "^1.0.0"
pkg-dir "^3.0.0"
find-file-up@^0.1.2: find-file-up@^0.1.2:
version "0.1.3" version "0.1.3"
resolved "https://registry.yarnpkg.com/find-file-up/-/find-file-up-0.1.3.tgz#cf68091bcf9f300a40da411b37da5cce5a2fbea0" resolved "https://registry.yarnpkg.com/find-file-up/-/find-file-up-0.1.3.tgz#cf68091bcf9f300a40da411b37da5cce5a2fbea0"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment