Unverified Commit 74cb4f4b authored by Joe Haddad's avatar Joe Haddad Committed by GitHub

Switch to using `.next/cache` for flying shuttle and terser (#7013)

parent c1eba492
import Sema from 'async-sema'
import crypto from 'crypto'
import findCacheDir from 'find-cache-dir'
import fs from 'fs'
import mkdirpModule from 'mkdirp'
import { CHUNK_GRAPH_MANIFEST } from 'next-server/constants'
......@@ -55,10 +54,13 @@ export class FlyingShuttle {
distDirectory: string
cacheIdentifier: string
}) {
this.shuttleDirectory = findCacheDir({
name: 'next-flying-shuttle',
create: true,
})!
mkdirpModule.sync(
(this.shuttleDirectory = path.join(
distDirectory,
'cache',
'next-flying-shuttle'
))
)
this.buildId = buildId
this.pagesDirectory = pagesDirectory
......
......@@ -74,7 +74,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
console.log(chalk.magenta('Building with Flying Shuttle enabled ...'))
console.log()
await recursiveDelete(distDir)
await recursiveDelete(distDir, /^(?!cache(?:[\/\\]|$)).*$/)
flyingShuttle = new FlyingShuttle({
buildId,
......
......@@ -73,6 +73,7 @@ export default function getBaseWebpackConfig (dir: string, {dev = false, debug =
sourceMap: false,
cache: true,
cpus: config.experimental.cpus,
distDir: distDir
}
let webpackConfig: webpack.Configuration = {
......
import { join } from 'path';
import minify from './minify';
import { promisify } from 'util';
import workerFarm from 'worker-farm';
import { writeFile, readFile } from 'fs';
import findCacheDir from 'find-cache-dir';
import serialize from 'serialize-javascript';
import Sema from 'async-sema';
import { join } from 'path'
import minify from './minify'
import { promisify } from 'util'
import workerFarm from 'worker-farm'
import { writeFile, readFile } from 'fs'
import serialize from 'serialize-javascript'
import Sema from 'async-sema'
import mkdirp from 'mkdirp'
const worker = require.resolve('./worker');
const worker = require.resolve('./worker')
const writeFileP = promisify(writeFile)
const readFileP = promisify(readFile)
export default class TaskRunner {
constructor(cpus) {
this.cacheDir = findCacheDir({ name: 'next-minifier', create: true })
constructor(distDir, cpus) {
mkdirp.sync((this.cacheDir = join(distDir, 'cache', 'next-minifier')))
// In some cases cpus() returns undefined
// https://github.com/nodejs/node/issues/19022
this.maxConcurrentWorkers = cpus
......@@ -23,8 +24,8 @@ export default class TaskRunner {
run(tasks, callback) {
/* istanbul ignore if */
if (!tasks.length) {
callback(null, []);
return;
callback(null, [])
return
}
if (this.maxConcurrentWorkers > 1) {
......@@ -34,58 +35,55 @@ export default class TaskRunner {
maxConcurrentWorkers: this.maxConcurrentWorkers,
maxConcurrentCallsPerWorker: 1,
}
: { maxConcurrentWorkers: this.maxConcurrentWorkers };
this.workers = workerFarm(workerOptions, worker);
: { maxConcurrentWorkers: this.maxConcurrentWorkers }
this.workers = workerFarm(workerOptions, worker)
this.boundWorkers = (options, cb) => {
try {
this.workers(serialize(options), cb);
this.workers(serialize(options), cb)
} catch (error) {
// worker-farm can fail with ENOMEM or something else
cb(error);
cb(error)
}
};
}
} else {
this.boundWorkers = (options, cb) => {
try {
cb(null, minify(options));
cb(null, minify(options))
} catch (error) {
cb(error);
cb(error)
}
};
}
}
let toRun = tasks.length;
const results = [];
let toRun = tasks.length
const results = []
const step = (index, data) => {
this.sema.release();
toRun -= 1;
results[index] = data;
this.sema.release()
toRun -= 1
results[index] = data
if (!toRun) {
callback(null, results);
callback(null, results)
}
};
}
tasks.forEach((task, index) => {
const cachePath = join(
this.cacheDir,
task.cacheKey
)
const cachePath = join(this.cacheDir, task.cacheKey)
const enqueue = () => {
this.boundWorkers(task, (error, data) => {
const result = error ? { error } : data;
const done = () => step(index, result);
const result = error ? { error } : data
const done = () => step(index, result)
if (this.cacheDir && !result.error) {
writeFileP(cachePath, JSON.stringify(data), 'utf8')
.then(done)
.catch(done)
} else {
done();
done()
}
});
};
})
}
this.sema.acquire().then(() => {
if (this.cacheDir) {
......@@ -93,15 +91,15 @@ export default class TaskRunner {
.then(data => step(index, JSON.parse(data)))
.catch(() => enqueue())
} else {
enqueue();
enqueue()
}
})
});
})
}
exit() {
if (this.workers) {
workerFarm.end(this.workers);
workerFarm.end(this.workers)
}
}
}
......@@ -12,7 +12,13 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => {
try {
await unlink(p)
} catch (e) {
if ((e.code === 'EBUSY' || e.code === 'ENOTEMPTY' || e.code === 'EPERM' || e.code === 'EMFILE') && t < 3) {
if (
(e.code === 'EBUSY' ||
e.code === 'ENOTEMPTY' ||
e.code === 'EPERM' ||
e.code === 'EMFILE') &&
t < 3
) {
await sleep(t * 100)
return unlinkFile(p, t++)
}
......@@ -28,34 +34,50 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => {
/**
* Recursively delete directory contents
* @param {string} dir Directory to delete the contents of
* @param {RegExp} filter Filter for the file name, only the name part is considered, not the full path
* @param {boolean} ensure Esures that parameter dir exists, this is not passed recursively
* @param {RegExp} [filter] Filter for the relative file path
* @param {boolean} [ensure] Esures that parameter dir exists, this is not passed recursively
* @param {string} [previousPath] Esures that parameter dir exists, this is not passed recursively
* @returns Promise void
*/
export async function recursiveDelete(dir: string, filter?: RegExp, ensure?: boolean): Promise<void> {
export async function recursiveDelete(
dir: string,
filter?: RegExp,
previousPath: string = '',
ensure?: boolean,
): Promise<void> {
let result
try {
result = await readdir(dir)
} catch (e) {
if (e.code === 'ENOENT' && !ensure) return
if (e.code === 'ENOENT' && !ensure) {
return
}
throw e
}
await Promise.all(result.map(async (part: string) => {
const absolutePath = join(dir, part)
const pathStat = await stat(absolutePath).catch((e) => {
if (e.code !== 'ENOENT') throw e
})
if (!pathStat) return
await Promise.all(
result.map(async (part: string) => {
const absolutePath = join(dir, part)
const pathStat = await stat(absolutePath).catch((e) => {
if (e.code !== 'ENOENT') throw e
})
if (!pathStat) {
return
}
if (pathStat.isDirectory()) {
await recursiveDelete(absolutePath, filter)
return rmdir(absolutePath)
}
if (pathStat.isDirectory()) {
const pp = join(previousPath, part)
await recursiveDelete(absolutePath, filter, pp)
if (!filter || filter.test(part)) {
return unlinkFile(absolutePath)
}
}))
if (!filter || filter.test(pp)) {
return rmdir(absolutePath)
}
return
}
if (!filter || filter.test(join(previousPath, part))) {
return unlinkFile(absolutePath)
}
}),
)
}
......@@ -60,7 +60,6 @@
"babel-plugin-transform-async-to-promises": "0.8.9",
"babel-plugin-transform-react-remove-prop-types": "0.4.15",
"chalk": "2.4.2",
"find-cache-dir": "2.0.0",
"find-up": "2.1.0",
"fresh": "0.5.2",
"launch-editor": "2.2.1",
......@@ -104,7 +103,6 @@
"@types/babel__traverse": "7.0.3",
"@types/cross-spawn": "6.0.0",
"@types/etag": "1.8.0",
"@types/find-cache-dir": "2.0.0",
"@types/find-up": "2.1.1",
"@types/fresh": "0.5.0",
"@types/loader-utils": "1.1.3",
......
......@@ -1578,11 +1578,6 @@
dependencies:
"@types/node" "*"
"@types/find-cache-dir@2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/find-cache-dir/-/find-cache-dir-2.0.0.tgz#6ee79b947b8e51ce8c565fc8278822b2605609db"
integrity sha512-LHAReDNv7IVTE2Q+nPcRBgUZAUKPJIvR7efMrWgx69442KMoMK+QYjtTtK9WGUdaqUYVLkd/0cvCfb55LFWsVw==
"@types/find-up@2.1.1":
version "2.1.1"
resolved "https://registry.yarnpkg.com/@types/find-up/-/find-up-2.1.1.tgz#1cd2d240f1ad1f48d32346074724dc3107248a11"
......@@ -5473,15 +5468,6 @@ finalhandler@1.1.1:
statuses "~1.4.0"
unpipe "~1.0.0"
find-cache-dir@2.0.0, find-cache-dir@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.0.0.tgz#4c1faed59f45184530fb9d7fa123a4d04a98472d"
integrity sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==
dependencies:
commondir "^1.0.1"
make-dir "^1.0.0"
pkg-dir "^3.0.0"
find-cache-dir@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-1.0.0.tgz#9288e3e9e3cc3748717d39eade17cf71fc30ee6f"
......@@ -5491,6 +5477,15 @@ find-cache-dir@^1.0.0:
make-dir "^1.0.0"
pkg-dir "^2.0.0"
find-cache-dir@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-2.0.0.tgz#4c1faed59f45184530fb9d7fa123a4d04a98472d"
integrity sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==
dependencies:
commondir "^1.0.1"
make-dir "^1.0.0"
pkg-dir "^3.0.0"
find-file-up@^0.1.2:
version "0.1.3"
resolved "https://registry.yarnpkg.com/find-file-up/-/find-file-up-0.1.3.tgz#cf68091bcf9f300a40da411b37da5cce5a2fbea0"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment