Commit 861edad4 authored by Connor Davis's avatar Connor Davis Committed by Tim Neutkens

Implement Recursive Delete (#6489)

Removes `rimraf` for a smaller custom lib

Benchmarks (in ms):
```
rimraf 1
518.536376
rimraf 2
416.112451
rimraf 3
451.905842
rimraf 4
525.117395
rimraf 5
434.230384
---- Average: 469.18ms
-----------
recursive delete 1
121.493979
recursive delete 2
130.335272
recursive delete 3
149.798316
recursive delete 4
182.184688
recursive delete 5
130.346207
--- Average: 142.83ms
```
`recursiveDelete` finishes in ~30% of the time it takes `rimraf` (3.3 times faster)
parent 068dd825
const { join } = require('path')
const { recursiveDelete } = require('next/dist/lib/recursive-delete')
const resolveDataDir = join(__dirname, `fixtures-${process.argv[2]}`)
async function test () {
const time = process.hrtime()
await recursiveDelete(resolveDataDir)
const hrtime = process.hrtime(time)
const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
const milliseconds = nanoseconds / 1e6
console.log(milliseconds)
}
test()
const { join } = require('path')
const { promisify } = require('util')
const rimrafMod = require('rimraf')
const resolveDataDir = join(__dirname, `fixtures-${process.argv[2]}`, '**/*')
const rimraf = promisify(rimrafMod)
async function test () {
const time = process.hrtime()
await rimraf(resolveDataDir)
const hrtime = process.hrtime(time)
const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
const milliseconds = nanoseconds / 1e6
console.log(milliseconds)
}
test()
# Uses https://github.com/divmain/fuzzponent
mkdir fixtures-1
cd fixtures-1
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 1"
node rimraf.js 1
mkdir fixtures-2
cd fixtures-2
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 2"
node rimraf.js 2
mkdir fixtures-3
cd fixtures-3
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 3"
node rimraf.js 3
mkdir fixtures-4
cd fixtures-4
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 4"
node rimraf.js 4
mkdir fixtures-5
cd fixtures-5
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 5"
node rimraf.js 5
echo "-----------"
cd fixtures-1
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 1"
node recursive-delete.js 1
cd fixtures-2
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 2"
node recursive-delete.js 2
cd fixtures-3
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 3"
node recursive-delete.js 3
cd fixtures-4
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 4"
node recursive-delete.js 4
cd fixtures-5
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 5"
node recursive-delete.js 5
rm -r fixtures-1 fixtures-2 fixtures-3 fixtures-4 fixtures-5
\ No newline at end of file
import rimrafModule from 'rimraf'
import { cpus } from 'os'
import { fork } from 'child_process'
import cp from 'recursive-copy'
......@@ -9,9 +8,9 @@ import loadConfig from 'next-server/next-config'
import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants'
import createProgress from 'tty-aware-progress'
import { promisify } from 'util'
import { recursiveDelete } from '../lib/recursive-delete'
const mkdirp = promisify(mkdirpModule)
const rimraf = promisify(rimrafModule)
export default async function (dir, options, configuration) {
function log (message) {
......@@ -55,7 +54,7 @@ export default async function (dir, options, configuration) {
// Initialize the output directory
const outDir = options.outdir
await rimraf(join(outDir, '*'))
await recursiveDelete(join(outDir))
await mkdirp(join(outDir, '_next', buildId))
// Copy static directory
......
import fs from 'fs'
import { join } from 'path'
import { promisify } from 'util'
const readdir = promisify(fs.readdir)
const stat = promisify(fs.stat)
const rmdir = promisify(fs.rmdir)
const unlink = promisify(fs.unlink)
const sleep = promisify(setTimeout)
const unlinkFile = async (p: string, t = 1): Promise<void> => {
try {
await unlink(p)
} catch (e) {
if ((e.code === 'EBUSY' || e.code === 'ENOTEMPTY' || e.code === 'EPERM' || e.code === 'EMFILE') && t < 3) {
await sleep(t * 100)
return unlinkFile(p, t++)
}
if (e.code === 'ENOENT') {
return
}
throw e
}
}
/**
* Recursively delete directory contents
* @param {string} dir Directory to delete the contents of
* @param {RegExp} filter Filter for the file name, only the name part is considered, not the full path
* @param {boolean} ensure Esures that parameter dir exists, this is not passed recursively
* @returns Promise void
*/
export async function recursiveDelete(dir: string, filter?: RegExp, ensure?: boolean): Promise<void> {
let result
try {
result = await readdir(dir)
} catch (e) {
if (e.code === 'ENOENT' && !ensure) return
throw e
}
await Promise.all(result.map(async (part: string) => {
const absolutePath = join(dir, part)
const pathStat = await stat(absolutePath).catch((e) => {
if (e.code !== 'ENOENT') throw e
})
if (!pathStat) return
if (pathStat.isDirectory()) {
await recursiveDelete(absolutePath, filter)
return rmdir(absolutePath)
}
if (!filter || filter.test(part)) {
return unlinkFile(absolutePath)
}
}))
}
......@@ -75,7 +75,6 @@
"react-is": "16.6.3",
"recursive-copy": "2.0.6",
"resolve": "1.5.0",
"rimraf": "2.6.3",
"serialize-javascript": "1.4.0",
"source-map": "0.6.1",
"strip-ansi": "3.0.1",
......@@ -113,7 +112,6 @@
"@types/nanoid": "1.2.0",
"@types/node-fetch": "2.1.4",
"@types/resolve": "0.0.8",
"@types/rimraf": "2.0.2",
"@zeit/ncc": "0.15.2",
"taskr": "1.1.0",
"typescript": "3.1.6",
......
......@@ -2,19 +2,16 @@ import { join, normalize } from 'path'
import WebpackDevMiddleware from 'webpack-dev-middleware'
import WebpackHotMiddleware from 'webpack-hot-middleware'
import errorOverlayMiddleware from './lib/error-overlay-middleware'
import rimrafModule from 'rimraf'
import onDemandEntryHandler, { normalizePage } from './on-demand-entry-handler'
import webpack from 'webpack'
import getBaseWebpackConfig from '../build/webpack-config'
import { IS_BUNDLED_PAGE_REGEX, ROUTE_NAME_REGEX, BLOCKED_PAGES, CLIENT_STATIC_FILES_RUNTIME_AMP } from 'next-server/constants'
import { NEXT_PROJECT_ROOT_DIST_CLIENT } from '../lib/constants'
import { route } from 'next-server/dist/server/router'
import { promisify } from 'util'
import { createPagesMapping, createEntrypoints } from '../build/entries'
import { watchCompiler } from '../build/output'
import { findPageFile } from './lib/find-page-file'
const rimraf = promisify(rimrafModule)
import { recursiveDelete } from '../lib/recursive-delete'
export async function renderScriptError (res, error) {
// Asks CDNs and others to not to cache the errored page
......@@ -162,7 +159,7 @@ export default class HotReloader {
}
async clean () {
return rimraf(join(this.dir, this.config.distDir), { force: true })
return recursiveDelete(join(this.dir, this.config.distDir))
}
async getWebpackConfig () {
......
......@@ -15,14 +15,35 @@ import {
import ssr from './ssr'
import browser from './browser'
import dev from './dev'
import { promisify } from 'util'
import fs from 'fs'
jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
const writeFile = promisify(fs.writeFile)
const mkdir = promisify(fs.mkdir)
const access = promisify(fs.access)
const appDir = join(__dirname, '../')
const context = {}
const devContext = {}
describe('Static Export', () => {
it('should delete existing exported files', async () => {
const outdir = join(appDir, 'out')
const tempfile = join(outdir, 'temp.txt')
await mkdir(outdir).catch((e) => { if (e.code !== 'EEXIST') throw e })
await writeFile(tempfile, 'Hello there')
await nextBuild(appDir)
await nextExport(appDir, { outdir })
let doesNotExist = false
await access(tempfile).catch((e) => {
if (e.code === 'ENOENT') doesNotExist = true
})
expect(doesNotExist).toBe(true)
})
beforeAll(async () => {
const outdir = join(appDir, 'out')
await nextBuild(appDir)
......
test_resolvedata
\ No newline at end of file
/* eslint-env jest */
import { recursiveDelete } from 'next/dist/lib/recursive-delete'
import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
import cp from 'recursive-copy'
import { join } from 'path'
const resolveDataDir = join(__dirname, '..', 'isolated', '_resolvedata')
const testResolveDataDir = join(__dirname, '..', 'isolated', 'test_resolvedata')
describe('recursiveDelete', () => {
it('should work', async () => {
await cp(
resolveDataDir,
testResolveDataDir,
{ expand: true }
)
await recursiveDelete(testResolveDataDir)
const result = await recursiveReadDir(testResolveDataDir, /.*/)
expect(result.length).toBe(0)
})
})
......@@ -1579,25 +1579,11 @@
dependencies:
"@types/node" "*"
"@types/events@*":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
"@types/fresh@0.5.0":
version "0.5.0"
resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f"
integrity sha512-eGPzuyc6wZM3sSHJdF7NM2jW6B/xsB014Rqg/iDa6xY02mlfy1w/TE2sYhR8vbHxkzJOXiGo6NuIk3xk35vsgQ==
"@types/glob@*":
version "7.1.1"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
dependencies:
"@types/events" "*"
"@types/minimatch" "*"
"@types/node" "*"
"@types/loader-utils@1.1.3":
version "1.1.3"
resolved "https://registry.yarnpkg.com/@types/loader-utils/-/loader-utils-1.1.3.tgz#82b9163f2ead596c68a8c03e450fbd6e089df401"
......@@ -1611,11 +1597,6 @@
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d"
integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==
"@types/minimatch@*":
version "3.0.3"
resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
"@types/mkdirp@0.5.2":
version "0.5.2"
resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.2.tgz#503aacfe5cc2703d5484326b1b27efa67a339c1f"
......@@ -1681,14 +1662,6 @@
dependencies:
"@types/node" "*"
"@types/rimraf@2.0.2":
version "2.0.2"
resolved "https://registry.yarnpkg.com/@types/rimraf/-/rimraf-2.0.2.tgz#7f0fc3cf0ff0ad2a99bb723ae1764f30acaf8b6e"
integrity sha512-Hm/bnWq0TCy7jmjeN5bKYij9vw5GrDFWME4IuxV08278NtU/VdGbzsBohcCUJ7+QMqmUq5hpRKB39HeQWJjztQ==
dependencies:
"@types/glob" "*"
"@types/node" "*"
"@types/send@0.14.4":
version "0.14.4"
resolved "https://registry.yarnpkg.com/@types/send/-/send-0.14.4.tgz#d70458b030305999db619a7b057f7105058bd0ff"
......@@ -10849,7 +10822,7 @@ rgba-regex@^1.0.0:
resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3"
integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM=
rimraf@2, rimraf@2.6.3, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2:
rimraf@2, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2:
version "2.6.3"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment