Commit 861edad4 authored by Connor Davis's avatar Connor Davis Committed by Tim Neutkens

Implement Recursive Delete (#6489)

Removes `rimraf` for a smaller custom lib

Benchmarks (in ms):
```
rimraf 1
518.536376
rimraf 2
416.112451
rimraf 3
451.905842
rimraf 4
525.117395
rimraf 5
434.230384
---- Average: 469.18ms
-----------
recursive delete 1
121.493979
recursive delete 2
130.335272
recursive delete 3
149.798316
recursive delete 4
182.184688
recursive delete 5
130.346207
--- Average: 142.83ms
```
`recursiveDelete` finishes in ~30% of the time it takes `rimraf` (3.3 times faster)
parent 068dd825
const { join } = require('path')
const { recursiveDelete } = require('next/dist/lib/recursive-delete')
const resolveDataDir = join(__dirname, `fixtures-${process.argv[2]}`)
async function test () {
const time = process.hrtime()
await recursiveDelete(resolveDataDir)
const hrtime = process.hrtime(time)
const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
const milliseconds = nanoseconds / 1e6
console.log(milliseconds)
}
test()
const { join } = require('path')
const { promisify } = require('util')
const rimrafMod = require('rimraf')
const resolveDataDir = join(__dirname, `fixtures-${process.argv[2]}`, '**/*')
const rimraf = promisify(rimrafMod)
async function test () {
const time = process.hrtime()
await rimraf(resolveDataDir)
const hrtime = process.hrtime(time)
const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
const milliseconds = nanoseconds / 1e6
console.log(milliseconds)
}
test()
# Uses https://github.com/divmain/fuzzponent
mkdir fixtures-1
cd fixtures-1
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 1"
node rimraf.js 1
mkdir fixtures-2
cd fixtures-2
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 2"
node rimraf.js 2
mkdir fixtures-3
cd fixtures-3
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 3"
node rimraf.js 3
mkdir fixtures-4
cd fixtures-4
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 4"
node rimraf.js 4
mkdir fixtures-5
cd fixtures-5
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "rimraf 5"
node rimraf.js 5
echo "-----------"
cd fixtures-1
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 1"
node recursive-delete.js 1
cd fixtures-2
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 2"
node recursive-delete.js 2
cd fixtures-3
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 3"
node recursive-delete.js 3
cd fixtures-4
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 4"
node recursive-delete.js 4
cd fixtures-5
fuzzponent -d 2 -s 20 > output.txt
cd ..
echo "recursive delete 5"
node recursive-delete.js 5
rm -r fixtures-1 fixtures-2 fixtures-3 fixtures-4 fixtures-5
\ No newline at end of file
import rimrafModule from 'rimraf'
import { cpus } from 'os' import { cpus } from 'os'
import { fork } from 'child_process' import { fork } from 'child_process'
import cp from 'recursive-copy' import cp from 'recursive-copy'
...@@ -9,9 +8,9 @@ import loadConfig from 'next-server/next-config' ...@@ -9,9 +8,9 @@ import loadConfig from 'next-server/next-config'
import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants' import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants'
import createProgress from 'tty-aware-progress' import createProgress from 'tty-aware-progress'
import { promisify } from 'util' import { promisify } from 'util'
import { recursiveDelete } from '../lib/recursive-delete'
const mkdirp = promisify(mkdirpModule) const mkdirp = promisify(mkdirpModule)
const rimraf = promisify(rimrafModule)
export default async function (dir, options, configuration) { export default async function (dir, options, configuration) {
function log (message) { function log (message) {
...@@ -55,7 +54,7 @@ export default async function (dir, options, configuration) { ...@@ -55,7 +54,7 @@ export default async function (dir, options, configuration) {
// Initialize the output directory // Initialize the output directory
const outDir = options.outdir const outDir = options.outdir
await rimraf(join(outDir, '*')) await recursiveDelete(join(outDir))
await mkdirp(join(outDir, '_next', buildId)) await mkdirp(join(outDir, '_next', buildId))
// Copy static directory // Copy static directory
......
import fs from 'fs'
import { join } from 'path'
import { promisify } from 'util'
const readdir = promisify(fs.readdir)
const stat = promisify(fs.stat)
const rmdir = promisify(fs.rmdir)
const unlink = promisify(fs.unlink)
const sleep = promisify(setTimeout)
const unlinkFile = async (p: string, t = 1): Promise<void> => {
try {
await unlink(p)
} catch (e) {
if ((e.code === 'EBUSY' || e.code === 'ENOTEMPTY' || e.code === 'EPERM' || e.code === 'EMFILE') && t < 3) {
await sleep(t * 100)
return unlinkFile(p, t++)
}
if (e.code === 'ENOENT') {
return
}
throw e
}
}
/**
* Recursively delete directory contents
* @param {string} dir Directory to delete the contents of
* @param {RegExp} filter Filter for the file name, only the name part is considered, not the full path
* @param {boolean} ensure Esures that parameter dir exists, this is not passed recursively
* @returns Promise void
*/
export async function recursiveDelete(dir: string, filter?: RegExp, ensure?: boolean): Promise<void> {
let result
try {
result = await readdir(dir)
} catch (e) {
if (e.code === 'ENOENT' && !ensure) return
throw e
}
await Promise.all(result.map(async (part: string) => {
const absolutePath = join(dir, part)
const pathStat = await stat(absolutePath).catch((e) => {
if (e.code !== 'ENOENT') throw e
})
if (!pathStat) return
if (pathStat.isDirectory()) {
await recursiveDelete(absolutePath, filter)
return rmdir(absolutePath)
}
if (!filter || filter.test(part)) {
return unlinkFile(absolutePath)
}
}))
}
...@@ -75,7 +75,6 @@ ...@@ -75,7 +75,6 @@
"react-is": "16.6.3", "react-is": "16.6.3",
"recursive-copy": "2.0.6", "recursive-copy": "2.0.6",
"resolve": "1.5.0", "resolve": "1.5.0",
"rimraf": "2.6.3",
"serialize-javascript": "1.4.0", "serialize-javascript": "1.4.0",
"source-map": "0.6.1", "source-map": "0.6.1",
"strip-ansi": "3.0.1", "strip-ansi": "3.0.1",
...@@ -113,7 +112,6 @@ ...@@ -113,7 +112,6 @@
"@types/nanoid": "1.2.0", "@types/nanoid": "1.2.0",
"@types/node-fetch": "2.1.4", "@types/node-fetch": "2.1.4",
"@types/resolve": "0.0.8", "@types/resolve": "0.0.8",
"@types/rimraf": "2.0.2",
"@zeit/ncc": "0.15.2", "@zeit/ncc": "0.15.2",
"taskr": "1.1.0", "taskr": "1.1.0",
"typescript": "3.1.6", "typescript": "3.1.6",
......
...@@ -2,19 +2,16 @@ import { join, normalize } from 'path' ...@@ -2,19 +2,16 @@ import { join, normalize } from 'path'
import WebpackDevMiddleware from 'webpack-dev-middleware' import WebpackDevMiddleware from 'webpack-dev-middleware'
import WebpackHotMiddleware from 'webpack-hot-middleware' import WebpackHotMiddleware from 'webpack-hot-middleware'
import errorOverlayMiddleware from './lib/error-overlay-middleware' import errorOverlayMiddleware from './lib/error-overlay-middleware'
import rimrafModule from 'rimraf'
import onDemandEntryHandler, { normalizePage } from './on-demand-entry-handler' import onDemandEntryHandler, { normalizePage } from './on-demand-entry-handler'
import webpack from 'webpack' import webpack from 'webpack'
import getBaseWebpackConfig from '../build/webpack-config' import getBaseWebpackConfig from '../build/webpack-config'
import { IS_BUNDLED_PAGE_REGEX, ROUTE_NAME_REGEX, BLOCKED_PAGES, CLIENT_STATIC_FILES_RUNTIME_AMP } from 'next-server/constants' import { IS_BUNDLED_PAGE_REGEX, ROUTE_NAME_REGEX, BLOCKED_PAGES, CLIENT_STATIC_FILES_RUNTIME_AMP } from 'next-server/constants'
import { NEXT_PROJECT_ROOT_DIST_CLIENT } from '../lib/constants' import { NEXT_PROJECT_ROOT_DIST_CLIENT } from '../lib/constants'
import { route } from 'next-server/dist/server/router' import { route } from 'next-server/dist/server/router'
import { promisify } from 'util'
import { createPagesMapping, createEntrypoints } from '../build/entries' import { createPagesMapping, createEntrypoints } from '../build/entries'
import { watchCompiler } from '../build/output' import { watchCompiler } from '../build/output'
import { findPageFile } from './lib/find-page-file' import { findPageFile } from './lib/find-page-file'
import { recursiveDelete } from '../lib/recursive-delete'
const rimraf = promisify(rimrafModule)
export async function renderScriptError (res, error) { export async function renderScriptError (res, error) {
// Asks CDNs and others to not to cache the errored page // Asks CDNs and others to not to cache the errored page
...@@ -162,7 +159,7 @@ export default class HotReloader { ...@@ -162,7 +159,7 @@ export default class HotReloader {
} }
async clean () { async clean () {
return rimraf(join(this.dir, this.config.distDir), { force: true }) return recursiveDelete(join(this.dir, this.config.distDir))
} }
async getWebpackConfig () { async getWebpackConfig () {
......
...@@ -15,14 +15,35 @@ import { ...@@ -15,14 +15,35 @@ import {
import ssr from './ssr' import ssr from './ssr'
import browser from './browser' import browser from './browser'
import dev from './dev' import dev from './dev'
import { promisify } from 'util'
import fs from 'fs'
jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5 jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
const writeFile = promisify(fs.writeFile)
const mkdir = promisify(fs.mkdir)
const access = promisify(fs.access)
const appDir = join(__dirname, '../') const appDir = join(__dirname, '../')
const context = {} const context = {}
const devContext = {} const devContext = {}
describe('Static Export', () => { describe('Static Export', () => {
it('should delete existing exported files', async () => {
const outdir = join(appDir, 'out')
const tempfile = join(outdir, 'temp.txt')
await mkdir(outdir).catch((e) => { if (e.code !== 'EEXIST') throw e })
await writeFile(tempfile, 'Hello there')
await nextBuild(appDir)
await nextExport(appDir, { outdir })
let doesNotExist = false
await access(tempfile).catch((e) => {
if (e.code === 'ENOENT') doesNotExist = true
})
expect(doesNotExist).toBe(true)
})
beforeAll(async () => { beforeAll(async () => {
const outdir = join(appDir, 'out') const outdir = join(appDir, 'out')
await nextBuild(appDir) await nextBuild(appDir)
......
test_resolvedata
\ No newline at end of file
/* eslint-env jest */
import { recursiveDelete } from 'next/dist/lib/recursive-delete'
import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
import cp from 'recursive-copy'
import { join } from 'path'
const resolveDataDir = join(__dirname, '..', 'isolated', '_resolvedata')
const testResolveDataDir = join(__dirname, '..', 'isolated', 'test_resolvedata')
describe('recursiveDelete', () => {
it('should work', async () => {
await cp(
resolveDataDir,
testResolveDataDir,
{ expand: true }
)
await recursiveDelete(testResolveDataDir)
const result = await recursiveReadDir(testResolveDataDir, /.*/)
expect(result.length).toBe(0)
})
})
...@@ -1579,25 +1579,11 @@ ...@@ -1579,25 +1579,11 @@
dependencies: dependencies:
"@types/node" "*" "@types/node" "*"
"@types/events@*":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7"
integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==
"@types/fresh@0.5.0": "@types/fresh@0.5.0":
version "0.5.0" version "0.5.0"
resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f" resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f"
integrity sha512-eGPzuyc6wZM3sSHJdF7NM2jW6B/xsB014Rqg/iDa6xY02mlfy1w/TE2sYhR8vbHxkzJOXiGo6NuIk3xk35vsgQ== integrity sha512-eGPzuyc6wZM3sSHJdF7NM2jW6B/xsB014Rqg/iDa6xY02mlfy1w/TE2sYhR8vbHxkzJOXiGo6NuIk3xk35vsgQ==
"@types/glob@*":
version "7.1.1"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
dependencies:
"@types/events" "*"
"@types/minimatch" "*"
"@types/node" "*"
"@types/loader-utils@1.1.3": "@types/loader-utils@1.1.3":
version "1.1.3" version "1.1.3"
resolved "https://registry.yarnpkg.com/@types/loader-utils/-/loader-utils-1.1.3.tgz#82b9163f2ead596c68a8c03e450fbd6e089df401" resolved "https://registry.yarnpkg.com/@types/loader-utils/-/loader-utils-1.1.3.tgz#82b9163f2ead596c68a8c03e450fbd6e089df401"
...@@ -1611,11 +1597,6 @@ ...@@ -1611,11 +1597,6 @@
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d"
integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw== integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==
"@types/minimatch@*":
version "3.0.3"
resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
"@types/mkdirp@0.5.2": "@types/mkdirp@0.5.2":
version "0.5.2" version "0.5.2"
resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.2.tgz#503aacfe5cc2703d5484326b1b27efa67a339c1f" resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.2.tgz#503aacfe5cc2703d5484326b1b27efa67a339c1f"
...@@ -1681,14 +1662,6 @@ ...@@ -1681,14 +1662,6 @@
dependencies: dependencies:
"@types/node" "*" "@types/node" "*"
"@types/rimraf@2.0.2":
version "2.0.2"
resolved "https://registry.yarnpkg.com/@types/rimraf/-/rimraf-2.0.2.tgz#7f0fc3cf0ff0ad2a99bb723ae1764f30acaf8b6e"
integrity sha512-Hm/bnWq0TCy7jmjeN5bKYij9vw5GrDFWME4IuxV08278NtU/VdGbzsBohcCUJ7+QMqmUq5hpRKB39HeQWJjztQ==
dependencies:
"@types/glob" "*"
"@types/node" "*"
"@types/send@0.14.4": "@types/send@0.14.4":
version "0.14.4" version "0.14.4"
resolved "https://registry.yarnpkg.com/@types/send/-/send-0.14.4.tgz#d70458b030305999db619a7b057f7105058bd0ff" resolved "https://registry.yarnpkg.com/@types/send/-/send-0.14.4.tgz#d70458b030305999db619a7b057f7105058bd0ff"
...@@ -10849,7 +10822,7 @@ rgba-regex@^1.0.0: ...@@ -10849,7 +10822,7 @@ rgba-regex@^1.0.0:
resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3" resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3"
integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM= integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM=
rimraf@2, rimraf@2.6.3, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2: rimraf@2, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2:
version "2.6.3" version "2.6.3"
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA== integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment