Commit 5514949d authored by Connor Davis's avatar Connor Davis Committed by Tim Neutkens

Remove glob package (#6415)

We don't use a lot of the features of `glob`, so let's remove it in favor of a leaner approach using regex.

It's failing on windows and I have no idea why and don't own a windows machine 🤦🏼‍♂️

(Ignore some of the commits in here, I forgot to create the new branch before I started working)
parent 993cab89
# Uses https://github.com/divmain/fuzzponent
mkdir fixtures
cd fixtures
fuzzponent -d 2 -s 20
const { join } = require('path')
const { promisify } = require('util')
const globMod = require('glob')
const glob = promisify(globMod)
const resolveDataDir = join(__dirname, 'fixtures', '**/*')
async function test () {
const time = process.hrtime()
await glob(resolveDataDir)
const hrtime = process.hrtime(time)
const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
const milliseconds = nanoseconds / 1e6
console.log(milliseconds)
}
async function run () {
for (let i = 0; i < 50; i++) {
await test()
}
}
run()
const { join } = require('path')
const { recursiveReadDir } = require('next/dist/lib/recursive-readdir')
const resolveDataDir = join(__dirname, 'fixtures')
async function test () {
const time = process.hrtime()
await recursiveReadDir(resolveDataDir, /\.js$/)
const hrtime = process.hrtime(time)
const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
const milliseconds = nanoseconds / 1e6
console.log(milliseconds)
}
async function run () {
for (let i = 0; i < 50; i++) {
await test()
}
}
run()
...@@ -9,7 +9,7 @@ type PagesMapping = { ...@@ -9,7 +9,7 @@ type PagesMapping = {
export function createPagesMapping(pagePaths: string[], extensions: string[]): PagesMapping { export function createPagesMapping(pagePaths: string[], extensions: string[]): PagesMapping {
const pages: PagesMapping = pagePaths.reduce((result: PagesMapping, pagePath): PagesMapping => { const pages: PagesMapping = pagePaths.reduce((result: PagesMapping, pagePath): PagesMapping => {
const page = `/${pagePath.replace(new RegExp(`\\.+(${extensions.join('|')})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '') const page = `${pagePath.replace(new RegExp(`\\.+(${extensions.join('|')})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
result[page === '' ? '/' : page] = join(PAGES_DIR_ALIAS, pagePath).replace(/\\/g, '/') result[page === '' ? '/' : page] = join(PAGES_DIR_ALIAS, pagePath).replace(/\\/g, '/')
return result return result
}, {}) }, {})
......
...@@ -7,19 +7,16 @@ import { generateBuildId } from './generate-build-id' ...@@ -7,19 +7,16 @@ import { generateBuildId } from './generate-build-id'
import { writeBuildId } from './write-build-id' import { writeBuildId } from './write-build-id'
import { isWriteable } from './is-writeable' import { isWriteable } from './is-writeable'
import { runCompiler, CompilerResult } from './compiler' import { runCompiler, CompilerResult } from './compiler'
import globModule from 'glob' import {recursiveReadDir} from '../lib/recursive-readdir'
import { promisify } from 'util'
import { createPagesMapping, createEntrypoints } from './entries' import { createPagesMapping, createEntrypoints } from './entries'
import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages' import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages'
import chalk from 'chalk' import chalk from 'chalk'
const glob = promisify(globModule)
function collectPages( function collectPages(
directory: string, directory: string,
pageExtensions: string[] pageExtensions: string[]
): Promise<string[]> { ): Promise<string[]> {
return glob(`**/*.+(${pageExtensions.join('|')})`, { cwd: directory }) return recursiveReadDir(directory, new RegExp(`\\.(?:${pageExtensions.join('|')})$`))
} }
function printTreeView(list: string[]) { function printTreeView(list: string[]) {
......
import fs from 'fs'
import { join } from 'path'
import { promisify } from 'util'
const readdir = promisify(fs.readdir)
const stat = promisify(fs.stat)
/**
* Recursively read directory
* @param {string} dir Directory to read
* @param {RegExp} filter Filter for the file name, only the name part is considered, not the full path
* @param {string[]=[]} arr This doesn't have to be provided, it's used for the recursion
* @param {string=dir`} rootDir Used to replace the initial path, only the relative path is left, it's faster than path.relative.
* @returns Promise array holding all relative paths
*/
export async function recursiveReadDir(dir: string, filter: RegExp, arr: string[] = [], rootDir: string = dir): Promise<string[]> {
const result = await readdir(dir)
await Promise.all(result.map(async (part: string) => {
const absolutePath = join(dir, part)
const pathStat = await stat(absolutePath)
if (pathStat.isDirectory()) {
await recursiveReadDir(absolutePath, filter, arr, rootDir)
return
}
if (!filter.test(part)) {
return
}
arr.push(absolutePath.replace(rootDir, ''))
}))
return arr
}
...@@ -64,7 +64,6 @@ ...@@ -64,7 +64,6 @@
"find-cache-dir": "2.0.0", "find-cache-dir": "2.0.0",
"find-up": "2.1.0", "find-up": "2.1.0",
"fresh": "0.5.2", "fresh": "0.5.2",
"glob": "7.1.2",
"hoist-non-react-statics": "3.2.0", "hoist-non-react-statics": "3.2.0",
"launch-editor": "2.2.1", "launch-editor": "2.2.1",
"loader-utils": "1.1.0", "loader-utils": "1.1.0",
...@@ -111,7 +110,6 @@ ...@@ -111,7 +110,6 @@
"@types/cross-spawn": "6.0.0", "@types/cross-spawn": "6.0.0",
"@types/etag": "1.8.0", "@types/etag": "1.8.0",
"@types/fresh": "0.5.0", "@types/fresh": "0.5.0",
"@types/glob": "7.1.1",
"@types/loader-utils": "1.1.3", "@types/loader-utils": "1.1.3",
"@types/mkdirp": "0.5.2", "@types/mkdirp": "0.5.2",
"@types/nanoid": "1.2.0", "@types/nanoid": "1.2.0",
......
...@@ -8,12 +8,11 @@ import webpack from 'webpack' ...@@ -8,12 +8,11 @@ import webpack from 'webpack'
import getBaseWebpackConfig from '../build/webpack-config' import getBaseWebpackConfig from '../build/webpack-config'
import { IS_BUNDLED_PAGE_REGEX, ROUTE_NAME_REGEX, BLOCKED_PAGES } from 'next-server/constants' import { IS_BUNDLED_PAGE_REGEX, ROUTE_NAME_REGEX, BLOCKED_PAGES } from 'next-server/constants'
import { route } from 'next-server/dist/server/router' import { route } from 'next-server/dist/server/router'
import globModule from 'glob'
import { promisify } from 'util' import { promisify } from 'util'
import { createPagesMapping, createEntrypoints } from '../build/entries' import { createPagesMapping, createEntrypoints } from '../build/entries'
import { watchCompiler } from '../build/output' import { watchCompiler } from '../build/output'
import { findPageFile } from './lib/find-page-file'
const glob = promisify(globModule)
const rimraf = promisify(rimrafModule) const rimraf = promisify(rimrafModule)
export async function renderScriptError (res, error) { export async function renderScriptError (res, error) {
...@@ -166,8 +165,13 @@ export default class HotReloader { ...@@ -166,8 +165,13 @@ export default class HotReloader {
} }
async getWebpackConfig () { async getWebpackConfig () {
const pagePaths = await glob(`+(_app|_document).+(${this.config.pageExtensions.join('|')})`, { cwd: join(this.dir, 'pages') }) const pagesDir = join(this.dir, 'pages')
const pages = createPagesMapping(pagePaths, this.config.pageExtensions) const pagePaths = await Promise.all([
findPageFile(pagesDir, '/_app', this.config.pageExtensions),
findPageFile(pagesDir, '/_document', this.config.pageExtensions)
])
const pages = createPagesMapping(pagePaths.filter(i => i !== null), this.config.pageExtensions)
const entrypoints = createEntrypoints(pages, 'server', this.buildId, this.config) const entrypoints = createEntrypoints(pages, 'server', this.buildId, this.config)
return [ return [
getBaseWebpackConfig(this.dir, { dev: true, isServer: false, config: this.config, buildId: this.buildId, entrypoints: entrypoints.client }), getBaseWebpackConfig(this.dir, { dev: true, isServer: false, config: this.config, buildId: this.buildId, entrypoints: entrypoints.client }),
......
import { join } from 'path'
import {isWriteable} from '../../build/is-writeable'
export async function findPageFile(rootDir: string, normalizedPagePath: string, pageExtensions: string[]): Promise<string|null> {
for (const extension of pageExtensions) {
const relativePagePath = `${normalizedPagePath}.${extension}`
const pagePath = join(rootDir, relativePagePath)
if (await isWriteable(pagePath)) {
return relativePagePath
}
const relativePagePathWithIndex = join(normalizedPagePath, `index.${extension}`)
const pagePathWithIndex = join(rootDir, relativePagePathWithIndex)
if (await isWriteable(pagePathWithIndex)) {
return relativePagePathWithIndex
}
}
return null
}
...@@ -2,21 +2,17 @@ import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin' ...@@ -2,21 +2,17 @@ import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin'
import { EventEmitter } from 'events' import { EventEmitter } from 'events'
import { join } from 'path' import { join } from 'path'
import { parse } from 'url' import { parse } from 'url'
import fs from 'fs'
import { promisify } from 'util'
import globModule from 'glob'
import { pageNotFoundError } from 'next-server/dist/server/require' import { pageNotFoundError } from 'next-server/dist/server/require'
import { normalizePagePath } from 'next-server/dist/server/normalize-page-path' import { normalizePagePath } from 'next-server/dist/server/normalize-page-path'
import { ROUTE_NAME_REGEX, IS_BUNDLED_PAGE_REGEX } from 'next-server/constants' import { ROUTE_NAME_REGEX, IS_BUNDLED_PAGE_REGEX } from 'next-server/constants'
import { stringify } from 'querystring' import { stringify } from 'querystring'
import { findPageFile } from './lib/find-page-file'
import { isWriteable } from '../build/is-writeable'
const ADDED = Symbol('added') const ADDED = Symbol('added')
const BUILDING = Symbol('building') const BUILDING = Symbol('building')
const BUILT = Symbol('built') const BUILT = Symbol('built')
const glob = promisify(globModule)
const access = promisify(fs.access)
// Based on https://github.com/webpack/webpack/blob/master/lib/DynamicEntryPlugin.js#L29-L37 // Based on https://github.com/webpack/webpack/blob/master/lib/DynamicEntryPlugin.js#L29-L37
function addEntry (compilation, context, name, entry) { function addEntry (compilation, context, name, entry) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
...@@ -36,6 +32,7 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, { ...@@ -36,6 +32,7 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, {
maxInactiveAge, maxInactiveAge,
pagesBufferLength pagesBufferLength
}) { }) {
const pagesDir = join(dir, 'pages')
const clients = new Set() const clients = new Set()
const evtSourceHeaders = { const evtSourceHeaders = {
'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Origin': '*',
...@@ -61,9 +58,8 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, { ...@@ -61,9 +58,8 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, {
const allEntries = Object.keys(entries).map(async (page) => { const allEntries = Object.keys(entries).map(async (page) => {
const { name, absolutePagePath } = entries[page] const { name, absolutePagePath } = entries[page]
try { const pageExists = await isWriteable(absolutePagePath)
await access(absolutePagePath, (fs.constants || fs).W_OK) if (!pageExists) {
} catch (err) {
console.warn('Page was removed', page) console.warn('Page was removed', page)
delete entries[page] delete entries[page]
return return
...@@ -220,22 +216,18 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, { ...@@ -220,22 +216,18 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, {
throw pageNotFoundError(normalizedPagePath) throw pageNotFoundError(normalizedPagePath)
} }
const extensions = pageExtensions.join('|') let pagePath = await findPageFile(pagesDir, normalizedPagePath, pageExtensions)
const pagesDir = join(dir, 'pages')
let paths = await glob(`{${normalizedPagePath.slice(1)}/index,${normalizedPagePath.slice(1)}}.+(${extensions})`, { cwd: pagesDir })
// Default the /_error route to the Next.js provided default page // Default the /_error route to the Next.js provided default page
if (page === '/_error' && paths.length === 0) { if (page === '/_error' && pagePath === null) {
paths = ['next/dist/pages/_error'] pagePath = 'next/dist/pages/_error'
} }
if (paths.length === 0) { if (pagePath === null) {
throw pageNotFoundError(normalizedPagePath) throw pageNotFoundError(normalizedPagePath)
} }
const pagePath = paths[0] let pageUrl = `/${pagePath.replace(new RegExp(`\\.+(?:${pageExtensions.join('|')})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
let pageUrl = `/${pagePath.replace(new RegExp(`\\.+(${extensions})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
pageUrl = pageUrl === '' ? '/' : pageUrl pageUrl = pageUrl === '' ? '/' : pageUrl
const bundleFile = pageUrl === '/' ? '/index.js' : `${pageUrl}.js` const bundleFile = pageUrl === '/' ? '/index.js' : `${pageUrl}.js`
const name = join('static', buildId, 'pages', bundleFile) const name = join('static', buildId, 'pages', bundleFile)
......
/* eslint-env jest */
import { findPageFile } from 'next/dist/server/lib/find-page-file'
import { normalizePagePath } from 'next-server/dist/server/normalize-page-path'
import { join } from 'path'
const resolveDataDir = join(__dirname, '..', 'isolated', '_resolvedata')
const dirWithPages = join(resolveDataDir, 'readdir', 'pages')
describe('findPageFile', () => {
it('should work', async () => {
const pagePath = normalizePagePath('/nav/about')
const result = await findPageFile(dirWithPages, pagePath, ['jsx', 'js'])
expect(result).toMatch(/^[\\/]nav[\\/]about\.js/)
})
it('should work with nested index.js', async () => {
const pagePath = normalizePagePath('/nested')
const result = await findPageFile(dirWithPages, pagePath, ['jsx', 'js'])
expect(result).toMatch(/^[\\/]nested[\\/]index\.js/)
})
it('should prefer prefered.js before prefered/index.js', async () => {
const pagePath = normalizePagePath('/prefered')
const result = await findPageFile(dirWithPages, pagePath, ['jsx', 'js'])
expect(result).toMatch(/^[\\/]prefered\.js/)
})
})
/* eslint-env jest */
import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
import { join } from 'path'
const resolveDataDir = join(__dirname, '..', 'isolated', '_resolvedata')
const dirWithPages = join(resolveDataDir, 'readdir', 'pages')
describe('recursiveReadDir', () => {
it('should work', async () => {
const result = await recursiveReadDir(dirWithPages, /\.js/)
const pages = [/^[\\/]index\.js/, /^[\\/]prefered\.js/, /^[\\/]nav[\\/]about\.js/, /^[\\/]nav[\\/]index\.js/, /^[\\/]nested[\\/]index\.js/, /^[\\/]prefered[\\/]index\.js/, /^[\\/]nav[\\/]products[\\/]product\.js/]
expect(result.filter((item) => {
for (const page of pages) {
if (page.test(item)) {
return false
}
}
return true
}).length).toBe(0)
})
})
...@@ -1550,7 +1550,7 @@ ...@@ -1550,7 +1550,7 @@
resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f" resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f"
integrity sha512-eGPzuyc6wZM3sSHJdF7NM2jW6B/xsB014Rqg/iDa6xY02mlfy1w/TE2sYhR8vbHxkzJOXiGo6NuIk3xk35vsgQ== integrity sha512-eGPzuyc6wZM3sSHJdF7NM2jW6B/xsB014Rqg/iDa6xY02mlfy1w/TE2sYhR8vbHxkzJOXiGo6NuIk3xk35vsgQ==
"@types/glob@*", "@types/glob@7.1.1": "@types/glob@*":
version "7.1.1" version "7.1.1"
resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w== integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
...@@ -1599,9 +1599,9 @@ ...@@ -1599,9 +1599,9 @@
"@types/node" "*" "@types/node" "*"
"@types/node@*": "@types/node@*":
version "11.9.4" version "11.9.5"
resolved "https://registry.yarnpkg.com/@types/node/-/node-11.9.4.tgz#ceb0048a546db453f6248f2d1d95e937a6f00a14" resolved "https://registry.yarnpkg.com/@types/node/-/node-11.9.5.tgz#011eece9d3f839a806b63973e228f85967b79ed3"
integrity sha512-Zl8dGvAcEmadgs1tmSPcvwzO1YRsz38bVJQvH1RvRqSR9/5n61Q1ktcDL0ht3FXWR+ZpVmXVwN1LuH4Ax23NsA== integrity sha512-vVjM0SVzgaOUpflq4GYBvCpozes8OgIIS5gVXVka+OfK3hvnkC1i93U8WiY2OtNE4XUWyyy/86Kf6e0IHTQw1Q==
"@types/prop-types@*": "@types/prop-types@*":
version "15.5.9" version "15.5.9"
...@@ -1981,9 +1981,9 @@ ajv@^5.1.0: ...@@ -1981,9 +1981,9 @@ ajv@^5.1.0:
json-schema-traverse "^0.3.0" json-schema-traverse "^0.3.0"
ajv@^6.0.1, ajv@^6.1.0, ajv@^6.5.0, ajv@^6.5.5: ajv@^6.0.1, ajv@^6.1.0, ajv@^6.5.0, ajv@^6.5.5:
version "6.9.1" version "6.9.2"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.9.1.tgz#a4d3683d74abc5670e75f0b16520f70a20ea8dc1" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.9.2.tgz#4927adb83e7f48e5a32b45729744c71ec39c9c7b"
integrity sha512-XDN92U311aINL77ieWHmqCcNlwjoP5cHXDxIxbf2MaPYuCXOHS7gHH8jktxeK5omgd52XbSTX6a4Piwd1pQmzA== integrity sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==
dependencies: dependencies:
fast-deep-equal "^2.0.1" fast-deep-equal "^2.0.1"
fast-json-stable-stringify "^2.0.0" fast-json-stable-stringify "^2.0.0"
...@@ -5679,18 +5679,6 @@ glob-to-regexp@^0.3.0: ...@@ -5679,18 +5679,6 @@ glob-to-regexp@^0.3.0:
resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab"
integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=
glob@7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
integrity sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1: glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1:
version "7.1.3" version "7.1.3"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment