Unverified Commit 922a4583 authored by Shu Ding's avatar Shu Ding Committed by GitHub

Optimize next-app-loader file resolution (#51924)

This PR changes `fs.stat` calls (to check if a file exists or not) to be
aggregated as `fs.readdir` calls and cached during the entire loader
pass. This is because we have a huge amount of file to check, but they
are always in a small amount of directories.

For example, a route that's 5-directory deep (`/a/b/c/d/page.js`) can
have up to 4,000 special segments and metadata files to check. However
they're always under these 5 directories. So instead of 4,000 `fs.stat`
calls let's just do 5 `fs.readdir` calls.

Didn't measure it carefully but a quick local test shows a 20~30%
improvement.

<details>
<summary>Another idea for future improvements</summary>
Currently, we create a MissingModuleDependency for any combination of
possible metadata filename. But in reality, we only need to track them
incrementally by index. For example if `icon1.js` is missing, it's kinda
waste to still track `icon2.js`~`icon9.js` because `icon1.js` will be
added first. This change should at least reduce the number of file
watchers by 80% and the initial compilation time by 0.5~1s, depending on
the actual route.
</details>

---------
Co-authored-by: 's avatarTim Neutkens <tim@timneutkens.nl>
parent b236670f
...@@ -34,7 +34,7 @@ async function enumMetadataFiles( ...@@ -34,7 +34,7 @@ async function enumMetadataFiles(
: [] : []
) )
for (const name of possibleFileNames) { for (const name of possibleFileNames) {
const resolved = await metadataResolver(path.join(dir, name), extensions) const resolved = await metadataResolver(dir, name, extensions)
if (resolved) { if (resolved) {
collectedFiles.push(resolved) collectedFiles.push(resolved)
} }
...@@ -123,14 +123,15 @@ export async function createStaticMetadataFromRoute( ...@@ -123,14 +123,15 @@ export async function createStaticMetadataFromRoute(
}) })
} }
await Promise.all([ // Intentially make these serial to reuse directory access cache.
collectIconModuleIfExists('icon'), await collectIconModuleIfExists('icon')
collectIconModuleIfExists('apple'), await collectIconModuleIfExists('apple')
collectIconModuleIfExists('openGraph'), await collectIconModuleIfExists('openGraph')
collectIconModuleIfExists('twitter'), await collectIconModuleIfExists('twitter')
isRootLayoutOrRootPage && collectIconModuleIfExists('favicon'), if (isRootLayoutOrRootPage) {
isRootLayoutOrRootPage && collectIconModuleIfExists('manifest'), await collectIconModuleIfExists('favicon')
]) await collectIconModuleIfExists('manifest')
}
return hasStaticMetadataFiles ? staticImagesMetadata : null return hasStaticMetadataFiles ? staticImagesMetadata : null
} }
......
...@@ -21,7 +21,6 @@ import { AppPathnameNormalizer } from '../../../server/future/normalizers/built/ ...@@ -21,7 +21,6 @@ import { AppPathnameNormalizer } from '../../../server/future/normalizers/built/
import { RouteKind } from '../../../server/future/route-kind' import { RouteKind } from '../../../server/future/route-kind'
import { AppRouteRouteModuleOptions } from '../../../server/future/route-modules/app-route/module' import { AppRouteRouteModuleOptions } from '../../../server/future/route-modules/app-route/module'
import { AppBundlePathNormalizer } from '../../../server/future/normalizers/built/app/app-bundle-path-normalizer' import { AppBundlePathNormalizer } from '../../../server/future/normalizers/built/app/app-bundle-path-normalizer'
import { FileType, fileExists } from '../../../lib/file-exists'
import { MiddlewareConfig } from '../../analysis/get-page-static-info' import { MiddlewareConfig } from '../../analysis/get-page-static-info'
export type AppLoaderOptions = { export type AppLoaderOptions = {
...@@ -59,7 +58,8 @@ type PathResolver = ( ...@@ -59,7 +58,8 @@ type PathResolver = (
pathname: string pathname: string
) => Promise<string | undefined> | string | undefined ) => Promise<string | undefined> | string | undefined
export type MetadataResolver = ( export type MetadataResolver = (
pathname: string, dir: string,
filename: string,
extensions: readonly string[] extensions: readonly string[]
) => Promise<string | undefined> ) => Promise<string | undefined>
...@@ -511,16 +511,45 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -511,16 +511,45 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
return createAbsolutePath(appDir, pathToResolve) return createAbsolutePath(appDir, pathToResolve)
} }
// Cached checker to see if a file exists in a given directory.
// This can be more efficient than checking them with `fs.stat` one by one
// because all the thousands of files are likely in a few possible directories.
// Note that it should only be cached for this compilation, not globally.
const filesInDir = new Map<string, Set<string>>()
const fileExistsInDirectory = async (dirname: string, fileName: string) => {
const existingFiles = filesInDir.get(dirname)
if (existingFiles) {
return existingFiles.has(fileName)
}
try {
const files = await fs.readdir(dirname, { withFileTypes: true })
const fileNames = new Set<string>()
for (const file of files) {
if (file.isFile()) {
fileNames.add(file.name)
}
}
filesInDir.set(dirname, fileNames)
return fileNames.has(fileName)
} catch (err) {
return false
}
}
const resolver: PathResolver = async (pathname) => { const resolver: PathResolver = async (pathname) => {
const absolutePath = createAbsolutePath(appDir, pathname) const absolutePath = createAbsolutePath(appDir, pathname)
const filenameIndex = absolutePath.lastIndexOf(path.sep)
const dirname = absolutePath.slice(0, filenameIndex)
const filename = absolutePath.slice(filenameIndex + 1)
let result: string | undefined let result: string | undefined
for (const ext of extensions) { for (const ext of extensions) {
const absolutePathWithExtension = `${absolutePath}${ext}` const absolutePathWithExtension = `${absolutePath}${ext}`
if ( if (
!result && !result &&
(await fileExists(absolutePathWithExtension, FileType.File)) (await fileExistsInDirectory(dirname, `${filename}${ext}`))
) { ) {
result = absolutePathWithExtension result = absolutePathWithExtension
} }
...@@ -532,18 +561,20 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -532,18 +561,20 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
return result return result
} }
const metadataResolver: MetadataResolver = async (pathname, exts) => { const metadataResolver: MetadataResolver = async (
const absolutePath = createAbsolutePath(appDir, pathname) dirname,
filename,
exts
) => {
const absoluteDir = createAbsolutePath(appDir, dirname)
let result: string | undefined let result: string | undefined
for (const ext of exts) { for (const ext of exts) {
// Compared to `resolver` above the exts do not have the `.` included already, so it's added here. // Compared to `resolver` above the exts do not have the `.` included already, so it's added here.
const absolutePathWithExtension = `${absolutePath}.${ext}` const filenameWithExt = `${filename}.${ext}`
if ( const absolutePathWithExtension = `${absoluteDir}${path.sep}${filenameWithExt}`
!result && if (!result && (await fileExistsInDirectory(dirname, filenameWithExt))) {
(await fileExists(absolutePathWithExtension, FileType.File))
) {
result = absolutePathWithExtension result = absolutePathWithExtension
} }
// Call `addMissingDependency` for all files even if they didn't match, // Call `addMissingDependency` for all files even if they didn't match,
...@@ -611,7 +642,8 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -611,7 +642,8 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
throw new Error(message) throw new Error(message)
} }
// Get the new result with the created root layout. // Clear fs cache, get the new result with the created root layout.
filesInDir.clear()
treeCodeResult = await createTreeCodeFromPath(pagePath, { treeCodeResult = await createTreeCodeFromPath(pagePath, {
resolveDir, resolveDir,
resolver, resolver,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment