Unverified Commit dd714796 authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Optimize next-app-loader resolving speed (#50745)

## What?

We recently implemented an optimized resolving method for `app` in Turbopack, this ports some of the main changes in that resolving logic to optimize `next-app-loader` which during compilation resolves the tree structure that we use to render in `app-render.tsx`.

Here's the results for a page that is nested a few levels deep on vercel.com using App Router. These results only cover `next-app-loader`, not any modules compiled below it.

### Before

<img width="671" alt="CleanShot 2023-06-03 at 22 36 26@2x" src="https://github.com/vercel/next.js/assets/6324199/0edeb060-2460-4a7d-95a7-1c22ea26a065">

### After

<img width="673" alt="CleanShot 2023-06-03 at 22 55 10@2x" src="https://github.com/vercel/next.js/assets/6324199/f40964fc-b169-4d95-8711-73cbff3ec76a">


## Raw numbers

<table>
<tr>
 <td>Before</td>
 <td>After</td>
 <td>Delta</td>
 <td>Delta (percent)</td>
</tr>
<tr>
 <td>1.620 ms</td>
 <td>76.39 ms</td>
 <td>-1.543.61 ms</td>
  <td>-95.2%</td>
</tr>
</table>

## How?

Changed the resolving logic to use `fileExists`, looping over the provided pageExtensions.
For Turbopack we have a process that does only one pass for generating all trees. That also only reads directories instead of checking individual files, which is even better (<5ms for generating all possible trees) but this PR is a quick win that has a big impact already without refactoring the entire entries generation in webpack.
parent 18d112fb
...@@ -28,7 +28,7 @@ import { ...@@ -28,7 +28,7 @@ import {
PAGES_DIR_ALIAS, PAGES_DIR_ALIAS,
INSTRUMENTATION_HOOK_FILENAME, INSTRUMENTATION_HOOK_FILENAME,
} from '../lib/constants' } from '../lib/constants'
import { fileExists } from '../lib/file-exists' import { FileType, fileExists } from '../lib/file-exists'
import { findPagesDir } from '../lib/find-pages-dir' import { findPagesDir } from '../lib/find-pages-dir'
import loadCustomRoutes, { import loadCustomRoutes, {
CustomRoutes, CustomRoutes,
...@@ -588,7 +588,7 @@ export default async function build( ...@@ -588,7 +588,7 @@ export default async function build(
for (const page in mappedPages) { for (const page in mappedPages) {
const hasPublicPageFile = await fileExists( const hasPublicPageFile = await fileExists(
path.join(publicDir, page === '/' ? '/index' : page), path.join(publicDir, page === '/' ? '/index' : page),
'file' FileType.File
) )
if (hasPublicPageFile) { if (hasPublicPageFile) {
conflictingPublicFiles.push(page) conflictingPublicFiles.push(page)
......
import type webpack from 'webpack'
import type { import type {
CollectingMetadata, CollectingMetadata,
PossibleStaticMetadataFileNameConvention, PossibleStaticMetadataFileNameConvention,
...@@ -7,6 +6,7 @@ import path from 'path' ...@@ -7,6 +6,7 @@ import path from 'path'
import { stringify } from 'querystring' import { stringify } from 'querystring'
import { STATIC_METADATA_IMAGES } from '../../../../lib/metadata/is-metadata-route' import { STATIC_METADATA_IMAGES } from '../../../../lib/metadata/is-metadata-route'
import { WEBPACK_RESOURCE_QUERIES } from '../../../../lib/constants' import { WEBPACK_RESOURCE_QUERIES } from '../../../../lib/constants'
import { MetadataResolver } from '../next-app-loader'
const METADATA_TYPE = 'metadata' const METADATA_TYPE = 'metadata'
...@@ -16,16 +16,14 @@ async function enumMetadataFiles( ...@@ -16,16 +16,14 @@ async function enumMetadataFiles(
filename: string, filename: string,
extensions: readonly string[], extensions: readonly string[],
{ {
resolvePath, metadataResolver,
loaderContext,
// When set to true, possible filename without extension could: icon, icon0, ..., icon9 // When set to true, possible filename without extension could: icon, icon0, ..., icon9
numericSuffix, numericSuffix,
}: { }: {
resolvePath: (pathname: string) => Promise<string> metadataResolver: MetadataResolver
loaderContext: webpack.LoaderContext<any>
numericSuffix: boolean numericSuffix: boolean
} }
) { ): Promise<string[]> {
const collectedFiles: string[] = [] const collectedFiles: string[] = []
const possibleFileNames = [filename].concat( const possibleFileNames = [filename].concat(
...@@ -36,19 +34,9 @@ async function enumMetadataFiles( ...@@ -36,19 +34,9 @@ async function enumMetadataFiles(
: [] : []
) )
for (const name of possibleFileNames) { for (const name of possibleFileNames) {
for (const ext of extensions) { const resolved = await metadataResolver(path.join(dir, name), extensions)
const pathname = path.join(dir, `${name}.${ext}`) if (resolved) {
try { collectedFiles.push(resolved)
const resolved = await resolvePath(pathname)
loaderContext.addDependency(resolved)
collectedFiles.push(resolved)
} catch (err: any) {
if (!err.message.includes("Can't resolve")) {
throw err
}
loaderContext.addMissingDependency(pathname)
}
} }
} }
...@@ -59,16 +47,14 @@ export async function createStaticMetadataFromRoute( ...@@ -59,16 +47,14 @@ export async function createStaticMetadataFromRoute(
resolvedDir: string, resolvedDir: string,
{ {
segment, segment,
resolvePath, metadataResolver,
isRootLayoutOrRootPage, isRootLayoutOrRootPage,
loaderContext,
pageExtensions, pageExtensions,
basePath, basePath,
}: { }: {
segment: string segment: string
resolvePath: (pathname: string) => Promise<string> metadataResolver: MetadataResolver
isRootLayoutOrRootPage: boolean isRootLayoutOrRootPage: boolean
loaderContext: webpack.LoaderContext<any>
pageExtensions: string[] pageExtensions: string[]
basePath: string basePath: string
} }
...@@ -82,11 +68,6 @@ export async function createStaticMetadataFromRoute( ...@@ -82,11 +68,6 @@ export async function createStaticMetadataFromRoute(
manifest: undefined, manifest: undefined,
} }
const opts = {
resolvePath,
loaderContext,
}
async function collectIconModuleIfExists( async function collectIconModuleIfExists(
type: PossibleStaticMetadataFileNameConvention type: PossibleStaticMetadataFileNameConvention
) { ) {
...@@ -96,7 +77,7 @@ export async function createStaticMetadataFromRoute( ...@@ -96,7 +77,7 @@ export async function createStaticMetadataFromRoute(
resolvedDir, resolvedDir,
'manifest', 'manifest',
staticManifestExtension.concat(pageExtensions), staticManifestExtension.concat(pageExtensions),
{ ...opts, numericSuffix: false } { metadataResolver, numericSuffix: false }
) )
if (manifestFile.length > 0) { if (manifestFile.length > 0) {
hasStaticMetadataFiles = true hasStaticMetadataFiles = true
...@@ -116,7 +97,7 @@ export async function createStaticMetadataFromRoute( ...@@ -116,7 +97,7 @@ export async function createStaticMetadataFromRoute(
...STATIC_METADATA_IMAGES[type].extensions, ...STATIC_METADATA_IMAGES[type].extensions,
...(type === 'favicon' ? [] : pageExtensions), ...(type === 'favicon' ? [] : pageExtensions),
], ],
{ ...opts, numericSuffix: true } { metadataResolver, numericSuffix: true }
) )
resolvedMetadataFiles resolvedMetadataFiles
.sort((a, b) => a.localeCompare(b)) .sort((a, b) => a.localeCompare(b))
......
...@@ -5,7 +5,6 @@ import type { ModuleReference, CollectedMetadata } from './metadata/types' ...@@ -5,7 +5,6 @@ import type { ModuleReference, CollectedMetadata } from './metadata/types'
import path from 'path' import path from 'path'
import { stringify } from 'querystring' import { stringify } from 'querystring'
import chalk from 'next/dist/compiled/chalk' import chalk from 'next/dist/compiled/chalk'
import { NODE_RESOLVE_OPTIONS } from '../../webpack-config'
import { getModuleBuildInfo } from './get-module-build-info' import { getModuleBuildInfo } from './get-module-build-info'
import { verifyRootLayout } from '../../../lib/verifyRootLayout' import { verifyRootLayout } from '../../../lib/verifyRootLayout'
import * as Log from '../../output/log' import * as Log from '../../output/log'
...@@ -22,6 +21,7 @@ import { AppPathnameNormalizer } from '../../../server/future/normalizers/built/ ...@@ -22,6 +21,7 @@ import { AppPathnameNormalizer } from '../../../server/future/normalizers/built/
import { RouteKind } from '../../../server/future/route-kind' import { RouteKind } from '../../../server/future/route-kind'
import { AppRouteRouteModuleOptions } from '../../../server/future/route-modules/app-route/module' import { AppRouteRouteModuleOptions } from '../../../server/future/route-modules/app-route/module'
import { AppBundlePathNormalizer } from '../../../server/future/normalizers/built/app/app-bundle-path-normalizer' import { AppBundlePathNormalizer } from '../../../server/future/normalizers/built/app/app-bundle-path-normalizer'
import { FileType, fileExists } from '../../../lib/file-exists'
export type AppLoaderOptions = { export type AppLoaderOptions = {
name: string name: string
...@@ -40,8 +40,6 @@ export type AppLoaderOptions = { ...@@ -40,8 +40,6 @@ export type AppLoaderOptions = {
} }
type AppLoader = webpack.LoaderDefinitionFunction<AppLoaderOptions> type AppLoader = webpack.LoaderDefinitionFunction<AppLoaderOptions>
const isNotResolvedError = (err: any) => err.message.includes("Can't resolve")
const FILE_TYPES = { const FILE_TYPES = {
layout: 'layout', layout: 'layout',
template: 'template', template: 'template',
...@@ -53,10 +51,13 @@ const FILE_TYPES = { ...@@ -53,10 +51,13 @@ const FILE_TYPES = {
const GLOBAL_ERROR_FILE_TYPE = 'global-error' const GLOBAL_ERROR_FILE_TYPE = 'global-error'
const PAGE_SEGMENT = 'page$' const PAGE_SEGMENT = 'page$'
type DirResolver = (pathToResolve: string) => string
type PathResolver = ( type PathResolver = (
pathname: string
) => Promise<string | undefined> | string | undefined
export type MetadataResolver = (
pathname: string, pathname: string,
resolveDir?: boolean, extensions: readonly string[]
internal?: boolean
) => Promise<string | undefined> ) => Promise<string | undefined>
export type ComponentsType = { export type ComponentsType = {
...@@ -73,14 +74,14 @@ async function createAppRouteCode({ ...@@ -73,14 +74,14 @@ async function createAppRouteCode({
name, name,
page, page,
pagePath, pagePath,
resolver, resolveAppRoute,
pageExtensions, pageExtensions,
nextConfigOutput, nextConfigOutput,
}: { }: {
name: string name: string
page: string page: string
pagePath: string pagePath: string
resolver: PathResolver resolveAppRoute: PathResolver
pageExtensions: string[] pageExtensions: string[]
nextConfigOutput: NextConfig['output'] nextConfigOutput: NextConfig['output']
}): Promise<string> { }): Promise<string> {
...@@ -90,7 +91,7 @@ async function createAppRouteCode({ ...@@ -90,7 +91,7 @@ async function createAppRouteCode({
// This, when used with the resolver will give us the pathname to the built // This, when used with the resolver will give us the pathname to the built
// route handler file. // route handler file.
let resolvedPagePath = await resolver(routePath) let resolvedPagePath = await resolveAppRoute(routePath)
if (!resolvedPagePath) { if (!resolvedPagePath) {
throw new Error( throw new Error(
`Invariant: could not resolve page path for ${name} at ${routePath}` `Invariant: could not resolve page path for ${name} at ${routePath}`
...@@ -181,15 +182,16 @@ const isDirectory = async (pathname: string) => { ...@@ -181,15 +182,16 @@ const isDirectory = async (pathname: string) => {
async function createTreeCodeFromPath( async function createTreeCodeFromPath(
pagePath: string, pagePath: string,
{ {
resolveDir,
resolver, resolver,
resolvePath,
resolveParallelSegments, resolveParallelSegments,
loaderContext, metadataResolver,
pageExtensions, pageExtensions,
basePath, basePath,
}: { }: {
resolveDir: DirResolver
resolver: PathResolver resolver: PathResolver
resolvePath: (pathname: string) => Promise<string> metadataResolver: MetadataResolver
resolveParallelSegments: ( resolveParallelSegments: (
pathname: string pathname: string
) => [key: string, segment: string | string[]][] ) => [key: string, segment: string | string[]][]
...@@ -197,7 +199,12 @@ async function createTreeCodeFromPath( ...@@ -197,7 +199,12 @@ async function createTreeCodeFromPath(
pageExtensions: string[] pageExtensions: string[]
basePath: string basePath: string
} }
) { ): Promise<{
treeCode: string
pages: string
rootLayout: string | undefined
globalError: string | undefined
}> {
const splittedPath = pagePath.split(/[\\/]/) const splittedPath = pagePath.split(/[\\/]/)
const appDirPrefix = splittedPath[0] const appDirPrefix = splittedPath[0]
const pages: string[] = [] const pages: string[] = []
...@@ -208,9 +215,8 @@ async function createTreeCodeFromPath( ...@@ -208,9 +215,8 @@ async function createTreeCodeFromPath(
async function resolveAdjacentParallelSegments( async function resolveAdjacentParallelSegments(
segmentPath: string segmentPath: string
): Promise<string[]> { ): Promise<string[]> {
const absoluteSegmentPath = await resolver( const absoluteSegmentPath = await resolveDir(
`${appDirPrefix}${segmentPath}`, `${appDirPrefix}${segmentPath}`
true
) )
if (!absoluteSegmentPath) { if (!absoluteSegmentPath) {
...@@ -264,24 +270,17 @@ async function createTreeCodeFromPath( ...@@ -264,24 +270,17 @@ async function createTreeCodeFromPath(
let metadata: Awaited<ReturnType<typeof createStaticMetadataFromRoute>> = let metadata: Awaited<ReturnType<typeof createStaticMetadataFromRoute>> =
null null
try { const routerDirPath = `${appDirPrefix}${segmentPath}`
const routerDirPath = `${appDirPrefix}${segmentPath}` const resolvedRouteDir = await resolveDir(routerDirPath)
const resolvedRouteDir = await resolver(routerDirPath, true)
if (resolvedRouteDir) {
if (resolvedRouteDir) { metadata = await createStaticMetadataFromRoute(resolvedRouteDir, {
metadata = await createStaticMetadataFromRoute(resolvedRouteDir, { basePath,
basePath, segment: segmentPath,
segment: segmentPath, metadataResolver,
resolvePath, isRootLayoutOrRootPage,
isRootLayoutOrRootPage, pageExtensions,
loaderContext, })
pageExtensions,
})
}
} catch (err: any) {
if (isNotResolvedError(err)) {
throw err
}
} }
for (const [parallelKey, parallelSegment] of parallelSegments) { for (const [parallelKey, parallelSegment] of parallelSegments) {
...@@ -448,20 +447,6 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -448,20 +447,6 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
const extensions = pageExtensions.map((extension) => `.${extension}`) const extensions = pageExtensions.map((extension) => `.${extension}`)
const resolveOptions: any = {
...NODE_RESOLVE_OPTIONS,
extensions,
}
const resolve = this.getResolve(resolveOptions)
// a resolver for internal next files. We need to override the extensions, in case
// a project doesn't have the same ones as used by next.
const internalResolve = this.getResolve({
...resolveOptions,
extensions: [...extensions, '.js', '.jsx', '.ts', '.tsx'],
})
const normalizedAppPaths = const normalizedAppPaths =
typeof appPaths === 'string' ? [appPaths] : appPaths || [] typeof appPaths === 'string' ? [appPaths] : appPaths || []
...@@ -496,29 +481,55 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -496,29 +481,55 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
return Object.entries(matched) return Object.entries(matched)
} }
const resolver: PathResolver = async (pathname, resolveDir, internal) => {
if (resolveDir) {
return createAbsolutePath(appDir, pathname)
}
try { const resolveDir: DirResolver = (pathToResolve) => {
const resolved = await (internal ? internalResolve : resolve)( return createAbsolutePath(appDir, pathToResolve)
this.rootContext, }
pathname
) const resolveAppRoute: PathResolver = (pathToResolve) => {
this.addDependency(resolved) return createAbsolutePath(appDir, pathToResolve)
return resolved }
} catch (err: any) {
const absolutePath = createAbsolutePath(appDir, pathname) const resolver: PathResolver = async (pathname) => {
for (const ext of extensions) { const absolutePath = createAbsolutePath(appDir, pathname)
const absolutePathWithExtension = `${absolutePath}${ext}`
let result: string | undefined
for (const ext of extensions) {
const absolutePathWithExtension = `${absolutePath}${ext}`
if (
!result &&
(await fileExists(absolutePathWithExtension, FileType.File))
) {
// Ensures we call `addMissingDependency` for all files that didn't match
result = absolutePathWithExtension
} else {
this.addMissingDependency(absolutePathWithExtension) this.addMissingDependency(absolutePathWithExtension)
} }
if (isNotResolvedError(err)) { }
return undefined
return result
}
const metadataResolver: MetadataResolver = async (pathname, exts) => {
const absolutePath = createAbsolutePath(appDir, pathname)
let result: string | undefined
for (const ext of exts) {
// Compared to `resolver` above the exts do not have the `.` included already, so it's added here.
const absolutePathWithExtension = `${absolutePath}.${ext}`
if (
!result &&
(await fileExists(absolutePathWithExtension, FileType.File))
) {
result = absolutePathWithExtension
} else {
this.addMissingDependency(absolutePathWithExtension)
} }
throw err
} }
return result
} }
if (isAppRouteRoute(name)) { if (isAppRouteRoute(name)) {
...@@ -527,27 +538,23 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -527,27 +538,23 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
page: loaderOptions.page, page: loaderOptions.page,
name, name,
pagePath, pagePath,
resolver, resolveAppRoute,
pageExtensions, pageExtensions,
nextConfigOutput, nextConfigOutput,
}) })
} }
const { let treeCodeResult = await createTreeCodeFromPath(pagePath, {
treeCode, resolveDir,
pages: pageListCode,
rootLayout,
globalError,
} = await createTreeCodeFromPath(pagePath, {
resolver, resolver,
resolvePath: (pathname: string) => resolve(this.rootContext, pathname), metadataResolver,
resolveParallelSegments, resolveParallelSegments,
loaderContext: this, loaderContext: this,
pageExtensions, pageExtensions,
basePath, basePath,
}) })
if (!rootLayout) { if (!treeCodeResult.rootLayout) {
if (!isDev) { if (!isDev) {
// If we're building and missing a root layout, exit the build // If we're building and missing a root layout, exit the build
Log.error( Log.error(
...@@ -581,18 +588,29 @@ const nextAppLoader: AppLoader = async function nextAppLoader() { ...@@ -581,18 +588,29 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
throw new Error(message) throw new Error(message)
} }
// Get the new result with the created root layout.
treeCodeResult = await createTreeCodeFromPath(pagePath, {
resolveDir,
resolver,
metadataResolver,
resolveParallelSegments,
loaderContext: this,
pageExtensions,
basePath,
})
} }
} }
const result = ` const result = `
export ${treeCode} export ${treeCodeResult.treeCode}
export ${pageListCode} export ${treeCodeResult.pages}
export { default as AppRouter } from 'next/dist/client/components/app-router' export { default as AppRouter } from 'next/dist/client/components/app-router'
export { default as LayoutRouter } from 'next/dist/client/components/layout-router' export { default as LayoutRouter } from 'next/dist/client/components/layout-router'
export { default as RenderFromTemplateContext } from 'next/dist/client/components/render-from-template-context' export { default as RenderFromTemplateContext } from 'next/dist/client/components/render-from-template-context'
export { default as GlobalError } from ${JSON.stringify( export { default as GlobalError } from ${JSON.stringify(
globalError || 'next/dist/client/components/error-boundary' treeCodeResult.globalError || 'next/dist/client/components/error-boundary'
)} )}
export { staticGenerationAsyncStorage } from 'next/dist/client/components/static-generation-async-storage' export { staticGenerationAsyncStorage } from 'next/dist/client/components/static-generation-async-storage'
......
...@@ -18,7 +18,7 @@ import { Telemetry } from '../telemetry/storage' ...@@ -18,7 +18,7 @@ import { Telemetry } from '../telemetry/storage'
import loadConfig from '../server/config' import loadConfig from '../server/config'
import { findPagesDir } from '../lib/find-pages-dir' import { findPagesDir } from '../lib/find-pages-dir'
import { findRootDir } from '../lib/find-root' import { findRootDir } from '../lib/find-root'
import { fileExists } from '../lib/file-exists' import { fileExists, FileType } from '../lib/file-exists'
import { getNpxCommand } from '../lib/helpers/get-npx-command' import { getNpxCommand } from '../lib/helpers/get-npx-command'
import Watchpack from 'next/dist/compiled/watchpack' import Watchpack from 'next/dist/compiled/watchpack'
import stripAnsi from 'next/dist/compiled/strip-ansi' import stripAnsi from 'next/dist/compiled/strip-ansi'
...@@ -161,7 +161,7 @@ const nextDev: CliCommand = async (argv) => { ...@@ -161,7 +161,7 @@ const nextDev: CliCommand = async (argv) => {
dir = getProjectDir(process.env.NEXT_PRIVATE_DEV_DIR || args._[0]) dir = getProjectDir(process.env.NEXT_PRIVATE_DEV_DIR || args._[0])
// Check if pages dir exists and warn if not // Check if pages dir exists and warn if not
if (!(await fileExists(dir, 'directory'))) { if (!(await fileExists(dir, FileType.Directory))) {
printAndExit(`> No such directory exists as the project root: ${dir}`) printAndExit(`> No such directory exists as the project root: ${dir}`)
} }
......
import { constants, promises } from 'fs' import { constants, promises } from 'fs'
import isError from './is-error' import isError from './is-error'
export enum FileType {
File = 'file',
Directory = 'directory',
}
export async function fileExists( export async function fileExists(
fileName: string, fileName: string,
type?: 'file' | 'directory' type?: FileType
): Promise<boolean> { ): Promise<boolean> {
try { try {
if (type === 'file') { if (type === FileType.File) {
const stats = await promises.stat(fileName) const stats = await promises.stat(fileName)
return stats.isFile() return stats.isFile()
} else if (type === 'directory') { } else if (type === FileType.Directory) {
const stats = await promises.stat(fileName) const stats = await promises.stat(fileName)
return stats.isDirectory() return stats.isDirectory()
} else { } else {
......
...@@ -6,7 +6,7 @@ import { ...@@ -6,7 +6,7 @@ import {
hasNecessaryDependencies, hasNecessaryDependencies,
NecessaryDependencies, NecessaryDependencies,
} from './has-necessary-dependencies' } from './has-necessary-dependencies'
import { fileExists } from './file-exists' import { fileExists, FileType } from './file-exists'
import { FatalError } from './fatal-error' import { FatalError } from './fatal-error'
import { recursiveDelete } from './recursive-delete' import { recursiveDelete } from './recursive-delete'
import * as Log from '../build/output/log' import * as Log from '../build/output/log'
...@@ -44,7 +44,10 @@ async function copyPartytownStaticFiles( ...@@ -44,7 +44,10 @@ async function copyPartytownStaticFiles(
staticDir: string staticDir: string
) { ) {
const partytownLibDir = path.join(staticDir, '~partytown') const partytownLibDir = path.join(staticDir, '~partytown')
const hasPartytownLibDir = await fileExists(partytownLibDir, 'directory') const hasPartytownLibDir = await fileExists(
partytownLibDir,
FileType.Directory
)
if (hasPartytownLibDir) { if (hasPartytownLibDir) {
await recursiveDelete(partytownLibDir) await recursiveDelete(partytownLibDir)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment