Unverified Commit 13f03030 authored by Wyatt Johnson's avatar Wyatt Johnson Committed by GitHub

Reduce FS access for incremental cache (#57902)

This passes down the route kind information to the incrememntal cache so it no longer needs to test some files existing in order to validate if the file exists or not for a route.
parent 7b524fa2
...@@ -81,6 +81,7 @@ import type { BuildManifest } from '../server/get-page-files' ...@@ -81,6 +81,7 @@ import type { BuildManifest } from '../server/get-page-files'
import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path' import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
import { getPagePath } from '../server/require' import { getPagePath } from '../server/require'
import * as ciEnvironment from '../telemetry/ci-info' import * as ciEnvironment from '../telemetry/ci-info'
import { import {
eventBuildOptimize, eventBuildOptimize,
eventCliSession, eventCliSession,
...@@ -155,6 +156,7 @@ import { collectBuildTraces } from './collect-build-traces' ...@@ -155,6 +156,7 @@ import { collectBuildTraces } from './collect-build-traces'
import type { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints-plugin' import type { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints-plugin'
import { formatManifest } from './manifests/formatter/format-manifest' import { formatManifest } from './manifests/formatter/format-manifest'
import { getStartServerInfo, logStartInfo } from '../server/lib/app-info-log' import { getStartServerInfo, logStartInfo } from '../server/lib/app-info-log'
import type { NextEnabledDirectories } from '../server/base-server'
interface ExperimentalBypassForInfo { interface ExperimentalBypassForInfo {
experimentalBypassFor?: RouteHas[] experimentalBypassFor?: RouteHas[]
...@@ -347,7 +349,6 @@ export default async function build( ...@@ -347,7 +349,6 @@ export default async function build(
const isCompile = buildMode === 'experimental-compile' const isCompile = buildMode === 'experimental-compile'
const isGenerate = buildMode === 'experimental-generate' const isGenerate = buildMode === 'experimental-generate'
let hasAppDir = false
try { try {
const nextBuildSpan = trace('next-build', undefined, { const nextBuildSpan = trace('next-build', undefined, {
buildMode: buildMode, buildMode: buildMode,
...@@ -432,11 +433,14 @@ export default async function build( ...@@ -432,11 +433,14 @@ export default async function build(
setGlobal('telemetry', telemetry) setGlobal('telemetry', telemetry)
const publicDir = path.join(dir, 'public') const publicDir = path.join(dir, 'public')
const isAppDirEnabled = true
const { pagesDir, appDir } = findPagesDir(dir) const { pagesDir, appDir } = findPagesDir(dir)
NextBuildContext.pagesDir = pagesDir NextBuildContext.pagesDir = pagesDir
NextBuildContext.appDir = appDir NextBuildContext.appDir = appDir
hasAppDir = Boolean(appDir)
const enabledDirectories: NextEnabledDirectories = {
app: typeof appDir === 'string',
pages: typeof pagesDir === 'string',
}
const isSrcDir = path const isSrcDir = path
.relative(dir, pagesDir || appDir || '') .relative(dir, pagesDir || appDir || '')
...@@ -1292,8 +1296,9 @@ export default async function build( ...@@ -1292,8 +1296,9 @@ export default async function build(
} = await initializeIncrementalCache({ } = await initializeIncrementalCache({
fs: nodeFs, fs: nodeFs,
dev: false, dev: false,
appDir: isAppDirEnabled, pagesDir: true,
fetchCache: isAppDirEnabled, appDir: true,
fetchCache: true,
flushToDisk: config.experimental.isrFlushToDisk, flushToDisk: config.experimental.isrFlushToDisk,
serverDistDir: path.join(distDir, 'server'), serverDistDir: path.join(distDir, 'server'),
fetchCacheKeyPrefix: config.experimental.fetchCacheKeyPrefix, fetchCacheKeyPrefix: config.experimental.fetchCacheKeyPrefix,
...@@ -1317,7 +1322,7 @@ export default async function build( ...@@ -1317,7 +1322,7 @@ export default async function build(
incrementalCacheIpcPort, incrementalCacheIpcPort,
incrementalCacheIpcValidationKey incrementalCacheIpcValidationKey
) )
const appStaticWorkers = isAppDirEnabled const appStaticWorkers = appDir
? createStaticWorker( ? createStaticWorker(
incrementalCacheIpcPort, incrementalCacheIpcPort,
incrementalCacheIpcValidationKey incrementalCacheIpcValidationKey
...@@ -2032,7 +2037,7 @@ export default async function build( ...@@ -2032,7 +2037,7 @@ export default async function build(
(combinedPages.length > 0 || (combinedPages.length > 0 ||
useStaticPages404 || useStaticPages404 ||
useDefaultStatic500 || useDefaultStatic500 ||
isAppDirEnabled) appDir)
) { ) {
const staticGenerationSpan = const staticGenerationSpan =
nextBuildSpan.traceChild('static-generation') nextBuildSpan.traceChild('static-generation')
...@@ -2172,7 +2177,7 @@ export default async function build( ...@@ -2172,7 +2177,7 @@ export default async function build(
const exportOptions: ExportAppOptions = { const exportOptions: ExportAppOptions = {
nextConfig: exportConfig, nextConfig: exportConfig,
hasAppDir, enabledDirectories,
silent: false, silent: false,
buildExport: true, buildExport: true,
debugOutput, debugOutput,
...@@ -2870,7 +2875,7 @@ export default async function build( ...@@ -2870,7 +2875,7 @@ export default async function build(
const options: ExportAppOptions = { const options: ExportAppOptions = {
buildExport: false, buildExport: false,
nextConfig: config, nextConfig: config,
hasAppDir, enabledDirectories,
silent: true, silent: true,
threads: config.experimental.cpus, threads: config.experimental.cpus,
outdir: path.join(dir, configOutDir), outdir: path.join(dir, configOutDir),
......
...@@ -69,8 +69,8 @@ import { nodeFs } from '../server/lib/node-fs-methods' ...@@ -69,8 +69,8 @@ import { nodeFs } from '../server/lib/node-fs-methods'
import * as ciEnvironment from '../telemetry/ci-info' import * as ciEnvironment from '../telemetry/ci-info'
import { normalizeAppPath } from '../shared/lib/router/utils/app-paths' import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
import { denormalizeAppPagePath } from '../shared/lib/page-path/denormalize-app-path' import { denormalizeAppPagePath } from '../shared/lib/page-path/denormalize-app-path'
import { AppRouteRouteModule } from '../server/future/route-modules/app-route/module.compiled'
import { RouteKind } from '../server/future/route-kind' import { RouteKind } from '../server/future/route-kind'
import { isAppRouteRouteModule } from '../server/future/route-modules/checks'
export type ROUTER_TYPE = 'pages' | 'app' export type ROUTER_TYPE = 'pages' | 'app'
...@@ -1267,6 +1267,8 @@ export async function buildAppStaticPaths({ ...@@ -1267,6 +1267,8 @@ export async function buildAppStaticPaths({
const incrementalCache = new IncrementalCache({ const incrementalCache = new IncrementalCache({
fs: nodeFs, fs: nodeFs,
dev: true, dev: true,
// Enabled both for build as we're only writing this cache, not reading it.
pagesDir: true,
appDir: true, appDir: true,
flushToDisk: isrFlushToDisk, flushToDisk: isrFlushToDisk,
serverDistDir: path.join(distDir, 'server'), serverDistDir: path.join(distDir, 'server'),
...@@ -1506,7 +1508,7 @@ export async function isPageStatic({ ...@@ -1506,7 +1508,7 @@ export async function isPageStatic({
const { tree, staticGenerationAsyncStorage, serverHooks } = ComponentMod const { tree, staticGenerationAsyncStorage, serverHooks } = ComponentMod
const generateParams: GenerateParams = const generateParams: GenerateParams =
routeModule && AppRouteRouteModule.is(routeModule) routeModule && isAppRouteRouteModule(routeModule)
? [ ? [
{ {
config: { config: {
......
import type { NextEnabledDirectories } from '../../server/base-server'
import path from 'path' import path from 'path'
import fs from 'fs'
import { IncrementalCache } from '../../server/lib/incremental-cache' import { IncrementalCache } from '../../server/lib/incremental-cache'
import { hasNextSupport } from '../../telemetry/ci-info' import { hasNextSupport } from '../../telemetry/ci-info'
import { nodeFs } from '../../server/lib/node-fs-methods'
export function createIncrementalCache( export function createIncrementalCache(
incrementalCacheHandlerPath: string | undefined, incrementalCacheHandlerPath: string | undefined,
isrMemoryCacheSize: number | undefined, isrMemoryCacheSize: number | undefined,
fetchCacheKeyPrefix: string | undefined, fetchCacheKeyPrefix: string | undefined,
distDir: string, distDir: string,
dir: string dir: string,
enabledDirectories: NextEnabledDirectories
) { ) {
// Custom cache handler overrides. // Custom cache handler overrides.
let CacheHandler: any let CacheHandler: any
...@@ -37,13 +40,9 @@ export function createIncrementalCache( ...@@ -37,13 +40,9 @@ export function createIncrementalCache(
}, },
notFoundRoutes: [], notFoundRoutes: [],
}), }),
fs: { fs: nodeFs,
readFile: fs.promises.readFile, pagesDir: enabledDirectories.pages,
readFileSync: fs.readFileSync, appDir: enabledDirectories.app,
writeFile: (f, d) => fs.promises.writeFile(f, d),
mkdir: (d) => fs.promises.mkdir(d, { recursive: true }),
stat: (f) => fs.promises.stat(f),
},
serverDistDir: path.join(distDir, 'server'), serverDistDir: path.join(distDir, 'server'),
CurCacheHandler: CacheHandler, CurCacheHandler: CacheHandler,
minimalMode: hasNextSupport, minimalMode: hasNextSupport,
......
...@@ -215,6 +215,8 @@ export async function exportAppImpl( ...@@ -215,6 +215,8 @@ export async function exportAppImpl(
// attempt to load global env values so they are available in next.config.js // attempt to load global env values so they are available in next.config.js
span.traceChild('load-dotenv').traceFn(() => loadEnvConfig(dir, false, Log)) span.traceChild('load-dotenv').traceFn(() => loadEnvConfig(dir, false, Log))
const { enabledDirectories } = options
const nextConfig = const nextConfig =
options.nextConfig || options.nextConfig ||
(await span (await span
...@@ -444,7 +446,7 @@ export async function exportAppImpl( ...@@ -444,7 +446,7 @@ export async function exportAppImpl(
} }
let serverActionsManifest let serverActionsManifest
if (options.hasAppDir) { if (enabledDirectories.app) {
serverActionsManifest = require(join( serverActionsManifest = require(join(
distDir, distDir,
SERVER_DIRECTORY, SERVER_DIRECTORY,
...@@ -487,15 +489,15 @@ export async function exportAppImpl( ...@@ -487,15 +489,15 @@ export async function exportAppImpl(
nextScriptWorkers: nextConfig.experimental.nextScriptWorkers, nextScriptWorkers: nextConfig.experimental.nextScriptWorkers,
optimizeFonts: nextConfig.optimizeFonts as FontConfig, optimizeFonts: nextConfig.optimizeFonts as FontConfig,
largePageDataBytes: nextConfig.experimental.largePageDataBytes, largePageDataBytes: nextConfig.experimental.largePageDataBytes,
serverComponents: options.hasAppDir,
serverActions: nextConfig.experimental.serverActions, serverActions: nextConfig.experimental.serverActions,
serverComponents: enabledDirectories.app,
nextFontManifest: require(join( nextFontManifest: require(join(
distDir, distDir,
'server', 'server',
`${NEXT_FONT_MANIFEST}.json` `${NEXT_FONT_MANIFEST}.json`
)), )),
images: nextConfig.images, images: nextConfig.images,
...(options.hasAppDir ...(enabledDirectories.app
? { ? {
serverActionsManifest, serverActionsManifest,
} }
...@@ -698,6 +700,7 @@ export async function exportAppImpl( ...@@ -698,6 +700,7 @@ export async function exportAppImpl(
incrementalCacheHandlerPath: incrementalCacheHandlerPath:
nextConfig.experimental.incrementalCacheHandlerPath, nextConfig.experimental.incrementalCacheHandlerPath,
enableExperimentalReact: needsExperimentalReact(nextConfig), enableExperimentalReact: needsExperimentalReact(nextConfig),
enabledDirectories,
}) })
}) })
......
...@@ -8,6 +8,7 @@ import type { FontConfig } from '../server/font-utils' ...@@ -8,6 +8,7 @@ import type { FontConfig } from '../server/font-utils'
import type { ExportPathMap, NextConfigComplete } from '../server/config-shared' import type { ExportPathMap, NextConfigComplete } from '../server/config-shared'
import type { Span } from '../trace' import type { Span } from '../trace'
import type { Revalidate } from '../server/lib/revalidate' import type { Revalidate } from '../server/lib/revalidate'
import type { NextEnabledDirectories } from '../server/base-server'
export interface AmpValidation { export interface AmpValidation {
page: string page: string
...@@ -58,6 +59,7 @@ export interface ExportPageInput { ...@@ -58,6 +59,7 @@ export interface ExportPageInput {
fetchCacheKeyPrefix?: string fetchCacheKeyPrefix?: string
nextConfigOutput?: NextConfigComplete['output'] nextConfigOutput?: NextConfigComplete['output']
enableExperimentalReact?: boolean enableExperimentalReact?: boolean
enabledDirectories: NextEnabledDirectories
} }
export type ExportedPageFile = { export type ExportedPageFile = {
...@@ -98,7 +100,7 @@ export type ExportWorker = ( ...@@ -98,7 +100,7 @@ export type ExportWorker = (
export interface ExportAppOptions { export interface ExportAppOptions {
outdir: string outdir: string
hasAppDir: boolean enabledDirectories: NextEnabledDirectories
silent?: boolean silent?: boolean
threads?: number threads?: number
debugOutput?: boolean debugOutput?: boolean
......
...@@ -65,6 +65,7 @@ async function exportPageImpl( ...@@ -65,6 +65,7 @@ async function exportPageImpl(
enableExperimentalReact, enableExperimentalReact,
ampValidatorPath, ampValidatorPath,
trailingSlash, trailingSlash,
enabledDirectories,
} = input } = input
if (enableExperimentalReact) { if (enableExperimentalReact) {
...@@ -222,7 +223,8 @@ async function exportPageImpl( ...@@ -222,7 +223,8 @@ async function exportPageImpl(
isrMemoryCacheSize, isrMemoryCacheSize,
fetchCacheKeyPrefix, fetchCacheKeyPrefix,
distDir, distDir,
dir dir,
enabledDirectories
) )
: undefined : undefined
......
import type { IncomingHttpHeaders } from 'node:http'
import { FLIGHT_PARAMETERS } from '../../client/components/app-router-headers'
/**
* Removes the flight headers from the request.
*
* @param req the request to strip the headers from
*/
export function stripFlightHeaders(headers: IncomingHttpHeaders) {
for (const [header] of FLIGHT_PARAMETERS) {
delete headers[header.toLowerCase()]
}
}
This diff is collapsed.
...@@ -508,7 +508,7 @@ export default class DevServer extends Server { ...@@ -508,7 +508,7 @@ export default class DevServer extends Server {
} }
protected getAppPathsManifest(): PagesManifest | undefined { protected getAppPathsManifest(): PagesManifest | undefined {
if (!this.hasAppDir) return undefined if (!this.enabledDirectories.app) return undefined
return ( return (
NodeManifestLoader.require( NodeManifestLoader.require(
......
...@@ -14,9 +14,7 @@ import { setHttpClientAndAgentOptions } from '../setup-http-agent-env' ...@@ -14,9 +14,7 @@ import { setHttpClientAndAgentOptions } from '../setup-http-agent-env'
import type { IncrementalCache } from '../lib/incremental-cache' import type { IncrementalCache } from '../lib/incremental-cache'
import * as serverHooks from '../../client/components/hooks-server-context' import * as serverHooks from '../../client/components/hooks-server-context'
import { staticGenerationAsyncStorage } from '../../client/components/static-generation-async-storage.external' import { staticGenerationAsyncStorage } from '../../client/components/static-generation-async-storage.external'
import { isAppRouteRouteModule } from '../future/route-modules/checks'
const { AppRouteRouteModule } =
require('../future/route-modules/app-route/module.compiled') as typeof import('../future/route-modules/app-route/module')
type RuntimeConfig = { type RuntimeConfig = {
configFileName: string configFileName: string
...@@ -88,7 +86,7 @@ export async function loadStaticPaths({ ...@@ -88,7 +86,7 @@ export async function loadStaticPaths({
if (isAppPath) { if (isAppPath) {
const { routeModule } = components const { routeModule } = components
const generateParams: GenerateParams = const generateParams: GenerateParams =
routeModule && AppRouteRouteModule.is(routeModule) routeModule && isAppRouteRouteModule(routeModule)
? [ ? [
{ {
config: { config: {
......
...@@ -32,7 +32,6 @@ import * as Log from '../../../../build/output/log' ...@@ -32,7 +32,6 @@ import * as Log from '../../../../build/output/log'
import { autoImplementMethods } from './helpers/auto-implement-methods' import { autoImplementMethods } from './helpers/auto-implement-methods'
import { getNonStaticMethods } from './helpers/get-non-static-methods' import { getNonStaticMethods } from './helpers/get-non-static-methods'
import { appendMutableCookies } from '../../../web/spec-extension/adapters/request-cookies' import { appendMutableCookies } from '../../../web/spec-extension/adapters/request-cookies'
import { RouteKind } from '../../route-kind'
import { parsedUrlQueryToParams } from './helpers/parsed-url-query-to-params' import { parsedUrlQueryToParams } from './helpers/parsed-url-query-to-params'
import * as serverHooks from '../../../../client/components/hooks-server-context' import * as serverHooks from '../../../../client/components/hooks-server-context'
...@@ -162,10 +161,6 @@ export class AppRouteRouteModule extends RouteModule< ...@@ -162,10 +161,6 @@ export class AppRouteRouteModule extends RouteModule<
private readonly nonStaticMethods: ReadonlyArray<HTTP_METHOD> | false private readonly nonStaticMethods: ReadonlyArray<HTTP_METHOD> | false
private readonly dynamic: AppRouteUserlandModule['dynamic'] private readonly dynamic: AppRouteUserlandModule['dynamic']
public static is(route: RouteModule): route is AppRouteRouteModule {
return route.definition.kind === RouteKind.APP_ROUTE
}
constructor({ constructor({
userland, userland,
definition, definition,
......
import type { AppRouteRouteModule } from './app-route/module'
import type { AppPageRouteModule } from './app-page/module'
import type { PagesRouteModule } from './pages/module'
import type { PagesAPIRouteModule } from './pages-api/module'
import type { RouteModule } from './route-module'
import { RouteKind } from '../route-kind'
export function isAppRouteRouteModule(
routeModule: RouteModule
): routeModule is AppRouteRouteModule {
return routeModule.definition.kind === RouteKind.APP_ROUTE
}
export function isAppPageRouteModule(
routeModule: RouteModule
): routeModule is AppPageRouteModule {
return routeModule.definition.kind === RouteKind.APP_PAGE
}
export function isPagesRouteModule(
routeModule: RouteModule
): routeModule is PagesRouteModule {
return routeModule.definition.kind === RouteKind.PAGES
}
export function isPagesAPIRouteModule(
routeModule: RouteModule
): routeModule is PagesAPIRouteModule {
return routeModule.definition.kind === RouteKind.PAGES_API
}
import type { RouteMetadata } from '../../../export/routes/types' import type { RouteMetadata } from '../../../export/routes/types'
import type { CacheHandler, CacheHandlerContext, CacheHandlerValue } from './' import type { CacheHandler, CacheHandlerContext, CacheHandlerValue } from './'
import type { CacheFs } from '../../../shared/lib/utils' import type { CacheFs } from '../../../shared/lib/utils'
import type { CachedFetchValue } from '../../response-cache' import type {
CachedFetchValue,
IncrementalCacheKindHint,
} from '../../response-cache'
import LRUCache from 'next/dist/compiled/lru-cache' import LRUCache from 'next/dist/compiled/lru-cache'
import path from '../../../shared/lib/isomorphic/path' import path from '../../../shared/lib/isomorphic/path'
...@@ -27,6 +30,7 @@ export default class FileSystemCache implements CacheHandler { ...@@ -27,6 +30,7 @@ export default class FileSystemCache implements CacheHandler {
private flushToDisk?: FileSystemCacheContext['flushToDisk'] private flushToDisk?: FileSystemCacheContext['flushToDisk']
private serverDistDir: FileSystemCacheContext['serverDistDir'] private serverDistDir: FileSystemCacheContext['serverDistDir']
private appDir: boolean private appDir: boolean
private pagesDir: boolean
private tagsManifestPath?: string private tagsManifestPath?: string
private revalidatedTags: string[] private revalidatedTags: string[]
...@@ -35,6 +39,7 @@ export default class FileSystemCache implements CacheHandler { ...@@ -35,6 +39,7 @@ export default class FileSystemCache implements CacheHandler {
this.flushToDisk = ctx.flushToDisk this.flushToDisk = ctx.flushToDisk
this.serverDistDir = ctx.serverDistDir this.serverDistDir = ctx.serverDistDir
this.appDir = !!ctx._appDir this.appDir = !!ctx._appDir
this.pagesDir = !!ctx._pagesDir
this.revalidatedTags = ctx.revalidatedTags this.revalidatedTags = ctx.revalidatedTags
if (ctx.maxMemoryCacheSize && !memoryCache) { if (ctx.maxMemoryCacheSize && !memoryCache) {
...@@ -111,11 +116,11 @@ export default class FileSystemCache implements CacheHandler { ...@@ -111,11 +116,11 @@ export default class FileSystemCache implements CacheHandler {
{ {
tags, tags,
softTags, softTags,
fetchCache, kindHint,
}: { }: {
tags?: string[] tags?: string[]
softTags?: string[] softTags?: string[]
fetchCache?: boolean kindHint?: IncrementalCacheKindHint
} = {} } = {}
) { ) {
let data = memoryCache?.get(key) let data = memoryCache?.get(key)
...@@ -123,10 +128,7 @@ export default class FileSystemCache implements CacheHandler { ...@@ -123,10 +128,7 @@ export default class FileSystemCache implements CacheHandler {
// let's check the disk for seed data // let's check the disk for seed data
if (!data && process.env.NEXT_RUNTIME !== 'edge') { if (!data && process.env.NEXT_RUNTIME !== 'edge') {
try { try {
const { filePath } = await this.getFsPath({ const filePath = this.getFilePath(`${key}.body`, 'app')
pathname: `${key}.body`,
appDir: true,
})
const fileData = await this.fs.readFile(filePath) const fileData = await this.fs.readFile(filePath)
const { mtime } = await this.fs.stat(filePath) const { mtime } = await this.fs.stat(filePath)
...@@ -149,14 +151,22 @@ export default class FileSystemCache implements CacheHandler { ...@@ -149,14 +151,22 @@ export default class FileSystemCache implements CacheHandler {
} }
try { try {
const { filePath, isAppPath } = await this.getFsPath({ // Determine the file kind if we didn't know it already.
pathname: fetchCache ? key : `${key}.html`, let kind = kindHint
fetchCache, if (!kind) {
}) kind = this.detectFileKind(`${key}.html`)
}
const isAppPath = kind === 'app'
const filePath = this.getFilePath(
kind === 'fetch' ? key : `${key}.html`,
kind
)
const fileData = await this.fs.readFile(filePath, 'utf8') const fileData = await this.fs.readFile(filePath, 'utf8')
const { mtime } = await this.fs.stat(filePath) const { mtime } = await this.fs.stat(filePath)
if (fetchCache) { if (kind === 'fetch') {
const lastModified = mtime.getTime() const lastModified = mtime.getTime()
const parsedData: CachedFetchValue = JSON.parse(fileData) const parsedData: CachedFetchValue = JSON.parse(fileData)
data = { data = {
...@@ -177,22 +187,12 @@ export default class FileSystemCache implements CacheHandler { ...@@ -177,22 +187,12 @@ export default class FileSystemCache implements CacheHandler {
} else { } else {
const pageData = isAppPath const pageData = isAppPath
? await this.fs.readFile( ? await this.fs.readFile(
( this.getFilePath(`${key}.rsc`, 'app'),
await this.getFsPath({
pathname: `${key}.rsc`,
appDir: true,
})
).filePath,
'utf8' 'utf8'
) )
: JSON.parse( : JSON.parse(
await this.fs.readFile( await this.fs.readFile(
( this.getFilePath(`${key}.json`, 'pages'),
await this.getFsPath({
pathname: `${key}.json`,
appDir: false,
})
).filePath,
'utf8' 'utf8'
) )
) )
...@@ -299,10 +299,7 @@ export default class FileSystemCache implements CacheHandler { ...@@ -299,10 +299,7 @@ export default class FileSystemCache implements CacheHandler {
if (!this.flushToDisk) return if (!this.flushToDisk) return
if (data?.kind === 'ROUTE') { if (data?.kind === 'ROUTE') {
const { filePath } = await this.getFsPath({ const filePath = this.getFilePath(`${key}.body`, 'app')
pathname: `${key}.body`,
appDir: true,
})
await this.fs.mkdir(path.dirname(filePath)) await this.fs.mkdir(path.dirname(filePath))
await this.fs.writeFile(filePath, data.body) await this.fs.writeFile(filePath, data.body)
...@@ -321,20 +318,18 @@ export default class FileSystemCache implements CacheHandler { ...@@ -321,20 +318,18 @@ export default class FileSystemCache implements CacheHandler {
if (data?.kind === 'PAGE') { if (data?.kind === 'PAGE') {
const isAppPath = typeof data.pageData === 'string' const isAppPath = typeof data.pageData === 'string'
const { filePath: htmlPath } = await this.getFsPath({ const htmlPath = this.getFilePath(
pathname: `${key}.html`, `${key}.html`,
appDir: isAppPath, isAppPath ? 'app' : 'pages'
}) )
await this.fs.mkdir(path.dirname(htmlPath)) await this.fs.mkdir(path.dirname(htmlPath))
await this.fs.writeFile(htmlPath, data.html) await this.fs.writeFile(htmlPath, data.html)
await this.fs.writeFile( await this.fs.writeFile(
( this.getFilePath(
await this.getFsPath({ `${key}.${isAppPath ? 'rsc' : 'json'}`,
pathname: `${key}.${isAppPath ? 'rsc' : 'json'}`, isAppPath ? 'app' : 'pages'
appDir: isAppPath, ),
})
).filePath,
isAppPath ? data.pageData : JSON.stringify(data.pageData) isAppPath ? data.pageData : JSON.stringify(data.pageData)
) )
...@@ -351,10 +346,7 @@ export default class FileSystemCache implements CacheHandler { ...@@ -351,10 +346,7 @@ export default class FileSystemCache implements CacheHandler {
) )
} }
} else if (data?.kind === 'FETCH') { } else if (data?.kind === 'FETCH') {
const { filePath } = await this.getFsPath({ const filePath = this.getFilePath(key, 'fetch')
pathname: key,
fetchCache: true,
})
await this.fs.mkdir(path.dirname(filePath)) await this.fs.mkdir(path.dirname(filePath))
await this.fs.writeFile( await this.fs.writeFile(
filePath, filePath,
...@@ -366,51 +358,61 @@ export default class FileSystemCache implements CacheHandler { ...@@ -366,51 +358,61 @@ export default class FileSystemCache implements CacheHandler {
} }
} }
private async getFsPath({ private detectFileKind(pathname: string) {
pathname, if (!this.appDir && !this.pagesDir) {
appDir, throw new Error(
fetchCache, "Invariant: Can't determine file path kind, no page directory enabled"
}: { )
pathname: string }
appDir?: boolean
fetchCache?: boolean // If app directory isn't enabled, then assume it's pages and avoid the fs
}): Promise<{ // hit.
filePath: string if (!this.appDir && this.pagesDir) {
isAppPath: boolean return 'pages'
}> { }
if (fetchCache) { // Otherwise assume it's a pages file if the pages directory isn't enabled.
// we store in .next/cache/fetch-cache so it can be persisted else if (this.appDir && !this.pagesDir) {
// across deploys return 'app'
return { }
filePath: path.join(
// If both are enabled, we need to test each in order, starting with
// `pages`.
let filePath = this.getFilePath(pathname, 'pages')
if (this.fs.existsSync(filePath)) {
return 'pages'
}
filePath = this.getFilePath(pathname, 'app')
if (this.fs.existsSync(filePath)) {
return 'app'
}
throw new Error(
`Invariant: Unable to determine file path kind for ${pathname}`
)
}
private getFilePath(
pathname: string,
kind: 'app' | 'fetch' | 'pages'
): string {
switch (kind) {
case 'fetch':
// we store in .next/cache/fetch-cache so it can be persisted
// across deploys
return path.join(
this.serverDistDir, this.serverDistDir,
'..', '..',
'cache', 'cache',
'fetch-cache', 'fetch-cache',
pathname pathname
), )
isAppPath: false, case 'pages':
} return path.join(this.serverDistDir, 'pages', pathname)
} case 'app':
let isAppPath = false return path.join(this.serverDistDir, 'app', pathname)
let filePath = path.join(this.serverDistDir, 'pages', pathname) default:
throw new Error("Invariant: Can't determine file path kind")
if (!this.appDir || appDir === false)
return {
filePath,
isAppPath,
}
try {
await this.fs.readFile(filePath)
return {
filePath,
isAppPath,
}
} catch (err) {
return {
filePath: path.join(this.serverDistDir, 'app', pathname),
isAppPath: true,
}
} }
} }
} }
...@@ -3,7 +3,10 @@ import type { PrerenderManifest } from '../../../build' ...@@ -3,7 +3,10 @@ import type { PrerenderManifest } from '../../../build'
import type { import type {
IncrementalCacheValue, IncrementalCacheValue,
IncrementalCacheEntry, IncrementalCacheEntry,
IncrementalCache as IncrementalCacheType,
IncrementalCacheKindHint,
} from '../../response-cache' } from '../../response-cache'
import FetchCache from './fetch-cache' import FetchCache from './fetch-cache'
import FileSystemCache from './file-system-cache' import FileSystemCache from './file-system-cache'
import path from '../../../shared/lib/isomorphic/path' import path from '../../../shared/lib/isomorphic/path'
...@@ -30,6 +33,7 @@ export interface CacheHandlerContext { ...@@ -30,6 +33,7 @@ export interface CacheHandlerContext {
prerenderManifest?: PrerenderManifest prerenderManifest?: PrerenderManifest
revalidatedTags: string[] revalidatedTags: string[]
_appDir: boolean _appDir: boolean
_pagesDir: boolean
_requestHeaders: IncrementalCache['requestHeaders'] _requestHeaders: IncrementalCache['requestHeaders']
} }
...@@ -57,7 +61,7 @@ export class CacheHandler { ...@@ -57,7 +61,7 @@ export class CacheHandler {
public async revalidateTag(_tag: string): Promise<void> {} public async revalidateTag(_tag: string): Promise<void> {}
} }
export class IncrementalCache { export class IncrementalCache implements IncrementalCacheType {
dev?: boolean dev?: boolean
cacheHandler?: CacheHandler cacheHandler?: CacheHandler
prerenderManifest: PrerenderManifest prerenderManifest: PrerenderManifest
...@@ -75,6 +79,7 @@ export class IncrementalCache { ...@@ -75,6 +79,7 @@ export class IncrementalCache {
fs, fs,
dev, dev,
appDir, appDir,
pagesDir,
flushToDisk, flushToDisk,
fetchCache, fetchCache,
minimalMode, minimalMode,
...@@ -90,6 +95,7 @@ export class IncrementalCache { ...@@ -90,6 +95,7 @@ export class IncrementalCache {
fs?: CacheFs fs?: CacheFs
dev: boolean dev: boolean
appDir?: boolean appDir?: boolean
pagesDir?: boolean
fetchCache?: boolean fetchCache?: boolean
minimalMode?: boolean minimalMode?: boolean
serverDistDir?: string serverDistDir?: string
...@@ -165,6 +171,7 @@ export class IncrementalCache { ...@@ -165,6 +171,7 @@ export class IncrementalCache {
serverDistDir, serverDistDir,
revalidatedTags, revalidatedTags,
maxMemoryCacheSize, maxMemoryCacheSize,
_pagesDir: !!pagesDir,
_appDir: !!appDir, _appDir: !!appDir,
_requestHeaders: requestHeaders, _requestHeaders: requestHeaders,
fetchCacheKeyPrefix, fetchCacheKeyPrefix,
...@@ -397,7 +404,7 @@ export class IncrementalCache { ...@@ -397,7 +404,7 @@ export class IncrementalCache {
async get( async get(
cacheKey: string, cacheKey: string,
ctx: { ctx: {
fetchCache?: boolean kindHint?: IncrementalCacheKindHint
revalidate?: number | false revalidate?: number | false
fetchUrl?: string fetchUrl?: string
fetchIdx?: number fetchIdx?: number
...@@ -425,12 +432,13 @@ export class IncrementalCache { ...@@ -425,12 +432,13 @@ export class IncrementalCache {
// so that getStaticProps is always called for easier debugging // so that getStaticProps is always called for easier debugging
if ( if (
this.dev && this.dev &&
(!ctx.fetchCache || this.requestHeaders['cache-control'] === 'no-cache') (ctx.kindHint !== 'fetch' ||
this.requestHeaders['cache-control'] === 'no-cache')
) { ) {
return null return null
} }
cacheKey = this._getPathname(cacheKey, ctx.fetchCache) cacheKey = this._getPathname(cacheKey, ctx.kindHint === 'fetch')
let entry: IncrementalCacheEntry | null = null let entry: IncrementalCacheEntry | null = null
let revalidate = ctx.revalidate let revalidate = ctx.revalidate
...@@ -479,7 +487,7 @@ export class IncrementalCache { ...@@ -479,7 +487,7 @@ export class IncrementalCache {
revalidateAfter = this.calculateRevalidate( revalidateAfter = this.calculateRevalidate(
cacheKey, cacheKey,
cacheData?.lastModified || Date.now(), cacheData?.lastModified || Date.now(),
this.dev && !ctx.fetchCache this.dev && ctx.kindHint !== 'fetch'
) )
isStale = isStale =
revalidateAfter !== false && revalidateAfter < Date.now() revalidateAfter !== false && revalidateAfter < Date.now()
......
import _fs from 'fs'
import type { CacheFs } from '../../shared/lib/utils' import type { CacheFs } from '../../shared/lib/utils'
import fs from 'fs'
export const nodeFs: CacheFs = { export const nodeFs: CacheFs = {
readFile: _fs.promises.readFile, existsSync: fs.existsSync,
readFileSync: _fs.readFileSync, readFile: fs.promises.readFile,
writeFile: (f, d) => _fs.promises.writeFile(f, d), readFileSync: fs.readFileSync,
mkdir: (dir) => _fs.promises.mkdir(dir, { recursive: true }), writeFile: (f, d) => fs.promises.writeFile(f, d),
stat: (f) => _fs.promises.stat(f), mkdir: (dir) => fs.promises.mkdir(dir, { recursive: true }),
stat: (f) => fs.promises.stat(f),
} }
...@@ -522,7 +522,7 @@ export function patchFetch({ ...@@ -522,7 +522,7 @@ export function patchFetch({
const entry = staticGenerationStore.isOnDemandRevalidate const entry = staticGenerationStore.isOnDemandRevalidate
? null ? null
: await staticGenerationStore.incrementalCache.get(cacheKey, { : await staticGenerationStore.incrementalCache.get(cacheKey, {
fetchCache: true, kindHint: 'fetch',
revalidate, revalidate,
fetchUrl, fetchUrl,
fetchIdx, fetchIdx,
......
...@@ -55,6 +55,7 @@ import type { ...@@ -55,6 +55,7 @@ import type {
NormalizedRouteManifest, NormalizedRouteManifest,
LoadedRenderOpts, LoadedRenderOpts,
RouteHandler, RouteHandler,
NextEnabledDirectories,
} from './base-server' } from './base-server'
import BaseServer, { NoFallbackError } from './base-server' import BaseServer, { NoFallbackError } from './base-server'
import { getMaybePagePath, getPagePath, requireFontManifest } from './require' import { getMaybePagePath, getPagePath, requireFontManifest } from './require'
...@@ -310,7 +311,8 @@ export default class NextNodeServer extends BaseServer { ...@@ -310,7 +311,8 @@ export default class NextNodeServer extends BaseServer {
dev, dev,
requestHeaders, requestHeaders,
requestProtocol, requestProtocol,
appDir: this.hasAppDir, pagesDir: this.enabledDirectories.pages,
appDir: this.enabledDirectories.app,
allowedRevalidateHeaderKeys: allowedRevalidateHeaderKeys:
this.nextConfig.experimental.allowedRevalidateHeaderKeys, this.nextConfig.experimental.allowedRevalidateHeaderKeys,
minimalMode: this.minimalMode, minimalMode: this.minimalMode,
...@@ -342,7 +344,7 @@ export default class NextNodeServer extends BaseServer { ...@@ -342,7 +344,7 @@ export default class NextNodeServer extends BaseServer {
} }
protected getAppPathsManifest(): PagesManifest | undefined { protected getAppPathsManifest(): PagesManifest | undefined {
if (!this.hasAppDir) return undefined if (!this.enabledDirectories.app) return undefined
return loadManifest(join(this.serverDistDir, APP_PATHS_MANIFEST)) return loadManifest(join(this.serverDistDir, APP_PATHS_MANIFEST))
} }
...@@ -352,7 +354,7 @@ export default class NextNodeServer extends BaseServer { ...@@ -352,7 +354,7 @@ export default class NextNodeServer extends BaseServer {
pathname, pathname,
this.distDir, this.distDir,
this.nextConfig.i18n?.locales, this.nextConfig.i18n?.locales,
this.hasAppDir this.enabledDirectories.app
) )
} }
...@@ -371,8 +373,13 @@ export default class NextNodeServer extends BaseServer { ...@@ -371,8 +373,13 @@ export default class NextNodeServer extends BaseServer {
} }
} }
protected getHasAppDir(dev: boolean): boolean { protected getEnabledDirectories(dev: boolean): NextEnabledDirectories {
return Boolean(findDir(dev ? this.dir : this.serverDistDir, 'app')) const dir = dev ? this.dir : this.serverDistDir
return {
app: findDir(dir, 'app') ? true : false,
pages: findDir(dir, 'pages') ? true : false,
}
} }
protected sendRenderResult( protected sendRenderResult(
...@@ -484,7 +491,7 @@ export default class NextNodeServer extends BaseServer { ...@@ -484,7 +491,7 @@ export default class NextNodeServer extends BaseServer {
// https://github.com/vercel/next.js/blob/df7cbd904c3bd85f399d1ce90680c0ecf92d2752/packages/next/server/render.tsx#L947-L952 // https://github.com/vercel/next.js/blob/df7cbd904c3bd85f399d1ce90680c0ecf92d2752/packages/next/server/render.tsx#L947-L952
renderOpts.nextFontManifest = this.nextFontManifest renderOpts.nextFontManifest = this.nextFontManifest
if (this.hasAppDir && renderOpts.isAppPath) { if (this.enabledDirectories.app && renderOpts.isAppPath) {
return lazyRenderAppPage( return lazyRenderAppPage(
req.originalRequest, req.originalRequest,
res.originalResponse, res.originalResponse,
...@@ -569,7 +576,12 @@ export default class NextNodeServer extends BaseServer { ...@@ -569,7 +576,12 @@ export default class NextNodeServer extends BaseServer {
} }
protected getPagePath(pathname: string, locales?: string[]): string { protected getPagePath(pathname: string, locales?: string[]): string {
return getPagePath(pathname, this.distDir, locales, this.hasAppDir) return getPagePath(
pathname,
this.distDir,
locales,
this.enabledDirectories.app
)
} }
protected async renderPageComponent( protected async renderPageComponent(
...@@ -1263,7 +1275,7 @@ export default class NextNodeServer extends BaseServer { ...@@ -1263,7 +1275,7 @@ export default class NextNodeServer extends BaseServer {
const { req, res, query } = ctx const { req, res, query } = ctx
const is404 = res.statusCode === 404 const is404 = res.statusCode === 404
if (is404 && this.hasAppDir) { if (is404 && this.enabledDirectories.app) {
const notFoundPathname = this.renderOpts.dev const notFoundPathname = this.renderOpts.dev
? '/not-found' ? '/not-found'
: '/_not-found' : '/_not-found'
......
...@@ -3,7 +3,10 @@ import type { ...@@ -3,7 +3,10 @@ import type {
ResponseCacheEntry, ResponseCacheEntry,
ResponseGenerator, ResponseGenerator,
IncrementalCacheItem, IncrementalCacheItem,
ResponseCacheBase,
IncrementalCacheKindHint,
} from './types' } from './types'
import { RouteKind } from '../future/route-kind'
import RenderResult from '../render-result' import RenderResult from '../render-result'
import { Batcher } from '../../lib/batcher' import { Batcher } from '../../lib/batcher'
...@@ -11,7 +14,7 @@ import { scheduleOnNextTick } from '../../lib/scheduler' ...@@ -11,7 +14,7 @@ import { scheduleOnNextTick } from '../../lib/scheduler'
export * from './types' export * from './types'
export default class ResponseCache { export default class ResponseCache implements ResponseCacheBase {
private readonly batcher = Batcher.create< private readonly batcher = Batcher.create<
{ key: string; isOnDemandRevalidate: boolean }, { key: string; isOnDemandRevalidate: boolean },
ResponseCacheEntry | null, ResponseCacheEntry | null,
...@@ -46,6 +49,7 @@ export default class ResponseCache { ...@@ -46,6 +49,7 @@ export default class ResponseCache {
key: string | null, key: string | null,
responseGenerator: ResponseGenerator, responseGenerator: ResponseGenerator,
context: { context: {
routeKind?: RouteKind
isOnDemandRevalidate?: boolean isOnDemandRevalidate?: boolean
isPrefetch?: boolean isPrefetch?: boolean
incrementalCache: IncrementalCache incrementalCache: IncrementalCache
...@@ -70,11 +74,22 @@ export default class ResponseCache { ...@@ -70,11 +74,22 @@ export default class ResponseCache {
return this.previousCacheItem.entry return this.previousCacheItem.entry
} }
// Coerce the kindHint into a given kind for the incremental cache.
let kindHint: IncrementalCacheKindHint | undefined
if (
context.routeKind === RouteKind.APP_PAGE ||
context.routeKind === RouteKind.APP_ROUTE
) {
kindHint = 'app'
} else if (context.routeKind === RouteKind.PAGES) {
kindHint = 'pages'
}
let resolved = false let resolved = false
let cachedResponse: IncrementalCacheItem = null let cachedResponse: IncrementalCacheItem = null
try { try {
cachedResponse = !this.minimalMode cachedResponse = !this.minimalMode
? await incrementalCache.get(key) ? await incrementalCache.get(key, { kindHint })
: null : null
if (cachedResponse && !isOnDemandRevalidate) { if (cachedResponse && !isOnDemandRevalidate) {
......
import type { OutgoingHttpHeaders } from 'http' import type { OutgoingHttpHeaders } from 'http'
import type RenderResult from '../render-result' import type RenderResult from '../render-result'
import type { Revalidate } from '../lib/revalidate' import type { Revalidate } from '../lib/revalidate'
import type { RouteKind } from '../../server/future/route-kind'
export interface ResponseCacheBase { export interface ResponseCacheBase {
get( get(
...@@ -10,6 +11,12 @@ export interface ResponseCacheBase { ...@@ -10,6 +11,12 @@ export interface ResponseCacheBase {
isOnDemandRevalidate?: boolean isOnDemandRevalidate?: boolean
isPrefetch?: boolean isPrefetch?: boolean
incrementalCache: IncrementalCache incrementalCache: IncrementalCache
/**
* This is a hint to the cache to help it determine what kind of route
* this is so it knows where to look up the cache entry from. If not
* provided it will test the filesystem to check.
*/
routeKind?: RouteKind
} }
): Promise<ResponseCacheEntry | null> ): Promise<ResponseCacheEntry | null>
} }
...@@ -120,10 +127,18 @@ export type IncrementalCacheItem = { ...@@ -120,10 +127,18 @@ export type IncrementalCacheItem = {
isMiss?: boolean isMiss?: boolean
} | null } | null
export type IncrementalCacheKindHint = 'app' | 'pages' | 'fetch'
export interface IncrementalCache { export interface IncrementalCache {
get: ( get: (
key: string, key: string,
ctx?: { fetchCache?: boolean } ctx?: {
/**
* The kind of cache entry to get. If not provided it will try to
* determine the kind from the filesystem.
*/
kindHint?: IncrementalCacheKindHint
}
) => Promise<IncrementalCacheItem> ) => Promise<IncrementalCacheItem>
set: ( set: (
key: string, key: string,
......
...@@ -62,7 +62,8 @@ export default class NextWebServer extends BaseServer<WebServerOptions> { ...@@ -62,7 +62,8 @@ export default class NextWebServer extends BaseServer<WebServerOptions> {
dev, dev,
requestHeaders, requestHeaders,
requestProtocol: 'https', requestProtocol: 'https',
appDir: this.hasAppDir, pagesDir: this.enabledDirectories.pages,
appDir: this.enabledDirectories.app,
allowedRevalidateHeaderKeys: allowedRevalidateHeaderKeys:
this.nextConfig.experimental.allowedRevalidateHeaderKeys, this.nextConfig.experimental.allowedRevalidateHeaderKeys,
minimalMode: this.minimalMode, minimalMode: this.minimalMode,
...@@ -87,8 +88,11 @@ export default class NextWebServer extends BaseServer<WebServerOptions> { ...@@ -87,8 +88,11 @@ export default class NextWebServer extends BaseServer<WebServerOptions> {
return this.serverOptions.webServerConfig.extendRenderOpts.buildId return this.serverOptions.webServerConfig.extendRenderOpts.buildId
} }
protected getHasAppDir() { protected getEnabledDirectories() {
return this.serverOptions.webServerConfig.pagesType === 'app' return {
app: this.serverOptions.webServerConfig.pagesType === 'app',
pages: this.serverOptions.webServerConfig.pagesType === 'pages',
}
} }
protected getPagesManifest() { protected getPagesManifest() {
......
...@@ -107,7 +107,7 @@ export function unstable_cache<T extends Callback>( ...@@ -107,7 +107,7 @@ export function unstable_cache<T extends Callback>(
store?.isOnDemandRevalidate || incrementalCache.isOnDemandRevalidate store?.isOnDemandRevalidate || incrementalCache.isOnDemandRevalidate
) && ) &&
(await incrementalCache?.get(cacheKey, { (await incrementalCache?.get(cacheKey, {
fetchCache: true, kindHint: 'fetch',
revalidate: options.revalidate, revalidate: options.revalidate,
tags, tags,
softTags: implicitTags, softTags: implicitTags,
......
...@@ -448,6 +448,7 @@ export class MiddlewareNotFoundError extends Error { ...@@ -448,6 +448,7 @@ export class MiddlewareNotFoundError extends Error {
} }
export interface CacheFs { export interface CacheFs {
existsSync: typeof fs.existsSync
readFile: typeof fs.promises.readFile readFile: typeof fs.promises.readFile
readFileSync: typeof fs.readFileSync readFileSync: typeof fs.readFileSync
writeFile(f: string, d: any): Promise<void> writeFile(f: string, d: any): Promise<void>
......
...@@ -9,6 +9,7 @@ describe('FileSystemCache', () => { ...@@ -9,6 +9,7 @@ describe('FileSystemCache', () => {
it('set image route', async () => { it('set image route', async () => {
const fsCache = new FileSystemCache({ const fsCache = new FileSystemCache({
_appDir: true, _appDir: true,
_pagesDir: true,
_requestHeaders: {}, _requestHeaders: {},
flushToDisk: true, flushToDisk: true,
fs: nodeFs, fs: nodeFs,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment