Unverified Commit 0c3b063d authored by Hendrik Liebau's avatar Hendrik Liebau Committed by GitHub

Log fetch requests during client-side navigation (#67925)

When the config `logging.fetches` (see
https://nextjs.org/docs/app/api-reference/next-config-js/logging) is
defined, any fetch requests that are done in server components should be
logged. This is currently not the case for client-side navigations in
the app router. This PR fixes that, and also cleans up the log format a
bit. In addition, cached requests for the purpose of HMR refreshes (see
#67527) are explicitly omitted from the logs.

**Before**

<img width="963" alt="fetch logs before"
src="https://github.com/user-attachments/assets/f3d50495-1168-4e75-933e-635931f6efb0">

**After**

<img width="908" alt="fetch logs after"
src="https://github.com/user-attachments/assets/0933964c-1e02-4f14-b059-1ce536abec71">

Note that warnings for conflicting fetch cache configs are now
co-located with the logged fetch metrics. If `logging.fetches` is not
enabled, the warnings are still logged, as was the case before. They are
now only logged in dev mode though, which was not the case before.
parent eeff5557
...@@ -435,7 +435,9 @@ async function generateDynamicFlightRenderResult( ...@@ -435,7 +435,9 @@ async function generateDynamicFlightRenderResult(
} }
) )
return new FlightRenderResult(flightReadableStream) return new FlightRenderResult(flightReadableStream, {
fetchMetrics: ctx.staticGenerationStore.fetchMetrics,
})
} }
type RenderToStreamResult = { type RenderToStreamResult = {
......
import { RSC_CONTENT_TYPE_HEADER } from '../../client/components/app-router-headers' import { RSC_CONTENT_TYPE_HEADER } from '../../client/components/app-router-headers'
import RenderResult from '../render-result' import RenderResult, { type RenderResultMetadata } from '../render-result'
/** /**
* Flight Response is always set to RSC_CONTENT_TYPE_HEADER to ensure it does not get interpreted as HTML. * Flight Response is always set to RSC_CONTENT_TYPE_HEADER to ensure it does not get interpreted as HTML.
*/ */
export class FlightRenderResult extends RenderResult { export class FlightRenderResult extends RenderResult {
constructor(response: string | ReadableStream<Uint8Array>) { constructor(
super(response, { contentType: RSC_CONTENT_TYPE_HEADER, metadata: {} }) response: string | ReadableStream<Uint8Array>,
metadata: RenderResultMetadata = {}
) {
super(response, { contentType: RSC_CONTENT_TYPE_HEADER, metadata })
} }
} }
...@@ -19,7 +19,8 @@ export type FetchMetric = { ...@@ -19,7 +19,8 @@ export type FetchMetric = {
method: string method: string
status: number status: number
cacheReason: string cacheReason: string
cacheStatus: 'hit' | 'miss' | 'skip' cacheStatus: 'hit' | 'miss' | 'skip' | 'hmr'
cacheWarning?: string
} }
export type FetchMetrics = Array<FetchMetric> export type FetchMetrics = Array<FetchMetric>
......
...@@ -11,7 +11,6 @@ import { ...@@ -11,7 +11,6 @@ import {
NEXT_CACHE_TAG_MAX_ITEMS, NEXT_CACHE_TAG_MAX_ITEMS,
NEXT_CACHE_TAG_MAX_LENGTH, NEXT_CACHE_TAG_MAX_LENGTH,
} from '../../lib/constants' } from '../../lib/constants'
import * as Log from '../../build/output/log'
import { markCurrentScopeAsDynamic } from '../app-render/dynamic-rendering' import { markCurrentScopeAsDynamic } from '../app-render/dynamic-rendering'
import type { FetchMetric } from '../base-http' import type { FetchMetric } from '../base-http'
import { createDedupeFetch } from './dedupe-fetch' import { createDedupeFetch } from './dedupe-fetch'
...@@ -328,6 +327,7 @@ function createPatchedFetcher( ...@@ -328,6 +327,7 @@ function createPatchedFetcher(
let currentFetchCacheConfig = getRequestMeta('cache') let currentFetchCacheConfig = getRequestMeta('cache')
let cacheReason = '' let cacheReason = ''
let cacheWarning: string | undefined
if ( if (
typeof currentFetchCacheConfig === 'string' && typeof currentFetchCacheConfig === 'string' &&
...@@ -336,9 +336,7 @@ function createPatchedFetcher( ...@@ -336,9 +336,7 @@ function createPatchedFetcher(
// when providing fetch with a Request input, it'll automatically set a cache value of 'default' // when providing fetch with a Request input, it'll automatically set a cache value of 'default'
// we only want to warn if the user is explicitly setting a cache value // we only want to warn if the user is explicitly setting a cache value
if (!(isRequestInput && currentFetchCacheConfig === 'default')) { if (!(isRequestInput && currentFetchCacheConfig === 'default')) {
Log.warn( cacheWarning = `Specified "cache: ${currentFetchCacheConfig}" and "revalidate: ${currentFetchRevalidate}", only one should be specified.`
`fetch for ${fetchUrl} on ${staticGenerationStore.route} specified "cache: ${currentFetchCacheConfig}" and "revalidate: ${currentFetchRevalidate}", only one should be specified.`
)
} }
currentFetchCacheConfig = undefined currentFetchCacheConfig = undefined
} }
...@@ -587,6 +585,7 @@ function createPatchedFetcher( ...@@ -587,6 +585,7 @@ function createPatchedFetcher(
finalRevalidate === 0 || cacheReasonOverride finalRevalidate === 0 || cacheReasonOverride
? 'skip' ? 'skip'
: 'miss', : 'miss',
cacheWarning,
status: res.status, status: res.status,
method: clonedInit.method || 'GET', method: clonedInit.method || 'GET',
}) })
...@@ -647,6 +646,7 @@ function createPatchedFetcher( ...@@ -647,6 +646,7 @@ function createPatchedFetcher(
let handleUnlock = () => Promise.resolve() let handleUnlock = () => Promise.resolve()
let cacheReasonOverride let cacheReasonOverride
let isForegroundRevalidate = false let isForegroundRevalidate = false
let isHmrRefreshCache = false
if (cacheKey && staticGenerationStore.incrementalCache) { if (cacheKey && staticGenerationStore.incrementalCache) {
let cachedFetchData: CachedFetchData | undefined let cachedFetchData: CachedFetchData | undefined
...@@ -657,6 +657,8 @@ function createPatchedFetcher( ...@@ -657,6 +657,8 @@ function createPatchedFetcher(
) { ) {
cachedFetchData = cachedFetchData =
requestStore.serverComponentsHmrCache.get(cacheKey) requestStore.serverComponentsHmrCache.get(cacheKey)
isHmrRefreshCache = true
} }
if (isCacheableRevalidate && !cachedFetchData) { if (isCacheableRevalidate && !cachedFetchData) {
...@@ -712,7 +714,8 @@ function createPatchedFetcher( ...@@ -712,7 +714,8 @@ function createPatchedFetcher(
start: fetchStart, start: fetchStart,
url: fetchUrl, url: fetchUrl,
cacheReason, cacheReason,
cacheStatus: 'hit', cacheStatus: isHmrRefreshCache ? 'hmr' : 'hit',
cacheWarning,
status: cachedFetchData.status || 200, status: cachedFetchData.status || 200,
method: init?.method || 'GET', method: init?.method || 'GET',
}) })
......
...@@ -12,7 +12,6 @@ import type { MiddlewareManifest } from '../build/webpack/plugins/middleware-plu ...@@ -12,7 +12,6 @@ import type { MiddlewareManifest } from '../build/webpack/plugins/middleware-plu
import type RenderResult from './render-result' import type RenderResult from './render-result'
import type { FetchEventResult } from './web/types' import type { FetchEventResult } from './web/types'
import type { PrerenderManifest } from '../build' import type { PrerenderManifest } from '../build'
import type { FetchMetric } from './base-http'
import type { PagesManifest } from '../build/webpack/plugins/pages-manifest-plugin' import type { PagesManifest } from '../build/webpack/plugins/pages-manifest-plugin'
import type { NextParsedUrlQuery, NextUrlWithParsedQuery } from './request-meta' import type { NextParsedUrlQuery, NextUrlWithParsedQuery } from './request-meta'
import type { Params } from '../shared/lib/router/utils/route-matcher' import type { Params } from '../shared/lib/router/utils/route-matcher'
...@@ -1123,7 +1122,7 @@ export default class NextNodeServer extends BaseServer< ...@@ -1123,7 +1122,7 @@ export default class NextNodeServer extends BaseServer<
const shouldTruncateUrl = !loggingFetchesConfig?.fullUrl const shouldTruncateUrl = !loggingFetchesConfig?.fullUrl
if (this.renderOpts.dev) { if (this.renderOpts.dev) {
const { blue, green, yellow, red, gray, white } = const { blue, green, yellow, red, gray, white, bold } =
require('../lib/picocolors') as typeof import('../lib/picocolors') require('../lib/picocolors') as typeof import('../lib/picocolors')
const { originalResponse } = normalizedRes const { originalResponse } = normalizedRes
...@@ -1159,6 +1158,7 @@ export default class NextNodeServer extends BaseServer< ...@@ -1159,6 +1158,7 @@ export default class NextNodeServer extends BaseServer<
const loggingUrl = isRSC const loggingUrl = isRSC
? stripNextRscUnionQuery(requestUrl) ? stripNextRscUnionQuery(requestUrl)
: requestUrl : requestUrl
const indentation = '│ '
writeStdoutLine( writeStdoutLine(
`${method} ${loggingUrl} ${color( `${method} ${loggingUrl} ${color(
...@@ -1166,30 +1166,16 @@ export default class NextNodeServer extends BaseServer< ...@@ -1166,30 +1166,16 @@ export default class NextNodeServer extends BaseServer<
)} in ${reqDuration}ms` )} in ${reqDuration}ms`
) )
if (fetchMetrics.length && enabledVerboseLogging) { for (let i = 0; i < fetchMetrics.length; i++) {
const calcNestedLevel = ( const metric = fetchMetrics[i]
prevMetrics: FetchMetric[], let { cacheStatus, cacheReason, cacheWarning, url } = metric
start: number
): string => { if (cacheStatus === 'hmr') {
let nestedLevel = 0 // Cache hits during HMR refreshes are intentionally not logged.
continue
for (let i = 0; i < prevMetrics.length; i++) {
const metric = prevMetrics[i]
const prevMetric = prevMetrics[i - 1]
if (
metric.end <= start &&
!(prevMetric && prevMetric.start < metric.end)
) {
nestedLevel += 1
}
}
return nestedLevel === 0 ? ' ' : ' │ '.repeat(nestedLevel)
} }
for (let i = 0; i < fetchMetrics.length; i++) { if (enabledVerboseLogging) {
const metric = fetchMetrics[i]
let { cacheStatus, cacheReason } = metric
let cacheReasonStr = '' let cacheReasonStr = ''
let cacheColor let cacheColor
...@@ -1203,7 +1189,6 @@ export default class NextNodeServer extends BaseServer< ...@@ -1203,7 +1189,6 @@ export default class NextNodeServer extends BaseServer<
`Cache ${status} reason: (${white(cacheReason)})` `Cache ${status} reason: (${white(cacheReason)})`
) )
} }
let url = metric.url
if (url.length > 48) { if (url.length > 48) {
const parsed = new URL(url) const parsed = new URL(url)
...@@ -1229,27 +1214,30 @@ export default class NextNodeServer extends BaseServer< ...@@ -1229,27 +1214,30 @@ export default class NextNodeServer extends BaseServer<
} }
const status = cacheColor(`(cache ${cacheStatus})`) const status = cacheColor(`(cache ${cacheStatus})`)
const newLineLeadingChar = '│'
const nestedIndent = calcNestedLevel(
fetchMetrics.slice(0, i + 1),
metric.start
)
writeStdoutLine( writeStdoutLine(
`${newLineLeadingChar}${nestedIndent}${white( `${indentation}${white(
metric.method metric.method
)} ${white(url)} ${metric.status} in ${duration}ms ${status}` )} ${white(url)} ${metric.status} in ${duration}ms ${status}`
) )
if (cacheReasonStr) { if (cacheReasonStr) {
const nextNestedIndent = calcNestedLevel( writeStdoutLine(`${indentation}${indentation}${cacheReasonStr}`)
fetchMetrics.slice(0, i + 1), }
metric.start
)
if (cacheWarning) {
writeStdoutLine( writeStdoutLine(
`${newLineLeadingChar}${nextNestedIndent}${newLineLeadingChar} ${cacheReasonStr}` `${indentation}${indentation}${yellow(bold('⚠'))} ${white(cacheWarning)}`
) )
} }
} else if (cacheWarning) {
writeStdoutLine(
`${indentation}${white(metric.method)} ${white(url)}`
)
writeStdoutLine(
`${indentation}${indentation}${yellow(bold('⚠'))} ${white(cacheWarning)}`
)
} }
} }
delete normalizedReq.fetchMetrics delete normalizedReq.fetchMetrics
......
...@@ -2237,8 +2237,9 @@ describe('app-dir static/dynamic handling', () => { ...@@ -2237,8 +2237,9 @@ describe('app-dir static/dynamic handling', () => {
let prevHtml = await res.text() let prevHtml = await res.text()
let prev$ = cheerio.load(prevHtml) let prev$ = cheerio.load(prevHtml)
const cliOutputLength = next.cliOutput.length
await check(async () => { await retry(async () => {
const curRes = await next.fetch('/force-cache') const curRes = await next.fetch('/force-cache')
expect(curRes.status).toBe(200) expect(curRes.status).toBe(200)
...@@ -2258,14 +2259,18 @@ describe('app-dir static/dynamic handling', () => { ...@@ -2258,14 +2259,18 @@ describe('app-dir static/dynamic handling', () => {
expect(cur$('#data-auto-cache').text()).toBe( expect(cur$('#data-auto-cache').text()).toBe(
prev$('#data-auto-cache').text() prev$('#data-auto-cache').text()
) )
})
return 'success' if (isNextDev) {
}, 'success') await retry(() => {
const cliOutput = next.cliOutput
.slice(cliOutputLength)
.replace(/in \d+ms/g, 'in 0ms') // stub request durations
if (!isNextDeploy) { expect(stripAnsi(cliOutput)).toContain(`
expect(next.cliOutput).toContain( │ GET https://next-data-api-en../api/random?d4 200 in 0ms (cache hit)
'fetch for https://next-data-api-endpoint.vercel.app/api/random?d4 on /force-cache specified "cache: force-cache" and "revalidate: 3", only one should be specified.' │ │ ⚠ Specified "cache: force-cache" and "revalidate: 3", only one should be specified.`)
) })
} }
}) })
......
...@@ -126,12 +126,15 @@ describe('app-dir - logging', () => { ...@@ -126,12 +126,15 @@ describe('app-dir - logging', () => {
const outputIndex = next.cliOutput.length const outputIndex = next.cliOutput.length
await next.fetch('/default-cache') await next.fetch('/default-cache')
const expectedUrl = withFullUrlFetches
? 'https://next-data-api-endpoint.vercel.app/api/random'
: 'https://next-data-api-en../api/random'
await retry(() => { await retry(() => {
const logs = stripAnsi(next.cliOutput.slice(outputIndex)) const logs = stripAnsi(next.cliOutput.slice(outputIndex))
expect(logs).toContain(' GET /default-cache') expect(logs).toIncludeRepeated(' GET /default-cache', 1)
expect(logs).toContain(' │ GET ') expect(logs).toIncludeRepeated(` │ GET ${expectedUrl}`, 7)
expect(logs).toContain(' │ │ Cache skipped reason') expect(logs).toIncludeRepeated(' │ │ Cache skipped reason', 3)
expect(logs).toContain(' │ │ GET ')
}) })
}) })
...@@ -191,6 +194,40 @@ describe('app-dir - logging', () => { ...@@ -191,6 +194,40 @@ describe('app-dir - logging', () => {
expect(logs).not.toContain('/_next/static') expect(logs).not.toContain('/_next/static')
expect(logs).not.toContain('?_rsc') expect(logs).not.toContain('?_rsc')
}) })
it('should log requests for client-side navigations', async () => {
const outputIndex = next.cliOutput.length
const browser = await next.browser('/')
await browser.elementById('nav-default-cache').click()
await browser.waitForElementByCss('h1')
const expectedUrl = withFullUrlFetches
? 'https://next-data-api-endpoint.vercel.app/api/random'
: 'https://next-data-api-en../api/random'
await retry(() => {
const logs = stripAnsi(next.cliOutput.slice(outputIndex))
expect(logs).toIncludeRepeated(` │ GET ${expectedUrl}`, 7)
})
})
it('should not log requests for HMR refreshes', async () => {
const browser = await next.browser('/default-cache')
let headline = await browser.waitForElementByCss('h1').text()
expect(headline).toBe('Default Cache')
const outputIndex = next.cliOutput.length
await next.patchFile(
'app/default-cache/page.js',
(content) => content.replace('Default Cache', 'Hello!'),
async () => {
headline = await browser.waitForElementByCss('h1').text()
expect(headline).toBe('Hello!')
const logs = stripAnsi(next.cliOutput.slice(outputIndex))
expect(logs).not.toInclude(` │ GET `)
}
)
})
} }
} else { } else {
// No fetches logging enabled // No fetches logging enabled
......
import { check } from 'next-test-utils' import { retry } from 'next-test-utils'
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
describe('app-dir - fetch warnings', () => { describe('app-dir - fetch warnings', () => {
const { next, skipped } = nextTestSetup({ const { next, skipped, isNextDev } = nextTestSetup({
skipDeployment: true, skipDeployment: true,
files: __dirname, files: __dirname,
}) })
...@@ -20,33 +20,37 @@ describe('app-dir - fetch warnings', () => { ...@@ -20,33 +20,37 @@ describe('app-dir - fetch warnings', () => {
await next.fetch('/cache-revalidate') await next.fetch('/cache-revalidate')
}) })
it('should log when request input is a string', async () => { if (isNextDev) {
await check(() => { it('should log when request input is a string', async () => {
return next.cliOutput.includes( await retry(() => {
'fetch for https://next-data-api-endpoint.vercel.app/api/random?request-string on /cache-revalidate specified "cache: force-cache" and "revalidate: 3", only one should be specified' expect(next.cliOutput).toInclude(`
) │ GET https://next-data-api-endpoint.vercel.app/api/random?request-string
? 'success' │ │ ⚠ Specified "cache: force-cache" and "revalidate: 3", only one should be specified.`)
: 'fail' })
}, 'success') })
})
it('should log when request input is a Request instance', async () => { it('should log when request input is a Request instance', async () => {
await check(() => { await retry(() => {
return next.cliOutput.includes( expect(next.cliOutput).toInclude(`
'fetch for https://next-data-api-endpoint.vercel.app/api/random?request-input-cache-override on /cache-revalidate specified "cache: force-cache" and "revalidate: 3", only one should be specified.' │ GET https://next-data-api-endpoint.vercel.app/api/random?request-input-cache-override
) │ │ ⚠ Specified "cache: force-cache" and "revalidate: 3", only one should be specified.`)
? 'success' })
: 'fail' })
}, 'success')
})
it('should not log when overriding cache within the Request object', async () => { it('should not log when overriding cache within the Request object', async () => {
await check(() => { await retry(() => {
return next.cliOutput.includes( expect(next.cliOutput).not.toInclude(`
`fetch for https://next-data-api-endpoint.vercel.app/api/random?request-input on /cache-revalidate specified "cache: default" and "revalidate: 3", only one should be specified.` │ GET https://next-data-api-endpoint.vercel.app/api/random?request-input
) │ │ ⚠ Specified "cache: force-cache" and "revalidate: 3", only one should be specified.`)
? 'fail' })
: 'success' })
}, 'success') } else {
}) it('should not log fetch warnings in production', async () => {
await retry(() => {
expect(next.cliOutput).not.toInclude(
'⚠ Specified "cache: force-cache" and "revalidate: 3", only one should be specified.'
)
})
})
}
}) })
...@@ -8,7 +8,7 @@ import { ChildProcess } from 'child_process' ...@@ -8,7 +8,7 @@ import { ChildProcess } from 'child_process'
import { createNextInstall } from '../create-next-install' import { createNextInstall } from '../create-next-install'
import { Span } from 'next/dist/trace' import { Span } from 'next/dist/trace'
import webdriver from '../next-webdriver' import webdriver from '../next-webdriver'
import { renderViaHTTP, fetchViaHTTP, findPort } from 'next-test-utils' import { renderViaHTTP, fetchViaHTTP, findPort, retry } from 'next-test-utils'
import cheerio from 'cheerio' import cheerio from 'cheerio'
import { once } from 'events' import { once } from 'events'
import { BrowserInterface } from '../browsers/base' import { BrowserInterface } from '../browsers/base'
...@@ -490,19 +490,31 @@ export class NextInstance { ...@@ -490,19 +490,31 @@ export class NextInstance {
public async patchFile( public async patchFile(
filename: string, filename: string,
content: string | ((contents: string) => string) content: string | ((content: string) => string),
retryWithTempContent?: (context: { newFile: boolean }) => Promise<void>
): Promise<{ newFile: boolean }> { ): Promise<{ newFile: boolean }> {
const outputPath = path.join(this.testDir, filename) const outputPath = path.join(this.testDir, filename)
const newFile = !existsSync(outputPath) const newFile = !existsSync(outputPath)
await fs.mkdir(path.dirname(outputPath), { recursive: true }) await fs.mkdir(path.dirname(outputPath), { recursive: true })
const previousContent = newFile ? undefined : await this.readFile(filename)
await fs.writeFile( await fs.writeFile(
outputPath, outputPath,
typeof content === 'function' typeof content === 'function' ? content(previousContent) : content
? content(await this.readFile(filename))
: content
) )
if (retryWithTempContent) {
try {
await retry(() => retryWithTempContent({ newFile }))
} finally {
if (previousContent === undefined) {
await fs.rm(outputPath)
} else {
await fs.writeFile(outputPath, previousContent)
}
}
}
return { newFile } return { newFile }
} }
......
...@@ -152,7 +152,8 @@ export class NextDevInstance extends NextInstance { ...@@ -152,7 +152,8 @@ export class NextDevInstance extends NextInstance {
public override async patchFile( public override async patchFile(
filename: string, filename: string,
content: string | ((contents: string) => string) content: string | ((contents: string) => string),
retryWithTempContent?: (context: { newFile: boolean }) => Promise<void>
) { ) {
const isServerRunning = this.childProcess && !this.isStopping const isServerRunning = this.childProcess && !this.isStopping
const cliOutputLength = this.cliOutput.length const cliOutputLength = this.cliOutput.length
...@@ -161,20 +162,28 @@ export class NextDevInstance extends NextInstance { ...@@ -161,20 +162,28 @@ export class NextDevInstance extends NextInstance {
await this.handleDevWatchDelayBeforeChange(filename) await this.handleDevWatchDelayBeforeChange(filename)
} }
const { newFile } = await super.patchFile(filename, content) const waitForChanges = async ({ newFile }: { newFile: boolean }) => {
if (isServerRunning) {
if (isServerRunning) { if (newFile) {
if (newFile) { await this.handleDevWatchDelayAfterChange(filename)
await this.handleDevWatchDelayAfterChange(filename) } else if (filename.startsWith('next.config')) {
} else if (filename.startsWith('next.config')) {
await retry(() => {
if (!this.cliOutput.slice(cliOutputLength).includes('Ready in')) { if (!this.cliOutput.slice(cliOutputLength).includes('Ready in')) {
throw new Error('Server has not finished restarting.') throw new Error('Server has not finished restarting.')
} }
}) }
} }
} }
if (retryWithTempContent) {
return super.patchFile(filename, content, async ({ newFile }) => {
await waitForChanges({ newFile })
await retryWithTempContent({ newFile })
})
}
const { newFile } = await super.patchFile(filename, content)
await retry(() => waitForChanges({ newFile }))
return { newFile } return { newFile }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment