Unverified Commit 4e0731fb authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Fix app static generation cases (#41172)

Co-authored-by: 's avatarTim Neutkens <tim@timneutkens.nl>
Co-authored-by: 's avatarkodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
parent dd3e005c
......@@ -237,7 +237,6 @@ function fillCacheWithPrefetchedSubTreeData(
const existingChildCacheNode = existingChildSegmentMap.get(segmentForCache)
// In case of last segment start the fetch at this level and don't copy further down.
if (isLastEntry) {
if (!existingChildCacheNode) {
existingChildSegmentMap.set(segmentForCache, {
......@@ -772,8 +771,6 @@ function clientReducer(
href
)
// Fill in the cache with blank that holds the `data` field.
// TODO-APP: segments.slice(1) strips '', we can get rid of '' altogether.
// Copy subTreeData for the root node of the cache.
cache.subTreeData = state.cache.subTreeData
......@@ -782,6 +779,7 @@ function clientReducer(
const res = fillCacheWithDataProperty(
cache,
state.cache,
// TODO-APP: segments.slice(1) strips '', we can get rid of '' altogether.
segments.slice(1),
() => fetchServerResponse(url, optimisticTree)
)
......@@ -839,7 +837,7 @@ function clientReducer(
const flightDataPath = flightData[0]
// The one before last item is the router state tree patch
const [treePatch] = flightDataPath.slice(-2)
const [treePatch, subTreeData] = flightDataPath.slice(-2)
// Path without the last segment, router state, and the subTreeData
const flightSegmentPath = flightDataPath.slice(0, -3)
......@@ -865,10 +863,14 @@ function clientReducer(
mutable.previousTree = state.tree
mutable.patchedTree = newTree
// Copy subTreeData for the root node of the cache.
cache.subTreeData = state.cache.subTreeData
// Create a copy of the existing cache with the subTreeData applied.
fillCacheWithNewSubTreeData(cache, state.cache, flightDataPath)
if (flightDataPath.length === 2) {
cache.subTreeData = subTreeData
} else {
// Copy subTreeData for the root node of the cache.
cache.subTreeData = state.cache.subTreeData
// Create a copy of the existing cache with the subTreeData applied.
fillCacheWithNewSubTreeData(cache, state.cache, flightDataPath)
}
return {
// Set href.
......@@ -939,7 +941,7 @@ function clientReducer(
// Slices off the last segment (which is at -3) as it doesn't exist in the tree yet
const treePath = flightDataPath.slice(0, -3)
const [treePatch] = flightDataPath.slice(-2)
const [treePatch, subTreeData] = flightDataPath.slice(-2)
const newTree = applyRouterStatePatchToTree(
// TODO-APP: remove ''
......@@ -962,9 +964,14 @@ function clientReducer(
mutable.patchedTree = newTree
// Copy subTreeData for the root node of the cache.
cache.subTreeData = state.cache.subTreeData
fillCacheWithNewSubTreeData(cache, state.cache, flightDataPath)
// Root refresh
if (flightDataPath.length === 2) {
cache.subTreeData = subTreeData
} else {
// Copy subTreeData for the root node of the cache.
cache.subTreeData = state.cache.subTreeData
fillCacheWithNewSubTreeData(cache, state.cache, flightDataPath)
}
return {
// Keep href as it was set during navigate / restore
......
......@@ -96,6 +96,7 @@ interface RenderOpts {
defaultLocale?: string
domainLocales?: DomainLocale[]
trailingSlash?: boolean
supportsDynamicHTML?: boolean
}
type ComponentModule = ComponentType<{}> & {
......@@ -389,6 +390,7 @@ export default async function exportPage({
? requireFontManifest(distDir, serverless)
: null,
locale: locale as string,
supportsDynamicHTML: false,
}
// during build we attempt rendering app dir paths
......@@ -406,8 +408,7 @@ export default async function exportPage({
res as any,
page,
query,
curRenderOpts as any,
true
curRenderOpts as any
)
const html = result?.toUnchunkedString()
const flightData = (curRenderOpts as any).pageData
......
......@@ -34,6 +34,7 @@ import { REDIRECT_ERROR_CODE } from '../client/components/redirect'
import { NextCookies } from './web/spec-extension/cookies'
import { DYNAMIC_ERROR_CODE } from '../client/components/hooks-server-context'
import { NOT_FOUND_ERROR_CODE } from '../client/components/not-found'
import { Writable } from 'stream'
const INTERNAL_HEADERS_INSTANCE = Symbol('internal for headers readonly')
......@@ -694,9 +695,22 @@ export async function renderToHTMLOrFlight(
res: ServerResponse,
pathname: string,
query: NextParsedUrlQuery,
renderOpts: RenderOpts,
isStaticGeneration: boolean = false
renderOpts: RenderOpts
): Promise<RenderResult | null> {
/**
* Rules of Static & Dynamic HTML:
*
* 1.) We must generate static HTML unless the caller explicitly opts
* in to dynamic HTML support.
*
* 2.) If dynamic HTML support is requested, we must honor that request
* or throw an error. It is the sole responsibility of the caller to
* ensure they aren't e.g. requesting dynamic HTML for an AMP page.
*
* These rules help ensure that other existing features like request caching,
* coalescing, and ISR continue working as intended.
*/
const isStaticGeneration = renderOpts.supportsDynamicHTML !== true
const isFlight = req.headers.__rsc__ !== undefined
const capturedErrors: Error[] = []
......@@ -719,7 +733,6 @@ export async function renderToHTMLOrFlight(
subresourceIntegrityManifest,
serverComponentManifest,
serverCSSManifest = {},
supportsDynamicHTML,
ComponentMod,
dev,
fontLoaderManifest,
......@@ -758,10 +771,10 @@ export async function renderToHTMLOrFlight(
/**
* Router state provided from the client-side router. Used to handle rendering from the common layout down.
*/
const providedFlightRouterState: FlightRouterState = isFlight
let providedFlightRouterState: FlightRouterState = isFlight
? req.headers.__next_router_state_tree__
? JSON.parse(req.headers.__next_router_state_tree__ as string)
: {}
: undefined
: undefined
/**
......@@ -1160,23 +1173,22 @@ export async function renderToHTMLOrFlight(
}
}
/**
* Rules of Static & Dynamic HTML:
*
* 1.) We must generate static HTML unless the caller explicitly opts
* in to dynamic HTML support.
*
* 2.) If dynamic HTML support is requested, we must honor that request
* or throw an error. It is the sole responsibility of the caller to
* ensure they aren't e.g. requesting dynamic HTML for an AMP page.
*
* These rules help ensure that other existing features like request caching,
* coalescing, and ISR continue working as intended.
*/
const generateStaticHTML = supportsDynamicHTML !== true
const streamToBufferedResult = async (
renderResult: RenderResult
): Promise<string> => {
const renderChunks: Buffer[] = []
const writable = new Writable({
write(chunk, _encoding, callback) {
renderChunks.push(chunk)
callback()
},
})
await renderResult.pipe(writable)
return Buffer.concat(renderChunks).toString()
}
// Handle Flight render request. This is only used when client-side navigating. E.g. when you `router.push('/dashboard')` or `router.reload()`.
if (isFlight) {
const generateFlight = async (): Promise<RenderResult> => {
// TODO-APP: throw on invalid flightRouterState
/**
* Use router state to decide at what common layout to render the page.
......@@ -1308,15 +1320,13 @@ export async function renderToHTMLOrFlight(
}
).pipeThrough(createBufferedTransformStream())
if (generateStaticHTML) {
let staticHtml = Buffer.from(
(await readable.getReader().read()).value || ''
).toString()
return new FlightRenderResult(staticHtml)
}
return new FlightRenderResult(readable)
}
if (isFlight && !isStaticGeneration) {
return generateFlight()
}
// Below this line is handling for rendering to HTML.
// Create full component tree from root to leaf.
......@@ -1450,7 +1460,7 @@ export async function renderToHTMLOrFlight(
return await continueFromInitialStream(renderStream, {
dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML: generateStaticHTML,
generateStaticHTML: isStaticGeneration,
getServerInsertedHTML,
serverInsertedHTMLToHead: true,
polyfills,
......@@ -1482,7 +1492,7 @@ export async function renderToHTMLOrFlight(
return await continueFromInitialStream(renderStream, {
dataStream: serverComponentsInlinedTransformStream?.readable,
generateStaticHTML: generateStaticHTML,
generateStaticHTML: isStaticGeneration,
getServerInsertedHTML,
serverInsertedHTMLToHead: true,
polyfills,
......@@ -1490,31 +1500,34 @@ export async function renderToHTMLOrFlight(
})
}
}
const renderResult = new RenderResult(await bodyResult())
if (generateStaticHTML) {
const readable = await bodyResult()
let staticHtml = Buffer.from(
(await readable.getReader().read()).value || ''
).toString()
if (isStaticGeneration) {
const htmlResult = await streamToBufferedResult(renderResult)
// if we encountered any unexpected errors during build
// we fail the prerendering phase and the build
if (capturedErrors.length > 0) {
throw capturedErrors[0]
}
// const before = Buffer.concat(
// serverComponentsRenderOpts.rscChunks
// ).toString()
// TODO-APP: derive this from same pass to prevent additional
// render during static generation
const filteredFlightData = await streamToBufferedResult(
await generateFlight()
)
;(renderOpts as any).pageData = Buffer.concat(
serverComponentsRenderOpts.rscChunks
).toString()
;(renderOpts as any).pageData = filteredFlightData
;(renderOpts as any).revalidate =
typeof staticGenerationStore?.revalidate === 'undefined'
? defaultRevalidate
: staticGenerationStore?.revalidate
return new RenderResult(staticHtml)
return new RenderResult(htmlResult)
}
return new RenderResult(await bodyResult())
return renderResult
}
const initialStaticGenerationStore = {
......
......@@ -158,7 +158,7 @@ export class WrappedBuildError extends Error {
}
type ResponsePayload = {
type: 'html' | 'json'
type: 'html' | 'json' | 'rsc'
body: RenderResult
revalidateOptions?: any
}
......@@ -238,7 +238,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
protected abstract getPublicDir(): string
protected abstract getHasStaticDir(): boolean
protected abstract getHasAppDir(): boolean
protected abstract getHasAppDir(dev: boolean): boolean
protected abstract getPagesManifest(): PagesManifest | undefined
protected abstract getAppPathsManifest(): PagesManifest | undefined
protected abstract getBuildId(): string
......@@ -287,7 +287,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
res: BaseNextResponse,
options: {
result: RenderResult
type: 'html' | 'json'
type: 'html' | 'json' | 'rsc'
generateEtags: boolean
poweredByHeader: boolean
options?: PayloadOptions
......@@ -355,7 +355,6 @@ export default abstract class Server<ServerOptions extends Options = Options> {
: require('path').join(this.dir, this.nextConfig.distDir)
this.publicDir = this.getPublicDir()
this.hasStaticDir = !minimalMode && this.getHasStaticDir()
this.hasAppDir = this.getHasAppDir()
// Only serverRuntimeConfig needs the default
// publicRuntimeConfig gets it's default in client/index.js
......@@ -369,6 +368,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
this.buildId = this.getBuildId()
this.minimalMode = minimalMode || !!process.env.NEXT_PRIVATE_MINIMAL_MODE
this.hasAppDir = this.getHasAppDir(dev)
const serverComponents = this.hasAppDir
this.serverComponentManifest = serverComponents
? this.getServerComponentManifest()
......@@ -1001,17 +1001,6 @@ export default abstract class Server<ServerOptions extends Options = Options> {
const hasGetInitialProps = !!components.Component?.getInitialProps
let isSSG = !!components.getStaticProps
// Toggle whether or not this is a Data request
const isDataReq =
!!(
query.__nextDataReq ||
(req.headers['x-nextjs-data'] &&
(this.serverOptions as any).webServerConfig)
) &&
(isSSG || hasServerProps)
delete query.__nextDataReq
// Compute the iSSG cache key. We use the rewroteUrl since
// pages with fallback: false are allowed to be rewritten to
// and we need to look up the path by the rewritten path
......@@ -1040,8 +1029,36 @@ export default abstract class Server<ServerOptions extends Options = Options> {
if (hasFallback || staticPaths?.includes(resolvedUrlPathname)) {
isSSG = true
} else if (!this.renderOpts.dev) {
const manifest = this.getPrerenderManifest()
isSSG =
isSSG || !!manifest.routes[pathname === '/index' ? '/' : pathname]
}
}
// Toggle whether or not this is a Data request
let isDataReq =
!!(
query.__nextDataReq ||
(req.headers['x-nextjs-data'] &&
(this.serverOptions as any).webServerConfig)
) &&
(isSSG || hasServerProps)
if (isAppPath && req.headers['__rsc__']) {
if (isSSG) {
isDataReq = true
// strip header so we generate HTML still
if (
opts.runtime !== 'experimental-edge' ||
(this.serverOptions as any).webServerConfig
) {
delete req.headers['__rsc__']
delete req.headers['__next_router_state_tree__']
}
}
}
delete query.__nextDataReq
// normalize req.url for SSG paths as it is not exposed
// to getStaticProps and the asPath should not expose /_next/data
......@@ -1206,7 +1223,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
}
let ssgCacheKey =
isPreviewMode || !isSSG || opts.supportsDynamicHTML || isFlightRequest
isPreviewMode || !isSSG || opts.supportsDynamicHTML
? null // Preview mode, manual revalidate, flight request can bypass the cache
: `${locale ? `/${locale}` : ''}${
(pathname === '/' || resolvedUrlPathname === '/') && locale
......@@ -1564,9 +1581,13 @@ export default abstract class Server<ServerOptions extends Options = Options> {
throw new Error('invariant SSG should not return an image cache value')
} else {
return {
type: isDataReq ? 'json' : 'html',
type: isDataReq ? (isAppPath ? 'rsc' : 'json') : 'html',
body: isDataReq
? RenderResult.fromStatic(JSON.stringify(cachedData.pageData))
? RenderResult.fromStatic(
isAppPath
? (cachedData.pageData as string)
: JSON.stringify(cachedData.pageData)
)
: cachedData.html,
revalidateOptions,
}
......
......@@ -475,11 +475,16 @@ export default class NextNodeServer extends BaseServer {
]
}
protected getHasAppDir(): boolean {
const appDirectory = join(this.dir, 'app')
return (
fs.existsSync(appDirectory) && fs.statSync(appDirectory).isDirectory()
)
protected getHasAppDir(dev: boolean): boolean {
const appDirectory = dev
? join(this.dir, 'app')
: join(this.serverDistDir, 'app')
try {
return fs.statSync(appDirectory).isDirectory()
} catch (err) {
return false
}
}
protected generateStaticRoutes(): Route[] {
......
import type { ServerResponse } from 'http'
import { Writable } from 'stream'
type ContentTypeOption = string | undefined
......@@ -27,7 +28,7 @@ export default class RenderResult {
return this._result
}
pipe(res: ServerResponse): Promise<void> {
pipe(res: ServerResponse | Writable): Promise<void> {
if (typeof this._result === 'string') {
throw new Error(
'invariant: static responses cannot be piped. This is a bug in Next.js'
......
......@@ -49,7 +49,7 @@ export async function sendRenderResult({
req: IncomingMessage
res: ServerResponse
result: RenderResult
type: 'html' | 'json'
type: 'html' | 'json' | 'rsc'
generateEtags: boolean
poweredByHeader: boolean
options?: PayloadOptions
......@@ -78,6 +78,8 @@ export async function sendRenderResult({
'Content-Type',
resultContentType
? resultContentType
: type === 'rsc'
? 'application/octet-stream'
: type === 'json'
? 'application/json'
: 'text/html; charset=utf-8'
......
......@@ -378,8 +378,7 @@ export default class NextWebServer extends BaseServer<WebServerOptions> {
Object.assign(renderOpts, {
disableOptimizedLoading: true,
runtime: 'experimental-edge',
}),
!!pagesRenderToHTML
})
)
} else {
throw new Error(`Invariant: curRenderToHTML is missing`)
......
......@@ -4,7 +4,8 @@ import { promisify } from 'util'
import path, { join } from 'path'
import { createNext, FileRef } from 'e2e-utils'
import { NextInstance } from 'test/lib/next-modes/base'
import { fetchViaHTTP, normalizeRegEx } from 'next-test-utils'
import { check, fetchViaHTTP, normalizeRegEx } from 'next-test-utils'
import webdriver from 'next-webdriver'
const glob = promisify(globOrig)
......@@ -56,6 +57,8 @@ describe('app-dir static/dynamic handling', () => {
'blog/tim.rsc',
'blog/tim/first-post.html',
'blog/tim/first-post.rsc',
'dynamic-no-gen-params-ssr/[slug]/page.js',
'dynamic-no-gen-params/[slug]/page.js',
'ssr-auto/page.js',
'ssr-forced/page.js',
])
......@@ -162,6 +165,32 @@ describe('app-dir static/dynamic handling', () => {
}
})
it('should work with forced dynamic path', async () => {
for (const slug of ['first', 'second']) {
const res = await fetchViaHTTP(
next.url,
`/dynamic-no-gen-params-ssr/${slug}`,
undefined,
{ redirect: 'manual' }
)
expect(res.status).toBe(200)
expect(await res.text()).toContain(`${slug}`)
}
})
it('should work with dynamic path no generateStaticParams', async () => {
for (const slug of ['first', 'second']) {
const res = await fetchViaHTTP(
next.url,
`/dynamic-no-gen-params/${slug}`,
undefined,
{ redirect: 'manual' }
)
expect(res.status).toBe(200)
expect(await res.text()).toContain(`${slug}`)
}
})
it('should handle dynamicParams: true correctly', async () => {
const paramsToCheck = [
{
......@@ -200,6 +229,41 @@ describe('app-dir static/dynamic handling', () => {
}
})
it('should navigate to static path correctly', async () => {
const browser = await webdriver(next.url, '/blog/tim')
await browser.eval('window.beforeNav = 1')
expect(await browser.eval('document.documentElement.innerHTML')).toContain(
'/blog/[author]'
)
await browser.elementByCss('#author-2').click()
await check(async () => {
const params = JSON.parse(await browser.elementByCss('#params').text())
return params.author === 'seb' ? 'found' : params
}, 'found')
expect(await browser.eval('window.beforeNav')).toBe(1)
await browser.elementByCss('#author-1-post-1').click()
await check(async () => {
const params = JSON.parse(await browser.elementByCss('#params').text())
return params.author === 'tim' && params.slug === 'first-post'
? 'found'
: params
}, 'found')
expect(await browser.eval('window.beforeNav')).toBe(1)
await browser.back()
await check(async () => {
const params = JSON.parse(await browser.elementByCss('#params').text())
return params.author === 'seb' ? 'found' : params
}, 'found')
expect(await browser.eval('window.beforeNav')).toBe(1)
})
it('should ssr dynamically when detected automatically', async () => {
const initialRes = await fetchViaHTTP(next.url, '/ssr-auto', undefined, {
redirect: 'manual',
......
import Link from 'next/link'
export const config = {
dynamicParams: false,
}
......@@ -8,6 +10,36 @@ export default function Page({ params }) {
<p id="page">/blog/[author]</p>
<p id="params">{JSON.stringify(params)}</p>
<p id="date">{Date.now()}</p>
<Link href="/blog/tim">
<a id="author-1">/blog/tim</a>
</Link>
<br />
<Link href="/blog/seb">
<a id="author-2">/blog/seb</a>
</Link>
<br />
<Link href="/blog/styfle">
<a id="author-3">/blog/styfle</a>
</Link>
<br />
<Link href="/blog/tim/first-post">
<a id="author-1-post-1">/blog/tim/first-post</a>
</Link>
<br />
<Link href="/blog/seb/second-post">
<a id="author-2-post-1">/blog/seb/second-post</a>
</Link>
<br />
<Link href="/blog/styfle/first-post">
<a id="author-3-post-1">/blog/styfle/first-post</a>
</Link>
<br />
<Link href="/dynamic-no-gen-params/first">
<a id="dynamic-no-params">/dynamic-no-gen-params/first</a>
</Link>
<br />
</>
)
}
export const config = {
revalidate: 0,
}
export default function Page({ params }) {
return (
<>
<p id="page">/dynamic-no-gen-params-ssr</p>
<p id="params">{JSON.stringify(params)}</p>
</>
)
}
import Link from 'next/link'
export default function Page({ params }) {
return (
<>
<p id="page">/dynamic-no-gen-params</p>
<p id="params">{JSON.stringify(params)}</p>
<Link href="/dynamic-no-gen-params/second">
<a id="dynamic-no-params-again">/dynamic-no-gen-params/second</a>
</Link>
<br />
<Link href="/blog/styfle">
<a id="to-blog">/blog/styfle</a>
</Link>
<br />
</>
)
}
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ children }) {
return (
<html>
......
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ children }) {
return (
<html>
......
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ children }) {
return (
<html>
......
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ children }) {
return (
<html>
......
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ one, two }) {
return (
<html>
......
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ children }) {
return (
<html>
......
// TODO-APP: remove after fixing filtering static flight data
export const config = {
revalidate: 0,
}
export default function Root({ children }) {
return children
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment