Unverified Commit 7c408e7a authored by Jamie's avatar Jamie Committed by GitHub

Fix ISR page re-rendering after revalidate expiry (Fork of #24807) (#27335)



Fixes: #24806 
Fixes: #26689
Fixes: #27325
Closes: #24807

@tommarshall has done us a huge favor with his PR https://github.com/vercel/next.js/pull/24807 which outlines exactly the issue and a pragmatic solution.

I'm not trying to "steal their work", however, the PR seems to have been stuck for some months. I think there's huge value in this for myself and others as essentially **ISR is broken** for people running Next.js at scale 😱 

This PR has cherry-picked @tommarshall's fine fix and added some integrations tests around page revalidation and the edge case when the cache size is exhausted.

✏️ Edits are enabled, so feel free great Vercel staff and other maintainers to improve my bad tests or surrounding code 🙇 

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added

parent ade52553
......@@ -80,6 +80,11 @@ export class IncrementalCache {
)
}
if (process.env.__NEXT_TEST_MAX_ISR_CACHE) {
// Allow cache size to be overridden for testing purposes
max = parseInt(process.env.__NEXT_TEST_MAX_ISR_CACHE, 10)
}
this.cache = new LRUCache({
// default to 50MB limit
max: max || 50 * 1024 * 1024,
......@@ -95,13 +100,15 @@ export class IncrementalCache {
return path.join(this.incrementalOptions.pagesDir!, `${pathname}.${ext}`)
}
private calculateRevalidate(pathname: string): number | false {
private calculateRevalidate(
pathname: string,
fromTime: number
): number | false {
pathname = toRoute(pathname)
// in development we don't have a prerender-manifest
// and default to always revalidating to allow easier debugging
const curTime = new Date().getTime()
if (this.incrementalOptions.dev) return curTime - 1000
if (this.incrementalOptions.dev) return new Date().getTime() - 1000
const { initialRevalidateSeconds } = this.prerenderManifest.routes[
pathname
......@@ -110,7 +117,7 @@ export class IncrementalCache {
}
const revalidateAfter =
typeof initialRevalidateSeconds === 'number'
? initialRevalidateSeconds * 1000 + curTime
? initialRevalidateSeconds * 1000 + fromTime
: initialRevalidateSeconds
return revalidateAfter
......@@ -135,16 +142,15 @@ export class IncrementalCache {
}
try {
const html = await promises.readFile(
this.getSeedPath(pathname, 'html'),
'utf8'
)
const htmlPath = this.getSeedPath(pathname, 'html')
const html = await promises.readFile(htmlPath, 'utf8')
const { mtime } = await promises.stat(htmlPath)
const pageData = JSON.parse(
await promises.readFile(this.getSeedPath(pathname, 'json'), 'utf8')
)
data = {
revalidateAfter: this.calculateRevalidate(pathname),
revalidateAfter: this.calculateRevalidate(pathname, mtime.getTime()),
value: {
kind: 'PAGE',
html,
......@@ -199,7 +205,7 @@ export class IncrementalCache {
pathname = normalizePagePath(pathname)
this.cache.set(pathname, {
revalidateAfter: this.calculateRevalidate(pathname),
revalidateAfter: this.calculateRevalidate(pathname, new Date().getTime()),
value: data,
})
......
export async function getStaticProps() {
return {
props: {
world: 'world',
time: new Date().getTime(),
other: Math.random(),
},
revalidate: 1,
}
}
const Page = ({ world, time, other }) => {
return (
<div>
<p>hello {world}</p>
<span>time: {time}</span>
<span>other: {other}</span>
</div>
)
}
export default Page
export async function getStaticProps() {
return {
props: {
world: 'world',
time: new Date().getTime(),
other: Math.random(),
},
revalidate: 1,
}
}
const Page = ({ world, time, other }) => {
return (
<div>
<p>hello {world}</p>
<span>time: {time}</span>
<span>other: {other}</span>
</div>
)
}
export default Page
export async function getStaticProps() {
return {
props: {
world: 'world',
time: new Date().getTime(),
other: Math.random(),
},
revalidate: 1,
}
}
const Page = ({ world, time, other }) => {
return (
<div>
<p>hello {world}</p>
<span>time: {time}</span>
<span>other: {other}</span>
</div>
)
}
export default Page
export async function getStaticProps() {
return {
props: {
world: 'world',
time: new Date().getTime(),
other: Math.random(),
},
revalidate: 1,
}
}
const Page = ({ world, time, other }) => {
return (
<div>
<p>hello {world}</p>
<span>time: {time}</span>
<span>other: {other}</span>
</div>
)
}
export default Page
/* eslint-env jest */
import fs from 'fs-extra'
import {
findPort,
killApp,
nextBuild,
nextStart,
renderViaHTTP,
waitFor,
getPageFileFromPagesManifest,
} from 'next-test-utils'
import { join } from 'path'
jest.setTimeout(1000 * 60 * 2)
const appDir = join(__dirname, '..')
let app
let appPort
let buildId
function runTests(route, routePath, serverless) {
it(`[${route}] should regenerate page when revalidate time exceeded`, async () => {
const fileName = join(
appDir,
'.next',
'server',
getPageFileFromPagesManifest(appDir, routePath).replace('.js', '.html')
)
const initialHtmlFile = await fs.readFile(fileName, 'utf8')
await waitFor(1000) // Wait revalidate duration
expect(await renderViaHTTP(appPort, route)).toBe(initialHtmlFile)
await waitFor(500) // Wait for regeneration to occur
const regeneratedFileHtml = await fs.readFile(fileName, 'utf8')
expect(regeneratedFileHtml).not.toBe(initialHtmlFile)
expect(await renderViaHTTP(appPort, route)).toBe(regeneratedFileHtml)
})
it(`[${route}] should regenerate /_next/data when revalidate time exceeded`, async () => {
const fileName = join(
appDir,
'.next',
'server',
getPageFileFromPagesManifest(appDir, routePath).replace('.js', '.json')
)
const route = join(`/_next/data/${buildId}`, `${routePath}.json`)
const initialFileJson = await fs.readFile(fileName, 'utf8')
await waitFor(1000) // Wait revalidate duration
expect(JSON.parse(await renderViaHTTP(appPort, route))).toEqual(
JSON.parse(initialFileJson)
)
await waitFor(500) // Wait for regeneration to occur
const regeneratedFileJson = await fs.readFile(fileName, 'utf8')
expect(regeneratedFileJson).not.toBe(initialFileJson)
expect(JSON.parse(await renderViaHTTP(appPort, route))).toEqual(
JSON.parse(regeneratedFileJson)
)
})
}
describe('SSG Prerender Revalidate', () => {
describe('production mode', () => {
beforeAll(async () => {
await fs.remove(join(appDir, '.next'))
await nextBuild(appDir, [])
appPort = await findPort()
app = await nextStart(appDir, appPort, {})
buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
})
afterAll(() => killApp(app))
runTests('/', '/')
runTests('/named', '/named')
runTests('/nested', '/nested')
runTests('/nested/named', '/nested/named')
})
// Regression test for https://github.com/vercel/next.js/issues/24806
describe('[regression] production mode and incremental cache size exceeded', () => {
beforeAll(async () => {
await fs.remove(join(appDir, '.next'))
await nextBuild(appDir, [])
appPort = await findPort()
app = await nextStart(appDir, appPort, {
// The lowest size of the LRU cache that can be set is "1"
// this will cause the cache size to always be exceeded
env: { __NEXT_TEST_MAX_ISR_CACHE: 1 },
})
buildId = await fs.readFile(join(appDir, '.next/BUILD_ID'), 'utf8')
})
afterAll(() => killApp(app))
runTests('/', '/')
runTests('/named', '/named')
runTests('/nested', '/nested')
runTests('/nested/named', '/nested/named')
})
})
......@@ -1646,6 +1646,7 @@ const runTests = (dev = false, isEmulatedServerless = false) => {
it('should handle revalidating HTML correctly', async () => {
const route = '/blog/post-2/comment-2'
await renderViaHTTP(appPort, route)
const initialHtml = await renderViaHTTP(appPort, route)
expect(initialHtml).toMatch(/Post:.*?post-2/)
expect(initialHtml).toMatch(/Comment:.*?comment-2/)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment