Unverified Commit 2c23da07 authored by Jimmy Lai's avatar Jimmy Lai Committed by GitHub

misc: fix benchmark script (#44592)

Fixing + doing some maintenance as I was using it for some things.

Changes:
- fixed a bug related to how React is injected
- added a progress tracker for the numbers of requests
- retry on 500
- added a param for crashing the lambda manually (useful for cold boots)
- added a param to skip build, useful if you want to retest the same variation
- don't crash when there's no data to compare

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [ ] [e2e](https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs) tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm build && pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)
parent 1dc8da96
...@@ -4,16 +4,24 @@ import console from 'console' ...@@ -4,16 +4,24 @@ import console from 'console'
import chalk from 'chalk' import chalk from 'chalk'
import PQueue from 'p-queue' import PQueue from 'p-queue'
import { generateProjects, cleanupProjectFolders } from './project-utils.js' import {
generateProjects,
cleanupProjectFolders,
TEST_PROJECT_NAME,
} from './project-utils.js'
import { printBenchmarkResults } from './chart.js' import { printBenchmarkResults } from './chart.js'
import { genRetryableRequest } from './gen-request.js' import { genRetryableRequest } from './gen-request.js'
const program = new Command() const program = new Command()
const queue = new PQueue({ concurrency: 25 }) const queue = new PQueue({ concurrency: 50 })
const TTFB_OUTLIERS_THRESHOLD = 250 const TTFB_OUTLIERS_THRESHOLD = 1500
let progress = 0
program.option('-p, --path <path>') program.option('-p, --path <path>')
program.option('-s, --skip-build', 'Skip build step')
program.option('-f, --force-crash', 'Force function crash')
program.parse(process.argv) program.parse(process.argv)
...@@ -23,8 +31,23 @@ if (options.path) { ...@@ -23,8 +31,23 @@ if (options.path) {
console.log('Running benchmark for path: ', options.path) console.log('Running benchmark for path: ', options.path)
} }
if (options.skipBuild) {
console.log('Skipping build step')
}
if (options.forceCrash) {
console.log('Forcing function crash')
}
export const forceCrash = options.forceCrash
try { try {
const [originDeploymentURL, headDeploymentURL] = await generateProjects() const [originDeploymentURL, headDeploymentURL] = options.skipBuild
? [
`https://${TEST_PROJECT_NAME}-origin.vercel.app`,
`https://${TEST_PROJECT_NAME}-head.vercel.app`,
]
: await generateProjects()
const originBenchmarkURL = `${originDeploymentURL}${options.path || ''}` const originBenchmarkURL = `${originDeploymentURL}${options.path || ''}`
const headBenchmarkURL = `${headDeploymentURL}${options.path || ''}` const headBenchmarkURL = `${headDeploymentURL}${options.path || ''}`
...@@ -60,11 +83,22 @@ try { ...@@ -60,11 +83,22 @@ try {
} }
async function runBenchmark(url) { async function runBenchmark(url) {
progress = 0
process.stdout.write(`Sending requests to ${url} ...\n`)
process.stdout.write(`Progress: ${++progress}/500`)
return ( return (
await Promise.all( await Promise.all(
Array.from({ length: 500 }).map(() => Array.from({ length: 500 }).map(async () => {
queue.add(() => genRetryableRequest(url)) const p = await queue.add(() => genRetryableRequest(url))
) refreshProgress()
return p
})
) )
).filter(Boolean) ).filter(Boolean)
} }
function refreshProgress() {
process.stdout.clearLine()
process.stdout.cursorTo(0)
process.stdout.write(`Requests sent: ${++progress}/500`)
}
...@@ -8,7 +8,3 @@ export default function Root({ children }) { ...@@ -8,7 +8,3 @@ export default function Root({ children }) {
</html> </html>
) )
} }
export const config = {
runtime: 'experimental-edge',
}
import * as React from 'react' import * as React from 'react'
import { cookies } from 'next/headers'
// if (!('hot' in Math)) Math.hot = false if (!('hot' in Math)) Math.hot = false
export default function page() { export default function page() {
// const previous = Math.hot // make the page dynamic
// Math.hot = true cookies()
// return <div>{previous ? 'HOT' : 'COLD'}</div> const previous = Math.hot
return <div>hello</div> Math.hot = true
}
// crash the server after responding
if (process.env.CRASH_FUNCTION) {
setTimeout(() => {
throw new Error('crash')
}, 500)
}
export const config = { return <div>{previous ? 'HOT' : 'COLD'}</div>
runtime: 'experimental-edge',
} }
...@@ -8,13 +8,16 @@ export async function getServerSideProps() { ...@@ -8,13 +8,16 @@ export async function getServerSideProps() {
const wasHot = Math.hot const wasHot = Math.hot
Math.hot = true Math.hot = true
// crash the server after responding
if (process.env.CRASH_FUNCTION) {
setTimeout(() => {
throw new Error('crash')
}, 700)
}
return { return {
props: { props: {
hot: wasHot, hot: wasHot,
}, },
} }
} }
export const config = {
runtime: 'experimental-edge',
}
...@@ -56,6 +56,11 @@ export function printBenchmarkResults({ origin, head }, metricSelector) { ...@@ -56,6 +56,11 @@ export function printBenchmarkResults({ origin, head }, metricSelector) {
(results) => results.map(metricSelector).filter(Boolean) (results) => results.map(metricSelector).filter(Boolean)
) )
if (processedHeadData.length === 0 || processedOriginData.length === 0) {
console.log('No data to compare, skipping')
return
}
const [originMetrics, headMetrics] = [ const [originMetrics, headMetrics] = [
processedOriginData, processedOriginData,
processedHeadData, processedHeadData,
......
...@@ -4,12 +4,11 @@ import timer from '@szmarczak/http-timer' ...@@ -4,12 +4,11 @@ import timer from '@szmarczak/http-timer'
// a wrapper around genAsyncRequest that will retry the request 5 times if it fails // a wrapper around genAsyncRequest that will retry the request 5 times if it fails
export async function genRetryableRequest(url) { export async function genRetryableRequest(url) {
let retries = 0 let retries = 0
while (retries < 5) { while (retries < 10) {
try { try {
return await genAsyncRequest(url) return await genAsyncRequest(url)
} catch (err) {} } catch (err) {}
retries++ retries++
await new Promise((r) => setTimeout(r, 1000))
} }
throw new Error(`Failed to fetch ${url}, too many retries`) throw new Error(`Failed to fetch ${url}, too many retries`)
} }
...@@ -25,6 +24,9 @@ async function genAsyncRequest(url) { ...@@ -25,6 +24,9 @@ async function genAsyncRequest(url) {
body += data body += data
}) })
response.on('end', () => { response.on('end', () => {
if (response.statusCode !== 200) {
reject(new Error(`Failed to fetch ${url}`))
}
resolve({ resolve({
...response.timings.phases, ...response.timings.phases,
cold: !body.includes('HOT'), cold: !body.includes('HOT'),
......
...@@ -46,7 +46,7 @@ async function getCurrentRootReactPackagesVersions() { ...@@ -46,7 +46,7 @@ async function getCurrentRootReactPackagesVersions() {
await fs.readFile('../../package.json', 'utf8') await fs.readFile('../../package.json', 'utf8')
) )
return { return {
react: packageJson.devDependencies['react-exp'], react: packageJson.devDependencies['react'],
'react-dom': packageJson.devDependencies['react-dom-exp'], 'react-dom': packageJson.devDependencies['react-dom'],
} }
} }
...@@ -7,10 +7,11 @@ import path from 'path' ...@@ -7,10 +7,11 @@ import path from 'path'
import url from 'url' import url from 'url'
import { generatePackageJson } from './generate-package-json.js' import { generatePackageJson } from './generate-package-json.js'
import { Listr } from 'listr2' import { Listr } from 'listr2'
import { forceCrash } from './bench.js'
config() config()
const TEST_PROJECT_NAME = process.env.VERCEL_TEST_PROJECT_NAME export const TEST_PROJECT_NAME = process.env.VERCEL_TEST_PROJECT_NAME
const ORIGIN_PROJECT_NAME = TEST_PROJECT_NAME + '-origin' const ORIGIN_PROJECT_NAME = TEST_PROJECT_NAME + '-origin'
const HEAD_PROJECT_NAME = TEST_PROJECT_NAME + '-head' const HEAD_PROJECT_NAME = TEST_PROJECT_NAME + '-head'
...@@ -197,6 +198,7 @@ export async function deployProject(projectName, appFolder) { ...@@ -197,6 +198,7 @@ export async function deployProject(projectName, appFolder) {
: []), : []),
'--force', '--force',
...vercelFlags, ...vercelFlags,
...(forceCrash ? ['--env', 'CRASH_FUNCTION=1'] : []),
], ],
{ {
cwd: appFolder, cwd: appFolder,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment