Unverified Commit 2c23da07 authored by Jimmy Lai's avatar Jimmy Lai Committed by GitHub

misc: fix benchmark script (#44592)

Fixing + doing some maintenance as I was using it for some things.

Changes:
- fixed a bug related to how React is injected
- added a progress tracker for the numbers of requests
- retry on 500
- added a param for crashing the lambda manually (useful for cold boots)
- added a param to skip build, useful if you want to retest the same variation
- don't crash when there's no data to compare

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [ ] [e2e](https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs) tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm build && pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)
parent 1dc8da96
......@@ -4,16 +4,24 @@ import console from 'console'
import chalk from 'chalk'
import PQueue from 'p-queue'
import { generateProjects, cleanupProjectFolders } from './project-utils.js'
import {
generateProjects,
cleanupProjectFolders,
TEST_PROJECT_NAME,
} from './project-utils.js'
import { printBenchmarkResults } from './chart.js'
import { genRetryableRequest } from './gen-request.js'
const program = new Command()
const queue = new PQueue({ concurrency: 25 })
const TTFB_OUTLIERS_THRESHOLD = 250
const queue = new PQueue({ concurrency: 50 })
const TTFB_OUTLIERS_THRESHOLD = 1500
let progress = 0
program.option('-p, --path <path>')
program.option('-s, --skip-build', 'Skip build step')
program.option('-f, --force-crash', 'Force function crash')
program.parse(process.argv)
......@@ -23,8 +31,23 @@ if (options.path) {
console.log('Running benchmark for path: ', options.path)
}
if (options.skipBuild) {
console.log('Skipping build step')
}
if (options.forceCrash) {
console.log('Forcing function crash')
}
export const forceCrash = options.forceCrash
try {
const [originDeploymentURL, headDeploymentURL] = await generateProjects()
const [originDeploymentURL, headDeploymentURL] = options.skipBuild
? [
`https://${TEST_PROJECT_NAME}-origin.vercel.app`,
`https://${TEST_PROJECT_NAME}-head.vercel.app`,
]
: await generateProjects()
const originBenchmarkURL = `${originDeploymentURL}${options.path || ''}`
const headBenchmarkURL = `${headDeploymentURL}${options.path || ''}`
......@@ -60,11 +83,22 @@ try {
}
async function runBenchmark(url) {
progress = 0
process.stdout.write(`Sending requests to ${url} ...\n`)
process.stdout.write(`Progress: ${++progress}/500`)
return (
await Promise.all(
Array.from({ length: 500 }).map(() =>
queue.add(() => genRetryableRequest(url))
)
Array.from({ length: 500 }).map(async () => {
const p = await queue.add(() => genRetryableRequest(url))
refreshProgress()
return p
})
)
).filter(Boolean)
}
function refreshProgress() {
process.stdout.clearLine()
process.stdout.cursorTo(0)
process.stdout.write(`Requests sent: ${++progress}/500`)
}
......@@ -8,7 +8,3 @@ export default function Root({ children }) {
</html>
)
}
export const config = {
runtime: 'experimental-edge',
}
import * as React from 'react'
import { cookies } from 'next/headers'
// if (!('hot' in Math)) Math.hot = false
if (!('hot' in Math)) Math.hot = false
export default function page() {
// const previous = Math.hot
// Math.hot = true
// return <div>{previous ? 'HOT' : 'COLD'}</div>
return <div>hello</div>
}
// make the page dynamic
cookies()
const previous = Math.hot
Math.hot = true
// crash the server after responding
if (process.env.CRASH_FUNCTION) {
setTimeout(() => {
throw new Error('crash')
}, 500)
}
export const config = {
runtime: 'experimental-edge',
return <div>{previous ? 'HOT' : 'COLD'}</div>
}
......@@ -8,13 +8,16 @@ export async function getServerSideProps() {
const wasHot = Math.hot
Math.hot = true
// crash the server after responding
if (process.env.CRASH_FUNCTION) {
setTimeout(() => {
throw new Error('crash')
}, 700)
}
return {
props: {
hot: wasHot,
},
}
}
export const config = {
runtime: 'experimental-edge',
}
......@@ -56,6 +56,11 @@ export function printBenchmarkResults({ origin, head }, metricSelector) {
(results) => results.map(metricSelector).filter(Boolean)
)
if (processedHeadData.length === 0 || processedOriginData.length === 0) {
console.log('No data to compare, skipping')
return
}
const [originMetrics, headMetrics] = [
processedOriginData,
processedHeadData,
......
......@@ -4,12 +4,11 @@ import timer from '@szmarczak/http-timer'
// a wrapper around genAsyncRequest that will retry the request 5 times if it fails
export async function genRetryableRequest(url) {
let retries = 0
while (retries < 5) {
while (retries < 10) {
try {
return await genAsyncRequest(url)
} catch (err) {}
retries++
await new Promise((r) => setTimeout(r, 1000))
}
throw new Error(`Failed to fetch ${url}, too many retries`)
}
......@@ -25,6 +24,9 @@ async function genAsyncRequest(url) {
body += data
})
response.on('end', () => {
if (response.statusCode !== 200) {
reject(new Error(`Failed to fetch ${url}`))
}
resolve({
...response.timings.phases,
cold: !body.includes('HOT'),
......
......@@ -46,7 +46,7 @@ async function getCurrentRootReactPackagesVersions() {
await fs.readFile('../../package.json', 'utf8')
)
return {
react: packageJson.devDependencies['react-exp'],
'react-dom': packageJson.devDependencies['react-dom-exp'],
react: packageJson.devDependencies['react'],
'react-dom': packageJson.devDependencies['react-dom'],
}
}
......@@ -7,10 +7,11 @@ import path from 'path'
import url from 'url'
import { generatePackageJson } from './generate-package-json.js'
import { Listr } from 'listr2'
import { forceCrash } from './bench.js'
config()
const TEST_PROJECT_NAME = process.env.VERCEL_TEST_PROJECT_NAME
export const TEST_PROJECT_NAME = process.env.VERCEL_TEST_PROJECT_NAME
const ORIGIN_PROJECT_NAME = TEST_PROJECT_NAME + '-origin'
const HEAD_PROJECT_NAME = TEST_PROJECT_NAME + '-head'
......@@ -197,6 +198,7 @@ export async function deployProject(projectName, appFolder) {
: []),
'--force',
...vercelFlags,
...(forceCrash ? ['--env', 'CRASH_FUNCTION=1'] : []),
],
{
cwd: appFolder,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment