Unverified Commit f8e47d49 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

test: scope e2e timeouts to startup and per-it (#89929)

This aims to avoid us having tests stall for upwards of 240s when a
single assertion in the test stalls. This is very common currently. To
also avoid wasting the entire time of resetting the test suite this also
allows 1 retry for individual failed assertions. This won't always work
since some assertions rely on previous setup so this also keeps the
entire test suite retry handling as a fallback.

This change is not applied for dev tests as they check HMR and compile
lazily.

## Summary
- keep elevated Jest timeout for test setup/startup in e2e-utils
- enforce a 60s default timeout for individual `it`/`test` cases
- apply one retry per individual test via `jest.retryTimes(1)`

## Verification
- pnpm testonly test/development/gssp-notfound/index.test.ts
- pnpm testonly test/development/enoent-during-require/index.test.ts
- pnpm testonly test/e2e/custom-app-render/custom-app-render.test.ts

Tested against our deploy tests to see if this helps:

Current run
[21967823508](https://github.com/vercel/next.js/actions/runs/21967823508)

Jobs: 24 (with test-runner output: 20)
Whole-suite retry starts: 11 (retry 1: 8, retry 2+: 3)
Jobs that needed suite retries: 8
Unique test files retried: 7

Previous run
[21964085768](https://github.com/vercel/next.js/actions/runs/21964085768)

Jobs: 26 (with test-runner output: 20)
Whole-suite retry starts: 18 (retry 1: 13, retry 2+: 5)
Jobs that needed suite retries: 9
Unique test files retried: 12
parent 5aa4568b
...@@ -10,21 +10,57 @@ import { shouldUseTurbopack } from '../next-test-utils' ...@@ -10,21 +10,57 @@ import { shouldUseTurbopack } from '../next-test-utils'
export type { NextInstance } export type { NextInstance }
// increase timeout to account for pnpm install time const individualTestTimeout = 60 * 1000
// if either test runs for the --turbo or have a custom timeout, set reduced timeout instead.
// this is due to current --turbo test have a lot of tests fails with timeouts, ends up the whole // Keep a higher timeout for setup hooks (e.g. initial createNext/startup),
// test job exceeds the 6 hours limit. // but enforce 60s per test case via wrapped `it`/`test` for non-dev modes.
let testTimeout = (process.platform === 'win32' ? 240 : 120) * 1000 let setupTimeout = (process.platform === 'win32' ? 240 : 120) * 1000
if (process.env.NEXT_E2E_TEST_TIMEOUT) { if (process.env.NEXT_E2E_TEST_TIMEOUT) {
try { const parsedTimeout = Number.parseInt(process.env.NEXT_E2E_TEST_TIMEOUT, 10)
testTimeout = parseInt(process.env.NEXT_E2E_TEST_TIMEOUT, 10) if (!Number.isNaN(parsedTimeout)) {
} catch (_) { setupTimeout = parsedTimeout
// ignore
} }
} }
jest.setTimeout(testTimeout) jest.setTimeout(setupTimeout)
type E2ETestGlobal = typeof globalThis & {
__NEXT_E2E_TEST_CONFIG_PATCHED__?: boolean
__NEXT_E2E_WRAPPED_TEST_FNS__?: WeakMap<Function, Function>
}
const wrapJestTestFn = <T extends Function>(fn: T): T => {
const e2eGlobal = global as E2ETestGlobal
const wrappedFns =
e2eGlobal.__NEXT_E2E_WRAPPED_TEST_FNS__ ??
(e2eGlobal.__NEXT_E2E_WRAPPED_TEST_FNS__ = new WeakMap())
const existing = wrappedFns.get(fn)
if (existing) return existing as T
const wrapped = new Proxy(fn, {
apply(target, thisArg, argArray: unknown[]) {
const args = [...argArray]
if (
args.length >= 2 &&
typeof args[1] === 'function' &&
args[2] === undefined
) {
args[2] = individualTestTimeout
}
const result = Reflect.apply(target, thisArg, args)
return typeof result === 'function' ? wrapJestTestFn(result) : result
},
get(target, prop, receiver) {
const value = Reflect.get(target, prop, receiver)
return typeof value === 'function' ? wrapJestTestFn(value) : value
},
})
wrappedFns.set(fn, wrapped)
return wrapped as T
}
const testsFolder = path.join(__dirname, '..', '..') const testsFolder = path.join(__dirname, '..', '..')
...@@ -80,6 +116,22 @@ if (testModeFromFile === 'e2e') { ...@@ -80,6 +116,22 @@ if (testModeFromFile === 'e2e') {
testMode = 'start' testMode = 'start'
} }
const e2eGlobal = global as E2ETestGlobal
if (!e2eGlobal.__NEXT_E2E_TEST_CONFIG_PATCHED__) {
if (testMode !== 'dev') {
if (typeof global.it === 'function') {
global.it = wrapJestTestFn(global.it) as jest.It
}
if (typeof global.test === 'function') {
global.test = wrapJestTestFn(global.test) as jest.It
}
jest.retryTimes(1)
}
e2eGlobal.__NEXT_E2E_TEST_CONFIG_PATCHED__ = true
}
if (testMode === 'dev') { if (testMode === 'dev') {
;(global as any).isNextDev = true ;(global as any).isNextDev = true
} else if (testMode === 'deploy') { } else if (testMode === 'deploy') {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment