Unverified Commit 3da24ee7 authored by Hendrik Liebau's avatar Hendrik Liebau Committed by GitHub

[ci] Run flakiness detection under concurrent load (#100525)

The flakiness detection jobs ran three attempts sequentially with
concurrency set to one, while regular test jobs run up to eight suites
at once. The response-read flake addressed by #100514 passed all three
checks for #100447, which prompted this change.

The detector now queues three runs per suite in one runner and uses the
job's configured concurrency. Each run has an isolated application
directory and separate results, traces, and failure tracking. Retries
and result caching remain disabled so every run executes and any failure
fails the job.

The testing guide documents how to keep concurrent runs independent and
allocate listening ports dynamically.
parent 2d139451
...@@ -72,6 +72,8 @@ the new checks can be added to the existing test suite. ...@@ -72,6 +72,8 @@ the new checks can be added to the existing test suite.
### Best Practices ### Best Practices
- Keep each test run independent. Flakiness detection runs multiple copies of the same suite concurrently. Store mutable files in the isolated application directory or a unique temporary directory.
- Allocate listening ports dynamically. Use `server.listen(0)` for a local test server, read its assigned port from `server.address()`, and pass its URL to the fixture through `next.env` before `next.start()`. Use `findPort()` from `next-test-utils` when a server requires a port before it starts. Fixed listening ports can cause concurrent runs to fail with `EADDRINUSE`.
- When checking for a condition that might take time, - When checking for a condition that might take time,
ensure it is waited for either using the browser `waitForElement` or using the `check` util in `next-test-utils`. ensure it is waited for either using the browser `waitForElement` or using the `check` util in `next-test-utils`.
- When applying a fix, ensure the test fails without the fix. - When applying a fix, ensure the test fails without the fix.
......
This diff is collapsed.
...@@ -13,7 +13,7 @@ import { ...@@ -13,7 +13,7 @@ import {
* CLI Options: * CLI Options:
* --mode: test mode (dev, deploy, start) * --mode: test mode (dev, deploy, start)
* --group: current group number / total groups * --group: current group number / total groups
* --flake-detection: run tests multiple times to detect flaky * --flake-detection: run each suite three times with shared concurrency
*/ */
async function main() { async function main() {
const argv = await yargs(process.argv.slice(2)) const argv = await yargs(process.argv.slice(2))
...@@ -23,7 +23,6 @@ async function main() { ...@@ -23,7 +23,6 @@ async function main() {
const testMode = argv.mode const testMode = argv.mode
const isFlakeDetectionMode = argv['flake-detection'] const isFlakeDetectionMode = argv['flake-detection']
const attempts = isFlakeDetectionMode ? 3 : 1
// Left unset when the env var is not set, so that // Left unset when the env var is not set, so that
// wait-for-preview-tarball.mjs owns the default. // wait-for-preview-tarball.mjs owns the default.
const previewBuildsBaseUrl = process.env.NEXT_TEST_PREVIEW_BUILDS_BASE_URL const previewBuildsBaseUrl = process.env.NEXT_TEST_PREVIEW_BUILDS_BASE_URL
...@@ -92,7 +91,7 @@ async function main() { ...@@ -92,7 +91,7 @@ async function main() {
return return
} }
const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0'] const RUN_TESTS_ARGS = ['run-tests.js', '--retries', '0']
// Only override the test version for deploy tests, as they need to run against // Only override the test version for deploy tests, as they need to run against
// the artifacts for the pull request. Otherwise, we don't need to specify this property, // the artifacts for the pull request. Otherwise, we don't need to specify this property,
// as tests will run against the local version of Next.js. // as tests will run against the local version of Next.js.
...@@ -131,49 +130,42 @@ async function main() { ...@@ -131,49 +130,42 @@ async function main() {
} }
if (isFlakeDetectionMode) { if (isFlakeDetectionMode) {
for (let i = 0; i < attempts; i++) { console.log(
console.log( `Running each ${testMode} suite three times with shared concurrency (Turbopack)`
`\n\nRun ${i + 1}/${attempts} for ${testMode} tests (Turbopack)` )
) await execa('node', [...RUN_TESTS_ARGS, '--runs', '3', ...currentTests], {
await execa('node', [...RUN_TESTS_ARGS, ...currentTests], { ...EXECA_OPTS_STDIO,
...EXECA_OPTS_STDIO, env: {
env: { ...process.env,
...process.env, NEXT_TEST_MODE: testMode,
NEXT_TEST_MODE: testMode, NEXT_TEST_VERSION: nextTestVersion,
NEXT_TEST_VERSION: nextTestVersion, NEXT_EXTERNAL_TESTS_FILTERS,
NEXT_EXTERNAL_TESTS_FILTERS, NEXT_FLAKE_DETECTION: '1',
NEXT_FLAKE_DETECTION: '1', IS_TURBOPACK_TEST: '1',
IS_TURBOPACK_TEST: '1', TURBOPACK_BUILD:
TURBOPACK_BUILD: testMode === 'start' || testMode === 'deploy' ? '1' : undefined,
testMode === 'start' || testMode === 'deploy' ? '1' : undefined, TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined,
TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined, },
}, })
})
}
} else { } else {
for (let i = 0; i < attempts; i++) { console.log(`Running ${testMode} tests`)
console.log(`\n\nRun ${i + 1}/${attempts} for ${testMode} tests`) await execa('node', [...RUN_TESTS_ARGS, '-c', '1', ...currentTests], {
...EXECA_OPTS_STDIO,
await execa('node', [...RUN_TESTS_ARGS, ...currentTests], { env: {
...EXECA_OPTS_STDIO, ...process.env,
env: { NEXT_EXTERNAL_TESTS_FILTERS,
...process.env, NEXT_TEST_MODE: testMode,
NEXT_EXTERNAL_TESTS_FILTERS, NEXT_TEST_VERSION: nextTestVersion,
NEXT_TEST_MODE: testMode, TURBOPACK_BUILD:
NEXT_TEST_VERSION: nextTestVersion, process.env.IS_TURBOPACK_TEST &&
TURBOPACK_BUILD: (testMode === 'start' || testMode === 'deploy')
process.env.IS_TURBOPACK_TEST && ? '1'
(testMode === 'start' || testMode === 'deploy') : undefined,
? '1' TURBOPACK_DEV:
: undefined, process.env.IS_TURBOPACK_TEST && testMode === 'dev' ? '1' : undefined,
TURBOPACK_DEV: NEXT_TEST_SKIP_RESULT_CACHE: '1',
process.env.IS_TURBOPACK_TEST && testMode === 'dev' },
? '1' })
: undefined,
NEXT_TEST_SKIP_RESULT_CACHE: '1',
},
})
}
} }
} }
......
...@@ -118,6 +118,9 @@ export class Playwright<TCurrent = undefined> { ...@@ -118,6 +118,9 @@ export class Playwright<TCurrent = undefined> {
`${path `${path
.relative(path.join(__dirname, '../../'), process.env.TEST_FILE_PATH!) .relative(path.join(__dirname, '../../'), process.env.TEST_FILE_PATH!)
.replace(/\//g, '-')}`, .replace(/\//g, '-')}`,
...(process.env.NEXT_TEST_RUN_NUMBER
? [`run-${process.env.NEXT_TEST_RUN_NUMBER}`]
: []),
`playwright-${this.activeTrace}-${Date.now()}.zip` `playwright-${this.activeTrace}-${Date.now()}.zip`
) )
......
...@@ -953,6 +953,9 @@ export class NextInstance { ...@@ -953,6 +953,9 @@ export class NextInstance {
process.env.TEST_FILE_PATH! process.env.TEST_FILE_PATH!
) )
.replace(/\//g, '-')}`, .replace(/\//g, '-')}`,
...(process.env.NEXT_TEST_RUN_NUMBER
? [`run-${process.env.NEXT_TEST_RUN_NUMBER}`]
: []),
`next-trace` `next-trace`
), ),
{ recursive: true } { recursive: true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment