Unverified Commit 3da24ee7 authored by Hendrik Liebau's avatar Hendrik Liebau Committed by GitHub

[ci] Run flakiness detection under concurrent load (#100525)

The flakiness detection jobs ran three attempts sequentially with
concurrency set to one, while regular test jobs run up to eight suites
at once. The response-read flake addressed by #100514 passed all three
checks for #100447, which prompted this change.

The detector now queues three runs per suite in one runner and uses the
job's configured concurrency. Each run has an isolated application
directory and separate results, traces, and failure tracking. Retries
and result caching remain disabled so every run executes and any failure
fails the job.

The testing guide documents how to keep concurrent runs independent and
allocate listening ports dynamically.
parent 2d139451
......@@ -72,6 +72,8 @@ the new checks can be added to the existing test suite.
### Best Practices
- Keep each test run independent. Flakiness detection runs multiple copies of the same suite concurrently. Store mutable files in the isolated application directory or a unique temporary directory.
- Allocate listening ports dynamically. Use `server.listen(0)` for a local test server, read its assigned port from `server.address()`, and pass its URL to the fixture through `next.env` before `next.start()`. Use `findPort()` from `next-test-utils` when a server requires a port before it starts. Fixed listening ports can cause concurrent runs to fail with `EADDRINUSE`.
- When checking for a condition that might take time,
ensure it is waited for either using the browser `waitForElement` or using the `check` util in `next-test-utils`.
- When applying a fix, ensure the test fails without the fix.
......
This diff is collapsed.
......@@ -13,7 +13,7 @@ import {
* CLI Options:
* --mode: test mode (dev, deploy, start)
* --group: current group number / total groups
* --flake-detection: run tests multiple times to detect flaky
* --flake-detection: run each suite three times with shared concurrency
*/
async function main() {
const argv = await yargs(process.argv.slice(2))
......@@ -23,7 +23,6 @@ async function main() {
const testMode = argv.mode
const isFlakeDetectionMode = argv['flake-detection']
const attempts = isFlakeDetectionMode ? 3 : 1
// Left unset when the env var is not set, so that
// wait-for-preview-tarball.mjs owns the default.
const previewBuildsBaseUrl = process.env.NEXT_TEST_PREVIEW_BUILDS_BASE_URL
......@@ -92,7 +91,7 @@ async function main() {
return
}
const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0']
const RUN_TESTS_ARGS = ['run-tests.js', '--retries', '0']
// Only override the test version for deploy tests, as they need to run against
// the artifacts for the pull request. Otherwise, we don't need to specify this property,
// as tests will run against the local version of Next.js.
......@@ -131,49 +130,42 @@ async function main() {
}
if (isFlakeDetectionMode) {
for (let i = 0; i < attempts; i++) {
console.log(
`\n\nRun ${i + 1}/${attempts} for ${testMode} tests (Turbopack)`
)
await execa('node', [...RUN_TESTS_ARGS, ...currentTests], {
...EXECA_OPTS_STDIO,
env: {
...process.env,
NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion,
NEXT_EXTERNAL_TESTS_FILTERS,
NEXT_FLAKE_DETECTION: '1',
IS_TURBOPACK_TEST: '1',
TURBOPACK_BUILD:
testMode === 'start' || testMode === 'deploy' ? '1' : undefined,
TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined,
},
})
}
console.log(
`Running each ${testMode} suite three times with shared concurrency (Turbopack)`
)
await execa('node', [...RUN_TESTS_ARGS, '--runs', '3', ...currentTests], {
...EXECA_OPTS_STDIO,
env: {
...process.env,
NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion,
NEXT_EXTERNAL_TESTS_FILTERS,
NEXT_FLAKE_DETECTION: '1',
IS_TURBOPACK_TEST: '1',
TURBOPACK_BUILD:
testMode === 'start' || testMode === 'deploy' ? '1' : undefined,
TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined,
},
})
} else {
for (let i = 0; i < attempts; i++) {
console.log(`\n\nRun ${i + 1}/${attempts} for ${testMode} tests`)
await execa('node', [...RUN_TESTS_ARGS, ...currentTests], {
...EXECA_OPTS_STDIO,
env: {
...process.env,
NEXT_EXTERNAL_TESTS_FILTERS,
NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion,
TURBOPACK_BUILD:
process.env.IS_TURBOPACK_TEST &&
(testMode === 'start' || testMode === 'deploy')
? '1'
: undefined,
TURBOPACK_DEV:
process.env.IS_TURBOPACK_TEST && testMode === 'dev'
? '1'
: undefined,
NEXT_TEST_SKIP_RESULT_CACHE: '1',
},
})
}
console.log(`Running ${testMode} tests`)
await execa('node', [...RUN_TESTS_ARGS, '-c', '1', ...currentTests], {
...EXECA_OPTS_STDIO,
env: {
...process.env,
NEXT_EXTERNAL_TESTS_FILTERS,
NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion,
TURBOPACK_BUILD:
process.env.IS_TURBOPACK_TEST &&
(testMode === 'start' || testMode === 'deploy')
? '1'
: undefined,
TURBOPACK_DEV:
process.env.IS_TURBOPACK_TEST && testMode === 'dev' ? '1' : undefined,
NEXT_TEST_SKIP_RESULT_CACHE: '1',
},
})
}
}
......
......@@ -118,6 +118,9 @@ export class Playwright<TCurrent = undefined> {
`${path
.relative(path.join(__dirname, '../../'), process.env.TEST_FILE_PATH!)
.replace(/\//g, '-')}`,
...(process.env.NEXT_TEST_RUN_NUMBER
? [`run-${process.env.NEXT_TEST_RUN_NUMBER}`]
: []),
`playwright-${this.activeTrace}-${Date.now()}.zip`
)
......
......@@ -953,6 +953,9 @@ export class NextInstance {
process.env.TEST_FILE_PATH!
)
.replace(/\//g, '-')}`,
...(process.env.NEXT_TEST_RUN_NUMBER
? [`run-${process.env.NEXT_TEST_RUN_NUMBER}`]
: []),
`next-trace`
),
{ recursive: true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment