Unverified Commit efd6b682 authored by Justin Ridgewell's avatar Justin Ridgewell Committed by GitHub

turbopack: only skip known flakey test cases (#55976)

### What?

A follow up to https://github.com/vercel/next.js/pull/55975, this will
list every flakey test in the turbopack suite so that we can run the
remaining passing tests.

### Why?

More tests is more better.

### How?

Trial and error. Gonna let CI tell me what's flakey with a few runs.

Closes WEB-1651
parent 2554fa5f
...@@ -240,7 +240,7 @@ async function main() { ...@@ -240,7 +240,7 @@ async function main() {
const info = externalTestsFilterLists[test.file] const info = externalTestsFilterLists[test.file]
// only run filtered mode when there are failing tests. // only run filtered mode when there are failing tests.
// When the whole test suite passes we can run all tests, including newly added ones. // When the whole test suite passes we can run all tests, including newly added ones.
if (info.failed.length > 0) { if (info.failed.length > 0 || info.flakey.length > 0) {
test.cases = info.passed test.cases = info.passed
} }
return test return test
...@@ -443,7 +443,7 @@ ${ENDGROUP}`) ...@@ -443,7 +443,7 @@ ${ENDGROUP}`)
...Object.entries(env).map((e) => `${e[0]}=${e[1]}`), ...Object.entries(env).map((e) => `${e[0]}=${e[1]}`),
jestPath, jestPath,
...args.map((a) => `'${a}'`), ...args.map((a) => `'${a}'`),
].join(' ') ].join(' ') + '\n'
) )
const child = spawn(jestPath, args, { const child = spawn(jestPath, args, {
......
...@@ -20,17 +20,23 @@ const INITIALIZING_TEST_CASES = [ ...@@ -20,17 +20,23 @@ const INITIALIZING_TEST_CASES = [
'should build successfully', 'should build successfully',
] ]
const SKIPPED_TEST_SUITES = new Set([ const SKIPPED_TEST_SUITES = {
'test/integration/router-rerender/test/index.test.js', 'test/integration/router-rerender/test/index.test.js': [],
'test/e2e/basepath.test.ts', 'test/e2e/basepath.test.ts': [],
'test/development/acceptance-app/ReactRefreshRequire.test.ts', 'test/development/acceptance-app/ReactRefreshRequire.test.ts': [],
'test/integration/dynamic-routing/test/middleware.test.js', 'test/integration/dynamic-routing/test/middleware.test.js': [],
'test/integration/css/test/css-modules.test.js', 'test/integration/css/test/css-modules.test.js': [],
'test/development/acceptance/ReactRefreshRequire.test.ts', 'test/development/acceptance/ReactRefreshRequire.test.ts': [],
'test/integration/custom-routes/test/index.test.js', 'test/integration/custom-routes/test/index.test.js': [],
'test/integration/absolute-assetprefix/test/index.test.js', 'test/integration/absolute-assetprefix/test/index.test.js': [],
'test/e2e/middleware-rewrites/test/index.test.ts', 'test/e2e/middleware-rewrites/test/index.test.ts': [],
]) 'test/integration/dynamic-routing/test/index.test.js': [
'Dynamic Routing production mode should have correct cache entries on prefetch',
],
'test/development/acceptance-app/ReactRefreshLogBox-builtins.test.ts': [
'ReactRefreshLogBox app turbo Module not found missing global CSS',
],
}
async function updatePassingTests() { async function updatePassingTests() {
const passing = { __proto__: null } const passing = { __proto__: null }
...@@ -42,17 +48,19 @@ async function updatePassingTests() { ...@@ -42,17 +48,19 @@ async function updatePassingTests() {
for (const testResult of result.data.testResults) { for (const testResult of result.data.testResults) {
const filepath = stripWorkingPath(testResult.name) const filepath = stripWorkingPath(testResult.name)
if (SKIPPED_TEST_SUITES.has(filepath)) continue
const fileResults = (passing[filepath] ??= { const fileResults = (passing[filepath] ??= {
passed: [], passed: [],
failed: [], failed: [],
pending: [], pending: [],
flakey: [],
runtimeError, runtimeError,
}) })
const skips = SKIPPED_TEST_SUITES[filepath] ?? []
let initializationFailed = false let initializationFailed = false
for (const testCase of testResult.assertionResults) { for (const testCase of testResult.assertionResults) {
let { fullName, status } = testCase let { fullName, status } = testCase
if ( if (
status === 'failed' && status === 'failed' &&
INITIALIZING_TEST_CASES.some((name) => fullName.includes(name)) INITIALIZING_TEST_CASES.some((name) => fullName.includes(name))
...@@ -61,6 +69,10 @@ async function updatePassingTests() { ...@@ -61,6 +69,10 @@ async function updatePassingTests() {
} else if (initializationFailed) { } else if (initializationFailed) {
status = 'failed' status = 'failed'
} }
if (shouldSkip(fullName, skips)) {
status = 'flakey'
}
const statusArray = fileResults[status] const statusArray = fileResults[status]
if (!statusArray) { if (!statusArray) {
throw new Error(`unexpected status "${status}"`) throw new Error(`unexpected status "${status}"`)
...@@ -87,11 +99,9 @@ async function updatePassingTests() { ...@@ -87,11 +99,9 @@ async function updatePassingTests() {
const newData = passing[file] const newData = passing[file]
const oldData = oldPassingData[file] const oldData = oldPassingData[file]
if (!newData) continue if (!newData) continue
// We only want to keep test cases from the old data that are still exiting
oldData.passed = oldData.passed.filter( // We want to find old passing tests that are now failing, and report them.
(name) => newData.failed.includes(name) || newData.passed.includes(name) // Tests are allowed transition to skipped or flakey.
)
// Grab test cases that passed before, but fail now
const shouldPass = new Set( const shouldPass = new Set(
oldData.passed.filter((name) => newData.failed.includes(name)) oldData.passed.filter((name) => newData.failed.includes(name))
) )
...@@ -102,9 +112,7 @@ async function updatePassingTests() { ...@@ -102,9 +112,7 @@ async function updatePassingTests() {
) )
} }
// Merge the old passing tests with the new ones // Merge the old passing tests with the new ones
newData.passed = [ newData.passed = [...new Set([...shouldPass, ...newData.passed])].sort()
...new Set([...oldData.passed, ...newData.passed]),
].sort()
// but remove them also from the failed list // but remove them also from the failed list
newData.failed = newData.failed newData.failed = newData.failed
.filter((name) => !shouldPass.has(name)) .filter((name) => !shouldPass.has(name))
...@@ -127,6 +135,19 @@ async function updatePassingTests() { ...@@ -127,6 +135,19 @@ async function updatePassingTests() {
) )
} }
function shouldSkip(name, skips) {
for (const skip of skips) {
if (typeof skip === 'string') {
// exact match
if (name === skip) return true
} else {
// regex
if (skip.test(name)) return true
}
}
return false
}
function stripWorkingPath(path) { function stripWorkingPath(path) {
if (!path.startsWith(WORKING_PATH)) { if (!path.startsWith(WORKING_PATH)) {
throw new Error( throw new Error(
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment