Unverified Commit e2554bc5 authored by Sebastian "Sebbie" Silbermann's avatar Sebastian "Sebbie" Silbermann Committed by GitHub

[test] Let pending test finish on abort (#86307)

parent 0469aec4
...@@ -501,10 +501,11 @@ ${ENDGROUP}`) ...@@ -501,10 +501,11 @@ ${ENDGROUP}`)
`jest${process.platform === 'win32' ? '.CMD' : ''}` `jest${process.platform === 'win32' ? '.CMD' : ''}`
) )
let firstError = true let firstError = true
let killed = false const testController = new AbortController()
const testSignal = testController.signal
let hadFailures = false let hadFailures = false
const runTest = (/** @type {TestFile} */ test, isFinalRun, isRetry) => const runTestOnce = (/** @type {TestFile} */ test, isFinalRun, isRetry) =>
new Promise((resolve, reject) => { new Promise((resolve, reject) => {
const start = new Date().getTime() const start = new Date().getTime()
let outputChunks = [] let outputChunks = []
...@@ -618,11 +619,11 @@ ${ENDGROUP}`) ...@@ -618,11 +619,11 @@ ${ENDGROUP}`)
if (hideOutput) { if (hideOutput) {
await outputSema.acquire() await outputSema.acquire()
const isExpanded = const isExpanded =
firstError && !killed && !shouldContinueTestsOnError firstError && !testSignal.aborted && !shouldContinueTestsOnError
if (isExpanded) { if (isExpanded) {
firstError = false firstError = false
process.stdout.write(`❌ ${test.file} output:\n`) process.stdout.write(`❌ ${test.file} output:\n`)
} else if (killed) { } else if (testSignal.aborted) {
process.stdout.write(`${GROUP}${test.file} output (killed)\n`) process.stdout.write(`${GROUP}${test.file} output (killed)\n`)
} else { } else {
process.stdout.write(`${GROUP}❌ ${test.file} output\n`) process.stdout.write(`${GROUP}❌ ${test.file} output\n`)
...@@ -636,7 +637,7 @@ ${ENDGROUP}`) ...@@ -636,7 +637,7 @@ ${ENDGROUP}`)
output += chunk.toString() output += chunk.toString()
} }
if (process.env.CI && !killed) { if (process.env.CI && !testSignal.aborted) {
errorsPerTests.set(test.file, output) errorsPerTests.set(test.file, output)
} }
...@@ -682,10 +683,105 @@ ${ENDGROUP}`) ...@@ -682,10 +683,105 @@ ${ENDGROUP}`)
}) })
}) })
const runTest = async (/** @type {TestFile} */ test) => {
let passed = false
const shouldSkipRetries = skipRetryTestManifest.find((t) =>
t.includes(test.file)
)
const numRetries = shouldSkipRetries ? 0 : originalRetries
if (shouldSkipRetries) {
console.log(
`Skipping retry for ${test.file} due to skipRetryTestManifest`
)
}
for (let i = 0; i < numRetries + 1; i++) {
try {
console.log(`Starting ${test.file} retry ${i}/${numRetries}`)
const time = await runTestOnce(
test,
shouldSkipRetries || i === numRetries,
shouldSkipRetries || i > 0
)
timings.push({
file: test.file,
time,
})
passed = true
console.log(
`${test.file} finished on retry ${i}/${numRetries} in ${time / 1000}s`
)
break
} catch (err) {
if (i < numRetries) {
try {
let testDir = path.dirname(path.join(__dirname, test.file))
// if test is nested in a test folder traverse up a dir to ensure
// we clean up relevant test files
if (testDir.endsWith('/test') || testDir.endsWith('\\test')) {
testDir = path.join(testDir, '..')
}
console.log('Cleaning test files at', testDir)
await exec(`git clean -fdx "${testDir}"`)
await exec(`git checkout "${testDir}"`)
} catch (err) {}
} else {
console.error(`${test.file} failed due to ${err}`)
}
}
}
if (!passed) {
hadFailures = true
const error = new Error(
// "failed to pass within" is a keyword parsed by next-pr-webhook
`${test.file} failed to pass within ${numRetries} retries`
)
console.error(error.message)
if (!shouldContinueTestsOnError) {
testController.abort(error)
} else {
console.log(
`CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed`
)
}
}
// Emit test output if test failed or if we're continuing tests on error
// This is parsed by the commenter webhook to notify about failing tests
if ((!passed || shouldContinueTestsOnError) && isTestJob) {
try {
const testsOutput = await fsp.readFile(
`${test.file}${RESULTS_EXT}`,
'utf8'
)
const obj = JSON.parse(testsOutput)
obj.processEnv = {
NEXT_TEST_MODE: process.env.NEXT_TEST_MODE,
HEADLESS: process.env.HEADLESS,
}
await outputSema.acquire()
if (GROUP) console.log(`${GROUP}Result as JSON for tooling`)
console.log(
`--test output start--`,
JSON.stringify(obj),
`--test output end--`
)
if (ENDGROUP) console.log(ENDGROUP)
outputSema.release()
} catch (err) {
console.log(`Failed to load test output`, err)
}
}
}
const directorySemas = new Map() const directorySemas = new Map()
const originalRetries = numRetries const originalRetries = numRetries
await Promise.all( const results = await Promise.allSettled(
tests.map(async (test) => { tests.map(async (test) => {
const dirName = path.dirname(test.file) const dirName = path.dirname(test.file)
let dirSema = directorySemas.get(dirName) let dirSema = directorySemas.get(dirName)
...@@ -695,109 +791,39 @@ ${ENDGROUP}`) ...@@ -695,109 +791,39 @@ ${ENDGROUP}`)
if (/^test[/\\]integration/.test(test.file) && dirSema === undefined) { if (/^test[/\\]integration/.test(test.file) && dirSema === undefined) {
directorySemas.set(dirName, (dirSema = new Sema(1))) directorySemas.set(dirName, (dirSema = new Sema(1)))
} }
// TODO: Use explicit resource managment instead of this acquire/release pattern
// once CI runs with Node.js 24+.
if (dirSema) await dirSema.acquire() if (dirSema) await dirSema.acquire()
await sema.acquire() await sema.acquire()
let passed = false
const shouldSkipRetries = skipRetryTestManifest.find((t) => try {
t.includes(test.file) if (testSignal.aborted) {
) // We already logged the abort reason. No need to include it in cause.
const numRetries = shouldSkipRetries ? 0 : originalRetries const error = new Error(`Skipped due to abort.`)
if (shouldSkipRetries) { error.name = test.file
console.log( throw error
`Skipping retry for ${test.file} due to skipRetryTestManifest`
)
}
for (let i = 0; i < numRetries + 1; i++) {
try {
console.log(`Starting ${test.file} retry ${i}/${numRetries}`)
const time = await runTest(
test,
shouldSkipRetries || i === numRetries,
shouldSkipRetries || i > 0
)
timings.push({
file: test.file,
time,
})
passed = true
console.log(
`Finished ${test.file} on retry ${i}/${numRetries} in ${
time / 1000
}s`
)
break
} catch (err) {
if (i < numRetries) {
try {
let testDir = path.dirname(path.join(__dirname, test.file))
// if test is nested in a test folder traverse up a dir to ensure
// we clean up relevant test files
if (testDir.endsWith('/test') || testDir.endsWith('\\test')) {
testDir = path.join(testDir, '..')
}
console.log('Cleaning test files at', testDir)
await exec(`git clean -fdx "${testDir}"`)
await exec(`git checkout "${testDir}"`)
} catch (err) {}
} else {
console.error(`${test.file} failed due to ${err}`)
}
}
}
if (!passed) {
console.error(
`${test.file} failed to pass within ${numRetries} retries`
)
if (!shouldContinueTestsOnError) {
killed = true
children.forEach((child) => child.kill())
cleanUpAndExit(1)
} else {
hadFailures = true
console.log(
`CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed`
)
} }
}
// Emit test output if test failed or if we're continuing tests on error await runTest(test)
// This is parsed by the commenter webhook to notify about failing tests } finally {
if ((!passed || shouldContinueTestsOnError) && isTestJob) { sema.release()
try { if (dirSema) dirSema.release()
const testsOutput = await fsp.readFile(
`${test.file}${RESULTS_EXT}`,
'utf8'
)
const obj = JSON.parse(testsOutput)
obj.processEnv = {
NEXT_TEST_MODE: process.env.NEXT_TEST_MODE,
HEADLESS: process.env.HEADLESS,
}
await outputSema.acquire()
if (GROUP) console.log(`${GROUP}Result as JSON for tooling`)
console.log(
`--test output start--`,
JSON.stringify(obj),
`--test output end--`
)
if (ENDGROUP) console.log(ENDGROUP)
outputSema.release()
} catch (err) {
console.log(`Failed to load test output`, err)
}
} }
sema.release()
if (dirSema) dirSema.release()
}) })
) )
for (const result of results) {
if (result.status === 'rejected') {
hadFailures = true
console.error(result.reason)
}
}
if (hadFailures && !shouldContinueTestsOnError) {
// TODO: Does it make sense to update timings if there were failures if without shouldContinueTestsOnError?
return hadFailures
}
if (options.timings) { if (options.timings) {
const curTimings = {} const curTimings = {}
// let junitData = `<testsuites name="jest tests">` // let junitData = `<testsuites name="jest tests">`
...@@ -857,20 +883,20 @@ ${ENDGROUP}`) ...@@ -857,20 +883,20 @@ ${ENDGROUP}`)
} }
} }
// Return whether there were any failures
return hadFailures return hadFailures
} }
main() main().then(
.then((hadFailures) => { (hadFailures) => {
if (hadFailures) { if (hadFailures) {
console.error('Some tests failed') console.error('Some tests failed')
cleanUpAndExit(1) return cleanUpAndExit(1)
} else { } else {
cleanUpAndExit(0) return cleanUpAndExit(0)
} }
}) },
.catch((err) => { (reason) => {
console.error(err) console.error(reason)
cleanUpAndExit(1) return cleanUpAndExit(1)
}) }
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment