Unverified Commit 62590c40 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Update E2E tests workflow (#85485)

Adds an option to allow continuing tests on error to find all failures
in single run also marks jobs as failed if any test failed while
running.
parent d35c7366
...@@ -91,7 +91,7 @@ jobs: ...@@ -91,7 +91,7 @@ jobs:
afterBuild: | afterBuild: |
# e2e and ${{ inputs.test_type }} tests with `node run-tests.js` # e2e and ${{ inputs.test_type }} tests with `node run-tests.js`
export NEXT_TEST_CONTINUE_ON_ERROR=TRUE export NEXT_TEST_CONTINUE_ON_ERROR=true
export NEXT_TEST_MODE=${{ export NEXT_TEST_MODE=${{
inputs.test_type == 'development' && 'dev' || 'start' inputs.test_type == 'development' && 'dev' || 'start'
}} }}
...@@ -123,7 +123,7 @@ jobs: ...@@ -123,7 +123,7 @@ jobs:
afterBuild: | afterBuild: |
# legacy integration tests with `node run-tests.js` # legacy integration tests with `node run-tests.js`
export NEXT_TEST_CONTINUE_ON_ERROR=TRUE export NEXT_TEST_CONTINUE_ON_ERROR=true
# HACK: Despite the name, these environment variables are just used to # HACK: Despite the name, these environment variables are just used to
# gate tests, so they're applicable to both turbopack and rspack tests # gate tests, so they're applicable to both turbopack and rspack tests
......
...@@ -19,6 +19,10 @@ on: ...@@ -19,6 +19,10 @@ on:
description: Override the proxy address to use for the test description: Override the proxy address to use for the test
default: '' default: ''
type: string type: string
continueOnError:
description: whether the tests should continue on failure
default: false
type: boolean
env: env:
TURBO_TEAM: 'vercel' TURBO_TEAM: 'vercel'
...@@ -81,6 +85,7 @@ jobs: ...@@ -81,6 +85,7 @@ jobs:
NEXT_E2E_TEST_TIMEOUT=240000 \ NEXT_E2E_TEST_TIMEOUT=240000 \
NEXT_TEST_MODE=deploy \ NEXT_TEST_MODE=deploy \
IS_WEBPACK_TEST=1 \ IS_WEBPACK_TEST=1 \
NEXT_TEST_CONTINUE_ON_ERROR="${{ github.event.inputs.continueOnError || false }}" \
NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \ NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \ NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \ VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
......
...@@ -55,7 +55,9 @@ const DEFAULT_CONCURRENCY = 2 ...@@ -55,7 +55,9 @@ const DEFAULT_CONCURRENCY = 2
const RESULTS_EXT = `.results.json` const RESULTS_EXT = `.results.json`
const isTestJob = !!process.env.NEXT_TEST_JOB const isTestJob = !!process.env.NEXT_TEST_JOB
// Check env to see if test should continue even if some of test fails // Check env to see if test should continue even if some of test fails
const shouldContinueTestsOnError = !!process.env.NEXT_TEST_CONTINUE_ON_ERROR const shouldContinueTestsOnError =
process.env.NEXT_TEST_CONTINUE_ON_ERROR === 'true'
// Check env to load a list of test paths to skip retry. This is to be used in conjunction with NEXT_TEST_CONTINUE_ON_ERROR, // Check env to load a list of test paths to skip retry. This is to be used in conjunction with NEXT_TEST_CONTINUE_ON_ERROR,
// When try to run all of the tests regardless of pass / fail and want to skip retrying `known` failed tests. // When try to run all of the tests regardless of pass / fail and want to skip retrying `known` failed tests.
// manifest should be a json file with an array of test paths. // manifest should be a json file with an array of test paths.
...@@ -500,6 +502,7 @@ ${ENDGROUP}`) ...@@ -500,6 +502,7 @@ ${ENDGROUP}`)
) )
let firstError = true let firstError = true
let killed = false let killed = false
let hadFailures = false
const runTest = (/** @type {TestFile} */ test, isFinalRun, isRetry) => const runTest = (/** @type {TestFile} */ test, isFinalRun, isRetry) =>
new Promise((resolve, reject) => { new Promise((resolve, reject) => {
...@@ -756,6 +759,7 @@ ${ENDGROUP}`) ...@@ -756,6 +759,7 @@ ${ENDGROUP}`)
children.forEach((child) => child.kill()) children.forEach((child) => child.kill())
cleanUpAndExit(1) cleanUpAndExit(1)
} else { } else {
hadFailures = true
console.log( console.log(
`CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed` `CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed`
) )
...@@ -852,10 +856,20 @@ ${ENDGROUP}`) ...@@ -852,10 +856,20 @@ ${ENDGROUP}`)
} }
} }
} }
// Return whether there were any failures
return hadFailures
} }
main() main()
.then(() => cleanUpAndExit(0)) .then((hadFailures) => {
if (hadFailures) {
console.error('Some tests failed')
cleanUpAndExit(1)
} else {
cleanUpAndExit(0)
}
})
.catch((err) => { .catch((err) => {
console.error(err) console.error(err)
cleanUpAndExit(1) cleanUpAndExit(1)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment