Unverified Commit bcb63f32 authored by OJ Kwon's avatar OJ Kwon Committed by GitHub

ci(workflow): enable test trace upload (#51107)



### What?

WEB-1150. 

This PR is an attempt to upload next.js's test results into datadog test trace to track its status. 

Originally it tried to use automatic trace injection (dd-trace/ci/init). However, due to some of custom environments / setup we use it was not compatible out of the box. Instead, this PR injects a new test reportert to generate junit report, then upload it at once at the end of the testing pipeline.

The reporter is configured to run when necessary variables set, local run should not be affected. 

One thing to note is this report will not count retry results, as it'll create duplicated test entry with multiple results since we runts jest per individual test. This'll allow to detect flaky test easier, but also it means we'll get bit of skewed test results compare to the real world as first failure will be accounted as test fail immediately. 
parent 272ffffd
...@@ -23,6 +23,7 @@ env: ...@@ -23,6 +23,7 @@ env:
# canary next-swc binaries in the monorepo # canary next-swc binaries in the monorepo
NEXT_SKIP_NATIVE_POSTINSTALL: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DATADOG_TRACE_NEXTJS_TEST: 'true'
DD_ENV: 'ci' DD_ENV: 'ci'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
NEXT_TEST_JOB: 1 NEXT_TEST_JOB: 1
...@@ -181,6 +182,27 @@ jobs: ...@@ -181,6 +182,27 @@ jobs:
afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/integration/production/test/index.test.js && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/integration/production/test/index.test.js test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/integration/production/test/index.test.js && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/integration/production/test/index.test.js test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts
secrets: inherit secrets: inherit
report-test-results:
needs: ['test-dev', 'test-prod', 'test-integration', 'test-turbopack-dev']
runs-on: [self-hosted, linux, x64]
name: upload test trace to datadog
# We'll not block the CI if the upload fails
continue-on-error: true
steps:
- uses: actions/download-artifact@v3
with:
name: Test trace reports
path: test
- name: Upload traces
run: |
ls -al ./test
npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci@2.14.0
jrm ./nextjs-test-result-junit.xml "test/test-junit-report/**/*.xml"
jrm ./turbopack-test-result-junit.xml "test/turbopack-test-junit-report/**/*.xml"
# Put a separate tag for the tests with turbopack to distinguish between same test names
DD_ENV=ci datadog-ci junit upload --tags test.type:nextjs --service nextjs ./nextjs-test-result-junit.xml
DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack --service nextjs ./turbopack-test-result-junit.xml
tests-pass: tests-pass:
needs: needs:
[ [
......
...@@ -53,6 +53,7 @@ env: ...@@ -53,6 +53,7 @@ env:
# canary next-swc binaries in the monorepo # canary next-swc binaries in the monorepo
NEXT_SKIP_NATIVE_POSTINSTALL: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DATADOG_TRACE_NEXTJS_TEST: 'true'
DD_ENV: 'ci' DD_ENV: 'ci'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
NEXT_TEST_JOB: 1 NEXT_TEST_JOB: 1
...@@ -140,3 +141,13 @@ jobs: ...@@ -140,3 +141,13 @@ jobs:
with: with:
name: turbo run summary name: turbo run summary
path: .turbo/runs path: .turbo/runs
- name: Upload test reports artifact
uses: actions/upload-artifact@v3
if: ${{ inputs.afterBuild }}
with:
name: Test trace reports
path: |
test/test-junit-report
test/turbopack-test-junit-report
if-no-files-found: ignore
...@@ -25,5 +25,33 @@ const customJestConfig = { ...@@ -25,5 +25,33 @@ const customJestConfig = {
}, },
} }
// Check if the environment variable is set to enable test trace,
// Insert a reporter to generate a junit report to upload.
// This won't count for the retry to avoid duplicated test being reported twice
// - which means our test trace will report test results for the flaky test as failed without retry.
const shouldEnableTestTrace =
process.env.DATADOG_API_KEY &&
process.env.DATADOG_TRACE_NEXTJS_TEST &&
!process.env.IS_RETRY
if (shouldEnableTestTrace) {
if (!customJestConfig.reporters) {
customJestConfig.reporters = ['default']
}
const outputDirectory = process.env.__INTERNAL_NEXT_DEV_TEST_TURBO_DEV
? '<rootDir>/turbopack-test-junit-report'
: '<rootDir>/test-junit-report'
customJestConfig.reporters.push([
'jest-junit',
{
outputDirectory,
reportTestSuiteErrors: 'true',
uniqueOutputName: 'true',
outputName: 'nextjs-test-junit',
},
])
}
// createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async // createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async
module.exports = createJestConfig(customJestConfig) module.exports = createJestConfig(customJestConfig)
...@@ -152,6 +152,7 @@ ...@@ -152,6 +152,7 @@
"isomorphic-unfetch": "3.0.0", "isomorphic-unfetch": "3.0.0",
"jest": "27.0.6", "jest": "27.0.6",
"jest-extended": "1.2.1", "jest-extended": "1.2.1",
"jest-junit": "16.0.0",
"json5": "2.2.3", "json5": "2.2.3",
"ky": "0.19.1", "ky": "0.19.1",
"ky-universal": "0.6.0", "ky-universal": "0.6.0",
......
...@@ -111,6 +111,7 @@ importers: ...@@ -111,6 +111,7 @@ importers:
isomorphic-unfetch: 3.0.0 isomorphic-unfetch: 3.0.0
jest: 27.0.6 jest: 27.0.6
jest-extended: 1.2.1 jest-extended: 1.2.1
jest-junit: 16.0.0
json5: 2.2.3 json5: 2.2.3
ky: 0.19.1 ky: 0.19.1
ky-universal: 0.6.0 ky-universal: 0.6.0
...@@ -277,6 +278,7 @@ importers: ...@@ -277,6 +278,7 @@ importers:
isomorphic-unfetch: 3.0.0 isomorphic-unfetch: 3.0.0
jest: 27.0.6 jest: 27.0.6
jest-extended: 1.2.1 jest-extended: 1.2.1
jest-junit: 16.0.0
json5: 2.2.3 json5: 2.2.3
ky: 0.19.1 ky: 0.19.1
ky-universal: 0.6.0_ky@0.19.1 ky-universal: 0.6.0_ky@0.19.1
...@@ -15600,6 +15602,16 @@ packages: ...@@ -15600,6 +15602,16 @@ packages:
- supports-color - supports-color
dev: true dev: true
/jest-junit/16.0.0:
resolution: {integrity: sha512-A94mmw6NfJab4Fg/BlvVOUXzXgF0XIH6EmTgJ5NDPp4xoKq0Kr7sErb+4Xs9nZvu58pJojz5RFGpqnZYJTrRfQ==}
engines: {node: '>=10.12.0'}
dependencies:
mkdirp: 1.0.4
strip-ansi: 6.0.1
uuid: 8.3.2
xml: 1.0.1
dev: true
/jest-leak-detector/27.0.6: /jest-leak-detector/27.0.6:
resolution: {integrity: sha512-2/d6n2wlH5zEcdctX4zdbgX8oM61tb67PQt4Xh8JFAIy6LRKUnX528HulkaG6nD5qDl5vRV1NXejCe1XRCH5gQ==} resolution: {integrity: sha512-2/d6n2wlH5zEcdctX4zdbgX8oM61tb67PQt4Xh8JFAIy6LRKUnX528HulkaG6nD5qDl5vRV1NXejCe1XRCH5gQ==}
engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
...@@ -25397,6 +25409,10 @@ packages: ...@@ -25397,6 +25409,10 @@ packages:
resolution: {integrity: sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==} resolution: {integrity: sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==}
dev: true dev: true
/xml/1.0.1:
resolution: {integrity: sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==}
dev: true
/xmlchars/2.2.0: /xmlchars/2.2.0:
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
dev: true dev: true
......
...@@ -322,33 +322,6 @@ async function main() { ...@@ -322,33 +322,6 @@ async function main() {
const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
let shouldEnableTestTrace = false
let traceInitPath = null
try {
traceInitPath = require.resolve('dd-trace/ci/init')
// Enable trace only if tracer (dd-trace) can be resolved
shouldEnableTestTrace =
process.env.DATADOG_API_KEY && process.env.DATADOG_TRACE_NEXTJS_TEST
} catch (e) {
shouldEnableTestTrace = false
}
const traceEnv =
shouldEnableTestTrace && traceInitPath
? {
DD_API_KEY: process.env.DATADOG_API_KEY,
DD_CIVISIBILITY_AGENTLESS_ENABLED: 1,
DD_ENV: 'ci',
DD_SERVICE: 'nextjs',
NODE_OPTIONS: `-r ${traceInitPath}`,
}
: {}
if (shouldEnableTestTrace) {
console.log(`Running test with Datadog tracing enabled`)
}
const child = spawn( const child = spawn(
jestPath, jestPath,
[ [
...@@ -368,7 +341,7 @@ async function main() { ...@@ -368,7 +341,7 @@ async function main() {
stdio: ['ignore', 'pipe', 'pipe'], stdio: ['ignore', 'pipe', 'pipe'],
env: { env: {
...process.env, ...process.env,
...traceEnv, IS_RETRY: isRetry ? 'true' : undefined,
RECORD_REPLAY: shouldRecordTestWithReplay, RECORD_REPLAY: shouldRecordTestWithReplay,
// run tests in headless mode by default // run tests in headless mode by default
HEADLESS: 'true', HEADLESS: 'true',
......
...@@ -2,3 +2,5 @@ ...@@ -2,3 +2,5 @@
.vscode .vscode
e2e/**/tsconfig.json e2e/**/tsconfig.json
test-junit-report/
turbopack-test-junit-report/
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment