Unverified Commit 8bdda745 authored by Leah's avatar Leah Committed by GitHub

chore(CI): improve datadog reporting (#58267)

parent f514684b
...@@ -301,12 +301,12 @@ jobs: ...@@ -301,12 +301,12 @@ jobs:
- name: 'check build cache status' - name: 'check build cache status'
id: check-did-build id: check-did-build
run: if [[ ! -z $(ls packages/next-swc/native) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi run: if [[ ! -z $(ls packages/next-swc/native) ]]; then echo "DID_BUILD=true" >> $GITHUB_OUTPUT; fi
# Trying to upload metrics for the Turbopack to datadog's CI pipeline execution # Try to upload metrics for Turbopack to datadog's CI pipeline execution
- name: 'Collect turbopack build metrics' - name: 'Collect turbopack build metrics'
id: check-turbopack-bytesize id: check-turbopack-bytesize
if: ${{ steps.check-did-build.outputs.DID_BUILD == 'yup' }} if: ${{ steps.check-did-build.outputs.DID_BUILD == 'true' }}
continue-on-error: true continue-on-error: true
run: | run: |
mkdir -p ./turbopack-bin-size mkdir -p ./turbopack-bin-size
...@@ -322,7 +322,7 @@ jobs: ...@@ -322,7 +322,7 @@ jobs:
done done
- name: Upload turbopack bytesize artifact - name: Upload turbopack bytesize artifact
if: ${{ steps.check-did-build.outputs.DID_BUILD == 'yup' }} if: ${{ steps.check-did-build.outputs.DID_BUILD == 'true' }}
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: turbopack-bytesize name: turbopack-bytesize
...@@ -492,23 +492,26 @@ jobs: ...@@ -492,23 +492,26 @@ jobs:
NEXT_SKIP_NATIVE_POSTINSTALL: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1
upload_turbopack_bytesize: upload_turbopack_bytesize:
name: Upload Turbopack Bytesize trace to Datadog name: Upload Turbopack Bytesize metrics to Datadog
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [build-native] needs: [build-native]
env: env:
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
steps: steps:
- name: Collect bytesize traces - name: Collect bytesize metrics
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3
with: with:
name: turbopack-bytesize name: turbopack-bytesize
path: turbopack-bin-size path: turbopack-bin-size
- name: Upload to Datadog - name: Upload to Datadog
run: | run: |
ls -al turbopack-bin-size ls -al turbopack-bin-size
npm install -g @datadog/datadog-ci@2.14.0 @aws-sdk/property-provider@3
for filename in turbopack-bin-size/*; do for filename in turbopack-bin-size/*; do
export BYTESIZE+=" --metrics $(cat $filename)" export BYTESIZE+=" --metrics $(cat $filename)"
done done
echo "Reporting $BYTESIZE" echo "Reporting $BYTESIZE"
datadog-ci metric --no-fail --level pipeline $BYTESIZE
npx @datadog/datadog-ci@2.23.1 metric --no-fail --level pipeline $BYTESIZE
...@@ -22,7 +22,7 @@ env: ...@@ -22,7 +22,7 @@ env:
# canary next-swc binaries in the monorepo # canary next-swc binaries in the monorepo
NEXT_SKIP_NATIVE_POSTINSTALL: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DATADOG_TRACE_NEXTJS_TEST: 'true' NEXT_JUNIT_TEST_REPORT: 'true'
DD_ENV: 'ci' DD_ENV: 'ci'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
NEXT_TEST_JOB: 1 NEXT_TEST_JOB: 1
...@@ -240,11 +240,28 @@ jobs: ...@@ -240,11 +240,28 @@ jobs:
'test-turbopack-dev', 'test-turbopack-dev',
'test-turbopack-integration', 'test-turbopack-integration',
] ]
uses: ./.github/workflows/build_reusable.yml if: always()
runs-on: ubuntu-latest
name: report test results to datadog
steps:
- name: Download test report artifacts
id: download-test-reports
uses: actions/download-artifact@v3
with: with:
skipForDocsOnly: 'yes' name: test-reports
uploadTestTrace: 'yes' path: test
secrets: inherit
- name: Upload test report to datadog
run: |
if [ -d ./test/test-junit-report ]; then
# Add a `test.type` tag to distinguish between turbopack and next.js runs
DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs --service nextjs ./test/test-junit-report
fi
if [ -d ./test/turbopack-test-junit-report ]; then
# Add a `test.type` tag to distinguish between turbopack and next.js runs
DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:turbopack --service nextjs ./test/turbopack-test-junit-report
fi
tests-pass: tests-pass:
needs: needs:
......
...@@ -49,10 +49,6 @@ on: ...@@ -49,10 +49,6 @@ on:
required: false required: false
description: 'if swc artifact needs uploading' description: 'if swc artifact needs uploading'
type: string type: string
uploadTestTrace:
required: false
description: 'if test trace needs uploading'
type: string
rustCacheKey: rustCacheKey:
required: false required: false
description: 'rustCacheKey to cache shared target assets' description: 'rustCacheKey to cache shared target assets'
...@@ -73,7 +69,7 @@ env: ...@@ -73,7 +69,7 @@ env:
# canary next-swc binaries in the monorepo # canary next-swc binaries in the monorepo
NEXT_SKIP_NATIVE_POSTINSTALL: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DATADOG_TRACE_NEXTJS_TEST: 'true' NEXT_JUNIT_TEST_REPORT: 'true'
DD_ENV: 'ci' DD_ENV: 'ci'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
NEXT_TEST_JOB: 1 NEXT_TEST_JOB: 1
...@@ -187,6 +183,7 @@ jobs: ...@@ -187,6 +183,7 @@ jobs:
with: with:
name: turbo run summary name: turbo run summary
path: .turbo/runs path: .turbo/runs
if-no-files-found: ignore
- name: Upload bundle analyzer artifacts - name: Upload bundle analyzer artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
...@@ -195,33 +192,12 @@ jobs: ...@@ -195,33 +192,12 @@ jobs:
name: webpack bundle analysis stats name: webpack bundle analysis stats
path: packages/next/dist/compiled/next-server/report.*.html path: packages/next/dist/compiled/next-server/report.*.html
- name: Upload test reports artifact - name: Upload test report artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
if: ${{ inputs.afterBuild }} if: ${{ inputs.afterBuild && always() }}
with: with:
name: Test trace reports name: test-reports
path: | path: |
test/test-junit-report test/test-junit-report
test/turbopack-test-junit-report test/turbopack-test-junit-report
if-no-files-found: ignore if-no-files-found: ignore
- name: Download test reports artifact
id: download-test-reports
uses: actions/download-artifact@v3
continue-on-error: true
if: ${{ inputs.uploadTestTrace == 'yes' && (inputs.skipForDocsOnly != 'yes' || steps.docs-change.outputs.DOCS_CHANGE == 'nope') }}
with:
name: Test trace reports
path: test
- name: Upload test trace to datadog
if: ${{ inputs.uploadTestTrace == 'yes' && (inputs.skipForDocsOnly != 'yes' || steps.docs-change.outputs.DOCS_CHANGE == 'nope') }}
continue-on-error: true
run: |
ls -al ./test
npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci@2.14.0 @aws-sdk/property-provider@3
jrm ./nextjs-test-result-junit.xml "test/test-junit-report/**/*.xml"
jrm ./turbopack-test-result-junit.xml "test/turbopack-test-junit-report/**/*.xml"
# Put a separate tag for the tests with turbopack to distinguish between same test names
DD_ENV=ci datadog-ci junit upload --tags test.type:nextjs --service nextjs ./nextjs-test-result-junit.xml
DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack --service nextjs ./turbopack-test-result-junit.xml
...@@ -30,7 +30,7 @@ env: ...@@ -30,7 +30,7 @@ env:
NEXT_TELEMETRY_DISABLED: 1 NEXT_TELEMETRY_DISABLED: 1
TEST_CONCURRENCY: 6 TEST_CONCURRENCY: 6
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DATADOG_TRACE_NEXTJS_TEST: 'true' NEXT_JUNIT_TEST_REPORT: 'true'
DD_ENV: 'ci' DD_ENV: 'ci'
# Turbopack specific customization for the test runner # Turbopack specific customization for the test runner
TURBOPACK: 1 TURBOPACK: 1
...@@ -96,10 +96,10 @@ jobs: ...@@ -96,10 +96,10 @@ jobs:
# marker to parse log output, do not delete / change. # marker to parse log output, do not delete / change.
NEXT_INTEGRATION_TEST: true NEXT_INTEGRATION_TEST: true
- name: Upload test reports artifact - name: Upload test report artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: Test trace reports name: test-reports
path: | path: |
test/turbopack-test-junit-report test/turbopack-test-junit-report
...@@ -140,10 +140,10 @@ jobs: ...@@ -140,10 +140,10 @@ jobs:
env: env:
NEXT_INTEGRATION_TEST: true NEXT_INTEGRATION_TEST: true
- name: Upload test reports artifact - name: Upload test report artifacts
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: Test trace reports name: test-reports
path: | path: |
test/turbopack-test-junit-report test/turbopack-test-junit-report
...@@ -183,9 +183,9 @@ jobs: ...@@ -183,9 +183,9 @@ jobs:
passed-test-path-list.json passed-test-path-list.json
slack-payload.json slack-payload.json
upload_test_trace: upload_test_report:
needs: [test-dev, test-integration] needs: [test-dev, test-integration]
name: Upload test trace to datadog name: Upload test report to datadog
runs-on: runs-on:
- 'self-hosted' - 'self-hosted'
- 'linux' - 'linux'
...@@ -194,27 +194,14 @@ jobs: ...@@ -194,27 +194,14 @@ jobs:
if: always() if: always()
steps: steps:
- name: Checkout - name: Download test report artifacts
uses: actions/checkout@v3
- name: Download test reports artifact
id: download-test-reports id: download-test-reports
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3
with: with:
name: Test trace reports name: test-reports
path: test path: test/reports
- name: Upload test trace to datadog
- name: Upload to datadog
run: | run: |
npm install -g @datadog/datadog-ci@2.18.1 # We'll tag this to the "Turbopack" datadog service, not "nextjs"
ls -al ./test/*.xml DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:turbopack.daily --service Turbopack ./test/report
# We'll tag this to the Turbopack service, not the next.js
DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack.daily --service Turbopack test
# For the debugging purpose, upload the test trace to github artifact.
# So we can manually analyze test reports.
- name: Upload test reports artifact
uses: actions/upload-artifact@v3
with:
name: Merged test trace reports
path: |
nextjs-test-result-junit.xml
./**/*.xml
...@@ -50,23 +50,22 @@ jobs: ...@@ -50,23 +50,22 @@ jobs:
- run: RESET_VC_PROJECT=true node scripts/reset-vercel-project.mjs - run: RESET_VC_PROJECT=true node scripts/reset-vercel-project.mjs
name: Reset test project name: Reset test project
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.35.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && corepack enable > /dev/null && DATADOG_TRACE_NEXTJS_TEST=TRUE DATADOG_API_KEY=${DATADOG_API_KEY} DD_ENV=ci VERCEL_TEST_TOKEN=${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TEAM=vtest314-next-e2e-tests NEXT_TEST_JOB=1 NEXT_TEST_MODE=deploy TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} NEXT_TEST_CONTINUE_ON_ERROR=1 xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/2 -c 1 >> /proc/1/fd/1" - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.35.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && corepack enable > /dev/null && NEXT_JUNIT_TEST_REPORT=true DATADOG_API_KEY=${DATADOG_API_KEY} DD_ENV=ci VERCEL_TEST_TOKEN=${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TEAM=vtest314-next-e2e-tests NEXT_TEST_JOB=1 NEXT_TEST_MODE=deploy TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} NEXT_TEST_CONTINUE_ON_ERROR=1 xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/2 -c 1 >> /proc/1/fd/1"
name: Run test/e2e (deploy) name: Run test/e2e (deploy)
- name: Upload test trace - name: Upload test report
if: always() if: always()
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: test-trace name: test-reports
if-no-files-found: ignore if-no-files-found: ignore
retention-days: 2 retention-days: 2
path: | path: |
test/traces test/test-junit-report
- name: Upload test trace to datadog - name: Upload test report to datadog
continue-on-error: true continue-on-error: true
run: | run: |
ls -al ./test ls -al ./test/*junit
npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci@2.14.0 @aws-sdk/property-provider@3
jrm ./nextjs-test-result-junit.xml "test/test-junit-report/**/*.xml" DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./test/test-junit-report
DD_ENV=ci datadog-ci junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./nextjs-test-result-junit.xml
...@@ -22,14 +22,14 @@ const customJestConfig = { ...@@ -22,14 +22,14 @@ const customJestConfig = {
}, },
} }
// Check if the environment variable is set to enable test trace, // Check if the environment variable is set to enable test report,
// Insert a reporter to generate a junit report to upload. // Insert a reporter to generate a junit report to upload.
// This won't count for the retry to avoid duplicated test being reported twice //
// - which means our test trace will report test results for the flaky test as failed without retry. // This won't count retries to avoid tests being reported twice.
const shouldEnableTestTrace = // Our test report will report test results for flaky tests as failed without retry.
process.env.DATADOG_API_KEY && process.env.DATADOG_TRACE_NEXTJS_TEST const enableTestReport = !!process.env.NEXT_JUNIT_TEST_REPORT
if (shouldEnableTestTrace) { if (enableTestReport) {
if (!customJestConfig.reporters) { if (!customJestConfig.reporters) {
customJestConfig.reporters = ['default'] customJestConfig.reporters = ['default']
} }
...@@ -45,6 +45,7 @@ if (shouldEnableTestTrace) { ...@@ -45,6 +45,7 @@ if (shouldEnableTestTrace) {
reportTestSuiteErrors: 'true', reportTestSuiteErrors: 'true',
uniqueOutputName: 'true', uniqueOutputName: 'true',
outputName: 'nextjs-test-junit', outputName: 'nextjs-test-junit',
addFileAttribute: 'true',
}, },
]) ])
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment