Unverified Commit 9ac41a44 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

refactor(CI): Extend areweturboyet integration test jobs for all bundlers (#76361)

Use the reusable configs defined in #76251 for rspack.

Development tests:
https://github.com/vercel/next.js/actions/runs/13514384329
Production tests:
https://github.com/vercel/next.js/actions/runs/13514385828

Patch the `build-rspack-dev-tests-manifest.js` script:

```
diff --git a/test/build-rspack-dev-tests-manifest.js b/test/build-rspack-dev-tests-manifest.js
index 22f8401ef7..8cc5ed4af6 100644
--- a/test/build-rspack-dev-tests-manifest.js
+++ b/test/build-rspack-dev-tests-manifest.js
@@ -91,7 +91,10 @@ async function fetchLatestTestArtifact() {
   const res = JSON.parse(stdout)

   for (const artifact of res.artifacts) {
-    if (artifact.expired || artifact.workflow_run.head_branch !== 'canary') {
+    if (
+      artifact.expired ||
+      artifact.workflow_run.head_branch !== 'bgw/round-robin-tests'
+    ) {
       continue
     }
```

```
node test/build-turbopack-dev-tests-manifest.js
```

Inspect the generated JSON:
https://gist.github.com/bgw/35b7f9d09cd95f64087a69ff5c7f8972
parent 8d43c66a
......@@ -33,6 +33,9 @@ on:
integration_timeout_minutes:
type: number
default: 30
num_retries:
type: number
default: 2
jobs:
# First, build Next.js to execute across tests.
......@@ -97,8 +100,9 @@ jobs:
${{ inputs.run_before_test }}
node run-tests.js \
-g ${{ matrix.group }}/${{ inputs.e2e_groups }} \
-c $TEST_CONCURRENCY \
--group ${{ matrix.group }}/${{ inputs.e2e_groups }} \
--concurrency $TEST_CONCURRENCY \
--retries ${{ inputs.num_retries }} \
--type ${{ inputs.test_type }}
stepName: test-${{ inputs.name }}-${{ matrix.group }}
timeout_minutes: ${{ inputs.e2e_timeout_minutes }}
......@@ -135,8 +139,9 @@ jobs:
${{ inputs.run_before_test }}
node run-tests.js \
-g ${{ matrix.group }}/${{ inputs.integration_groups }} \
-c $TEST_CONCURRENCY \
--group ${{ matrix.group }}/${{ inputs.integration_groups }} \
--concurrency $TEST_CONCURRENCY \
--retries ${{ inputs.num_retries }} \
--type integration
stepName: test-${{ inputs.name }}-integration-${{ matrix.group }}
timeout_minutes: ${{ inputs.integration_timeout_minutes }}
......
# Reusable workflow to execute certain version of Next.js integration tests
# with Rspack.
#
# Refer test.yml for how this workflow is being initialized
# - Workflow can specify `inputs.version` to specify which version of next.js to use, otherwise will use latest release version.
name: Rspack Next.js production integration tests
on:
schedule:
- cron: '0 6 * * *'
workflow_dispatch:
inputs:
# Allow to specify Next.js version to run integration test against.
# If not specified, will use latest release version including canary.
version:
description: Next.js version, sha, branch to test
type: string
default: 'canary'
# The base of the test results to compare against. If not specified, will try to compare with latest main branch's test results.
diff_base:
type: string
default: 'none'
# Workflow-common env variables
env:
# Enabling backtrace will makes snapshot tests fail
RUST_BACKTRACE: 0
NEXT_TELEMETRY_DISABLED: 1
TEST_CONCURRENCY: 6
NEXT_JUNIT_TEST_REPORT: 'true'
__INTERNAL_CUSTOM_TURBOPACK_BINDINGS: ${{ github.workspace }}/packages/next-swc/native/next-swc.linux-x64-gnu.node
NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ github.workspace }}/integration-test-data/test-results/main/failed-test-path-list.json
NEXT_TEST_CONTINUE_ON_ERROR: TRUE
NEXT_E2E_TEST_TIMEOUT: 240000
NEXT_TEST_JOB: 1
# Run an hour earlier than the turbopack tests, so we don't overwhelm the CI
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
# First, build Next.js to execute across tests.
setup_nextjs:
name: Setup Next.js build
uses: ./.github/workflows/setup-nextjs-build.yml
test-dev:
name: Rspack integration tests
uses: ./.github/workflows/integration_tests_reusable.yml
with:
nodeVersion: 18.18.2
version: ${{ inputs.version || 'canary' }}
# Actual test scheduling. These jobs mimic the same jobs in Next.js repo,
# which we do allow some of duplications to make it easier to update if upstream changes.
# Refer build_and_test.yml in the Next.js repo for more details.
test-production:
# This job name is being used in github action to collect test results. Do not change it, or should update
# ./.github/actions/next-integration-stat to match the new name.
name: Next.js integration test (Production)
# Currently it is possible test grouping puts large number of failing tests suites in a single group,
# which ends up job timeouts. Temporarily relieve the timeout until we make progresses on the failing suites.
# ref: https://github.com/vercel/turbo/pull/5668
# timeout-minutes: 180
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
needs: [setup_nextjs]
strategy:
fail-fast: false
matrix:
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
steps:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 18.18.2
check-latest: true
- uses: actions/cache/restore@v3
id: restore-build
with:
path: ./*
key: ${{ inputs.version || 'canary' }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt}}-${{ github.run_number }}
fail-on-cache-miss: true
- name: Enable corepack and install yarn
run: |
corepack enable
corepack prepare --activate yarn@1.22.19
- name: Setup playwright
run: |
pnpm playwright install
- name: Run test/production
run: |
NEXT_TEST_MODE=start NEXT_RSPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 node run-tests.js -g ${{ matrix.group }}/12 -c ${TEST_CONCURRENCY} --type production
# It is currently expected to fail some of next.js integration test, do not fail CI check.
continue-on-error: true
- name: Upload test report artifacts
uses: actions/upload-artifact@v4
with:
name: test-reports-start-${{ matrix.group }}
if-no-files-found: 'error'
path: |
test/rspack-test-junit-report
test-integration-production:
name: Next.js integration test (Integration)
needs: [setup_nextjs]
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
timeout-minutes: 180
strategy:
fail-fast: false
matrix:
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
steps:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 18.18.2
check-latest: true
- uses: actions/cache/restore@v3
id: restore-build
with:
path: ./*
key: ${{ inputs.version || 'canary' }}-${{ github.sha }}
fail-on-cache-miss: true
- name: Enable corepack and install yarn
run: |
corepack enable
corepack prepare --activate yarn@1.22.19
- name: Setup playwright
run: |
pnpm playwright install
- name: Run test/integration
run: |
NEXT_RSPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 node run-tests.js -g ${{ matrix.group }}/12 -c ${TEST_CONCURRENCY} --type integration
continue-on-error: true
- name: Upload test report artifacts
uses: actions/upload-artifact@v4
with:
name: test-reports-build-integration-${{ matrix.group }}
if-no-files-found: 'error'
path: |
test/rspack-test-junit-report
# Collect integration test results from execute_tests,
# Store it as github artifact for next step to consume.
collect_nextjs_production_integration_stat:
needs: [test-production, test-integration-production]
name: Next.js integration test production status report
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
if: always()
permissions:
pull-requests: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Collect integration test stat
uses: ./.github/actions/next-integration-stat
with:
diff_base: ${{ inputs.diff_base || 'none' }}
- name: Store artifacts
uses: actions/upload-artifact@v4
with:
name: test-results-rspack-production
path: |
nextjs-test-results.json
failed-test-path-list.json
passed-test-path-list.json
upload_test_report:
needs: [test-production, test-integration-production]
name: Upload test report to datadog
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
if: ${{ !cancelled() }}
steps:
- name: Download test report artifacts
id: download-test-reports
uses: actions/download-artifact@v4
with:
pattern: test-reports-*
path: test/reports
merge-multiple: true
- name: Upload to datadog
env:
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DD_ENV: 'ci'
run: |
# We'll tag this to the "rspack" datadog service, not "nextjs"
npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:rspack-build.daily --service rspack-build ./test/reports
name: rspack-production
test_type: production
run_before_test: |
export NEXT_RSPACK=1 \
NEXT_TEST_USE_RSPACK=1 \
BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 \
BUILTIN_APP_LOADER=1 \
BUILTIN_SWC_LOADER=1
# Failing tests take longer (due to timeouts and retries). Since we have
# many failing tests, we need smaller groups and longer timeouts, in case
# a group gets stuck with a cluster of failing tests.
e2e_groups: 12
integration_groups: 12
e2e_timeout_minutes: 90
integration_timeout_minutes: 90
# Don't retry, instead of the default of twice, many of these tests are
# currently failing, and it's wasteful to keep retrying them.
num_retries: 0
secrets: inherit
# Reusable workflow to execute certain version of Next.js integration tests
# with Rspack.
#
# Refer test.yml for how this workflow is being initialized
# - Workflow can specify `inputs.version` to specify which version of next.js to use, otherwise will use latest release version.
name: Rspack Next.js development integration tests
on:
schedule:
- cron: '0 6 * * *'
workflow_dispatch:
inputs:
# Allow to specify Next.js version to run integration test against.
# If not specified, will use latest release version including canary.
version:
description: Next.js version, sha, branch to test
type: string
default: 'canary'
# The base of the test results to compare against. If not specified, will try to compare with latest main branch's test results.
diff_base:
type: string
default: 'none'
# Workflow-common env variables
env:
# Enabling backtrace will makes snapshot tests fail
RUST_BACKTRACE: 0
NEXT_TELEMETRY_DISABLED: 1
TEST_CONCURRENCY: 6
NEXT_JUNIT_TEST_REPORT: 'true'
__INTERNAL_CUSTOM_TURBOPACK_BINDINGS: ${{ github.workspace }}/packages/next-swc/native/next-swc.linux-x64-gnu.node
NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ github.workspace }}/integration-test-data/test-results/main/failed-test-path-list.json
NEXT_TEST_CONTINUE_ON_ERROR: TRUE
NEXT_E2E_TEST_TIMEOUT: 240000
NEXT_TEST_JOB: 1
# Run an hour earlier than the turbopack tests, so we don't overwhelm the CI
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
# First, build next-dev and Next.js both to execute across tests.
setup_nextjs:
name: Setup Next.js build
uses: ./.github/workflows/setup-nextjs-build.yml
with:
nodeVersion: 18.18.2
version: ${{ inputs.version || 'canary' }}
# Actual test scheduling. These jobs mimic the same jobs in Next.js repo,
# which we do allow some of duplications to make it easier to update if upstream changes.
# Refer build_and_test.yml in the Next.js repo for more details.
test-dev:
# This job name is being used in github action to collect test results. Do not change it, or should update
# ./.github/actions/next-integration-stat to match the new name.
name: Next.js integration test (Development)
# Currently it is possible test grouping puts large number of failing tests suites in a single group,
# which ends up job timeouts. Temporarily relieve the timeout until we make progresses on the failing suites.
# ref: https://github.com/vercel/turbo/pull/5668
# timeout-minutes: 180
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
needs: [setup_nextjs]
strategy:
fail-fast: false
matrix:
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
steps:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 18.18.2
check-latest: true
- uses: actions/cache/restore@v3
id: restore-build
with:
path: ./*
key: ${{ inputs.version || 'canary' }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt}}-${{ github.run_number }}
fail-on-cache-miss: true
- name: Enable corepack and install yarn
run: |
corepack enable
corepack prepare --activate yarn@1.22.19
- name: Setup playwright
run: |
pnpm playwright install
- name: Run test/development
run: |
NEXT_TEST_MODE=dev NEXT_RSPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 node run-tests.js -g ${{ matrix.group }}/12 -c ${TEST_CONCURRENCY} --type development
# It is currently expected to fail some of next.js integration test, do not fail CI check.
continue-on-error: true
- name: Upload test report artifacts
uses: actions/upload-artifact@v4
with:
name: test-reports-dev-${{ matrix.group }}
if-no-files-found: 'error'
path: |
test/rspack-test-junit-report
test-integration-development:
name: Next.js integration test (Integration)
needs: [setup_nextjs]
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
timeout-minutes: 180
strategy:
fail-fast: false
matrix:
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
steps:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 18.18.2
check-latest: true
- uses: actions/cache/restore@v3
id: restore-build
with:
path: ./*
key: ${{ inputs.version || 'canary' }}-${{ github.sha }}
fail-on-cache-miss: true
- name: Enable corepack and install yarn
run: |
corepack enable
corepack prepare --activate yarn@1.22.19
- name: Setup playwright
run: |
pnpm playwright install
- name: Run test/integration
run: |
NEXT_RSPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 node run-tests.js -g ${{ matrix.group }}/12 -c ${TEST_CONCURRENCY} --type integration
continue-on-error: true
- name: Upload test report artifacts
uses: actions/upload-artifact@v4
with:
name: test-reports-dev-integration-${{ matrix.group }}
if-no-files-found: 'error'
path: |
test/rspack-test-junit-report
# Collect integration test results from execute_tests,
# Store it as github artifact for next step to consume.
collect_nextjs_development_integration_stat:
needs: [test-dev, test-integration-development]
name: Next.js integration test development status report
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
if: always()
permissions:
pull-requests: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Collect integration test stat
uses: ./.github/actions/next-integration-stat
with:
diff_base: ${{ inputs.diff_base || 'none' }}
- name: Store artifacts
uses: actions/upload-artifact@v4
with:
name: test-results-rspack-development
path: |
nextjs-test-results.json
failed-test-path-list.json
passed-test-path-list.json
upload_test_report:
needs: [test-dev, test-integration-development]
name: Upload test report to datadog
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
if: ${{ !cancelled() }}
steps:
- name: Download test report artifacts
id: download-test-reports
uses: actions/download-artifact@v4
with:
pattern: test-reports-*
path: test/reports
merge-multiple: true
- name: Upload to datadog
env:
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DD_ENV: 'ci'
run: |
# We'll tag this to the "rspack" datadog service, not "nextjs"
npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:rspack.daily --service rspack ./test/reports
name: Rspack integration tests
uses: ./.github/workflows/integration_tests_reusable.yml
with:
name: rspack-development
test_type: development
run_before_test: |
export NEXT_RSPACK=1 \
NEXT_TEST_USE_RSPACK=1 \
BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 \
BUILTIN_APP_LOADER=1 \
BUILTIN_SWC_LOADER=1
# Failing tests take longer (due to timeouts and retries). Since we have
# many failing tests, we need smaller groups and longer timeouts, in case
# a group gets stuck with a cluster of failing tests.
e2e_groups: 16
integration_groups: 16
e2e_timeout_minutes: 90
integration_timeout_minutes: 90
# Don't retry, instead of the default of twice, many of these tests are
# currently failing, and it's wasteful to keep retrying them.
num_retries: 0
secrets: inherit
......@@ -3,7 +3,7 @@ name: Update Rspack test manifest
on:
schedule:
# Every day at 9AM https://crontab.guru/#0_9_*_*_*
# Every day at 7AM https://crontab.guru/#0_7_*_*_*
- cron: '0 7 * * *'
workflow_dispatch:
......
......@@ -3,7 +3,7 @@ name: Update Turbopack test manifest
on:
schedule:
# Every day at 9AM https://crontab.guru/#0_9_*_*_*
# Every day at 7AM UTC https://crontab.guru/#0_7_*_*_*
- cron: '0 7 * * *'
workflow_dispatch:
......
......@@ -16,21 +16,21 @@
"test-types": "tsc",
"test-unit": "jest test/unit/ packages/next/ packages/font",
"test-dev": "cross-env NEXT_TEST_MODE=dev pnpm testheadless",
"test-dev-rspack": "cross-env NEXT_TEST_MODE=dev NEXT_TEST_USE_RSPACK=1 BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 BUILTIN_APP_LOADER=1 BUILTIN_SWC_LOADER=1 pnpm testheadless",
"test-dev-rspack": "pnpm run with-rspack pnpm run test-dev",
"test-dev-turbo": "cross-env NEXT_TEST_MODE=dev TURBOPACK=1 TURBOPACK_DEV=1 pnpm testheadless",
"test-start": "cross-env NEXT_TEST_MODE=start pnpm testheadless",
"test-start-rspack": "cross-env NEXT_TEST_MODE=start NEXT_RSPACK=1 BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 BUILTIN_APP_LOADER=1 BUILTIN_SWC_LOADER=1 pnpm testheadless",
"test-start-rspack": "pnpm run with-rspack pnpm run test-start",
"test-start-turbo": "cross-env NEXT_TEST_MODE=start TURBOPACK=1 TURBOPACK_BUILD=1 pnpm testheadless",
"test-deploy": "cross-env NEXT_TEST_MODE=deploy pnpm testheadless",
"testonly-dev": "cross-env NEXT_TEST_MODE=dev pnpm testonly",
"testonly-dev-rspack": "cross-env NEXT_TEST_MODE=dev NEXT_RSPACK=1 pnpm testonly",
"testonly-dev-rspack": "pnpm run with-rspack pnpm run testonly-dev",
"testonly-dev-turbo": "cross-env NEXT_TEST_MODE=dev TURBOPACK=1 TURBOPACK_DEV=1 pnpm testonly",
"testonly-start": "cross-env NEXT_TEST_MODE=start pnpm testonly",
"testonly-start-rspack": "cross-env NEXT_TEST_MODE=start NEXT_RSPACK=1 pnpm testonly",
"testonly-start-rspack": "pnpm run with-rspack pnpm run testonly-start",
"testonly-start-turbo": "cross-env NEXT_TEST_MODE=start TURBOPACK=1 TURBOPACK_BUILD=1 pnpm testonly",
"testonly-deploy": "cross-env NEXT_TEST_MODE=deploy pnpm testonly",
"test": "pnpm testheadless",
"test-rspack": "cross-env NEXT_RSPACK=1 pnpm testheadless",
"test-rspack": "pnpm run with-rspack pnpm run test",
"test-turbo": "cross-env TURBOPACK=1 TURBOPACK_DEV=1 TURBOPACK_BUILD=1 pnpm testheadless",
"testonly": "jest --runInBand",
"testheadless": "cross-env HEADLESS=true pnpm testonly",
......@@ -76,7 +76,8 @@
"update-error-codes": "cd packages/next && pnpm taskr compile check_error_codes",
"storybook": "turbo run storybook",
"build-storybook": "turbo run build-storybook",
"test-storybook": "turbo run test-storybook"
"test-storybook": "turbo run test-storybook",
"with-rspack": "cross-env NEXT_RSPACK=1 NEXT_TEST_USE_RSPACK=1 BUILTIN_FLIGHT_CLIENT_ENTRY_PLUGIN=1 BUILTIN_APP_LOADER=1 BUILTIN_SWC_LOADER=1"
},
"devDependencies": {
"@actions/core": "1.10.1",
......
......@@ -134,7 +134,7 @@ async function createNextInstall({
}
if (useRspack) {
combinedDependencies['@next/plugin-rspack'] = pkgPaths = pkgPaths.get(
combinedDependencies['@next/plugin-rspack'] = pkgPaths.get(
'@next/plugin-rspack'
)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment