Unverified Commit a7c61c10 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Run the deployment tests for turbopack and webpack (#84360)

Fork the set of deployment tests so they run with both turbopack and
webpack. Now that turbopack is the default bundler we need this
coverage.

Because we are now running twice as many tests, decrease concurrency,
otherwise we hit rate limits on vercel. There were attempts to improve
rate limit recovery in the vercel CLI
(https://github.com/vercel/vercel/pull/14443 and
https://github.com/vercel/vercel/pull/14407) which helped but did not
solve the issue. Also there was an investigation into the API service
where we discovered some suspicious but ultimately correct code
(https://github.com/vercel/api/pull/55967).

The basic issue is that the vercel CLI polls `api-deployments-get`
fairly aggressively at the beginning of a deploymnet, and since we start
so many deployments in parallel we will always hit the rate limits. The
recovery logic in the CLI is good but has (reasonably) a fixed set of
retries it will attempt, inevitably some tasks get unlucky. So retrying
the test does work, but really we should slow down which is what we do
here. 😥


https://github.com/vercel/next.js/actions/runs/20290161194/job/58272593085

Fixes PACK-5613

---------
Co-authored-by: 's avatarJJ Kasper <jj@jjsweb.site>
parent 85d74bb4
...@@ -114,7 +114,7 @@ env: ...@@ -114,7 +114,7 @@ env:
VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }} VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }} VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbopack-e2e-tests VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
NEXT_TEST_PREFER_OFFLINE: 1 NEXT_TEST_PREFER_OFFLINE: 1
NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }} NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }}
NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }} NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }}
......
...@@ -35,7 +35,7 @@ env: ...@@ -35,7 +35,7 @@ env:
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }} VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbopack-e2e-tests VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }} VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
run-name: test-e2e-deploy ${{ inputs.nextVersion || (github.event_name == 'release' && github.event.release.tag_name) || 'canary' }} run-name: test-e2e-deploy ${{ inputs.nextVersion || (github.event_name == 'release' && github.event.release.tag_name) || 'canary' }}
...@@ -76,8 +76,8 @@ jobs: ...@@ -76,8 +76,8 @@ jobs:
name: Extract `next` version name: Extract `next` version
run: echo 'value=${{ fromJson(steps.nextPackageInfo.outputs.value).version }}' >> "$GITHUB_OUTPUT" run: echo 'value=${{ fromJson(steps.nextPackageInfo.outputs.value).version }}' >> "$GITHUB_OUTPUT"
test-deploy: test-deploy-webpack:
name: Run Deploy Tests name: Run Deploy Tests (Webpack)
needs: setup needs: setup
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
secrets: inherit secrets: inherit
...@@ -99,7 +99,49 @@ jobs: ...@@ -99,7 +99,49 @@ jobs:
node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
skipNativeBuild: 'yes' skipNativeBuild: 'yes'
skipNativeInstall: 'no' skipNativeInstall: 'no'
stepName: 'test-deploy-${{ matrix.group }}' stepName: 'test-deploy-webpack-${{ matrix.group }}'
timeout_minutes: 180
runs_on_labels: '["ubuntu-latest"]'
overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
test-deploy-turbopack:
name: Run Deploy Tests (Turbopack)
needs: [setup]
uses: ./.github/workflows/build_reusable.yml
secrets: inherit
strategy:
fail-fast: false
matrix:
group:
[
1/12,
2/12,
3/12,
4/12,
5/12,
6/12,
7/12,
8/12,
9/12,
10/12,
11/12,
12/12,
]
with:
afterBuild: |
npm i -g vercel@latest && \
NEXT_E2E_TEST_TIMEOUT=240000 \
NEXT_TEST_MODE=deploy \
IS_TURBOPACK_TEST=1 \
TURBOPACK_BUILD=1 \
NEXT_TEST_CONTINUE_ON_ERROR="${{ github.event.inputs.continueOnError || false }}" \
NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
skipNativeBuild: 'yes'
skipNativeInstall: 'no'
stepName: 'test-deploy-turbopack-${{ matrix.group }}'
timeout_minutes: 180 timeout_minutes: 180
runs_on_labels: '["ubuntu-latest"]' runs_on_labels: '["ubuntu-latest"]'
overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }} overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
...@@ -147,7 +189,7 @@ jobs: ...@@ -147,7 +189,7 @@ jobs:
overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }} overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
report-test-results-to-datadog: report-test-results-to-datadog:
needs: test-deploy needs: [test-deploy-turbopack, test-deploy-webpack]
if: ${{ always() }} if: ${{ always() }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
...@@ -238,8 +280,8 @@ jobs: ...@@ -238,8 +280,8 @@ jobs:
update-prs: update-prs:
name: Update prs as ready for review name: Update prs as ready for review
needs: [test-deploy, create-draft-prs] needs: [test-deploy-turbopack, test-deploy-webpack, create-draft-prs]
if: ${{ needs.test-deploy.result == 'success' && github.repository_owner == 'vercel' && github.event_name == 'release' }} if: ${{ needs.test-deploy-webpack.result == 'success' && needs.test-deploy-turbopack.result == 'success' && github.repository_owner == 'vercel' && github.event_name == 'release' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
fail-fast: false fail-fast: false
......
...@@ -11,12 +11,12 @@ import getChangedTests from './get-changed-tests.mjs' ...@@ -11,12 +11,12 @@ import getChangedTests from './get-changed-tests.mjs'
* --flake-detection: run tests multiple times to detect flaky * --flake-detection: run tests multiple times to detect flaky
*/ */
async function main() { async function main() {
let argv = await yargs(process.argv.slice(2)) const argv = await yargs(process.argv.slice(2))
.string('mode') .string('mode')
.string('group') .string('group')
.boolean('flake-detection').argv .boolean('flake-detection').argv
let testMode = argv.mode const testMode = argv.mode
const isFlakeDetectionMode = argv['flake-detection'] const isFlakeDetectionMode = argv['flake-detection']
const attempts = isFlakeDetectionMode ? 3 : 1 const attempts = isFlakeDetectionMode ? 3 : 1
...@@ -149,7 +149,7 @@ async function main() { ...@@ -149,7 +149,7 @@ async function main() {
) )
} }
if (isFlakeDetectionMode && testMode !== 'deploy') { if (isFlakeDetectionMode) {
for (let i = 0; i < attempts; i++) { for (let i = 0; i < attempts; i++) {
console.log( console.log(
`\n\nRun ${i + 1}/${attempts} for ${testMode} tests (Turbopack)` `\n\nRun ${i + 1}/${attempts} for ${testMode} tests (Turbopack)`
...@@ -160,8 +160,10 @@ async function main() { ...@@ -160,8 +160,10 @@ async function main() {
...process.env, ...process.env,
NEXT_TEST_MODE: testMode, NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion, NEXT_TEST_VERSION: nextTestVersion,
NEXT_EXTERNAL_TESTS_FILTERS,
IS_TURBOPACK_TEST: '1', IS_TURBOPACK_TEST: '1',
TURBOPACK_BUILD: testMode === 'start' ? '1' : undefined, TURBOPACK_BUILD:
testMode === 'start' || testMode === 'deploy' ? '1' : undefined,
TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined, TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined,
}, },
}) })
......
...@@ -252,7 +252,6 @@ export async function createNext( ...@@ -252,7 +252,6 @@ export async function createNext(
rootSpan.traceChild('init next deploy instance').traceFn(() => { rootSpan.traceChild('init next deploy instance').traceFn(() => {
nextInstance = new NextDeployInstance({ nextInstance = new NextDeployInstance({
...opts, ...opts,
turbo: false,
}) })
}) })
} else { } else {
...@@ -260,7 +259,6 @@ export async function createNext( ...@@ -260,7 +259,6 @@ export async function createNext(
rootSpan.traceChild('init next start instance').traceFn(() => { rootSpan.traceChild('init next start instance').traceFn(() => {
nextInstance = new NextStartInstance({ nextInstance = new NextStartInstance({
...opts, ...opts,
turbo: false,
}) })
}) })
} }
......
...@@ -238,7 +238,6 @@ export class NextDeployInstance extends NextInstance { ...@@ -238,7 +238,6 @@ export class NextDeployInstance extends NextInstance {
if (TEST_TEAM_NAME) { if (TEST_TEAM_NAME) {
vercelFlags.push('--scope', TEST_TEAM_NAME) vercelFlags.push('--scope', TEST_TEAM_NAME)
} }
const vercelEnv = { ...process.env } const vercelEnv = { ...process.env }
// If the token is available in the environment, use it as the token in the // If the token is available in the environment, use it as the token in the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment