Unverified Commit a7c61c10 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Run the deployment tests for turbopack and webpack (#84360)

Fork the set of deployment tests so they run with both turbopack and
webpack. Now that turbopack is the default bundler we need this
coverage.

Because we are now running twice as many tests, decrease concurrency,
otherwise we hit rate limits on vercel. There were attempts to improve
rate limit recovery in the vercel CLI
(https://github.com/vercel/vercel/pull/14443 and
https://github.com/vercel/vercel/pull/14407) which helped but did not
solve the issue. Also there was an investigation into the API service
where we discovered some suspicious but ultimately correct code
(https://github.com/vercel/api/pull/55967).

The basic issue is that the vercel CLI polls `api-deployments-get`
fairly aggressively at the beginning of a deploymnet, and since we start
so many deployments in parallel we will always hit the rate limits. The
recovery logic in the CLI is good but has (reasonably) a fixed set of
retries it will attempt, inevitably some tasks get unlucky. So retrying
the test does work, but really we should slow down which is what we do
here. 😥


https://github.com/vercel/next.js/actions/runs/20290161194/job/58272593085

Fixes PACK-5613

---------
Co-authored-by: 's avatarJJ Kasper <jj@jjsweb.site>
parent 85d74bb4
......@@ -114,7 +114,7 @@ env:
VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbopack-e2e-tests
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
NEXT_TEST_PREFER_OFFLINE: 1
NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }}
NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }}
......
......@@ -35,7 +35,7 @@ env:
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbopack-e2e-tests
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
run-name: test-e2e-deploy ${{ inputs.nextVersion || (github.event_name == 'release' && github.event.release.tag_name) || 'canary' }}
......@@ -76,8 +76,8 @@ jobs:
name: Extract `next` version
run: echo 'value=${{ fromJson(steps.nextPackageInfo.outputs.value).version }}' >> "$GITHUB_OUTPUT"
test-deploy:
name: Run Deploy Tests
test-deploy-webpack:
name: Run Deploy Tests (Webpack)
needs: setup
uses: ./.github/workflows/build_reusable.yml
secrets: inherit
......@@ -99,7 +99,49 @@ jobs:
node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
skipNativeBuild: 'yes'
skipNativeInstall: 'no'
stepName: 'test-deploy-${{ matrix.group }}'
stepName: 'test-deploy-webpack-${{ matrix.group }}'
timeout_minutes: 180
runs_on_labels: '["ubuntu-latest"]'
overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
test-deploy-turbopack:
name: Run Deploy Tests (Turbopack)
needs: [setup]
uses: ./.github/workflows/build_reusable.yml
secrets: inherit
strategy:
fail-fast: false
matrix:
group:
[
1/12,
2/12,
3/12,
4/12,
5/12,
6/12,
7/12,
8/12,
9/12,
10/12,
11/12,
12/12,
]
with:
afterBuild: |
npm i -g vercel@latest && \
NEXT_E2E_TEST_TIMEOUT=240000 \
NEXT_TEST_MODE=deploy \
IS_TURBOPACK_TEST=1 \
TURBOPACK_BUILD=1 \
NEXT_TEST_CONTINUE_ON_ERROR="${{ github.event.inputs.continueOnError || false }}" \
NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
skipNativeBuild: 'yes'
skipNativeInstall: 'no'
stepName: 'test-deploy-turbopack-${{ matrix.group }}'
timeout_minutes: 180
runs_on_labels: '["ubuntu-latest"]'
overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
......@@ -147,7 +189,7 @@ jobs:
overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
report-test-results-to-datadog:
needs: test-deploy
needs: [test-deploy-turbopack, test-deploy-webpack]
if: ${{ always() }}
runs-on: ubuntu-latest
......@@ -238,8 +280,8 @@ jobs:
update-prs:
name: Update prs as ready for review
needs: [test-deploy, create-draft-prs]
if: ${{ needs.test-deploy.result == 'success' && github.repository_owner == 'vercel' && github.event_name == 'release' }}
needs: [test-deploy-turbopack, test-deploy-webpack, create-draft-prs]
if: ${{ needs.test-deploy-webpack.result == 'success' && needs.test-deploy-turbopack.result == 'success' && github.repository_owner == 'vercel' && github.event_name == 'release' }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
......
......@@ -11,12 +11,12 @@ import getChangedTests from './get-changed-tests.mjs'
* --flake-detection: run tests multiple times to detect flaky
*/
async function main() {
let argv = await yargs(process.argv.slice(2))
const argv = await yargs(process.argv.slice(2))
.string('mode')
.string('group')
.boolean('flake-detection').argv
let testMode = argv.mode
const testMode = argv.mode
const isFlakeDetectionMode = argv['flake-detection']
const attempts = isFlakeDetectionMode ? 3 : 1
......@@ -149,7 +149,7 @@ async function main() {
)
}
if (isFlakeDetectionMode && testMode !== 'deploy') {
if (isFlakeDetectionMode) {
for (let i = 0; i < attempts; i++) {
console.log(
`\n\nRun ${i + 1}/${attempts} for ${testMode} tests (Turbopack)`
......@@ -160,8 +160,10 @@ async function main() {
...process.env,
NEXT_TEST_MODE: testMode,
NEXT_TEST_VERSION: nextTestVersion,
NEXT_EXTERNAL_TESTS_FILTERS,
IS_TURBOPACK_TEST: '1',
TURBOPACK_BUILD: testMode === 'start' ? '1' : undefined,
TURBOPACK_BUILD:
testMode === 'start' || testMode === 'deploy' ? '1' : undefined,
TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined,
},
})
......
......@@ -252,7 +252,6 @@ export async function createNext(
rootSpan.traceChild('init next deploy instance').traceFn(() => {
nextInstance = new NextDeployInstance({
...opts,
turbo: false,
})
})
} else {
......@@ -260,7 +259,6 @@ export async function createNext(
rootSpan.traceChild('init next start instance').traceFn(() => {
nextInstance = new NextStartInstance({
...opts,
turbo: false,
})
})
}
......
......@@ -238,7 +238,6 @@ export class NextDeployInstance extends NextInstance {
if (TEST_TEAM_NAME) {
vercelFlags.push('--scope', TEST_TEAM_NAME)
}
const vercelEnv = { ...process.env }
// If the token is available in the environment, use it as the token in the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment