Unverified Commit 31921728 authored by Balázs Orbán's avatar Balázs Orbán Committed by GitHub

chore(test): run related E2E deploy tests on PRs (#63763)

parent 754fadac
name: Test E2E (Vercel Deploy), related
on:
pull_request:
types: [opened, synchronize]
jobs:
test:
if: github.repository_owner == 'vercel'
runs-on: ubuntu-latest
env:
CARGO_PROFILE_RELEASE_LTO: 'true'
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DD_ENV: 'ci'
NAPI_CLI_VERSION: 2.16.2
NEXT_JUNIT_TEST_REPORT: 'true'
NEXT_TELEMETRY_DISABLED: 1
NEXT_TEST_JOB: 1
NEXT_TEST_MODE: 'deploy'
NODE_LTS_VERSION: 20
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
TURBO_REMOTE_ONLY: 'true'
TURBO_TEAM: 'vercel'
TURBO_VERSION: 1.12.5
VERCEL_TEST_TEAM: vtest314-next-e2e-tests
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
strategy:
fail-fast: false
steps:
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_LTS_VERSION }}
check-latest: true
- name: Setup pnpm
run: corepack enable
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 25
- name: Setup tests
run: |
pnpm install
pnpm run build
npm i -g vercel@latest
node scripts/run-e2e-test-project-reset.mjs
- name: Run tests
run: |
docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.41.2-jammy /bin/bash -c "cd /work && \
NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && \
corepack enable > /dev/null && \
NEXT_JUNIT_TEST_REPORT=${{ env.NEXT_JUNIT_TEST_REPORT }} \
DATADOG_API_KEY=${{ env.DATADOG_API_KEY }} \
DD_ENV=${{ env.DD_ENV }} \
VERCEL_TEST_TOKEN=${{ env.VERCEL_TEST_TOKEN }} \
VERCEL_TEST_TEAM=${{ env.VERCEL_TEST_TEAM }} \
NEXT_TEST_JOB=${{ env.NEXT_TEST_JOB }} \
NEXT_TEST_MODE=${{ env.NEXT_TEST_MODE }} \
TEST_TIMINGS_TOKEN=${{ env.TEST_TIMINGS_TOKEN }} \
xvfb-run node run-tests.js --related --timings -c 1 >> /proc/1/fd/1"
- name: Save test report as artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: test-reports
if-no-files-found: ignore
retention-days: 2
path: test/test-junit-report
- name: Upload test report to Datadog
continue-on-error: true
run: |
pnpx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./test/test-junit-report
name: test-e2e-deploy
name: Test E2E (Vercel Deploy), scheduled
on:
schedule:
......@@ -8,22 +8,27 @@ on:
workflow_dispatch:
jobs:
build:
test:
if: github.repository_owner == 'vercel'
runs-on: ubuntu-latest
env:
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
VERCEL_TEST_TEAM: vtest314-next-e2e-tests
CARGO_PROFILE_RELEASE_LTO: 'true'
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
DD_ENV: 'ci'
NAPI_CLI_VERSION: 2.16.2
TURBO_VERSION: 1.12.5
NEXT_JUNIT_TEST_REPORT: 'true'
NEXT_TELEMETRY_DISABLED: 1
NEXT_TEST_CONTINUE_ON_ERROR: 1
NEXT_TEST_JOB: 1
NEXT_TEST_MODE: 'deploy'
NODE_LTS_VERSION: 20
CARGO_PROFILE_RELEASE_LTO: 'true'
TURBO_TEAM: 'vercel'
TURBO_REMOTE_ONLY: 'true'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
NEXT_TELEMETRY_DISABLED: 1
TURBO_REMOTE_ONLY: 'true'
TURBO_TEAM: 'vercel'
TURBO_VERSION: 1.12.5
VERCEL_TEST_TEAM: vtest314-next-e2e-tests
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
strategy:
fail-fast: false
......@@ -31,42 +36,52 @@ jobs:
group: [1, 2]
steps:
- name: Setup node
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: ${{ env.NODE_LTS_VERSION }}
check-latest: true
- run: corepack enable
- uses: actions/checkout@v4
- name: Setup pnpm
run: corepack enable
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 25
- run: pnpm install
- run: pnpm run build
- run: npm i -g vercel@latest
- run: node scripts/run-e2e-test-project-reset.mjs
name: Reset test project
- name: Setup tests
run: |
pnpm install
pnpm run build
npm i -g vercel@latest
node scripts/run-e2e-test-project-reset.mjs
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.35.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && corepack enable > /dev/null && NEXT_JUNIT_TEST_REPORT=true DATADOG_API_KEY=${DATADOG_API_KEY} DD_ENV=ci VERCEL_TEST_TOKEN=${{ secrets.VERCEL_TEST_TOKEN }} VERCEL_TEST_TEAM=vtest314-next-e2e-tests NEXT_TEST_JOB=1 NEXT_TEST_MODE=deploy TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/2 -c 1 >> /proc/1/fd/1"
name: Run test/e2e (deploy)
- name: Run tests
run: |
docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.41.2-jammy /bin/bash -c "cd /work && \
NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && \
corepack enable > /dev/null && \
NEXT_JUNIT_TEST_REPORT=${{ env.NEXT_JUNIT_TEST_REPORT }} \
DATADOG_API_KEY=${{ env.DATADOG_API_KEY }} \
DD_ENV=${{ env.DD_ENV }} \
VERCEL_TEST_TOKEN=${{ env.VERCEL_TEST_TOKEN }} \
VERCEL_TEST_TEAM=${{ env.VERCEL_TEST_TEAM }} \
NEXT_TEST_JOB=${{ env.NEXT_TEST_JOB }} \
NEXT_TEST_MODE=${{ env.NEXT_TEST_MODE }} \
TEST_TIMINGS_TOKEN=${{ env.TEST_TIMINGS_TOKEN }} \
xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/2 -c 1 >> /proc/1/fd/1"
- name: Upload test report
- name: Save test report as artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: test-reports
if-no-files-found: ignore
retention-days: 2
path: |
test/test-junit-report
path: test/test-junit-report
- name: Upload test report to datadog
- name: Upload test report to Datadog
continue-on-error: true
run: |
ls -al ./test/*junit
DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./test/test-junit-report
pnpx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./test/test-junit-report
......@@ -102,7 +102,7 @@ import { AppRouteRouteMatcherProvider } from './future/route-matcher-providers/a
import { PagesAPIRouteMatcherProvider } from './future/route-matcher-providers/pages-api-route-matcher-provider'
import { PagesRouteMatcherProvider } from './future/route-matcher-providers/pages-route-matcher-provider'
import { ServerManifestLoader } from './future/route-matcher-providers/helpers/manifest-loaders/server-manifest-loader'
import { getTracer, SpanKind } from './lib/trace/tracer'
import { getTracer, isBubbledError, SpanKind } from './lib/trace/tracer'
import { BaseServerSpan } from './lib/trace/constants'
import { I18NProvider } from './future/helpers/i18n-provider'
import { sendResponse } from './send-response'
......@@ -1390,7 +1390,11 @@ export default abstract class Server<
return this.renderError(null, req, res, '/_error', {})
}
if (this.minimalMode || this.renderOpts.dev || (err as any).bubble) {
if (
this.minimalMode ||
this.renderOpts.dev ||
(isBubbledError(err) && err.bubble)
) {
throw err
}
this.logError(getProperError(err))
......
import type { FetchEventResult } from '../../web/types'
import type { SpanTypes } from './constants'
import { LogSpanAllowList, NextVanillaSpanAllowlist } from './constants'
......@@ -36,10 +37,22 @@ const isPromise = <T>(p: any): p is Promise<T> => {
return p !== null && typeof p === 'object' && typeof p.then === 'function'
}
type BubbledError = Error & { bubble?: boolean }
export class BubbledError extends Error {
constructor(
public readonly bubble?: boolean,
public readonly result?: FetchEventResult
) {
super()
}
}
export function isBubbledError(error: unknown): error is BubbledError {
if (typeof error !== 'object' || error === null) return false
return error instanceof BubbledError
}
const closeSpanWithError = (span: Span, error?: Error) => {
if ((error as BubbledError | undefined)?.bubble === true) {
if (isBubbledError(error) && error.bubble) {
span.setAttribute('next.bubble', true)
} else {
if (error) {
......@@ -307,7 +320,7 @@ class NextTracerImpl implements NextTracer {
}
try {
if (fn.length > 1) {
return fn(span, (err?: Error) => closeSpanWithError(span, err))
return fn(span, (err) => closeSpanWithError(span, err))
}
const result = fn(span)
......
......@@ -88,7 +88,7 @@ import {
INSTRUMENTATION_HOOK_FILENAME,
RSC_PREFETCH_SUFFIX,
} from '../lib/constants'
import { getTracer } from './lib/trace/tracer'
import { BubbledError, getTracer } from './lib/trace/tracer'
import { NextNodeServerSpan } from './lib/trace/constants'
import { nodeFs } from './lib/node-fs-methods'
import { getRouteRegex } from '../shared/lib/router/utils/route-regex'
......@@ -1675,10 +1675,7 @@ export default class NextNodeServer extends BaseServer<
if ('response' in result) {
if (isMiddlewareInvoke) {
bubblingResult = true
const err = new Error()
;(err as any).result = result
;(err as any).bubble = true
throw err
throw new BubbledError(true, result)
}
for (const [key, value] of Object.entries(
......
......@@ -25,6 +25,8 @@ let argv = require('yargs/yargs')(process.argv.slice(2))
.string('g')
.alias('g', 'group')
.number('c')
.boolean('related')
.alias('r', 'related')
.alias('c', 'concurrency').argv
function escapeRegexp(str) {
......@@ -197,6 +199,7 @@ async function main() {
group: argv.group ?? false,
testPattern: argv.testPattern ?? false,
type: argv.type ?? false,
related: argv.related ?? false,
retries: argv.retries ?? DEFAULT_NUM_RETRIES,
}
let numRetries = options.retries
......@@ -232,12 +235,22 @@ async function main() {
let prevTimings
if (tests.length === 0) {
/** @type {RegExp | undefined} */
let testPatternRegex
if (options.testPattern) {
testPatternRegex = new RegExp(options.testPattern)
}
if (options.related) {
const { getRelatedTests } = await import('./scripts/run-related-test.mjs')
const tests = await getRelatedTests()
if (tests.length)
testPatternRegex = new RegExp(tests.map(escapeRegexp).join('|'))
console.log('Running related tests:', testPatternRegex.toString())
}
tests = (
await glob('**/*.test.{js,ts,tsx}', {
nodir: true,
......
......@@ -19,7 +19,9 @@ async function getChangedFilesFromPackages(baseBranch = 'canary') {
await exec('git config --global --add safe.directory /work')
await exec(`git remote set-branches --add origin ${baseBranch}`)
await exec(`git fetch origin ${baseBranch} --depth=20`)
const { stdout } = await exec(`git diff --name-only ${baseBranch}`)
const { stdout } = await exec(
`git diff 'origin/${baseBranch}...' --name-only`
)
return stdout
.trim()
.split('\n')
......
import { createNextDescribe } from 'e2e-utils'
import { createNextDescribe, isNextStart } from 'e2e-utils'
import { links } from './components/links'
async function measure(stream: NodeJS.ReadableStream) {
......@@ -304,12 +304,18 @@ createNextDescribe(
'text/html; charset=utf-8'
)
if (!isNextDev) {
if (isNextStart) {
expect(res.headers.get('cache-control')).toEqual(
's-maxage=31536000, stale-while-revalidate'
)
}
if (isNextDeploy) {
expect(res.headers.get('cache-control')).toEqual(
'public, max-age=0, must-revalidate'
)
}
if (signal === 'redirect()') {
const location = res.headers.get('location')
expect(typeof location).toEqual('string')
......
{
"packages/next/src/server": ["e2e/app-dir/ppr-*"]
"packages/next/src/server": ["e2e/app-dir/ppr-full", "e2e/app-dir/ppr-errors"]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment