Unverified Commit 445d880e authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

test: Don't run integration tests twice for Turbopack (#83335)

This should reduce some duplicate CI jobs.
This split existed to support separate dev/prod manifests for skipping tests. That's not needed anymore, so we can just switch to the Webpack setup. This manual partitioning of `integration` tests into dev vs prod was brittle anyway
We still need this setup for rspack though
parent c979cefa
...@@ -266,14 +266,27 @@ jobs: ...@@ -266,14 +266,27 @@ jobs:
secrets: inherit secrets: inherit
test-turbopack-integration: test-turbopack-integration:
name: test turbopack development integration name: test turbopack integration
needs: ['optimize-ci', 'changes', 'build-next', 'build-native'] needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }} if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
group: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6] group:
- 1/13
- 2/13
- 3/13
- 4/13
- 5/13
- 6/13
- 7/13
- 8/13
- 9/13
- 10/13
- 11/13
- 12/13
- 13/13
# Empty value uses default # Empty value uses default
# TODO: Run with React 18. # TODO: Run with React 18.
# Integration tests use the installed React version in next/package.json. # Integration tests use the installed React version in next/package.json.
...@@ -285,7 +298,6 @@ jobs: ...@@ -285,7 +298,6 @@ jobs:
nodeVersion: 20.9.0 nodeVersion: 20.9.0
afterBuild: | afterBuild: |
export IS_TURBOPACK_TEST=1 export IS_TURBOPACK_TEST=1
export TURBOPACK_DEV=1
export NEXT_TEST_REACT_VERSION="${{ matrix.react }}" export NEXT_TEST_REACT_VERSION="${{ matrix.react }}"
export __NEXT_EXPERIMENTAL_STRICT_ROUTE_TYPES=true export __NEXT_EXPERIMENTAL_STRICT_ROUTE_TYPES=true
export RUST_BACKTRACE=1 export RUST_BACKTRACE=1
...@@ -326,38 +338,6 @@ jobs: ...@@ -326,38 +338,6 @@ jobs:
stepName: 'test-turbopack-production-react-${{ matrix.react }}-${{ matrix.group }}' stepName: 'test-turbopack-production-react-${{ matrix.react }}-${{ matrix.group }}'
secrets: inherit secrets: inherit
test-turbopack-production-integration:
name: test turbopack production integration
needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}
strategy:
fail-fast: false
matrix:
group: [1/10, 2/10, 3/10, 4/10, 5/10, 6/10, 7/10, 8/10, 9/10, 10/10]
# Empty value uses default
# TODO: Run with React 18.
# Integration tests use the installed React version in next/package.json.
# We can't easily switch like we do for e2e tests.
# Skipping this dimension until we can figure out a way to test multiple React versions.
react: ['']
uses: ./.github/workflows/build_reusable.yml
with:
nodeVersion: 20.9.0
afterBuild: |
export IS_TURBOPACK_TEST=1
export TURBOPACK_BUILD=1
export NEXT_TEST_REACT_VERSION="${{ matrix.react }}"
export __NEXT_EXPERIMENTAL_STRICT_ROUTE_TYPES=true
export RUST_BACKTRACE=1
node run-tests.js \
--timings \
-g ${{ matrix.group }} \
--type integration
stepName: 'test-turbopack-production-integration-react-${{ matrix.react }}-${{ matrix.group }}'
secrets: inherit
test-rspack-dev: test-rspack-dev:
name: test rspack dev name: test rspack dev
needs: ['optimize-ci', 'changes', 'build-next', 'build-native'] needs: ['optimize-ci', 'changes', 'build-next', 'build-native']
...@@ -1059,7 +1039,6 @@ jobs: ...@@ -1059,7 +1039,6 @@ jobs:
'test-new-tests-deploy', 'test-new-tests-deploy',
'test-new-tests-deploy-cache-components', 'test-new-tests-deploy-cache-components',
'test-turbopack-production', 'test-turbopack-production',
'test-turbopack-production-integration',
'test-unit-windows', 'test-unit-windows',
'test-dev-windows', 'test-dev-windows',
'test-integration-windows', 'test-integration-windows',
......
...@@ -25,8 +25,8 @@ ...@@ -25,8 +25,8 @@
"test-dev-experimental-webpack": "pnpm run with-webpack pnpm test-dev-experimental-inner", "test-dev-experimental-webpack": "pnpm run with-webpack pnpm test-dev-experimental-inner",
"test-dev-rspack": "pnpm run with-rspack pnpm run test-dev-inner", "test-dev-rspack": "pnpm run with-rspack pnpm run test-dev-inner",
"test-dev-experimental-rspack": "pnpm run with-rspack pnpm run test-dev-experimental-inner", "test-dev-experimental-rspack": "pnpm run with-rspack pnpm run test-dev-experimental-inner",
"test-dev-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_DEV=1 pnpm test-dev-inner", "test-dev-turbo": "pnpm run with-turbo pnpm test-dev-inner",
"test-dev-experimental-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_DEV=1 pnpm run with-experimental pnpm test-dev-inner", "test-dev-experimental-turbo": "pnpm run with-turbo pnpm run with-experimental pnpm test-dev-inner",
"test-start-inner": "cross-env NEXT_TEST_MODE=start pnpm testheadless", "test-start-inner": "cross-env NEXT_TEST_MODE=start pnpm testheadless",
"test-start": "pnpm run test-start-webpack", "test-start": "pnpm run test-start-webpack",
"test-start-webpack": "pnpm run with-webpack pnpm run test-start-inner", "test-start-webpack": "pnpm run with-webpack pnpm run test-start-inner",
...@@ -35,31 +35,31 @@ ...@@ -35,31 +35,31 @@
"test-start-experimental-webpack": "pnpm run with-webpack pnpm run test-start-experimental-inner", "test-start-experimental-webpack": "pnpm run with-webpack pnpm run test-start-experimental-inner",
"test-start-rspack": "pnpm run with-rspack pnpm run test-start-inner", "test-start-rspack": "pnpm run with-rspack pnpm run test-start-inner",
"test-start-experimental-rspack": "pnpm run with-rspack pnpm run test-start-experimental-inner", "test-start-experimental-rspack": "pnpm run with-rspack pnpm run test-start-experimental-inner",
"test-start-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_BUILD=1 pnpm test-start-inner", "test-start-turbo": "pnpm run with-turbo pnpm test-start-inner",
"test-start-experimental-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_BUILD=1 pnpm run with-experimental pnpm test-start-experimental-inner", "test-start-experimental-turbo": "pnpm run with-turbo pnpm run with-experimental pnpm test-start-experimental-inner",
"test-deploy-inner": "cross-env NEXT_TEST_MODE=deploy pnpm testheadless", "test-deploy-inner": "cross-env NEXT_TEST_MODE=deploy pnpm testheadless",
"test-deploy": "pnpm run test-deploy-webpack", "test-deploy": "pnpm run test-deploy-webpack",
"test-deploy-webpack": "pnpm run with-webpack pnpm test-deploy-inner", "test-deploy-webpack": "pnpm run with-webpack pnpm test-deploy-inner",
"test-deploy-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_BUILD=1 pnpm test-deploy-inner", "test-deploy-turbo": "pnpm run with-turbo pnpm test-deploy-inner",
"testonly-dev-inner": "cross-env NEXT_TEST_MODE=dev pnpm testonly", "testonly-dev-inner": "cross-env NEXT_TEST_MODE=dev pnpm testonly",
"testonly-dev": "pnpm run testonly-dev-webpack", "testonly-dev": "pnpm run testonly-dev-webpack",
"testonly-dev-webpack": "pnpm run with-webpack pnpm run testonly-dev-inner", "testonly-dev-webpack": "pnpm run with-webpack pnpm run testonly-dev-inner",
"testonly-dev-rspack": "pnpm run with-rspack pnpm run testonly-dev-inner", "testonly-dev-rspack": "pnpm run with-rspack pnpm run testonly-dev-inner",
"testonly-dev-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_DEV=1 pnpm testonly-dev-inner", "testonly-dev-turbo": "pnpm run with-turbo pnpm testonly-dev-inner",
"testonly-start-inner": "cross-env NEXT_TEST_MODE=start pnpm testonly", "testonly-start-inner": "cross-env NEXT_TEST_MODE=start pnpm testonly",
"testonly-start": "pnpm run testonly-start-webpack", "testonly-start": "pnpm run testonly-start-webpack",
"testonly-start-webpack": "pnpm run with-webpack pnpm run testonly-start-inner", "testonly-start-webpack": "pnpm run with-webpack pnpm run testonly-start-inner",
"testonly-start-rspack": "pnpm run with-rspack pnpm run testonly-start-inner", "testonly-start-rspack": "pnpm run with-rspack pnpm run testonly-start-inner",
"testonly-start-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_BUILD=1 pnpm testonly-start-inner", "testonly-start-turbo": "pnpm run with-turbo pnpm testonly-start-inner",
"testonly-deploy-inner": "cross-env NEXT_TEST_MODE=deploy pnpm testonly", "testonly-deploy-inner": "cross-env NEXT_TEST_MODE=deploy pnpm testonly",
"testonly-deploy": "pnpm run testonly-deploy-webpack", "testonly-deploy": "pnpm run testonly-deploy-webpack",
"testonly-deploy-webpack": "pnpm run with-webpack pnpm run testonly-deploy-inner", "testonly-deploy-webpack": "pnpm run with-webpack pnpm run testonly-deploy-inner",
"testonly-deploy-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_BUILD=1 pnpm testonly-deploy-inner", "testonly-deploy-turbo": "pnpm run with-turbo pnpm testonly-deploy-inner",
"test-inner": "pnpm testheadless", "test-inner": "pnpm testheadless",
"test": "pnpm test-webpack", "test": "pnpm test-webpack",
"test-webpack": "pnpm run with-webpack pnpm run test-inner", "test-webpack": "pnpm run with-webpack pnpm run test-inner",
"test-rspack": "pnpm run with-rspack pnpm run test-inner", "test-rspack": "pnpm run with-rspack pnpm run test-inner",
"test-turbo": "cross-env IS_TURBOPACK_TEST=1 TURBOPACK_DEV=1 TURBOPACK_BUILD=1 pnpm test-inner", "test-turbo": "pnpm run with-turbo pnpm test-inner",
"testonly": "jest --runInBand", "testonly": "jest --runInBand",
"testheadless": "cross-env HEADLESS=true pnpm testonly", "testheadless": "cross-env HEADLESS=true pnpm testonly",
"genstats": "cross-env LOCAL_STATS=true node .github/actions/next-stats-action/src/index.js", "genstats": "cross-env LOCAL_STATS=true node .github/actions/next-stats-action/src/index.js",
...@@ -109,6 +109,7 @@ ...@@ -109,6 +109,7 @@
"build-storybook": "turbo run build-storybook", "build-storybook": "turbo run build-storybook",
"test-storybook": "turbo run test-storybook", "test-storybook": "turbo run test-storybook",
"with-rspack": "cross-env NEXT_RSPACK=1 NEXT_TEST_USE_RSPACK=1", "with-rspack": "cross-env NEXT_RSPACK=1 NEXT_TEST_USE_RSPACK=1",
"with-turbo": "cross-env IS_TURBOPACK_TEST=1",
"with-webpack": "cross-env IS_WEBPACK_TEST=1", "with-webpack": "cross-env IS_WEBPACK_TEST=1",
"with-experimental": "cross-env __NEXT_CACHE_COMPONENTS=true __NEXT_EXPERIMENTAL_APP_NEW_SCROLL_HANDLER=true __NEXT_EXPERIMENTAL_DEBUG_CHANNEL=true" "with-experimental": "cross-env __NEXT_CACHE_COMPONENTS=true __NEXT_EXPERIMENTAL_APP_NEW_SCROLL_HANDLER=true __NEXT_EXPERIMENTAL_DEBUG_CHANNEL=true"
}, },
......
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
// Only Turbopack runs the transform on the layout once in edge and non-edge contexts import { waitForRedbox } from 'next-test-utils'
// so we only test this on Turbopack
import {
waitForRedbox,
getRedboxDescription,
getRedboxSource,
} from 'next-test-utils'
// Filter out browser log lines from CLI output to avoid counting forwarded browser errors // Filter out browser log lines from CLI output to avoid counting forwarded browser errors
function filterBrowserLogs(output: string): string { function filterBrowserLogs(output: string): string {
...@@ -15,6 +9,9 @@ function filterBrowserLogs(output: string): string { ...@@ -15,6 +9,9 @@ function filterBrowserLogs(output: string): string {
.filter((line) => !line.includes('[browser]')) .filter((line) => !line.includes('[browser]'))
.join('\n') .join('\n')
} }
// Only Turbopack runs the transform on the layout once in edge and non-edge contexts
// so we only test this on Turbopack
;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)( ;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)(
'cache-components-edge-deduplication', 'cache-components-edge-deduplication',
() => { () => {
...@@ -38,24 +35,19 @@ function filterBrowserLogs(output: string): string { ...@@ -38,24 +35,19 @@ function filterBrowserLogs(output: string): string {
if (isNextDev) { if (isNextDev) {
const browser = await next.browser('/edge-with-layout/edge') const browser = await next.browser('/edge-with-layout/edge')
waitForRedbox(browser) waitForRedbox(browser)
const redbox = { await expect(browser).toDisplayRedbox(`
description: await getRedboxDescription(browser), {
source: await getRedboxSource(browser), "description": "Ecmascript file had an error",
} "environmentLabel": null,
expect(redbox.description).toMatchInlineSnapshot( "label": "Build Error",
`"Ecmascript file had an error"` "source": "./app/edge-with-layout/edge/page.tsx (1:14)
)
expect(redbox.source).toMatchInlineSnapshot(`
"./app/edge-with-layout/edge/page.tsx (1:14)
Ecmascript file had an error Ecmascript file had an error
> 1 | export const runtime = 'edge' > 1 | export const runtime = 'edge'
| ^^^^^^^ | ^^^^^^^",
2 | "stack": [],
3 | export default function Page() { }
4 | return <div>Test page under app/</div>
Route segment config "runtime" is not compatible with \`nextConfig.cacheComponents\`. Please remove it."
`) `)
// Count occurrences of the layout error at the specific location // Count occurrences of the layout error at the specific location
// Filter out browser logs to avoid counting forwarded browser errors // Filter out browser logs to avoid counting forwarded browser errors
const filteredOutput = filterBrowserLogs(next.cliOutput) const filteredOutput = filterBrowserLogs(next.cliOutput)
...@@ -63,7 +55,7 @@ function filterBrowserLogs(output: string): string { ...@@ -63,7 +55,7 @@ function filterBrowserLogs(output: string): string {
/\.\/app\/edge-with-layout\/layout\.tsx:1:14/g /\.\/app\/edge-with-layout\/layout\.tsx:1:14/g
) )
// We don't show an error stack, just the individual error messages at each location // We don't show an error stack, just the individual error messages at each location
expect(layoutErrorMatches?.length).toBe(1) expect(layoutErrorMatches.length).toBe(1)
} else { } else {
// Check that both the layout and edge page errors appear // Check that both the layout and edge page errors appear
expect(next.cliOutput).toContain('./app/edge-with-layout/layout.tsx') expect(next.cliOutput).toContain('./app/edge-with-layout/layout.tsx')
...@@ -80,7 +72,7 @@ function filterBrowserLogs(output: string): string { ...@@ -80,7 +72,7 @@ function filterBrowserLogs(output: string): string {
) )
// Should appear exactly twice: once in the formatted error message, once in the stack trace // Should appear exactly twice: once in the formatted error message, once in the stack trace
expect(layoutErrorMatches?.length).toBe(2) expect(layoutErrorMatches.length).toBe(2)
} }
}) })
} }
......
...@@ -57,7 +57,7 @@ describe('build trace with extra entries', () => { ...@@ -57,7 +57,7 @@ describe('build trace with extra entries', () => {
] ]
// Skip hello.json check for Turbopack as it doesn't support webpack entry modifications // Skip hello.json check for Turbopack as it doesn't support webpack entry modifications
if (!process.env.TURBOPACK_BUILD) { if (!process.env.IS_TURBOPACK_TEST) {
expect(tracedFiles.some((file) => file.endsWith('hello.json'))).toBe( expect(tracedFiles.some((file) => file.endsWith('hello.json'))).toBe(
true true
) )
......
...@@ -72,13 +72,13 @@ describe('build trace with extra entries', () => { ...@@ -72,13 +72,13 @@ describe('build trace with extra entries', () => {
) )
// Skip hello.json check for Turbopack as it doesn't support webpack entry modifications // Skip hello.json check for Turbopack as it doesn't support webpack entry modifications
if (!process.env.TURBOPACK_BUILD) { if (!process.env.IS_TURBOPACK_TEST) {
expect( expect(
appTrace.files.some((file) => file.endsWith('hello.json')) appTrace.files.some((file) => file.endsWith('hello.json'))
).toBe(true) ).toBe(true)
} }
// Skip lib/get-data.js check for Turbopack as it doesn't support webpack entry modifications // Skip lib/get-data.js check for Turbopack as it doesn't support webpack entry modifications
if (!process.env.TURBOPACK_BUILD) { if (!process.env.IS_TURBOPACK_TEST) {
expect( expect(
appTrace.files.some((file) => file.endsWith('lib/get-data.js')) appTrace.files.some((file) => file.endsWith('lib/get-data.js'))
).toBe(true) ).toBe(true)
......
...@@ -1489,9 +1489,12 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1489,9 +1489,12 @@ export function runTests(ctx: RunTestsCtx) {
it('should set cache-control to immutable for static images', async () => { it('should set cache-control to immutable for static images', async () => {
if (!ctx.isDev) { if (!ctx.isDev) {
const filename = 'test' const filename = fs
.readdirSync(join(ctx.appDir, '.next/static/media'))
.find((f) => /^test\.[0-9a-f]+\.jpg$/.test(f))
expect(filename).toBeString()
const query = { const query = {
url: `/_next/static/media/${filename}.fab2915d.jpg`, url: `/_next/static/media/${filename}`,
w: ctx.w, w: ctx.w,
q: ctx.q, q: ctx.q,
} }
...@@ -1504,7 +1507,7 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1504,7 +1507,7 @@ export function runTests(ctx: RunTestsCtx) {
) )
expect(res1.headers.get('Vary')).toBe('Accept') expect(res1.headers.get('Vary')).toBe('Accept')
expect(res1.headers.get('Content-Disposition')).toBe( expect(res1.headers.get('Content-Disposition')).toBe(
`${contentDispositionType}; filename="${filename}.webp"` `${contentDispositionType}; filename="test.webp"`
) )
await expectWidth(res1, ctx.w) await expectWidth(res1, ctx.w)
...@@ -1516,7 +1519,7 @@ export function runTests(ctx: RunTestsCtx) { ...@@ -1516,7 +1519,7 @@ export function runTests(ctx: RunTestsCtx) {
) )
expect(res2.headers.get('Vary')).toBe('Accept') expect(res2.headers.get('Vary')).toBe('Accept')
expect(res2.headers.get('Content-Disposition')).toBe( expect(res2.headers.get('Content-Disposition')).toBe(
`${contentDispositionType}; filename="${filename}.webp"` `${contentDispositionType}; filename="test.webp"`
) )
await expectWidth(res2, ctx.w) await expectWidth(res2, ctx.w)
} }
......
import { debugPrint, getFullUrl, waitFor } from 'next-test-utils' import { debugPrint, getFullUrl } from 'next-test-utils'
import os from 'os' import os from 'os'
import { import {
Playwright, Playwright,
...@@ -216,13 +216,5 @@ export default async function webdriver( ...@@ -216,13 +216,5 @@ export default async function webdriver(
debugPrint(`Hydration complete for ${fullUrl}`) debugPrint(`Hydration complete for ${fullUrl}`)
} }
// This is a temporary workaround for turbopack starting watching too late.
// So we delay file changes to give it some time
// to connect the WebSocket and start watching.
// TODO: Is this still needed? Can we wait in a more useful way, like a socket connection event?
if (process.env.IS_TURBOPACK_TEST && process.env.TURBOPACK_DEV) {
debugPrint(`Waiting for for turbopack watcher to start`)
await waitFor(1000)
}
return browser return browser
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment