Unverified Commit 90f022ae authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Update test config (#43661)

Updates our test config to take advantage of more concurrency and also
updates to leverage the playwright docker image to reduce flakes from
actions/setup-node stalling or playwright dependencies stalling on `apt
install`.

This reduces our test times from upwards of 30 minutes down to back
around 15 minutes.
parent 2fcc1e2c
......@@ -261,15 +261,8 @@ jobs:
fail-fast: false
matrix:
node: [16, 18]
group: [1, 2]
group: [1, 2, 3, 4]
steps:
- name: Setup node
uses: actions/setup-node@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
node-version: ${{ matrix.node }}
check-latest: true
- run: echo ${{needs.build.outputs.docsChange}}
# https://github.com/actions/virtual-environments/issues/1187
......@@ -289,16 +282,7 @@ jobs:
name: next-swc-test-binary
path: packages/next-swc/native
- run: npm i -g pnpm@${PNPM_VERSION}
if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}
# - run: npx @replayio/playwright install chromium
# if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: node run-tests.js --type development --timings -g ${{ matrix.group }}/2
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type development --timings -g ${{ matrix.group }}/4 >> /proc/1/fd/1"
name: Run test/development
if: ${{needs.build.outputs.docsChange == 'nope'}}
# env:
......@@ -338,15 +322,8 @@ jobs:
fail-fast: false
matrix:
node: [16, 18]
group: [1, 2, 3]
group: [1, 2, 3, 4, 5, 6, 7]
steps:
- name: Setup node
uses: actions/setup-node@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
node-version: ${{ matrix.node }}
check-latest: true
- run: echo ${{needs.build.outputs.docsChange}}
# https://github.com/actions/virtual-environments/issues/1187
......@@ -366,16 +343,7 @@ jobs:
name: next-swc-test-binary
path: packages/next-swc/native
- run: npm i -g pnpm@${PNPM_VERSION}
if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}
# - run: npx @replayio/playwright install chromium
# if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/7 >> /proc/1/fd/1"
name: Run test/e2e (dev)
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
......@@ -415,15 +383,8 @@ jobs:
fail-fast: false
matrix:
node: [16, 18]
group: [1, 2]
group: [1, 2, 3]
steps:
- name: Setup node
uses: actions/setup-node@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
node-version: ${{ matrix.node }}
check-latest: true
- run: echo ${{needs.build.outputs.docsChange}}
# https://github.com/actions/virtual-environments/issues/1187
......@@ -443,16 +404,7 @@ jobs:
name: next-swc-test-binary
path: packages/next-swc/native
- run: npm i -g pnpm@${PNPM_VERSION}
if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}
# - run: npx @replayio/playwright install chromium
# if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: node run-tests.js --type production --timings -g ${{ matrix.group }}/2
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type production --timings -g ${{ matrix.group }}/3 >> /proc/1/fd/1"
name: Run test/production
if: ${{needs.build.outputs.docsChange == 'nope'}}
# env:
......@@ -482,15 +434,8 @@ jobs:
fail-fast: false
matrix:
node: [16, 18]
group: [1, 2, 3]
group: [1, 2, 3, 4, 5, 6, 7]
steps:
- name: Setup node
uses: actions/setup-node@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
node-version: ${{ matrix.node }}
check-latest: true
- run: echo ${{needs.build.outputs.docsChange}}
# https://github.com/actions/virtual-environments/issues/1187
......@@ -510,16 +455,7 @@ jobs:
name: next-swc-test-binary
path: packages/next-swc/native
- run: npm i -g pnpm@${PNPM_VERSION}
if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}
# - run: npx @replayio/playwright install chromium
# if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/7 >> /proc/1/fd/1"
name: Run test/e2e (production)
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
......@@ -571,15 +507,13 @@ jobs:
18,
19,
20,
21,
22,
23,
24,
25,
]
steps:
- name: Setup node
uses: actions/setup-node@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
node-version: 16
check-latest: true
- run: echo ${{needs.build.outputs.docsChange}}
# https://github.com/actions/virtual-environments/issues/1187
......@@ -599,16 +533,7 @@ jobs:
name: next-swc-test-binary
path: packages/next-swc/native
- run: npm i -g pnpm@${PNPM_VERSION}
if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}
# - run: npx @replayio/playwright install chromium
# if: ${{needs.build.outputs.docsChange == 'nope'}}
- run: xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/20
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v16 | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/25 >> /proc/1/fd/1"
if: ${{needs.build.outputs.docsChange == 'nope'}}
# env:
# RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testIntegration / Group ${{ matrix.group }}
......
......@@ -57,6 +57,7 @@
"@edge-runtime/jest-environment": "2.0.0",
"@fullhuman/postcss-purgecss": "1.3.0",
"@mdx-js/loader": "^1.5.1",
"@mdx-js/react": "^1.6.18",
"@next/bundle-analyzer": "workspace:*",
"@next/env": "workspace:*",
"@next/eslint-plugin-next": "workspace:*",
......@@ -114,7 +115,9 @@
"alex": "9.1.0",
"amphtml-validator": "1.0.35",
"async-sema": "3.0.1",
"body-parser": "1.20.1",
"browserslist": "4.20.2",
"buffer": "5.6.0",
"chalk": "5.0.1",
"cheerio": "0.22.0",
"cookie": "0.4.1",
......@@ -140,6 +143,7 @@
"express": "4.17.0",
"faker": "5.5.3",
"faunadb": "2.6.1",
"find-up": "4.1.0",
"firebase": "7.14.5",
"flat": "5.0.2",
"form-data": "4.0.0",
......@@ -149,15 +153,18 @@
"gzip-size": "5.1.1",
"html-validator": "5.1.18",
"husky": "8.0.0",
"http-proxy": "1.18.1",
"image-size": "0.9.3",
"is-animated": "2.0.2",
"isomorphic-unfetch": "3.0.0",
"jest": "27.0.6",
"jest-extended": "1.2.1",
"json5": "2.2.1",
"ky": "0.19.1",
"ky-universal": "0.6.0",
"lerna": "4.0.0",
"lint-staged": "10.1.7",
"lodash": "4.17.20",
"lost": "8.3.1",
"minimatch": "3.0.4",
"moment": "^2.24.0",
......@@ -197,7 +204,9 @@
"selenium-webdriver": "4.0.0-beta.4",
"semver": "7.3.7",
"shell-quote": "1.7.3",
"strip-ansi": "6.0.0",
"styled-components": "6.0.0-beta.5",
"styled-jsx": "5.1.0",
"styled-jsx-plugin-postcss": "3.0.2",
"swr": "2.0.0-rc.0",
"tailwindcss": "1.1.3",
......@@ -206,9 +215,12 @@
"tsec": "0.2.1",
"turbo": "1.6.3",
"typescript": "4.8.2",
"unfetch": "4.2.0",
"wait-port": "0.2.2",
"whatwg-fetch": "3.0.0",
"webpack": "5.74.0",
"webpack-bundle-analyzer": "4.7.0"
"webpack-bundle-analyzer": "4.7.0",
"ws": "8.2.3"
},
"resolutions": {
"browserslist": "4.20.2",
......
This diff is collapsed.
......@@ -2,7 +2,6 @@ import { createNext } from 'e2e-utils'
import { NextInstance } from 'test/lib/next-modes/base'
import { check, renderViaHTTP } from 'next-test-utils'
import webdriver from 'next-webdriver'
// @ts-expect-error missing types
import stripAnsi from 'strip-ansi'
describe('typescript-auto-install', () => {
......
......@@ -301,22 +301,29 @@ describe('create next app', () => {
it('should exit if the folder is not writable', async () => {
await useTempDir(async (cwd) => {
const projectName = 'not-writable'
// if the folder isn't able to be write restricted we can't test
// this so skip
if (
await fs
.writeFile(path.join(cwd, 'test'), 'hello')
.then(() => true)
.catch(() => false)
) {
console.warn(
`Test folder is not write restricted skipping write permission test`
)
return
}
const res = await run([projectName, '--js', '--eslint'], {
cwd,
reject: false,
})
if (process.platform === 'win32') {
expect(res.exitCode).toBe(0)
const files = ['package.json']
projectFilesShouldExist({ cwd, projectName, files })
return
}
expect(res.exitCode).toBe(1)
expect(res.stderr).toMatch(
/you do not have write permissions for this folder/
)
expect(res.exitCode).toBe(1)
}, 0o500)
})
......
......@@ -20,7 +20,7 @@ class App extends React.Component {
const { refHeight } = this.state
return (
<div ref={this.divRef}>
<div ref={this.divRef} style={{ height: 76, overflow: 'hidden' }}>
<h1>DOM Ref test using 9.2.0</h1>
<code id="ref-val">{`this component is ${refHeight}px tall`}</code>
</div>
......
......@@ -290,6 +290,7 @@ describe('Prerender Preview Mode', () => {
it('should return cookies to be expired after dev server reboot', async () => {
await killApp(app)
appPort = await findPort()
app = await launchApp(appDir, appPort)
const res = await fetchViaHTTP(
......
......@@ -92,6 +92,7 @@ export default function EvilPage(): JSX.Element {
}
`
)
appPort = await findPort()
app = await launchApp(appDir, appPort)
const $ = await get$('/hello')
......
......@@ -17,7 +17,7 @@ export async function useTempDir(
await fs.mkdirp(folder)
if (mode) {
fs.chmod(folder, mode)
await fs.chmod(folder, mode)
}
try {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment