Unverified Commit 02c02a47 authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

split out flake detection from utility to detect changed tests (#67611)

This splits out the existing logic for detecting changed/added tests
into a separate util, so it can be leveraged by #67612.

No changes in functionality, aside from replacing informational logs
with `console.log` rather than `console.error` and a tweak to arg
parsing.
parent 1309deea
...@@ -291,7 +291,7 @@ jobs: ...@@ -291,7 +291,7 @@ jobs:
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
with: with:
afterBuild: node scripts/test-new-tests.mjs --dev-mode --group ${{ matrix.group }} afterBuild: node scripts/test-new-tests.mjs --flake-detection --mode dev --group ${{ matrix.group }}
stepName: 'test-new-tests-dev-${{matrix.group}}' stepName: 'test-new-tests-dev-${{matrix.group}}'
secrets: inherit secrets: inherit
...@@ -308,7 +308,7 @@ jobs: ...@@ -308,7 +308,7 @@ jobs:
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
with: with:
afterBuild: node scripts/test-new-tests.mjs --prod-mode --group ${{ matrix.group }} afterBuild: node scripts/test-new-tests.mjs --flake-detection --mode start --group ${{ matrix.group }}
stepName: 'test-new-tests-start-${{matrix.group}}' stepName: 'test-new-tests-start-${{matrix.group}}'
secrets: inherit secrets: inherit
......
// @ts-check
import fs from 'fs/promises'
import execa from 'execa'
import path from 'path'
/**
* Detects changed tests files by comparing the current branch with `origin/canary`
* Returns tests separated by test mode (dev/prod)
*/
export default async function getChangedTests() {
let eventData = {}
/** @type import('execa').Options */
const EXECA_OPTS = { shell: true }
/** @type import('execa').Options */
const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' }
try {
eventData =
JSON.parse(
await fs.readFile(process.env.GITHUB_EVENT_PATH || '', 'utf8')
)['pull_request'] || {}
} catch (_) {}
const branchName =
eventData?.head?.ref ||
process.env.GITHUB_REF_NAME ||
(await execa('git rev-parse --abbrev-ref HEAD', EXECA_OPTS)).stdout
const remoteUrl =
eventData?.head?.repo?.full_name ||
process.env.GITHUB_REPOSITORY ||
(await execa('git remote get-url origin', EXECA_OPTS)).stdout
const isCanary =
branchName.trim() === 'canary' && remoteUrl.includes('vercel/next.js')
if (isCanary) {
console.log(`Skipping flake detection for canary`)
return { devTests: [], prodTests: [] }
}
try {
await execa('git remote set-branches --add origin canary', EXECA_OPTS_STDIO)
await execa('git fetch origin canary --depth=20', EXECA_OPTS_STDIO)
} catch (err) {
console.error(await execa('git remote -v', EXECA_OPTS_STDIO))
console.error(`Failed to fetch origin/canary`, err)
}
const changesResult = await execa(
`git diff origin/canary --name-only`,
EXECA_OPTS
).catch((err) => {
console.error(err)
return { stdout: '', stderr: '' }
})
console.log(
{
branchName,
remoteUrl,
isCanary,
},
`\ngit diff:\n${changesResult.stderr}\n${changesResult.stdout}`
)
const changedFiles = changesResult.stdout.split('\n')
// run each test 3 times in each test mode (if E2E) with no-retrying
// and if any fail it's flakey
const devTests = []
const prodTests = []
for (let file of changedFiles) {
// normalize slashes
file = file.replace(/\\/g, '/')
const fileExists = await fs
.access(path.join(process.cwd(), file), fs.constants.F_OK)
.then(() => true)
.catch(() => false)
if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) {
if (file.startsWith('test/e2e/')) {
devTests.push(file)
prodTests.push(file)
} else if (file.startsWith('test/prod')) {
prodTests.push(file)
} else if (file.startsWith('test/development')) {
devTests.push(file)
}
}
}
console.log(
'Detected tests:',
JSON.stringify(
{
devTests,
prodTests,
},
null,
2
)
)
return { devTests, prodTests }
}
// @ts-check // @ts-check
import fs from 'fs/promises'
import execa from 'execa' import execa from 'execa'
import path from 'path'
import yargs from 'yargs' import yargs from 'yargs'
import getChangedTests from './get-changed-tests.mjs'
/**
* Run tests for added/changed tests in the current branch
* CLI Options:
* --mode: test mode (dev, deploy, start)
* --group: current group number / total groups
* --flake-detection: run tests multiple times to detect flaky
*/
async function main() { async function main() {
let argv = await yargs(process.argv.slice(2)) let argv = await yargs(process.argv.slice(2))
.boolean('dev-mode') .string('mode')
.string('group').argv .string('group')
.boolean('flake-detection').argv
let testMode = argv.mode
const attempts = argv['flake-detection'] ? 3 : 1
if (testMode && !['dev', 'deploy', 'start'].includes(testMode)) {
throw new Error(
`Invalid test mode: ${testMode}. Must be one of: dev, deploy, start`
)
}
let testMode = argv.devMode ? 'dev' : 'start'
const rawGroup = argv['group'] const rawGroup = argv['group']
let currentGroup = 1 let currentGroup = 1
let groupTotal = 1 let groupTotal = 1
...@@ -20,101 +35,12 @@ async function main() { ...@@ -20,101 +35,12 @@ async function main() {
.map((item) => Number(item)) .map((item) => Number(item))
} }
let eventData = {}
/** @type import('execa').Options */ /** @type import('execa').Options */
const EXECA_OPTS = { shell: true } const EXECA_OPTS = { shell: true }
/** @type import('execa').Options */ /** @type import('execa').Options */
const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' } const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' }
try { const { devTests, prodTests } = await getChangedTests()
eventData =
JSON.parse(
await fs.readFile(process.env.GITHUB_EVENT_PATH || '', 'utf8')
)['pull_request'] || {}
} catch (_) {}
// detect changed test files
const branchName =
eventData?.head?.ref ||
process.env.GITHUB_REF_NAME ||
(await execa('git rev-parse --abbrev-ref HEAD', EXECA_OPTS)).stdout
const remoteUrl =
eventData?.head?.repo?.full_name ||
process.env.GITHUB_REPOSITORY ||
(await execa('git remote get-url origin', EXECA_OPTS)).stdout
const isCanary =
branchName.trim() === 'canary' && remoteUrl.includes('vercel/next.js')
if (isCanary) {
console.error(`Skipping flake detection for canary`)
return
}
try {
await execa('git remote set-branches --add origin canary', EXECA_OPTS_STDIO)
await execa('git fetch origin canary --depth=20', EXECA_OPTS_STDIO)
} catch (err) {
console.error(await execa('git remote -v', EXECA_OPTS_STDIO))
console.error(`Failed to fetch origin/canary`, err)
}
const changesResult = await execa(
`git diff origin/canary --name-only`,
EXECA_OPTS
).catch((err) => {
console.error(err)
return { stdout: '', stderr: '' }
})
console.error(
{
branchName,
remoteUrl,
isCanary,
testMode,
},
`\ngit diff:\n${changesResult.stderr}\n${changesResult.stdout}`
)
const changedFiles = changesResult.stdout.split('\n')
// run each test 3 times in each test mode (if E2E) with no-retrying
// and if any fail it's flakey
const devTests = []
const prodTests = []
for (let file of changedFiles) {
// normalize slashes
file = file.replace(/\\/g, '/')
const fileExists = await fs
.access(path.join(process.cwd(), file), fs.constants.F_OK)
.then(() => true)
.catch(() => false)
if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) {
if (file.startsWith('test/e2e/')) {
devTests.push(file)
prodTests.push(file)
} else if (file.startsWith('test/prod')) {
prodTests.push(file)
} else if (file.startsWith('test/development')) {
devTests.push(file)
}
}
}
console.log(
'Detected tests:',
JSON.stringify(
{
devTests,
prodTests,
},
null,
2
)
)
let currentTests = testMode === 'dev' ? devTests : prodTests let currentTests = testMode === 'dev' ? devTests : prodTests
...@@ -153,8 +79,8 @@ async function main() { ...@@ -153,8 +79,8 @@ async function main() {
const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0'] const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0']
for (let i = 0; i < 3; i++) { for (let i = 0; i < attempts; i++) {
console.log(`\n\nRun ${i + 1} for ${testMode} tests`) console.log(`\n\nRun ${i + 1}/${attempts} for ${testMode} tests`)
await execa('node', [...RUN_TESTS_ARGS, ...currentTests], { await execa('node', [...RUN_TESTS_ARGS, ...currentTests], {
...EXECA_OPTS_STDIO, ...EXECA_OPTS_STDIO,
env: { env: {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment