Unverified Commit b0aeccca authored by Justin Ridgewell's avatar Justin Ridgewell Committed by GitHub

Allow filtering individual test cases inside test files (#55786)

### What?

Updates `run-test.js` to allow running individual test cases inside a test file.

### Why?

So that we can dramatically increase Turbopack's test coverage. Turbopack has implemented most (but not all) necessary features from the webpack bundles. But a single failing test case would prevent us from running any case inside a test file. With case filtering, we're able to run the cases we know will pass and prevent regressions on those.

### How?

Case filtering is only exposed via the `NEXT_EXTERNAL_TESTS_FILTERS` ENV, which points to a JSON file containing a map of test files with the test cases inside those files that are known to pass.

The known-passing test cases can be updated after Turbopack's daily integration test run by running `test/build-turbopack-tests-manifest.js`, which will update the `test/turbopack-tests-manifest.json` manifest.

Closes WEB-1640
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 8f7d24fa
...@@ -33,9 +33,6 @@ jobs: ...@@ -33,9 +33,6 @@ jobs:
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
with: with:
skipInstallBuild: 'yes' skipInstallBuild: 'yes'
# Ensure all of the tests listed in turbopack test manifest actually exists;
# i.e after removing an actual test and if it still stays in the manifest
afterBuild: node -e 'require(\"./test/turbopack-tests-manifest.js\").verifyEnabledTestPath()'
secrets: inherit secrets: inherit
build-next: build-next:
...@@ -137,7 +134,7 @@ jobs: ...@@ -137,7 +134,7 @@ jobs:
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
with: with:
skipForDocsOnly: 'yes' skipForDocsOnly: 'yes'
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.js" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/development)/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/development)/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY}
secrets: inherit secrets: inherit
test-turbopack-integration: test-turbopack-integration:
...@@ -151,7 +148,7 @@ jobs: ...@@ -151,7 +148,7 @@ jobs:
with: with:
nodeVersion: 16 nodeVersion: 16
skipForDocsOnly: 'yes' skipForDocsOnly: 'yes'
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.js" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration
secrets: inherit secrets: inherit
# --type production also runs tests/e2e # --type production also runs tests/e2e
...@@ -166,7 +163,7 @@ jobs: ...@@ -166,7 +163,7 @@ jobs:
with: with:
nodeVersion: 16 nodeVersion: 16
skipForDocsOnly: 'yes' skipForDocsOnly: 'yes'
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.js" TURBOPACK=1 NEXT_TEST_MODE=start node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type production afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_TEST_MODE=start node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type production
secrets: inherit secrets: inherit
test-next-swc-wasm: test-next-swc-wasm:
......
This diff is collapsed.
const fetch = require('node-fetch')
const fs = require('fs')
const override = process.argv.includes('--override')
// TODO: Switch to nextjs-integration-test-data branch once https://github.com/vercel/turbo/pull/5999 is merged.
const RESULT_URL =
'https://raw.githubusercontent.com/vercel/turbo/nextjs-integration-test-data/test-results/main/nextjs-test-results.json'
const PASSING_JSON_PATH = `${__dirname}/turbopack-tests-manifest.json`
const WORKING_PATH = '/home/runner/work/turbo/turbo/'
const INITIALIZING_TEST_CASES = [
'compile successfully',
'should build successfully',
]
const SKIPPED_TEST_SUITES = new Set([
'test/integration/router-rerender/test/index.test.js',
'test/e2e/basepath.test.ts',
'test/development/acceptance-app/ReactRefreshRequire.test.ts',
'test/integration/dynamic-routing/test/middleware.test.js',
'test/integration/css/test/css-modules.test.js',
'test/development/acceptance/ReactRefreshRequire.test.ts',
'test/integration/custom-routes/test/index.test.js',
'test/integration/absolute-assetprefix/test/index.test.js',
'test/e2e/middleware-rewrites/test/index.test.ts',
])
async function updatePassingTests() {
const passing = { __proto__: null }
const res = await fetch(RESULT_URL)
const results = await res.json()
for (const result of results.result) {
const runtimeError = result.data.numRuntimeErrorTestSuites > 0
for (const testResult of result.data.testResults) {
const filepath = stripWorkingPath(testResult.name)
for (const file of duplicateFileNames(filepath)) {
if (SKIPPED_TEST_SUITES.has(file)) continue
const fileResults = (passing[file] ??= {
passed: [],
failed: [],
pending: [],
runtimeError,
})
let initializationFailed = false
for (const testCase of testResult.assertionResults) {
let { fullName, status } = testCase
if (
status === 'failed' &&
INITIALIZING_TEST_CASES.some((name) => fullName.includes(name))
) {
initializationFailed = true
} else if (initializationFailed) {
status = 'failed'
}
const statusArray = fileResults[status]
if (!statusArray) {
throw new Error(`unexpected status "${status}"`)
}
statusArray.push(fullName)
}
}
}
}
for (const info of Object.values(passing)) {
info.failed = [...new Set(info.failed)]
info.pending = [...new Set(info.pending)]
info.passed = [
...new Set(info.passed.filter((name) => !info.failed.includes(name))),
]
}
if (!override) {
const oldPassingData = JSON.parse(
fs.readFileSync(PASSING_JSON_PATH, 'utf8')
)
for (const file of Object.keys(oldPassingData)) {
const newData = passing[file]
const oldData = oldPassingData[file]
if (!newData) continue
// We only want to keep test cases from the old data that are still exiting
oldData.passed = oldData.passed.filter(
(name) => newData.failed.includes(name) || newData.passed.includes(name)
)
// Grab test cases that passed before, but fail now
const shouldPass = new Set(
oldData.passed.filter((name) => newData.failed.includes(name))
)
if (shouldPass.size > 0) {
const list = JSON.stringify([...shouldPass], 0, 2)
console.log(
`${file} has ${shouldPass.size} test(s) that should pass but failed: ${list}`
)
}
// Merge the old passing tests with the new ones
newData.passed = [...new Set([...oldData.passed, ...newData.passed])]
// but remove them also from the failed list
newData.failed = newData.failed.filter((name) => !shouldPass.has(name))
}
}
fs.writeFileSync(PASSING_JSON_PATH, JSON.stringify(passing, null, 2))
}
function stripWorkingPath(path) {
if (!path.startsWith(WORKING_PATH)) {
throw new Error(
`found unexpected working path in "${path}", expected it to begin with ${WORKING_PATH}`
)
}
return path.slice(WORKING_PATH.length)
}
function duplicateFileNames(path) {
if (path.includes('/src/')) {
const dist = path.replace('/src/', '/dist/').replace(/.tsx?$/, '.js')
if (fs.existsSync(`${__dirname}/../${dist}`)) {
return [path, dist]
}
}
return [path]
}
updatePassingTests()
...@@ -31,6 +31,9 @@ describe('Build Output', () => { ...@@ -31,6 +31,9 @@ describe('Build Output', () => {
)} };` )} };`
) )
} }
;({ stdout } = await nextBuild(appDir, [], {
stdout: true,
}))
}) })
if (hasExperimentalConfig) { if (hasExperimentalConfig) {
...@@ -40,10 +43,6 @@ describe('Build Output', () => { ...@@ -40,10 +43,6 @@ describe('Build Output', () => {
} }
it('should not include internal pages', async () => { it('should not include internal pages', async () => {
;({ stdout } = await nextBuild(appDir, [], {
stdout: true,
}))
expect(stdout).toMatch(/\/ (.* )?\d{1,} B/) expect(stdout).toMatch(/\/ (.* )?\d{1,} B/)
expect(stdout).toMatch(/\+ First Load JS shared by all [ 0-9.]* kB/) expect(stdout).toMatch(/\+ First Load JS shared by all [ 0-9.]* kB/)
expect(stdout).toMatch(/ chunks\/main-[0-9a-z]{16}\.js [ 0-9.]* kB/) expect(stdout).toMatch(/ chunks\/main-[0-9a-z]{16}\.js [ 0-9.]* kB/)
......
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment