Commit 6b6a8edb authored by Jimmy Lai's avatar Jimmy Lai

stack: merge ci-coverage into cache-fix base

parents e3e435e2 7ef10df5
...@@ -95,6 +95,15 @@ Analyze PR status including CI failures and review comments. ...@@ -95,6 +95,15 @@ Analyze PR status including CI failures and review comments.
- **MEDIUM priority**: Identify root cause pattern, address open suggestions - **MEDIUM priority**: Identify root cause pattern, address open suggestions
- **LOW priority**: Mark as likely flaky/transient, note resolved/nitpick comments - **LOW priority**: Mark as likely flaky/transient, note resolved/nitpick comments
8. When proposing local repro commands, **always include the exact env vars from the CI job** (shown in the "Job Environment Variables" section of index.md). Key variables that change behavior:
- `IS_WEBPACK_TEST=1` forces webpack (turbopack is default locally)
- `NEXT_SKIP_ISOLATE=1` skips packing next.js into a separate project (hides module resolution failures)
- Feature flags like `__NEXT_USE_NODE_STREAMS=true`, `__NEXT_CACHE_COMPONENTS=true` change build-time DefinePlugin replacements
- Example: a failure in "test node streams prod" needs `IS_WEBPACK_TEST=1 __NEXT_USE_NODE_STREAMS=true __NEXT_CACHE_COMPONENTS=true __NEXT_EXPERIMENTAL_DEBUG_CHANNEL=true NEXT_TEST_MODE=start`
- Never use `NEXT_SKIP_ISOLATE=1` when verifying module resolution or build-time compilation fixes
9. The script automatically checks the last 3 main branch CI runs for known flaky tests. Check the **"Known Flaky Tests"** section in index.md and the `flaky-tests.json` file. Tests listed there also fail on main and are likely pre-existing flakes, not caused by the PR. Mark them as **FLAKY (pre-existing)** in your summary table. Use `--skip-flaky-check` to skip this step if it's too slow.
- Do not try to fix these failures or address review comments without user confirmation. - Do not try to fix these failures or address review comments without user confirmation.
- If failures would require complex analysis and there are multiple problems, only do some basic analysis and point out that further investigation is needed and could be performed when requested. - If failures would require complex analysis and there are multiple problems, only do some basic analysis and point out that further investigation is needed and could be performed when requested.
......
...@@ -1021,6 +1021,63 @@ jobs: ...@@ -1021,6 +1021,63 @@ jobs:
stepName: 'test-cache-components-prod-${{ matrix.group }}' stepName: 'test-cache-components-prod-${{ matrix.group }}'
secrets: inherit secrets: inherit
test-node-streams-dev:
name: test node streams dev
needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}
strategy:
fail-fast: false
matrix:
group: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6]
uses: ./.github/workflows/build_reusable.yml
with:
# Dev tests use Turbopack (default bundler) to cover turbopack code paths.
# Prod tests below use IS_WEBPACK_TEST=1 to cover webpack code paths.
afterBuild: |
export __NEXT_USE_NODE_STREAMS=true
export __NEXT_CACHE_COMPONENTS=true
export __NEXT_EXPERIMENTAL_DEBUG_CHANNEL=true
export NEXT_EXTERNAL_TESTS_FILTERS="test/cache-components-tests-manifest.json,test/use-node-streams-tests-manifest.json"
export NEXT_TEST_MODE=dev
export IS_TURBOPACK_TEST=1
export TURBOPACK_DEV=1
node run-tests.js \
--timings \
-g ${{ matrix.group }} \
--type development
stepName: 'test-node-streams-dev-${{ matrix.group }}'
secrets: inherit
test-node-streams-prod:
name: test node streams prod
needs: ['optimize-ci', 'changes', 'build-native', 'build-next']
if: ${{ needs.optimize-ci.outputs.skip == 'false' && needs.changes.outputs.docs-only == 'false' }}
strategy:
fail-fast: false
matrix:
group: [1/7, 2/7, 3/7, 4/7, 5/7, 6/7, 7/7]
uses: ./.github/workflows/build_reusable.yml
with:
# Prod tests use webpack (IS_WEBPACK_TEST=1) to cover webpack code paths.
# Dev tests above use Turbopack (default) to cover turbopack code paths.
afterBuild: |
export __NEXT_USE_NODE_STREAMS=true
export __NEXT_CACHE_COMPONENTS=true
export __NEXT_EXPERIMENTAL_DEBUG_CHANNEL=true
export NEXT_EXTERNAL_TESTS_FILTERS="test/cache-components-tests-manifest.json,test/use-node-streams-tests-manifest.json"
export NEXT_TEST_MODE=start
export IS_WEBPACK_TEST=1
node run-tests.js \
--timings \
-g ${{ matrix.group }} \
--type production
stepName: 'test-node-streams-prod-${{ matrix.group }}'
secrets: inherit
tests-pass: tests-pass:
needs: needs:
[ [
...@@ -1039,6 +1096,8 @@ jobs: ...@@ -1039,6 +1096,8 @@ jobs:
'test-cache-components-dev', 'test-cache-components-dev',
'test-cache-components-prod', 'test-cache-components-prod',
'test-cache-components-integration', 'test-cache-components-integration',
'test-node-streams-dev',
'test-node-streams-prod',
'test-cargo-unit', 'test-cargo-unit',
'rust-check', 'rust-check',
'rustdoc-check', 'rustdoc-check',
......
This diff is collapsed.
#!/bin/bash
# Benchmark script for comparing web streams vs node streams performance.
# Uses the minimal server (bench/next-minimal-server) for lowest overhead.
# Warms up with 50 requests, then runs two phases:
# Phase 1: 10s at concurrency=1 (single-client latency)
# Phase 2: 10s at concurrency=100 (throughput under load)
# Reports throughput and latency percentiles for each phase.
#
# Usage:
# ./benchmark.sh [duration] [warmup_requests]
#
# Defaults: 10s duration per phase, 50 warmup requests
set -euo pipefail
DURATION=${1:-10}
WARMUP_REQS=${2:-50}
PORT=3199
NEXT_BIN="../../packages/next/dist/bin/next"
MINIMAL_SERVER="../next-minimal-server/bin/minimal-server.js"
if ! command -v npx &>/dev/null; then
echo "npx is required (for autocannon)"
exit 1
fi
cleanup() {
lsof -ti :"$PORT" 2>/dev/null | xargs kill -9 2>/dev/null || true
}
trap cleanup EXIT
start_server() {
cleanup
sleep 0.5
PORT=$PORT node "$MINIMAL_SERVER" &>/dev/null &
SERVER_PID=$!
# Wait for server to be ready
local retries=0
while ! curl -sf "http://localhost:$PORT" >/dev/null 2>&1; do
retries=$((retries + 1))
if [ "$retries" -gt 30 ]; then
echo "ERROR: Server failed to start after 15s"
exit 1
fi
sleep 0.5
done
}
stop_server() {
kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
cleanup
sleep 1
}
warmup() {
echo " Warming up ($WARMUP_REQS requests)..."
for i in $(seq 1 "$WARMUP_REQS"); do
curl -sf "http://localhost:$PORT" >/dev/null 2>&1 || true
done
sleep 0.5
}
run_phase() {
local label="$1"
local connections="$2"
echo ""
echo " --- $label (${DURATION}s, c=$connections) ---"
local result
result=$(npx autocannon -d "$DURATION" -c "$connections" -j "http://localhost:$PORT" 2>/dev/null)
node -e "
const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));
const r = d.requests;
const l = d.latency;
console.log(' Throughput:');
console.log(' avg: ' + r.average + ' req/s');
console.log(' mean: ' + r.mean + ' req/s');
console.log(' total: ' + r.total + ' requests in ${DURATION}s');
console.log(' Latency:');
console.log(' avg: ' + l.average.toFixed(2) + ' ms');
console.log(' p50: ' + l.p50.toFixed(2) + ' ms');
console.log(' p90: ' + l.p90.toFixed(2) + ' ms');
console.log(' p99: ' + l.p99.toFixed(2) + ' ms');
console.log(' max: ' + l.max.toFixed(2) + ' ms');
" <<< "$result"
}
run_benchmark() {
local mode="$1"
echo ""
echo "============================================"
echo " $mode"
echo "============================================"
start_server
warmup
run_phase "Single client" 1
run_phase "Under load" 100
stop_server
}
echo "Benchmark: web streams vs node streams"
echo "======================================="
echo "Duration: ${DURATION}s per phase | Warmup: ${WARMUP_REQS} reqs"
echo "Server: minimal-server (minimalMode: true)"
# --- Web Streams (default) ---
cat > next.config.js <<'CONF'
module.exports = {}
CONF
echo ""
echo "Building (web streams)..."
node "$NEXT_BIN" build &>/dev/null
run_benchmark "Web Streams (default)"
# --- Node Streams ---
cat > next.config.js <<'CONF'
module.exports = {
experimental: {
useNodeStreams: true,
},
}
CONF
echo ""
echo "Building (node streams)..."
node "$NEXT_BIN" build &>/dev/null
run_benchmark "Node Streams (useNodeStreams: true)"
# Restore config
cat > next.config.js <<'CONF'
module.exports = {}
CONF
echo ""
echo "Done."
module.exports = { module.exports = {}
experimental: {
serverMinification: true,
},
}
...@@ -26,13 +26,13 @@ const nextServer = new NextServer({ ...@@ -26,13 +26,13 @@ const nextServer = new NextServer({
const requestHandler = nextServer.getRequestHandler() const requestHandler = nextServer.getRequestHandler()
const port = parseInt(process.env.PORT, 10) || 3000
require('http') require('http')
.createServer((req, res) => { .createServer((req, res) => {
console.time('next-request') return requestHandler(req, res)
return requestHandler(req, res).finally(() => {
console.timeEnd('next-request')
})
}) })
.listen(3000, () => { .listen(port, () => {
console.timeEnd('next-cold-start') console.timeEnd('next-cold-start')
console.log('Listening on port ' + port)
}) })
...@@ -83,6 +83,60 @@ function isBot(username) { ...@@ -83,6 +83,60 @@ function isBot(username) {
return username.endsWith('-bot') || username.endsWith('[bot]') return username.endsWith('-bot') || username.endsWith('[bot]')
} }
/**
* Parses the build_and_test.yml workflow to extract env vars from afterBuild
* sections. Returns a map of job display name prefix → env var list.
*/
function getJobEnvVarsFromWorkflow() {
const workflowPath = path.join(
__dirname,
'..',
'.github',
'workflows',
'build_and_test.yml'
)
try {
const content = require('fs').readFileSync(workflowPath, 'utf8')
const envMap = {}
// Match job blocks: " job-id:\n name: display name\n" ... "afterBuild: |"
const jobRegex =
/^ {2}([\w-]+):\s*\n\s+name:\s*(.+)\n[\s\S]*?afterBuild:\s*\|\n([\s\S]*?)(?=\n\s+stepName:)/gm
let match
while ((match = jobRegex.exec(content)) !== null) {
const displayName = match[2].trim()
const afterBuild = match[3]
const exports = []
for (const line of afterBuild.split('\n')) {
const exportMatch = line.match(
/^\s*export\s+([\w]+)=["']?([^"'\s]+)["']?/
)
if (exportMatch) {
exports.push(`${exportMatch[1]}=${exportMatch[2]}`)
}
}
if (exports.length > 0) {
envMap[displayName] = exports
}
}
return envMap
} catch {
return {}
}
}
/**
* Given a job name like "test node streams prod (4/7) / build" and the env map,
* returns the relevant env vars or null.
*/
function getEnvVarsForJob(jobName, envMap) {
for (const [prefix, vars] of Object.entries(envMap)) {
if (jobName.startsWith(prefix)) {
return vars
}
}
return null
}
// ============================================================================ // ============================================================================
// Data Fetching Functions // Data Fetching Functions
// ============================================================================ // ============================================================================
...@@ -413,7 +467,9 @@ function generateIndexMd( ...@@ -413,7 +467,9 @@ function generateIndexMd(
runMetadata, runMetadata,
categorizedJobs, categorizedJobs,
jobTestCounts, jobTestCounts,
reviewData reviewData,
jobEnvMap,
flakyTests
) { ) {
const { failed, inProgress, queued, succeeded, cancelled, skipped } = const { failed, inProgress, queued, succeeded, cancelled, skipped } =
categorizedJobs categorizedJobs
...@@ -493,6 +549,41 @@ function generateIndexMd( ...@@ -493,6 +549,41 @@ function generateIndexMd(
) )
} }
lines.push('') lines.push('')
// Show env vars for failed jobs if they differ from defaults
if (jobEnvMap && Object.keys(jobEnvMap).length > 0) {
const jobEnvGroups = new Map()
for (const job of failed) {
const envVars = getEnvVarsForJob(job.name, jobEnvMap)
if (envVars) {
const key = envVars.join(', ')
if (!jobEnvGroups.has(key)) {
jobEnvGroups.set(key, [])
}
jobEnvGroups.get(key).push(job.name)
}
}
if (jobEnvGroups.size > 0) {
lines.push('### Job Environment Variables', '')
for (const [envStr, jobNames] of jobEnvGroups) {
const prefix = jobNames[0].replace(/ \(.*/, '')
lines.push(`**${prefix}**: \`${envStr}\``, '')
}
}
}
// Known flaky tests section
if (flakyTests && flakyTests.size > 0) {
lines.push('### Known Flaky Tests (failing on 2+ branches)', '')
lines.push(
'These tests also failed in recent CI runs across multiple different branches and are likely pre-existing flakes, not caused by this PR:',
''
)
for (const testPath of [...flakyTests].sort()) {
lines.push(`- \`${testPath}\``)
}
lines.push('')
}
} }
// In-progress jobs section (only when CI is running) // In-progress jobs section (only when CI is running)
...@@ -838,6 +929,146 @@ function generateThreadMd(thread, index) { ...@@ -838,6 +929,146 @@ function generateThreadMd(thread, index) {
return lines.join('\n') return lines.join('\n')
} }
// ============================================================================
// Flaky Test Detection
// ============================================================================
/**
* Fetches recent failed CI runs across all branches and identifies tests that
* fail on multiple different branches (indicating flakiness, not branch-specific bugs).
* Excludes the current PR's branch to avoid self-matching.
* Returns a Set of test file paths that are likely flaky.
*/
async function getFlakyTests(currentBranch, runsToCheck = 5) {
console.log(
`Checking last ${runsToCheck} failed CI runs across all branches for known flaky tests...`
)
// Get recent failed build-and-test runs across ALL branches
const jqQuery = `.workflow_runs[] | select(.conclusion == "failure") | {id, head_branch}`
let output
try {
output = exec(
`gh api "repos/vercel/next.js/actions/workflows/57419851/runs?status=completed&per_page=30" --jq '${jqQuery}'`
)
} catch {
console.log(' Could not fetch CI runs, skipping flaky check')
return new Set()
}
if (!output.trim()) {
console.log(' No failed runs found')
return new Set()
}
// Filter out the current branch and take up to runsToCheck
const allRuns = output
.split('\n')
.filter((line) => line.trim())
.map((line) => JSON.parse(line))
.filter((run) => run.head_branch !== currentBranch)
.slice(0, runsToCheck)
if (allRuns.length === 0) {
console.log(' No failed runs from other branches found')
return new Set()
}
const branchCount = new Set(allRuns.map((r) => r.head_branch)).size
console.log(
` Checking ${allRuns.length} runs from ${branchCount} different branches...`
)
// Fetch failed jobs for all runs in parallel
const runJobResults = await Promise.all(
allRuns.map(async (run) => {
try {
const jobsJq = '.jobs[] | select(.conclusion == "failure") | {id, name}'
const jobsOutput = exec(
`gh api "repos/vercel/next.js/actions/runs/${run.id}/jobs?per_page=100" --jq '${jobsJq}'`
)
if (!jobsOutput.trim()) return { run, jobs: [] }
const jobs = jobsOutput
.split('\n')
.filter((line) => line.trim())
.map((line) => JSON.parse(line))
// Skip runs with 20+ failed jobs (likely systemic, not flaky)
if (jobs.length > 20) return { run, jobs: [] }
return { run, jobs }
} catch {
return { run, jobs: [] }
}
})
)
// Collect all (job, branch) pairs, then fetch logs in parallel (batch of 5)
const jobBranchPairs = []
for (const { run, jobs } of runJobResults) {
for (const job of jobs) {
jobBranchPairs.push({ job, branch: run.head_branch })
}
}
console.log(` Fetching logs for ${jobBranchPairs.length} failed jobs...`)
// Map: testPath → Set of branches where it failed
const testFailBranches = new Map()
// Process in batches of 5 to avoid overwhelming the API
const BATCH_SIZE = 5
for (let i = 0; i < jobBranchPairs.length; i += BATCH_SIZE) {
const batch = jobBranchPairs.slice(i, i + BATCH_SIZE)
const results = await Promise.all(
batch.map(async ({ job, branch }) => {
try {
const logs = exec(
`gh api "repos/vercel/next.js/actions/jobs/${job.id}/logs"`
)
return { logs, branch }
} catch {
return { logs: null, branch }
}
})
)
for (const { logs, branch } of results) {
if (!logs) continue
const testResults = extractTestOutputJson(logs)
for (const result of testResults) {
if (result.testResults) {
for (const tr of result.testResults) {
const hasFailed = tr.assertionResults?.some(
(a) => a.status === 'failed'
)
if (hasFailed) {
const shortPath = tr.name?.replace(/.*\/(test\/)/, '$1')
if (shortPath) {
if (!testFailBranches.has(shortPath)) {
testFailBranches.set(shortPath, new Set())
}
testFailBranches.get(shortPath).add(branch)
}
}
}
}
}
}
}
// A test is flaky if it fails on 2+ different branches
const flakyTestFiles = new Set()
for (const [testPath, branches] of testFailBranches) {
if (branches.size >= 2) {
flakyTestFiles.add(testPath)
}
}
console.log(
` Found ${flakyTestFiles.size} flaky tests (failing on 2+ different branches)`
)
return flakyTestFiles
}
// ============================================================================ // ============================================================================
// Main Function // Main Function
// ============================================================================ // ============================================================================
...@@ -972,7 +1203,8 @@ async function main() { ...@@ -972,7 +1203,8 @@ async function main() {
runMetadata, runMetadata,
emptyCategorizedJobs, emptyCategorizedJobs,
{}, {},
reviewData reviewData,
{}
) )
) )
process.exit(0) process.exit(0)
...@@ -1082,19 +1314,34 @@ async function main() { ...@@ -1082,19 +1314,34 @@ async function main() {
} }
} }
// Step 8: Generate index.md // Step 8: Check for known flaky tests across branches (skip with --skip-flaky-check)
let flakyTests = new Set()
if (!process.argv.includes('--skip-flaky-check')) {
flakyTests = await getFlakyTests(branchInfo.branchName, 5)
if (flakyTests.size > 0) {
await fs.writeFile(
path.join(OUTPUT_DIR, 'flaky-tests.json'),
JSON.stringify([...flakyTests].sort(), null, 2)
)
}
}
// Step 9: Generate index.md
console.log('Generating index.md...') console.log('Generating index.md...')
// Update categorizedJobs.failed with full processed metadata // Update categorizedJobs.failed with full processed metadata
const finalCategorizedJobs = { const finalCategorizedJobs = {
...categorizedJobs, ...categorizedJobs,
failed: processedFailedJobs, failed: processedFailedJobs,
} }
const jobEnvMap = getJobEnvVarsFromWorkflow()
const indexMd = generateIndexMd( const indexMd = generateIndexMd(
branchInfo, branchInfo,
runMetadata, runMetadata,
finalCategorizedJobs, finalCategorizedJobs,
jobTestCounts, jobTestCounts,
reviewData reviewData,
jobEnvMap,
flakyTests
) )
await fs.writeFile(path.join(OUTPUT_DIR, 'index.md'), indexMd) await fs.writeFile(path.join(OUTPUT_DIR, 'index.md'), indexMd)
......
{
"version": 2,
"suites": {},
"rules": {
"include": [
"test/e2e/**/*.test.{t,j}s{,x}",
"test/integration/app-*/**/*.test.{t,j}s{,x}",
"test/production/app-*/**/*.test.{t,j}s{,x}",
"test/development/app-*/**/*.test.{t,j}s{,x}",
"test/development/acceptance-app/**/*.test.{t,j}s{,x}"
],
"exclude": []
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment