Unverified Commit 99e09714 authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

CI: use KV for test timing data (#83745)

Next.js tests are grouped based on timing data to better distribute them
for parallelization.

Previously this relied on a gist file that we would write to in CI.
However, this has more recently started resulting in 401s.

The actual reason for the 401s is the Turbo task wasn't propagating the
required environment variable to the task. It was mostly working by
accident, because we would store timing data on disk for the runner,
which would have been satisfied by a job that ran before it.

However despite that, relying on a gist file to read/write timing data
didn't feel like the write abstraction. I decided to refactor it to use
KV instead.

This also makes the handling more resilient to missing timing data. We
shouldn't fail the build if we can't fetch timings, as there's already
handling to fallback to round robin.
parent bc84caf5
......@@ -100,7 +100,9 @@ env:
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
NEXT_JUNIT_TEST_REPORT: 'true'
DD_ENV: 'ci'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
# Vercel KV Store for test timings
KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
NEXT_TEST_JOB: 1
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
VERCEL_TEST_TEAM: vtest314-next-e2e-tests
......
......@@ -16,7 +16,9 @@ env:
# we build a dev binary for use in CI so skip downloading
# canary next-swc binaries in the monorepo
NEXT_SKIP_NATIVE_POSTINSTALL: 1
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
# Vercel KV Store for test timings
KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
NEXT_TEST_JOB: 1
NEXT_DISABLE_SWC_WASM: 1
......
......@@ -152,7 +152,7 @@
"@typescript-eslint/eslint-plugin": "^8.36.0",
"@typescript-eslint/parser": "^8.36.0",
"@vercel/devlow-bench": "workspace:*",
"@vercel/fetch": "6.1.1",
"@vercel/kv": "3.0.0",
"@vercel/og": "0.7.2",
"abort-controller": "3.0.0",
"alex": "9.1.0",
......
......@@ -225,9 +225,9 @@ importers:
'@vercel/devlow-bench':
specifier: workspace:*
version: link:turbopack/packages/devlow-bench
'@vercel/fetch':
specifier: 6.1.1
version: 6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13))
'@vercel/kv':
specifier: 3.0.0
version: 3.0.0
'@vercel/og':
specifier: 0.7.2
version: 0.7.2
......@@ -5463,9 +5463,6 @@ packages:
'@types/aria-query@5.0.1':
resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==}
'@types/async-retry@1.2.1':
resolution: {integrity: sha512-yMQ6CVgICWtyFNBqJT3zqOc+TnqqEPLo4nKJNPFwcialiylil38Ie6q1ENeFTjvaLOkVim9K5LisHgAKJWidGQ==}
'@types/async-retry@1.4.2':
resolution: {integrity: sha512-GUDuJURF0YiJZ+CBjNQA0+vbP/VHlJbB0sFqkzsV7EcOPRfurVonXpXKAt3w8qIjM1TEzpz6hc6POocPvHOS3w==}
......@@ -5686,9 +5683,6 @@ packages:
'@types/long@4.0.1':
resolution: {integrity: sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==}
'@types/lru-cache@4.1.1':
resolution: {integrity: sha512-8mNEUG6diOrI6pMqOHrHPDBB1JsrpedeMK9AWGzVCQ7StRRribiT9BRvUmF8aUws9iBbVlgVekOT5Sgzc1MTKw==}
'@types/mdast@3.0.10':
resolution: {integrity: sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==}
......@@ -5710,9 +5704,6 @@ packages:
'@types/mute-stream@0.0.4':
resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==}
'@types/node-fetch@2.3.2':
resolution: {integrity: sha512-yW0EOebSsQme9yKu09XbdDfle4/SmWZMK4dfteWcSLCYNQQcF+YOv0kIrvm+9pO11/ghA4E6A+RNQqvYj4Nr3A==}
'@types/node-fetch@2.6.1':
resolution: {integrity: sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==}
......@@ -6028,21 +6019,12 @@ packages:
'@ungap/structured-clone@1.2.0':
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
'@vercel/fetch-cached-dns@2.0.2':
resolution: {integrity: sha512-gDqKEV8CeY2YmCdZpP1rn3tFK1L07Vw2+HYkCK8zpRHOVGr/sP8yhBsW+C/yqGVj0i9z/rIvqIHe5emvRvxwgw==}
peerDependencies:
node-fetch: '*'
'@upstash/redis@1.35.3':
resolution: {integrity: sha512-hSjv66NOuahW3MisRGlSgoszU2uONAY2l5Qo3Sae8OT3/Tng9K+2/cBRuyPBX8egwEGcNNCF9+r0V6grNnhL+w==}
'@vercel/fetch-retry@5.0.3':
resolution: {integrity: sha512-DIIoBY92r+sQ6iHSf5WjKiYvkdsDIMPWKYATlE0KcUAj2RV6SZK9UWpUzBRKsofXqedOqpVjrI0IE6AWL7JRtg==}
peerDependencies:
node-fetch: '*'
'@vercel/fetch@6.1.1':
resolution: {integrity: sha512-nddCkgpA0aVIqOlzh+qVlzDNcQq0cSnqefM+x6SciGI4GCvVZeaZ7WEowgX8I/HwBAq8Uj5Bdnd+r0+sYsJsig==}
peerDependencies:
'@types/node-fetch': '2'
node-fetch: '2'
'@vercel/kv@3.0.0':
resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==}
engines: {node: '>=14.6'}
'@vercel/ncc@0.34.0':
resolution: {integrity: sha512-G9h5ZLBJ/V57Ou9vz5hI8pda/YQX5HQszCs3AmIus3XzsmRn/0Ptic5otD3xVST8QLKk7AMk7AqpsyQGN7MZ9A==}
......@@ -6162,9 +6144,6 @@ packages:
'@xtuc/long@4.2.2':
resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==}
'@zeit/dns-cached-resolve@2.1.2':
resolution: {integrity: sha512-A/5gbBskKPETTBqHwvlaW1Ri2orO62yqoFoXdxna1SQ7A/lXjpWgpJ1wdY3IQEcz5LydpS4sJ8SzI2gFyyLEhg==}
JSONStream@1.3.5:
resolution: {integrity: sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==}
hasBin: true
......@@ -6264,10 +6243,6 @@ packages:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
engines: {node: '>= 6.0.0'}
agentkeepalive@3.4.1:
resolution: {integrity: sha512-MPIwsZU9PP9kOrZpyu2042kYA8Fdt/AedQYkYXucHgF9QoD9dXVp0ypuGnHXSR0hTstBxdt85Xkh4JolYfK5wg==}
engines: {node: '>= 4.0.0'}
agentkeepalive@4.1.4:
resolution: {integrity: sha512-+V/rGa3EuU74H6wR04plBb7Ks10FbtUQgRj/FQOG7uUIEuaINI+AiqJR1k6t3SVNs7o7ZjIdus6706qqzVq8jQ==}
engines: {node: '>= 8.0.0'}
......@@ -6778,6 +6753,7 @@ packages:
binary-install@1.1.0:
resolution: {integrity: sha512-rkwNGW+3aQVSZoD0/o3mfPN6Yxh3Id0R/xzTVBVVpGNlVz8EGwusksxRlbk/A5iKTZt9zkMn3qIqmAt3vpfbzg==}
engines: {node: '>=10'}
deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
bindings@1.5.0:
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
......@@ -8867,6 +8843,7 @@ packages:
eslint-plugin-markdown@3.0.1:
resolution: {integrity: sha512-8rqoc148DWdGdmYF6WSQFT3uQ6PO7zXYgeBpHAOAakX/zpq+NvFYbDA/H7PYzHajwtmaOzAwfxyl++x0g1/N9A==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
deprecated: Please use @eslint/markdown instead
peerDependencies:
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
......@@ -13993,6 +13970,7 @@ packages:
engines: {node: '>=0.6.0', teleport: '>=0.2.0'}
deprecated: |-
You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.
(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)
qs@6.11.0:
......@@ -16072,6 +16050,9 @@ packages:
unbox-primitive@1.0.2:
resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==}
uncrypto@0.1.3:
resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==}
undici-types@6.19.8:
resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
......@@ -21388,8 +21369,6 @@ snapshots:
'@types/aria-query@5.0.1': {}
'@types/async-retry@1.2.1': {}
'@types/async-retry@1.4.2':
dependencies:
'@types/retry': 0.12.0
......@@ -21648,8 +21627,6 @@ snapshots:
'@types/long@4.0.1': {}
'@types/lru-cache@4.1.1': {}
'@types/mdast@3.0.10':
dependencies:
'@types/unist': 2.0.3
......@@ -21670,10 +21647,6 @@ snapshots:
dependencies:
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
'@types/node-fetch@2.3.2':
dependencies:
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
'@types/node-fetch@2.6.1':
dependencies:
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
......@@ -22113,31 +22086,13 @@ snapshots:
'@ungap/structured-clone@1.2.0': {}
'@vercel/fetch-cached-dns@2.0.2(node-fetch@2.6.7(encoding@0.1.13))':
'@upstash/redis@1.35.3':
dependencies:
'@types/node-fetch': 2.3.2
'@zeit/dns-cached-resolve': 2.1.2
node-fetch: 2.6.7(encoding@0.1.13)
uncrypto: 0.1.3
'@vercel/fetch-retry@5.0.3(node-fetch@2.6.7(encoding@0.1.13))':
'@vercel/kv@3.0.0':
dependencies:
async-retry: 1.3.1
debug: 3.2.7
node-fetch: 2.6.7(encoding@0.1.13)
transitivePeerDependencies:
- supports-color
'@vercel/fetch@6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13))':
dependencies:
'@types/async-retry': 1.2.1
'@types/node-fetch': 2.6.1
'@vercel/fetch-cached-dns': 2.0.2(node-fetch@2.6.7(encoding@0.1.13))
'@vercel/fetch-retry': 5.0.3(node-fetch@2.6.7(encoding@0.1.13))
agentkeepalive: 3.4.1
debug: 3.1.0
node-fetch: 2.6.7(encoding@0.1.13)
transitivePeerDependencies:
- supports-color
'@upstash/redis': 1.35.3
'@vercel/ncc@0.34.0': {}
......@@ -22331,14 +22286,6 @@ snapshots:
'@xtuc/long@4.2.2': {}
'@zeit/dns-cached-resolve@2.1.2':
dependencies:
'@types/async-retry': 1.2.1
'@types/lru-cache': 4.1.1
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
async-retry: 1.2.3
lru-cache: 5.1.1
JSONStream@1.3.5:
dependencies:
jsonparse: 1.3.1
......@@ -22423,10 +22370,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
agentkeepalive@3.4.1:
dependencies:
humanize-ms: 1.2.1
agentkeepalive@4.1.4:
dependencies:
debug: 4.1.1
......@@ -34731,6 +34674,8 @@ snapshots:
has-symbols: 1.0.3
which-boxed-primitive: 1.0.2
uncrypto@0.1.3: {}
undici-types@6.19.8: {}
undici@5.26.3:
......@@ -4,10 +4,7 @@ const path = require('path')
const _glob = require('glob')
const { existsSync } = require('fs')
const fsp = require('fs/promises')
const nodeFetch = require('node-fetch')
const vercelFetch = require('@vercel/fetch')
// @ts-expect-error
const fetch = vercelFetch(nodeFetch)
const { createClient } = require('@vercel/kv')
const { promisify } = require('util')
const { Sema } = require('async-sema')
const { spawn, exec: execOrig } = require('child_process')
......@@ -65,14 +62,47 @@ const shouldContinueTestsOnError = !!process.env.NEXT_TEST_CONTINUE_ON_ERROR
const skipRetryTestManifest = process.env.NEXT_TEST_SKIP_RETRY_MANIFEST
? require(process.env.NEXT_TEST_SKIP_RETRY_MANIFEST)
: []
const TIMINGS_API = `https://api.github.com/gists/4500dd89ae2f5d70d9aaceb191f528d1`
const TIMINGS_API_HEADERS = {
Accept: 'application/vnd.github.v3+json',
...(process.env.TEST_TIMINGS_TOKEN
? {
Authorization: `Bearer ${process.env.TEST_TIMINGS_TOKEN}`,
const KV_TIMINGS_KEY = 'test-timings'
const kvClient =
process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN
? createClient({
url: process.env.KV_REST_API_URL,
token: process.env.KV_REST_API_TOKEN,
})
: null
/**
* Retry a KV operation with exponential backoff
* @param {() => Promise<any>} operation - The async operation to retry
* @param {string} operationName - Name of the operation for logging
* @param {number} maxRetries - Maximum number of retries (default: 3)
* @returns {Promise<any>} The result of the operation
*/
async function retryKVOperation(operation, operationName, maxRetries = 3) {
let lastError
let retries = maxRetries
while (retries > 0) {
try {
return await operation()
} catch (err) {
lastError = err
retries--
if (retries > 0) {
const delay = (maxRetries - retries + 1) * 5 // 5s, 10s, 15s backoff
console.log(
`KV ${operationName} failed, retrying in ${delay}s. Error:`,
err.message
)
await new Promise((resolve) => setTimeout(resolve, delay * 1000))
}
: {}),
}
}
throw new Error(
`Failed to ${operationName} after ${maxRetries} retries: ${lastError?.message}`
)
}
const testFilters = {
......@@ -168,28 +198,19 @@ const isMatchingPattern = (pattern, file) => {
}
async function getTestTimings() {
let timingsRes
const doFetch = () =>
fetch(TIMINGS_API, {
headers: {
...TIMINGS_API_HEADERS,
},
})
timingsRes = await doFetch()
if (timingsRes.status === 403) {
const delay = 15
console.log(`Got 403 response waiting ${delay} seconds before retry`)
await new Promise((resolve) => setTimeout(resolve, delay * 1000))
timingsRes = await doFetch()
if (!kvClient) {
console.warn('KV client not configured, skipping timing fetch')
return null
}
if (!timingsRes.ok) {
throw new Error(`request status: ${timingsRes.status}`)
}
const timingsData = await timingsRes.json()
return JSON.parse(timingsData.files['test-timings.json'].content)
const timings = await retryKVOperation(async () => {
const data = await kvClient.get(KV_TIMINGS_KEY)
if (!data) {
console.log('No timing data found in KV store')
}
return data
}, 'fetch timings')
return timings || null
}
async function main() {
......@@ -243,6 +264,10 @@ async function main() {
process.env.NEXT_TEST_MODE
)
// Only fetch/update shared timing data during grouped CI runs to avoid
// individual test runs from polluting the timing data
const shouldUseSharedTimings = options.timings && options.group
/** @type TestFile[] */
let tests = argv._.filter((arg) =>
arg.toString().match(/\.test\.(js|ts|tsx)/)
......@@ -302,30 +327,45 @@ async function main() {
//
}
if (options.timings && options.group) {
if (shouldUseSharedTimings) {
console.log('Fetching previous timings data')
const timingsFile = path.join(process.cwd(), 'test-timings.json')
try {
const timingsFile = path.join(process.cwd(), 'test-timings.json')
try {
prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8'))
console.log('Loaded test timings from disk successfully')
} catch (_) {
console.error('failed to load from disk', _)
}
prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8'))
console.log('Loaded test timings from disk successfully')
} catch (_) {
console.error(
'Failed to load test timings from disk. Proceeding to fetch from KV store. Original error: ',
_
)
}
if (!prevTimings) {
if (!prevTimings) {
try {
prevTimings = await getTestTimings()
console.log('Fetched previous timings data successfully')
if (prevTimings) {
console.log('Fetched previous timings data successfully from KV')
} else {
console.log('No previous timings data available')
}
} catch (kvError) {
console.warn(
'Failed to fetch timings from KV, continuing without timing data:',
kvError.message
)
prevTimings = null
}
if (options.writeTimings) {
if (options.writeTimings) {
if (prevTimings) {
await fsp.writeFile(timingsFile, JSON.stringify(prevTimings))
console.log('Wrote previous timings data to', timingsFile)
await cleanUpAndExit(0)
} else {
console.log('No timings data to write')
}
await cleanUpAndExit(0)
}
} catch (err) {
console.log(`Failed to fetch timings data`, err)
await cleanUpAndExit(1)
}
}
......@@ -389,6 +429,14 @@ async function main() {
// tests tend not to get clustered together
tests = tests.filter((_value, idx) => idx % groupTotal === groupPos - 1)
console.log('Splitting without timings')
// Warn in CI that tests are not optimally distributed
if (process.env.GITHUB_ACTIONS) {
core.warning(
`Test timing data unavailable for group ${options.group}. Tests are being distributed round-robin, which may increase CI time. ` +
`Consider checking KV store connectivity if this persists.`
)
}
}
}
......@@ -773,43 +821,34 @@ ${ENDGROUP}`)
// junitData += `</testsuites>`
// console.log('output timing data to junit.xml')
if (prevTimings && process.env.TEST_TIMINGS_TOKEN) {
try {
const newTimings = {
...(await getTestTimings()),
...curTimings,
}
if (shouldUseSharedTimings) {
if (kvClient) {
try {
// Fetch existing timings and merge with new ones
const existingTimings = (await getTestTimings()) || {}
const newTimings = {
...existingTimings,
...curTimings,
}
for (const test of Object.keys(newTimings)) {
if (!existsSync(path.join(__dirname, test))) {
console.log('removing stale timing', test)
delete newTimings[test]
// Clean up stale timings for deleted tests
for (const test of Object.keys(newTimings)) {
if (!existsSync(path.join(__dirname, test))) {
console.log('removing stale timing', test)
delete newTimings[test]
}
}
}
const timingsRes = await fetch(TIMINGS_API, {
method: 'PATCH',
headers: {
...TIMINGS_API_HEADERS,
},
body: JSON.stringify({
files: {
'test-timings.json': {
content: JSON.stringify(newTimings),
},
},
}),
})
if (!timingsRes.ok) {
throw new Error(`request status: ${timingsRes.status}`)
// Update KV store with retries
await retryKVOperation(async () => {
await kvClient.set(KV_TIMINGS_KEY, newTimings)
console.log('Successfully updated test timings in KV store')
}, 'update timings')
} catch (err) {
console.log('Failed to update timings data', err)
}
const result = await timingsRes.json()
console.log(
`Sent updated timings successfully. API URL: "${result?.url}" HTML URL: "${result?.html_url}"`
)
} catch (err) {
console.log('Failed to update timings data', err)
} else {
console.warn('KV client not configured, skipping timing update')
}
}
}
......
......@@ -23,7 +23,8 @@
"//#typescript": {},
"//#get-test-timings": {
"inputs": ["run-tests.js"],
"outputs": ["test-timings.json"]
"outputs": ["test-timings.json"],
"env": ["KV_REST_API_URL", "KV_REST_API_TOKEN"]
}
},
"ui": "tui"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment