Unverified Commit c7d18554 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

[backport] CI: use KV for test timing data (#83860)

Co-authored-by: 's avatarZack Tanner <1939140+ztanner@users.noreply.github.com>
parent d960fd03
......@@ -95,7 +95,9 @@ env:
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
NEXT_JUNIT_TEST_REPORT: 'true'
DD_ENV: 'ci'
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
# Vercel KV Store for test timings
KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
NEXT_TEST_JOB: 1
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
VERCEL_TEST_TEAM: vtest314-next-e2e-tests
......
......@@ -16,7 +16,9 @@ env:
# we build a dev binary for use in CI so skip downloading
# canary next-swc binaries in the monorepo
NEXT_SKIP_NATIVE_POSTINSTALL: 1
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
# Vercel KV Store for test timings
KV_REST_API_URL: ${{ secrets.KV_REST_API_URL }}
KV_REST_API_TOKEN: ${{ secrets.KV_REST_API_TOKEN }}
NEXT_TEST_JOB: 1
NEXT_DISABLE_SWC_WASM: 1
......
......@@ -145,7 +145,7 @@
"@typescript-eslint/eslint-plugin": "^8.36.0",
"@typescript-eslint/parser": "^8.36.0",
"@vercel/devlow-bench": "workspace:*",
"@vercel/fetch": "6.1.1",
"@vercel/kv": "3.0.0",
"@vercel/og": "0.7.2",
"abort-controller": "3.0.0",
"alex": "9.1.0",
......
......@@ -223,9 +223,9 @@ importers:
'@vercel/devlow-bench':
specifier: workspace:*
version: link:turbopack/packages/devlow-bench
'@vercel/fetch':
specifier: 6.1.1
version: 6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13))
'@vercel/kv':
specifier: 3.0.0
version: 3.0.0
'@vercel/og':
specifier: 0.7.2
version: 0.7.2
......@@ -5454,9 +5454,6 @@ packages:
'@types/aria-query@5.0.1':
resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==}
'@types/async-retry@1.2.1':
resolution: {integrity: sha512-yMQ6CVgICWtyFNBqJT3zqOc+TnqqEPLo4nKJNPFwcialiylil38Ie6q1ENeFTjvaLOkVim9K5LisHgAKJWidGQ==}
'@types/async-retry@1.4.2':
resolution: {integrity: sha512-GUDuJURF0YiJZ+CBjNQA0+vbP/VHlJbB0sFqkzsV7EcOPRfurVonXpXKAt3w8qIjM1TEzpz6hc6POocPvHOS3w==}
......@@ -5677,9 +5674,6 @@ packages:
'@types/long@4.0.1':
resolution: {integrity: sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==}
'@types/lru-cache@4.1.1':
resolution: {integrity: sha512-8mNEUG6diOrI6pMqOHrHPDBB1JsrpedeMK9AWGzVCQ7StRRribiT9BRvUmF8aUws9iBbVlgVekOT5Sgzc1MTKw==}
'@types/mdast@3.0.10':
resolution: {integrity: sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==}
......@@ -5701,9 +5695,6 @@ packages:
'@types/mute-stream@0.0.4':
resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==}
'@types/node-fetch@2.3.2':
resolution: {integrity: sha512-yW0EOebSsQme9yKu09XbdDfle4/SmWZMK4dfteWcSLCYNQQcF+YOv0kIrvm+9pO11/ghA4E6A+RNQqvYj4Nr3A==}
'@types/node-fetch@2.6.1':
resolution: {integrity: sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==}
......@@ -6019,21 +6010,12 @@ packages:
'@ungap/structured-clone@1.2.0':
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
'@vercel/fetch-cached-dns@2.0.2':
resolution: {integrity: sha512-gDqKEV8CeY2YmCdZpP1rn3tFK1L07Vw2+HYkCK8zpRHOVGr/sP8yhBsW+C/yqGVj0i9z/rIvqIHe5emvRvxwgw==}
peerDependencies:
node-fetch: '*'
'@upstash/redis@1.35.3':
resolution: {integrity: sha512-hSjv66NOuahW3MisRGlSgoszU2uONAY2l5Qo3Sae8OT3/Tng9K+2/cBRuyPBX8egwEGcNNCF9+r0V6grNnhL+w==}
'@vercel/fetch-retry@5.0.3':
resolution: {integrity: sha512-DIIoBY92r+sQ6iHSf5WjKiYvkdsDIMPWKYATlE0KcUAj2RV6SZK9UWpUzBRKsofXqedOqpVjrI0IE6AWL7JRtg==}
peerDependencies:
node-fetch: '*'
'@vercel/fetch@6.1.1':
resolution: {integrity: sha512-nddCkgpA0aVIqOlzh+qVlzDNcQq0cSnqefM+x6SciGI4GCvVZeaZ7WEowgX8I/HwBAq8Uj5Bdnd+r0+sYsJsig==}
peerDependencies:
'@types/node-fetch': '2'
node-fetch: '2'
'@vercel/kv@3.0.0':
resolution: {integrity: sha512-pKT8fRnfyYk2MgvyB6fn6ipJPCdfZwiKDdw7vB+HL50rjboEBHDVBEcnwfkEpVSp2AjNtoaOUH7zG+bVC/rvSg==}
engines: {node: '>=14.6'}
'@vercel/ncc@0.34.0':
resolution: {integrity: sha512-G9h5ZLBJ/V57Ou9vz5hI8pda/YQX5HQszCs3AmIus3XzsmRn/0Ptic5otD3xVST8QLKk7AMk7AqpsyQGN7MZ9A==}
......@@ -6153,9 +6135,6 @@ packages:
'@xtuc/long@4.2.2':
resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==}
'@zeit/dns-cached-resolve@2.1.2':
resolution: {integrity: sha512-A/5gbBskKPETTBqHwvlaW1Ri2orO62yqoFoXdxna1SQ7A/lXjpWgpJ1wdY3IQEcz5LydpS4sJ8SzI2gFyyLEhg==}
JSONStream@1.3.5:
resolution: {integrity: sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==}
hasBin: true
......@@ -6255,10 +6234,6 @@ packages:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
engines: {node: '>= 6.0.0'}
agentkeepalive@3.4.1:
resolution: {integrity: sha512-MPIwsZU9PP9kOrZpyu2042kYA8Fdt/AedQYkYXucHgF9QoD9dXVp0ypuGnHXSR0hTstBxdt85Xkh4JolYfK5wg==}
engines: {node: '>= 4.0.0'}
agentkeepalive@4.1.4:
resolution: {integrity: sha512-+V/rGa3EuU74H6wR04plBb7Ks10FbtUQgRj/FQOG7uUIEuaINI+AiqJR1k6t3SVNs7o7ZjIdus6706qqzVq8jQ==}
engines: {node: '>= 8.0.0'}
......@@ -6769,6 +6744,7 @@ packages:
binary-install@1.1.0:
resolution: {integrity: sha512-rkwNGW+3aQVSZoD0/o3mfPN6Yxh3Id0R/xzTVBVVpGNlVz8EGwusksxRlbk/A5iKTZt9zkMn3qIqmAt3vpfbzg==}
engines: {node: '>=10'}
deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
bindings@1.5.0:
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
......@@ -8858,6 +8834,7 @@ packages:
eslint-plugin-markdown@3.0.1:
resolution: {integrity: sha512-8rqoc148DWdGdmYF6WSQFT3uQ6PO7zXYgeBpHAOAakX/zpq+NvFYbDA/H7PYzHajwtmaOzAwfxyl++x0g1/N9A==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
deprecated: Please use @eslint/markdown instead
peerDependencies:
eslint: ^6.0.0 || ^7.0.0 || ^8.0.0
......@@ -16069,6 +16046,9 @@ packages:
unbox-primitive@1.0.2:
resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==}
uncrypto@0.1.3:
resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==}
undici-types@6.19.8:
resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
......@@ -21377,8 +21357,6 @@ snapshots:
'@types/aria-query@5.0.1': {}
'@types/async-retry@1.2.1': {}
'@types/async-retry@1.4.2':
dependencies:
'@types/retry': 0.12.0
......@@ -21637,8 +21615,6 @@ snapshots:
'@types/long@4.0.1': {}
'@types/lru-cache@4.1.1': {}
'@types/mdast@3.0.10':
dependencies:
'@types/unist': 2.0.3
......@@ -21659,10 +21635,6 @@ snapshots:
dependencies:
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
'@types/node-fetch@2.3.2':
dependencies:
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
'@types/node-fetch@2.6.1':
dependencies:
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
......@@ -22102,31 +22074,13 @@ snapshots:
'@ungap/structured-clone@1.2.0': {}
'@vercel/fetch-cached-dns@2.0.2(node-fetch@2.6.7(encoding@0.1.13))':
'@upstash/redis@1.35.3':
dependencies:
'@types/node-fetch': 2.3.2
'@zeit/dns-cached-resolve': 2.1.2
node-fetch: 2.6.7(encoding@0.1.13)
uncrypto: 0.1.3
'@vercel/fetch-retry@5.0.3(node-fetch@2.6.7(encoding@0.1.13))':
'@vercel/kv@3.0.0':
dependencies:
async-retry: 1.3.1
debug: 3.2.7
node-fetch: 2.6.7(encoding@0.1.13)
transitivePeerDependencies:
- supports-color
'@vercel/fetch@6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7(encoding@0.1.13))':
dependencies:
'@types/async-retry': 1.2.1
'@types/node-fetch': 2.6.1
'@vercel/fetch-cached-dns': 2.0.2(node-fetch@2.6.7(encoding@0.1.13))
'@vercel/fetch-retry': 5.0.3(node-fetch@2.6.7(encoding@0.1.13))
agentkeepalive: 3.4.1
debug: 3.1.0
node-fetch: 2.6.7(encoding@0.1.13)
transitivePeerDependencies:
- supports-color
'@upstash/redis': 1.35.3
'@vercel/ncc@0.34.0': {}
......@@ -22320,14 +22274,6 @@ snapshots:
'@xtuc/long@4.2.2': {}
'@zeit/dns-cached-resolve@2.1.2':
dependencies:
'@types/async-retry': 1.2.1
'@types/lru-cache': 4.1.1
'@types/node': 20.17.6(patch_hash=rvl3vkomen3tospgr67bzubfyu)
async-retry: 1.2.3
lru-cache: 5.1.1
JSONStream@1.3.5:
dependencies:
jsonparse: 1.3.1
......@@ -22412,10 +22358,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
agentkeepalive@3.4.1:
dependencies:
humanize-ms: 1.2.1
agentkeepalive@4.1.4:
dependencies:
debug: 4.1.1
......@@ -34722,6 +34664,8 @@ snapshots:
has-symbols: 1.0.3
which-boxed-primitive: 1.0.2
uncrypto@0.1.3: {}
undici-types@6.19.8: {}
undici@5.26.3:
......@@ -4,10 +4,7 @@ const path = require('path')
const _glob = require('glob')
const { existsSync } = require('fs')
const fsp = require('fs/promises')
const nodeFetch = require('node-fetch')
const vercelFetch = require('@vercel/fetch')
// @ts-expect-error
const fetch = vercelFetch(nodeFetch)
const { createClient } = require('@vercel/kv')
const { promisify } = require('util')
const { Sema } = require('async-sema')
const { spawn, exec: execOrig } = require('child_process')
......@@ -65,14 +62,47 @@ const shouldContinueTestsOnError = !!process.env.NEXT_TEST_CONTINUE_ON_ERROR
const skipRetryTestManifest = process.env.NEXT_TEST_SKIP_RETRY_MANIFEST
? require(process.env.NEXT_TEST_SKIP_RETRY_MANIFEST)
: []
const TIMINGS_API = `https://api.github.com/gists/4500dd89ae2f5d70d9aaceb191f528d1`
const TIMINGS_API_HEADERS = {
Accept: 'application/vnd.github.v3+json',
...(process.env.TEST_TIMINGS_TOKEN
? {
Authorization: `Bearer ${process.env.TEST_TIMINGS_TOKEN}`,
const KV_TIMINGS_KEY = 'test-timings'
const kvClient =
process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN
? createClient({
url: process.env.KV_REST_API_URL,
token: process.env.KV_REST_API_TOKEN,
})
: null
/**
* Retry a KV operation with exponential backoff
* @param {() => Promise<any>} operation - The async operation to retry
* @param {string} operationName - Name of the operation for logging
* @param {number} maxRetries - Maximum number of retries (default: 3)
* @returns {Promise<any>} The result of the operation
*/
async function retryKVOperation(operation, operationName, maxRetries = 3) {
let lastError
let retries = maxRetries
while (retries > 0) {
try {
return await operation()
} catch (err) {
lastError = err
retries--
if (retries > 0) {
const delay = (maxRetries - retries + 1) * 5 // 5s, 10s, 15s backoff
console.log(
`KV ${operationName} failed, retrying in ${delay}s. Error:`,
err.message
)
await new Promise((resolve) => setTimeout(resolve, delay * 1000))
}
: {}),
}
}
throw new Error(
`Failed to ${operationName} after ${maxRetries} retries: ${lastError?.message}`
)
}
const testFilters = {
......@@ -168,28 +198,19 @@ const isMatchingPattern = (pattern, file) => {
}
async function getTestTimings() {
let timingsRes
const doFetch = () =>
fetch(TIMINGS_API, {
headers: {
...TIMINGS_API_HEADERS,
},
})
timingsRes = await doFetch()
if (timingsRes.status === 403) {
const delay = 15
console.log(`Got 403 response waiting ${delay} seconds before retry`)
await new Promise((resolve) => setTimeout(resolve, delay * 1000))
timingsRes = await doFetch()
if (!kvClient) {
console.warn('KV client not configured, skipping timing fetch')
return null
}
if (!timingsRes.ok) {
throw new Error(`request status: ${timingsRes.status}`)
}
const timingsData = await timingsRes.json()
return JSON.parse(timingsData.files['test-timings.json'].content)
const timings = await retryKVOperation(async () => {
const data = await kvClient.get(KV_TIMINGS_KEY)
if (!data) {
console.log('No timing data found in KV store')
}
return data
}, 'fetch timings')
return timings || null
}
async function main() {
......@@ -243,6 +264,10 @@ async function main() {
process.env.NEXT_TEST_MODE
)
// Only fetch/update shared timing data during grouped CI runs to avoid
// individual test runs from polluting the timing data
const shouldUseSharedTimings = options.timings && options.group
/** @type TestFile[] */
let tests = argv._.filter((arg) =>
arg.toString().match(/\.test\.(js|ts|tsx)/)
......@@ -302,30 +327,45 @@ async function main() {
//
}
if (options.timings && options.group) {
if (shouldUseSharedTimings) {
console.log('Fetching previous timings data')
const timingsFile = path.join(process.cwd(), 'test-timings.json')
try {
const timingsFile = path.join(process.cwd(), 'test-timings.json')
try {
prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8'))
console.log('Loaded test timings from disk successfully')
} catch (_) {
console.error('failed to load from disk', _)
}
prevTimings = JSON.parse(await fsp.readFile(timingsFile, 'utf8'))
console.log('Loaded test timings from disk successfully')
} catch (_) {
console.error(
'Failed to load test timings from disk. Proceeding to fetch from KV store. Original error: ',
_
)
}
if (!prevTimings) {
if (!prevTimings) {
try {
prevTimings = await getTestTimings()
console.log('Fetched previous timings data successfully')
if (prevTimings) {
console.log('Fetched previous timings data successfully from KV')
} else {
console.log('No previous timings data available')
}
} catch (kvError) {
console.warn(
'Failed to fetch timings from KV, continuing without timing data:',
kvError.message
)
prevTimings = null
}
if (options.writeTimings) {
if (options.writeTimings) {
if (prevTimings) {
await fsp.writeFile(timingsFile, JSON.stringify(prevTimings))
console.log('Wrote previous timings data to', timingsFile)
await cleanUpAndExit(0)
} else {
console.log('No timings data to write')
}
await cleanUpAndExit(0)
}
} catch (err) {
console.log(`Failed to fetch timings data`, err)
await cleanUpAndExit(1)
}
}
......@@ -389,6 +429,14 @@ async function main() {
// tests tend not to get clustered together
tests = tests.filter((_value, idx) => idx % groupTotal === groupPos - 1)
console.log('Splitting without timings')
// Warn in CI that tests are not optimally distributed
if (process.env.GITHUB_ACTIONS) {
core.warning(
`Test timing data unavailable for group ${options.group}. Tests are being distributed round-robin, which may increase CI time. ` +
`Consider checking KV store connectivity if this persists.`
)
}
}
}
......@@ -774,43 +822,34 @@ ${ENDGROUP}`)
// junitData += `</testsuites>`
// console.log('output timing data to junit.xml')
if (prevTimings && process.env.TEST_TIMINGS_TOKEN) {
try {
const newTimings = {
...(await getTestTimings()),
...curTimings,
}
if (shouldUseSharedTimings) {
if (kvClient) {
try {
// Fetch existing timings and merge with new ones
const existingTimings = (await getTestTimings()) || {}
const newTimings = {
...existingTimings,
...curTimings,
}
for (const test of Object.keys(newTimings)) {
if (!existsSync(path.join(__dirname, test))) {
console.log('removing stale timing', test)
delete newTimings[test]
// Clean up stale timings for deleted tests
for (const test of Object.keys(newTimings)) {
if (!existsSync(path.join(__dirname, test))) {
console.log('removing stale timing', test)
delete newTimings[test]
}
}
}
const timingsRes = await fetch(TIMINGS_API, {
method: 'PATCH',
headers: {
...TIMINGS_API_HEADERS,
},
body: JSON.stringify({
files: {
'test-timings.json': {
content: JSON.stringify(newTimings),
},
},
}),
})
if (!timingsRes.ok) {
throw new Error(`request status: ${timingsRes.status}`)
// Update KV store with retries
await retryKVOperation(async () => {
await kvClient.set(KV_TIMINGS_KEY, newTimings)
console.log('Successfully updated test timings in KV store')
}, 'update timings')
} catch (err) {
console.log('Failed to update timings data', err)
}
const result = await timingsRes.json()
console.log(
`Sent updated timings successfully. API URL: "${result?.url}" HTML URL: "${result?.html_url}"`
)
} catch (err) {
console.log('Failed to update timings data', err)
} else {
console.warn('KV client not configured, skipping timing update')
}
}
}
......
......@@ -23,7 +23,8 @@
"//#typescript": {},
"//#get-test-timings": {
"inputs": ["run-tests.js"],
"outputs": ["test-timings.json"]
"outputs": ["test-timings.json"],
"env": ["KV_REST_API_URL", "KV_REST_API_TOKEN"]
}
},
"ui": "tui"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment