Unverified Commit fc73de6c authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

test: expire stale test timings after 30 days instead of deleting them

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent c8b481fe
......@@ -250,8 +250,68 @@ async function retryKVOperation(operation, operationName, maxRetries = 3) {
// field = test file, value = seconds), so parallel shards update individual
// fields atomically instead of overwriting each other's read-modify-write.
// `test-timings-v2:variants` lists the variants that have been recorded.
//
// Entries are kept for TIMINGS_TTL_SECONDS after their last write, because
// branches whose checkout differs from the writer's (older or newer) may
// still rely on them:
// - `test-timings-v2-updated:<variant>` records, per test file, when its
// timing was last written (epoch ms). A file missing from the writer's
// checkout is only deleted once that is longer ago than the TTL.
// - Both per-variant keys get the TTL as a Redis expiry, refreshed on every
// write, so a variant no CI job writes any more expires on its own; the
// fetch prunes it from the index afterwards.
const KV_TIMINGS_V2_PREFIX = 'test-timings-v2:'
const KV_TIMINGS_V2_VARIANTS_KEY = `${KV_TIMINGS_V2_PREFIX}variants`
const KV_TIMINGS_V2_UPDATED_PREFIX = 'test-timings-v2-updated:'
const TIMINGS_TTL_SECONDS = 30 * 24 * 60 * 60
// Atomically handles stale-timing candidates (files missing from the
// writer's checkout), re-reading each on the server so that an entry
// refreshed by a concurrent writer is never deleted:
// - timing gone in the meantime: nothing to do
// - no (valid) last-write timestamp: start the TTL clock now
// - last written before the cutoff: delete timing and timestamp
// KEYS: timings hash, last-write hash. ARGV: now (ms), cutoff (ms), TTL (s),
// files... Returns the deleted files.
const CLEANUP_STALE_TIMINGS_SCRIPT = `
local deleted = {}
local cutoff = tonumber(ARGV[2])
for i = 4, #ARGV do
local file = ARGV[i]
if redis.call('HEXISTS', KEYS[1], file) == 1 then
local updated = tonumber(redis.call('HGET', KEYS[2], file) or '')
if not updated then
redis.call('HSET', KEYS[2], file, ARGV[1])
elseif updated < cutoff then
redis.call('HDEL', KEYS[1], file)
redis.call('HDEL', KEYS[2], file)
deleted[#deleted + 1] = file
end
end
end
if redis.call('TTL', KEYS[2]) == -1 then
redis.call('EXPIRE', KEYS[2], ARGV[3])
end
return deleted
`
// Atomically removes a variant from the index if its timings hash no longer
// exists (expired), checked at removal time so a variant created
// concurrently stays indexed. Otherwise gives both per-variant keys the TTL
// if they have none yet (keys written before TTLs existed) without extending
// an existing one, so reading never keeps a variant alive.
// KEYS: index set, timings hash, last-write hash. ARGV: variant, TTL (s).
const MAINTAIN_VARIANT_SCRIPT = `
if redis.call('EXISTS', KEYS[2]) == 0 then
return redis.call('SREM', KEYS[1], ARGV[1])
end
for i = 2, 3 do
if redis.call('TTL', KEYS[i]) == -1 then
redis.call('EXPIRE', KEYS[i], ARGV[2])
end
end
return 0
`
/**
* Identifies the CI variant a run's timings belong to, from the env vars that
......@@ -315,7 +375,8 @@ function computeMaxTimings(variants) {
/**
* Normalizes loaded timings data. Accepts the versioned format written by
* `fetchTestTimings`, as well as a legacy flat `{ [file]: seconds }` map (or
* the `{}` the workflow writes when no timings could be fetched).
* the `{}` the workflow writes when no timings could be fetched). The
* cross-variant `max` is always derived from `variants`, never stored.
*
* @param {unknown} data
* @returns {TestTimings | null}
......@@ -339,10 +400,7 @@ function normalizeTimings(data) {
}
return {
variants,
max:
record.max && typeof record.max === 'object'
? record.max
: computeMaxTimings(variants),
max: computeMaxTimings(variants),
legacy:
record.legacy && typeof record.legacy === 'object' ? record.legacy : {},
}
......@@ -393,7 +451,8 @@ function toTimingsMap(hash) {
/**
* Fetches the timings of every variant, plus the legacy shared map as a
* fallback, in the versioned format consumed by `normalizeTimings`.
* Returns `null` when no timings are available at all.
* Returns `null` when no timings are available at all. Also prunes expired
* variants from the index (see MAINTAIN_VARIANT_SCRIPT).
*
* @param {any} client KV client
* @param {typeof retryKVOperation} retry
......@@ -419,6 +478,22 @@ async function fetchTestTimings(client, retry) {
if (Object.keys(variantTimings).length > 0) {
result[name] = variantTimings
}
try {
await client.eval(
MAINTAIN_VARIANT_SCRIPT,
[
KV_TIMINGS_V2_VARIANTS_KEY,
`${KV_TIMINGS_V2_PREFIX}${name}`,
`${KV_TIMINGS_V2_UPDATED_PREFIX}${name}`,
],
[name, String(TIMINGS_TTL_SECONDS)]
)
} catch (err) {
console.log(
`Failed to maintain timings of variant ${name}:`,
err.message
)
}
}
return result
}, 'fetch variant timings')
......@@ -440,17 +515,12 @@ async function fetchTestTimings(client, retry) {
console.log('No timing data found in KV store')
return null
}
return {
version: 2,
variants,
max: computeMaxTimings(variants),
legacy,
}
return { version: 2, variants, legacy }
}
/**
* Records this run's timings for its variant. Only touches the variant's own
* hash and the variant index; other variants and the legacy key are left as
* keys and the variant index; other variants and the legacy key are left as
* they are.
*
* @param {any} client KV client
......@@ -458,30 +528,60 @@ async function fetchTestTimings(client, retry) {
* @param {string} variant
* @param {Record<string, number>} curTimings seconds per test file
* @param {(file: string) => boolean} fileExists
* @param {number} [now] epoch ms
*/
async function writeTestTimings(
client,
retry,
variant,
curTimings,
fileExists
fileExists,
now = Date.now()
) {
const key = `${KV_TIMINGS_V2_PREFIX}${variant}`
if (Object.keys(curTimings).length > 0) {
await retry(async () => {
await client.hset(key, curTimings)
await client.sadd(KV_TIMINGS_V2_VARIANTS_KEY, variant)
}, 'update timings')
const updatedKey = `${KV_TIMINGS_V2_UPDATED_PREFIX}${variant}`
const files = Object.keys(curTimings)
if (files.length > 0) {
const updated = Object.fromEntries(files.map((file) => [file, now]))
await retry(
() =>
client
.multi()
.hset(key, curTimings)
.hset(updatedKey, updated)
.sadd(KV_TIMINGS_V2_VARIANTS_KEY, variant)
.expire(key, TIMINGS_TTL_SECONDS)
.expire(updatedKey, TIMINGS_TTL_SECONDS)
.exec(),
'update timings'
)
console.log(`Successfully updated test timings for variant ${variant}`)
}
// Clean up stale timings for deleted tests, within this variant only
// Timings of files missing from this checkout are only deleted once no
// branch has written them for TIMINGS_TTL_SECONDS (within this variant)
const recorded =
(await retry(() => client.hgetall(key), 'read timings')) || {}
const stale = Object.keys(recorded).filter((file) => !fileExists(file))
if (stale.length > 0) {
for (const file of stale) console.log('removing stale timing', file)
await retry(() => client.hdel(key, ...stale), 'remove stale timings')
const candidates = Object.keys(recorded).filter((file) => !fileExists(file))
if (candidates.length > 0) {
/** @type {string[] | null} */
const deleted = await retry(
() =>
client.eval(
CLEANUP_STALE_TIMINGS_SCRIPT,
[key, updatedKey],
[
String(now),
String(now - TIMINGS_TTL_SECONDS * 1000),
String(TIMINGS_TTL_SECONDS),
...candidates,
]
),
'clean up stale timings'
)
for (const file of deleted || []) {
console.log('removing stale timing', file)
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment