Unverified Commit fc73de6c authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

test: expire stale test timings after 30 days instead of deleting them

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent c8b481fe
...@@ -250,8 +250,68 @@ async function retryKVOperation(operation, operationName, maxRetries = 3) { ...@@ -250,8 +250,68 @@ async function retryKVOperation(operation, operationName, maxRetries = 3) {
// field = test file, value = seconds), so parallel shards update individual // field = test file, value = seconds), so parallel shards update individual
// fields atomically instead of overwriting each other's read-modify-write. // fields atomically instead of overwriting each other's read-modify-write.
// `test-timings-v2:variants` lists the variants that have been recorded. // `test-timings-v2:variants` lists the variants that have been recorded.
//
// Entries are kept for TIMINGS_TTL_SECONDS after their last write, because
// branches whose checkout differs from the writer's (older or newer) may
// still rely on them:
// - `test-timings-v2-updated:<variant>` records, per test file, when its
// timing was last written (epoch ms). A file missing from the writer's
// checkout is only deleted once that is longer ago than the TTL.
// - Both per-variant keys get the TTL as a Redis expiry, refreshed on every
// write, so a variant no CI job writes any more expires on its own; the
// fetch prunes it from the index afterwards.
const KV_TIMINGS_V2_PREFIX = 'test-timings-v2:' const KV_TIMINGS_V2_PREFIX = 'test-timings-v2:'
const KV_TIMINGS_V2_VARIANTS_KEY = `${KV_TIMINGS_V2_PREFIX}variants` const KV_TIMINGS_V2_VARIANTS_KEY = `${KV_TIMINGS_V2_PREFIX}variants`
const KV_TIMINGS_V2_UPDATED_PREFIX = 'test-timings-v2-updated:'
const TIMINGS_TTL_SECONDS = 30 * 24 * 60 * 60
// Atomically handles stale-timing candidates (files missing from the
// writer's checkout), re-reading each on the server so that an entry
// refreshed by a concurrent writer is never deleted:
// - timing gone in the meantime: nothing to do
// - no (valid) last-write timestamp: start the TTL clock now
// - last written before the cutoff: delete timing and timestamp
// KEYS: timings hash, last-write hash. ARGV: now (ms), cutoff (ms), TTL (s),
// files... Returns the deleted files.
const CLEANUP_STALE_TIMINGS_SCRIPT = `
local deleted = {}
local cutoff = tonumber(ARGV[2])
for i = 4, #ARGV do
local file = ARGV[i]
if redis.call('HEXISTS', KEYS[1], file) == 1 then
local updated = tonumber(redis.call('HGET', KEYS[2], file) or '')
if not updated then
redis.call('HSET', KEYS[2], file, ARGV[1])
elseif updated < cutoff then
redis.call('HDEL', KEYS[1], file)
redis.call('HDEL', KEYS[2], file)
deleted[#deleted + 1] = file
end
end
end
if redis.call('TTL', KEYS[2]) == -1 then
redis.call('EXPIRE', KEYS[2], ARGV[3])
end
return deleted
`
// Atomically removes a variant from the index if its timings hash no longer
// exists (expired), checked at removal time so a variant created
// concurrently stays indexed. Otherwise gives both per-variant keys the TTL
// if they have none yet (keys written before TTLs existed) without extending
// an existing one, so reading never keeps a variant alive.
// KEYS: index set, timings hash, last-write hash. ARGV: variant, TTL (s).
const MAINTAIN_VARIANT_SCRIPT = `
if redis.call('EXISTS', KEYS[2]) == 0 then
return redis.call('SREM', KEYS[1], ARGV[1])
end
for i = 2, 3 do
if redis.call('TTL', KEYS[i]) == -1 then
redis.call('EXPIRE', KEYS[i], ARGV[2])
end
end
return 0
`
/** /**
* Identifies the CI variant a run's timings belong to, from the env vars that * Identifies the CI variant a run's timings belong to, from the env vars that
...@@ -315,7 +375,8 @@ function computeMaxTimings(variants) { ...@@ -315,7 +375,8 @@ function computeMaxTimings(variants) {
/** /**
* Normalizes loaded timings data. Accepts the versioned format written by * Normalizes loaded timings data. Accepts the versioned format written by
* `fetchTestTimings`, as well as a legacy flat `{ [file]: seconds }` map (or * `fetchTestTimings`, as well as a legacy flat `{ [file]: seconds }` map (or
* the `{}` the workflow writes when no timings could be fetched). * the `{}` the workflow writes when no timings could be fetched). The
* cross-variant `max` is always derived from `variants`, never stored.
* *
* @param {unknown} data * @param {unknown} data
* @returns {TestTimings | null} * @returns {TestTimings | null}
...@@ -339,10 +400,7 @@ function normalizeTimings(data) { ...@@ -339,10 +400,7 @@ function normalizeTimings(data) {
} }
return { return {
variants, variants,
max: max: computeMaxTimings(variants),
record.max && typeof record.max === 'object'
? record.max
: computeMaxTimings(variants),
legacy: legacy:
record.legacy && typeof record.legacy === 'object' ? record.legacy : {}, record.legacy && typeof record.legacy === 'object' ? record.legacy : {},
} }
...@@ -393,7 +451,8 @@ function toTimingsMap(hash) { ...@@ -393,7 +451,8 @@ function toTimingsMap(hash) {
/** /**
* Fetches the timings of every variant, plus the legacy shared map as a * Fetches the timings of every variant, plus the legacy shared map as a
* fallback, in the versioned format consumed by `normalizeTimings`. * fallback, in the versioned format consumed by `normalizeTimings`.
* Returns `null` when no timings are available at all. * Returns `null` when no timings are available at all. Also prunes expired
* variants from the index (see MAINTAIN_VARIANT_SCRIPT).
* *
* @param {any} client KV client * @param {any} client KV client
* @param {typeof retryKVOperation} retry * @param {typeof retryKVOperation} retry
...@@ -419,6 +478,22 @@ async function fetchTestTimings(client, retry) { ...@@ -419,6 +478,22 @@ async function fetchTestTimings(client, retry) {
if (Object.keys(variantTimings).length > 0) { if (Object.keys(variantTimings).length > 0) {
result[name] = variantTimings result[name] = variantTimings
} }
try {
await client.eval(
MAINTAIN_VARIANT_SCRIPT,
[
KV_TIMINGS_V2_VARIANTS_KEY,
`${KV_TIMINGS_V2_PREFIX}${name}`,
`${KV_TIMINGS_V2_UPDATED_PREFIX}${name}`,
],
[name, String(TIMINGS_TTL_SECONDS)]
)
} catch (err) {
console.log(
`Failed to maintain timings of variant ${name}:`,
err.message
)
}
} }
return result return result
}, 'fetch variant timings') }, 'fetch variant timings')
...@@ -440,17 +515,12 @@ async function fetchTestTimings(client, retry) { ...@@ -440,17 +515,12 @@ async function fetchTestTimings(client, retry) {
console.log('No timing data found in KV store') console.log('No timing data found in KV store')
return null return null
} }
return { return { version: 2, variants, legacy }
version: 2,
variants,
max: computeMaxTimings(variants),
legacy,
}
} }
/** /**
* Records this run's timings for its variant. Only touches the variant's own * Records this run's timings for its variant. Only touches the variant's own
* hash and the variant index; other variants and the legacy key are left as * keys and the variant index; other variants and the legacy key are left as
* they are. * they are.
* *
* @param {any} client KV client * @param {any} client KV client
...@@ -458,30 +528,60 @@ async function fetchTestTimings(client, retry) { ...@@ -458,30 +528,60 @@ async function fetchTestTimings(client, retry) {
* @param {string} variant * @param {string} variant
* @param {Record<string, number>} curTimings seconds per test file * @param {Record<string, number>} curTimings seconds per test file
* @param {(file: string) => boolean} fileExists * @param {(file: string) => boolean} fileExists
* @param {number} [now] epoch ms
*/ */
async function writeTestTimings( async function writeTestTimings(
client, client,
retry, retry,
variant, variant,
curTimings, curTimings,
fileExists fileExists,
now = Date.now()
) { ) {
const key = `${KV_TIMINGS_V2_PREFIX}${variant}` const key = `${KV_TIMINGS_V2_PREFIX}${variant}`
if (Object.keys(curTimings).length > 0) { const updatedKey = `${KV_TIMINGS_V2_UPDATED_PREFIX}${variant}`
await retry(async () => { const files = Object.keys(curTimings)
await client.hset(key, curTimings) if (files.length > 0) {
await client.sadd(KV_TIMINGS_V2_VARIANTS_KEY, variant) const updated = Object.fromEntries(files.map((file) => [file, now]))
}, 'update timings') await retry(
() =>
client
.multi()
.hset(key, curTimings)
.hset(updatedKey, updated)
.sadd(KV_TIMINGS_V2_VARIANTS_KEY, variant)
.expire(key, TIMINGS_TTL_SECONDS)
.expire(updatedKey, TIMINGS_TTL_SECONDS)
.exec(),
'update timings'
)
console.log(`Successfully updated test timings for variant ${variant}`) console.log(`Successfully updated test timings for variant ${variant}`)
} }
// Clean up stale timings for deleted tests, within this variant only // Timings of files missing from this checkout are only deleted once no
// branch has written them for TIMINGS_TTL_SECONDS (within this variant)
const recorded = const recorded =
(await retry(() => client.hgetall(key), 'read timings')) || {} (await retry(() => client.hgetall(key), 'read timings')) || {}
const stale = Object.keys(recorded).filter((file) => !fileExists(file)) const candidates = Object.keys(recorded).filter((file) => !fileExists(file))
if (stale.length > 0) { if (candidates.length > 0) {
for (const file of stale) console.log('removing stale timing', file) /** @type {string[] | null} */
await retry(() => client.hdel(key, ...stale), 'remove stale timings') const deleted = await retry(
() =>
client.eval(
CLEANUP_STALE_TIMINGS_SCRIPT,
[key, updatedKey],
[
String(now),
String(now - TIMINGS_TTL_SECONDS * 1000),
String(TIMINGS_TTL_SECONDS),
...candidates,
]
),
'clean up stale timings'
)
for (const file of deleted || []) {
console.log('removing stale timing', file)
}
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment