Unverified Commit 1bf6c2f4 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Fix flaky turbo cache pull with retries and graceful fallback (#89575)

- Add retry logic (3 attempts with 5s delay) for transient network
failures
- Don't fail the job when cache restoration fails - let workflow build
from source
- Add error handling for dry run failures and JSON parsing

This attempts to address transient failures seen here
https://github.com/vercel/next.js/actions/runs/21735816168/job/62700523424?pr=89574
parent 488d90d1
...@@ -3,69 +3,122 @@ ...@@ -3,69 +3,122 @@
const { spawn } = require('child_process') const { spawn } = require('child_process')
;(async function () { const MAX_ATTEMPTS = 3
const target = process.argv[process.argv.length - 1] const RETRY_DELAY_MS = 5000
let turboResult = '' /**
const turboCommand = `pnpm dlx turbo@${process.env.TURBO_VERSION || 'latest'}` * @param {number} ms
*/
await new Promise((resolve, reject) => { function sleep(ms) {
const child = spawn( return new Promise((resolve) => setTimeout(resolve, ms))
'/bin/bash', }
['-c', `${turboCommand} run cache-build-native --dry=json -- ${target}`],
{
stdio: 'pipe',
}
)
child.stderr.on('data', (data) => { /**
process.stderr.write(data) * @param {string} command
* @param {{ stdio?: 'pipe' | 'inherit', captureOutput?: boolean }} options
* @returns {Promise<{ code: number | null, signal: string | null, output: string }>}
*/
function runCommand(
command,
{ stdio = 'inherit', captureOutput = false } = {}
) {
return new Promise((resolve) => {
let output = ''
const child = spawn('/bin/bash', ['-c', command], {
stdio: captureOutput ? 'pipe' : stdio,
}) })
child.stdout.on('data', (data) => { if (captureOutput) {
child.stdout?.on('data', (data) => {
process.stdout.write(data) process.stdout.write(data)
turboResult += data.toString() output += data.toString()
}) })
child.stderr?.on('data', (data) => {
process.stderr.write(data)
})
}
child.on('exit', (code, signal) => { child.on('exit', (code, signal) => {
resolve({ code, signal, output })
})
})
}
/**
* @param {string} command
* @param {number} maxAttempts
* @returns {Promise<boolean>}
*/
async function runWithRetry(command, maxAttempts) {
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
console.log(`Attempt ${attempt}/${maxAttempts}...`)
const { code, signal } = await runCommand(command)
if (!code && !signal) {
return true // success
}
console.warn(
`Attempt ${attempt} failed (exit code ${code}, signal ${signal})`
)
if (attempt < maxAttempts) {
console.log(`Retrying in ${RETRY_DELAY_MS / 1000}s...`)
await sleep(RETRY_DELAY_MS)
}
}
return false // all attempts failed
}
;(async function () {
const target = process.argv[process.argv.length - 1]
const turboCommand = `pnpm dlx turbo@${process.env.TURBO_VERSION || 'latest'}`
// First, do a dry run to check cache status
const { code, signal, output } = await runCommand(
`${turboCommand} run cache-build-native --dry=json -- ${target}`,
{ captureOutput: true }
)
if (code || signal) { if (code || signal) {
return reject( console.warn(
new Error(`invalid exit code ${code} or signal ${signal}`) `Dry run failed (exit code ${code}, signal ${signal}). Continuing without cache.`
) )
return
} }
resolve(0)
})
})
const turboData = JSON.parse(turboResult) let turboData
try {
turboData = JSON.parse(output)
} catch (e) {
console.warn(`Failed to parse turbo output: ${e.message}`)
return
}
const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>') const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>')
if (!task) { if (!task) {
console.warn(`Failed to find related turbo task`, turboResult) console.warn(`Failed to find related turbo task`, output)
return return
} }
// pull cache if it was available // Pull cache if it was available
if (task.cache.local || task.cache.remote) { if (task.cache.local || task.cache.remote) {
console.log('Cache Status', task.taskId, task.hash, task.cache) console.log('Cache Status', task.taskId, task.hash, task.cache)
await new Promise((resolve, reject) => {
const child = spawn( const success = await runWithRetry(
'/bin/bash', `${turboCommand} run cache-build-native -- ${target}`,
['-c', `${turboCommand} run cache-build-native -- ${target}`], MAX_ATTEMPTS
{
stdio: 'inherit',
}
) )
child.on('exit', (code, signal) => {
if (code || signal) { if (!success) {
return reject( // Don't fail the job - the workflow will check if build exists
new Error(`invalid exit code ${code} or signal ${signal}`) // and build from source if needed
console.warn(
`Cache restoration failed after ${MAX_ATTEMPTS} attempts. ` +
`Build will proceed from source.`
) )
} }
resolve(0)
})
})
} else { } else {
console.warn(`No turbo cache was available, continuing...`) console.warn(`No turbo cache was available, continuing...`)
console.warn(task) console.warn(task)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment