Unverified Commit dd2b8fd7 authored by Matt Mastracci's avatar Matt Mastracci Committed by GitHub

fix(ci): improve Docker image caching and native build skip optimization (#92633)

## Summary

- **turbo-cache.mjs**: Increase stream buffer from 16KB to 16MB (fixes
throughput stalls when piping large artifacts). Add transfer stats
tracking with 5s stall warning. `getToFile()` now returns `{ ok, stats
}`.
- **docker-image-cache.js**: Switch from `docker save/load` to
`export/import` (drops redundant base image layers). Use `zstd
--long=27` for better compression. Add 3x retry logic. Add arch-aware
cache key to prevent cross-arch cache poisoning. Add download stats
logging.
- **native-cache.js**: Consume new `getToFile()` return shape, log
download stats.
- **build_and_deploy.yml**: Reorder to try native binary cache restore
BEFORE Docker image restore. On cache hit, skip the expensive Docker
image download entirely.
parent 42b2a625
......@@ -248,22 +248,22 @@ jobs:
- name: Clear native build
run: rm -rf packages/next-swc/native
# Build or restore the Docker image via turbo-cache.js.
- name: Build/restore Docker image
if: ${{ matrix.docker }}
run: node scripts/docker-image-cache.js
# Compute Rust input fingerprint (turbo hashes all Rust sources)
# Try to restore the native binary BEFORE loading the Docker image.
# If the binary cache hits, we skip the expensive Docker image restore entirely.
- name: Rust fingerprint
if: ${{ matrix.docker }}
run: pnpm dlx turbo@${TURBO_VERSION} run rust-fingerprint ${TURBO_ARGS}
# Try to restore previously-built native binary from turbo cache
- name: Restore native binary cache
if: ${{ matrix.docker }}
id: native-cache
run: node scripts/native-cache.js --restore --target ${{ matrix.target }} && echo "HIT=yes" >> $GITHUB_OUTPUT || echo "HIT=no" >> $GITHUB_OUTPUT
# Only load the Docker image if we need to compile (native cache miss)
- name: Build/restore Docker image
if: ${{ matrix.docker && steps.native-cache.outputs.HIT != 'yes' }}
run: node scripts/docker-image-cache.js
- name: Build in docker
if: ${{ matrix.docker && steps.native-cache.outputs.HIT != 'yes' }}
run: |
......
......@@ -4,14 +4,16 @@
//
// Computes a cache key from the Dockerfile + rust-toolchain.toml contents,
// then checks the turbo cache API via scripts/turbo-cache.mjs.
// Images are compressed with zstd before upload (~2.8GB → ~500MB).
// Uses docker export/import (flat filesystem) instead of save/load (layered)
// to avoid including redundant base image layers. Compressed with zstd.
//
// Usage:
// node scripts/docker-image-cache.js # restore from cache or build + upload
// node scripts/docker-image-cache.js --force # always rebuild and re-upload
const { execSync, spawn } = require('child_process')
const { createHash } = require('crypto')
const { execSync } = require('child_process')
const crypto = require('crypto')
const { createHash } = crypto
const path = require('path')
const fs = require('fs')
const os = require('os')
......@@ -27,21 +29,29 @@ const { values: flags } = parseArgs({
const REPO_ROOT = path.resolve(__dirname, '..')
const IMAGE_NAME = 'next-swc-builder:latest'
// Files that determine the docker image content — if any change, rebuild.
// docker export/import strips all image metadata. These --change flags
// restore the ENV and WORKDIR that the Dockerfile sets, so that tools
// like cargo, rustc, napi, sccache are found in PATH.
const DOCKER_IMPORT_CHANGES = [
'ENV PATH=/root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
'ENV DEBIAN_FRONTEND=noninteractive',
'WORKDIR /build',
]
// Files baked into the Docker image — only these affect the image content.
// Scripts that run on the host (docker-image-cache.js, docker-native-build.*)
// are NOT included since they're mounted at runtime, not COPY'd.
const CACHE_INPUTS = [
path.join(REPO_ROOT, 'scripts/native-builder.Dockerfile'),
path.join(REPO_ROOT, 'scripts/docker-image-cache.js'),
path.join(REPO_ROOT, 'scripts/docker-native-build.js'),
path.join(REPO_ROOT, 'scripts/docker-native-build.sh'),
path.join(REPO_ROOT, 'rust-toolchain.toml'),
]
function computeCacheKey() {
// Turbo cache keys must be hex-only (^[a-fA-F0-9]+$).
const hash = createHash('sha256')
hash.update('docker-image-v3\0')
hash.update('docker-image-v4\0')
// Include host architecture — the image contains native binaries
// (Rust toolchain, cargo-xwin, lld-link symlink) that are arch-specific.
// (Rust toolchain, cargo-xwin, etc.) that are arch-specific.
hash.update(`arch:${os.arch()}\0`)
for (const file of CACHE_INPUTS) {
hash.update(file + '\0')
......@@ -68,37 +78,23 @@ function buildImage() {
}
function tmpFile(name) {
return path.join(process.env.RUNNER_TEMP || os.tmpdir(), name)
const suffix = crypto.randomBytes(6).toString('hex')
return path.join(process.env.RUNNER_TEMP || os.tmpdir(), `${name}.${suffix}`)
}
function sh(cmd) {
execSync(cmd, { stdio: 'inherit', shell: true })
}
/** Pipe a Node.js Readable stream into a shell command's stdin. */
function pipeToShell(stream, cmd) {
return new Promise((resolve, reject) => {
const child = spawn(cmd, {
stdio: ['pipe', 'inherit', 'inherit'],
shell: true,
})
stream.pipe(child.stdin)
stream.on('error', (err) => {
child.kill()
reject(err)
})
child.on('error', reject)
child.on('close', (code) => {
if (code !== 0) reject(new Error(`Command failed with exit code ${code}`))
else resolve()
})
})
}
async function main() {
const cache = await import('./turbo-cache.mjs')
const key = computeCacheKey()
console.log(`Docker image cache key: ${key}`)
// Show redacted endpoint for debugging (scheme + first 2 chars of host)
const apiUrl = new URL(process.env.TURBO_API || 'https://vercel.com')
const redactedApi = `${apiUrl.protocol}//${apiUrl.hostname.slice(0, 2)}***`
console.log(`Docker image: ${IMAGE_NAME}`)
console.log(`Cache key: ${key}`)
console.log(`Cache endpoint: ${redactedApi}`)
if (!process.env.TURBO_TOKEN) {
console.log('No TURBO_TOKEN — building without cache')
......@@ -112,47 +108,70 @@ async function main() {
console.log(hit ? 'Cache HIT' : 'Cache MISS')
if (hit) {
try {
console.log('Streaming cached image through zstd into docker load...')
const stream = await cache.getStream(key)
await pipeToShell(stream, `zstd -d | docker load`)
console.log('Docker image restored from turbo cache')
return
} catch (e) {
console.log(`WARNING: Failed to restore image: ${e.message}`)
console.log('Discarding cached image and rebuilding from scratch')
// Remove the partially-loaded image if it exists
const zstFile = tmpFile('docker-image-cache.tar.zst')
let restored = false
for (let attempt = 1; attempt <= 3; attempt++) {
try {
execSync(`docker rmi -f ${IMAGE_NAME}`, { stdio: 'ignore' })
} catch {}
console.log(
`Downloading cached image${attempt > 1 ? ` (retry ${attempt})` : ''}...`
)
const result = await cache.getToFile(key, zstFile, { retries: 0 })
if (!result.ok) throw new Error('download failed')
if (result.stats) {
console.log(`Downloaded: ${cache.formatStats(result.stats)}`)
}
console.log('Decompressing and importing into Docker...')
const changeFlags = DOCKER_IMPORT_CHANGES.map(
(c) => `--change '${c}'`
).join(' ')
sh(
`zstd -d -c --long=27 --threads=0 ${zstFile} | docker import ${changeFlags} - ${IMAGE_NAME}`
)
console.log('Docker image restored from turbo cache')
restored = true
break
} catch (e) {
console.log(`WARNING: Attempt ${attempt} failed: ${e.message}`)
try {
execSync(`docker rmi -f ${IMAGE_NAME}`, { stdio: 'ignore' })
} catch {}
} finally {
try {
fs.unlinkSync(zstFile)
} catch {}
}
}
if (restored) return
console.log('All restore attempts failed — rebuilding from scratch')
}
}
// Cache miss or --force: always rebuild since inputs changed
buildImage()
// Compress and upload
console.log('Compressing docker image with zstd...')
const zstdFile = tmpFile('docker-image-cache.tar.zst')
// Export and compress with zstd (docker export produces uncompressed tar).
const zstFile = tmpFile('docker-image-cache.tar.zst')
const containerName = `next-swc-export-${process.pid}`
try {
sh(`docker save ${IMAGE_NAME} | zstd -3 -T0 -o ${zstdFile}`)
sh(`docker create --name ${containerName} ${IMAGE_NAME} true`)
sh(`docker export ${containerName} | zstd -1 -T0 --long=27 -o ${zstFile}`)
sh(`docker rm ${containerName}`)
const size = fs.statSync(zstdFile).size
const size = fs.statSync(zstFile).size
console.log(
`Compressed: ${(size / 1024 / 1024).toFixed(0)} MB — uploading...`
`Exported + compressed: ${(size / 1024 / 1024).toFixed(0)} MB — uploading...`
)
try {
// Stream upload from file (avoids 2GB Buffer limit)
await cache.put(key, zstdFile)
await cache.put(key, zstFile)
console.log('Docker image uploaded to turbo cache')
} catch (e) {
console.log(`WARNING: Failed to upload: ${e.message}`)
}
} finally {
try {
fs.unlinkSync(zstdFile)
fs.unlinkSync(zstFile)
} catch {}
}
}
......
......@@ -75,14 +75,18 @@ async function restore() {
console.log('Native cache HIT — downloading...')
const tarFile = tmpFile('native-cache.tar.zst')
const ok = await cache.getToFile(key, tarFile)
if (!ok) {
const result = await cache.getToFile(key, tarFile)
if (!result.ok) {
console.log('Download failed')
return false
}
const size = fs.statSync(tarFile).size
console.log(`Downloaded ${(size / 1024 / 1024).toFixed(0)} MB`)
if (result.stats) {
console.log(`Downloaded (${cache.formatStats(result.stats)})`)
} else {
const size = fs.statSync(tarFile).size
console.log(`Downloaded ${(size / 1024 / 1024).toFixed(0)} MB`)
}
fs.mkdirSync(NATIVE_DIR, { recursive: true })
sh(`zstd -d -c "${tarFile}" | tar xf - -C "${NATIVE_DIR}"`)
fs.unlinkSync(tarFile)
......
......@@ -64,8 +64,15 @@ export async function get(key) {
/**
* Download an artifact as a Node.js Readable stream. Throws on failure.
*
* The returned stream has a `.stats` property with transfer metrics
* (populated as data flows):
* { totalBytes, startTime, endTime, maxStallMs, stallWarned }
*
* A console warning is printed (once) if no data arrives for 5+ seconds.
* The stream is destroyed if no data arrives within `stallTimeout` ms.
*/
export async function getStream(key) {
export async function getStream(key, { stallTimeout = 30_000 } = {}) {
const res = await fetch(artifactUrl(key), {
method: 'GET',
headers: baseHeaders(),
......@@ -76,27 +83,125 @@ export async function getStream(key) {
// Use a large buffer to avoid backpressure stalls — the default
// highWaterMark for Readable.fromWeb() is only 16KB which throttles
// throughput when piping large artifacts to shell commands.
return Readable.fromWeb(res.body, { highWaterMark: 16 * 1024 * 1024 })
const stream = Readable.fromWeb(res.body, {
highWaterMark: 16 * 1024 * 1024,
})
const stats = {
totalBytes: 0,
startTime: Date.now(),
endTime: 0,
maxStallMs: 0,
stallWarned: false,
}
stream.stats = stats
let lastDataTime = Date.now()
// Stall detection: warn once at 5s, destroy at stallTimeout.
let timer = setTimeout(() => {
stream.destroy(new Error(`Download stalled: no data for ${stallTimeout}ms`))
}, stallTimeout)
const STALL_WARN_MS = 5_000
let warnTimer = setTimeout(() => {
if (!stats.stallWarned) {
stats.stallWarned = true
const elapsed = ((Date.now() - stats.startTime) / 1000).toFixed(1)
const mb = (stats.totalBytes / 1024 / 1024).toFixed(1)
console.log(
`WARNING: download stall detected — ${mb} MB received in ${elapsed}s, no data for 5s+`
)
}
}, STALL_WARN_MS)
stream.on('data', (chunk) => {
const now = Date.now()
const gap = now - lastDataTime
if (gap > stats.maxStallMs) stats.maxStallMs = gap
lastDataTime = now
stats.totalBytes += chunk.length
clearTimeout(timer)
timer = setTimeout(() => {
stream.destroy(
new Error(`Download stalled: no data for ${stallTimeout}ms`)
)
}, stallTimeout)
clearTimeout(warnTimer)
warnTimer = setTimeout(() => {
if (!stats.stallWarned) {
stats.stallWarned = true
const elapsed = ((Date.now() - stats.startTime) / 1000).toFixed(1)
const mb = (stats.totalBytes / 1024 / 1024).toFixed(1)
console.log(
`WARNING: download stall detected — ${mb} MB received in ${elapsed}s, no data for 5s+`
)
}
}, STALL_WARN_MS)
})
stream.on('end', () => {
clearTimeout(timer)
clearTimeout(warnTimer)
stats.endTime = Date.now()
})
stream.on('error', () => {
clearTimeout(timer)
clearTimeout(warnTimer)
stats.endTime = Date.now()
})
return stream
}
/** Format transfer stats as a human-readable string. */
export function formatStats(stats) {
const duration = ((stats.endTime - stats.startTime) / 1000).toFixed(1)
const mb = (stats.totalBytes / 1024 / 1024).toFixed(1)
const speed = (
stats.totalBytes /
1024 /
1024 /
((stats.endTime - stats.startTime) / 1000)
).toFixed(1)
const stall =
stats.maxStallMs > 1000
? `, max stall ${(stats.maxStallMs / 1000).toFixed(1)}s`
: ''
return `${mb} MB in ${duration}s (${speed} MB/s${stall})`
}
/**
* Download an artifact to a file. Returns true on hit, false on miss.
* Download an artifact to a file.
* Returns { ok: true, stats } on hit, { ok: false } on miss/failure.
* Uses streaming to handle files larger than 2GB.
* Retries up to `retries` times on stall or network errors.
*/
export async function getToFile(key, destPath) {
try {
const stream = await getStream(key)
await new Promise((resolve, reject) => {
const ws = fs.createWriteStream(destPath)
stream.pipe(ws)
ws.on('finish', resolve)
ws.on('error', reject)
stream.on('error', reject)
})
return true
} catch {
return false
export async function getToFile(key, destPath, { retries = 2 } = {}) {
for (let attempt = 0; attempt <= retries; attempt++) {
try {
const stream = await getStream(key)
await new Promise((resolve, reject) => {
const ws = fs.createWriteStream(destPath)
stream.pipe(ws)
ws.on('finish', resolve)
ws.on('error', reject)
stream.on('error', reject)
})
return { ok: true, stats: stream.stats }
} catch (e) {
if (attempt < retries) {
console.log(
`Download attempt ${attempt + 1} failed: ${e.message} — retrying...`
)
continue
}
return { ok: false }
}
}
return { ok: false }
}
/**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment