Unverified Commit e9150cad authored by Matt Mastracci's avatar Matt Mastracci Committed by GitHub

fix(ci): improve Docker image cache streaming throughput and arch-awareness (#92617)

## What
- Increase `highWaterMark` from 16KB (Node.js default) to 16MB for
turbo-cache stream downloads and uploads.
- Include host architecture (`os.arch()`) in Docker image cache key. 

## Why

The tiny default buffer in `Readable.fromWeb()` causes constant
backpressure stalls when piping large artifacts (~500MB Docker images)
through `zstd -d | docker load`.

The image contains native binaries (Rust toolchain, lld-link symlink)
that are arch-specific — without this, an image built on x86_64 could be
restored on aarch64 with broken binaries.
parent ba33b4e3
......@@ -40,6 +40,9 @@ function computeCacheKey() {
// Turbo cache keys must be hex-only (^[a-fA-F0-9]+$).
const hash = createHash('sha256')
hash.update('docker-image-v3\0')
// Include host architecture — the image contains native binaries
// (Rust toolchain, cargo-xwin, lld-link symlink) that are arch-specific.
hash.update(`arch:${os.arch()}\0`)
for (const file of CACHE_INPUTS) {
hash.update(file + '\0')
hash.update(fs.readFileSync(file))
......
......@@ -73,7 +73,10 @@ export async function getStream(key) {
if (!res.ok) {
throw new Error(`GET ${key} failed: ${res.status} ${res.statusText}`)
}
return Readable.fromWeb(res.body)
// Use a large buffer to avoid backpressure stalls — the default
// highWaterMark for Readable.fromWeb() is only 16KB which throttles
// throughput when piping large artifacts to shell commands.
return Readable.fromWeb(res.body, { highWaterMark: 16 * 1024 * 1024 })
}
/**
......@@ -116,7 +119,9 @@ export async function put(key, data) {
let body
if (isFile) {
// Stream from file — avoids loading into memory
body = Readable.toWeb(fs.createReadStream(data))
body = Readable.toWeb(
fs.createReadStream(data, { highWaterMark: 16 * 1024 * 1024 })
)
} else {
body = data
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment