Unverified Commit f98a3f11 authored by Matt Mastracci's avatar Matt Mastracci Committed by GitHub

[turbopack] Rust dependency caching via sccache (#91873)

## What

Use sccache, backed by the Vercel artifact cache, when compiling Rust
code.

## Why

Each CI runner compiles ~960 external Rust crates from scratch (~5 min)
on binary cache misses, even though `Cargo.lock` rarely changes between
commits. Some workflows don't cache and always build from scratch.

## Notes

Builds will use a local cache on each runner, but we've got a lot of
these runners and they'll take some time to populate that local, fastest
level of cache.

Windows builds are currently hitting ~68% of cached compilations for a
reason that isn't totally clear. Linux builds are caching correctly,
however. A follow-up pass at this work will use an incremental turborepo
caching strategy to improve this, and another pass will experiment with
cross-compiling Windows from Linux which will likely also help.

Linux builds that don't change Rust files at all are nearly
instantaneous (as before), and Linux builds that change Rust files will
complete in approximately 1/2 the time. `check` builds are ~1min faster
(~25% faster).

LTO-heavy runs like the benchmark builders are currently still slow
(approx unchanged), needs further investigation (may be related to git
hash embedding?).
parent 18164506
...@@ -53,7 +53,12 @@ rustflags = ["-Clink-arg=--icf=all"] ...@@ -53,7 +53,12 @@ rustflags = ["-Clink-arg=--icf=all"]
# rust-lld is not yet the default on Windows (tracking: rust-lang/rust#71520). # rust-lld is not yet the default on Windows (tracking: rust-lang/rust#71520).
[target.'cfg(windows)'] [target.'cfg(windows)']
linker = "rust-lld" linker = "rust-lld"
rustflags = ["-Ctarget-feature=+crt-static"] rustflags = [
"-Ctarget-feature=+crt-static",
# /Brepro makes MSVC link.exe produce deterministic PE timestamps,
# improving sccache hit rates by making .rlib/.rmeta output reproducible.
"-Clink-arg=/Brepro",
]
# Musl targets: disable static CRT linking. We produce dynamic .node shared # Musl targets: disable static CRT linking. We produce dynamic .node shared
# libraries (N-API addons), not static binaries. # libraries (N-API addons), not static binaries.
......
name: 'sccache with Vercel Artifacts'
description: 'Start sccache with multi-level disk+Vercel Artifacts caching for per-crate Rust compilation caching. Automatically stops and prints stats after the job completes.'
inputs:
base-dir:
description: 'SCCACHE_BASE_DIR for path normalization (default: $GITHUB_WORKSPACE)'
required: false
default: ''
turbo-token:
description: 'Fallback TURBO_TOKEN if not set in environment (e.g. from secrets.TURBO_REMOTE_CACHE_TOKEN)'
required: false
default: ''
runs:
using: 'node20'
main: 'main.js'
post: 'main.js'
post-if: 'always()'
const { execFileSync } = require('child_process')
const path = require('path')
const fs = require('fs')
const isPost = !!process.env.STATE_isPost
// Signal to the post phase that main ran
fs.appendFileSync(process.env.GITHUB_STATE, 'isPost=true\n')
const script = path.join(__dirname, isPost ? 'stop.sh' : 'start.sh')
try {
execFileSync('bash', [script], { stdio: 'inherit', env: process.env })
} catch (e) {
if (isPost) return // don't fail the job on cleanup
throw e
}
#!/usr/bin/env bash
set -euo pipefail
# Normalize TURBO_API and TURBO_TOKEN: prefer runner .bashrc values,
# fall back to Vercel's public API and the secret passed as an action input.
if [ -z "${TURBO_API:-}" ]; then
export TURBO_API="https://api.vercel.com"
echo "TURBO_API=${TURBO_API}" >> "$GITHUB_ENV"
fi
if [ -z "${TURBO_TOKEN:-}" ]; then
if [ -n "${INPUT_TURBO_TOKEN:-}" ]; then
export TURBO_TOKEN="${INPUT_TURBO_TOKEN}"
echo "TURBO_TOKEN=${TURBO_TOKEN}" >> "$GITHUB_ENV"
else
echo "WARNING: no TURBO_TOKEN available"
fi
fi
if [ -z "${TURBO_TEAM:-}" ]; then
export TURBO_TEAM="vercel"
echo "TURBO_TEAM=${TURBO_TEAM}" >> "$GITHUB_ENV"
fi
echo "::add-mask::${TURBO_TOKEN:-}"
echo "Cache endpoint: ${TURBO_API:0:9}..."
echo "TURBO_TOKEN: ${TURBO_TOKEN:0:3}..."
echo "TURBO_TEAM: ${TURBO_TEAM}"
# Kill stale processes from cancelled builds on self-hosted runners.
sccache --stop-server 2>/dev/null || true
pkill -9 -x cargo 2>/dev/null || true
pkill -9 -x rustc 2>/dev/null || true
# Install sccache via turbo task (cached by scripts/sccache-version).
TURBO="pnpm dlx turbo@${TURBO_VERSION:-latest}"
$TURBO run build-sccache ${TURBO_ARGS:-}
SCCACHE_PATH="${GITHUB_WORKSPACE}/target/sccache/bin"
export PATH="${SCCACHE_PATH}:${PATH}"
echo "${SCCACHE_PATH}" >> "$GITHUB_PATH"
sccache --version
# Set env vars for the sccache server (export) and subsequent steps (GITHUB_ENV).
set_env() {
export "$1=$2"
echo "$1=$2" >> "$GITHUB_ENV"
}
set_env RUSTC_WRAPPER sccache
set_env SCCACHE_BASEDIRS "${INPUT_BASE_DIR:-${GITHUB_WORKSPACE}}"
set_env CARGO_INCREMENTAL 0
set_env SCCACHE_IDLE_TIMEOUT 0
set_env SCCACHE_DIR "${HOME}/.sccache"
set_env SCCACHE_RUST_CRATE_TYPE_ALLOW_HASH v1
set_env SCCACHE_ERROR_LOG "${RUNNER_TEMP:-/tmp}/sccache-error.log"
# Gracefully fall back to local compilation if the remote cache is unreachable.
set_env SCCACHE_IGNORE_SERVER_IO_ERROR 1
# Configure remote cache if token is available, otherwise disk-only.
if [ -n "${TURBO_TOKEN:-}" ]; then
set_env SCCACHE_MULTILEVEL_CHAIN "disk,vercel_artifacts"
set_env SCCACHE_VERCEL_ARTIFACTS_ENDPOINT "${TURBO_API}"
set_env SCCACHE_VERCEL_ARTIFACTS_TOKEN "${TURBO_TOKEN}"
set_env SCCACHE_VERCEL_ARTIFACTS_TEAM_SLUG "${TURBO_TEAM}"
fi
# Start the sccache daemon.
echo "SCCACHE_BASEDIRS=${SCCACHE_BASEDIRS}"
sccache --start-server 2>&1 || echo "WARNING: sccache failed to start"
sccache --show-stats 2>&1 | grep -E "Cache location|Base directories" || true
#!/usr/bin/env bash
echo "=== sccache stats ==="
sccache --show-stats || true
sccache --stop-server 2>/dev/null || true
# Show server error log if present
LOG="${RUNNER_TEMP:-/tmp}/sccache-error.log"
if [ -f "$LOG" ]; then
echo "=== sccache error log (last 30 lines) ==="
tail -30 "$LOG"
fi
...@@ -253,18 +253,19 @@ jobs: ...@@ -253,18 +253,19 @@ jobs:
if: ${{ matrix.docker }} if: ${{ matrix.docker }}
run: node scripts/docker-image-cache.js run: node scripts/docker-image-cache.js
# Try to restore previously-built native binary from turbo cache # Compute Rust input fingerprint (turbo hashes all Rust sources)
- name: pull build cache - name: Rust fingerprint
if: ${{ matrix.docker }} if: ${{ matrix.docker }}
run: TURBO_VERSION=${TURBO_VERSION} node ./scripts/pull-turbo-cache.js ${{ matrix.target }} run: pnpm dlx turbo@${TURBO_VERSION} run rust-fingerprint ${TURBO_ARGS}
- name: check build exists # Try to restore previously-built native binary from turbo cache
- name: Restore native binary cache
if: ${{ matrix.docker }} if: ${{ matrix.docker }}
run: if [ -f packages/next-swc/native/next-swc.*.node ]; then echo "BUILD_EXISTS=yes" >> $GITHUB_OUTPUT; else echo "BUILD_EXISTS=no" >> $GITHUB_OUTPUT; fi id: native-cache
id: build-exists run: node scripts/native-cache.js --restore --target ${{ matrix.target }} && echo "HIT=yes" >> $GITHUB_OUTPUT || echo "HIT=no" >> $GITHUB_OUTPUT
- name: Build in docker - name: Build in docker
if: ${{ matrix.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }} if: ${{ matrix.docker && steps.native-cache.outputs.HIT != 'yes' }}
run: | run: |
docker run --rm \ docker run --rm \
-e CI -e RUST_BACKTRACE -e CARGO_TERM_COLOR \ -e CI -e RUST_BACKTRACE -e CARGO_TERM_COLOR \
...@@ -282,15 +283,12 @@ jobs: ...@@ -282,15 +283,12 @@ jobs:
${{ matrix.docker }} \ ${{ matrix.docker }} \
-xeo pipefail scripts/docker-native-build.sh -xeo pipefail scripts/docker-native-build.sh
- name: cache build - name: Save native binary cache
if: ${{ matrix.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }} if: ${{ matrix.docker && steps.native-cache.outputs.HIT != 'yes' }}
run: pnpm dlx turbo@${TURBO_VERSION} run cache-build-native --force -- ${{ matrix.target }} run: node scripts/native-cache.js --save --target ${{ matrix.target }}
- name: 'Build' - name: 'Build'
if: ${{ !matrix.docker }} if: ${{ !matrix.docker }}
env:
# We don't want incremental builds on CI
CARGO_INCREMENTAL: 0
run: | run: |
echo "Host arch: $(uname -m)" echo "Host arch: $(uname -m)"
echo "Node arch: $(node -e 'console.log(process.arch)')" echo "Node arch: $(node -e 'console.log(process.arch)')"
......
...@@ -192,7 +192,7 @@ jobs: ...@@ -192,7 +192,7 @@ jobs:
needsRust: 'yes' needsRust: 'yes'
needsNextest: 'yes' needsNextest: 'yes'
skipNativeBuild: 'yes' skipNativeBuild: 'yes'
afterBuild: pnpm dlx turbo@${TURBO_VERSION} run test-cargo-unit --remote-cache-timeout 60 --log-order stream afterBuild: pnpm dlx turbo@${TURBO_VERSION} run test-cargo-unit ${TURBO_ARGS}
stepName: 'test-cargo-unit' stepName: 'test-cargo-unit'
secrets: inherit secrets: inherit
...@@ -214,7 +214,7 @@ jobs: ...@@ -214,7 +214,7 @@ jobs:
needsRust: 'yes' needsRust: 'yes'
skipInstallBuild: 'yes' skipInstallBuild: 'yes'
skipNativeBuild: 'yes' skipNativeBuild: 'yes'
afterBuild: pnpm dlx turbo@${TURBO_VERSION} run rust-check afterBuild: pnpm dlx turbo@${TURBO_VERSION} run rust-check ${TURBO_ARGS}
stepName: 'rust-check' stepName: 'rust-check'
secrets: inherit secrets: inherit
...@@ -602,7 +602,7 @@ jobs: ...@@ -602,7 +602,7 @@ jobs:
afterBuild: | afterBuild: |
rustup target add wasm32-unknown-unknown rustup target add wasm32-unknown-unknown
node ./scripts/normalize-version-bump.js node ./scripts/normalize-version-bump.js
pnpm dlx turbo@${TURBO_VERSION} run build-wasm -- --target nodejs pnpm dlx turbo@${TURBO_VERSION} run build-wasm ${TURBO_ARGS} -- --target nodejs
git checkout . git checkout .
export NEXT_TEST_MODE=start export NEXT_TEST_MODE=start
...@@ -630,7 +630,7 @@ jobs: ...@@ -630,7 +630,7 @@ jobs:
skipNativeInstall: 'yes' skipNativeInstall: 'yes'
afterBuild: | afterBuild: |
rustup target add wasm32-wasip1-threads rustup target add wasm32-wasip1-threads
pnpm dlx turbo@${TURBO_VERSION} run build-native-wasi pnpm dlx turbo@${TURBO_VERSION} run build-native-wasi ${TURBO_ARGS}
stepName: 'test-next-napi-bindings-wasi' stepName: 'test-next-napi-bindings-wasi'
secrets: inherit secrets: inherit
......
...@@ -93,6 +93,7 @@ on: ...@@ -93,6 +93,7 @@ on:
env: env:
NAPI_CLI_VERSION: 2.18.4 NAPI_CLI_VERSION: 2.18.4
TURBO_VERSION: 2.9.4 TURBO_VERSION: 2.9.4
TURBO_ARGS: '-v --env-mode loose --remote-cache-timeout 300 --log-order stream'
NODE_LTS_VERSION: 20.9.0 NODE_LTS_VERSION: 20.9.0
# run-tests.js reads `TEST_CONCURRENCY` if no explicit `--concurrency` or `-c` # run-tests.js reads `TEST_CONCURRENCY` if no explicit `--concurrency` or `-c`
# argument is provided # argument is provided
...@@ -102,7 +103,6 @@ env: ...@@ -102,7 +103,6 @@ env:
TURBO_TEAM: 'vtest314-next-e2e-tests' TURBO_TEAM: 'vtest314-next-e2e-tests'
TURBO_CACHE: 'remote:rw' TURBO_CACHE: 'remote:rw'
TURBO_TOKEN: ${{ secrets.TURBO_REMOTE_CACHE_TOKEN }}
NEXT_TELEMETRY_DISABLED: 1 NEXT_TELEMETRY_DISABLED: 1
# allow not skipping install-native postinstall script if we don't have a binary available already # allow not skipping install-native postinstall script if we don't have a binary available already
...@@ -243,14 +243,6 @@ jobs: ...@@ -243,14 +243,6 @@ jobs:
- run: corepack prepare --activate yarn@1.22.19 && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" - run: corepack prepare --activate yarn@1.22.19 && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}"
- name: Cache on ${{ github.ref_name }}
uses: ijjk/rust-cache@a34594c450817c9860143c79797a8770c4e587f5 # turbo-cache-v1.0.9
if: ${{ inputs.rustCacheKey }}
with:
cache-provider: 'turbo'
save-if: ${{ github.ref_name == 'canary' }}
shared-key: ${{ inputs.rustCacheKey }}-${{ inputs.buildNativeTarget }}-build-${{ inputs.rustBuildProfile }}-${{ hashFiles('.cargo/config.toml') }}
# clean up any previous artifacts to avoid hitting disk space limits # clean up any previous artifacts to avoid hitting disk space limits
- run: git clean -xdf && rm -rf /tmp/next-repo-*; rm -rf /tmp/next-install-* /tmp/yarn-* /tmp/ncc-cache target - run: git clean -xdf && rm -rf /tmp/next-repo-*; rm -rf /tmp/next-install-* /tmp/yarn-* /tmp/ncc-cache target
...@@ -260,14 +252,17 @@ jobs: ...@@ -260,14 +252,17 @@ jobs:
git config --global user.name "vercel-ci-bot" git config --global user.name "vercel-ci-bot"
git config --global user.email "infra+ci@vercel.com" git config --global user.email "infra+ci@vercel.com"
- run: cargo clean
if: ${{ inputs.skipNativeBuild != 'yes' || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' }}
# normalize versions before build-native for better cache hits # normalize versions before build-native for better cache hits
- run: node scripts/normalize-version-bump.js - run: node scripts/normalize-version-bump.js
name: normalize versions name: normalize versions
- run: pnpm dlx turbo@${TURBO_VERSION} run build-native-${{ inputs.rustBuildProfile }} -v --env-mode loose --remote-cache-timeout 90 --summarize -- --target ${{ inputs.buildNativeTarget }} - name: Start sccache
uses: ./.github/actions/sccache
if: ${{ inputs.skipNativeBuild != 'yes' || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' }}
with:
turbo-token: ${{ secrets.TURBO_REMOTE_CACHE_TOKEN }}
- run: pnpm dlx turbo@${TURBO_VERSION} run build-native-${{ inputs.rustBuildProfile }} ${TURBO_ARGS} --summarize -- --target ${{ inputs.buildNativeTarget }}
if: ${{ inputs.skipNativeBuild != 'yes' }} if: ${{ inputs.skipNativeBuild != 'yes' }}
- name: Upload next-swc artifact - name: Upload next-swc artifact
......
...@@ -15,7 +15,6 @@ on: ...@@ -15,7 +15,6 @@ on:
env: env:
TURBOPACK_BENCH_COUNTS: '100' TURBOPACK_BENCH_COUNTS: '100'
TURBOPACK_BENCH_PROGRESS: '1' TURBOPACK_BENCH_PROGRESS: '1'
NODE_LTS_VERSION: 20 NODE_LTS_VERSION: 20
jobs: jobs:
...@@ -46,6 +45,11 @@ jobs: ...@@ -46,6 +45,11 @@ jobs:
- run: pnpm install - run: pnpm install
working-directory: turbopack/benchmark-apps working-directory: turbopack/benchmark-apps
- name: Start sccache
uses: ./.github/actions/sccache
with:
turbo-token: ${{ secrets.TURBO_REMOTE_CACHE_TOKEN }}
- name: Build benchmarks for tests - name: Build benchmarks for tests
timeout-minutes: 120 timeout-minutes: 120
run: | run: |
......
...@@ -19,8 +19,6 @@ concurrency: ...@@ -19,8 +19,6 @@ concurrency:
env: env:
CI: 1 CI: 1
CARGO_INCREMENTAL: 0
# For faster CI
RUST_LOG: 'off' RUST_LOG: 'off'
TURBO_TEAM: 'vercel' TURBO_TEAM: 'vercel'
TURBO_CACHE: 'remote:rw' TURBO_CACHE: 'remote:rw'
...@@ -40,12 +38,10 @@ jobs: ...@@ -40,12 +38,10 @@ jobs:
with: with:
tool: cargo-codspeed@3.0.5 tool: cargo-codspeed@3.0.5
- name: Cache on ${{ github.ref_name }} - name: Start sccache
uses: ijjk/rust-cache@a34594c450817c9860143c79797a8770c4e587f5 # turbo-cache-v1.0.9 uses: ./.github/actions/sccache
with: with:
save-if: 'true' turbo-token: ${{ secrets.TURBO_REMOTE_CACHE_TOKEN }}
cache-provider: 'turbo'
shared-key: build-turbopack-benchmark-small-apps-${{ hashFiles('.cargo/config.toml') }}
- name: Install pnpm dependencies - name: Install pnpm dependencies
working-directory: turbopack/benchmark-apps working-directory: turbopack/benchmark-apps
...@@ -80,12 +76,10 @@ jobs: ...@@ -80,12 +76,10 @@ jobs:
with: with:
tool: cargo-codspeed@3.0.5 tool: cargo-codspeed@3.0.5
- name: Cache on ${{ github.ref_name }} - name: Start sccache
uses: ijjk/rust-cache@a34594c450817c9860143c79797a8770c4e587f5 # turbo-cache-v1.0.9 uses: ./.github/actions/sccache
with: with:
save-if: 'true' turbo-token: ${{ secrets.TURBO_REMOTE_CACHE_TOKEN }}
cache-provider: 'turbo'
shared-key: build-turbopack-benchmark-analyzer-${{ hashFiles('.cargo/config.toml') }}
- name: Install pnpm dependencies - name: Install pnpm dependencies
working-directory: turbopack/benchmark-apps working-directory: turbopack/benchmark-apps
...@@ -121,6 +115,11 @@ jobs: ...@@ -121,6 +115,11 @@ jobs:
with: with:
tool: cargo-codspeed@3.0.5 tool: cargo-codspeed@3.0.5
- name: Start sccache
uses: ./.github/actions/sccache
with:
turbo-token: ${{ secrets.TURBO_REMOTE_CACHE_TOKEN }}
- name: Build the benchmark target(s) - name: Build the benchmark target(s)
env: env:
CODSPEED_RUNNER_MODE: instrumentation CODSPEED_RUNNER_MODE: instrumentation
......
...@@ -2005,6 +2005,22 @@ dependencies = [ ...@@ -2005,6 +2005,22 @@ dependencies = [
"syn 2.0.104", "syn 2.0.104",
] ]
[[package]]
name = "ctor"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "352d39c2f7bef1d6ad73db6f5160efcaed66d94ef8c6c573a8410c00bf909a98"
dependencies = [
"ctor-proc-macro",
"dtor",
]
[[package]]
name = "ctor-proc-macro"
version = "0.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
[[package]] [[package]]
name = "ctrlc" name = "ctrlc"
version = "3.4.7" version = "3.4.7"
...@@ -2381,6 +2397,21 @@ dependencies = [ ...@@ -2381,6 +2397,21 @@ dependencies = [
"dtoa", "dtoa",
] ]
[[package]]
name = "dtor"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1057d6c64987086ff8ed0fd3fbf377a6b7d205cc7715868cd401705f715cbe4"
dependencies = [
"dtor-proc-macro",
]
[[package]]
name = "dtor-proc-macro"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5"
[[package]] [[package]]
name = "dunce" name = "dunce"
version = "1.0.4" version = "1.0.4"
...@@ -4622,7 +4653,7 @@ checksum = "214f07a80874bb96a8433b3cdfc84980d56c7b02e1a0d7ba4ba0db5cef785e2b" ...@@ -4622,7 +4653,7 @@ checksum = "214f07a80874bb96a8433b3cdfc84980d56c7b02e1a0d7ba4ba0db5cef785e2b"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bitflags 2.9.1", "bitflags 2.9.1",
"ctor", "ctor 0.2.0",
"napi-derive", "napi-derive",
"napi-sys", "napi-sys",
"once_cell", "once_cell",
...@@ -10080,6 +10111,7 @@ version = "0.1.0" ...@@ -10080,6 +10111,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bincode 2.0.1", "bincode 2.0.1",
"ctor 0.8.0",
"serde", "serde",
"tokio", "tokio",
"trybuild", "trybuild",
......
...@@ -258,6 +258,7 @@ console-subscriber = "0.4.1" ...@@ -258,6 +258,7 @@ console-subscriber = "0.4.1"
const_format = "0.2.30" const_format = "0.2.30"
crc32fast = "1.4.2" crc32fast = "1.4.2"
criterion = { package = "codspeed-criterion-compat", version = "4.3.0" } criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
ctor = "0.8"
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5.8"
dashmap = "6.1.0" dashmap = "6.1.0"
data-encoding = "2.3.3" data-encoding = "2.3.3"
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
"build-native-no-plugin": "napi build --platform -p next-napi-bindings --cargo-cwd ../../ --cargo-name next_napi_bindings --features image-webp --js false native", "build-native-no-plugin": "napi build --platform -p next-napi-bindings --cargo-cwd ../../ --cargo-name next_napi_bindings --features image-webp --js false native",
"build-native-no-plugin-release": "napi build --platform -p next-napi-bindings --cargo-cwd ../../ --cargo-name next_napi_bindings --release --features image-webp,tracing/release_max_level_trace --js false native", "build-native-no-plugin-release": "napi build --platform -p next-napi-bindings --cargo-cwd ../../ --cargo-name next_napi_bindings --release --features image-webp,tracing/release_max_level_trace --js false native",
"build-native-wasi": "npx --package=@napi-rs/cli@3.0.0-alpha.45 napi build --platform --target wasm32-wasip1-threads -p next-napi-bindings --cwd ../../ --output-dir packages/next-swc/native --no-default-features", "build-native-wasi": "npx --package=@napi-rs/cli@3.0.0-alpha.45 napi build --platform --target wasm32-wasip1-threads -p next-napi-bindings --cwd ../../ --output-dir packages/next-swc/native --no-default-features",
"build-sccache": "node ../../scripts/build-sccache.js",
"build-wasm": "wasm-pack build ../../crates/wasm --scope=next", "build-wasm": "wasm-pack build ../../crates/wasm --scope=next",
"cache-build-native": "[ -d native ] && echo $(ls native)", "cache-build-native": "[ -d native ] && echo $(ls native)",
"rust-fingerprint": "node ../../scripts/rust-fingerprint.js", "rust-fingerprint": "node ../../scripts/rust-fingerprint.js",
......
...@@ -2,9 +2,8 @@ ...@@ -2,9 +2,8 @@
"$schema": "https://turborepo.org/schema.json", "$schema": "https://turborepo.org/schema.json",
"extends": ["//"], "extends": ["//"],
"tasks": { "tasks": {
// Fingerprint all Rust inputs into a single stamp file. Every Rust task // Fingerprint all Rust inputs into a single stamp file so other tasks
// depends on this, so they only need inputs: [stamp file] instead of // only need inputs: [stamp file] instead of repeating the full glob list.
// repeating the full list of Rust source globs.
"rust-fingerprint": { "rust-fingerprint": {
"inputs": [ "inputs": [
"../../.cargo/**", "../../.cargo/**",
...@@ -19,8 +18,10 @@ ...@@ -19,8 +18,10 @@
"outputs": ["../../target/.rust-fingerprint"], "outputs": ["../../target/.rust-fingerprint"],
"cache": false, "cache": false,
}, },
"build-sccache": {
// "auto" is used by the workspace `pnpm build-all` script. "inputs": ["../../scripts/sccache-version"],
"outputs": ["../../target/sccache/**"],
},
"build-native-auto": { "build-native-auto": {
"dependsOn": ["rust-fingerprint"], "dependsOn": ["rust-fingerprint"],
"inputs": ["../../target/.rust-fingerprint"], "inputs": ["../../target/.rust-fingerprint"],
......
#!/usr/bin/env node
const { execSync } = require('child_process')
const fs = require('fs')
const path = require('path')
const root = path.resolve(__dirname, '..')
const raw = fs
.readFileSync(path.join(__dirname, 'sccache-version'), 'utf-8')
.trim()
const [url, ref] = raw.split('#')
const args = [
'install',
'--git',
url,
'--locked',
'--root',
path.join(root, 'target', 'sccache'),
]
if (ref) {
// Use --rev for commit hashes, --branch for branch names
const flag = /^[0-9a-f]{7,40}$/.test(ref) ? '--rev' : '--branch'
args.push(flag, ref)
}
args.push('sccache')
execSync(`cargo ${args.join(' ')}`, { stdio: 'inherit', cwd: root })
...@@ -116,11 +116,19 @@ if [ "$ARCH" = "aarch64" ]; then ...@@ -116,11 +116,19 @@ if [ "$ARCH" = "aarch64" ]; then
export JEMALLOC_SYS_WITH_LG_PAGE=16 export JEMALLOC_SYS_WITH_LG_PAGE=16
fi fi
# Verify sccache is available if RUSTC_WRAPPER is set to it.
# Cached docker images may not have sccache installed yet.
if [ "${RUSTC_WRAPPER:-}" = "sccache" ] && ! command -v sccache &>/dev/null; then
echo "WARNING: RUSTC_WRAPPER=sccache but sccache not found — disabling"
unset RUSTC_WRAPPER
fi
echo "--- Build environment ---" echo "--- Build environment ---"
node -v node -v
rustc --version rustc --version
echo "Target: $TARGET" echo "Target: $TARGET"
echo "RUSTFLAGS: $RUSTFLAGS" echo "RUSTFLAGS: $RUSTFLAGS"
echo "RUSTC_WRAPPER: ${RUSTC_WRAPPER:-<unset>}"
echo "-------------------------" echo "-------------------------"
rustup target add "$TARGET" rustup target add "$TARGET"
...@@ -128,6 +136,12 @@ cd packages/next-swc ...@@ -128,6 +136,12 @@ cd packages/next-swc
npm run "$BUILD_TASK" -- --target "$TARGET" npm run "$BUILD_TASK" -- --target "$TARGET"
llvm-strip -x native/next-swc.*.node llvm-strip -x native/next-swc.*.node
# Show sccache stats if available
if command -v sccache &>/dev/null; then
echo "--- sccache stats ---"
sccache --show-stats || true
fi
# Post-build verification # Post-build verification
echo "--- Dynamic libraries ---" echo "--- Dynamic libraries ---"
readelf -d native/next-swc.*.node | grep NEEDED readelf -d native/next-swc.*.node | grep NEEDED
......
...@@ -41,7 +41,7 @@ RUN HOST_ARCH=$(dpkg --print-architecture) && \ ...@@ -41,7 +41,7 @@ RUN HOST_ARCH=$(dpkg --print-architecture) && \
"deb [arch=${FOREIGN_ARCH}] ${FOREIGN_MIRROR} focal-updates main universe" \ "deb [arch=${FOREIGN_ARCH}] ${FOREIGN_MIRROR} focal-updates main universe" \
"deb [arch=${FOREIGN_ARCH}] ${FOREIGN_MIRROR} focal-security main universe" \ "deb [arch=${FOREIGN_ARCH}] ${FOREIGN_MIRROR} focal-security main universe" \
> /etc/apt/sources.list > /etc/apt/sources.list
# Core build tools + GNU cross-compilation sysroots + Node.js 20 via nodesource. # Core build tools + GNU cross-compilation sysroots + Node.js 20 via nodesource.
# crossbuild-essential installs headers + libs in the multiarch layout # crossbuild-essential installs headers + libs in the multiarch layout
# that clang finds via --target. Both archs installed so the image # that clang finds via --target. Both archs installed so the image
...@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi ...@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
apt-get install -y --no-install-recommends \ apt-get install -y --no-install-recommends \
nodejs \ nodejs \
clang lld llvm pkg-config wget git xz-utils \ clang lld llvm pkg-config wget git xz-utils libssl-dev \
crossbuild-essential-amd64 crossbuild-essential-arm64 \ crossbuild-essential-amd64 crossbuild-essential-arm64 \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
...@@ -85,13 +85,15 @@ RUN rustup target add \ ...@@ -85,13 +85,15 @@ RUN rustup target add \
x86_64-unknown-linux-musl \ x86_64-unknown-linux-musl \
aarch64-unknown-linux-musl aarch64-unknown-linux-musl
# Install @napi-rs/cli and cargo-rustflags globally. # Install @napi-rs/cli, cargo-rustflags, and sccache.
# cargo-rustflags resolves the effective RUSTFLAGS for a target by querying # Use cargo-binstall for sccache (pre-built binary, much faster than compiling).
# cargo's own config resolution (handles cfg() predicates, --config overlays).
RUN npm i -g @napi-rs/cli@2.18.4 && \ RUN npm i -g @napi-rs/cli@2.18.4 && \
cargo install cargo-rustflags cargo install cargo-rustflags && \
BINSTALL_ARCH=$(uname -m) && \
curl -fsSL "https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-${BINSTALL_ARCH}-unknown-linux-musl.tgz" | tar xz -C /root/.cargo/bin && \
cargo binstall sccache@0.14.0 --no-confirm
# Verify installations # Verify installations
RUN node --version && rustc --version && napi -h > /dev/null && cargo rustflags --help > /dev/null RUN node --version && rustc --version && napi -h > /dev/null && cargo rustflags --help > /dev/null && sccache --version
WORKDIR /build WORKDIR /build
#!/usr/bin/env node
//
// Cache the compiled next-swc .node binary in the turbo remote cache.
//
// Uses the turbo-computed rust fingerprint (target/.rust-fingerprint) as
// the cache key, combined with the target triple. Run `turbo run
// rust-fingerprint` first to compute it.
//
// Usage:
// node scripts/native-cache.js --restore --target x86_64-unknown-linux-gnu
// node scripts/native-cache.js --save --target x86_64-unknown-linux-gnu
const { execSync } = require('child_process')
const { createHash } = require('crypto')
const path = require('path')
const fs = require('fs')
const os = require('os')
const { parseArgs } = require('node:util')
const { values: flags } = parseArgs({
args: process.argv.slice(2),
options: {
restore: { type: 'boolean', default: false },
save: { type: 'boolean', default: false },
target: { type: 'string', default: '' },
},
strict: false,
})
const REPO_ROOT = path.resolve(__dirname, '..')
const NATIVE_DIR = path.join(REPO_ROOT, 'packages/next-swc/native')
const FINGERPRINT = path.join(REPO_ROOT, 'target/.rust-fingerprint')
function computeCacheKey() {
if (!fs.existsSync(FINGERPRINT)) {
console.error(
'target/.rust-fingerprint not found — run `turbo run rust-fingerprint` first'
)
process.exit(1)
}
const turboHash = fs.readFileSync(FINGERPRINT, 'utf-8').trim()
// Combine turbo's input hash with the target triple for a unique key.
// Must be hex-only (turbo cache API requirement).
const hash = createHash('sha256')
hash.update(`native-cache-v1\0`)
hash.update(`turbo=${turboHash}\0`)
hash.update(`target=${flags.target}\0`)
return hash.digest('hex')
}
function tmpFile(name) {
return path.join(process.env.RUNNER_TEMP || os.tmpdir(), name)
}
function sh(cmd) {
execSync(cmd, { stdio: 'inherit', shell: true })
}
async function restore() {
const cache = await import('./turbo-cache.mjs')
const key = computeCacheKey()
console.log(`Native cache key: ${key.slice(0, 16)}...`)
console.log(` target: ${flags.target}`)
if (!process.env.TURBO_TOKEN) {
console.log('No TURBO_TOKEN — skipping cache restore')
return false
}
const hit = await cache.exists(key)
if (!hit) {
console.log('Native cache MISS')
return false
}
console.log('Native cache HIT — downloading...')
const tarFile = tmpFile('native-cache.tar.zst')
const ok = await cache.getToFile(key, tarFile)
if (!ok) {
console.log('Download failed')
return false
}
const size = fs.statSync(tarFile).size
console.log(`Downloaded ${(size / 1024 / 1024).toFixed(0)} MB`)
fs.mkdirSync(NATIVE_DIR, { recursive: true })
sh(`zstd -d -c "${tarFile}" | tar xf - -C "${NATIVE_DIR}"`)
fs.unlinkSync(tarFile)
// Verify
const nodes = fs.readdirSync(NATIVE_DIR).filter((f) => f.endsWith('.node'))
if (nodes.length > 0) {
console.log(`Restored: ${nodes.join(', ')}`)
return true
}
console.log('WARNING: tar extracted but no .node files found')
return false
}
async function save() {
const cache = await import('./turbo-cache.mjs')
const key = computeCacheKey()
console.log(`Native cache key: ${key.slice(0, 16)}...`)
if (!process.env.TURBO_TOKEN) {
console.log('No TURBO_TOKEN — skipping cache save')
return
}
const nodes = fs.existsSync(NATIVE_DIR)
? fs.readdirSync(NATIVE_DIR).filter((f) => f.endsWith('.node'))
: []
if (nodes.length === 0) {
console.log('No .node files to cache')
return
}
if (await cache.exists(key)) {
console.log('Already cached — skipping')
return
}
const tarFile = tmpFile('native-cache.tar.zst')
const fileList = nodes.join(' ')
sh(`tar cf - -C "${NATIVE_DIR}" ${fileList} | zstd -3 -T0 -o "${tarFile}"`)
const size = fs.statSync(tarFile).size
console.log(
`Compressed: ${(size / 1024 / 1024).toFixed(0)} MB — uploading...`
)
try {
await cache.put(key, tarFile)
console.log('Native cache saved')
} catch (e) {
console.log(`WARNING: Failed to save: ${e.message}`)
}
fs.unlinkSync(tarFile)
}
async function main() {
if (flags.restore) {
const ok = await restore()
process.exit(ok ? 0 : 1)
} else if (flags.save) {
await save()
} else {
console.error('Usage: --restore or --save (with --target)')
process.exit(1)
}
}
main().catch((e) => {
console.error(e)
process.exit(1)
})
#!/usr/bin/env node
// @ts-check
const { spawn } = require('child_process')
const MAX_ATTEMPTS = 3
const RETRY_DELAY_MS = 5000
/**
* @param {number} ms
*/
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
/**
* @param {string} command
* @param {{ stdio?: 'pipe' | 'inherit', captureOutput?: boolean }} options
* @returns {Promise<{ code: number | null, signal: string | null, output: string }>}
*/
function runCommand(
command,
{ stdio = 'inherit', captureOutput = false } = {}
) {
return new Promise((resolve) => {
let output = ''
const child = spawn('/bin/bash', ['-c', command], {
stdio: captureOutput ? 'pipe' : stdio,
})
if (captureOutput) {
child.stdout?.on('data', (data) => {
process.stdout.write(data)
output += data.toString()
})
child.stderr?.on('data', (data) => {
process.stderr.write(data)
})
}
child.on('exit', (code, signal) => {
resolve({ code, signal, output })
})
})
}
/**
* @param {string} command
* @param {number} maxAttempts
* @returns {Promise<boolean>}
*/
async function runWithRetry(command, maxAttempts) {
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
console.log(`Attempt ${attempt}/${maxAttempts}...`)
const { code, signal } = await runCommand(command)
if (!code && !signal) {
return true // success
}
console.warn(
`Attempt ${attempt} failed (exit code ${code}, signal ${signal})`
)
if (attempt < maxAttempts) {
console.log(`Retrying in ${RETRY_DELAY_MS / 1000}s...`)
await sleep(RETRY_DELAY_MS)
}
}
return false // all attempts failed
}
;(async function () {
const target = process.argv[process.argv.length - 1]
const turboCommand = `pnpm dlx turbo@${process.env.TURBO_VERSION || 'latest'}`
// First, do a dry run to check cache status
const { code, signal, output } = await runCommand(
`${turboCommand} run cache-build-native --dry=json -- ${target}`,
{ captureOutput: true }
)
if (code || signal) {
console.warn(
`Dry run failed (exit code ${code}, signal ${signal}). Continuing without cache.`
)
return
}
let turboData
try {
turboData = JSON.parse(output)
} catch (e) {
console.warn(`Failed to parse turbo output: ${e.message}`)
return
}
const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>')
if (!task) {
console.warn(`Failed to find related turbo task`, output)
return
}
// Pull cache if it was available
if (task.cache.local || task.cache.remote) {
console.log('Cache Status', task.taskId, task.hash, task.cache)
const success = await runWithRetry(
`${turboCommand} run cache-build-native -- ${target}`,
MAX_ATTEMPTS
)
if (!success) {
// Don't fail the job - the workflow will check if build exists
// and build from source if needed
console.warn(
`Cache restoration failed after ${MAX_ATTEMPTS} attempts. ` +
`Build will proceed from source.`
)
}
} else {
console.warn(`No turbo cache was available, continuing...`)
console.warn(task)
}
})()
https://github.com/vercel/sccache#b9566621e5daba0d898b73cb6bdcadd3eb989a0f
...@@ -76,6 +76,26 @@ export async function getStream(key) { ...@@ -76,6 +76,26 @@ export async function getStream(key) {
return Readable.fromWeb(res.body) return Readable.fromWeb(res.body)
} }
/**
* Download an artifact to a file. Returns true on hit, false on miss.
* Uses streaming to handle files larger than 2GB.
*/
export async function getToFile(key, destPath) {
try {
const stream = await getStream(key)
await new Promise((resolve, reject) => {
const ws = fs.createWriteStream(destPath)
stream.pipe(ws)
ws.on('finish', resolve)
ws.on('error', reject)
stream.on('error', reject)
})
return true
} catch {
return false
}
}
/** /**
* Upload an artifact. * Upload an artifact.
* @param {string} key - hex-only cache key * @param {string} key - hex-only cache key
......
...@@ -7,6 +7,7 @@ edition = "2024" ...@@ -7,6 +7,7 @@ edition = "2024"
[dev-dependencies] [dev-dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
ctor = { workspace = true }
bincode = { workspace = true } bincode = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
......
// Unset RUSTC_WRAPPER before trybuild tests run. When sccache wraps rustc, it
// emits "warning: ignoring -C extra-filename flag due to -o flag" which pollutes
// trybuild's stderr snapshot comparisons. Unsetting it here means only the
// sub-compilations trybuild spawns are affected — the main test binary was already
// compiled with sccache.
#[ctor::ctor]
fn unset_rustc_wrapper() {
unsafe { std::env::remove_var("RUSTC_WRAPPER") };
}
#[test] #[test]
fn derive_operation_value() { fn derive_operation_value() {
let t = trybuild::TestCases::new(); let t = trybuild::TestCases::new();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment