Unverified Commit 2f645475 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

Apply native LTO to release build paths

Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent bfa7abf2
...@@ -196,7 +196,7 @@ jobs: ...@@ -196,7 +196,7 @@ jobs:
id: install-rust id: install-rust
uses: ./.github/actions/setup-rust uses: ./.github/actions/setup-rust
background: true background: true
if: ${{ inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' }} if: ${{ inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' || (inputs.rustBuildProfile == 'release' && inputs.skipNativeBuild != 'yes') }}
- name: Install fnm - name: Install fnm
run: | run: |
...@@ -289,11 +289,11 @@ jobs: ...@@ -289,11 +289,11 @@ jobs:
run: cargo binstall --no-confirm --locked cargo-nextest@0.9.133 run: cargo binstall --no-confirm --locked cargo-nextest@0.9.133
- run: rustc --version - run: rustc --version
if: ${{ inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' }} if: ${{ inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' || (inputs.rustBuildProfile == 'release' && inputs.skipNativeBuild != 'yes') }}
- name: Start sccache - name: Start sccache
uses: ./.github/actions/sccache uses: ./.github/actions/sccache
if: ${{ runner.os != 'Windows' && (inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes') }} if: ${{ runner.os != 'Windows' && (inputs.uploadNativeArtifact || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' || (inputs.rustBuildProfile == 'release' && inputs.skipNativeBuild != 'yes')) }}
# Infer the Rust target triple from the runner's OS/arch. napi would build # Infer the Rust target triple from the runner's OS/arch. napi would build
# for the host target if `--target` were omitted, but Turborepo does not # for the host target if `--target` were omitted, but Turborepo does not
...@@ -322,12 +322,66 @@ jobs: ...@@ -322,12 +322,66 @@ jobs:
name: next-swc-${{ steps.native-target.outputs.target }} name: next-swc-${{ steps.native-target.outputs.target }}
path: packages/next-swc/native/ path: packages/next-swc/native/
- name: Restore upstream LLVM for release native LTO
if: ${{ inputs.skipNativeBuild != 'yes' && (inputs.uploadNativeArtifact || steps.native-download.outcome != 'success') && inputs.rustBuildProfile == 'release' && contains(fromJSON('["x86_64-unknown-linux-gnu","aarch64-unknown-linux-gnu","aarch64-apple-darwin"]'), steps.native-target.outputs.target) }}
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: ${{ runner.temp }}/llvm-18.1.8
key: native-llvm-18.1.8-${{ runner.os }}-${{ runner.arch }}
- name: Install upstream LLVM for release native LTO
if: ${{ inputs.skipNativeBuild != 'yes' && (inputs.uploadNativeArtifact || steps.native-download.outcome != 'success') && inputs.rustBuildProfile == 'release' && contains(fromJSON('["x86_64-unknown-linux-gnu","aarch64-unknown-linux-gnu","aarch64-apple-darwin"]'), steps.native-target.outputs.target) }}
env:
LLVM_BIN_DIR: ${{ runner.temp }}/llvm-18.1.8/bin
LLVM_ROOT: ${{ runner.temp }}/llvm-18.1.8
run: |
if [[ ! -x "${LLVM_BIN_DIR}/clang" ]]; then
if [[ "${RUNNER_OS}" == "Linux" ]]; then
case "${RUNNER_ARCH}" in
X64) TINFO_PACKAGE=libtinfo5 ;;
ARM64) TINFO_PACKAGE=libtinfo6 ;;
*) echo "Unsupported Linux architecture: ${RUNNER_ARCH}" >&2; exit 1 ;;
esac
sudo apt-get update
sudo apt-get install -y "${TINFO_PACKAGE}" xz-utils
fi
scripts/install-native-llvm.sh "${LLVM_ROOT}"
fi
cargo binstall --no-confirm cargo-rustflags@0.4.0
- name: Build native binary - name: Build native binary
if: ${{ inputs.skipNativeBuild != 'yes' && (inputs.uploadNativeArtifact || steps.native-download.outcome != 'success') }} if: ${{ inputs.skipNativeBuild != 'yes' && (inputs.uploadNativeArtifact || steps.native-download.outcome != 'success') }}
env: env:
LLVM_BIN_DIR: ${{ runner.temp }}/llvm-18.1.8/bin
RUST_BUILD_PROFILE: ${{ inputs.rustBuildProfile }} RUST_BUILD_PROFILE: ${{ inputs.rustBuildProfile }}
NATIVE_TARGET: ${{ steps.native-target.outputs.target }} NATIVE_TARGET: ${{ steps.native-target.outputs.target }}
run: pnpm dlx turbo@${TURBO_VERSION} run "build-native-${RUST_BUILD_PROFILE}" ${TURBO_ARGS} --summarize -- --target "${NATIVE_TARGET}" run: |
export TARGET="${NATIVE_TARGET}"
export BUILD_TASK="build-native-${RUST_BUILD_PROFILE}"
export NATIVE_LTO_DETECT_ONLY=1
source scripts/configure-native-lto.sh
unset NATIVE_LTO_DETECT_ONLY
if native_lto_is_enabled; then
RUSTFLAGS=$(cargo rustflags --target "${TARGET}")
export RUSTFLAGS
if [[ "${TARGET}" == "aarch64-apple-darwin" ]]; then
export MACOSX_DEPLOYMENT_TARGET=11.0
fi
source scripts/configure-native-lto.sh
fi
pnpm dlx turbo@${TURBO_VERSION} run "${BUILD_TASK}" ${TURBO_ARGS} --summarize -- --target "${TARGET}"
if native_lto_is_enabled; then
scripts/verify-native-lto.sh
if [[ "${TARGET}" == "aarch64-apple-darwin" ]]; then
MIN_OS=$(otool -l packages/next-swc/native/next-swc.*.node | awk '$1 == "minos" { print $2; exit }')
if [[ "${MIN_OS}" != "${MACOSX_DEPLOYMENT_TARGET}" ]]; then
echo "Expected macOS ${MACOSX_DEPLOYMENT_TARGET}, found ${MIN_OS:-unknown}" >&2
exit 1
fi
fi
fi
- name: Upload native artifact - name: Upload native artifact
if: ${{ inputs.uploadNativeArtifact && inputs.skipNativeBuild != 'yes' }} if: ${{ inputs.uploadNativeArtifact && inputs.skipNativeBuild != 'yes' }}
......
...@@ -16,11 +16,13 @@ ...@@ -16,11 +16,13 @@
"../../Cargo.lock", "../../Cargo.lock",
"../../rust-toolchain.toml", "../../rust-toolchain.toml",
"../../scripts/configure-native-lto.sh", "../../scripts/configure-native-lto.sh",
"../../scripts/docker-native-build.js",
"../../scripts/docker-native-build.sh", "../../scripts/docker-native-build.sh",
"../../scripts/install-native-llvm.sh", "../../scripts/install-native-llvm.sh",
"../../scripts/native-builder.Dockerfile", "../../scripts/native-builder.Dockerfile",
"../../scripts/verify-native-lto.sh", "../../scripts/verify-native-lto.sh",
"../../.github/workflows/build_and_deploy.yml", "../../.github/workflows/build_and_deploy.yml",
"../../.github/workflows/build_reusable.yml",
], ],
"outputs": ["../../target/.rust-fingerprint"], "outputs": ["../../target/.rust-fingerprint"],
"cache": false, "cache": false,
......
...@@ -146,4 +146,34 @@ if ( ...@@ -146,4 +146,34 @@ if (
exit 1 exit 1
fi fi
echo 'native LTO target matrix passed' assert_installer_config() {
local host=$1
local expected_archive=$2
local expected_sha=$3
local output
output=$(NATIVE_LLVM_HOST="${host}" NATIVE_LLVM_PRINT_CONFIG=1 \
"${REPO_ROOT}/scripts/install-native-llvm.sh" /unused)
[[ "${output}" == *"archive=${expected_archive}"* ]]
[[ "${output}" == *"sha256=${expected_sha}"* ]]
}
assert_installer_config \
Linux-x86_64 \
clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz \
54ec30358afcc9fb8aa74307db3046f5187f9fb89fb37064cdde906e062ebf36
assert_installer_config \
Linux-aarch64 \
clang+llvm-18.1.8-aarch64-linux-gnu.tar.xz \
dcaa1bebbfbb86953fdfbdc7f938800229f75ad26c5c9375ef242edad737d999
assert_installer_config \
Darwin-arm64 \
clang+llvm-18.1.8-arm64-apple-macos11.tar.xz \
4573b7f25f46d2a9c8882993f091c52f416c83271db6f5b213c93f0bd0346a10
if NATIVE_LLVM_HOST=Linux-ppc64le NATIVE_LLVM_PRINT_CONFIG=1 \
"${REPO_ROOT}/scripts/install-native-llvm.sh" /unused >/dev/null 2>&1; then
echo 'Expected unsupported LLVM build host to fail' >&2
exit 1
fi
echo 'native LTO target and installer matrix passed'
...@@ -88,16 +88,29 @@ if (targets.length === 0) { ...@@ -88,16 +88,29 @@ if (targets.length === 0) {
function buildImage() { function buildImage() {
console.log(`Building Docker image: ${DOCKER_IMAGE}`) console.log(`Building Docker image: ${DOCKER_IMAGE}`)
const ctx = fs.mkdtempSync(path.join(os.tmpdir(), 'next-swc-docker-')) const ctx = fs.mkdtempSync(path.join(os.tmpdir(), 'next-swc-docker-'))
const scriptsDir = path.join(ctx, 'scripts')
fs.mkdirSync(scriptsDir)
fs.copyFileSync( fs.copyFileSync(
path.join(REPO_ROOT, 'rust-toolchain.toml'), path.join(REPO_ROOT, 'rust-toolchain.toml'),
path.join(ctx, 'rust-toolchain.toml') path.join(ctx, 'rust-toolchain.toml')
) )
fs.copyFileSync(
path.join(REPO_ROOT, 'scripts/install-native-llvm.sh'),
path.join(scriptsDir, 'install-native-llvm.sh')
)
const installNativeLlvm =
!quick &&
targets.some(({ target }) =>
['x86_64-unknown-linux-gnu', 'aarch64-unknown-linux-gnu'].includes(target)
)
try { try {
execFileSync( execFileSync(
'docker', 'docker',
[ [
'build', 'build',
...(rebuild ? ['--no-cache'] : []), ...(rebuild ? ['--no-cache'] : []),
'--build-arg',
`INSTALL_NATIVE_LLVM=${installNativeLlvm ? '1' : '0'}`,
'-t', '-t',
DOCKER_IMAGE, DOCKER_IMAGE,
'-f', '-f',
......
...@@ -8,23 +8,35 @@ set -euo pipefail ...@@ -8,23 +8,35 @@ set -euo pipefail
readonly LLVM_VERSION=18.1.8 readonly LLVM_VERSION=18.1.8
readonly DESTINATION=${1:?usage: install-native-llvm.sh <destination>} readonly DESTINATION=${1:?usage: install-native-llvm.sh <destination>}
case "$(uname -s)-$(uname -m)" in readonly BUILD_HOST=${NATIVE_LLVM_HOST:-"$(uname -s)-$(uname -m)"}
case "${BUILD_HOST}" in
Linux-x86_64) Linux-x86_64)
readonly ARCHIVE_NAME="clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-18.04.tar.xz" readonly ARCHIVE_NAME="clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-18.04.tar.xz"
readonly ARCHIVE_SHA256=54ec30358afcc9fb8aa74307db3046f5187f9fb89fb37064cdde906e062ebf36 readonly ARCHIVE_SHA256=54ec30358afcc9fb8aa74307db3046f5187f9fb89fb37064cdde906e062ebf36
readonly RESOURCE_LIB="" readonly RESOURCE_LIB=""
;; ;;
Linux-aarch64 | Linux-arm64)
readonly ARCHIVE_NAME="clang+llvm-${LLVM_VERSION}-aarch64-linux-gnu.tar.xz"
readonly ARCHIVE_SHA256=dcaa1bebbfbb86953fdfbdc7f938800229f75ad26c5c9375ef242edad737d999
readonly RESOURCE_LIB=""
;;
Darwin-arm64) Darwin-arm64)
readonly ARCHIVE_NAME="clang+llvm-${LLVM_VERSION}-arm64-apple-macos11.tar.xz" readonly ARCHIVE_NAME="clang+llvm-${LLVM_VERSION}-arm64-apple-macos11.tar.xz"
readonly ARCHIVE_SHA256=4573b7f25f46d2a9c8882993f091c52f416c83271db6f5b213c93f0bd0346a10 readonly ARCHIVE_SHA256=4573b7f25f46d2a9c8882993f091c52f416c83271db6f5b213c93f0bd0346a10
readonly RESOURCE_LIB="clang+llvm-${LLVM_VERSION}-arm64-apple-macos11/lib/clang/18/lib/darwin" readonly RESOURCE_LIB="clang+llvm-${LLVM_VERSION}-arm64-apple-macos11/lib/clang/18/lib/darwin"
;; ;;
*) *)
echo "Unsupported LLVM build host: $(uname -s)-$(uname -m)" >&2 echo "Unsupported LLVM build host: ${BUILD_HOST}" >&2
exit 1 exit 1
;; ;;
esac esac
if [[ "${NATIVE_LLVM_PRINT_CONFIG:-}" == "1" ]]; then
printf 'archive=%s\nsha256=%s\n' "${ARCHIVE_NAME}" "${ARCHIVE_SHA256}"
exit 0
fi
if [[ -x "${DESTINATION}/bin/clang" ]]; then if [[ -x "${DESTINATION}/bin/clang" ]]; then
installed_version=$("${DESTINATION}/bin/clang" --version | sed -n 's/^.*clang version \([0-9][0-9.]*\).*$/\1/p' | head -1) installed_version=$("${DESTINATION}/bin/clang" --version | sed -n 's/^.*clang version \([0-9][0-9.]*\).*$/\1/p' | head -1)
if [[ "${installed_version}" == "${LLVM_VERSION}" ]]; then if [[ "${installed_version}" == "${LLVM_VERSION}" ]]; then
...@@ -73,5 +85,7 @@ fi ...@@ -73,5 +85,7 @@ fi
tar -xJf "${ARCHIVE_PATH}" -C "${DESTINATION}" --strip-components=1 "${paths[@]}" tar -xJf "${ARCHIVE_PATH}" -C "${DESTINATION}" --strip-components=1 "${paths[@]}"
"${DESTINATION}/bin/clang" --version if [[ "${NATIVE_LLVM_SKIP_EXECUTION_CHECK:-}" != "1" ]]; then
"${DESTINATION}/bin/llvm-ar" --version | head -1 "${DESTINATION}/bin/clang" --version
"${DESTINATION}/bin/llvm-ar" --version | head -1
fi
...@@ -51,7 +51,7 @@ RUN HOST_ARCH=$(dpkg --print-architecture) && \ ...@@ -51,7 +51,7 @@ RUN HOST_ARCH=$(dpkg --print-architecture) && \
# works on either host architecture. # works on either host architecture.
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
curl ca-certificates \ curl ca-certificates \
clang lld llvm file pkg-config wget git xz-utils libssl-dev libtinfo5 \ clang lld llvm file pkg-config wget git xz-utils libssl-dev libtinfo5 libtinfo6 \
crossbuild-essential-amd64 crossbuild-essential-arm64 \ crossbuild-essential-amd64 crossbuild-essential-arm64 \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment