Unverified Commit 6f8c5803 authored by OJ Kwon's avatar OJ Kwon Committed by GitHub

feat(turbopack): Experimental wasm build (#57906)

### What?

continuation of https://github.com/vercel/next.js/pull/57851, since it is from a remote branch that I don't have access to write.
Co-authored-by: 's avatarMaia Teegarden <2865858+padmaia@users.noreply.github.com>
Co-authored-by: 's avatarTim Neutkens <6324199+timneutkens@users.noreply.github.com>
parent f979341f
......@@ -340,12 +340,65 @@ jobs:
name: turbo run summary
path: .turbo/runs
build-wasm:
strategy:
matrix:
target: [web, nodejs]
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
steps:
- uses: actions/checkout@v3
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_LTS_VERSION }}
check-latest: true
- run: corepack enable
- name: Install Rust
uses: ./.github/actions/setup-rust
with:
targets: wasm32-unknown-unknown
- run: npm i -g turbo@${{ env.TURBO_VERSION }}
- name: Install wasm-pack
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- name: normalize versions
run: node scripts/normalize-version-bump.js
- name: Build
run: turbo run build-wasm -vvv --remote-cache-timeout 90 --summarize -- --target ${{ matrix.target }} --features tracing/release_max_level_info
- name: Add target to folder name
run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm'
- name: Upload turbo summary artifact
uses: actions/upload-artifact@v3
with:
name: turbo run summary
path: .turbo/runs
- name: Upload swc artifact
uses: actions/upload-artifact@v3
with:
name: wasm-binaries
path: packages/next-swc/crates/wasm/pkg-*
publishRelease:
if: ${{ needs.build.outputs.isRelease == 'true' }}
name: Potentially publish release
runs-on: ubuntu-latest
needs:
- build
- build-wasm
- build-native
permissions:
contents: write
......@@ -376,6 +429,11 @@ jobs:
name: next-swc-binaries
path: packages/next-swc/native
- uses: actions/download-artifact@v3
with:
name: wasm-binaries
path: packages/next-swc/crates/wasm
- run: npm i -g npm@9.6.7 # need latest version for provenance (pinning to avoid bugs)
- run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
- run: ./scripts/publish-native.js
......
......@@ -152,14 +152,14 @@ jobs:
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration
secrets: inherit
# test-next-swc-wasm:
# name: test next-swc wasm
# needs: ['build-native', 'build-next']
# uses: ./.github/workflows/build_reusable.yml
# with:
# skipForDocsOnly: 'yes'
# afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
# secrets: inherit
test-next-swc-wasm:
name: test next-swc wasm
needs: ['build-native', 'build-next']
uses: ./.github/workflows/build_reusable.yml
with:
skipForDocsOnly: 'yes'
afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
secrets: inherit
test-unit:
name: test unit
......@@ -262,6 +262,7 @@ jobs:
'test-cargo-integration',
'test-cargo-bench',
'rust-check',
'test-next-swc-wasm',
'test-turbopack-dev',
'test-turbopack-integration',
]
......
......@@ -8,4 +8,6 @@ pnpm build
By default, the latest canary of the `next-swc` binaries will be installed and used. If you are actively working on Rust code or you need to test out the most recent Rust code that hasn't been published as a canary yet, you can [install Rust](https://www.rust-lang.org/tools/install) and run `pnpm --filter=@next/swc build-native`.
If you want to test out the wasm build locally, you will need to [install wasm-pack](https://rustwasm.github.io/wasm-pack/installer/). Run `pnpm --filter=@next/swc build-wasm --target <wasm_target>` to build and `node ./scripts/setup-wasm.mjs` to copy it into your `node_modules`. Run next with `NODE_OPTIONS='--no-addons'` to force it to use the wasm binary.
If you need to clean the project for any reason, use `pnpm clean`.
......@@ -17,6 +17,7 @@ We chose to build on SWC for a few reasons:
- **Extensibility:** SWC can be used as a Crate inside Next.js, without having to fork the library or workaround design constraints.
- **Performance:** We were able to achieve ~3x faster Fast Refresh and ~5x faster builds in Next.js by switching to SWC, with more room for optimization still in progress.
- **WebAssembly:** Rust's support for WASM is essential for supporting all possible platforms and taking Next.js development everywhere.
- **Community:** The Rust community and ecosystem are amazing and still growing.
## Supported Features
......
......@@ -1030,7 +1030,7 @@ export default async function build(
const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir })
let binding = await loadBindings()
let binding = await loadBindings(config?.experimental?.useWasmBinary)
let root =
turboNextBuildRoot ??
......
......@@ -89,7 +89,7 @@ export default function nextJest(options: { dir?: string } = {}) {
: customJestConfig) ?? {}
// eagerly load swc bindings instead of waiting for transform calls
await loadBindings()
await loadBindings(nextConfig?.experimental?.useWasmBinary)
if (lockfilePatchPromise.cur) {
await lockfilePatchPromise.cur
......
......@@ -31,7 +31,7 @@ const infoLog = (...args: any[]) => {
* Based on napi-rs's target triples, returns triples that have corresponding next-swc binaries.
*/
export const getSupportedArchTriples: () => Record<string, any> = () => {
const { darwin, win32, linux } = platformArchTriples
const { darwin, win32, linux, freebsd, android } = platformArchTriples
return {
darwin,
......@@ -48,6 +48,16 @@ export const getSupportedArchTriples: () => Record<string, any> = () => {
(triple: { abi: string }) => triple.abi !== 'gnux32'
),
arm64: linux.arm64,
// This target is being deprecated, however we keep it in `knownDefaultWasmFallbackTriples` for now
arm: linux.arm,
},
// Below targets are being deprecated, however we keep it in `knownDefaultWasmFallbackTriples` for now
freebsd: {
x64: freebsd.x64,
},
android: {
arm64: android.arm64,
arm: android.arm,
},
}
}
......@@ -67,7 +77,7 @@ const triples = (() => {
if (rawTargetTriple) {
Log.warn(
`Trying to load next-swc for target triple ${rawTargetTriple}, but this target is not supported`
`Trying to load next-swc for target triple ${rawTargetTriple}, but there next-swc does not have native bindings support`
)
} else {
Log.warn(
......@@ -97,6 +107,21 @@ function checkVersionMismatch(pkgData: any) {
}
}
// These are the platforms we'll try to load wasm bindings first,
// only try to load native bindings if loading wasm binding somehow fails.
// Fallback to native binding is for migration period only,
// once we can verify loading-wasm-first won't cause visible regressions,
// we'll not include native bindings for these platform at all.
const knownDefaultWasmFallbackTriples = [
'x86_64-unknown-freebsd',
'aarch64-linux-android',
'arm-linux-androideabi',
'armv7-unknown-linux-gnueabihf',
'i686-pc-windows-msvc',
// WOA targets are TBD, while current userbase is small we may support it in the future
//'aarch64-pc-windows-msvc',
]
// The last attempt's error code returned when cjs require to native bindings fails.
// If node.js throws an error without error code, this should be `unknown` instead of undefined.
// For the wasm-first targets (`knownDefaultWasmFallbackTriples`) this will be `unsupported_target`.
......@@ -149,7 +174,9 @@ export interface Binding {
teardownCrashReporter?: any
}
export async function loadBindings(): Promise<Binding> {
export async function loadBindings(
useWasmBinary: boolean = false
): Promise<Binding> {
if (pendingBindings) {
return pendingBindings
}
......@@ -176,6 +203,27 @@ export async function loadBindings(): Promise<Binding> {
}
let attempts: any[] = []
const disableWasmFallback = process.env.NEXT_DISABLE_SWC_WASM
const unsupportedPlatform = triples.some(
(triple: any) =>
!!triple?.raw && knownDefaultWasmFallbackTriples.includes(triple.raw)
)
const shouldLoadWasmFallbackFirst =
!disableWasmFallback && unsupportedPlatform && useWasmBinary
if (!unsupportedPlatform && useWasmBinary) {
Log.warn(
`experimental.useWasmBinary is not an option for supported platform ${PlatformName}/${ArchName} and will be ignored.`
)
}
if (shouldLoadWasmFallbackFirst) {
lastNativeBindingsLoadErrorCode = 'unsupported_target'
const fallbackBindings = await tryLoadWasmWithFallback(attempts)
if (fallbackBindings) {
return resolve(fallbackBindings)
}
}
// Trickle down loading `fallback` bindings:
//
......@@ -184,6 +232,7 @@ export async function loadBindings(): Promise<Binding> {
// that host system where generated package lock is not matching to the guest system running on, try to manually
// download corresponding target triple and load it. This won't be triggered if native bindings are failed to load
// with other reasons than `ERR_MODULE_NOT_FOUND`.
// - Lastly, falls back to wasm binding where possible.
try {
return resolve(loadNative())
} catch (a) {
......@@ -201,7 +250,7 @@ export async function loadBindings(): Promise<Binding> {
attempts = attempts.concat(a)
}
logLoadFailure(attempts)
logLoadFailure(attempts, true)
})
return pendingBindings
}
......@@ -230,8 +279,6 @@ async function tryLoadNativeWithFallback(attempts: Array<string>) {
return undefined
}
// We may end up using this function in the future if/when we support wasm builds again
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function tryLoadWasmWithFallback(attempts: any) {
try {
let bindings = await loadWasm('')
......@@ -284,12 +331,18 @@ function loadBindingsSync() {
attempts = attempts.concat(a)
}
// we can leverage the wasm bindings if they are already
// loaded
if (wasmBindings) {
return wasmBindings
}
logLoadFailure(attempts)
}
let loggingLoadFailure = false
function logLoadFailure(attempts: any) {
function logLoadFailure(attempts: any, triedWasm = false) {
// make sure we only emit the event and log the failure once
if (loggingLoadFailure) return
loggingLoadFailure = true
......@@ -300,6 +353,7 @@ function logLoadFailure(attempts: any) {
// @ts-expect-error TODO: this event has a wrong type.
eventSwcLoadFailure({
wasm: triedWasm ? 'failed' : undefined,
nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode,
})
.then(() => lockfilePatchPromise.cur || Promise.resolve())
......
......@@ -370,7 +370,7 @@ export default async function getBaseWebpackConfig(
// eagerly load swc bindings instead of waiting for transform calls
if (!babelConfigFile && isClient) {
await loadBindings()
await loadBindings(config.experimental.useWasmBinary)
}
if (!loggedIgnoredCompilerOptions && !useSWCLoader && config.compiler) {
......
......@@ -59,6 +59,9 @@ async function getNextConfig() {
return {
output: config.output ?? 'N/A',
experimental: {
useWasmBinary: config.experimental?.useWasmBinary,
},
}
}
......@@ -350,8 +353,11 @@ async function printVerbose() {
// First, try to load next-swc via loadBindings.
try {
let nextConfig = await getNextConfig()
const { loadBindings } = require('../build/swc')
const bindings = await loadBindings()
const bindings = await loadBindings(
nextConfig.experimental?.useWasmBinary
)
// Run arbitary function to verify the bindings are loaded correctly.
const target = bindings.getTargetTriple()
......
......@@ -367,6 +367,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
serverSourceMaps: z.boolean().optional(),
bundlePagesExternals: z.boolean().optional(),
staticWorkerRequestDeduping: z.boolean().optional(),
useWasmBinary: z.boolean().optional(),
})
.optional(),
exportPathMap: z
......
......@@ -338,6 +338,8 @@ export interface ExperimentalConfig {
* Uses an IPC server to dedupe build-time requests to the cache handler
*/
staticWorkerRequestDeduping?: boolean
useWasmBinary?: boolean
}
export type ExportPathMap = {
......@@ -787,7 +789,6 @@ export const defaultConfig: NextConfig = {
typedRoutes: false,
instrumentationHook: false,
bundlePagesExternals: false,
staticWorkerRequestDeduping: false,
},
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment