Unverified Commit 6f8c5803 authored by OJ Kwon's avatar OJ Kwon Committed by GitHub

feat(turbopack): Experimental wasm build (#57906)

### What?

continuation of https://github.com/vercel/next.js/pull/57851, since it is from a remote branch that I don't have access to write.
Co-authored-by: 's avatarMaia Teegarden <2865858+padmaia@users.noreply.github.com>
Co-authored-by: 's avatarTim Neutkens <6324199+timneutkens@users.noreply.github.com>
parent f979341f
...@@ -340,12 +340,65 @@ jobs: ...@@ -340,12 +340,65 @@ jobs:
name: turbo run summary name: turbo run summary
path: .turbo/runs path: .turbo/runs
build-wasm:
strategy:
matrix:
target: [web, nodejs]
runs-on:
- 'self-hosted'
- 'linux'
- 'x64'
- 'metal'
steps:
- uses: actions/checkout@v3
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_LTS_VERSION }}
check-latest: true
- run: corepack enable
- name: Install Rust
uses: ./.github/actions/setup-rust
with:
targets: wasm32-unknown-unknown
- run: npm i -g turbo@${{ env.TURBO_VERSION }}
- name: Install wasm-pack
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- name: normalize versions
run: node scripts/normalize-version-bump.js
- name: Build
run: turbo run build-wasm -vvv --remote-cache-timeout 90 --summarize -- --target ${{ matrix.target }} --features tracing/release_max_level_info
- name: Add target to folder name
run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm'
- name: Upload turbo summary artifact
uses: actions/upload-artifact@v3
with:
name: turbo run summary
path: .turbo/runs
- name: Upload swc artifact
uses: actions/upload-artifact@v3
with:
name: wasm-binaries
path: packages/next-swc/crates/wasm/pkg-*
publishRelease: publishRelease:
if: ${{ needs.build.outputs.isRelease == 'true' }} if: ${{ needs.build.outputs.isRelease == 'true' }}
name: Potentially publish release name: Potentially publish release
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: needs:
- build - build
- build-wasm
- build-native - build-native
permissions: permissions:
contents: write contents: write
...@@ -376,6 +429,11 @@ jobs: ...@@ -376,6 +429,11 @@ jobs:
name: next-swc-binaries name: next-swc-binaries
path: packages/next-swc/native path: packages/next-swc/native
- uses: actions/download-artifact@v3
with:
name: wasm-binaries
path: packages/next-swc/crates/wasm
- run: npm i -g npm@9.6.7 # need latest version for provenance (pinning to avoid bugs) - run: npm i -g npm@9.6.7 # need latest version for provenance (pinning to avoid bugs)
- run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc - run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
- run: ./scripts/publish-native.js - run: ./scripts/publish-native.js
......
...@@ -152,14 +152,14 @@ jobs: ...@@ -152,14 +152,14 @@ jobs:
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration
secrets: inherit secrets: inherit
# test-next-swc-wasm: test-next-swc-wasm:
# name: test next-swc wasm name: test next-swc wasm
# needs: ['build-native', 'build-next'] needs: ['build-native', 'build-next']
# uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
# with: with:
# skipForDocsOnly: 'yes' skipForDocsOnly: 'yes'
# afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
# secrets: inherit secrets: inherit
test-unit: test-unit:
name: test unit name: test unit
...@@ -262,6 +262,7 @@ jobs: ...@@ -262,6 +262,7 @@ jobs:
'test-cargo-integration', 'test-cargo-integration',
'test-cargo-bench', 'test-cargo-bench',
'rust-check', 'rust-check',
'test-next-swc-wasm',
'test-turbopack-dev', 'test-turbopack-dev',
'test-turbopack-integration', 'test-turbopack-integration',
] ]
......
...@@ -8,4 +8,6 @@ pnpm build ...@@ -8,4 +8,6 @@ pnpm build
By default, the latest canary of the `next-swc` binaries will be installed and used. If you are actively working on Rust code or you need to test out the most recent Rust code that hasn't been published as a canary yet, you can [install Rust](https://www.rust-lang.org/tools/install) and run `pnpm --filter=@next/swc build-native`. By default, the latest canary of the `next-swc` binaries will be installed and used. If you are actively working on Rust code or you need to test out the most recent Rust code that hasn't been published as a canary yet, you can [install Rust](https://www.rust-lang.org/tools/install) and run `pnpm --filter=@next/swc build-native`.
If you want to test out the wasm build locally, you will need to [install wasm-pack](https://rustwasm.github.io/wasm-pack/installer/). Run `pnpm --filter=@next/swc build-wasm --target <wasm_target>` to build and `node ./scripts/setup-wasm.mjs` to copy it into your `node_modules`. Run next with `NODE_OPTIONS='--no-addons'` to force it to use the wasm binary.
If you need to clean the project for any reason, use `pnpm clean`. If you need to clean the project for any reason, use `pnpm clean`.
...@@ -17,6 +17,7 @@ We chose to build on SWC for a few reasons: ...@@ -17,6 +17,7 @@ We chose to build on SWC for a few reasons:
- **Extensibility:** SWC can be used as a Crate inside Next.js, without having to fork the library or workaround design constraints. - **Extensibility:** SWC can be used as a Crate inside Next.js, without having to fork the library or workaround design constraints.
- **Performance:** We were able to achieve ~3x faster Fast Refresh and ~5x faster builds in Next.js by switching to SWC, with more room for optimization still in progress. - **Performance:** We were able to achieve ~3x faster Fast Refresh and ~5x faster builds in Next.js by switching to SWC, with more room for optimization still in progress.
- **WebAssembly:** Rust's support for WASM is essential for supporting all possible platforms and taking Next.js development everywhere.
- **Community:** The Rust community and ecosystem are amazing and still growing. - **Community:** The Rust community and ecosystem are amazing and still growing.
## Supported Features ## Supported Features
......
...@@ -1030,7 +1030,7 @@ export default async function build( ...@@ -1030,7 +1030,7 @@ export default async function build(
const turboJson = findUp.sync('turbo.json', { cwd: dir }) const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow // eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir }) const packagePath = findUp.sync('package.json', { cwd: dir })
let binding = await loadBindings() let binding = await loadBindings(config?.experimental?.useWasmBinary)
let root = let root =
turboNextBuildRoot ?? turboNextBuildRoot ??
......
...@@ -89,7 +89,7 @@ export default function nextJest(options: { dir?: string } = {}) { ...@@ -89,7 +89,7 @@ export default function nextJest(options: { dir?: string } = {}) {
: customJestConfig) ?? {} : customJestConfig) ?? {}
// eagerly load swc bindings instead of waiting for transform calls // eagerly load swc bindings instead of waiting for transform calls
await loadBindings() await loadBindings(nextConfig?.experimental?.useWasmBinary)
if (lockfilePatchPromise.cur) { if (lockfilePatchPromise.cur) {
await lockfilePatchPromise.cur await lockfilePatchPromise.cur
......
...@@ -31,7 +31,7 @@ const infoLog = (...args: any[]) => { ...@@ -31,7 +31,7 @@ const infoLog = (...args: any[]) => {
* Based on napi-rs's target triples, returns triples that have corresponding next-swc binaries. * Based on napi-rs's target triples, returns triples that have corresponding next-swc binaries.
*/ */
export const getSupportedArchTriples: () => Record<string, any> = () => { export const getSupportedArchTriples: () => Record<string, any> = () => {
const { darwin, win32, linux } = platformArchTriples const { darwin, win32, linux, freebsd, android } = platformArchTriples
return { return {
darwin, darwin,
...@@ -48,6 +48,16 @@ export const getSupportedArchTriples: () => Record<string, any> = () => { ...@@ -48,6 +48,16 @@ export const getSupportedArchTriples: () => Record<string, any> = () => {
(triple: { abi: string }) => triple.abi !== 'gnux32' (triple: { abi: string }) => triple.abi !== 'gnux32'
), ),
arm64: linux.arm64, arm64: linux.arm64,
// This target is being deprecated, however we keep it in `knownDefaultWasmFallbackTriples` for now
arm: linux.arm,
},
// Below targets are being deprecated, however we keep it in `knownDefaultWasmFallbackTriples` for now
freebsd: {
x64: freebsd.x64,
},
android: {
arm64: android.arm64,
arm: android.arm,
}, },
} }
} }
...@@ -67,7 +77,7 @@ const triples = (() => { ...@@ -67,7 +77,7 @@ const triples = (() => {
if (rawTargetTriple) { if (rawTargetTriple) {
Log.warn( Log.warn(
`Trying to load next-swc for target triple ${rawTargetTriple}, but this target is not supported` `Trying to load next-swc for target triple ${rawTargetTriple}, but there next-swc does not have native bindings support`
) )
} else { } else {
Log.warn( Log.warn(
...@@ -97,6 +107,21 @@ function checkVersionMismatch(pkgData: any) { ...@@ -97,6 +107,21 @@ function checkVersionMismatch(pkgData: any) {
} }
} }
// These are the platforms we'll try to load wasm bindings first,
// only try to load native bindings if loading wasm binding somehow fails.
// Fallback to native binding is for migration period only,
// once we can verify loading-wasm-first won't cause visible regressions,
// we'll not include native bindings for these platform at all.
const knownDefaultWasmFallbackTriples = [
'x86_64-unknown-freebsd',
'aarch64-linux-android',
'arm-linux-androideabi',
'armv7-unknown-linux-gnueabihf',
'i686-pc-windows-msvc',
// WOA targets are TBD, while current userbase is small we may support it in the future
//'aarch64-pc-windows-msvc',
]
// The last attempt's error code returned when cjs require to native bindings fails. // The last attempt's error code returned when cjs require to native bindings fails.
// If node.js throws an error without error code, this should be `unknown` instead of undefined. // If node.js throws an error without error code, this should be `unknown` instead of undefined.
// For the wasm-first targets (`knownDefaultWasmFallbackTriples`) this will be `unsupported_target`. // For the wasm-first targets (`knownDefaultWasmFallbackTriples`) this will be `unsupported_target`.
...@@ -149,7 +174,9 @@ export interface Binding { ...@@ -149,7 +174,9 @@ export interface Binding {
teardownCrashReporter?: any teardownCrashReporter?: any
} }
export async function loadBindings(): Promise<Binding> { export async function loadBindings(
useWasmBinary: boolean = false
): Promise<Binding> {
if (pendingBindings) { if (pendingBindings) {
return pendingBindings return pendingBindings
} }
...@@ -176,6 +203,27 @@ export async function loadBindings(): Promise<Binding> { ...@@ -176,6 +203,27 @@ export async function loadBindings(): Promise<Binding> {
} }
let attempts: any[] = [] let attempts: any[] = []
const disableWasmFallback = process.env.NEXT_DISABLE_SWC_WASM
const unsupportedPlatform = triples.some(
(triple: any) =>
!!triple?.raw && knownDefaultWasmFallbackTriples.includes(triple.raw)
)
const shouldLoadWasmFallbackFirst =
!disableWasmFallback && unsupportedPlatform && useWasmBinary
if (!unsupportedPlatform && useWasmBinary) {
Log.warn(
`experimental.useWasmBinary is not an option for supported platform ${PlatformName}/${ArchName} and will be ignored.`
)
}
if (shouldLoadWasmFallbackFirst) {
lastNativeBindingsLoadErrorCode = 'unsupported_target'
const fallbackBindings = await tryLoadWasmWithFallback(attempts)
if (fallbackBindings) {
return resolve(fallbackBindings)
}
}
// Trickle down loading `fallback` bindings: // Trickle down loading `fallback` bindings:
// //
...@@ -184,6 +232,7 @@ export async function loadBindings(): Promise<Binding> { ...@@ -184,6 +232,7 @@ export async function loadBindings(): Promise<Binding> {
// that host system where generated package lock is not matching to the guest system running on, try to manually // that host system where generated package lock is not matching to the guest system running on, try to manually
// download corresponding target triple and load it. This won't be triggered if native bindings are failed to load // download corresponding target triple and load it. This won't be triggered if native bindings are failed to load
// with other reasons than `ERR_MODULE_NOT_FOUND`. // with other reasons than `ERR_MODULE_NOT_FOUND`.
// - Lastly, falls back to wasm binding where possible.
try { try {
return resolve(loadNative()) return resolve(loadNative())
} catch (a) { } catch (a) {
...@@ -201,7 +250,7 @@ export async function loadBindings(): Promise<Binding> { ...@@ -201,7 +250,7 @@ export async function loadBindings(): Promise<Binding> {
attempts = attempts.concat(a) attempts = attempts.concat(a)
} }
logLoadFailure(attempts) logLoadFailure(attempts, true)
}) })
return pendingBindings return pendingBindings
} }
...@@ -230,8 +279,6 @@ async function tryLoadNativeWithFallback(attempts: Array<string>) { ...@@ -230,8 +279,6 @@ async function tryLoadNativeWithFallback(attempts: Array<string>) {
return undefined return undefined
} }
// We may end up using this function in the future if/when we support wasm builds again
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function tryLoadWasmWithFallback(attempts: any) { async function tryLoadWasmWithFallback(attempts: any) {
try { try {
let bindings = await loadWasm('') let bindings = await loadWasm('')
...@@ -284,12 +331,18 @@ function loadBindingsSync() { ...@@ -284,12 +331,18 @@ function loadBindingsSync() {
attempts = attempts.concat(a) attempts = attempts.concat(a)
} }
// we can leverage the wasm bindings if they are already
// loaded
if (wasmBindings) {
return wasmBindings
}
logLoadFailure(attempts) logLoadFailure(attempts)
} }
let loggingLoadFailure = false let loggingLoadFailure = false
function logLoadFailure(attempts: any) { function logLoadFailure(attempts: any, triedWasm = false) {
// make sure we only emit the event and log the failure once // make sure we only emit the event and log the failure once
if (loggingLoadFailure) return if (loggingLoadFailure) return
loggingLoadFailure = true loggingLoadFailure = true
...@@ -300,6 +353,7 @@ function logLoadFailure(attempts: any) { ...@@ -300,6 +353,7 @@ function logLoadFailure(attempts: any) {
// @ts-expect-error TODO: this event has a wrong type. // @ts-expect-error TODO: this event has a wrong type.
eventSwcLoadFailure({ eventSwcLoadFailure({
wasm: triedWasm ? 'failed' : undefined,
nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode, nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode,
}) })
.then(() => lockfilePatchPromise.cur || Promise.resolve()) .then(() => lockfilePatchPromise.cur || Promise.resolve())
......
...@@ -370,7 +370,7 @@ export default async function getBaseWebpackConfig( ...@@ -370,7 +370,7 @@ export default async function getBaseWebpackConfig(
// eagerly load swc bindings instead of waiting for transform calls // eagerly load swc bindings instead of waiting for transform calls
if (!babelConfigFile && isClient) { if (!babelConfigFile && isClient) {
await loadBindings() await loadBindings(config.experimental.useWasmBinary)
} }
if (!loggedIgnoredCompilerOptions && !useSWCLoader && config.compiler) { if (!loggedIgnoredCompilerOptions && !useSWCLoader && config.compiler) {
......
...@@ -59,6 +59,9 @@ async function getNextConfig() { ...@@ -59,6 +59,9 @@ async function getNextConfig() {
return { return {
output: config.output ?? 'N/A', output: config.output ?? 'N/A',
experimental: {
useWasmBinary: config.experimental?.useWasmBinary,
},
} }
} }
...@@ -350,8 +353,11 @@ async function printVerbose() { ...@@ -350,8 +353,11 @@ async function printVerbose() {
// First, try to load next-swc via loadBindings. // First, try to load next-swc via loadBindings.
try { try {
let nextConfig = await getNextConfig()
const { loadBindings } = require('../build/swc') const { loadBindings } = require('../build/swc')
const bindings = await loadBindings() const bindings = await loadBindings(
nextConfig.experimental?.useWasmBinary
)
// Run arbitary function to verify the bindings are loaded correctly. // Run arbitary function to verify the bindings are loaded correctly.
const target = bindings.getTargetTriple() const target = bindings.getTargetTriple()
......
...@@ -367,6 +367,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() => ...@@ -367,6 +367,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
serverSourceMaps: z.boolean().optional(), serverSourceMaps: z.boolean().optional(),
bundlePagesExternals: z.boolean().optional(), bundlePagesExternals: z.boolean().optional(),
staticWorkerRequestDeduping: z.boolean().optional(), staticWorkerRequestDeduping: z.boolean().optional(),
useWasmBinary: z.boolean().optional(),
}) })
.optional(), .optional(),
exportPathMap: z exportPathMap: z
......
...@@ -338,6 +338,8 @@ export interface ExperimentalConfig { ...@@ -338,6 +338,8 @@ export interface ExperimentalConfig {
* Uses an IPC server to dedupe build-time requests to the cache handler * Uses an IPC server to dedupe build-time requests to the cache handler
*/ */
staticWorkerRequestDeduping?: boolean staticWorkerRequestDeduping?: boolean
useWasmBinary?: boolean
} }
export type ExportPathMap = { export type ExportPathMap = {
...@@ -787,7 +789,6 @@ export const defaultConfig: NextConfig = { ...@@ -787,7 +789,6 @@ export const defaultConfig: NextConfig = {
typedRoutes: false, typedRoutes: false,
instrumentationHook: false, instrumentationHook: false,
bundlePagesExternals: false, bundlePagesExternals: false,
staticWorkerRequestDeduping: false,
}, },
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment