Unverified Commit e7bbe9fe authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

feat(next): add experimental next build --wasi

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 96d955d0
......@@ -680,6 +680,7 @@ jobs:
# compile is simply never built.
cargo build -p next-napi-bindings --target wasm32-wasip1-threads
timeout 60 node scripts/wasi-test-host/napi-async-smoke.mjs
timeout 300 node scripts/wasi-test-host/next-build-smoke.mjs
cargo build --release -p next-napi-bindings --target wasm32-wasip1-threads
timeout 120 node scripts/wasi-test-host/napi-async-smoke.mjs \
target/wasm32-wasip1-threads/release/next_napi_bindings.wasm --error-path
......
......@@ -208,6 +208,7 @@ program
.option('--turbopack', 'Builds using Turbopack.')
.option('--webpack', 'Builds using the bundled webpack.')
.option('--custom-webpack', 'Builds using your project-installed webpack.')
.option('--wasi', 'Builds using the experimental WASI bindings.')
.addOption(
new Option(
'--experimental-build-mode [mode]',
......
......@@ -128,7 +128,12 @@ async function handleCJS({
// resulting loading Development React on Production
const { loadBindings } = require('../swc') as typeof import('../swc')
const bindings = await loadBindings()
const { code } = await bindings.transform(nextConfigString, swcOptions)
// emnapi's JavaScript async-work pool requires another host Worker ABI. The synchronous N-API
// transform uses the already-selected WASI binding and is sufficient for one config file.
const { code } =
bindings.bindingType === 'wasi'
? bindings.transformSync(nextConfigString, swcOptions)
: await bindings.transform(nextConfigString, swcOptions)
// register require hook only if require exists
if (code.includes('require(')) {
......
This diff is collapsed.
......@@ -37,8 +37,13 @@ export interface TurbopackProjectCallbacks {
onBeforeDeferredEntries?: () => Promise<void>
}
export type BindingType = 'native' | 'wasm' | 'wasi'
export interface Binding {
/** Runtime kind. `isWasm` remains true for both wasm-bindgen and N-API/WASI bindings. */
bindingType: BindingType
isWasm: boolean
supportsTurbopack: boolean
turbo: {
createProject(
options: ProjectOptions,
......
......@@ -7,8 +7,9 @@
* import. The common host support lives in `./wasi-runtime`; `scripts/wasi-test-host/` consumes the
* compiled copy so the production loader and test runner cannot drift.
*
* This is groundwork only — nothing here is wired into the SWC-only wasm fallback in `./index.ts`,
* and packaging is deliberately out of scope, so loading Turbopack from wasm does not work yet.
* `next build --wasi` uses this N-API/WASI host. It is intentionally separate from the legacy
* SWC-only wasm-bindgen fallback in `./index.ts` because the two bindings have different ABIs and
* capabilities.
*/
import path from 'node:path'
......
......@@ -2,6 +2,7 @@ import path from 'path'
import { validateTurboNextConfig } from '../../lib/turbopack-warning'
import { seedTurbopackCacheIfNeeded } from '../../lib/turbopack-cache-seed'
import { NextBuildContext } from '../build-context'
import * as Log from '../output/log'
import { createDefineEnv, getBindingsSync } from '../swc'
import {
handleRouteType,
......@@ -47,7 +48,7 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{
const startTime = process.hrtime()
const bindings = getBindingsSync() // our caller should have already loaded these
if (bindings.isWasm) {
if (!bindings.supportsTurbopack) {
throw new Error(
`Turbopack is not supported on this platform (${process.platform}/${process.arch}) because native bindings are not available. ` +
`Only WebAssembly (WASM) bindings were loaded, and Turbopack requires native bindings.\n\n` +
......@@ -64,8 +65,15 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{
const hasDeferredEntries =
(config.experimental.deferredEntries?.length ?? 0) > 0
const persistentCaching =
const configuredPersistentCaching =
config.experimental?.turbopackFileSystemCacheForBuild || false
if (bindings.bindingType === 'wasi' && configuredPersistentCaching) {
Log.warn(
'Turbopack filesystem caching is disabled for --wasi because WASI file descriptors cannot be shared across pthread instances.'
)
}
const persistentCaching =
bindings.bindingType === 'wasi' ? false : configuredPersistentCaching
const rootPath = config.turbopack?.root || config.outputFileTracingRoot || dir
// Shared options for createProject calls
......
......@@ -24,6 +24,7 @@ export type NextBuildOptions = {
turbopack?: boolean
webpack?: boolean
customWebpack?: boolean
wasi?: boolean
experimentalDebugMemoryUsage: boolean
experimentalAppOnly?: boolean
experimentalTurbo?: boolean
......@@ -77,6 +78,18 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
printAndExit('--analyze is only compatible with the Turbopack bundler.')
}
if (options.wasi && bundler !== Bundler.Turbopack) {
printAndExit('--wasi is only compatible with the Turbopack bundler.')
}
if (options.wasi && (analyze || experimentalAnalyze)) {
printAndExit('--wasi does not yet support --analyze.')
}
if (options.wasi) {
// Private process-wide selection is inherited by Next build workers. It is set before config
// loading so a next.config.ts transform uses the same N-API/WASI binding as the build.
process.env.NEXT_PRIVATE_BUILD_WASI = '1'
}
if (!mangling) {
warn(
`Mangling is disabled. ${italic('Note: This may affect performance and should only be used for debugging purposes.')}`
......@@ -124,6 +137,7 @@ const nextBuild = async (options: NextBuildOptions, directory?: string) => {
experimentalBuildMode:
experimentalBuildMode !== 'default' ? experimentalBuildMode : undefined,
experimentalCpuProf: options.experimentalCpuProf,
wasi: options.wasi,
}).filter(([_, value]) => value !== undefined && value !== false)
)
......
......@@ -48,6 +48,7 @@ node --test packages/next/src/build/swc/wasi-loader.test.mjs
source scripts/setup-wasi-env.sh
cargo build -p next-napi-bindings --target wasm32-wasip1-threads
node scripts/wasi-test-host/napi-async-smoke.mjs
node scripts/wasi-test-host/next-build-smoke.mjs
```
The async smoke uses the full emnapi runtime installed by `setup-wasi-env.sh`. It proves real
......@@ -56,6 +57,11 @@ runner does not provide N-API imports and cannot run `next-napi-bindings` test a
It materializes the built `next` package in its temporary project: on Node 20.9, the WASI resolver
cannot follow the workspace's symlinked `node_modules/next`, which would prevent the error-path
smoke from reaching its intentionally invalid Sass configuration.
The build smoke invokes the normal `next build --wasi` CLI with a TypeScript config and proves the
parent plus every build worker selected WASI without creating persistent cache files. Its temporary
project materializes the built `next` package and its compile-time dependencies rather than relying
on symlinks the Node 20.9 WASI filesystem resolver cannot traverse. Its CLI runs from the same copy
that the project builds against, so Node workers share the expected package module identity.
(Pass the file. `node --test <dir>` tries to resolve the directory as a module and fails.)
......
import assert from 'node:assert/strict'
import { execFile as execFileCallback } from 'node:child_process'
import {
copyFile,
cp,
mkdir,
mkdtemp,
readFile,
readdir,
realpath,
rm,
symlink,
writeFile,
} from 'node:fs/promises'
import { createRequire } from 'node:module'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { promisify } from 'node:util'
const execFile = promisify(execFileCallback)
const repoRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../..'
)
const scratchRoot = await mkdtemp(
path.join(path.dirname(repoRoot), 'next-wasi-build-smoke-')
)
const sitePath = path.join(scratchRoot, 'site')
const packagePath = path.join(scratchRoot, 'wasi-package')
const targetLog = path.join(scratchRoot, 'targets.jsonl')
const nextBin = path.join(sitePath, 'node_modules/next/dist/bin/next')
const wasmPath = path.resolve(
process.argv[2] ??
path.join(
repoRoot,
'target/wasm32-wasip1-threads/debug/next_napi_bindings.wasm'
)
)
const emnapiLinkDir = process.env.EMNAPI_LINK_DIR
assert.ok(
emnapiLinkDir,
'EMNAPI_LINK_DIR is missing; source scripts/setup-wasi-env.sh first'
)
const emnapiNodeModules = path.resolve(emnapiLinkDir, '../../..')
const require = createRequire(import.meta.url)
async function writeConfig(experimental = {}) {
await writeFile(
path.join(sitePath, 'next.config.ts'),
`import type { NextConfig } from 'next'
const config: NextConfig = {
turbopack: { root: ${JSON.stringify(path.dirname(repoRoot))} },
experimental: ${JSON.stringify(experimental)},
}
export default config
`
)
}
async function runBuild(args, env = {}) {
return execFile(process.execPath, [nextBin, 'build', ...args], {
cwd: sitePath,
env: {
...process.env,
NEXT_TELEMETRY_DISABLED: '1',
NEXT_TEST_WASI_DIR: packagePath,
NEXT_TEST_NATIVE_DIR: path.join(scratchRoot, 'native-must-not-load'),
NEXT_TEST_WASI_TARGET_LOG: targetLog,
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--max-old-space-size=4096']
.filter(Boolean)
.join(' '),
...env,
},
maxBuffer: 50 * 1024 * 1024,
timeout: 240_000,
})
}
async function expectFailure(args, pattern, env) {
let failure
try {
await runBuild(args, env)
} catch (error) {
failure = error
}
assert.ok(failure, `next build ${args.join(' ')} unexpectedly succeeded`)
assert.match(`${failure.stdout ?? ''}\n${failure.stderr ?? ''}`, pattern)
}
try {
await mkdir(path.join(sitePath, 'app'), { recursive: true })
await mkdir(path.join(packagePath, 'node_modules/@emnapi'), {
recursive: true,
})
// The WASI filesystem resolver on Node 20.9 cannot follow the workspace's
// symlinked node_modules/next. Put the built package at the project's actual
// resolution path while retaining the other workspace dependencies.
await mkdir(path.join(sitePath, 'node_modules/next'), { recursive: true })
await copyFile(
path.join(repoRoot, 'packages/next/package.json'),
path.join(sitePath, 'node_modules/next/package.json')
)
await cp(
path.join(repoRoot, 'packages/next/dist'),
path.join(sitePath, 'node_modules/next/dist'),
{ recursive: true }
)
for (const name of ['index.d.ts', 'types.js', 'types.d.ts']) {
await copyFile(
path.join(repoRoot, 'packages/next', name),
path.join(sitePath, 'node_modules/next', name)
)
}
for (const name of ['types', 'image-types']) {
await cp(
path.join(repoRoot, 'packages/next', name),
path.join(sitePath, 'node_modules/next', name),
{ recursive: true }
)
}
for (const name of ['react', 'react-dom', '@swc/helpers']) {
await mkdir(path.dirname(path.join(sitePath, 'node_modules', name)), {
recursive: true,
})
await cp(
await realpath(path.join(repoRoot, 'node_modules', name)),
path.join(sitePath, 'node_modules', name),
{ recursive: true }
)
}
const reactDomRequire = createRequire(
path.join(
await realpath(path.join(repoRoot, 'node_modules/react-dom')),
'package.json'
)
)
await cp(
path.dirname(reactDomRequire.resolve('scheduler/package.json')),
path.join(sitePath, 'node_modules/scheduler'),
{ recursive: true }
)
await mkdir(path.join(sitePath, 'node_modules/@types'))
await mkdir(path.join(sitePath, 'node_modules/@next'))
for (const name of [
'typescript',
'@types/react',
'@types/node',
'@next/env',
'baseline-browser-mapping',
'caniuse-lite',
'postcss',
'styled-jsx',
]) {
let source
try {
source = await realpath(path.join(repoRoot, 'node_modules', name))
} catch (error) {
if (error.code !== 'ENOENT') throw error
source = await realpath(
path.join(repoRoot, 'packages/next/node_modules', name)
)
}
await symlink(source, path.join(sitePath, 'node_modules', name), 'dir')
}
await symlink(
path.join(emnapiNodeModules, '@emnapi/core'),
path.join(packagePath, 'node_modules/@emnapi/core')
)
await symlink(
path.join(emnapiNodeModules, '@emnapi/runtime'),
path.join(packagePath, 'node_modules/@emnapi/runtime')
)
await symlink(
path.dirname(require.resolve('@emnapi/wasi-threads/package.json')),
path.join(packagePath, 'node_modules/@emnapi/wasi-threads')
)
await copyFile(wasmPath, path.join(packagePath, 'next-swc.wasm32-wasi.wasm'))
await writeFile(
path.join(packagePath, 'package.json'),
JSON.stringify({
name: '@next/swc-wasm-wasi',
version: '0.0.0-test',
private: true,
})
)
await writeFile(
path.join(sitePath, 'package.json'),
JSON.stringify({
private: true,
dependencies: {
next: JSON.parse(
await readFile(path.join(sitePath, 'node_modules/next/package.json'))
).version,
react: JSON.parse(
await readFile(path.join(sitePath, 'node_modules/react/package.json'))
).version,
'react-dom': JSON.parse(
await readFile(
path.join(sitePath, 'node_modules/react-dom/package.json')
)
).version,
},
})
)
await writeFile(
path.join(sitePath, 'app/layout.tsx'),
`import './style.css'
export default function Layout({ children }: { children: React.ReactNode }) {
return <html><body>{children}</body></html>
}
`
)
await writeFile(
path.join(sitePath, 'app/page.tsx'),
`export default function Page() {
return <main>Built by WASI</main>
}
`
)
await writeFile(
path.join(sitePath, 'app/style.css'),
'main { color: rgb(0 100 200); }\n'
)
await writeConfig({ turbopackFileSystemCacheForBuild: true })
const buildHelp = await runBuild(['--help'])
assert.match(
buildHelp.stdout,
/--wasi\s+Builds using the experimental WASI bindings/
)
const devHelp = await execFile(process.execPath, [nextBin, 'dev', '--help'], {
cwd: sitePath,
})
assert.doesNotMatch(devHelp.stdout, /--wasi/)
const { stdout, stderr } = await runBuild(['--wasi'])
const output = `${stdout}\n${stderr}`
assert.match(output, /Running next\.config\.ts/)
assert.match(output, /Compiled successfully/)
assert.match(output, /Route \(app\)/)
assert.equal(
output.match(/Turbopack filesystem caching is disabled for --wasi/g)
?.length,
1
)
assert.equal(
await readFile(path.join(sitePath, '.next/BUILD_ID'), 'utf8').then(Boolean),
true
)
assert.match(
await readFile(path.join(sitePath, '.next/server/app/index.html'), 'utf8'),
/Built by WASI/
)
const targetRecords = (await readFile(targetLog, 'utf8'))
.trim()
.split('\n')
.map((line) => JSON.parse(line))
assert.ok(targetRecords.length >= 2)
assert.ok(targetRecords.some(({ worker }) => !worker))
assert.ok(targetRecords.some(({ worker }) => worker))
assert.ok(
targetRecords.every(({ target }) => target === 'wasm32-wasip1-threads')
)
async function findSst(directory) {
for (const entry of await readdir(directory, { withFileTypes: true })) {
const entryPath = path.join(directory, entry.name)
if (entry.isDirectory()) {
if (await findSst(entryPath)) return true
} else if (entry.name.endsWith('.sst')) {
return true
}
}
return false
}
assert.equal(await findSst(path.join(sitePath, '.next')), false)
await expectFailure(
['--wasi', '--webpack'],
/--wasi is only compatible with the Turbopack bundler/
)
await writeConfig({ useWasmBinary: true })
await expectFailure(
['--wasi'],
/cannot be combined with `experimental\.useWasmBinary: true`/
)
await writeConfig()
await expectFailure(
['--wasi'],
/Failed to initialize @next\/swc-wasm-wasi@.*from .*missing-wasi-package/,
{ NEXT_TEST_WASI_DIR: path.join(scratchRoot, 'missing-wasi-package') }
)
console.log(
'next build --wasi completed through the parent and build workers; negative CLI cases passed'
)
} finally {
await rm(scratchRoot, { recursive: true, force: true })
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment