Unverified Commit 96d955d0 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

fix(turbopack): bridge async N-API work on WASI

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 5e08d7d4
...@@ -679,6 +679,10 @@ jobs: ...@@ -679,6 +679,10 @@ jobs:
# next-napi-bindings), so nothing needs patching here: the wasm32 plugin path that does not # next-napi-bindings), so nothing needs patching here: the wasm32 plugin path that does not
# compile is simply never built. # compile is simply never built.
cargo build -p next-napi-bindings --target wasm32-wasip1-threads cargo build -p next-napi-bindings --target wasm32-wasip1-threads
timeout 60 node scripts/wasi-test-host/napi-async-smoke.mjs
cargo build --release -p next-napi-bindings --target wasm32-wasip1-threads
timeout 120 node scripts/wasi-test-host/napi-async-smoke.mjs \
target/wasm32-wasip1-threads/release/next_napi_bindings.wasm --error-path
# Run the crates whose tests can execute on wasm. This needs a host that supplies the # Run the crates whose tests can execute on wasm. This needs a host that supplies the
# `env.read_custom_section` import turbo-tasks' link-time registries depend on; no stock # `env.read_custom_section` import turbo-tasks' link-time registries depend on; no stock
......
...@@ -33,8 +33,9 @@ The script exports environment variables into the calling shell, which a subproc ...@@ -33,8 +33,9 @@ The script exports environment variables into the calling shell, which a subproc
Running it directly prints an error and does nothing. Running it directly prints an error and does nothing.
The script downloads the wasi-sdk matching your host architecture, verifies it against a pinned The script downloads the wasi-sdk matching your host architecture, verifies it against a pinned
sha256, installs emnapi, and exports the cross-compilation variables (`WASI_SDK_PATH`, sha256, installs exact compatible emnapi linker/runtime versions, and exports the cross-compilation
`EMNAPI_LINK_DIR`, and the `*_wasm32_wasip1_threads` compiler variables). Downloads are cached under variables (`WASI_SDK_PATH`, `EMNAPI_NODE_MODULES`, `EMNAPI_LINK_DIR`, and the
`*_wasm32_wasip1_threads` compiler variables). Downloads are cached under
`~/.cache/next-wasi-toolchain`, so re-sourcing in a new shell is fast. Set `WASI_SETUP_CACHE_DIR` to `~/.cache/next-wasi-toolchain`, so re-sourcing in a new shell is fast. Set `WASI_SETUP_CACHE_DIR` to
move the cache. move the cache.
......
...@@ -424,7 +424,10 @@ static PANIC_LOG: LazyLock<PathBuf> = LazyLock::new(|| { ...@@ -424,7 +424,10 @@ static PANIC_LOG: LazyLock<PathBuf> = LazyLock::new(|| {
// TODO: Now that we're passing the error to a JS callback, handle this logic in Next.js using the // TODO: Now that we're passing the error to a JS callback, handle this logic in Next.js using the
// logger there instead of writing directly to stderr. // logger there instead of writing directly to stderr.
pub fn log_internal_error_and_inform(internal_error: &anyhow::Error) { pub fn log_internal_error_and_inform(internal_error: &anyhow::Error) {
if cfg!(debug_assertions) // WASI preview1 does not implement `std::env::temp_dir()`. Always use stderr there, including
// release builds, so error reporting preserves the original diagnostic instead of trapping.
if cfg!(target_family = "wasm")
|| cfg!(debug_assertions)
|| env::var("SWC_DEBUG") == Ok("1".to_string()) || env::var("SWC_DEBUG") == Ok("1".to_string())
|| env::var("CI").is_ok_and(|v| !v.is_empty()) || env::var("CI").is_ok_and(|v| !v.is_empty())
// Next's run-tests unsets CI and sets NEXT_TEST_CI // Next's run-tests unsets CI and sets NEXT_TEST_CI
......
...@@ -62,6 +62,7 @@ async function listenForThreads(data: WasiThreadWorkerData) { ...@@ -62,6 +62,7 @@ async function listenForThreads(data: WasiThreadWorkerData) {
console.error(`wasi thread${suffix} failed:`, error) console.error(`wasi thread${suffix} failed:`, error)
process.exitCode = 1 process.exitCode = 1
}, },
beforeLoad: (worker) => napiModule.emnapi.addSendListener(worker),
}) })
const imports = createWasiImportObject({ const imports = createWasiImportObject({
module: wasmModule, module: wasmModule,
......
...@@ -198,6 +198,7 @@ test('passes Node parallelism to the main WASI instance', async () => { ...@@ -198,6 +198,7 @@ test('passes Node parallelism to the main WASI instance', async () => {
}), }),
napiModule: { napiModule: {
imports: {}, imports: {},
emnapi: { addSendListener: () => true },
init: (options) => initCalls.push(options), init: (options) => initCalls.push(options),
}, },
env: { KEEP: 'yes', TURBO_TASKS_AVAILABLE_PARALLELISM: '2' }, env: { KEEP: 'yes', TURBO_TASKS_AVAILABLE_PARALLELISM: '2' },
...@@ -287,6 +288,7 @@ test('transfers one compiled module through the shared thread manager', async () ...@@ -287,6 +288,7 @@ test('transfers one compiled module through the shared thread manager', async ()
shared: true, shared: true,
}) })
const ids = new Int32Array(new SharedArrayBuffer(4)) const ids = new Int32Array(new SharedArrayBuffer(4))
const listenerCalls = []
const { threadSpawn, manager } = await createThreadRuntime({ const { threadSpawn, manager } = await createThreadRuntime({
module, module,
memory, memory,
...@@ -301,11 +303,16 @@ test('transfers one compiled module through the shared thread manager', async () ...@@ -301,11 +303,16 @@ test('transfers one compiled module through the shared thread manager', async ()
workerPath: '/loader-worker.js', workerPath: '/loader-worker.js',
wasiThreadsModuleSpecifier: '/wasi-threads-provider.js', wasiThreadsModuleSpecifier: '/wasi-threads-provider.js',
onError: assert.fail, onError: assert.fail,
beforeLoad(worker) {
assert.deepEqual(worker.messages, [])
listenerCalls.push(worker)
},
WorkerClass: FakeWorker, WorkerClass: FakeWorker,
ThreadManagerClass: FakeThreadManager, ThreadManagerClass: FakeThreadManager,
}) })
assert.equal(threadSpawn(41), 1) assert.equal(threadSpawn(41), 1)
assert.deepEqual(listenerCalls, [workers[0]])
assert.equal(manager.module, module) assert.equal(manager.module, module)
assert.equal(manager.memory, memory) assert.equal(manager.memory, memory)
assert.equal(workers[0].options.workerData.threadIds, ids) assert.equal(workers[0].options.workerData.threadIds, ids)
......
...@@ -39,6 +39,9 @@ type ImportNamespace = Record<string, WebAssembly.ImportValue> ...@@ -39,6 +39,9 @@ type ImportNamespace = Record<string, WebAssembly.ImportValue>
export type NapiModuleLike = { export type NapiModuleLike = {
imports: WebAssembly.Imports imports: WebAssembly.Imports
emnapi: {
addSendListener(worker: unknown): boolean
}
init(options: { init(options: {
instance: WebAssembly.Instance instance: WebAssembly.Instance
module: WebAssembly.Module module: WebAssembly.Module
...@@ -183,6 +186,7 @@ export async function instantiateWasiNapiModule(options: { ...@@ -183,6 +186,7 @@ export async function instantiateWasiNapiModule(options: {
napiModuleSpecifier, napiModuleSpecifier,
}, },
onError: options.onThreadError, onError: options.onThreadError,
beforeLoad: (worker) => napiModule.emnapi.addSendListener(worker),
}) })
const imports = createWasiImportObject({ const imports = createWasiImportObject({
module, module,
......
...@@ -138,6 +138,7 @@ export async function createThreadRuntime(options: { ...@@ -138,6 +138,7 @@ export async function createThreadRuntime(options: {
workerPath?: string workerPath?: string
wasiThreadsModuleSpecifier?: string wasiThreadsModuleSpecifier?: string
onError: (error: Error, threadId: number | undefined) => void onError: (error: Error, threadId: number | undefined) => void
beforeLoad?: (worker: EmnapiWorkerLike) => void
WorkerClass?: WorkerConstructor WorkerClass?: WorkerConstructor
ThreadManagerClass?: ThreadManagerConstructor ThreadManagerClass?: ThreadManagerConstructor
}): Promise<{ }): Promise<{
...@@ -152,6 +153,7 @@ export async function createThreadRuntime(options: { ...@@ -152,6 +153,7 @@ export async function createThreadRuntime(options: {
workerPath = path.join(__dirname, 'wasi-loader-worker.js'), workerPath = path.join(__dirname, 'wasi-loader-worker.js'),
wasiThreadsModuleSpecifier = '@emnapi/wasi-threads', wasiThreadsModuleSpecifier = '@emnapi/wasi-threads',
onError, onError,
beforeLoad,
WorkerClass = Worker as unknown as WorkerConstructor, WorkerClass = Worker as unknown as WorkerConstructor,
} = options } = options
const ThreadManagerClass = const ThreadManagerClass =
...@@ -171,8 +173,8 @@ export async function createThreadRuntime(options: { ...@@ -171,8 +173,8 @@ export async function createThreadRuntime(options: {
const manager = new SharedIdThreadManager({ const manager = new SharedIdThreadManager({
reuseWorker: false, reuseWorker: false,
onCreateWorker: () => onCreateWorker: () => {
new WorkerClass(workerPath, { const worker = new WorkerClass(workerPath, {
workerData: { workerData: {
...workerData, ...workerData,
threadIds, threadIds,
...@@ -180,7 +182,13 @@ export async function createThreadRuntime(options: { ...@@ -180,7 +182,13 @@ export async function createThreadRuntime(options: {
}, },
stdout: false, stdout: false,
stderr: false, stderr: false,
}), })
// emnapi delivers async-work and thread-safe-function completions through messages from the
// pthread Worker. The listener must exist before ThreadManager sends its `load` message or a
// fast completion can be lost and the JavaScript Promise will never settle.
beforeLoad?.(worker)
return worker
},
printErr: (message) => onError(new Error(String(message)), undefined), printErr: (message) => onError(new Error(String(message)), undefined),
}) })
manager.init() manager.init()
......
...@@ -73,18 +73,26 @@ setup_wasi_env() { ...@@ -73,18 +73,26 @@ setup_wasi_env() {
# The archive must define emnapi_create_env / emnapi_delete_env, which only exist in emnapi v2. # The archive must define emnapi_create_env / emnapi_delete_env, which only exist in emnapi v2.
# That is still a prerelease, hence the exact alpha pin; move to the stable release once it ships. # That is still a prerelease, hence the exact alpha pin; move to the stable release once it ships.
# The archives are wasm objects, so they are arch-independent. # The archives are wasm objects, so they are arch-independent.
local emnapi_version=2.0.0-alpha.4 local emnapi_version=2.0.0-alpha.4 wasi_threads_version=2.1.0
local emnapi_dir="$cache_dir/emnapi" local emnapi_dir="$cache_dir/emnapi"
if [ ! -d "$emnapi_dir/node_modules/emnapi/lib/wasm32-wasip1-threads" ]; then if [ ! -d "$emnapi_dir/node_modules/emnapi/lib/wasm32-wasip1-threads" ] || \
[ ! -d "$emnapi_dir/node_modules/@emnapi/core" ] || \
[ ! -d "$emnapi_dir/node_modules/@emnapi/runtime" ] || \
[ ! -d "$emnapi_dir/node_modules/@emnapi/wasi-threads" ]; then
mkdir -p "$emnapi_dir" || return 1 mkdir -p "$emnapi_dir" || return 1
printf '{\n "name": "emnapi-scratch",\n "private": true\n}\n' > "$emnapi_dir/package.json" || return 1 printf '{\n "name": "emnapi-scratch",\n "private": true\n}\n' > "$emnapi_dir/package.json" || return 1
# pnpm is invoked from the repository root with --dir rather than by cd'ing into the scratch # pnpm is invoked from the repository root with --dir rather than by cd'ing into the scratch
# directory: corepack resolves the pnpm version from the nearest package.json, and outside the # directory: corepack resolves the pnpm version from the nearest package.json, and outside the
# repo there is none, so it would pick the latest pnpm (11.x), which cannot run on the pinned # repo there is none, so it would pick the latest pnpm (11.x), which cannot run on the pinned
# Node 20 (ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING). # Node 20 (ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING).
(cd "$repo_root" && pnpm --dir "$emnapi_dir" add "emnapi@${emnapi_version}") || return 1 (cd "$repo_root" && pnpm --dir "$emnapi_dir" add \
"emnapi@${emnapi_version}" \
"@emnapi/core@${emnapi_version}" \
"@emnapi/runtime@${emnapi_version}" \
"@emnapi/wasi-threads@${wasi_threads_version}") || return 1
fi fi
export EMNAPI_LINK_DIR="$emnapi_dir/node_modules/emnapi/lib/wasm32-wasip1-threads" export EMNAPI_NODE_MODULES="$emnapi_dir/node_modules"
export EMNAPI_LINK_DIR="$EMNAPI_NODE_MODULES/emnapi/lib/wasm32-wasip1-threads"
# Runs wasm test binaries under the Node WASI host, which supplies the `env.read_custom_section` # Runs wasm test binaries under the Node WASI host, which supplies the `env.read_custom_section`
# import that turbo-tasks' link-time registries need. # import that turbo-tasks' link-time registries need.
......
...@@ -45,8 +45,18 @@ Its own tests run against the same compiled production support: ...@@ -45,8 +45,18 @@ Its own tests run against the same compiled production support:
```sh ```sh
node --test packages/next/src/build/swc/wasi-loader.test.mjs node --test packages/next/src/build/swc/wasi-loader.test.mjs
source scripts/setup-wasi-env.sh
cargo build -p next-napi-bindings --target wasm32-wasip1-threads
node scripts/wasi-test-host/napi-async-smoke.mjs
``` ```
The async smoke uses the full emnapi runtime installed by `setup-wasi-env.sh`. It proves real
`projectNew` and `projectShutdown` promises complete across pthread Workers; the lower-crate Cargo
runner does not provide N-API imports and cannot run `next-napi-bindings` test artifacts directly.
It materializes the built `next` package in its temporary project: on Node 20.9, the WASI resolver
cannot follow the workspace's symlinked `node_modules/next`, which would prevent the error-path
smoke from reaching its intentionally invalid Sass configuration.
(Pass the file. `node --test <dir>` tries to resolve the directory as a module and fails.) (Pass the file. `node --test <dir>` tries to resolve the directory as a module and fails.)
## What it provides ## What it provides
...@@ -62,7 +72,8 @@ node --test packages/next/src/build/swc/wasi-loader.test.mjs ...@@ -62,7 +72,8 @@ node --test packages/next/src/build/swc/wasi-loader.test.mjs
`@emnapi/wasi-threads` owns Worker lifecycle, compiled-module transfer, load/start ordering, and `@emnapi/wasi-threads` owns Worker lifecycle, compiled-module transfer, load/start ordering, and
cleanup. A small local adapter remains because Rust imports the original one-argument cleanup. A small local adapter remains because Rust imports the original one-argument
`thread-spawn` ABI but does not export the `malloc`/`free` functions the package's high-level wrapper `thread-spawn` ABI but does not export the `malloc`/`free` functions the package's high-level wrapper
uses for that ABI. uses for that ABI. Every pthread Worker is registered with the main emnapi N-API instance before it
loads; async-work and thread-safe-function completions otherwise never reach the JavaScript Promise.
The imported shared memory uses 8,192 initial pages (512 MiB) and 65,536 maximum pages (4 GiB). The imported shared memory uses 8,192 initial pages (512 MiB) and 65,536 maximum pages (4 GiB).
Those values are also explicit `wasm32-wasip1-threads` linker flags in `.cargo/config.toml`; keep the Those values are also explicit `wasm32-wasip1-threads` linker flags in `.cargo/config.toml`; keep the
......
import assert from 'node:assert/strict'
import {
copyFile,
cp,
mkdir,
mkdtemp,
readFile,
realpath,
rm,
symlink,
writeFile,
} from 'node:fs/promises'
import { createRequire } from 'node:module'
import path from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
const repoRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../..'
)
const errorPath = process.argv.includes('--error-path')
const wasmArgument = process.argv
.slice(2)
.find((argument) => argument.endsWith('.wasm'))
const wasmPath = path.resolve(
wasmArgument ??
path.join(
repoRoot,
'target/wasm32-wasip1-threads/debug/next_napi_bindings.wasm'
)
)
const emnapiLinkDir = process.env.EMNAPI_LINK_DIR
assert.ok(
emnapiLinkDir,
'EMNAPI_LINK_DIR is missing; source scripts/setup-wasi-env.sh first'
)
const emnapiNodeModules =
process.env.EMNAPI_NODE_MODULES ?? path.resolve(emnapiLinkDir, '../../..')
const coreUrl = pathToFileURL(
path.join(emnapiNodeModules, '@emnapi/core/dist/emnapi-core.full.js')
).href
const runtimeUrl = pathToFileURL(
path.join(emnapiNodeModules, '@emnapi/runtime/dist/emnapi.js')
).href
const emnapiRequire = createRequire(
path.join(emnapiNodeModules, '../package.json')
)
const wasiThreadsUrl = pathToFileURL(
emnapiRequire.resolve('@emnapi/wasi-threads')
).href
const workerPath = path.join(
repoRoot,
'packages/next/dist/build/swc/wasi-loader-worker.js'
)
const [
{ instantiateWasiNapiModule },
{ createNapiModule },
{ getDefaultContext },
] = await Promise.all([
import(
pathToFileURL(
path.join(repoRoot, 'packages/next/dist/build/swc/wasi-loader.js')
).href
),
import(coreUrl),
import(runtimeUrl),
])
const napiModule = createNapiModule({
context: getDefaultContext(),
asyncWorkPoolSize: 0,
})
const loaded = await instantiateWasiNapiModule({
bytes: new Uint8Array(await readFile(wasmPath)),
napiModule,
args: [wasmPath],
env: {
...process.env,
TURBO_TASKS_AVAILABLE_PARALLELISM: '2',
},
// This is trusted Next.js code, not a sandbox. Matching native path semantics requires host
// absolute paths to remain guest absolute paths.
preopens: { '/': '/' },
workerPath,
napiModuleSpecifier: coreUrl,
wasiThreadsModuleSpecifier: wasiThreadsUrl,
onThreadError(error, threadId) {
throw new Error(`WASI worker ${threadId ?? 'unknown'} failed`, {
cause: error,
})
},
})
assert.equal(napiModule.exports.getTargetTriple(), 'wasm32-wasip1-threads')
assert.equal(Atomics.load(loaded.threadIds, 0), 2)
const fixtureRoot = path.join(
path.dirname(repoRoot),
'next-build-wasi-verification'
)
await mkdir(fixtureRoot, { recursive: true })
const fixturePath = await mkdtemp(
path.join(fixtureRoot, errorPath ? 'internal-error-' : 'async-work-')
)
await mkdir(path.join(fixturePath, 'app'))
await mkdir(path.join(fixturePath, 'node_modules/next'), { recursive: true })
// The WASI resolver on CI's Node 20.9 cannot traverse the workspace's symlinked
// node_modules/next. Materialize the package in the fixture so the error-path
// smoke actually reaches the Sass loader instead of returning a resolve issue.
await copyFile(
path.join(repoRoot, 'packages/next/package.json'),
path.join(fixturePath, 'node_modules/next/package.json')
)
await cp(
path.join(repoRoot, 'packages/next/dist'),
path.join(fixturePath, 'node_modules/next/dist'),
{ recursive: true }
)
for (const name of ['react', 'react-dom']) {
await symlink(
await realpath(path.join(repoRoot, 'node_modules', name)),
path.join(fixturePath, 'node_modules', name),
'dir'
)
}
await writeFile(
path.join(fixturePath, 'package.json'),
JSON.stringify({ private: true })
)
await writeFile(
path.join(fixturePath, 'app/layout.tsx'),
`${errorPath ? "import './style.scss'\n" : ''}export default function Layout({ children }) { return <html><body>{children}</body></html> }`
)
await writeFile(
path.join(fixturePath, 'app/page.tsx'),
'export default function Page() { return <main>WASI async smoke</main> }'
)
if (errorPath) {
// Force Sass loader configuration to run rather than relying on incidental graph traversal.
await writeFile(
path.join(fixturePath, 'app/style.scss'),
'main { color: red; }'
)
}
const rootPath = path.dirname(repoRoot)
const projectPath = path.relative(rootPath, fixturePath)
let reportedInternalError
let project
try {
const result = await napiModule.exports.projectNew(
{
rootPath,
projectPath,
additionalRoots: [],
distDir: path.join(fixturePath, '.next'),
watch: { enable: false },
nextConfig: JSON.stringify({
distDir: '.next',
distDirRoot: `${projectPath}/.next`,
pageExtensions: ['tsx', 'ts', 'jsx', 'js'],
...(errorPath ? { sassOptions: 'invalid' } : {}),
}),
env: [],
defineEnv: { client: [], edge: [], nodejs: [] },
dev: false,
encryptionKey: '00000000000000000000000000000000',
buildId: 'wasi-async-smoke',
previewProps: {
previewModeId: '00000000000000000000000000000000',
previewModeEncryptionKey: '00000000000000000000000000000000',
previewModeSigningKey: '00000000000000000000000000000000',
},
browserslistQuery: 'Chrome 111',
noMangling: false,
writeRoutesHashesManifest: false,
currentNodeJsVersion: process.versions.node,
isPersistentCachingEnabled: false,
nextVersion: 'test',
},
{
turbopackMemoryEviction: 'off',
dependencyTracking: false,
isCi: false,
isShortSession: true,
skipCompaction: true,
},
{
throwTurbopackInternalError(error, options) {
reportedInternalError =
error ?? new Error(options?.message ?? 'Turbopack internal error')
throw reportedInternalError
},
}
)
assert.ok(result.value?.project)
assert.ok(Array.isArray(result.issues))
project = result.value.project
if (errorPath) {
await assert.rejects(
napiModule.exports.projectWriteAllEntrypointsToDisk(project, true),
/Failed to write app endpoint|sass_options must be an object/
)
assert.match(
String(reportedInternalError),
/sass_options must be an object/
)
console.log(
'release internal error preserved the original Turbopack diagnostic'
)
}
await napiModule.exports.projectShutdown(project)
project = undefined
} finally {
if (project) await napiModule.exports.projectShutdown(project)
await rm(fixturePath, { recursive: true, force: true })
}
console.log(
'async N-API projectNew and projectShutdown promises resolved through WASI pthread workers'
)
// @emnapi/wasi-threads owns unref'ed Workers, but V8 may retain wasm pthread state long enough to
// delay Node's normal empty-loop exit. This smoke has completed every assertion, so exit directly.
process.exit(0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment