Unverified Commit 96d955d0 authored by vercel-fleet-prod[bot]'s avatar vercel-fleet-prod[bot] Committed by GitHub

fix(turbopack): bridge async N-API work on WASI

Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 5e08d7d4
......@@ -679,6 +679,10 @@ jobs:
# next-napi-bindings), so nothing needs patching here: the wasm32 plugin path that does not
# compile is simply never built.
cargo build -p next-napi-bindings --target wasm32-wasip1-threads
timeout 60 node scripts/wasi-test-host/napi-async-smoke.mjs
cargo build --release -p next-napi-bindings --target wasm32-wasip1-threads
timeout 120 node scripts/wasi-test-host/napi-async-smoke.mjs \
target/wasm32-wasip1-threads/release/next_napi_bindings.wasm --error-path
# Run the crates whose tests can execute on wasm. This needs a host that supplies the
# `env.read_custom_section` import turbo-tasks' link-time registries depend on; no stock
......
......@@ -33,8 +33,9 @@ The script exports environment variables into the calling shell, which a subproc
Running it directly prints an error and does nothing.
The script downloads the wasi-sdk matching your host architecture, verifies it against a pinned
sha256, installs emnapi, and exports the cross-compilation variables (`WASI_SDK_PATH`,
`EMNAPI_LINK_DIR`, and the `*_wasm32_wasip1_threads` compiler variables). Downloads are cached under
sha256, installs exact compatible emnapi linker/runtime versions, and exports the cross-compilation
variables (`WASI_SDK_PATH`, `EMNAPI_NODE_MODULES`, `EMNAPI_LINK_DIR`, and the
`*_wasm32_wasip1_threads` compiler variables). Downloads are cached under
`~/.cache/next-wasi-toolchain`, so re-sourcing in a new shell is fast. Set `WASI_SETUP_CACHE_DIR` to
move the cache.
......
......@@ -424,7 +424,10 @@ static PANIC_LOG: LazyLock<PathBuf> = LazyLock::new(|| {
// TODO: Now that we're passing the error to a JS callback, handle this logic in Next.js using the
// logger there instead of writing directly to stderr.
pub fn log_internal_error_and_inform(internal_error: &anyhow::Error) {
if cfg!(debug_assertions)
// WASI preview1 does not implement `std::env::temp_dir()`. Always use stderr there, including
// release builds, so error reporting preserves the original diagnostic instead of trapping.
if cfg!(target_family = "wasm")
|| cfg!(debug_assertions)
|| env::var("SWC_DEBUG") == Ok("1".to_string())
|| env::var("CI").is_ok_and(|v| !v.is_empty())
// Next's run-tests unsets CI and sets NEXT_TEST_CI
......
......@@ -62,6 +62,7 @@ async function listenForThreads(data: WasiThreadWorkerData) {
console.error(`wasi thread${suffix} failed:`, error)
process.exitCode = 1
},
beforeLoad: (worker) => napiModule.emnapi.addSendListener(worker),
})
const imports = createWasiImportObject({
module: wasmModule,
......
......@@ -198,6 +198,7 @@ test('passes Node parallelism to the main WASI instance', async () => {
}),
napiModule: {
imports: {},
emnapi: { addSendListener: () => true },
init: (options) => initCalls.push(options),
},
env: { KEEP: 'yes', TURBO_TASKS_AVAILABLE_PARALLELISM: '2' },
......@@ -287,6 +288,7 @@ test('transfers one compiled module through the shared thread manager', async ()
shared: true,
})
const ids = new Int32Array(new SharedArrayBuffer(4))
const listenerCalls = []
const { threadSpawn, manager } = await createThreadRuntime({
module,
memory,
......@@ -301,11 +303,16 @@ test('transfers one compiled module through the shared thread manager', async ()
workerPath: '/loader-worker.js',
wasiThreadsModuleSpecifier: '/wasi-threads-provider.js',
onError: assert.fail,
beforeLoad(worker) {
assert.deepEqual(worker.messages, [])
listenerCalls.push(worker)
},
WorkerClass: FakeWorker,
ThreadManagerClass: FakeThreadManager,
})
assert.equal(threadSpawn(41), 1)
assert.deepEqual(listenerCalls, [workers[0]])
assert.equal(manager.module, module)
assert.equal(manager.memory, memory)
assert.equal(workers[0].options.workerData.threadIds, ids)
......
......@@ -39,6 +39,9 @@ type ImportNamespace = Record<string, WebAssembly.ImportValue>
export type NapiModuleLike = {
imports: WebAssembly.Imports
emnapi: {
addSendListener(worker: unknown): boolean
}
init(options: {
instance: WebAssembly.Instance
module: WebAssembly.Module
......@@ -183,6 +186,7 @@ export async function instantiateWasiNapiModule(options: {
napiModuleSpecifier,
},
onError: options.onThreadError,
beforeLoad: (worker) => napiModule.emnapi.addSendListener(worker),
})
const imports = createWasiImportObject({
module,
......
......@@ -138,6 +138,7 @@ export async function createThreadRuntime(options: {
workerPath?: string
wasiThreadsModuleSpecifier?: string
onError: (error: Error, threadId: number | undefined) => void
beforeLoad?: (worker: EmnapiWorkerLike) => void
WorkerClass?: WorkerConstructor
ThreadManagerClass?: ThreadManagerConstructor
}): Promise<{
......@@ -152,6 +153,7 @@ export async function createThreadRuntime(options: {
workerPath = path.join(__dirname, 'wasi-loader-worker.js'),
wasiThreadsModuleSpecifier = '@emnapi/wasi-threads',
onError,
beforeLoad,
WorkerClass = Worker as unknown as WorkerConstructor,
} = options
const ThreadManagerClass =
......@@ -171,8 +173,8 @@ export async function createThreadRuntime(options: {
const manager = new SharedIdThreadManager({
reuseWorker: false,
onCreateWorker: () =>
new WorkerClass(workerPath, {
onCreateWorker: () => {
const worker = new WorkerClass(workerPath, {
workerData: {
...workerData,
threadIds,
......@@ -180,7 +182,13 @@ export async function createThreadRuntime(options: {
},
stdout: false,
stderr: false,
}),
})
// emnapi delivers async-work and thread-safe-function completions through messages from the
// pthread Worker. The listener must exist before ThreadManager sends its `load` message or a
// fast completion can be lost and the JavaScript Promise will never settle.
beforeLoad?.(worker)
return worker
},
printErr: (message) => onError(new Error(String(message)), undefined),
})
manager.init()
......
......@@ -73,18 +73,26 @@ setup_wasi_env() {
# The archive must define emnapi_create_env / emnapi_delete_env, which only exist in emnapi v2.
# That is still a prerelease, hence the exact alpha pin; move to the stable release once it ships.
# The archives are wasm objects, so they are arch-independent.
local emnapi_version=2.0.0-alpha.4
local emnapi_version=2.0.0-alpha.4 wasi_threads_version=2.1.0
local emnapi_dir="$cache_dir/emnapi"
if [ ! -d "$emnapi_dir/node_modules/emnapi/lib/wasm32-wasip1-threads" ]; then
if [ ! -d "$emnapi_dir/node_modules/emnapi/lib/wasm32-wasip1-threads" ] || \
[ ! -d "$emnapi_dir/node_modules/@emnapi/core" ] || \
[ ! -d "$emnapi_dir/node_modules/@emnapi/runtime" ] || \
[ ! -d "$emnapi_dir/node_modules/@emnapi/wasi-threads" ]; then
mkdir -p "$emnapi_dir" || return 1
printf '{\n "name": "emnapi-scratch",\n "private": true\n}\n' > "$emnapi_dir/package.json" || return 1
# pnpm is invoked from the repository root with --dir rather than by cd'ing into the scratch
# directory: corepack resolves the pnpm version from the nearest package.json, and outside the
# repo there is none, so it would pick the latest pnpm (11.x), which cannot run on the pinned
# Node 20 (ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING).
(cd "$repo_root" && pnpm --dir "$emnapi_dir" add "emnapi@${emnapi_version}") || return 1
(cd "$repo_root" && pnpm --dir "$emnapi_dir" add \
"emnapi@${emnapi_version}" \
"@emnapi/core@${emnapi_version}" \
"@emnapi/runtime@${emnapi_version}" \
"@emnapi/wasi-threads@${wasi_threads_version}") || return 1
fi
export EMNAPI_LINK_DIR="$emnapi_dir/node_modules/emnapi/lib/wasm32-wasip1-threads"
export EMNAPI_NODE_MODULES="$emnapi_dir/node_modules"
export EMNAPI_LINK_DIR="$EMNAPI_NODE_MODULES/emnapi/lib/wasm32-wasip1-threads"
# Runs wasm test binaries under the Node WASI host, which supplies the `env.read_custom_section`
# import that turbo-tasks' link-time registries need.
......
......@@ -45,8 +45,18 @@ Its own tests run against the same compiled production support:
```sh
node --test packages/next/src/build/swc/wasi-loader.test.mjs
source scripts/setup-wasi-env.sh
cargo build -p next-napi-bindings --target wasm32-wasip1-threads
node scripts/wasi-test-host/napi-async-smoke.mjs
```
The async smoke uses the full emnapi runtime installed by `setup-wasi-env.sh`. It proves real
`projectNew` and `projectShutdown` promises complete across pthread Workers; the lower-crate Cargo
runner does not provide N-API imports and cannot run `next-napi-bindings` test artifacts directly.
It materializes the built `next` package in its temporary project: on Node 20.9, the WASI resolver
cannot follow the workspace's symlinked `node_modules/next`, which would prevent the error-path
smoke from reaching its intentionally invalid Sass configuration.
(Pass the file. `node --test <dir>` tries to resolve the directory as a module and fails.)
## What it provides
......@@ -62,7 +72,8 @@ node --test packages/next/src/build/swc/wasi-loader.test.mjs
`@emnapi/wasi-threads` owns Worker lifecycle, compiled-module transfer, load/start ordering, and
cleanup. A small local adapter remains because Rust imports the original one-argument
`thread-spawn` ABI but does not export the `malloc`/`free` functions the package's high-level wrapper
uses for that ABI.
uses for that ABI. Every pthread Worker is registered with the main emnapi N-API instance before it
loads; async-work and thread-safe-function completions otherwise never reach the JavaScript Promise.
The imported shared memory uses 8,192 initial pages (512 MiB) and 65,536 maximum pages (4 GiB).
Those values are also explicit `wasm32-wasip1-threads` linker flags in `.cargo/config.toml`; keep the
......
import assert from 'node:assert/strict'
import {
copyFile,
cp,
mkdir,
mkdtemp,
readFile,
realpath,
rm,
symlink,
writeFile,
} from 'node:fs/promises'
import { createRequire } from 'node:module'
import path from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
const repoRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../..'
)
const errorPath = process.argv.includes('--error-path')
const wasmArgument = process.argv
.slice(2)
.find((argument) => argument.endsWith('.wasm'))
const wasmPath = path.resolve(
wasmArgument ??
path.join(
repoRoot,
'target/wasm32-wasip1-threads/debug/next_napi_bindings.wasm'
)
)
const emnapiLinkDir = process.env.EMNAPI_LINK_DIR
assert.ok(
emnapiLinkDir,
'EMNAPI_LINK_DIR is missing; source scripts/setup-wasi-env.sh first'
)
const emnapiNodeModules =
process.env.EMNAPI_NODE_MODULES ?? path.resolve(emnapiLinkDir, '../../..')
const coreUrl = pathToFileURL(
path.join(emnapiNodeModules, '@emnapi/core/dist/emnapi-core.full.js')
).href
const runtimeUrl = pathToFileURL(
path.join(emnapiNodeModules, '@emnapi/runtime/dist/emnapi.js')
).href
const emnapiRequire = createRequire(
path.join(emnapiNodeModules, '../package.json')
)
const wasiThreadsUrl = pathToFileURL(
emnapiRequire.resolve('@emnapi/wasi-threads')
).href
const workerPath = path.join(
repoRoot,
'packages/next/dist/build/swc/wasi-loader-worker.js'
)
const [
{ instantiateWasiNapiModule },
{ createNapiModule },
{ getDefaultContext },
] = await Promise.all([
import(
pathToFileURL(
path.join(repoRoot, 'packages/next/dist/build/swc/wasi-loader.js')
).href
),
import(coreUrl),
import(runtimeUrl),
])
const napiModule = createNapiModule({
context: getDefaultContext(),
asyncWorkPoolSize: 0,
})
const loaded = await instantiateWasiNapiModule({
bytes: new Uint8Array(await readFile(wasmPath)),
napiModule,
args: [wasmPath],
env: {
...process.env,
TURBO_TASKS_AVAILABLE_PARALLELISM: '2',
},
// This is trusted Next.js code, not a sandbox. Matching native path semantics requires host
// absolute paths to remain guest absolute paths.
preopens: { '/': '/' },
workerPath,
napiModuleSpecifier: coreUrl,
wasiThreadsModuleSpecifier: wasiThreadsUrl,
onThreadError(error, threadId) {
throw new Error(`WASI worker ${threadId ?? 'unknown'} failed`, {
cause: error,
})
},
})
assert.equal(napiModule.exports.getTargetTriple(), 'wasm32-wasip1-threads')
assert.equal(Atomics.load(loaded.threadIds, 0), 2)
const fixtureRoot = path.join(
path.dirname(repoRoot),
'next-build-wasi-verification'
)
await mkdir(fixtureRoot, { recursive: true })
const fixturePath = await mkdtemp(
path.join(fixtureRoot, errorPath ? 'internal-error-' : 'async-work-')
)
await mkdir(path.join(fixturePath, 'app'))
await mkdir(path.join(fixturePath, 'node_modules/next'), { recursive: true })
// The WASI resolver on CI's Node 20.9 cannot traverse the workspace's symlinked
// node_modules/next. Materialize the package in the fixture so the error-path
// smoke actually reaches the Sass loader instead of returning a resolve issue.
await copyFile(
path.join(repoRoot, 'packages/next/package.json'),
path.join(fixturePath, 'node_modules/next/package.json')
)
await cp(
path.join(repoRoot, 'packages/next/dist'),
path.join(fixturePath, 'node_modules/next/dist'),
{ recursive: true }
)
for (const name of ['react', 'react-dom']) {
await symlink(
await realpath(path.join(repoRoot, 'node_modules', name)),
path.join(fixturePath, 'node_modules', name),
'dir'
)
}
await writeFile(
path.join(fixturePath, 'package.json'),
JSON.stringify({ private: true })
)
await writeFile(
path.join(fixturePath, 'app/layout.tsx'),
`${errorPath ? "import './style.scss'\n" : ''}export default function Layout({ children }) { return <html><body>{children}</body></html> }`
)
await writeFile(
path.join(fixturePath, 'app/page.tsx'),
'export default function Page() { return <main>WASI async smoke</main> }'
)
if (errorPath) {
// Force Sass loader configuration to run rather than relying on incidental graph traversal.
await writeFile(
path.join(fixturePath, 'app/style.scss'),
'main { color: red; }'
)
}
const rootPath = path.dirname(repoRoot)
const projectPath = path.relative(rootPath, fixturePath)
let reportedInternalError
let project
try {
const result = await napiModule.exports.projectNew(
{
rootPath,
projectPath,
additionalRoots: [],
distDir: path.join(fixturePath, '.next'),
watch: { enable: false },
nextConfig: JSON.stringify({
distDir: '.next',
distDirRoot: `${projectPath}/.next`,
pageExtensions: ['tsx', 'ts', 'jsx', 'js'],
...(errorPath ? { sassOptions: 'invalid' } : {}),
}),
env: [],
defineEnv: { client: [], edge: [], nodejs: [] },
dev: false,
encryptionKey: '00000000000000000000000000000000',
buildId: 'wasi-async-smoke',
previewProps: {
previewModeId: '00000000000000000000000000000000',
previewModeEncryptionKey: '00000000000000000000000000000000',
previewModeSigningKey: '00000000000000000000000000000000',
},
browserslistQuery: 'Chrome 111',
noMangling: false,
writeRoutesHashesManifest: false,
currentNodeJsVersion: process.versions.node,
isPersistentCachingEnabled: false,
nextVersion: 'test',
},
{
turbopackMemoryEviction: 'off',
dependencyTracking: false,
isCi: false,
isShortSession: true,
skipCompaction: true,
},
{
throwTurbopackInternalError(error, options) {
reportedInternalError =
error ?? new Error(options?.message ?? 'Turbopack internal error')
throw reportedInternalError
},
}
)
assert.ok(result.value?.project)
assert.ok(Array.isArray(result.issues))
project = result.value.project
if (errorPath) {
await assert.rejects(
napiModule.exports.projectWriteAllEntrypointsToDisk(project, true),
/Failed to write app endpoint|sass_options must be an object/
)
assert.match(
String(reportedInternalError),
/sass_options must be an object/
)
console.log(
'release internal error preserved the original Turbopack diagnostic'
)
}
await napiModule.exports.projectShutdown(project)
project = undefined
} finally {
if (project) await napiModule.exports.projectShutdown(project)
await rm(fixturePath, { recursive: true, force: true })
}
console.log(
'async N-API projectNew and projectShutdown promises resolved through WASI pthread workers'
)
// @emnapi/wasi-threads owns unref'ed Workers, but V8 may retain wasm pthread state long enough to
// delay Node's normal empty-loop exit. This smoke has completed every assertion, so exit directly.
process.exit(0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment