Unverified Commit a4fff794 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: correctly reference injected variable for Wasm on Edge (#78771)

Closes PACK-4511

Concerning that we have test coverage for this never the deployment test never ran with Turbopack: https://github.com/vercel/next.js/pull/78768

This never (?) worked: 

```js
const wasm_vercel_og__resvg = require("...");
function loadWebAssemblyModule() {
	console.log(globalThis["wasm_vercel_og__resvg"]); // undefined
}
loadWebAssemblyModule();
```



Webpack generates the following, so let's do that as well
```js
const wasm_77d9faebf7af9e421806970ce10a58e9d83116d7 = require("/wasm/wasm_77d9faebf7af9e421806970ce10a58e9d83116d7.wasm");
.....
/***/ "(rsc)/../../node_modules/.pnpm/next@15.4.0-canary.12_react-dom@19.0.0_react@19.0.0/node_modules/next/dist/compiled/@vercel/og/resvg.wasm?module":
/***/ ((module) => {
module.exports = wasm_77d9faebf7af9e421806970ce10a58e9d83116d7;
/***/ }),
```
parent ec6c7108
......@@ -4508,6 +4508,7 @@ dependencies = [
"turbopack-env",
"turbopack-node",
"turbopack-nodejs",
"turbopack-wasm",
"vergen 9.0.5",
]
......@@ -10315,6 +10316,7 @@ dependencies = [
"turbo-tasks",
"turbo-tasks-build",
"turbo-tasks-fs",
"turbo-tasks-hash",
"turbopack-core",
"turbopack-ecmascript",
"wasmparser 0.110.0",
......
......@@ -40,6 +40,7 @@ turbopack-ecmascript = { workspace = true }
turbopack-env = { workspace = true }
turbopack-node = { workspace = true }
turbopack-nodejs = { workspace = true }
turbopack-wasm = { workspace = true }
[build-dependencies]
anyhow = { workspace = true }
......
......@@ -1556,7 +1556,7 @@ impl AppEndpoint {
};
let edge_function_definition = EdgeFunctionDefinition {
files: file_paths_from_root.into_iter().collect(),
wasm: wasm_paths_to_bindings(wasm_paths_from_root.into_iter().collect()),
wasm: wasm_paths_to_bindings(wasm_paths_from_root).await?,
assets: paths_to_bindings(all_assets),
name: app_entry.pathname.clone(),
page: app_entry.original_name.clone(),
......
......@@ -187,7 +187,7 @@ impl InstrumentationEndpoint {
let instrumentation_definition = InstrumentationDefinition {
files: file_paths_from_root,
wasm: wasm_paths_to_bindings(wasm_paths_from_root),
wasm: wasm_paths_to_bindings(wasm_paths_from_root).await?,
name: "instrumentation".into(),
..Default::default()
};
......
......@@ -304,7 +304,7 @@ impl MiddlewareEndpoint {
let edge_function_definition = EdgeFunctionDefinition {
files: file_paths_from_root,
wasm: wasm_paths_to_bindings(wasm_paths_from_root),
wasm: wasm_paths_to_bindings(wasm_paths_from_root).await?,
assets: paths_to_bindings(all_assets),
name: "middleware".into(),
page: "/".into(),
......
......@@ -30,8 +30,8 @@ use serde::{Deserialize, Serialize};
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{
fxindexmap, trace::TraceRawVcs, Completion, FxIndexMap, NonLocalValue, ResolvedVc, TaskInput,
Value, ValueToString, Vc,
fxindexmap, fxindexset, trace::TraceRawVcs, Completion, FxIndexMap, NonLocalValue, ResolvedVc,
TaskInput, Value, ValueToString, Vc,
};
use turbo_tasks_fs::{
self, File, FileContent, FileSystem, FileSystemPath, FileSystemPathOption, VirtualFileSystem,
......@@ -1375,7 +1375,7 @@ impl PageEndpoint {
"server/middleware-build-manifest.js".into(),
"server/next-font-manifest.js".into(),
];
let mut wasm_paths_from_root = vec![];
let mut wasm_paths_from_root = fxindexset![];
let node_root_value = node_root.await?;
......@@ -1404,7 +1404,7 @@ impl PageEndpoint {
let original_name = this.original_name.owned().await?;
let edge_function_definition = EdgeFunctionDefinition {
files: file_paths_from_root,
wasm: wasm_paths_to_bindings(wasm_paths_from_root),
wasm: wasm_paths_to_bindings(wasm_paths_from_root).await?,
assets: paths_to_bindings(all_assets),
name: pathname.clone(),
page: original_name.clone(),
......
......@@ -3,12 +3,15 @@ use next_core::{all_assets_from_entries, next_manifests::AssetBinding};
use serde::{Deserialize, Serialize};
use tracing::Instrument;
use turbo_rcstr::RcStr;
use turbo_tasks::{trace::TraceRawVcs, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, Vc};
use turbo_tasks::{
trace::TraceRawVcs, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc,
};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
};
use turbopack_wasm::wasm_edge_var_name;
/// A reference to a server file with content hash for change detection
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs, NonLocalValue)]
......@@ -109,8 +112,23 @@ pub(crate) async fn get_js_paths_from_root(
pub(crate) async fn get_wasm_paths_from_root(
root: &FileSystemPath,
output_assets: impl IntoIterator<Item = &ResolvedVc<Box<dyn OutputAsset>>>,
) -> Result<Vec<RcStr>> {
get_paths_from_root(root, output_assets, |path| path.ends_with(".wasm")).await
) -> Result<Vec<(RcStr, ResolvedVc<Box<dyn OutputAsset>>)>> {
output_assets
.into_iter()
.map(move |&file| async move {
let path = &*file.path().await?;
let Some(relative) = root.get_path_to(path) else {
return Ok(None);
};
Ok(if relative.ends_with(".wasm") {
Some((relative.into(), file))
} else {
None
})
})
.try_flat_join()
.await
}
pub(crate) async fn get_asset_paths_from_root(
......@@ -137,42 +155,19 @@ pub(crate) async fn get_font_paths_from_root(
.await
}
fn get_file_stem(path: &str) -> &str {
let file_name = if let Some((_, file_name)) = path.rsplit_once('/') {
file_name
} else {
path
};
if let Some((stem, _)) = file_name.split_once('.') {
if stem.is_empty() {
file_name
} else {
stem
}
} else {
file_name
}
}
pub(crate) fn wasm_paths_to_bindings(paths: Vec<RcStr>) -> Vec<AssetBinding> {
pub(crate) async fn wasm_paths_to_bindings(
paths: impl IntoIterator<Item = (RcStr, ResolvedVc<Box<dyn OutputAsset>>)>,
) -> Result<Vec<AssetBinding>> {
paths
.into_iter()
.map(|path| {
let stem = get_file_stem(&path);
// very simple escaping just replacing unsupported characters with `_`
let escaped = stem.replace(
|c: char| !c.is_ascii_alphanumeric() && c != '$' && c != '_',
"_",
);
AssetBinding {
name: format!("wasm_{}", escaped).into(),
.map(async |(path, asset)| {
Ok(AssetBinding {
name: wasm_edge_var_name(Vc::upcast(*asset)).owned().await?,
file_path: path,
}
})
})
.collect()
.try_join()
.await
}
pub(crate) fn paths_to_bindings(paths: Vec<RcStr>) -> Vec<AssetBinding> {
......
......@@ -14,10 +14,12 @@ declare var BACKEND: RuntimeBackend
declare var loadWebAssembly: (
source: SourceInfo,
wasmChunkPath: ChunkPath,
edgeModule: () => WebAssembly.Module,
imports: WebAssembly.Imports
) => Exports
declare var loadWebAssemblyModule: (
source: SourceInfo,
wasmChunkPath: ChunkPath
wasmChunkPath: ChunkPath,
edgeModule: () => WebAssembly.Module
) => WebAssembly.Module
declare var relativeURL: (inputUrl: string) => void
......@@ -31,6 +31,7 @@ function fetchWebAssembly(wasmChunkPath: ChunkPath) {
async function loadWebAssembly(
_source: unknown,
wasmChunkPath: ChunkPath,
_edgeModule: () => WebAssembly.Module,
importsObj: WebAssembly.Imports
): Promise<Exports> {
const req = fetchWebAssembly(wasmChunkPath)
......@@ -42,7 +43,8 @@ async function loadWebAssembly(
async function loadWebAssemblyModule(
_source: unknown,
wasmChunkPath: ChunkPath
wasmChunkPath: ChunkPath,
_edgeModule: () => WebAssembly.Module
): Promise<WebAssembly.Module> {
const req = fetchWebAssembly(wasmChunkPath)
......
......@@ -43,9 +43,10 @@ function augmentContext(
async function loadWebAssembly(
source: SourceInfo,
chunkPath: ChunkPath,
edgeModule: () => WebAssembly.Module,
imports: WebAssembly.Imports
): Promise<Exports> {
const module = await loadWebAssemblyModule(source, chunkPath)
const module = await loadWebAssemblyModule(source, chunkPath, edgeModule)
return await WebAssembly.instantiate(module, imports)
}
......@@ -62,26 +63,19 @@ function getFileStem(path: string): string {
return stem
}
type GlobalWithInjectedWebAssembly = typeof globalThis & {
[key: `wasm_${string}`]: WebAssembly.Module
}
async function loadWebAssemblyModule(
_source: SourceInfo,
chunkPath: ChunkPath
chunkPath: ChunkPath,
edgeModule: () => WebAssembly.Module
): Promise<WebAssembly.Module> {
const stem = getFileStem(chunkPath)
// very simple escaping just replacing unsupported characters with `_`
const escaped = stem.replace(/[^a-zA-Z0-9$_]/gi, '_')
const identifier: `wasm_${string}` = `wasm_${escaped}`
const module = (globalThis as GlobalWithInjectedWebAssembly)[identifier]
let module
try {
module = edgeModule()
} catch (_e) {}
if (!module) {
throw new Error(
`dynamically loading WebAssembly is not supported in this runtime and global \`${identifier}\` was not injected`
`dynamically loading WebAssembly is not supported in this runtime as global was not injected for chunk '${chunkPath}'`
)
}
......
......@@ -190,13 +190,20 @@ async function loadChunkAsyncByUrl(source: SourceInfo, chunkUrl: string) {
return loadChunkAsync(source, path)
}
function loadWebAssembly(chunkPath: ChunkPath, imports: WebAssembly.Imports) {
function loadWebAssembly(
chunkPath: ChunkPath,
_edgeModule: () => WebAssembly.Module,
imports: WebAssembly.Imports
) {
const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
return instantiateWebAssemblyFromPath(resolved, imports)
}
function loadWebAssemblyModule(chunkPath: ChunkPath) {
function loadWebAssemblyModule(
chunkPath: ChunkPath,
_edgeModule: () => WebAssembly.Module
) {
const resolved = path.resolve(RUNTIME_ROOT, chunkPath)
return compileWebAssemblyFromPath(resolved)
......
......@@ -45,9 +45,13 @@ type LoadChunk = (chunkPath: ChunkPath) => Promise<any> | undefined
type LoadChunkByUrl = (chunkUrl: ChunkUrl) => Promise<any> | undefined
type LoadWebAssembly = (
wasmChunkPath: ChunkPath,
edgeModule: () => WebAssembly.Module,
imports: WebAssembly.Imports
) => Exports
type LoadWebAssemblyModule = (wasmChunkPath: ChunkPath) => WebAssembly.Module
type LoadWebAssemblyModule = (
wasmChunkPath: ChunkPath,
edgeModule: () => WebAssembly.Module
) => WebAssembly.Module
type ModuleCache<M> = Record<ModuleId, M>
type ModuleFactories = Record<ModuleId, unknown>
......
......@@ -540,11 +540,11 @@ async function loadChunkAsyncByUrl(source, chunkUrl) {
const path1 = url.fileURLToPath(new URL(chunkUrl, RUNTIME_ROOT));
return loadChunkAsync(source, path1);
}
function loadWebAssembly(chunkPath, imports) {
function loadWebAssembly(chunkPath, _edgeModule, imports) {
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
return instantiateWebAssemblyFromPath(resolved, imports);
}
function loadWebAssemblyModule(chunkPath) {
function loadWebAssemblyModule(chunkPath, _edgeModule) {
const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
return compileWebAssemblyFromPath(resolved);
}
......
......@@ -1440,12 +1440,12 @@ function augmentContext(context) {
function fetchWebAssembly(wasmChunkPath) {
return fetch(getChunkRelativeUrl(wasmChunkPath));
}
async function loadWebAssembly(_source, wasmChunkPath, importsObj) {
async function loadWebAssembly(_source, wasmChunkPath, _edgeModule, importsObj) {
const req = fetchWebAssembly(wasmChunkPath);
const { instance } = await WebAssembly.instantiateStreaming(req, importsObj);
return instance.exports;
}
async function loadWebAssemblyModule(_source, wasmChunkPath) {
async function loadWebAssemblyModule(_source, wasmChunkPath, _edgeModule) {
const req = fetchWebAssembly(wasmChunkPath);
return await WebAssembly.compileStreaming(req);
}
......
......@@ -20,6 +20,7 @@ serde = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbopack-core = { workspace = true }
turbopack-ecmascript = { workspace = true }
wasmparser = "0.110.0"
......
......@@ -9,6 +9,12 @@
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::Vc;
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::asset::Asset;
pub(crate) mod analysis;
pub(crate) mod loader;
pub mod module_asset;
......@@ -16,6 +22,14 @@ pub(crate) mod output_asset;
pub mod raw;
pub mod source;
#[turbo_tasks::function]
pub async fn wasm_edge_var_name(asset: Vc<Box<dyn Asset>>) -> Result<Vc<RcStr>> {
let content = asset.content().file_content().await?;
Ok(Vc::cell(
format!("wasm_{:08x}", hash_xxh3_hash64(content)).into(),
))
}
pub fn register() {
turbo_tasks::register();
turbo_tasks_fs::register();
......
......@@ -8,7 +8,7 @@ use turbo_tasks_fs::File;
use turbopack_core::{asset::AssetContent, source::Source, virtual_source::VirtualSource};
use turbopack_ecmascript::utils::StringifyJs;
use crate::{analysis::analyze, source::WebAssemblySource};
use crate::{analysis::analyze, source::WebAssemblySource, wasm_edge_var_name};
/// Create a javascript loader to instantiate the WebAssembly module with the
/// necessary imports and exports to be processed by [turbopack_ecmascript].
......@@ -44,10 +44,11 @@ pub(crate) async fn instantiating_loader_source(
writedoc!(
code,
r#"
const {{ {exports} }} = await __turbopack_wasm__(wasmPath, {imports});
const {{ {exports} }} = await __turbopack_wasm__(wasmPath, () => {edgeVariable}, {imports});
export {{ {exports} }};
"#,
edgeVariable = wasm_edge_var_name(Vc::upcast(source)).await?,
imports = imports_obj,
exports = analysis.exports.join(", "),
)?;
......@@ -63,20 +64,23 @@ pub(crate) async fn instantiating_loader_source(
/// Create a javascript loader to compile the WebAssembly module and export it
/// without instantiating.
#[turbo_tasks::function]
pub(crate) fn compiling_loader_source(source: Vc<WebAssemblySource>) -> Vc<Box<dyn Source>> {
pub(crate) async fn compiling_loader_source(
source: Vc<WebAssemblySource>,
) -> Result<Vc<Box<dyn Source>>> {
let code: RcStr = formatdoc! {
r#"
import wasmPath from "WASM_PATH";
const mod = await __turbopack_wasm_module__(wasmPath);
const mod = await __turbopack_wasm_module__(wasmPath, () => {edgeVariable});
export default mod;
"#
"#,
edgeVariable = wasm_edge_var_name(Vc::upcast(source)).await?
}
.into();
Vc::upcast(VirtualSource::new(
Ok(Vc::upcast(VirtualSource::new(
source.ident().path().append("_.loader.mjs".into()),
AssetContent::file(File::from(code).into()),
))
)))
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment