Turbopack: Cache compiled WASM SWC plugins

parent 77f02f85
...@@ -10681,6 +10681,7 @@ dependencies = [ ...@@ -10681,6 +10681,7 @@ dependencies = [
"turbo-rcstr", "turbo-rcstr",
"turbo-tasks", "turbo-tasks",
"turbo-tasks-fs", "turbo-tasks-fs",
"turbo-tasks-hash",
"turbopack-core", "turbopack-core",
"turbopack-ecmascript", "turbopack-ecmascript",
] ]
......
use std::path::PathBuf;
use anyhow::Result; use anyhow::Result;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc, trace::TraceRawVcs}; use turbo_tasks::{ResolvedVc, Vc, trace::TraceRawVcs};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::{FileSystemPath, to_sys_path};
use turbopack::module_options::ModuleRule; use turbopack::module_options::ModuleRule;
use turbopack_ecmascript::{CustomTransformer, TransformPlugin}; use turbopack_ecmascript::{CustomTransformer, TransformPlugin};
...@@ -39,16 +41,28 @@ pub async fn get_swc_ecma_transform_plugin_rule( ...@@ -39,16 +41,28 @@ pub async fn get_swc_ecma_transform_plugin_rule(
let plugin_configs = next_config.experimental_swc_plugins().await?; let plugin_configs = next_config.experimental_swc_plugins().await?;
if !plugin_configs.is_empty() { if !plugin_configs.is_empty() {
let enable_mdx_rs = next_config.mdx_rs().await?.is_some(); let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
get_swc_ecma_transform_rule_impl(project_path, &plugin_configs, enable_mdx_rs).await let cache_dir = resolve_plugin_cache_dir(next_config, &project_path).await?;
get_swc_ecma_transform_rule_impl(project_path, &plugin_configs, enable_mdx_rs, cache_dir)
.await
} else { } else {
Ok(None) Ok(None)
} }
} }
async fn resolve_plugin_cache_dir(
next_config: Vc<NextConfig>,
project_path: &FileSystemPath,
) -> Result<Option<PathBuf>> {
let dist_dir = next_config.dist_dir().await?;
let cache_root = project_path.join(&dist_dir)?.join("cache/swc-plugins")?;
to_sys_path(cache_root.clone()).await
}
pub async fn get_swc_ecma_transform_rule_impl( pub async fn get_swc_ecma_transform_rule_impl(
project_path: FileSystemPath, project_path: FileSystemPath,
plugin_configs: &[(RcStr, serde_json::Value)], plugin_configs: &[(RcStr, serde_json::Value)],
enable_mdx_rs: bool, enable_mdx_rs: bool,
cache_dir: Option<PathBuf>,
) -> Result<Option<ModuleRule>> { ) -> Result<Option<ModuleRule>> {
use anyhow::bail; use anyhow::bail;
use turbo_tasks::TryFlatJoinIterExt; use turbo_tasks::TryFlatJoinIterExt;
...@@ -70,6 +84,7 @@ pub async fn get_swc_ecma_transform_rule_impl( ...@@ -70,6 +84,7 @@ pub async fn get_swc_ecma_transform_rule_impl(
.iter() .iter()
.map(|(name, config)| { .map(|(name, config)| {
let project_path = project_path.clone(); let project_path = project_path.clone();
let cache_dir = cache_dir.clone();
async move { async move {
// [TODO]: SWC's current experimental config supports // [TODO]: SWC's current experimental config supports
...@@ -129,11 +144,15 @@ pub async fn get_swc_ecma_transform_rule_impl( ...@@ -129,11 +144,15 @@ pub async fn get_swc_ecma_transform_rule_impl(
bail!("Expected file content for plugin module"); bail!("Expected file content for plugin module");
}; };
Ok(Some(( let plugin_bytes = file.content().to_bytes().to_vec();
SwcPluginModule::new(name.clone(), file.content().to_bytes().to_vec()) let module = match cache_dir.as_deref() {
.resolved_cell(), Some(dir) => {
JsonValue(config.clone()), SwcPluginModule::new_with_fs_cache(name.clone(), plugin_bytes, dir)
))) }
None => SwcPluginModule::new(name.clone(), plugin_bytes),
};
Ok(Some((module.resolved_cell(), JsonValue(config.clone()))))
} }
}) })
.try_flat_join() .try_flat_join()
......
...@@ -29,6 +29,7 @@ rustc-hash = { workspace = true } ...@@ -29,6 +29,7 @@ rustc-hash = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbopack-core = { workspace = true } turbopack-core = { workspace = true }
turbopack-ecmascript = { workspace = true } turbopack-ecmascript = { workspace = true }
...@@ -44,4 +45,3 @@ swc_core = { workspace = true, features = [ ...@@ -44,4 +45,3 @@ swc_core = { workspace = true, features = [
swc_plugin_backend_wasmtime = { workspace = true } swc_plugin_backend_wasmtime = { workspace = true }
swc_emotion = { workspace = true } swc_emotion = { workspace = true }
swc_relay = { workspace = true } swc_relay = { workspace = true }
use std::path::Path;
use anyhow::Result; use anyhow::Result;
use async_trait::async_trait; use async_trait::async_trait;
use swc_core::{ use swc_core::{
ecma::ast::Program, ecma::ast::Program,
plugin_runner::plugin_module_bytes::{CompiledPluginModuleBytes, RawPluginModuleBytes}, plugin_runner::{
plugin_module_bytes::{CompiledPluginModuleBytes, RawPluginModuleBytes},
runtime::Runtime,
},
}; };
use swc_plugin_backend_wasmtime::WasmtimeRuntime; use swc_plugin_backend_wasmtime::WasmtimeRuntime;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64};
use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, StyledString}; use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, StyledString};
use turbopack_ecmascript::{CustomTransformer, TransformContext}; use turbopack_ecmascript::{CustomTransformer, TransformContext};
...@@ -42,6 +48,76 @@ impl SwcPluginModule { ...@@ -42,6 +48,76 @@ impl SwcPluginModule {
name: plugin_name, name: plugin_name,
} }
} }
/// Like [`Self::new`], but caches the compiled wasm artifact on disk so
/// that future builds skip the expensive WASM cold start.
///
/// Cache layout: `<cache_dir>/<runtime_id>/<safe_plugin_name>-<hash>.cwasm`
/// where `<cache_dir>` comes from the caller/framework,
/// `<runtime_identifier>` is the wasmtime serialization ID,
/// and the plugin name and hash provide a safe, unique identifier.
/// Collectively, these should be enough to bump the cache path if the
/// runtime version or plugin content changes.
///
/// Cache load failures (missing file, corruption, version skew) fall back
/// to a fresh compile. Cache write failures never fail the build, this is
/// only an optimization.
pub fn new_with_fs_cache(plugin_name: RcStr, plugin_bytes: Vec<u8>, cache_dir: &Path) -> Self {
let rt = WasmtimeRuntime;
let cache_path = cache_path_for(cache_dir, &rt, &plugin_name, &plugin_bytes);
if cache_path.exists() {
// UNSAFE: load_cache deserializes wasmtime AOT compiled bytes.
// The file was produced by store_cache below using the same
// runtime version, guaranteed by embedding the version in the file
// path.
if let Some(cache) = unsafe { rt.load_cache(&cache_path) } {
return Self {
plugin: CompiledPluginModuleBytes::new(plugin_name.to_string(), cache),
name: plugin_name,
};
}
// Fall through to compilation on failure
}
let cache = rt
.prepare_module(&plugin_bytes)
.expect("Failed to compile SWC WASM plugin");
if let Some(parent) = cache_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let _ = rt.store_cache(&cache_path, &cache);
Self {
plugin: CompiledPluginModuleBytes::new(plugin_name.to_string(), cache),
name: plugin_name,
}
}
}
fn cache_path_for(
cache_dir: &Path,
rt: &WasmtimeRuntime,
plugin_name: &str,
plugin_bytes: &[u8],
) -> std::path::PathBuf {
let content_hash = encode_hex(hash_xxh3_hash64(plugin_bytes));
// Plugin names frequently contain '@' and '/' (thanks, npm scoped packages)
// which are not safe filename characters. Sanitize to something friendlier
// using only the POSIX portable character set.
let safe_name: String = plugin_name
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' {
c
} else {
'_'
}
})
.collect();
cache_dir
.join(rt.identifier())
.join(format!("{safe_name}-{content_hash}.cwasm"))
} }
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment