Unverified Commit 509ed00f authored by LongYinan's avatar LongYinan Committed by GitHub

Calling turbopack from the next build CLI (#46602)

Close WEB-661
parent 17e1cc7a
...@@ -1498,6 +1498,8 @@ jobs: ...@@ -1498,6 +1498,8 @@ jobs:
with: with:
envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION VM_RELEASE envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION VM_RELEASE
usesh: true usesh: true
sync: rsync
copyback: false
mem: 6000 mem: 6000
prepare: | prepare: |
pkg install -y -f curl node libnghttp2 pkg install -y -f curl node libnghttp2
......
...@@ -3008,6 +3008,7 @@ version = "0.1.0" ...@@ -3008,6 +3008,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"mdxjs", "mdxjs",
"modularize_imports", "modularize_imports",
"next-build",
"next-dev", "next-dev",
"node-file-trace", "node-file-trace",
"styled_components", "styled_components",
...@@ -3017,6 +3018,19 @@ dependencies = [ ...@@ -3017,6 +3018,19 @@ dependencies = [
"testing", "testing",
] ]
[[package]]
name = "next-build"
version = "0.1.0"
dependencies = [
"anyhow",
"next-core",
"turbo-malloc",
"turbo-tasks",
"turbo-tasks-build",
"turbo-tasks-memory",
"vergen",
]
[[package]] [[package]]
name = "next-core" name = "next-core"
version = "0.1.0" version = "0.1.0"
......
...@@ -5,6 +5,7 @@ members = [ ...@@ -5,6 +5,7 @@ members = [
"crates/napi", "crates/napi",
"crates/wasm", "crates/wasm",
"crates/next-binding", "crates/next-binding",
"crates/next-build",
"crates/next-core", "crates/next-core",
"crates/next-dev", "crates/next-dev",
"crates/next-dev-tests", "crates/next-dev-tests",
......
...@@ -44,6 +44,7 @@ turbo-tasks = { workspace = true } ...@@ -44,6 +44,7 @@ turbo-tasks = { workspace = true }
turbo-tasks-memory = { workspace = true } turbo-tasks-memory = { workspace = true }
next-binding = { path = "../next-binding", features = [ next-binding = { path = "../next-binding", features = [
"__swc_core_binding_napi", "__swc_core_binding_napi",
"__turbo_next_build",
"__turbo_next_dev_server", "__turbo_next_dev_server",
"__turbo_node_file_trace", "__turbo_node_file_trace",
"__feature_mdx_rs", "__feature_mdx_rs",
......
use std::convert::TryFrom;
use crate::util::MapErr; use crate::util::MapErr;
use napi::bindgen_prelude::*; use napi::bindgen_prelude::*;
use next_binding::turbo::next_dev::{devserver_options::DevServerOptions, start_server}; use next_binding::turbo::{
next_build::{next_build as turbo_next_build, NextBuildOptions},
next_dev::{devserver_options::DevServerOptions, start_server},
};
#[napi] #[napi]
pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> { pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
let options: DevServerOptions = serde_json::from_slice(&options)?; let options: DevServerOptions = serde_json::from_slice(&options)?;
start_server(&options).await.convert_err() start_server(&options).await.convert_err()
} }
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct NextBuildContext {
pub dir: Option<String>,
pub app_dir: Option<String>,
pub pages_dir: Option<String>,
pub rewrites: Option<Rewrites>,
pub original_rewrites: Option<Rewrites>,
pub original_redirects: Option<Vec<Redirect>>,
}
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrites {
pub fallback: Vec<Rewrite>,
pub after_files: Vec<Rewrite>,
pub before_files: Vec<Rewrite>,
}
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrite {
pub source: String,
pub destination: String,
}
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Redirect {
pub source: String,
pub destination: String,
pub permanent: Option<bool>,
pub status_code: Option<u32>,
pub has: Option<RouteHas>,
pub missing: Option<RouteHas>,
}
#[derive(Debug)]
pub struct RouteHas {
pub r#type: RouteType,
pub key: Option<String>,
pub value: Option<String>,
}
#[derive(Debug)]
pub enum RouteType {
Header,
Query,
Cookie,
Host,
}
impl TryFrom<String> for RouteType {
type Error = napi::Error;
fn try_from(value: String) -> Result<Self> {
match value.as_str() {
"header" => Ok(RouteType::Header),
"query" => Ok(RouteType::Query),
"cookie" => Ok(RouteType::Cookie),
"host" => Ok(RouteType::Host),
_ => Err(napi::Error::new(
napi::Status::InvalidArg,
"Invalid route type",
)),
}
}
}
impl FromNapiValue for RouteHas {
unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
let object = Object::from_napi_value(env, napi_val)?;
let r#type = object.get_named_property::<String>("type")?;
Ok(RouteHas {
r#type: RouteType::try_from(r#type)?,
key: object.get("key")?,
value: object.get("value")?,
})
}
}
impl From<NextBuildContext> for NextBuildOptions {
fn from(value: NextBuildContext) -> Self {
Self {
dir: value.dir,
memory_limit: None,
full_stats: None,
}
}
}
#[napi]
pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
turbo_next_build(ctx.into()).await.convert_err()
}
...@@ -74,6 +74,7 @@ __swc_core_binding_wasm_plugin = ["swc_core/plugin_transform_host_js"] ...@@ -74,6 +74,7 @@ __swc_core_binding_wasm_plugin = ["swc_core/plugin_transform_host_js"]
__swc_core_testing_transform = ["swc_core/testing_transform"] __swc_core_testing_transform = ["swc_core/testing_transform"]
__turbo = [] __turbo = []
__turbo_next_build = ["__turbo", "next-build"]
__turbo_next_dev_server = ["__turbo", "next-dev/serializable"] __turbo_next_dev_server = ["__turbo", "next-dev/serializable"]
__turbo_node_file_trace = ["__turbo", "node-file-trace/node-api"] __turbo_node_file_trace = ["__turbo", "node-file-trace/node-api"]
...@@ -102,6 +103,9 @@ __swc_testing = ["__swc", "testing"] ...@@ -102,6 +103,9 @@ __swc_testing = ["__swc", "testing"]
[dependencies] [dependencies]
mdxjs = { optional = true, workspace = true } mdxjs = { optional = true, workspace = true }
modularize_imports = { optional = true, workspace = true } modularize_imports = { optional = true, workspace = true }
next-build = { optional = true, path = "../next-build", default-features = false, features = [
"custom_allocator",
] }
# TODO: Not sure what's going on, but using `workspace = true` noops `default-features = false`? # TODO: Not sure what's going on, but using `workspace = true` noops `default-features = false`?
next-dev = { optional = true, path = "../next-dev", default-features = false, features = [ next-dev = { optional = true, path = "../next-dev", default-features = false, features = [
"custom_allocator", "custom_allocator",
......
...@@ -21,6 +21,8 @@ pub mod swc { ...@@ -21,6 +21,8 @@ pub mod swc {
#[cfg(feature = "__turbo")] #[cfg(feature = "__turbo")]
pub mod turbo { pub mod turbo {
#[cfg(feature = "__turbo_next_build")]
pub use next_build;
#[cfg(feature = "__turbo_next_dev_server")] #[cfg(feature = "__turbo_next_dev_server")]
pub use next_dev; pub use next_dev;
#[cfg(feature = "__turbo_node_file_trace")] #[cfg(feature = "__turbo_node_file_trace")]
......
[package]
name = "next-build"
version = "0.1.0"
description = "TBD"
license = "MPL-2.0"
edition = "2021"
autobenches = false
[features]
next-font-local = ["next-core/next-font-local"]
native-tls = ["next-core/native-tls"]
rustls-tls = ["next-core/rustls-tls"]
custom_allocator = ["turbo-malloc/custom_allocator"]
[dependencies]
anyhow = "1.0.47"
next-core = { workspace = true }
turbo-malloc = { workspace = true, default-features = false }
turbo-tasks = { workspace = true }
turbo-tasks-memory = { workspace = true }
[build-dependencies]
turbo-tasks-build = { workspace = true }
vergen = { version = "7.3.2", default-features = false, features = [
"cargo",
"build",
] }
\ No newline at end of file
use turbo_tasks_build::{generate_register, rerun_if_glob};
use vergen::{vergen, Config};
fn main() {
generate_register();
rerun_if_glob("tests/integration/*/*", "tests/integration");
// Attempt to collect some build time env values but will skip if there are any
// errors.
let _ = vergen(Config::default());
}
use turbo_tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi};
use turbo_tasks_memory::MemoryBackend;
pub fn register() {
turbo_tasks::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
pub struct NextBuildOptions {
pub dir: Option<String>,
pub memory_limit: Option<usize>,
pub full_stats: Option<bool>,
}
pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> {
register();
let tt = TurboTasks::new(MemoryBackend::new(
options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024),
));
let stats_type = match options.full_stats {
Some(true) => StatsType::Full,
_ => StatsType::Essential,
};
tt.set_stats_type(stats_type);
let task = tt.spawn_root_task(move || {
Box::pin(async move {
// run next build here
Ok(NothingVc::new().into())
})
});
tt.wait_task_completion(task, true).await?;
Ok(())
}
...@@ -255,7 +255,8 @@ export default async function build( ...@@ -255,7 +255,8 @@ export default async function build(
debugOutput = false, debugOutput = false,
runLint = true, runLint = true,
noMangling = false, noMangling = false,
appDirOnly = false appDirOnly = false,
turboNextBuild = false
): Promise<void> { ): Promise<void> {
try { try {
const nextBuildSpan = trace('next-build', undefined, { const nextBuildSpan = trace('next-build', undefined, {
...@@ -1010,8 +1011,17 @@ export default async function build( ...@@ -1010,8 +1011,17 @@ export default async function build(
ignore: [] as string[], ignore: [] as string[],
})) }))
let binding = (await loadBindings()) as any
async function turbopackBuild() {
const turboNextBuildStart = process.hrtime()
await binding.turbo.nextBuild(NextBuildContext)
const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null }
}
const { duration: webpackBuildDuration, turbotraceContext } = const { duration: webpackBuildDuration, turbotraceContext } =
await webpackBuild() turboNextBuild ? await turbopackBuild() : await webpackBuild()
telemetry.record( telemetry.record(
eventBuildCompleted(pagesPaths, { eventBuildCompleted(pagesPaths, {
...@@ -1026,7 +1036,6 @@ export default async function build( ...@@ -1026,7 +1036,6 @@ export default async function build(
if (!turbotraceContext) { if (!turbotraceContext) {
return return
} }
let binding = (await loadBindings()) as any
if ( if (
!binding?.isWasm && !binding?.isWasm &&
typeof binding.turbo.startTrace === 'function' typeof binding.turbo.startTrace === 'function'
...@@ -1069,11 +1078,9 @@ export default async function build( ...@@ -1069,11 +1078,9 @@ export default async function build(
if (filesTracedFromEntries.length) { if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present // The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json // let's write the traced files into the first [entry].nft.json
// @ts-expect-error types const [[, entryName]] = Array.from<[string, string]>(
const [[, entryName]] = Array.from(entryNameMap.entries()).filter( entryNameMap.entries()
// @ts-expect-error types ).filter(([k]) => k.startsWith(turbotraceContextAppDir))
([k]) => k.startsWith(turbotraceContextAppDir)
)
const traceOutputPath = path.join( const traceOutputPath = path.join(
outputPath, outputPath,
`../${entryName}.js.nft.json` `../${entryName}.js.nft.json`
...@@ -1797,7 +1804,6 @@ export default async function build( ...@@ -1797,7 +1804,6 @@ export default async function build(
} else if (config.outputFileTracing) { } else if (config.outputFileTracing) {
let nodeFileTrace: any let nodeFileTrace: any
if (config.experimental.turbotrace) { if (config.experimental.turbotrace) {
let binding = (await loadBindings()) as any
if (!binding?.isWasm) { if (!binding?.isWasm) {
nodeFileTrace = binding.turbo.startTrace nodeFileTrace = binding.turbo.startTrace
} }
......
...@@ -471,6 +471,9 @@ function loadNative(isCustomTurbopack = false) { ...@@ -471,6 +471,9 @@ function loadNative(isCustomTurbopack = false) {
require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS).startDev(devOptions) require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS).startDev(devOptions)
} }
}, },
nextBuild: (options: unknown) => {
return bindings.nextBuild(options)
},
startTrace: (options = {}, turboTasks: unknown) => startTrace: (options = {}, turboTasks: unknown) =>
bindings.runTurboTracing( bindings.runTurboTracing(
toBuffer({ exact: true, ...options }), toBuffer({ exact: true, ...options }),
......
...@@ -409,7 +409,7 @@ async function webpackBuildWithWorker() { ...@@ -409,7 +409,7 @@ async function webpackBuildWithWorker() {
const combinedResult = { const combinedResult = {
duration: 0, duration: 0,
turbotraceContext: {} as any, turbotraceContext: {} as TurbotraceContext,
} }
// order matters here // order matters here
const ORDERED_COMPILER_NAMES = [ const ORDERED_COMPILER_NAMES = [
...@@ -447,9 +447,9 @@ async function webpackBuildWithWorker() { ...@@ -447,9 +447,9 @@ async function webpackBuildWithWorker() {
if (curResult.turbotraceContext?.entriesTrace) { if (curResult.turbotraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext combinedResult.turbotraceContext = curResult.turbotraceContext
const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) { if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace.entryNameMap = new Map( combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap entryNameMap
) )
} }
......
...@@ -17,6 +17,7 @@ const nextBuild: CliCommand = (argv) => { ...@@ -17,6 +17,7 @@ const nextBuild: CliCommand = (argv) => {
'--no-lint': Boolean, '--no-lint': Boolean,
'--no-mangling': Boolean, '--no-mangling': Boolean,
'--experimental-app-only': Boolean, '--experimental-app-only': Boolean,
'--experimental-turbo': Boolean,
// Aliases // Aliases
'-h': '--help', '-h': '--help',
'-d': '--debug', '-d': '--debug',
...@@ -76,7 +77,8 @@ const nextBuild: CliCommand = (argv) => { ...@@ -76,7 +77,8 @@ const nextBuild: CliCommand = (argv) => {
args['--debug'] || process.env.NEXT_DEBUG_BUILD, args['--debug'] || process.env.NEXT_DEBUG_BUILD,
!args['--no-lint'], !args['--no-lint'],
args['--no-mangling'], args['--no-mangling'],
args['--experimental-app-only'] args['--experimental-app-only'],
args['--experimental-turbo']
).catch((err) => { ).catch((err) => {
console.error('') console.error('')
if ( if (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment