Unverified Commit c6313606 authored by Alex Kirszenberg's avatar Alex Kirszenberg Committed by GitHub

Next Build Turbo POC (2) (#51546)

Another attempt at getting https://github.com/vercel/next.js/pull/49942 in.

This time, the mold install step is gated to Linux.
parent b714ac55
...@@ -98,6 +98,8 @@ jobs: ...@@ -98,6 +98,8 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64 docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
build: >- build: >-
set -e && set -e &&
apt update &&
apt install -y pkg-config &&
rustup toolchain install "${RUST_TOOLCHAIN}" && rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-gnu && rustup target add x86_64-unknown-linux-gnu &&
...@@ -110,7 +112,7 @@ jobs: ...@@ -110,7 +112,7 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
build: >- build: >-
set -e && set -e &&
apk add --no-cache libc6-compat && apk add --no-cache libc6-compat pkgconfig &&
rustup toolchain install "${RUST_TOOLCHAIN}" && rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-musl && rustup target add x86_64-unknown-linux-musl &&
...@@ -133,6 +135,8 @@ jobs: ...@@ -133,6 +135,8 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64 docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64
build: >- build: >-
set -e && set -e &&
apt update &&
apt install -y pkg-config &&
export JEMALLOC_SYS_WITH_LG_PAGE=16 && export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
rustup toolchain install "${RUST_TOOLCHAIN}" && rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" &&
...@@ -146,7 +150,7 @@ jobs: ...@@ -146,7 +150,7 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
build: >- build: >-
set -e && set -e &&
apk add --no-cache libc6-compat && apk add --no-cache libc6-compat pkgconfig &&
export JEMALLOC_SYS_WITH_LG_PAGE=16 && export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
rustup toolchain install "${RUST_TOOLCHAIN}" && rustup toolchain install "${RUST_TOOLCHAIN}" &&
......
...@@ -73,6 +73,7 @@ jobs: ...@@ -73,6 +73,7 @@ jobs:
skipInstallBuild: 'yes' skipInstallBuild: 'yes'
skipNativeBuild: 'yes' skipNativeBuild: 'yes'
afterBuild: turbo run test-cargo-unit afterBuild: turbo run test-cargo-unit
mold: 'yes'
secrets: inherit secrets: inherit
test-cargo-integration: test-cargo-integration:
......
...@@ -7,6 +7,12 @@ on: ...@@ -7,6 +7,12 @@ on:
required: false required: false
description: 'additional steps to run' description: 'additional steps to run'
type: string type: string
# Toggles the mold linker. The default linker is much slower and can run into OOMs.
# However, custom linkers won't work for wasm.
mold:
required: false
description: 'whether to use the mold linker'
type: string
skipInstallBuild: skipInstallBuild:
required: false required: false
description: 'whether to skip pnpm install && pnpm build' description: 'whether to skip pnpm install && pnpm build'
...@@ -88,6 +94,13 @@ jobs: ...@@ -88,6 +94,13 @@ jobs:
with: with:
components: rustfmt, clippy components: rustfmt, clippy
- name: 'Install mold linker'
if: ${{ inputs.mold == 'yes' }}
run: |
sudo apt update
sudo apt install -y mold
echo RUSTFLAGS=${RUSTFLAGS}\ -C\ link-arg=-fuse-ld=mold >> $GITHUB_ENV
- name: Install nextest - name: Install nextest
if: ${{ inputs.needsNextest == 'yes' }} if: ${{ inputs.needsNextest == 'yes' }}
uses: taiki-e/install-action@nextest uses: taiki-e/install-action@nextest
......
...@@ -3260,7 +3260,16 @@ name = "next-build" ...@@ -3260,7 +3260,16 @@ name = "next-build"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap 4.1.11",
"console-subscriber",
"dunce",
"next-core", "next-core",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
"turbo-tasks",
"turbopack-binding", "turbopack-binding",
"vergen", "vergen",
] ]
...@@ -7376,6 +7385,7 @@ dependencies = [ ...@@ -7376,6 +7385,7 @@ dependencies = [
"turbo-tasks-testing", "turbo-tasks-testing",
"turbopack", "turbopack",
"turbopack-bench", "turbopack-bench",
"turbopack-build",
"turbopack-cli-utils", "turbopack-cli-utils",
"turbopack-core", "turbopack-core",
"turbopack-dev", "turbopack-dev",
...@@ -7390,6 +7400,26 @@ dependencies = [ ...@@ -7390,6 +7400,26 @@ dependencies = [
"turbopack-test-utils", "turbopack-test-utils",
] ]
[[package]]
name = "turbopack-build"
version = "0.1.0"
source = "git+https://github.com/vercel/turbo.git?tag=turbopack-230615.1#1ff1956dc18ff1805b2ac87f21f79e1abea75fc8"
dependencies = [
"anyhow",
"indexmap",
"indoc",
"serde",
"serde_json",
"serde_qs",
"turbo-tasks",
"turbo-tasks-build",
"turbo-tasks-fs",
"turbopack-core",
"turbopack-css",
"turbopack-ecmascript",
"turbopack-ecmascript-runtime",
]
[[package]] [[package]]
name = "turbopack-cli-utils" name = "turbopack-cli-utils"
version = "0.1.0" version = "0.1.0"
......
...@@ -26,7 +26,7 @@ lto = true ...@@ -26,7 +26,7 @@ lto = true
[workspace.dependencies] [workspace.dependencies]
# Workspace crates # Workspace crates
next-build = { path = "packages/next-swc/crates/next-build" } next-build = { path = "packages/next-swc/crates/next-build", default-features = false }
next-core = { path = "packages/next-swc/crates/next-core", default-features = false } next-core = { path = "packages/next-swc/crates/next-core", default-features = false }
next-dev = { path = "packages/next-swc/crates/next-dev", default-features = false, features = [ next-dev = { path = "packages/next-swc/crates/next-dev", default-features = false, features = [
"serializable", "serializable",
......
...@@ -48,9 +48,9 @@ turbo-tasks = { workspace = true } ...@@ -48,9 +48,9 @@ turbo-tasks = { workspace = true }
once_cell = { workspace = true } once_cell = { workspace = true }
serde = "1" serde = "1"
serde_json = "1" serde_json = "1"
tracing = { version = "0.1.37" } tracing = { workspace = true }
tracing-futures = "0.2.5" tracing-futures = "0.2.5"
tracing-subscriber = "0.3.9" tracing-subscriber = { workspace = true }
tracing-chrome = "0.5.0" tracing-chrome = "0.5.0"
turbopack-binding = { workspace = true, features = [ turbopack-binding = { workspace = true, features = [
"__swc_core_binding_napi", "__swc_core_binding_napi",
......
use std::convert::TryFrom; use std::{
convert::{TryFrom, TryInto},
path::PathBuf,
};
use anyhow::Context;
use napi::bindgen_prelude::*; use napi::bindgen_prelude::*;
use next_build::{next_build as turbo_next_build, NextBuildOptions}; use next_build::{
build as turbo_next_build, build_options::BuildContext, BuildOptions as NextBuildOptions,
};
use next_core::next_config::{Rewrite, Rewrites, RouteHas};
use next_dev::{devserver_options::DevServerOptions, start_server}; use next_dev::{devserver_options::DevServerOptions, start_server};
use crate::util::MapErr; use crate::util::MapErr;
...@@ -15,95 +22,162 @@ pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> { ...@@ -15,95 +22,162 @@ pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
#[napi(object, object_to_js = false)] #[napi(object, object_to_js = false)]
#[derive(Debug)] #[derive(Debug)]
pub struct NextBuildContext { pub struct NextBuildContext {
// Added by Next.js for next build --turbo specifically.
/// The root directory of the workspace.
pub root: Option<String>,
/// The project's directory.
pub dir: Option<String>, pub dir: Option<String>,
pub app_dir: Option<String>,
pub pages_dir: Option<String>, /// The build ID.
pub rewrites: Option<Rewrites>, pub build_id: Option<String>,
pub original_rewrites: Option<Rewrites>,
pub original_redirects: Option<Vec<Redirect>>, /// The rewrites, as computed by Next.js.
pub rewrites: Option<NapiRewrites>,
// TODO(alexkirsz) These are detected directly by Turbopack for now.
// pub app_dir: Option<String>,
// pub pages_dir: Option<String>,
// TODO(alexkirsz) These are used to generate route types.
// pub original_rewrites: Option<Rewrites>,
// pub original_redirects: Option<Vec<Redirect>>,
} }
#[napi(object, object_to_js = false)] impl TryFrom<NextBuildContext> for NextBuildOptions {
#[derive(Debug)] type Error = napi::Error;
pub struct Rewrites {
pub fallback: Vec<Rewrite>, fn try_from(value: NextBuildContext) -> Result<Self> {
pub after_files: Vec<Rewrite>, Ok(Self {
pub before_files: Vec<Rewrite>, dir: value.dir.map(PathBuf::try_from).transpose()?,
root: value.root.map(PathBuf::try_from).transpose()?,
log_level: None,
show_all: true,
log_detail: true,
full_stats: true,
memory_limit: None,
build_context: Some(BuildContext {
build_id: value
.build_id
.context("NextBuildContext must provide a build ID")?,
rewrites: value
.rewrites
.context("NextBuildContext must provide rewrites")?
.into(),
}),
})
}
} }
/// Keep in sync with [`next_core::next_config::Rewrites`]
#[napi(object, object_to_js = false)] #[napi(object, object_to_js = false)]
#[derive(Debug)] #[derive(Debug)]
pub struct Rewrite { pub struct NapiRewrites {
pub source: String, pub fallback: Vec<NapiRewrite>,
pub destination: String, pub after_files: Vec<NapiRewrite>,
pub before_files: Vec<NapiRewrite>,
}
impl From<NapiRewrites> for Rewrites {
fn from(val: NapiRewrites) -> Self {
Rewrites {
fallback: val
.fallback
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
after_files: val
.after_files
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
before_files: val
.before_files
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
}
}
} }
/// Keep in sync with [`next_core::next_config::Rewrite`]
#[napi(object, object_to_js = false)] #[napi(object, object_to_js = false)]
#[derive(Debug)] #[derive(Debug)]
pub struct Redirect { pub struct NapiRewrite {
pub source: String, pub source: String,
pub destination: String, pub destination: String,
pub permanent: Option<bool>, pub base_path: Option<bool>,
pub status_code: Option<u32>, pub locale: Option<bool>,
pub has: Option<RouteHas>, pub has: Option<Vec<NapiRouteHas>>,
pub missing: Option<RouteHas>, pub missing: Option<Vec<NapiRouteHas>>,
} }
#[derive(Debug)] impl From<NapiRewrite> for Rewrite {
pub struct RouteHas { fn from(val: NapiRewrite) -> Self {
pub r#type: RouteType, Rewrite {
pub key: Option<String>, source: val.source,
pub value: Option<String>, destination: val.destination,
base_path: val.base_path,
locale: val.locale,
has: val
.has
.map(|has| has.into_iter().map(|has| has.into()).collect()),
missing: val
.missing
.map(|missing| missing.into_iter().map(|missing| missing.into()).collect()),
}
}
} }
/// Keep in sync with [`next_core::next_config::RouteHas`]
#[derive(Debug)] #[derive(Debug)]
pub enum RouteType { pub enum NapiRouteHas {
Header, Header { key: String, value: Option<String> },
Query, Query { key: String, value: Option<String> },
Cookie, Cookie { key: String, value: Option<String> },
Host, Host { value: String },
} }
impl TryFrom<String> for RouteType { impl FromNapiValue for NapiRouteHas {
type Error = napi::Error;
fn try_from(value: String) -> Result<Self> {
match value.as_str() {
"header" => Ok(RouteType::Header),
"query" => Ok(RouteType::Query),
"cookie" => Ok(RouteType::Cookie),
"host" => Ok(RouteType::Host),
_ => Err(napi::Error::new(
napi::Status::InvalidArg,
"Invalid route type",
)),
}
}
}
impl FromNapiValue for RouteHas {
unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> { unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
let object = Object::from_napi_value(env, napi_val)?; let object = Object::from_napi_value(env, napi_val)?;
let r#type = object.get_named_property::<String>("type")?; let type_ = object.get_named_property::<String>("type")?;
Ok(RouteHas { Ok(match type_.as_str() {
r#type: RouteType::try_from(r#type)?, "header" => NapiRouteHas::Header {
key: object.get("key")?, key: object.get_named_property("key")?,
value: object.get("value")?, value: object.get_named_property("value")?,
},
"query" => NapiRouteHas::Query {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"cookie" => NapiRouteHas::Cookie {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"host" => NapiRouteHas::Host {
value: object.get_named_property("value")?,
},
_ => {
return Err(napi::Error::new(
Status::GenericFailure,
format!("invalid type for RouteHas: {}", type_),
))
}
}) })
} }
} }
impl From<NextBuildContext> for NextBuildOptions { impl From<NapiRouteHas> for RouteHas {
fn from(value: NextBuildContext) -> Self { fn from(val: NapiRouteHas) -> Self {
Self { match val {
dir: value.dir, NapiRouteHas::Header { key, value } => RouteHas::Header { key, value },
memory_limit: None, NapiRouteHas::Query { key, value } => RouteHas::Query { key, value },
full_stats: None, NapiRouteHas::Cookie { key, value } => RouteHas::Cookie { key, value },
NapiRouteHas::Host { value } => RouteHas::Host { value },
} }
} }
} }
#[napi] #[napi]
pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> { pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
turbo_next_build(ctx.into()).await.convert_err() turbo_next_build(ctx.try_into()?).await.convert_err()
} }
...@@ -6,16 +6,66 @@ license = "MPL-2.0" ...@@ -6,16 +6,66 @@ license = "MPL-2.0"
edition = "2021" edition = "2021"
autobenches = false autobenches = false
[[bin]]
name = "next-build"
path = "src/main.rs"
bench = false
required-features = ["cli"]
[lib]
bench = false
[features] [features]
# By default, we enable native-tls for reqwest via downstream transitive features.
# This is for the convenience of running daily dev workflows, i.e running
# `cargo xxx` without explicitly specifying features, not that we want to
# promote this as default backend. Actual configuration is done when building next-swc,
# and also turbopack standalone when we have it.
default = ["cli", "custom_allocator", "native-tls"]
cli = ["clap"]
tokio_console = [
"dep:console-subscriber",
"tokio/tracing",
"turbo-tasks/tokio_tracing",
]
native-tls = ["next-core/native-tls"] native-tls = ["next-core/native-tls"]
rustls-tls = ["next-core/rustls-tls"] rustls-tls = ["next-core/rustls-tls"]
custom_allocator = ["turbopack-binding/__turbo_tasks_malloc", "turbopack-binding/__turbo_tasks_malloc_custom_allocator"] custom_allocator = [
"turbopack-binding/__turbo_tasks_malloc",
"turbopack-binding/__turbo_tasks_malloc_custom_allocator",
]
serializable = []
profile = []
[dependencies] [dependencies]
anyhow = "1.0.47" anyhow = { workspace = true }
clap = { workspace = true, features = ["derive", "env"], optional = true }
console-subscriber = { workspace = true, optional = true }
dunce = { workspace = true }
next-core = { workspace = true } next-core = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true, features = ["full"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
turbopack-binding = { workspace = true, features = ["__turbo_tasks", "__turbo_tasks_memory"] } turbopack-binding = { workspace = true, features = [
"__turbo_tasks",
"__turbo_tasks_memory",
"__turbo_tasks_env",
"__turbo_tasks_fs",
"__turbo_tasks_memory",
"__turbopack",
"__turbopack_build",
"__turbopack_cli_utils",
"__turbopack_core",
"__turbopack_dev",
"__turbopack_ecmascript",
"__turbopack_ecmascript_runtime",
"__turbopack_env",
"__turbopack_node",
] }
turbo-tasks = { workspace = true }
[build-dependencies] [build-dependencies]
turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] } turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
......
use std::path::PathBuf;
use next_core::{next_config::Rewrites, turbopack::core::issue::IssueSeverity};
#[derive(Clone, Debug)]
pub struct BuildOptions {
/// The root directory of the workspace.
pub root: Option<PathBuf>,
/// The project's directory.
pub dir: Option<PathBuf>,
/// The maximum memory to use for the build.
pub memory_limit: Option<usize>,
/// The log level to use for the build.
pub log_level: Option<IssueSeverity>,
/// Whether to show all logs.
pub show_all: bool,
/// Whether to show detailed logs.
pub log_detail: bool,
/// Whether to compute full stats.
pub full_stats: bool,
/// The Next.js build context.
pub build_context: Option<BuildContext>,
}
#[derive(Clone, Debug)]
pub struct BuildContext {
/// The build id.
pub build_id: String,
/// Next.js config rewrites.
pub rewrites: Rewrites,
}
use turbopack_binding::turbo::{ use turbopack_binding::turbo::{
tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi}, tasks::{run_once, TransientInstance, TurboTasks},
tasks_memory::MemoryBackend, tasks_memory::MemoryBackend,
}; };
pub fn register() { pub mod build_options;
turbopack_binding::turbo::tasks::register(); pub mod manifests;
include!(concat!(env!("OUT_DIR"), "/register.rs")); pub(crate) mod next_build;
} pub(crate) mod next_pages;
pub struct NextBuildOptions { use anyhow::Result;
pub dir: Option<String>, use turbo_tasks::{StatsType, TurboTasksBackendApi};
pub memory_limit: Option<usize>,
pub full_stats: Option<bool>, pub use self::build_options::BuildOptions;
}
pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> { pub async fn build(options: BuildOptions) -> Result<()> {
#[cfg(feature = "tokio_console")]
console_subscriber::init();
register(); register();
setup_tracing();
let tt = TurboTasks::new(MemoryBackend::new( let tt = TurboTasks::new(MemoryBackend::new(
options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024), options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024),
)); ));
let stats_type = match options.full_stats { let stats_type = match options.full_stats {
Some(true) => StatsType::Full, true => StatsType::Full,
_ => StatsType::Essential, false => StatsType::Essential,
}; };
tt.set_stats_type(stats_type); tt.set_stats_type(stats_type);
let task = tt.spawn_root_task(move || {
Box::pin(async move { run_once(tt, async move {
// run next build here next_build::next_build(TransientInstance::new(options)).await?;
Ok(NothingVc::new().into())
}) Ok(())
}); })
tt.wait_task_completion(task, true).await?; .await?;
Ok(()) Ok(())
} }
fn setup_tracing() {
use tracing_subscriber::{prelude::*, EnvFilter, Registry};
let subscriber = Registry::default();
let stdout_log = tracing_subscriber::fmt::layer().pretty();
let subscriber = subscriber.with(stdout_log);
let subscriber = subscriber.with(EnvFilter::from_default_env());
subscriber.init();
}
pub fn register() {
turbopack_binding::turbo::tasks::register();
turbopack_binding::turbo::tasks_fs::register();
turbopack_binding::turbopack::turbopack::register();
turbopack_binding::turbopack::core::register();
turbopack_binding::turbopack::node::register();
turbopack_binding::turbopack::dev::register();
turbopack_binding::turbopack::build::register();
next_core::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
use std::path::PathBuf;
use anyhow::Result;
use clap::Parser;
use next_build::BuildOptions;
use turbopack_binding::turbopack::cli_utils::issue::IssueSeverityCliOption;
#[global_allocator]
static ALLOC: turbopack_binding::turbo::malloc::TurboMalloc =
turbopack_binding::turbo::malloc::TurboMalloc;
#[derive(Debug, Parser)]
#[clap(author, version, about, long_about = None)]
pub struct BuildCliArgs {
/// The directory of the Next.js application.
/// If no directory is provided, the current directory will be used.
#[clap(value_parser)]
pub dir: Option<PathBuf>,
/// The root directory of the project. Nothing outside of this directory can
/// be accessed. e. g. the monorepo root.
/// If no directory is provided, `dir` will be used.
#[clap(long, value_parser)]
pub root: Option<PathBuf>,
/// Display version of the binary. Noop if used in library mode.
#[clap(long)]
pub display_version: bool,
/// Filter by issue severity.
#[clap(short, long)]
pub log_level: Option<IssueSeverityCliOption>,
/// Show all log messages without limit.
#[clap(long)]
pub show_all: bool,
/// Expand the log details.
#[clap(long)]
pub log_detail: bool,
/// Whether to enable full task stats recording in Turbo Engine.
#[clap(long)]
pub full_stats: bool,
/// Enable experimental garbage collection with the provided memory limit in
/// MB.
#[clap(long)]
pub memory_limit: Option<usize>,
}
fn main() {
use turbopack_binding::turbo::malloc::TurboMalloc;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.on_thread_stop(|| {
TurboMalloc::thread_stop();
})
.build()
.unwrap()
.block_on(main_inner())
.unwrap()
}
async fn main_inner() -> Result<()> {
let args = BuildCliArgs::parse();
if args.display_version {
// Note: enabling git causes trouble with aarch64 linux builds with libz-sys
println!(
"Build Timestamp\t\t{:#?}",
option_env!("VERGEN_BUILD_TIMESTAMP").unwrap_or_else(|| "N/A")
);
println!(
"Build Version\t\t{:#?}",
option_env!("VERGEN_BUILD_SEMVER").unwrap_or_else(|| "N/A")
);
println!(
"Cargo Target Triple\t{:#?}",
option_env!("VERGEN_CARGO_TARGET_TRIPLE").unwrap_or_else(|| "N/A")
);
println!(
"Cargo Profile\t\t{:#?}",
option_env!("VERGEN_CARGO_PROFILE").unwrap_or_else(|| "N/A")
);
return Ok(());
}
next_build::build(BuildOptions {
dir: args.dir,
root: args.root,
memory_limit: args.memory_limit,
log_level: args.log_level.map(|l| l.0),
show_all: args.show_all,
log_detail: args.log_detail,
full_stats: args.full_stats,
build_context: None,
})
.await
}
//! Type definitions for the Next.js manifest formats.
use std::collections::HashMap;
use next_core::next_config::Rewrites;
use serde::Serialize;
#[derive(Serialize, Default, Debug)]
pub struct PagesManifest {
#[serde(flatten)]
pub pages: HashMap<String, String>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BuildManifest {
pub dev_files: Vec<String>,
pub amp_dev_files: Vec<String>,
pub polyfill_files: Vec<String>,
pub low_priority_files: Vec<String>,
pub root_main_files: Vec<String>,
pub pages: HashMap<String, Vec<String>>,
pub amp_first_pages: Vec<String>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase", tag = "version")]
pub enum MiddlewaresManifest {
#[serde(rename = "2")]
MiddlewaresManifestV2(MiddlewaresManifestV2),
#[serde(other)]
Unsupported,
}
impl Default for MiddlewaresManifest {
fn default() -> Self {
Self::MiddlewaresManifestV2(Default::default())
}
}
#[derive(Serialize, Default, Debug)]
pub struct MiddlewaresManifestV2 {
pub sorted_middleware: Vec<()>,
pub middleware: HashMap<String, ()>,
pub functions: HashMap<String, ()>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReactLoadableManifest {
#[serde(flatten)]
pub manifest: HashMap<String, ReactLoadableManifestEntry>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReactLoadableManifestEntry {
pub id: u32,
pub files: Vec<String>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct NextFontManifest {
pub pages: HashMap<String, Vec<String>>,
pub app: HashMap<String, Vec<String>>,
pub app_using_size_adjust: bool,
pub pages_using_size_adjust: bool,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct AppPathsManifest {
#[serde(flatten)]
pub edge_server_app_paths: PagesManifest,
#[serde(flatten)]
pub node_server_app_paths: PagesManifest,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ServerReferenceManifest {
#[serde(flatten)]
pub server_actions: ActionManifest,
#[serde(flatten)]
pub edge_server_actions: ActionManifest,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ActionManifest {
#[serde(flatten)]
pub actions: HashMap<String, ActionManifestEntry>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ActionManifestEntry {
pub workers: HashMap<String, ActionManifestWorkerEntry>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
#[serde(untagged)]
pub enum ActionManifestWorkerEntry {
String(String),
Number(f64),
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientReferenceManifest {
pub client_modules: ManifestNode,
pub ssr_module_mapping: HashMap<String, ManifestNode>,
#[serde(rename = "edgeSSRModuleMapping")]
pub edge_ssr_module_mapping: HashMap<String, ManifestNode>,
pub css_files: HashMap<String, Vec<String>>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientCssReferenceManifest {
pub css_imports: HashMap<String, Vec<String>>,
pub css_modules: HashMap<String, Vec<String>>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ManifestNode {
#[serde(flatten)]
pub module_exports: HashMap<String, ManifestNodeEntry>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ManifestNodeEntry {
pub id: ModuleId,
pub name: String,
pub chunks: Vec<String>,
pub r#async: bool,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
#[serde(untagged)]
pub enum ModuleId {
String(String),
Number(f64),
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct FontManifest(pub Vec<FontManifestEntry>);
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct FontManifestEntry {
pub url: String,
pub content: String,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct AppBuildManifest {
pub pages: HashMap<String, Vec<String>>,
}
// TODO(alexkirsz) Unify with the one for dev.
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientBuildManifest<'a> {
#[serde(rename = "__rewrites")]
pub rewrites: &'a Rewrites,
pub sorted_pages: &'a [String],
#[serde(flatten)]
pub pages: HashMap<String, Vec<&'a str>>,
}
This diff is collapsed.
use anyhow::{bail, Result};
use next_core::{
create_page_loader_entry_asset,
turbopack::core::{asset::AssetsVc, chunk::EvaluatableAssetsVc},
};
use turbopack_binding::{
turbo::{
tasks::{primitives::StringVc, Value},
tasks_fs::FileSystemPathVc,
},
turbopack::{
core::{
asset::AssetVc,
chunk::{ChunkableAsset, ChunkingContext, ChunkingContextVc},
context::{AssetContext, AssetContextVc},
reference_type::ReferenceType,
},
dev::DevChunkingContextVc,
ecmascript::EcmascriptModuleAssetVc,
},
};
#[turbo_tasks::value]
pub(crate) struct PagesBuildClientContext {
project_root: FileSystemPathVc,
client_root: FileSystemPathVc,
client_asset_context: AssetContextVc,
client_runtime_entries: EvaluatableAssetsVc,
}
#[turbo_tasks::value_impl]
impl PagesBuildClientContextVc {
#[turbo_tasks::function]
pub fn new(
project_root: FileSystemPathVc,
client_root: FileSystemPathVc,
client_asset_context: AssetContextVc,
client_runtime_entries: EvaluatableAssetsVc,
) -> PagesBuildClientContextVc {
PagesBuildClientContext {
project_root,
client_root,
client_asset_context,
client_runtime_entries,
}
.cell()
}
#[turbo_tasks::function]
async fn client_chunking_context(self) -> Result<ChunkingContextVc> {
let this = self.await?;
Ok(DevChunkingContextVc::builder(
this.project_root,
this.client_root,
this.client_root.join("static/chunks"),
this.client_root.join("static/media"),
this.client_asset_context.compile_time_info().environment(),
)
.build())
}
#[turbo_tasks::function]
pub async fn client_chunk(
self,
asset: AssetVc,
pathname: StringVc,
reference_type: Value<ReferenceType>,
) -> Result<AssetsVc> {
let this = self.await?;
let client_asset_page = this.client_asset_context.process(asset, reference_type);
let client_asset_page =
create_page_loader_entry_asset(this.client_asset_context, client_asset_page, pathname);
let Some(client_module_asset) = EcmascriptModuleAssetVc::resolve_from(client_asset_page).await? else {
bail!("Expected an EcmaScript module asset");
};
let client_chunking_context = self.client_chunking_context();
Ok(client_chunking_context.evaluated_chunk_group(
client_module_asset.as_root_chunk(client_chunking_context),
this.client_runtime_entries
.with_entry(client_module_asset.into()),
))
}
}
pub(crate) mod client_context;
pub(crate) mod node_context;
pub(crate) mod page_chunks;
use anyhow::{bail, Result};
use next_core::{next_client::RuntimeEntriesVc, turbopack::core::chunk::EvaluatableAssetsVc};
use turbopack_binding::{
turbo::{tasks::Value, tasks_fs::FileSystemPathVc},
turbopack::{
build::BuildChunkingContextVc,
core::{
asset::AssetVc,
context::{AssetContext, AssetContextVc},
reference_type::{EntryReferenceSubType, ReferenceType},
resolve::{parse::RequestVc, pattern::QueryMapVc},
},
ecmascript::EcmascriptModuleAssetVc,
},
};
#[turbo_tasks::value]
pub(crate) struct PagesBuildNodeContext {
project_root: FileSystemPathVc,
node_root: FileSystemPathVc,
node_asset_context: AssetContextVc,
node_runtime_entries: EvaluatableAssetsVc,
}
#[turbo_tasks::value_impl]
impl PagesBuildNodeContextVc {
#[turbo_tasks::function]
pub fn new(
project_root: FileSystemPathVc,
node_root: FileSystemPathVc,
node_asset_context: AssetContextVc,
node_runtime_entries: RuntimeEntriesVc,
) -> PagesBuildNodeContextVc {
PagesBuildNodeContext {
project_root,
node_root,
node_asset_context,
node_runtime_entries: node_runtime_entries.resolve_entries(node_asset_context),
}
.cell()
}
#[turbo_tasks::function]
pub async fn resolve_module(
self,
origin: FileSystemPathVc,
package: String,
path: String,
) -> Result<AssetVc> {
let this = self.await?;
let Some(asset) = this
.node_asset_context
.resolve_asset(
origin,
RequestVc::module(package.clone(), Value::new(path.clone().into()), QueryMapVc::none()),
this.node_asset_context.resolve_options(origin, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page))),
Value::new(ReferenceType::Entry(EntryReferenceSubType::Page))
)
.primary_assets()
.await?
.first()
.copied()
else {
bail!("module {}/{} not found in {}", package, path, origin.await?);
};
Ok(asset)
}
#[turbo_tasks::function]
async fn node_chunking_context(self) -> Result<BuildChunkingContextVc> {
let this = self.await?;
Ok(BuildChunkingContextVc::builder(
this.project_root,
this.node_root,
this.node_root.join("server/pages"),
this.node_root.join("server/assets"),
this.node_asset_context.compile_time_info().environment(),
)
.build())
}
#[turbo_tasks::function]
pub async fn node_chunk(
self,
asset: AssetVc,
reference_type: Value<ReferenceType>,
) -> Result<AssetVc> {
let this = self.await?;
let node_asset_page = this.node_asset_context.process(asset, reference_type);
let Some(node_module_asset) = EcmascriptModuleAssetVc::resolve_from(node_asset_page).await? else {
bail!("Expected an EcmaScript module asset");
};
let chunking_context = self.node_chunking_context();
Ok(chunking_context.generate_exported_chunk(node_module_asset, this.node_runtime_entries))
}
}
...@@ -15,7 +15,7 @@ pub mod manifest; ...@@ -15,7 +15,7 @@ pub mod manifest;
pub mod mode; pub mod mode;
mod next_build; mod next_build;
pub mod next_client; pub mod next_client;
mod next_client_chunks; pub mod next_client_chunks;
mod next_client_component; mod next_client_component;
pub mod next_config; pub mod next_config;
mod next_edge; mod next_edge;
...@@ -38,8 +38,10 @@ mod util; ...@@ -38,8 +38,10 @@ mod util;
mod web_entry_source; mod web_entry_source;
pub use app_source::create_app_source; pub use app_source::create_app_source;
pub use page_loader::create_page_loader_entry_asset;
pub use page_source::create_page_source; pub use page_source::create_page_source;
pub use turbopack_binding::{turbopack::node::source_map, *}; pub use turbopack_binding::{turbopack::node::source_map, *};
pub use util::{pathname_for_path, PathType};
pub use web_entry_source::create_web_entry_source; pub use web_entry_source::create_web_entry_source;
pub fn register() { pub fn register() {
......
...@@ -337,14 +337,14 @@ pub fn get_client_chunking_context( ...@@ -337,14 +337,14 @@ pub fn get_client_chunking_context(
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn get_client_assets_path( pub fn get_client_assets_path(
server_root: FileSystemPathVc, client_root: FileSystemPathVc,
ty: Value<ClientContextType>, ty: Value<ClientContextType>,
) -> FileSystemPathVc { ) -> FileSystemPathVc {
match ty.into_value() { match ty.into_value() {
ClientContextType::Pages { .. } ClientContextType::Pages { .. }
| ClientContextType::App { .. } | ClientContextType::App { .. }
| ClientContextType::Fallback => server_root.join("/_next/static/media"), | ClientContextType::Fallback => client_root.join("/_next/static/media"),
ClientContextType::Other => server_root.join("/_assets"), ClientContextType::Other => client_root.join("/_assets"),
} }
} }
......
...@@ -2,3 +2,10 @@ pub(crate) mod context; ...@@ -2,3 +2,10 @@ pub(crate) mod context;
pub(crate) mod runtime_entry; pub(crate) mod runtime_entry;
pub(crate) mod transforms; pub(crate) mod transforms;
pub(crate) mod transition; pub(crate) mod transition;
pub use context::{
get_client_compile_time_info, get_client_module_options_context,
get_client_resolve_options_context, get_client_runtime_entries, ClientContextType,
};
pub use runtime_entry::{RuntimeEntries, RuntimeEntriesVc, RuntimeEntry, RuntimeEntryVc};
pub use transition::NextClientTransition;
pub(crate) mod client_chunks_transition; pub(crate) mod client_chunks_transition;
pub(crate) mod in_chunking_context_asset; pub(crate) mod in_chunking_context_asset;
pub(crate) mod with_chunks; pub(crate) mod with_chunks;
pub use client_chunks_transition::NextClientChunksTransitionVc;
...@@ -678,7 +678,6 @@ pub async fn load_next_config_internal( ...@@ -678,7 +678,6 @@ pub async fn load_next_config_internal(
next_asset("entry/config/next.js"), next_asset("entry/config/next.js"),
Value::new(ReferenceType::Entry(EntryReferenceSubType::Undefined)), Value::new(ReferenceType::Entry(EntryReferenceSubType::Undefined)),
); );
let config_value = evaluate( let config_value = evaluate(
load_next_config_asset, load_next_config_asset,
project_path, project_path,
......
pub(crate) mod context; pub(crate) mod context;
pub(crate) mod resolve; pub(crate) mod resolve;
pub(crate) mod transforms; pub(crate) mod transforms;
pub use context::{
get_server_compile_time_info, get_server_module_options_context,
get_server_resolve_options_context, ServerContextType,
};
...@@ -54,48 +54,51 @@ pub struct PageLoaderAsset { ...@@ -54,48 +54,51 @@ pub struct PageLoaderAsset {
pub pathname: StringVc, pub pathname: StringVc,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::function]
impl PageLoaderAssetVc { pub async fn create_page_loader_entry_asset(
#[turbo_tasks::function] client_context: AssetContextVc,
async fn get_loader_entry_asset(self) -> Result<AssetVc> { entry_asset: AssetVc,
let this = &*self.await?; pathname: StringVc,
) -> Result<AssetVc> {
let mut result = RopeBuilder::default();
writeln!(
result,
"const PAGE_PATH = {};\n",
StringifyJs(&*pathname.await?)
)?;
let page_loader_path = next_js_file_path("entry/page-loader.ts");
let base_code = page_loader_path.read();
if let FileContent::Content(base_file) = &*base_code.await? {
result += base_file.content()
} else {
bail!("required file `entry/page-loader.ts` not found");
}
let mut result = RopeBuilder::default(); let file = File::from(result.build());
writeln!(
result,
"const PAGE_PATH = {};\n",
StringifyJs(&*this.pathname.await?)
)?;
let page_loader_path = next_js_file_path("entry/page-loader.ts");
let base_code = page_loader_path.read();
if let FileContent::Content(base_file) = &*base_code.await? {
result += base_file.content()
} else {
bail!("required file `entry/page-loader.ts` not found");
}
let file = File::from(result.build()); let virtual_asset = VirtualAssetVc::new(page_loader_path, file.into()).into();
Ok(VirtualAssetVc::new(page_loader_path, file.into()).into()) Ok(client_context.process(
} virtual_asset,
Value::new(ReferenceType::Internal(
InnerAssetsVc::cell(indexmap! {
"PAGE".to_string() => client_context.process(entry_asset, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)))
})
)))
)
}
#[turbo_tasks::value_impl]
impl PageLoaderAssetVc {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn get_page_chunks(self) -> Result<AssetsVc> { async fn get_page_chunks(self) -> Result<AssetsVc> {
let this = &*self.await?; let this = &*self.await?;
let loader_entry_asset = self.get_loader_entry_asset(); let page_loader_entry_asset =
create_page_loader_entry_asset(this.client_context, this.entry_asset, this.pathname);
let module = this.client_context.process(
loader_entry_asset,
Value::new(ReferenceType::Internal(
InnerAssetsVc::cell(indexmap! {
"PAGE".to_string() => this.client_context.process(this.entry_asset, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)))
})
)),
);
let Some(module) = EvaluatableAssetVc::resolve_from(module).await? else { let Some(module) = EvaluatableAssetVc::resolve_from(page_loader_entry_asset).await? else {
bail!("internal module must be evaluatable"); bail!("internal module must be evaluatable");
}; };
......
...@@ -107,6 +107,12 @@ pub struct PagesDirectoryStructure { ...@@ -107,6 +107,12 @@ pub struct PagesDirectoryStructure {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl PagesDirectoryStructureVc { impl PagesDirectoryStructureVc {
/// Returns the router path of this directory.
#[turbo_tasks::function]
pub async fn next_router_path(self) -> Result<FileSystemPathVc> {
Ok(self.await?.next_router_path)
}
/// Returns the path to the directory of this structure in the project file /// Returns the path to the directory of this structure in the project file
/// system. /// system.
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -38,7 +38,7 @@ use crate::{ ...@@ -38,7 +38,7 @@ use crate::{
get_client_asset_context, get_client_chunking_context, get_client_asset_context, get_client_chunking_context,
get_client_resolve_options_context, ClientContextType, get_client_resolve_options_context, ClientContextType,
}, },
runtime_entry::{RuntimeEntriesVc, RuntimeEntry}, RuntimeEntriesVc, RuntimeEntry,
}, },
next_config::NextConfigVc, next_config::NextConfigVc,
}; };
......
...@@ -210,6 +210,7 @@ export default async function build( ...@@ -210,6 +210,7 @@ export default async function build(
noMangling = false, noMangling = false,
appDirOnly = false, appDirOnly = false,
turboNextBuild = false, turboNextBuild = false,
turboNextBuildRoot = null,
buildMode: 'default' | 'experimental-compile' | 'experimental-generate' buildMode: 'default' | 'experimental-compile' | 'experimental-generate'
): Promise<void> { ): Promise<void> {
const isCompile = buildMode === 'experimental-compile' const isCompile = buildMode === 'experimental-compile'
...@@ -550,6 +551,23 @@ export default async function build( ...@@ -550,6 +551,23 @@ export default async function build(
app: appPageKeys.length > 0 ? appPageKeys : undefined, app: appPageKeys.length > 0 ? appPageKeys : undefined,
} }
if (turboNextBuild) {
// TODO(WEB-397) This is a temporary workaround to allow for filtering a
// subset of pages when building with --experimental-turbo, until we
// have complete support for all pages.
if (process.env.NEXT_TURBO_FILTER_PAGES) {
const filterPages = process.env.NEXT_TURBO_FILTER_PAGES.split(',')
pageKeys.pages = pageKeys.pages.filter((page) => {
return filterPages.some((filterPage) => {
return isMatch(page, filterPage)
})
})
}
// TODO(alexkirsz) Filter out app pages entirely as they are not supported yet.
pageKeys.app = undefined
}
const numConflictingAppPaths = conflictingAppPagePaths.length const numConflictingAppPaths = conflictingAppPagePaths.length
if (mappedAppPages && numConflictingAppPaths > 0) { if (mappedAppPages && numConflictingAppPaths > 0) {
Log.error( Log.error(
...@@ -932,7 +950,23 @@ export default async function build( ...@@ -932,7 +950,23 @@ export default async function build(
async function turbopackBuild() { async function turbopackBuild() {
const turboNextBuildStart = process.hrtime() const turboNextBuildStart = process.hrtime()
await binding.turbo.nextBuild(NextBuildContext)
const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir })
let root =
turboNextBuildRoot ??
(turboJson
? path.dirname(turboJson)
: packagePath
? path.dirname(packagePath)
: undefined)
await binding.turbo.nextBuild({
...NextBuildContext,
root,
})
const [duration] = process.hrtime(turboNextBuildStart) const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null } return { duration, turbotraceContext: null }
} }
......
...@@ -18,6 +18,7 @@ const nextBuild: CliCommand = (argv) => { ...@@ -18,6 +18,7 @@ const nextBuild: CliCommand = (argv) => {
'--no-mangling': Boolean, '--no-mangling': Boolean,
'--experimental-app-only': Boolean, '--experimental-app-only': Boolean,
'--experimental-turbo': Boolean, '--experimental-turbo': Boolean,
'--experimental-turbo-root': String,
'--build-mode': String, '--build-mode': String,
// Aliases // Aliases
'-h': '--help', '-h': '--help',
...@@ -86,6 +87,7 @@ const nextBuild: CliCommand = (argv) => { ...@@ -86,6 +87,7 @@ const nextBuild: CliCommand = (argv) => {
args['--no-mangling'], args['--no-mangling'],
args['--experimental-app-only'], args['--experimental-app-only'],
!!process.env.TURBOPACK, !!process.env.TURBOPACK,
args['--experimental-turbo-root'],
args['--build-mode'] || 'default' args['--build-mode'] || 'default'
).catch((err) => { ).catch((err) => {
console.error('') console.error('')
......
...@@ -38,10 +38,38 @@ ...@@ -38,10 +38,38 @@
"outputs": ["dist/**"] "outputs": ["dist/**"]
}, },
"typescript": {}, "typescript": {},
"rust-check": {}, "rust-check": {
"test-cargo-unit": {}, "inputs": [
"test-cargo-integration": {}, ".cargo/**",
"test-cargo-bench": {}, "packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"test-cargo-unit": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"test-cargo-integration": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"test-cargo-bench": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"//#get-test-timings": { "//#get-test-timings": {
"inputs": ["run-tests.js"], "inputs": ["run-tests.js"],
"outputs": ["test-timings.json"] "outputs": ["test-timings.json"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment