Unverified Commit c6313606 authored by Alex Kirszenberg's avatar Alex Kirszenberg Committed by GitHub

Next Build Turbo POC (2) (#51546)

Another attempt at getting https://github.com/vercel/next.js/pull/49942 in.

This time, the mold install step is gated to Linux.
parent b714ac55
......@@ -98,6 +98,8 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
build: >-
set -e &&
apt update &&
apt install -y pkg-config &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-gnu &&
......@@ -110,7 +112,7 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
build: >-
set -e &&
apk add --no-cache libc6-compat &&
apk add --no-cache libc6-compat pkgconfig &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-musl &&
......@@ -133,6 +135,8 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64
build: >-
set -e &&
apt update &&
apt install -y pkg-config &&
export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
......@@ -146,7 +150,7 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
build: >-
set -e &&
apk add --no-cache libc6-compat &&
apk add --no-cache libc6-compat pkgconfig &&
export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
......
......@@ -73,6 +73,7 @@ jobs:
skipInstallBuild: 'yes'
skipNativeBuild: 'yes'
afterBuild: turbo run test-cargo-unit
mold: 'yes'
secrets: inherit
test-cargo-integration:
......
......@@ -7,6 +7,12 @@ on:
required: false
description: 'additional steps to run'
type: string
# Toggles the mold linker. The default linker is much slower and can run into OOMs.
# However, custom linkers won't work for wasm.
mold:
required: false
description: 'whether to use the mold linker'
type: string
skipInstallBuild:
required: false
description: 'whether to skip pnpm install && pnpm build'
......@@ -88,6 +94,13 @@ jobs:
with:
components: rustfmt, clippy
- name: 'Install mold linker'
if: ${{ inputs.mold == 'yes' }}
run: |
sudo apt update
sudo apt install -y mold
echo RUSTFLAGS=${RUSTFLAGS}\ -C\ link-arg=-fuse-ld=mold >> $GITHUB_ENV
- name: Install nextest
if: ${{ inputs.needsNextest == 'yes' }}
uses: taiki-e/install-action@nextest
......
......@@ -3260,7 +3260,16 @@ name = "next-build"
version = "0.1.0"
dependencies = [
"anyhow",
"clap 4.1.11",
"console-subscriber",
"dunce",
"next-core",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
"turbo-tasks",
"turbopack-binding",
"vergen",
]
......@@ -7376,6 +7385,7 @@ dependencies = [
"turbo-tasks-testing",
"turbopack",
"turbopack-bench",
"turbopack-build",
"turbopack-cli-utils",
"turbopack-core",
"turbopack-dev",
......@@ -7390,6 +7400,26 @@ dependencies = [
"turbopack-test-utils",
]
[[package]]
name = "turbopack-build"
version = "0.1.0"
source = "git+https://github.com/vercel/turbo.git?tag=turbopack-230615.1#1ff1956dc18ff1805b2ac87f21f79e1abea75fc8"
dependencies = [
"anyhow",
"indexmap",
"indoc",
"serde",
"serde_json",
"serde_qs",
"turbo-tasks",
"turbo-tasks-build",
"turbo-tasks-fs",
"turbopack-core",
"turbopack-css",
"turbopack-ecmascript",
"turbopack-ecmascript-runtime",
]
[[package]]
name = "turbopack-cli-utils"
version = "0.1.0"
......
......@@ -26,7 +26,7 @@ lto = true
[workspace.dependencies]
# Workspace crates
next-build = { path = "packages/next-swc/crates/next-build" }
next-build = { path = "packages/next-swc/crates/next-build", default-features = false }
next-core = { path = "packages/next-swc/crates/next-core", default-features = false }
next-dev = { path = "packages/next-swc/crates/next-dev", default-features = false, features = [
"serializable",
......
......@@ -48,9 +48,9 @@ turbo-tasks = { workspace = true }
once_cell = { workspace = true }
serde = "1"
serde_json = "1"
tracing = { version = "0.1.37" }
tracing = { workspace = true }
tracing-futures = "0.2.5"
tracing-subscriber = "0.3.9"
tracing-subscriber = { workspace = true }
tracing-chrome = "0.5.0"
turbopack-binding = { workspace = true, features = [
"__swc_core_binding_napi",
......
use std::convert::TryFrom;
use std::{
convert::{TryFrom, TryInto},
path::PathBuf,
};
use anyhow::Context;
use napi::bindgen_prelude::*;
use next_build::{next_build as turbo_next_build, NextBuildOptions};
use next_build::{
build as turbo_next_build, build_options::BuildContext, BuildOptions as NextBuildOptions,
};
use next_core::next_config::{Rewrite, Rewrites, RouteHas};
use next_dev::{devserver_options::DevServerOptions, start_server};
use crate::util::MapErr;
......@@ -15,95 +22,162 @@ pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct NextBuildContext {
// Added by Next.js for next build --turbo specifically.
/// The root directory of the workspace.
pub root: Option<String>,
/// The project's directory.
pub dir: Option<String>,
pub app_dir: Option<String>,
pub pages_dir: Option<String>,
pub rewrites: Option<Rewrites>,
pub original_rewrites: Option<Rewrites>,
pub original_redirects: Option<Vec<Redirect>>,
/// The build ID.
pub build_id: Option<String>,
/// The rewrites, as computed by Next.js.
pub rewrites: Option<NapiRewrites>,
// TODO(alexkirsz) These are detected directly by Turbopack for now.
// pub app_dir: Option<String>,
// pub pages_dir: Option<String>,
// TODO(alexkirsz) These are used to generate route types.
// pub original_rewrites: Option<Rewrites>,
// pub original_redirects: Option<Vec<Redirect>>,
}
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrites {
pub fallback: Vec<Rewrite>,
pub after_files: Vec<Rewrite>,
pub before_files: Vec<Rewrite>,
impl TryFrom<NextBuildContext> for NextBuildOptions {
type Error = napi::Error;
fn try_from(value: NextBuildContext) -> Result<Self> {
Ok(Self {
dir: value.dir.map(PathBuf::try_from).transpose()?,
root: value.root.map(PathBuf::try_from).transpose()?,
log_level: None,
show_all: true,
log_detail: true,
full_stats: true,
memory_limit: None,
build_context: Some(BuildContext {
build_id: value
.build_id
.context("NextBuildContext must provide a build ID")?,
rewrites: value
.rewrites
.context("NextBuildContext must provide rewrites")?
.into(),
}),
})
}
}
/// Keep in sync with [`next_core::next_config::Rewrites`]
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrite {
pub source: String,
pub destination: String,
pub struct NapiRewrites {
pub fallback: Vec<NapiRewrite>,
pub after_files: Vec<NapiRewrite>,
pub before_files: Vec<NapiRewrite>,
}
impl From<NapiRewrites> for Rewrites {
fn from(val: NapiRewrites) -> Self {
Rewrites {
fallback: val
.fallback
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
after_files: val
.after_files
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
before_files: val
.before_files
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
}
}
}
/// Keep in sync with [`next_core::next_config::Rewrite`]
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Redirect {
pub struct NapiRewrite {
pub source: String,
pub destination: String,
pub permanent: Option<bool>,
pub status_code: Option<u32>,
pub has: Option<RouteHas>,
pub missing: Option<RouteHas>,
pub base_path: Option<bool>,
pub locale: Option<bool>,
pub has: Option<Vec<NapiRouteHas>>,
pub missing: Option<Vec<NapiRouteHas>>,
}
#[derive(Debug)]
pub struct RouteHas {
pub r#type: RouteType,
pub key: Option<String>,
pub value: Option<String>,
impl From<NapiRewrite> for Rewrite {
fn from(val: NapiRewrite) -> Self {
Rewrite {
source: val.source,
destination: val.destination,
base_path: val.base_path,
locale: val.locale,
has: val
.has
.map(|has| has.into_iter().map(|has| has.into()).collect()),
missing: val
.missing
.map(|missing| missing.into_iter().map(|missing| missing.into()).collect()),
}
}
}
/// Keep in sync with [`next_core::next_config::RouteHas`]
#[derive(Debug)]
pub enum RouteType {
Header,
Query,
Cookie,
Host,
pub enum NapiRouteHas {
Header { key: String, value: Option<String> },
Query { key: String, value: Option<String> },
Cookie { key: String, value: Option<String> },
Host { value: String },
}
impl TryFrom<String> for RouteType {
type Error = napi::Error;
fn try_from(value: String) -> Result<Self> {
match value.as_str() {
"header" => Ok(RouteType::Header),
"query" => Ok(RouteType::Query),
"cookie" => Ok(RouteType::Cookie),
"host" => Ok(RouteType::Host),
_ => Err(napi::Error::new(
napi::Status::InvalidArg,
"Invalid route type",
)),
}
}
}
impl FromNapiValue for RouteHas {
impl FromNapiValue for NapiRouteHas {
unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
let object = Object::from_napi_value(env, napi_val)?;
let r#type = object.get_named_property::<String>("type")?;
Ok(RouteHas {
r#type: RouteType::try_from(r#type)?,
key: object.get("key")?,
value: object.get("value")?,
let type_ = object.get_named_property::<String>("type")?;
Ok(match type_.as_str() {
"header" => NapiRouteHas::Header {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"query" => NapiRouteHas::Query {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"cookie" => NapiRouteHas::Cookie {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"host" => NapiRouteHas::Host {
value: object.get_named_property("value")?,
},
_ => {
return Err(napi::Error::new(
Status::GenericFailure,
format!("invalid type for RouteHas: {}", type_),
))
}
})
}
}
impl From<NextBuildContext> for NextBuildOptions {
fn from(value: NextBuildContext) -> Self {
Self {
dir: value.dir,
memory_limit: None,
full_stats: None,
impl From<NapiRouteHas> for RouteHas {
fn from(val: NapiRouteHas) -> Self {
match val {
NapiRouteHas::Header { key, value } => RouteHas::Header { key, value },
NapiRouteHas::Query { key, value } => RouteHas::Query { key, value },
NapiRouteHas::Cookie { key, value } => RouteHas::Cookie { key, value },
NapiRouteHas::Host { value } => RouteHas::Host { value },
}
}
}
#[napi]
pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
turbo_next_build(ctx.into()).await.convert_err()
turbo_next_build(ctx.try_into()?).await.convert_err()
}
......@@ -6,16 +6,66 @@ license = "MPL-2.0"
edition = "2021"
autobenches = false
[[bin]]
name = "next-build"
path = "src/main.rs"
bench = false
required-features = ["cli"]
[lib]
bench = false
[features]
# By default, we enable native-tls for reqwest via downstream transitive features.
# This is for the convenience of running daily dev workflows, i.e running
# `cargo xxx` without explicitly specifying features, not that we want to
# promote this as default backend. Actual configuration is done when building next-swc,
# and also turbopack standalone when we have it.
default = ["cli", "custom_allocator", "native-tls"]
cli = ["clap"]
tokio_console = [
"dep:console-subscriber",
"tokio/tracing",
"turbo-tasks/tokio_tracing",
]
native-tls = ["next-core/native-tls"]
rustls-tls = ["next-core/rustls-tls"]
custom_allocator = ["turbopack-binding/__turbo_tasks_malloc", "turbopack-binding/__turbo_tasks_malloc_custom_allocator"]
custom_allocator = [
"turbopack-binding/__turbo_tasks_malloc",
"turbopack-binding/__turbo_tasks_malloc_custom_allocator",
]
serializable = []
profile = []
[dependencies]
anyhow = "1.0.47"
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive", "env"], optional = true }
console-subscriber = { workspace = true, optional = true }
dunce = { workspace = true }
next-core = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true, features = ["full"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
turbopack-binding = { workspace = true, features = ["__turbo_tasks", "__turbo_tasks_memory"] }
turbopack-binding = { workspace = true, features = [
"__turbo_tasks",
"__turbo_tasks_memory",
"__turbo_tasks_env",
"__turbo_tasks_fs",
"__turbo_tasks_memory",
"__turbopack",
"__turbopack_build",
"__turbopack_cli_utils",
"__turbopack_core",
"__turbopack_dev",
"__turbopack_ecmascript",
"__turbopack_ecmascript_runtime",
"__turbopack_env",
"__turbopack_node",
] }
turbo-tasks = { workspace = true }
[build-dependencies]
turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
......
use std::path::PathBuf;
use next_core::{next_config::Rewrites, turbopack::core::issue::IssueSeverity};
#[derive(Clone, Debug)]
pub struct BuildOptions {
/// The root directory of the workspace.
pub root: Option<PathBuf>,
/// The project's directory.
pub dir: Option<PathBuf>,
/// The maximum memory to use for the build.
pub memory_limit: Option<usize>,
/// The log level to use for the build.
pub log_level: Option<IssueSeverity>,
/// Whether to show all logs.
pub show_all: bool,
/// Whether to show detailed logs.
pub log_detail: bool,
/// Whether to compute full stats.
pub full_stats: bool,
/// The Next.js build context.
pub build_context: Option<BuildContext>,
}
#[derive(Clone, Debug)]
pub struct BuildContext {
/// The build id.
pub build_id: String,
/// Next.js config rewrites.
pub rewrites: Rewrites,
}
use turbopack_binding::turbo::{
tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi},
tasks::{run_once, TransientInstance, TurboTasks},
tasks_memory::MemoryBackend,
};
pub fn register() {
turbopack_binding::turbo::tasks::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
pub mod build_options;
pub mod manifests;
pub(crate) mod next_build;
pub(crate) mod next_pages;
pub struct NextBuildOptions {
pub dir: Option<String>,
pub memory_limit: Option<usize>,
pub full_stats: Option<bool>,
}
use anyhow::Result;
use turbo_tasks::{StatsType, TurboTasksBackendApi};
pub use self::build_options::BuildOptions;
pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> {
pub async fn build(options: BuildOptions) -> Result<()> {
#[cfg(feature = "tokio_console")]
console_subscriber::init();
register();
setup_tracing();
let tt = TurboTasks::new(MemoryBackend::new(
options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024),
));
let stats_type = match options.full_stats {
Some(true) => StatsType::Full,
_ => StatsType::Essential,
true => StatsType::Full,
false => StatsType::Essential,
};
tt.set_stats_type(stats_type);
let task = tt.spawn_root_task(move || {
Box::pin(async move {
// run next build here
Ok(NothingVc::new().into())
})
});
tt.wait_task_completion(task, true).await?;
run_once(tt, async move {
next_build::next_build(TransientInstance::new(options)).await?;
Ok(())
})
.await?;
Ok(())
}
fn setup_tracing() {
use tracing_subscriber::{prelude::*, EnvFilter, Registry};
let subscriber = Registry::default();
let stdout_log = tracing_subscriber::fmt::layer().pretty();
let subscriber = subscriber.with(stdout_log);
let subscriber = subscriber.with(EnvFilter::from_default_env());
subscriber.init();
}
pub fn register() {
turbopack_binding::turbo::tasks::register();
turbopack_binding::turbo::tasks_fs::register();
turbopack_binding::turbopack::turbopack::register();
turbopack_binding::turbopack::core::register();
turbopack_binding::turbopack::node::register();
turbopack_binding::turbopack::dev::register();
turbopack_binding::turbopack::build::register();
next_core::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
use std::path::PathBuf;
use anyhow::Result;
use clap::Parser;
use next_build::BuildOptions;
use turbopack_binding::turbopack::cli_utils::issue::IssueSeverityCliOption;
#[global_allocator]
static ALLOC: turbopack_binding::turbo::malloc::TurboMalloc =
turbopack_binding::turbo::malloc::TurboMalloc;
#[derive(Debug, Parser)]
#[clap(author, version, about, long_about = None)]
pub struct BuildCliArgs {
/// The directory of the Next.js application.
/// If no directory is provided, the current directory will be used.
#[clap(value_parser)]
pub dir: Option<PathBuf>,
/// The root directory of the project. Nothing outside of this directory can
/// be accessed. e. g. the monorepo root.
/// If no directory is provided, `dir` will be used.
#[clap(long, value_parser)]
pub root: Option<PathBuf>,
/// Display version of the binary. Noop if used in library mode.
#[clap(long)]
pub display_version: bool,
/// Filter by issue severity.
#[clap(short, long)]
pub log_level: Option<IssueSeverityCliOption>,
/// Show all log messages without limit.
#[clap(long)]
pub show_all: bool,
/// Expand the log details.
#[clap(long)]
pub log_detail: bool,
/// Whether to enable full task stats recording in Turbo Engine.
#[clap(long)]
pub full_stats: bool,
/// Enable experimental garbage collection with the provided memory limit in
/// MB.
#[clap(long)]
pub memory_limit: Option<usize>,
}
fn main() {
use turbopack_binding::turbo::malloc::TurboMalloc;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.on_thread_stop(|| {
TurboMalloc::thread_stop();
})
.build()
.unwrap()
.block_on(main_inner())
.unwrap()
}
async fn main_inner() -> Result<()> {
let args = BuildCliArgs::parse();
if args.display_version {
// Note: enabling git causes trouble with aarch64 linux builds with libz-sys
println!(
"Build Timestamp\t\t{:#?}",
option_env!("VERGEN_BUILD_TIMESTAMP").unwrap_or_else(|| "N/A")
);
println!(
"Build Version\t\t{:#?}",
option_env!("VERGEN_BUILD_SEMVER").unwrap_or_else(|| "N/A")
);
println!(
"Cargo Target Triple\t{:#?}",
option_env!("VERGEN_CARGO_TARGET_TRIPLE").unwrap_or_else(|| "N/A")
);
println!(
"Cargo Profile\t\t{:#?}",
option_env!("VERGEN_CARGO_PROFILE").unwrap_or_else(|| "N/A")
);
return Ok(());
}
next_build::build(BuildOptions {
dir: args.dir,
root: args.root,
memory_limit: args.memory_limit,
log_level: args.log_level.map(|l| l.0),
show_all: args.show_all,
log_detail: args.log_detail,
full_stats: args.full_stats,
build_context: None,
})
.await
}
//! Type definitions for the Next.js manifest formats.
use std::collections::HashMap;
use next_core::next_config::Rewrites;
use serde::Serialize;
#[derive(Serialize, Default, Debug)]
pub struct PagesManifest {
#[serde(flatten)]
pub pages: HashMap<String, String>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BuildManifest {
pub dev_files: Vec<String>,
pub amp_dev_files: Vec<String>,
pub polyfill_files: Vec<String>,
pub low_priority_files: Vec<String>,
pub root_main_files: Vec<String>,
pub pages: HashMap<String, Vec<String>>,
pub amp_first_pages: Vec<String>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase", tag = "version")]
pub enum MiddlewaresManifest {
#[serde(rename = "2")]
MiddlewaresManifestV2(MiddlewaresManifestV2),
#[serde(other)]
Unsupported,
}
impl Default for MiddlewaresManifest {
fn default() -> Self {
Self::MiddlewaresManifestV2(Default::default())
}
}
#[derive(Serialize, Default, Debug)]
pub struct MiddlewaresManifestV2 {
pub sorted_middleware: Vec<()>,
pub middleware: HashMap<String, ()>,
pub functions: HashMap<String, ()>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReactLoadableManifest {
#[serde(flatten)]
pub manifest: HashMap<String, ReactLoadableManifestEntry>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReactLoadableManifestEntry {
pub id: u32,
pub files: Vec<String>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct NextFontManifest {
pub pages: HashMap<String, Vec<String>>,
pub app: HashMap<String, Vec<String>>,
pub app_using_size_adjust: bool,
pub pages_using_size_adjust: bool,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct AppPathsManifest {
#[serde(flatten)]
pub edge_server_app_paths: PagesManifest,
#[serde(flatten)]
pub node_server_app_paths: PagesManifest,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ServerReferenceManifest {
#[serde(flatten)]
pub server_actions: ActionManifest,
#[serde(flatten)]
pub edge_server_actions: ActionManifest,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ActionManifest {
#[serde(flatten)]
pub actions: HashMap<String, ActionManifestEntry>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ActionManifestEntry {
pub workers: HashMap<String, ActionManifestWorkerEntry>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
#[serde(untagged)]
pub enum ActionManifestWorkerEntry {
String(String),
Number(f64),
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientReferenceManifest {
pub client_modules: ManifestNode,
pub ssr_module_mapping: HashMap<String, ManifestNode>,
#[serde(rename = "edgeSSRModuleMapping")]
pub edge_ssr_module_mapping: HashMap<String, ManifestNode>,
pub css_files: HashMap<String, Vec<String>>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientCssReferenceManifest {
pub css_imports: HashMap<String, Vec<String>>,
pub css_modules: HashMap<String, Vec<String>>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ManifestNode {
#[serde(flatten)]
pub module_exports: HashMap<String, ManifestNodeEntry>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ManifestNodeEntry {
pub id: ModuleId,
pub name: String,
pub chunks: Vec<String>,
pub r#async: bool,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
#[serde(untagged)]
pub enum ModuleId {
String(String),
Number(f64),
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct FontManifest(pub Vec<FontManifestEntry>);
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct FontManifestEntry {
pub url: String,
pub content: String,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct AppBuildManifest {
pub pages: HashMap<String, Vec<String>>,
}
// TODO(alexkirsz) Unify with the one for dev.
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientBuildManifest<'a> {
#[serde(rename = "__rewrites")]
pub rewrites: &'a Rewrites,
pub sorted_pages: &'a [String],
#[serde(flatten)]
pub pages: HashMap<String, Vec<&'a str>>,
}
This diff is collapsed.
use anyhow::{bail, Result};
use next_core::{
create_page_loader_entry_asset,
turbopack::core::{asset::AssetsVc, chunk::EvaluatableAssetsVc},
};
use turbopack_binding::{
turbo::{
tasks::{primitives::StringVc, Value},
tasks_fs::FileSystemPathVc,
},
turbopack::{
core::{
asset::AssetVc,
chunk::{ChunkableAsset, ChunkingContext, ChunkingContextVc},
context::{AssetContext, AssetContextVc},
reference_type::ReferenceType,
},
dev::DevChunkingContextVc,
ecmascript::EcmascriptModuleAssetVc,
},
};
#[turbo_tasks::value]
pub(crate) struct PagesBuildClientContext {
project_root: FileSystemPathVc,
client_root: FileSystemPathVc,
client_asset_context: AssetContextVc,
client_runtime_entries: EvaluatableAssetsVc,
}
#[turbo_tasks::value_impl]
impl PagesBuildClientContextVc {
#[turbo_tasks::function]
pub fn new(
project_root: FileSystemPathVc,
client_root: FileSystemPathVc,
client_asset_context: AssetContextVc,
client_runtime_entries: EvaluatableAssetsVc,
) -> PagesBuildClientContextVc {
PagesBuildClientContext {
project_root,
client_root,
client_asset_context,
client_runtime_entries,
}
.cell()
}
#[turbo_tasks::function]
async fn client_chunking_context(self) -> Result<ChunkingContextVc> {
let this = self.await?;
Ok(DevChunkingContextVc::builder(
this.project_root,
this.client_root,
this.client_root.join("static/chunks"),
this.client_root.join("static/media"),
this.client_asset_context.compile_time_info().environment(),
)
.build())
}
#[turbo_tasks::function]
pub async fn client_chunk(
self,
asset: AssetVc,
pathname: StringVc,
reference_type: Value<ReferenceType>,
) -> Result<AssetsVc> {
let this = self.await?;
let client_asset_page = this.client_asset_context.process(asset, reference_type);
let client_asset_page =
create_page_loader_entry_asset(this.client_asset_context, client_asset_page, pathname);
let Some(client_module_asset) = EcmascriptModuleAssetVc::resolve_from(client_asset_page).await? else {
bail!("Expected an EcmaScript module asset");
};
let client_chunking_context = self.client_chunking_context();
Ok(client_chunking_context.evaluated_chunk_group(
client_module_asset.as_root_chunk(client_chunking_context),
this.client_runtime_entries
.with_entry(client_module_asset.into()),
))
}
}
pub(crate) mod client_context;
pub(crate) mod node_context;
pub(crate) mod page_chunks;
use anyhow::{bail, Result};
use next_core::{next_client::RuntimeEntriesVc, turbopack::core::chunk::EvaluatableAssetsVc};
use turbopack_binding::{
turbo::{tasks::Value, tasks_fs::FileSystemPathVc},
turbopack::{
build::BuildChunkingContextVc,
core::{
asset::AssetVc,
context::{AssetContext, AssetContextVc},
reference_type::{EntryReferenceSubType, ReferenceType},
resolve::{parse::RequestVc, pattern::QueryMapVc},
},
ecmascript::EcmascriptModuleAssetVc,
},
};
#[turbo_tasks::value]
pub(crate) struct PagesBuildNodeContext {
project_root: FileSystemPathVc,
node_root: FileSystemPathVc,
node_asset_context: AssetContextVc,
node_runtime_entries: EvaluatableAssetsVc,
}
#[turbo_tasks::value_impl]
impl PagesBuildNodeContextVc {
#[turbo_tasks::function]
pub fn new(
project_root: FileSystemPathVc,
node_root: FileSystemPathVc,
node_asset_context: AssetContextVc,
node_runtime_entries: RuntimeEntriesVc,
) -> PagesBuildNodeContextVc {
PagesBuildNodeContext {
project_root,
node_root,
node_asset_context,
node_runtime_entries: node_runtime_entries.resolve_entries(node_asset_context),
}
.cell()
}
#[turbo_tasks::function]
pub async fn resolve_module(
self,
origin: FileSystemPathVc,
package: String,
path: String,
) -> Result<AssetVc> {
let this = self.await?;
let Some(asset) = this
.node_asset_context
.resolve_asset(
origin,
RequestVc::module(package.clone(), Value::new(path.clone().into()), QueryMapVc::none()),
this.node_asset_context.resolve_options(origin, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page))),
Value::new(ReferenceType::Entry(EntryReferenceSubType::Page))
)
.primary_assets()
.await?
.first()
.copied()
else {
bail!("module {}/{} not found in {}", package, path, origin.await?);
};
Ok(asset)
}
#[turbo_tasks::function]
async fn node_chunking_context(self) -> Result<BuildChunkingContextVc> {
let this = self.await?;
Ok(BuildChunkingContextVc::builder(
this.project_root,
this.node_root,
this.node_root.join("server/pages"),
this.node_root.join("server/assets"),
this.node_asset_context.compile_time_info().environment(),
)
.build())
}
#[turbo_tasks::function]
pub async fn node_chunk(
self,
asset: AssetVc,
reference_type: Value<ReferenceType>,
) -> Result<AssetVc> {
let this = self.await?;
let node_asset_page = this.node_asset_context.process(asset, reference_type);
let Some(node_module_asset) = EcmascriptModuleAssetVc::resolve_from(node_asset_page).await? else {
bail!("Expected an EcmaScript module asset");
};
let chunking_context = self.node_chunking_context();
Ok(chunking_context.generate_exported_chunk(node_module_asset, this.node_runtime_entries))
}
}
......@@ -15,7 +15,7 @@ pub mod manifest;
pub mod mode;
mod next_build;
pub mod next_client;
mod next_client_chunks;
pub mod next_client_chunks;
mod next_client_component;
pub mod next_config;
mod next_edge;
......@@ -38,8 +38,10 @@ mod util;
mod web_entry_source;
pub use app_source::create_app_source;
pub use page_loader::create_page_loader_entry_asset;
pub use page_source::create_page_source;
pub use turbopack_binding::{turbopack::node::source_map, *};
pub use util::{pathname_for_path, PathType};
pub use web_entry_source::create_web_entry_source;
pub fn register() {
......
......@@ -337,14 +337,14 @@ pub fn get_client_chunking_context(
#[turbo_tasks::function]
pub fn get_client_assets_path(
server_root: FileSystemPathVc,
client_root: FileSystemPathVc,
ty: Value<ClientContextType>,
) -> FileSystemPathVc {
match ty.into_value() {
ClientContextType::Pages { .. }
| ClientContextType::App { .. }
| ClientContextType::Fallback => server_root.join("/_next/static/media"),
ClientContextType::Other => server_root.join("/_assets"),
| ClientContextType::Fallback => client_root.join("/_next/static/media"),
ClientContextType::Other => client_root.join("/_assets"),
}
}
......
......@@ -2,3 +2,10 @@ pub(crate) mod context;
pub(crate) mod runtime_entry;
pub(crate) mod transforms;
pub(crate) mod transition;
pub use context::{
get_client_compile_time_info, get_client_module_options_context,
get_client_resolve_options_context, get_client_runtime_entries, ClientContextType,
};
pub use runtime_entry::{RuntimeEntries, RuntimeEntriesVc, RuntimeEntry, RuntimeEntryVc};
pub use transition::NextClientTransition;
pub(crate) mod client_chunks_transition;
pub(crate) mod in_chunking_context_asset;
pub(crate) mod with_chunks;
pub use client_chunks_transition::NextClientChunksTransitionVc;
......@@ -678,7 +678,6 @@ pub async fn load_next_config_internal(
next_asset("entry/config/next.js"),
Value::new(ReferenceType::Entry(EntryReferenceSubType::Undefined)),
);
let config_value = evaluate(
load_next_config_asset,
project_path,
......
pub(crate) mod context;
pub(crate) mod resolve;
pub(crate) mod transforms;
pub use context::{
get_server_compile_time_info, get_server_module_options_context,
get_server_resolve_options_context, ServerContextType,
};
......@@ -54,48 +54,51 @@ pub struct PageLoaderAsset {
pub pathname: StringVc,
}
#[turbo_tasks::value_impl]
impl PageLoaderAssetVc {
#[turbo_tasks::function]
async fn get_loader_entry_asset(self) -> Result<AssetVc> {
let this = &*self.await?;
#[turbo_tasks::function]
pub async fn create_page_loader_entry_asset(
client_context: AssetContextVc,
entry_asset: AssetVc,
pathname: StringVc,
) -> Result<AssetVc> {
let mut result = RopeBuilder::default();
writeln!(
result,
"const PAGE_PATH = {};\n",
StringifyJs(&*pathname.await?)
)?;
let page_loader_path = next_js_file_path("entry/page-loader.ts");
let base_code = page_loader_path.read();
if let FileContent::Content(base_file) = &*base_code.await? {
result += base_file.content()
} else {
bail!("required file `entry/page-loader.ts` not found");
}
let mut result = RopeBuilder::default();
writeln!(
result,
"const PAGE_PATH = {};\n",
StringifyJs(&*this.pathname.await?)
)?;
let page_loader_path = next_js_file_path("entry/page-loader.ts");
let base_code = page_loader_path.read();
if let FileContent::Content(base_file) = &*base_code.await? {
result += base_file.content()
} else {
bail!("required file `entry/page-loader.ts` not found");
}
let file = File::from(result.build());
let file = File::from(result.build());
let virtual_asset = VirtualAssetVc::new(page_loader_path, file.into()).into();
Ok(VirtualAssetVc::new(page_loader_path, file.into()).into())
}
Ok(client_context.process(
virtual_asset,
Value::new(ReferenceType::Internal(
InnerAssetsVc::cell(indexmap! {
"PAGE".to_string() => client_context.process(entry_asset, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)))
})
)))
)
}
#[turbo_tasks::value_impl]
impl PageLoaderAssetVc {
#[turbo_tasks::function]
async fn get_page_chunks(self) -> Result<AssetsVc> {
let this = &*self.await?;
let loader_entry_asset = self.get_loader_entry_asset();
let module = this.client_context.process(
loader_entry_asset,
Value::new(ReferenceType::Internal(
InnerAssetsVc::cell(indexmap! {
"PAGE".to_string() => this.client_context.process(this.entry_asset, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)))
})
)),
);
let page_loader_entry_asset =
create_page_loader_entry_asset(this.client_context, this.entry_asset, this.pathname);
let Some(module) = EvaluatableAssetVc::resolve_from(module).await? else {
let Some(module) = EvaluatableAssetVc::resolve_from(page_loader_entry_asset).await? else {
bail!("internal module must be evaluatable");
};
......
......@@ -107,6 +107,12 @@ pub struct PagesDirectoryStructure {
#[turbo_tasks::value_impl]
impl PagesDirectoryStructureVc {
/// Returns the router path of this directory.
#[turbo_tasks::function]
pub async fn next_router_path(self) -> Result<FileSystemPathVc> {
Ok(self.await?.next_router_path)
}
/// Returns the path to the directory of this structure in the project file
/// system.
#[turbo_tasks::function]
......
......@@ -38,7 +38,7 @@ use crate::{
get_client_asset_context, get_client_chunking_context,
get_client_resolve_options_context, ClientContextType,
},
runtime_entry::{RuntimeEntriesVc, RuntimeEntry},
RuntimeEntriesVc, RuntimeEntry,
},
next_config::NextConfigVc,
};
......
......@@ -210,6 +210,7 @@ export default async function build(
noMangling = false,
appDirOnly = false,
turboNextBuild = false,
turboNextBuildRoot = null,
buildMode: 'default' | 'experimental-compile' | 'experimental-generate'
): Promise<void> {
const isCompile = buildMode === 'experimental-compile'
......@@ -550,6 +551,23 @@ export default async function build(
app: appPageKeys.length > 0 ? appPageKeys : undefined,
}
if (turboNextBuild) {
// TODO(WEB-397) This is a temporary workaround to allow for filtering a
// subset of pages when building with --experimental-turbo, until we
// have complete support for all pages.
if (process.env.NEXT_TURBO_FILTER_PAGES) {
const filterPages = process.env.NEXT_TURBO_FILTER_PAGES.split(',')
pageKeys.pages = pageKeys.pages.filter((page) => {
return filterPages.some((filterPage) => {
return isMatch(page, filterPage)
})
})
}
// TODO(alexkirsz) Filter out app pages entirely as they are not supported yet.
pageKeys.app = undefined
}
const numConflictingAppPaths = conflictingAppPagePaths.length
if (mappedAppPages && numConflictingAppPaths > 0) {
Log.error(
......@@ -932,7 +950,23 @@ export default async function build(
async function turbopackBuild() {
const turboNextBuildStart = process.hrtime()
await binding.turbo.nextBuild(NextBuildContext)
const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir })
let root =
turboNextBuildRoot ??
(turboJson
? path.dirname(turboJson)
: packagePath
? path.dirname(packagePath)
: undefined)
await binding.turbo.nextBuild({
...NextBuildContext,
root,
})
const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null }
}
......
......@@ -18,6 +18,7 @@ const nextBuild: CliCommand = (argv) => {
'--no-mangling': Boolean,
'--experimental-app-only': Boolean,
'--experimental-turbo': Boolean,
'--experimental-turbo-root': String,
'--build-mode': String,
// Aliases
'-h': '--help',
......@@ -86,6 +87,7 @@ const nextBuild: CliCommand = (argv) => {
args['--no-mangling'],
args['--experimental-app-only'],
!!process.env.TURBOPACK,
args['--experimental-turbo-root'],
args['--build-mode'] || 'default'
).catch((err) => {
console.error('')
......
......@@ -38,10 +38,38 @@
"outputs": ["dist/**"]
},
"typescript": {},
"rust-check": {},
"test-cargo-unit": {},
"test-cargo-integration": {},
"test-cargo-bench": {},
"rust-check": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"test-cargo-unit": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"test-cargo-integration": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"test-cargo-bench": {
"inputs": [
".cargo/**",
"packages/next-swc/crates/**",
"**/Cargo.toml",
"**/Cargo.lock"
]
},
"//#get-test-timings": {
"inputs": ["run-tests.js"],
"outputs": ["test-timings.json"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment