Unverified Commit 7d0bdab8 authored by Alex Kirszenberg's avatar Alex Kirszenberg Committed by GitHub

Next Build Turbo POC (#49942)

This contains the original POC for `next build --turbo`. The implementation is _just enough_ to get pages building, and doesn't support the app router yet.

I'll write more details here on the implementation and what the next steps are next week.

Necessary changes on the Turbo side: https://github.com/vercel/turbo/pull/4998
parent 5b7b91f3
......@@ -5,6 +5,10 @@ CARGO_WORKSPACE_DIR = { value = "", relative = true }
rustdocflags = []
[target.x86_64-unknown-linux-gnu]
# Should be kept in sync with turbopack's linker
rustflags = ["-C", "link-arg=-fuse-ld=mold"]
[target.x86_64-pc-windows-msvc]
linker = "rust-lld"
......
......@@ -52,6 +52,12 @@ runs:
echo CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse >> $GITHUB_ENV
fi
- shell: bash
run: |
: install mold linker
sudo apt update
sudo apt install -y mold
- name: 'Setup Rust toolchain'
uses: dtolnay/rust-toolchain@master
if: ${{ !inputs.skip-install }}
......
......@@ -3260,7 +3260,16 @@ name = "next-build"
version = "0.1.0"
dependencies = [
"anyhow",
"clap 4.1.11",
"console-subscriber",
"dunce",
"next-core",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
"turbo-tasks",
"turbopack-binding",
"vergen",
]
......@@ -7376,6 +7385,7 @@ dependencies = [
"turbo-tasks-testing",
"turbopack",
"turbopack-bench",
"turbopack-build",
"turbopack-cli-utils",
"turbopack-core",
"turbopack-dev",
......@@ -7390,6 +7400,26 @@ dependencies = [
"turbopack-test-utils",
]
[[package]]
name = "turbopack-build"
version = "0.1.0"
source = "git+https://github.com/vercel/turbo.git?tag=turbopack-230615.1#1ff1956dc18ff1805b2ac87f21f79e1abea75fc8"
dependencies = [
"anyhow",
"indexmap",
"indoc",
"serde",
"serde_json",
"serde_qs",
"turbo-tasks",
"turbo-tasks-build",
"turbo-tasks-fs",
"turbopack-core",
"turbopack-css",
"turbopack-ecmascript",
"turbopack-ecmascript-runtime",
]
[[package]]
name = "turbopack-cli-utils"
version = "0.1.0"
......
......@@ -48,9 +48,9 @@ turbo-tasks = { workspace = true }
once_cell = { workspace = true }
serde = "1"
serde_json = "1"
tracing = { version = "0.1.37" }
tracing = { workspace = true }
tracing-futures = "0.2.5"
tracing-subscriber = "0.3.9"
tracing-subscriber = { workspace = true }
tracing-chrome = "0.5.0"
turbopack-binding = { workspace = true, features = [
"__swc_core_binding_napi",
......
use std::convert::TryFrom;
use std::{
convert::{TryFrom, TryInto},
path::PathBuf,
};
use anyhow::Context;
use napi::bindgen_prelude::*;
use next_build::{next_build as turbo_next_build, NextBuildOptions};
use next_build::{
build as turbo_next_build, build_options::BuildContext, BuildOptions as NextBuildOptions,
};
use next_core::next_config::{Rewrite, Rewrites, RouteHas};
use next_dev::{devserver_options::DevServerOptions, start_server};
use crate::util::MapErr;
......@@ -15,95 +22,162 @@ pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct NextBuildContext {
// Added by Next.js for next build --turbo specifically.
/// The root directory of the workspace.
pub root: Option<String>,
/// The project's directory.
pub dir: Option<String>,
pub app_dir: Option<String>,
pub pages_dir: Option<String>,
pub rewrites: Option<Rewrites>,
pub original_rewrites: Option<Rewrites>,
pub original_redirects: Option<Vec<Redirect>>,
/// The build ID.
pub build_id: Option<String>,
/// The rewrites, as computed by Next.js.
pub rewrites: Option<NapiRewrites>,
// TODO(alexkirsz) These are detected directly by Turbopack for now.
// pub app_dir: Option<String>,
// pub pages_dir: Option<String>,
// TODO(alexkirsz) These are used to generate route types.
// pub original_rewrites: Option<Rewrites>,
// pub original_redirects: Option<Vec<Redirect>>,
}
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrites {
pub fallback: Vec<Rewrite>,
pub after_files: Vec<Rewrite>,
pub before_files: Vec<Rewrite>,
impl TryFrom<NextBuildContext> for NextBuildOptions {
type Error = napi::Error;
fn try_from(value: NextBuildContext) -> Result<Self> {
Ok(Self {
dir: value.dir.map(PathBuf::try_from).transpose()?,
root: value.root.map(PathBuf::try_from).transpose()?,
log_level: None,
show_all: true,
log_detail: true,
full_stats: true,
memory_limit: None,
build_context: Some(BuildContext {
build_id: value
.build_id
.context("NextBuildContext must provide a build ID")?,
rewrites: value
.rewrites
.context("NextBuildContext must provide rewrites")?
.into(),
}),
})
}
}
/// Keep in sync with [`next_core::next_config::Rewrites`]
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Rewrite {
pub source: String,
pub destination: String,
pub struct NapiRewrites {
pub fallback: Vec<NapiRewrite>,
pub after_files: Vec<NapiRewrite>,
pub before_files: Vec<NapiRewrite>,
}
impl From<NapiRewrites> for Rewrites {
fn from(val: NapiRewrites) -> Self {
Rewrites {
fallback: val
.fallback
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
after_files: val
.after_files
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
before_files: val
.before_files
.into_iter()
.map(|rewrite| rewrite.into())
.collect(),
}
}
}
/// Keep in sync with [`next_core::next_config::Rewrite`]
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct Redirect {
pub struct NapiRewrite {
pub source: String,
pub destination: String,
pub permanent: Option<bool>,
pub status_code: Option<u32>,
pub has: Option<RouteHas>,
pub missing: Option<RouteHas>,
pub base_path: Option<bool>,
pub locale: Option<bool>,
pub has: Option<Vec<NapiRouteHas>>,
pub missing: Option<Vec<NapiRouteHas>>,
}
#[derive(Debug)]
pub struct RouteHas {
pub r#type: RouteType,
pub key: Option<String>,
pub value: Option<String>,
impl From<NapiRewrite> for Rewrite {
fn from(val: NapiRewrite) -> Self {
Rewrite {
source: val.source,
destination: val.destination,
base_path: val.base_path,
locale: val.locale,
has: val
.has
.map(|has| has.into_iter().map(|has| has.into()).collect()),
missing: val
.missing
.map(|missing| missing.into_iter().map(|missing| missing.into()).collect()),
}
}
}
/// Keep in sync with [`next_core::next_config::RouteHas`]
#[derive(Debug)]
pub enum RouteType {
Header,
Query,
Cookie,
Host,
pub enum NapiRouteHas {
Header { key: String, value: Option<String> },
Query { key: String, value: Option<String> },
Cookie { key: String, value: Option<String> },
Host { value: String },
}
impl TryFrom<String> for RouteType {
type Error = napi::Error;
fn try_from(value: String) -> Result<Self> {
match value.as_str() {
"header" => Ok(RouteType::Header),
"query" => Ok(RouteType::Query),
"cookie" => Ok(RouteType::Cookie),
"host" => Ok(RouteType::Host),
_ => Err(napi::Error::new(
napi::Status::InvalidArg,
"Invalid route type",
)),
}
}
}
impl FromNapiValue for RouteHas {
impl FromNapiValue for NapiRouteHas {
unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
let object = Object::from_napi_value(env, napi_val)?;
let r#type = object.get_named_property::<String>("type")?;
Ok(RouteHas {
r#type: RouteType::try_from(r#type)?,
key: object.get("key")?,
value: object.get("value")?,
let type_ = object.get_named_property::<String>("type")?;
Ok(match type_.as_str() {
"header" => NapiRouteHas::Header {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"query" => NapiRouteHas::Query {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"cookie" => NapiRouteHas::Cookie {
key: object.get_named_property("key")?,
value: object.get_named_property("value")?,
},
"host" => NapiRouteHas::Host {
value: object.get_named_property("value")?,
},
_ => {
return Err(napi::Error::new(
Status::GenericFailure,
format!("invalid type for RouteHas: {}", type_),
))
}
})
}
}
impl From<NextBuildContext> for NextBuildOptions {
fn from(value: NextBuildContext) -> Self {
Self {
dir: value.dir,
memory_limit: None,
full_stats: None,
impl From<NapiRouteHas> for RouteHas {
fn from(val: NapiRouteHas) -> Self {
match val {
NapiRouteHas::Header { key, value } => RouteHas::Header { key, value },
NapiRouteHas::Query { key, value } => RouteHas::Query { key, value },
NapiRouteHas::Cookie { key, value } => RouteHas::Cookie { key, value },
NapiRouteHas::Host { value } => RouteHas::Host { value },
}
}
}
#[napi]
pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
turbo_next_build(ctx.into()).await.convert_err()
turbo_next_build(ctx.try_into()?).await.convert_err()
}
......@@ -6,16 +6,67 @@ license = "MPL-2.0"
edition = "2021"
autobenches = false
[[bin]]
name = "next-build"
path = "src/main.rs"
bench = false
required-features = ["cli"]
[lib]
bench = false
[features]
# By default, we enable native-tls for reqwest via downstream transitive features.
# This is for the convenience of running daily dev workflows, i.e running
# `cargo xxx` without explicitly specifying features, not that we want to
# promote this as default backend. Actual configuration is done when building next-swc,
# and also turbopack standalone when we have it.
default = ["cli", "custom_allocator", "native-tls"]
cli = ["clap"]
tokio_console = [
"dep:console-subscriber",
"tokio/tracing",
"turbo-tasks/tokio_tracing",
]
native-tls = ["next-core/native-tls"]
rustls-tls = ["next-core/rustls-tls"]
custom_allocator = ["turbopack-binding/__turbo_tasks_malloc", "turbopack-binding/__turbo_tasks_malloc_custom_allocator"]
custom_allocator = [
"turbopack-binding/__turbo_tasks_malloc",
"turbopack-binding/__turbo_tasks_malloc_custom_allocator",
]
serializable = []
profile = []
[dependencies]
anyhow = "1.0.47"
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive", "env"], optional = true }
console-subscriber = { workspace = true, optional = true }
dunce = { workspace = true }
next-core = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
turbopack-binding = { workspace = true, features = ["__turbo_tasks", "__turbo_tasks_memory"] }
turbopack-binding = { workspace = true, features = [
"__turbo_tasks",
"__turbo_tasks_malloc",
"__turbo_tasks_memory",
"__turbo_tasks_env",
"__turbo_tasks_fs",
"__turbo_tasks_memory",
"__turbopack",
"__turbopack_build",
"__turbopack_cli_utils",
"__turbopack_core",
"__turbopack_dev",
"__turbopack_ecmascript",
"__turbopack_ecmascript_runtime",
"__turbopack_env",
"__turbopack_node",
] }
turbo-tasks = { workspace = true }
[build-dependencies]
turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
......
use std::path::PathBuf;
use next_core::{next_config::Rewrites, turbopack::core::issue::IssueSeverity};
#[derive(Clone, Debug)]
pub struct BuildOptions {
/// The root directory of the workspace.
pub root: Option<PathBuf>,
/// The project's directory.
pub dir: Option<PathBuf>,
/// The maximum memory to use for the build.
pub memory_limit: Option<usize>,
/// The log level to use for the build.
pub log_level: Option<IssueSeverity>,
/// Whether to show all logs.
pub show_all: bool,
/// Whether to show detailed logs.
pub log_detail: bool,
/// Whether to compute full stats.
pub full_stats: bool,
/// The Next.js build context.
pub build_context: Option<BuildContext>,
}
#[derive(Clone, Debug)]
pub struct BuildContext {
/// The build id.
pub build_id: String,
/// Next.js config rewrites.
pub rewrites: Rewrites,
}
use turbopack_binding::turbo::{
tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi},
tasks::{run_once, TransientInstance, TurboTasks},
tasks_memory::MemoryBackend,
};
pub fn register() {
turbopack_binding::turbo::tasks::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
pub mod build_options;
pub mod manifests;
pub(crate) mod next_build;
pub(crate) mod next_pages;
pub struct NextBuildOptions {
pub dir: Option<String>,
pub memory_limit: Option<usize>,
pub full_stats: Option<bool>,
}
use anyhow::Result;
use turbo_tasks::{StatsType, TurboTasksBackendApi};
pub use self::build_options::BuildOptions;
pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> {
pub async fn build(options: BuildOptions) -> Result<()> {
#[cfg(feature = "tokio_console")]
console_subscriber::init();
register();
setup_tracing();
let tt = TurboTasks::new(MemoryBackend::new(
options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024),
));
let stats_type = match options.full_stats {
Some(true) => StatsType::Full,
_ => StatsType::Essential,
true => StatsType::Full,
false => StatsType::Essential,
};
tt.set_stats_type(stats_type);
let task = tt.spawn_root_task(move || {
Box::pin(async move {
// run next build here
Ok(NothingVc::new().into())
})
});
tt.wait_task_completion(task, true).await?;
run_once(tt, async move {
next_build::next_build(TransientInstance::new(options)).await?;
Ok(())
})
.await?;
Ok(())
}
fn setup_tracing() {
use tracing_subscriber::{prelude::*, EnvFilter, Registry};
let subscriber = Registry::default();
let stdout_log = tracing_subscriber::fmt::layer().pretty();
let subscriber = subscriber.with(stdout_log);
let subscriber = subscriber.with(EnvFilter::from_default_env());
subscriber.init();
}
pub fn register() {
turbopack_binding::turbo::tasks::register();
turbopack_binding::turbo::tasks_fs::register();
turbopack_binding::turbopack::turbopack::register();
turbopack_binding::turbopack::core::register();
turbopack_binding::turbopack::node::register();
turbopack_binding::turbopack::dev::register();
turbopack_binding::turbopack::build::register();
next_core::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
use std::path::PathBuf;
use anyhow::Result;
use clap::Parser;
use next_build::BuildOptions;
use turbopack_binding::turbopack::cli_utils::issue::IssueSeverityCliOption;
#[global_allocator]
static ALLOC: turbopack_binding::turbo::malloc::TurboMalloc =
turbopack_binding::turbo::malloc::TurboMalloc;
#[derive(Debug, Parser)]
#[clap(author, version, about, long_about = None)]
pub struct BuildCliArgs {
/// The directory of the Next.js application.
/// If no directory is provided, the current directory will be used.
#[clap(value_parser)]
pub dir: Option<PathBuf>,
/// The root directory of the project. Nothing outside of this directory can
/// be accessed. e. g. the monorepo root.
/// If no directory is provided, `dir` will be used.
#[clap(long, value_parser)]
pub root: Option<PathBuf>,
/// Display version of the binary. Noop if used in library mode.
#[clap(long)]
pub display_version: bool,
/// Filter by issue severity.
#[clap(short, long)]
pub log_level: Option<IssueSeverityCliOption>,
/// Show all log messages without limit.
#[clap(long)]
pub show_all: bool,
/// Expand the log details.
#[clap(long)]
pub log_detail: bool,
/// Whether to enable full task stats recording in Turbo Engine.
#[clap(long)]
pub full_stats: bool,
/// Enable experimental garbage collection with the provided memory limit in
/// MB.
#[clap(long)]
pub memory_limit: Option<usize>,
}
fn main() {
use turbopack_binding::turbo::malloc::TurboMalloc;
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.on_thread_stop(|| {
TurboMalloc::thread_stop();
})
.build()
.unwrap()
.block_on(main_inner())
.unwrap()
}
async fn main_inner() -> Result<()> {
let args = BuildCliArgs::parse();
if args.display_version {
// Note: enabling git causes trouble with aarch64 linux builds with libz-sys
println!(
"Build Timestamp\t\t{:#?}",
option_env!("VERGEN_BUILD_TIMESTAMP").unwrap_or_else(|| "N/A")
);
println!(
"Build Version\t\t{:#?}",
option_env!("VERGEN_BUILD_SEMVER").unwrap_or_else(|| "N/A")
);
println!(
"Cargo Target Triple\t{:#?}",
option_env!("VERGEN_CARGO_TARGET_TRIPLE").unwrap_or_else(|| "N/A")
);
println!(
"Cargo Profile\t\t{:#?}",
option_env!("VERGEN_CARGO_PROFILE").unwrap_or_else(|| "N/A")
);
return Ok(());
}
next_build::build(BuildOptions {
dir: args.dir,
root: args.root,
memory_limit: args.memory_limit,
log_level: args.log_level.map(|l| l.0),
show_all: args.show_all,
log_detail: args.log_detail,
full_stats: args.full_stats,
build_context: None,
})
.await
}
//! Type definitions for the Next.js manifest formats.
use std::collections::HashMap;
use next_core::next_config::Rewrites;
use serde::Serialize;
#[derive(Serialize, Default, Debug)]
pub struct PagesManifest {
#[serde(flatten)]
pub pages: HashMap<String, String>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BuildManifest {
pub dev_files: Vec<String>,
pub amp_dev_files: Vec<String>,
pub polyfill_files: Vec<String>,
pub low_priority_files: Vec<String>,
pub root_main_files: Vec<String>,
pub pages: HashMap<String, Vec<String>>,
pub amp_first_pages: Vec<String>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase", tag = "version")]
pub enum MiddlewaresManifest {
#[serde(rename = "2")]
MiddlewaresManifestV2(MiddlewaresManifestV2),
#[serde(other)]
Unsupported,
}
impl Default for MiddlewaresManifest {
fn default() -> Self {
Self::MiddlewaresManifestV2(Default::default())
}
}
#[derive(Serialize, Default, Debug)]
pub struct MiddlewaresManifestV2 {
pub sorted_middleware: Vec<()>,
pub middleware: HashMap<String, ()>,
pub functions: HashMap<String, ()>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReactLoadableManifest {
#[serde(flatten)]
pub manifest: HashMap<String, ReactLoadableManifestEntry>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReactLoadableManifestEntry {
pub id: u32,
pub files: Vec<String>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct NextFontManifest {
pub pages: HashMap<String, Vec<String>>,
pub app: HashMap<String, Vec<String>>,
pub app_using_size_adjust: bool,
pub pages_using_size_adjust: bool,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct AppPathsManifest {
#[serde(flatten)]
pub edge_server_app_paths: PagesManifest,
#[serde(flatten)]
pub node_server_app_paths: PagesManifest,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ServerReferenceManifest {
#[serde(flatten)]
pub server_actions: ActionManifest,
#[serde(flatten)]
pub edge_server_actions: ActionManifest,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ActionManifest {
#[serde(flatten)]
pub actions: HashMap<String, ActionManifestEntry>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ActionManifestEntry {
pub workers: HashMap<String, ActionManifestWorkerEntry>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
#[serde(untagged)]
pub enum ActionManifestWorkerEntry {
String(String),
Number(f64),
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientReferenceManifest {
pub client_modules: ManifestNode,
pub ssr_module_mapping: HashMap<String, ManifestNode>,
#[serde(rename = "edgeSSRModuleMapping")]
pub edge_ssr_module_mapping: HashMap<String, ManifestNode>,
pub css_files: HashMap<String, Vec<String>>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientCssReferenceManifest {
pub css_imports: HashMap<String, Vec<String>>,
pub css_modules: HashMap<String, Vec<String>>,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ManifestNode {
#[serde(flatten)]
pub module_exports: HashMap<String, ManifestNodeEntry>,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ManifestNodeEntry {
pub id: ModuleId,
pub name: String,
pub chunks: Vec<String>,
pub r#async: bool,
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
#[serde(untagged)]
pub enum ModuleId {
String(String),
Number(f64),
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct FontManifest(pub Vec<FontManifestEntry>);
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct FontManifestEntry {
pub url: String,
pub content: String,
}
#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct AppBuildManifest {
pub pages: HashMap<String, Vec<String>>,
}
// TODO(alexkirsz) Unify with the one for dev.
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ClientBuildManifest<'a> {
#[serde(rename = "__rewrites")]
pub rewrites: &'a Rewrites,
pub sorted_pages: &'a [String],
#[serde(flatten)]
pub pages: HashMap<String, Vec<&'a str>>,
}
This diff is collapsed.
use anyhow::{bail, Result};
use next_core::{
create_page_loader_entry_asset,
turbopack::core::{asset::AssetsVc, chunk::EvaluatableAssetsVc},
};
use turbopack_binding::{
turbo::{
tasks::{primitives::StringVc, Value},
tasks_fs::FileSystemPathVc,
},
turbopack::{
core::{
asset::AssetVc,
chunk::{ChunkableAsset, ChunkingContext, ChunkingContextVc},
context::{AssetContext, AssetContextVc},
reference_type::ReferenceType,
},
dev::DevChunkingContextVc,
ecmascript::EcmascriptModuleAssetVc,
},
};
#[turbo_tasks::value]
pub(crate) struct PagesBuildClientContext {
project_root: FileSystemPathVc,
client_root: FileSystemPathVc,
client_asset_context: AssetContextVc,
client_runtime_entries: EvaluatableAssetsVc,
}
#[turbo_tasks::value_impl]
impl PagesBuildClientContextVc {
#[turbo_tasks::function]
pub fn new(
project_root: FileSystemPathVc,
client_root: FileSystemPathVc,
client_asset_context: AssetContextVc,
client_runtime_entries: EvaluatableAssetsVc,
) -> PagesBuildClientContextVc {
PagesBuildClientContext {
project_root,
client_root,
client_asset_context,
client_runtime_entries,
}
.cell()
}
#[turbo_tasks::function]
async fn client_chunking_context(self) -> Result<ChunkingContextVc> {
let this = self.await?;
Ok(DevChunkingContextVc::builder(
this.project_root,
this.client_root,
this.client_root.join("static/chunks"),
this.client_root.join("static/media"),
this.client_asset_context.compile_time_info().environment(),
)
.build())
}
#[turbo_tasks::function]
pub async fn client_chunk(
self,
asset: AssetVc,
pathname: StringVc,
reference_type: Value<ReferenceType>,
) -> Result<AssetsVc> {
let this = self.await?;
let client_asset_page = this.client_asset_context.process(asset, reference_type);
let client_asset_page =
create_page_loader_entry_asset(this.client_asset_context, client_asset_page, pathname);
let Some(client_module_asset) = EcmascriptModuleAssetVc::resolve_from(client_asset_page).await? else {
bail!("Expected an EcmaScript module asset");
};
let client_chunking_context = self.client_chunking_context();
Ok(client_chunking_context.evaluated_chunk_group(
client_module_asset.as_root_chunk(client_chunking_context),
this.client_runtime_entries
.with_entry(client_module_asset.into()),
))
}
}
pub(crate) mod client_context;
pub(crate) mod node_context;
pub(crate) mod page_chunks;
use anyhow::{bail, Result};
use next_core::{next_client::RuntimeEntriesVc, turbopack::core::chunk::EvaluatableAssetsVc};
use turbopack_binding::{
turbo::{tasks::Value, tasks_fs::FileSystemPathVc},
turbopack::{
build::BuildChunkingContextVc,
core::{
asset::AssetVc,
context::{AssetContext, AssetContextVc},
reference_type::{EntryReferenceSubType, ReferenceType},
resolve::{parse::RequestVc, pattern::QueryMapVc},
},
ecmascript::EcmascriptModuleAssetVc,
},
};
#[turbo_tasks::value]
pub(crate) struct PagesBuildNodeContext {
project_root: FileSystemPathVc,
node_root: FileSystemPathVc,
node_asset_context: AssetContextVc,
node_runtime_entries: EvaluatableAssetsVc,
}
#[turbo_tasks::value_impl]
impl PagesBuildNodeContextVc {
#[turbo_tasks::function]
pub fn new(
project_root: FileSystemPathVc,
node_root: FileSystemPathVc,
node_asset_context: AssetContextVc,
node_runtime_entries: RuntimeEntriesVc,
) -> PagesBuildNodeContextVc {
PagesBuildNodeContext {
project_root,
node_root,
node_asset_context,
node_runtime_entries: node_runtime_entries.resolve_entries(node_asset_context),
}
.cell()
}
#[turbo_tasks::function]
pub async fn resolve_module(
self,
origin: FileSystemPathVc,
package: String,
path: String,
) -> Result<AssetVc> {
let this = self.await?;
let Some(asset) = this
.node_asset_context
.resolve_asset(
origin,
RequestVc::module(package.clone(), Value::new(path.clone().into()), QueryMapVc::none()),
this.node_asset_context.resolve_options(origin, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page))),
Value::new(ReferenceType::Entry(EntryReferenceSubType::Page))
)
.primary_assets()
.await?
.first()
.copied()
else {
bail!("module {}/{} not found in {}", package, path, origin.await?);
};
Ok(asset)
}
#[turbo_tasks::function]
async fn node_chunking_context(self) -> Result<BuildChunkingContextVc> {
let this = self.await?;
Ok(BuildChunkingContextVc::builder(
this.project_root,
this.node_root,
this.node_root.join("server/pages"),
this.node_root.join("server/assets"),
this.node_asset_context.compile_time_info().environment(),
)
.build())
}
#[turbo_tasks::function]
pub async fn node_chunk(
self,
asset: AssetVc,
reference_type: Value<ReferenceType>,
) -> Result<AssetVc> {
let this = self.await?;
let node_asset_page = this.node_asset_context.process(asset, reference_type);
let Some(node_module_asset) = EcmascriptModuleAssetVc::resolve_from(node_asset_page).await? else {
bail!("Expected an EcmaScript module asset");
};
let chunking_context = self.node_chunking_context();
Ok(chunking_context.generate_exported_chunk(node_module_asset, this.node_runtime_entries))
}
}
......@@ -15,7 +15,7 @@ pub mod manifest;
pub mod mode;
mod next_build;
pub mod next_client;
mod next_client_chunks;
pub mod next_client_chunks;
mod next_client_component;
pub mod next_config;
mod next_edge;
......@@ -38,8 +38,10 @@ mod util;
mod web_entry_source;
pub use app_source::create_app_source;
pub use page_loader::create_page_loader_entry_asset;
pub use page_source::create_page_source;
pub use turbopack_binding::{turbopack::node::source_map, *};
pub use util::{pathname_for_path, PathType};
pub use web_entry_source::create_web_entry_source;
pub fn register() {
......
......@@ -337,14 +337,14 @@ pub fn get_client_chunking_context(
#[turbo_tasks::function]
pub fn get_client_assets_path(
server_root: FileSystemPathVc,
client_root: FileSystemPathVc,
ty: Value<ClientContextType>,
) -> FileSystemPathVc {
match ty.into_value() {
ClientContextType::Pages { .. }
| ClientContextType::App { .. }
| ClientContextType::Fallback => server_root.join("/_next/static/media"),
ClientContextType::Other => server_root.join("/_assets"),
| ClientContextType::Fallback => client_root.join("/_next/static/media"),
ClientContextType::Other => client_root.join("/_assets"),
}
}
......
......@@ -2,3 +2,10 @@ pub(crate) mod context;
pub(crate) mod runtime_entry;
pub(crate) mod transforms;
pub(crate) mod transition;
pub use context::{
get_client_compile_time_info, get_client_module_options_context,
get_client_resolve_options_context, get_client_runtime_entries, ClientContextType,
};
pub use runtime_entry::{RuntimeEntries, RuntimeEntriesVc, RuntimeEntry, RuntimeEntryVc};
pub use transition::NextClientTransition;
pub(crate) mod client_chunks_transition;
pub(crate) mod in_chunking_context_asset;
pub(crate) mod with_chunks;
pub use client_chunks_transition::NextClientChunksTransitionVc;
......@@ -678,7 +678,6 @@ pub async fn load_next_config_internal(
next_asset("entry/config/next.js"),
Value::new(ReferenceType::Entry(EntryReferenceSubType::Undefined)),
);
let config_value = evaluate(
load_next_config_asset,
project_path,
......
pub(crate) mod context;
pub(crate) mod resolve;
pub(crate) mod transforms;
pub use context::{
get_server_compile_time_info, get_server_module_options_context,
get_server_resolve_options_context, ServerContextType,
};
......@@ -54,48 +54,51 @@ pub struct PageLoaderAsset {
pub pathname: StringVc,
}
#[turbo_tasks::value_impl]
impl PageLoaderAssetVc {
#[turbo_tasks::function]
async fn get_loader_entry_asset(self) -> Result<AssetVc> {
let this = &*self.await?;
#[turbo_tasks::function]
pub async fn create_page_loader_entry_asset(
client_context: AssetContextVc,
entry_asset: AssetVc,
pathname: StringVc,
) -> Result<AssetVc> {
let mut result = RopeBuilder::default();
writeln!(
result,
"const PAGE_PATH = {};\n",
StringifyJs(&*pathname.await?)
)?;
let page_loader_path = next_js_file_path("entry/page-loader.ts");
let base_code = page_loader_path.read();
if let FileContent::Content(base_file) = &*base_code.await? {
result += base_file.content()
} else {
bail!("required file `entry/page-loader.ts` not found");
}
let mut result = RopeBuilder::default();
writeln!(
result,
"const PAGE_PATH = {};\n",
StringifyJs(&*this.pathname.await?)
)?;
let page_loader_path = next_js_file_path("entry/page-loader.ts");
let base_code = page_loader_path.read();
if let FileContent::Content(base_file) = &*base_code.await? {
result += base_file.content()
} else {
bail!("required file `entry/page-loader.ts` not found");
}
let file = File::from(result.build());
let file = File::from(result.build());
let virtual_asset = VirtualAssetVc::new(page_loader_path, file.into()).into();
Ok(VirtualAssetVc::new(page_loader_path, file.into()).into())
}
Ok(client_context.process(
virtual_asset,
Value::new(ReferenceType::Internal(
InnerAssetsVc::cell(indexmap! {
"PAGE".to_string() => client_context.process(entry_asset, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)))
})
)))
)
}
#[turbo_tasks::value_impl]
impl PageLoaderAssetVc {
#[turbo_tasks::function]
async fn get_page_chunks(self) -> Result<AssetsVc> {
let this = &*self.await?;
let loader_entry_asset = self.get_loader_entry_asset();
let module = this.client_context.process(
loader_entry_asset,
Value::new(ReferenceType::Internal(
InnerAssetsVc::cell(indexmap! {
"PAGE".to_string() => this.client_context.process(this.entry_asset, Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)))
})
)),
);
let page_loader_entry_asset =
create_page_loader_entry_asset(this.client_context, this.entry_asset, this.pathname);
let Some(module) = EvaluatableAssetVc::resolve_from(module).await? else {
let Some(module) = EvaluatableAssetVc::resolve_from(page_loader_entry_asset).await? else {
bail!("internal module must be evaluatable");
};
......
......@@ -107,6 +107,12 @@ pub struct PagesDirectoryStructure {
#[turbo_tasks::value_impl]
impl PagesDirectoryStructureVc {
/// Returns the router path of this directory.
#[turbo_tasks::function]
pub async fn next_router_path(self) -> Result<FileSystemPathVc> {
Ok(self.await?.next_router_path)
}
/// Returns the path to the directory of this structure in the project file
/// system.
#[turbo_tasks::function]
......
......@@ -38,7 +38,7 @@ use crate::{
get_client_asset_context, get_client_chunking_context,
get_client_resolve_options_context, ClientContextType,
},
runtime_entry::{RuntimeEntriesVc, RuntimeEntry},
RuntimeEntriesVc, RuntimeEntry,
},
next_config::NextConfigVc,
};
......
......@@ -210,6 +210,7 @@ export default async function build(
noMangling = false,
appDirOnly = false,
turboNextBuild = false,
turboNextBuildRoot = null,
buildMode: 'default' | 'experimental-compile' | 'experimental-generate'
): Promise<void> {
const isCompile = buildMode === 'experimental-compile'
......@@ -550,6 +551,23 @@ export default async function build(
app: appPageKeys.length > 0 ? appPageKeys : undefined,
}
if (turboNextBuild) {
// TODO(WEB-397) This is a temporary workaround to allow for filtering a
// subset of pages when building with --experimental-turbo, until we
// have complete support for all pages.
if (process.env.NEXT_TURBO_FILTER_PAGES) {
const filterPages = process.env.NEXT_TURBO_FILTER_PAGES.split(',')
pageKeys.pages = pageKeys.pages.filter((page) => {
return filterPages.some((filterPage) => {
return isMatch(page, filterPage)
})
})
}
// TODO(alexkirsz) Filter out app pages entirely as they are not supported yet.
pageKeys.app = undefined
}
const numConflictingAppPaths = conflictingAppPagePaths.length
if (mappedAppPages && numConflictingAppPaths > 0) {
Log.error(
......@@ -932,7 +950,23 @@ export default async function build(
async function turbopackBuild() {
const turboNextBuildStart = process.hrtime()
await binding.turbo.nextBuild(NextBuildContext)
const turboJson = findUp.sync('turbo.json', { cwd: dir })
// eslint-disable-next-line no-shadow
const packagePath = findUp.sync('package.json', { cwd: dir })
let root =
turboNextBuildRoot ??
(turboJson
? path.dirname(turboJson)
: packagePath
? path.dirname(packagePath)
: undefined)
await binding.turbo.nextBuild({
...NextBuildContext,
root,
})
const [duration] = process.hrtime(turboNextBuildStart)
return { duration, turbotraceContext: null }
}
......
......@@ -18,6 +18,7 @@ const nextBuild: CliCommand = (argv) => {
'--no-mangling': Boolean,
'--experimental-app-only': Boolean,
'--experimental-turbo': Boolean,
'--experimental-turbo-root': String,
'--build-mode': String,
// Aliases
'-h': '--help',
......@@ -82,6 +83,7 @@ const nextBuild: CliCommand = (argv) => {
args['--no-mangling'],
args['--experimental-app-only'],
args['--experimental-turbo'],
args['--experimental-turbo-root'],
args['--build-mode'] || 'default'
).catch((err) => {
console.error('')
......
......@@ -38,10 +38,18 @@
"outputs": ["dist/**"]
},
"typescript": {},
"rust-check": {},
"test-cargo-unit": {},
"test-cargo-integration": {},
"test-cargo-bench": {},
"rust-check": {
"inputs": [".cargo/**", "**/*.rs", "**/Cargo.toml"]
},
"test-cargo-unit": {
"inputs": [".cargo/**", "**/*.rs", "**/Cargo.toml"]
},
"test-cargo-integration": {
"inputs": [".cargo/**", "**/*.rs", "**/Cargo.toml"]
},
"test-cargo-bench": {
"inputs": [".cargo/**", "**/*.rs", "**/Cargo.toml"]
},
"//#get-test-timings": {
"inputs": ["run-tests.js"],
"outputs": ["test-timings.json"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment