Unverified Commit a40a3501 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Chunking Refactoring followup fixes (#56789)

### What?

fix some minor bugs
Make the page-loader reuse the client chunks

Closes WEB-1777
parent 04bad16b
This diff is collapsed.
...@@ -40,11 +40,11 @@ swc_core = { version = "0.83.28", features = [ ...@@ -40,11 +40,11 @@ swc_core = { version = "0.83.28", features = [
testing = { version = "0.34.1" } testing = { version = "0.34.1" }
# Turbo crates # Turbo crates
turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.2" } turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.3" }
# [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros.. # [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros..
turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.2" } turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.3" }
# [TODO]: need to refactor embed_directory! macro usage in next-core # [TODO]: need to refactor embed_directory! macro usage in next-core
turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.2" } turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.3" }
# General Deps # General Deps
......
...@@ -563,11 +563,9 @@ impl PageEndpoint { ...@@ -563,11 +563,9 @@ impl PageEndpoint {
client_chunks.push(Vc::upcast(PageLoaderAsset::new( client_chunks.push(Vc::upcast(PageLoaderAsset::new(
this.pages_project.project().client_root(), this.pages_project.project().client_root(),
client_module_context,
Vc::upcast(client_chunking_context),
self.source(),
this.pathname, this.pathname,
self.client_relative_path(), self.client_relative_path(),
Vc::cell(client_chunks.clone()),
))); )));
Ok(Vc::cell(client_chunks)) Ok(Vc::cell(client_chunks))
......
...@@ -737,14 +737,6 @@ async fn insert_next_server_special_aliases( ...@@ -737,14 +737,6 @@ async fn insert_next_server_special_aliases(
}, },
), ),
); );
import_map.insert_exact_alias(
"server-only",
request_to_import_mapping(app_dir, "next/dist/compiled/server-only"),
);
import_map.insert_exact_alias(
"client-only",
request_to_import_mapping(app_dir, "next/dist/compiled/client-only"),
);
import_map.insert_exact_alias( import_map.insert_exact_alias(
"react", "react",
request_to_import_mapping( request_to_import_mapping(
...@@ -841,9 +833,9 @@ async fn insert_next_server_special_aliases( ...@@ -841,9 +833,9 @@ async fn insert_next_server_special_aliases(
project_path, project_path,
indexmap! { indexmap! {
"server-only" => "next/dist/compiled/server-only/empty".to_string(), "server-only" => "next/dist/compiled/server-only/empty".to_string(),
"client-only" => "next/dist/compiled/client-only/error".to_string(), "client-only" => "next/dist/compiled/client-only/index".to_string(),
"next/dist/compiled/server-only" => "next/dist/compiled/server-only/empty".to_string(), "next/dist/compiled/server-only" => "next/dist/compiled/server-only/empty".to_string(),
"next/dist/compiled/client-only" => "next/dist/compiled/client-only/error".to_string(), "next/dist/compiled/client-only" => "next/dist/compiled/client-only/index".to_string(),
}, },
); );
......
use anyhow::Result; use anyhow::Result;
use turbo_tasks::Vc; use turbo_tasks::{Value, Vc};
use turbopack_binding::{ use turbopack_binding::{
turbo::tasks_fs::{glob::Glob, FileJsonContent, FileSystemPath}, turbo::tasks_fs::{glob::Glob, FileJsonContent, FileSystemPath},
turbopack::core::{ turbopack::core::{
resolve::{ resolve::{
find_context_file, find_context_file,
node::node_cjs_resolve_options, node::{node_cjs_resolve_options, node_esm_resolve_options},
package_json, package_json,
parse::Request, parse::Request,
pattern::Pattern,
plugin::{ResolvePlugin, ResolvePluginCondition}, plugin::{ResolvePlugin, ResolvePluginCondition},
resolve, FindContextFileResult, ResolveResult, ResolveResultItem, ResolveResultOption, resolve, FindContextFileResult, ResolveResult, ResolveResultItem, ResolveResultOption,
}, },
...@@ -58,8 +59,17 @@ async fn is_node_resolveable( ...@@ -58,8 +59,17 @@ async fn is_node_resolveable(
context: Vc<FileSystemPath>, context: Vc<FileSystemPath>,
request: Vc<Request>, request: Vc<Request>,
expected: Vc<FileSystemPath>, expected: Vc<FileSystemPath>,
is_esm: bool,
) -> Result<Vc<bool>> { ) -> Result<Vc<bool>> {
let node_resolve_result = resolve(context, request, node_cjs_resolve_options(context.root())); let node_resolve_result = resolve(
context,
request,
if is_esm {
node_cjs_resolve_options(context.root())
} else {
node_esm_resolve_options(context.root())
},
);
let primary_node_assets = node_resolve_result.primary_sources().await?; let primary_node_assets = node_resolve_result.primary_sources().await?;
let Some(&node_asset) = primary_node_assets.first() else { let Some(&node_asset) = primary_node_assets.first() else {
// can't resolve request with node.js options // can't resolve request with node.js options
...@@ -153,11 +163,34 @@ impl ResolvePlugin for ExternalCjsModulesResolvePlugin { ...@@ -153,11 +163,34 @@ impl ResolvePlugin for ExternalCjsModulesResolvePlugin {
// check if we can resolve the package from the project dir with node.js resolve // check if we can resolve the package from the project dir with node.js resolve
// options (might be hidden by pnpm) // options (might be hidden by pnpm)
if *is_node_resolveable(self.project_path, request, fs_path).await? { if *is_node_resolveable(self.project_path, request, fs_path, false).await? {
// mark as external // We don't know if this is a ESM reference, so also check if it is resolveable
return Ok(ResolveResultOption::some( // as ESM.
ResolveResult::primary(ResolveResultItem::OriginalReferenceExternal).cell(), if *is_node_resolveable(self.project_path, request, fs_path, true).await? {
)); // mark as external
return Ok(ResolveResultOption::some(
ResolveResult::primary(ResolveResultItem::OriginalReferenceExternal).cell(),
));
} else if let Some(mut request_str) = request.await?.request() {
// When it's not resolveable as ESM, there is maybe an extension missing, try
// .js
if !request_str.ends_with(".js") {
request_str += ".js";
let request =
Request::parse(Value::new(Pattern::Constant(request_str.clone())));
if *is_node_resolveable(self.project_path, request, fs_path, false).await?
&& *is_node_resolveable(self.project_path, request, fs_path, true).await?
{
// mark as external, but with .js extension
return Ok(ResolveResultOption::some(
ResolveResult::primary(
ResolveResultItem::OriginalReferenceTypeExternal(request_str),
)
.cell(),
));
}
}
}
} }
Ok(ResolveResultOption::none()) Ok(ResolveResultOption::none())
......
...@@ -9,7 +9,7 @@ use turbopack_binding::{ ...@@ -9,7 +9,7 @@ use turbopack_binding::{
turbopack::{ turbopack::{
core::{ core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{ChunkData, ChunkingContext, ChunksData, EvaluatableAsset, EvaluatableAssets}, chunk::{ChunkData, ChunksData},
context::AssetContext, context::AssetContext,
ident::AssetIdent, ident::AssetIdent,
module::Module, module::Module,
...@@ -19,38 +19,12 @@ use turbopack_binding::{ ...@@ -19,38 +19,12 @@ use turbopack_binding::{
source::Source, source::Source,
virtual_source::VirtualSource, virtual_source::VirtualSource,
}, },
dev_server::source::{asset_graph::AssetGraphContentSource, ContentSource},
ecmascript::{chunk::EcmascriptChunkData, utils::StringifyJs}, ecmascript::{chunk::EcmascriptChunkData, utils::StringifyJs},
}, },
}; };
use crate::{embed_js::next_js_file_path, util::get_asset_path_from_pathname}; use crate::{embed_js::next_js_file_path, util::get_asset_path_from_pathname};
#[turbo_tasks::function]
pub async fn create_page_loader(
server_root: Vc<FileSystemPath>,
client_context: Vc<Box<dyn AssetContext>>,
client_chunking_context: Vc<Box<dyn ChunkingContext>>,
entry_asset: Vc<Box<dyn Source>>,
pathname: Vc<String>,
rebase_prefix_path: Vc<FileSystemPathOption>,
) -> Result<Vc<Box<dyn ContentSource>>> {
let asset = PageLoaderAsset {
server_root,
client_context,
client_chunking_context,
entry_asset,
pathname,
rebase_prefix_path,
}
.cell();
Ok(Vc::upcast(AssetGraphContentSource::new_lazy(
server_root,
Vc::upcast(asset),
)))
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn create_page_loader_entry_module( pub async fn create_page_loader_entry_module(
client_context: Vc<Box<dyn AssetContext>>, client_context: Vc<Box<dyn AssetContext>>,
...@@ -93,11 +67,9 @@ pub async fn create_page_loader_entry_module( ...@@ -93,11 +67,9 @@ pub async fn create_page_loader_entry_module(
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct PageLoaderAsset { pub struct PageLoaderAsset {
pub server_root: Vc<FileSystemPath>, pub server_root: Vc<FileSystemPath>,
pub client_context: Vc<Box<dyn AssetContext>>,
pub client_chunking_context: Vc<Box<dyn ChunkingContext>>,
pub entry_asset: Vc<Box<dyn Source>>,
pub pathname: Vc<String>, pub pathname: Vc<String>,
pub rebase_prefix_path: Vc<FileSystemPathOption>, pub rebase_prefix_path: Vc<FileSystemPathOption>,
pub page_chunks: Vc<OutputAssets>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -105,48 +77,26 @@ impl PageLoaderAsset { ...@@ -105,48 +77,26 @@ impl PageLoaderAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new( pub fn new(
server_root: Vc<FileSystemPath>, server_root: Vc<FileSystemPath>,
client_context: Vc<Box<dyn AssetContext>>,
client_chunking_context: Vc<Box<dyn ChunkingContext>>,
entry_asset: Vc<Box<dyn Source>>,
pathname: Vc<String>, pathname: Vc<String>,
rebase_prefix_path: Vc<FileSystemPathOption>, rebase_prefix_path: Vc<FileSystemPathOption>,
page_chunks: Vc<OutputAssets>,
) -> Vc<Self> { ) -> Vc<Self> {
Self { Self {
server_root, server_root,
client_context,
client_chunking_context,
entry_asset,
pathname, pathname,
rebase_prefix_path, rebase_prefix_path,
page_chunks,
} }
.cell() .cell()
} }
#[turbo_tasks::function]
async fn get_page_chunks(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = &*self.await?;
let page_loader_entry_asset =
create_page_loader_entry_module(this.client_context, this.entry_asset, this.pathname);
let Some(module) =
Vc::try_resolve_sidecast::<Box<dyn EvaluatableAsset>>(page_loader_entry_asset).await?
else {
bail!("internal module must be evaluatable");
};
Ok(this
.client_chunking_context
.evaluated_chunk_group(module.ident(), EvaluatableAssets::one(module)))
}
#[turbo_tasks::function] #[turbo_tasks::function]
async fn chunks_data( async fn chunks_data(
self: Vc<Self>, self: Vc<Self>,
rebase_prefix_path: Vc<FileSystemPathOption>, rebase_prefix_path: Vc<FileSystemPathOption>,
) -> Result<Vc<ChunksData>> { ) -> Result<Vc<ChunksData>> {
let this = self.await?; let this = self.await?;
let mut chunks = self.get_page_chunks(); let mut chunks = this.page_chunks;
// If we are provided a prefix path, we need to rewrite our chunk paths to // If we are provided a prefix path, we need to rewrite our chunk paths to
// remove that prefix. // remove that prefix.
...@@ -187,7 +137,7 @@ impl OutputAsset for PageLoaderAsset { ...@@ -187,7 +137,7 @@ impl OutputAsset for PageLoaderAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> { async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let chunks = self.get_page_chunks().await?; let chunks = self.await?.page_chunks.await?;
let mut references = Vec::with_capacity(chunks.len()); let mut references = Vec::with_capacity(chunks.len());
for &chunk in chunks.iter() { for &chunk in chunks.iter() {
......
...@@ -191,7 +191,7 @@ ...@@ -191,7 +191,7 @@
"@types/ws": "8.2.0", "@types/ws": "8.2.0",
"@vercel/ncc": "0.34.0", "@vercel/ncc": "0.34.0",
"@vercel/nft": "0.22.6", "@vercel/nft": "0.22.6",
"@vercel/turbopack-ecmascript-runtime": "https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2", "@vercel/turbopack-ecmascript-runtime": "https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3",
"acorn": "8.5.0", "acorn": "8.5.0",
"amphtml-validator": "1.0.35", "amphtml-validator": "1.0.35",
"anser": "1.4.9", "anser": "1.4.9",
......
...@@ -179,18 +179,16 @@ export async function loadBindings(): Promise<Binding> { ...@@ -179,18 +179,16 @@ export async function loadBindings(): Promise<Binding> {
return pendingBindings return pendingBindings
} }
if (process.platform === 'darwin') { // rust needs stdout to be blocking, otherwise it will throw an error (on macOS at least) when writing a lot of data (logs) to it
// rust needs stdout to be blocking, otherwise it will throw an error (on macOS at least) when writing a lot of data (logs) to it // see https://github.com/napi-rs/napi-rs/issues/1630
// see https://github.com/napi-rs/napi-rs/issues/1630 // and https://github.com/nodejs/node/blob/main/doc/api/process.md#a-note-on-process-io
// and https://github.com/nodejs/node/blob/main/doc/api/process.md#a-note-on-process-io if (process.stdout._handle != null) {
if (process.stdout._handle != null) { // @ts-ignore
// @ts-ignore process.stdout._handle.setBlocking(true)
process.stdout._handle.setBlocking(true) }
} if (process.stderr._handle != null) {
if (process.stderr._handle != null) { // @ts-ignore
// @ts-ignore process.stderr._handle.setBlocking(true)
process.stderr._handle.setBlocking(true)
}
} }
pendingBindings = new Promise(async (resolve, _reject) => { pendingBindings = new Promise(async (resolve, _reject) => {
......
...@@ -1061,8 +1061,8 @@ importers: ...@@ -1061,8 +1061,8 @@ importers:
specifier: 0.22.6 specifier: 0.22.6
version: 0.22.6 version: 0.22.6
'@vercel/turbopack-ecmascript-runtime': '@vercel/turbopack-ecmascript-runtime':
specifier: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2 specifier: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3
version: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2(react-refresh@0.12.0)(webpack@5.86.0)' version: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3(react-refresh@0.12.0)(webpack@5.86.0)'
acorn: acorn:
specifier: 8.5.0 specifier: 8.5.0
version: 8.5.0 version: 8.5.0
...@@ -26805,9 +26805,9 @@ packages: ...@@ -26805,9 +26805,9 @@ packages:
/zwitch@2.0.4: /zwitch@2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
'@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2(react-refresh@0.12.0)(webpack@5.86.0)': '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3(react-refresh@0.12.0)(webpack@5.86.0)':
resolution: {tarball: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2} resolution: {tarball: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3}
id: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2' id: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3'
name: '@vercel/turbopack-ecmascript-runtime' name: '@vercel/turbopack-ecmascript-runtime'
version: 0.0.0 version: 0.0.0
dependencies: dependencies:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment