Unverified Commit a40a3501 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Chunking Refactoring followup fixes (#56789)

### What?

fix some minor bugs
Make the page-loader reuse the client chunks

Closes WEB-1777
parent 04bad16b
This diff is collapsed.
......@@ -40,11 +40,11 @@ swc_core = { version = "0.83.28", features = [
testing = { version = "0.34.1" }
# Turbo crates
turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.2" }
turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.3" }
# [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros..
turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.2" }
turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.3" }
# [TODO]: need to refactor embed_directory! macro usage in next-core
turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.2" }
turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-231013.3" }
# General Deps
......
......@@ -563,11 +563,9 @@ impl PageEndpoint {
client_chunks.push(Vc::upcast(PageLoaderAsset::new(
this.pages_project.project().client_root(),
client_module_context,
Vc::upcast(client_chunking_context),
self.source(),
this.pathname,
self.client_relative_path(),
Vc::cell(client_chunks.clone()),
)));
Ok(Vc::cell(client_chunks))
......
......@@ -737,14 +737,6 @@ async fn insert_next_server_special_aliases(
},
),
);
import_map.insert_exact_alias(
"server-only",
request_to_import_mapping(app_dir, "next/dist/compiled/server-only"),
);
import_map.insert_exact_alias(
"client-only",
request_to_import_mapping(app_dir, "next/dist/compiled/client-only"),
);
import_map.insert_exact_alias(
"react",
request_to_import_mapping(
......@@ -841,9 +833,9 @@ async fn insert_next_server_special_aliases(
project_path,
indexmap! {
"server-only" => "next/dist/compiled/server-only/empty".to_string(),
"client-only" => "next/dist/compiled/client-only/error".to_string(),
"client-only" => "next/dist/compiled/client-only/index".to_string(),
"next/dist/compiled/server-only" => "next/dist/compiled/server-only/empty".to_string(),
"next/dist/compiled/client-only" => "next/dist/compiled/client-only/error".to_string(),
"next/dist/compiled/client-only" => "next/dist/compiled/client-only/index".to_string(),
},
);
......
use anyhow::Result;
use turbo_tasks::Vc;
use turbo_tasks::{Value, Vc};
use turbopack_binding::{
turbo::tasks_fs::{glob::Glob, FileJsonContent, FileSystemPath},
turbopack::core::{
resolve::{
find_context_file,
node::node_cjs_resolve_options,
node::{node_cjs_resolve_options, node_esm_resolve_options},
package_json,
parse::Request,
pattern::Pattern,
plugin::{ResolvePlugin, ResolvePluginCondition},
resolve, FindContextFileResult, ResolveResult, ResolveResultItem, ResolveResultOption,
},
......@@ -58,8 +59,17 @@ async fn is_node_resolveable(
context: Vc<FileSystemPath>,
request: Vc<Request>,
expected: Vc<FileSystemPath>,
is_esm: bool,
) -> Result<Vc<bool>> {
let node_resolve_result = resolve(context, request, node_cjs_resolve_options(context.root()));
let node_resolve_result = resolve(
context,
request,
if is_esm {
node_cjs_resolve_options(context.root())
} else {
node_esm_resolve_options(context.root())
},
);
let primary_node_assets = node_resolve_result.primary_sources().await?;
let Some(&node_asset) = primary_node_assets.first() else {
// can't resolve request with node.js options
......@@ -153,11 +163,34 @@ impl ResolvePlugin for ExternalCjsModulesResolvePlugin {
// check if we can resolve the package from the project dir with node.js resolve
// options (might be hidden by pnpm)
if *is_node_resolveable(self.project_path, request, fs_path).await? {
// mark as external
return Ok(ResolveResultOption::some(
ResolveResult::primary(ResolveResultItem::OriginalReferenceExternal).cell(),
));
if *is_node_resolveable(self.project_path, request, fs_path, false).await? {
// We don't know if this is a ESM reference, so also check if it is resolveable
// as ESM.
if *is_node_resolveable(self.project_path, request, fs_path, true).await? {
// mark as external
return Ok(ResolveResultOption::some(
ResolveResult::primary(ResolveResultItem::OriginalReferenceExternal).cell(),
));
} else if let Some(mut request_str) = request.await?.request() {
// When it's not resolveable as ESM, there is maybe an extension missing, try
// .js
if !request_str.ends_with(".js") {
request_str += ".js";
let request =
Request::parse(Value::new(Pattern::Constant(request_str.clone())));
if *is_node_resolveable(self.project_path, request, fs_path, false).await?
&& *is_node_resolveable(self.project_path, request, fs_path, true).await?
{
// mark as external, but with .js extension
return Ok(ResolveResultOption::some(
ResolveResult::primary(
ResolveResultItem::OriginalReferenceTypeExternal(request_str),
)
.cell(),
));
}
}
}
}
Ok(ResolveResultOption::none())
......
......@@ -9,7 +9,7 @@ use turbopack_binding::{
turbopack::{
core::{
asset::{Asset, AssetContent},
chunk::{ChunkData, ChunkingContext, ChunksData, EvaluatableAsset, EvaluatableAssets},
chunk::{ChunkData, ChunksData},
context::AssetContext,
ident::AssetIdent,
module::Module,
......@@ -19,38 +19,12 @@ use turbopack_binding::{
source::Source,
virtual_source::VirtualSource,
},
dev_server::source::{asset_graph::AssetGraphContentSource, ContentSource},
ecmascript::{chunk::EcmascriptChunkData, utils::StringifyJs},
},
};
use crate::{embed_js::next_js_file_path, util::get_asset_path_from_pathname};
#[turbo_tasks::function]
pub async fn create_page_loader(
server_root: Vc<FileSystemPath>,
client_context: Vc<Box<dyn AssetContext>>,
client_chunking_context: Vc<Box<dyn ChunkingContext>>,
entry_asset: Vc<Box<dyn Source>>,
pathname: Vc<String>,
rebase_prefix_path: Vc<FileSystemPathOption>,
) -> Result<Vc<Box<dyn ContentSource>>> {
let asset = PageLoaderAsset {
server_root,
client_context,
client_chunking_context,
entry_asset,
pathname,
rebase_prefix_path,
}
.cell();
Ok(Vc::upcast(AssetGraphContentSource::new_lazy(
server_root,
Vc::upcast(asset),
)))
}
#[turbo_tasks::function]
pub async fn create_page_loader_entry_module(
client_context: Vc<Box<dyn AssetContext>>,
......@@ -93,11 +67,9 @@ pub async fn create_page_loader_entry_module(
#[turbo_tasks::value(shared)]
pub struct PageLoaderAsset {
pub server_root: Vc<FileSystemPath>,
pub client_context: Vc<Box<dyn AssetContext>>,
pub client_chunking_context: Vc<Box<dyn ChunkingContext>>,
pub entry_asset: Vc<Box<dyn Source>>,
pub pathname: Vc<String>,
pub rebase_prefix_path: Vc<FileSystemPathOption>,
pub page_chunks: Vc<OutputAssets>,
}
#[turbo_tasks::value_impl]
......@@ -105,48 +77,26 @@ impl PageLoaderAsset {
#[turbo_tasks::function]
pub fn new(
server_root: Vc<FileSystemPath>,
client_context: Vc<Box<dyn AssetContext>>,
client_chunking_context: Vc<Box<dyn ChunkingContext>>,
entry_asset: Vc<Box<dyn Source>>,
pathname: Vc<String>,
rebase_prefix_path: Vc<FileSystemPathOption>,
page_chunks: Vc<OutputAssets>,
) -> Vc<Self> {
Self {
server_root,
client_context,
client_chunking_context,
entry_asset,
pathname,
rebase_prefix_path,
page_chunks,
}
.cell()
}
#[turbo_tasks::function]
async fn get_page_chunks(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let this = &*self.await?;
let page_loader_entry_asset =
create_page_loader_entry_module(this.client_context, this.entry_asset, this.pathname);
let Some(module) =
Vc::try_resolve_sidecast::<Box<dyn EvaluatableAsset>>(page_loader_entry_asset).await?
else {
bail!("internal module must be evaluatable");
};
Ok(this
.client_chunking_context
.evaluated_chunk_group(module.ident(), EvaluatableAssets::one(module)))
}
#[turbo_tasks::function]
async fn chunks_data(
self: Vc<Self>,
rebase_prefix_path: Vc<FileSystemPathOption>,
) -> Result<Vc<ChunksData>> {
let this = self.await?;
let mut chunks = self.get_page_chunks();
let mut chunks = this.page_chunks;
// If we are provided a prefix path, we need to rewrite our chunk paths to
// remove that prefix.
......@@ -187,7 +137,7 @@ impl OutputAsset for PageLoaderAsset {
#[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<OutputAssets>> {
let chunks = self.get_page_chunks().await?;
let chunks = self.await?.page_chunks.await?;
let mut references = Vec::with_capacity(chunks.len());
for &chunk in chunks.iter() {
......
......@@ -191,7 +191,7 @@
"@types/ws": "8.2.0",
"@vercel/ncc": "0.34.0",
"@vercel/nft": "0.22.6",
"@vercel/turbopack-ecmascript-runtime": "https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2",
"@vercel/turbopack-ecmascript-runtime": "https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3",
"acorn": "8.5.0",
"amphtml-validator": "1.0.35",
"anser": "1.4.9",
......
......@@ -179,18 +179,16 @@ export async function loadBindings(): Promise<Binding> {
return pendingBindings
}
if (process.platform === 'darwin') {
// rust needs stdout to be blocking, otherwise it will throw an error (on macOS at least) when writing a lot of data (logs) to it
// see https://github.com/napi-rs/napi-rs/issues/1630
// and https://github.com/nodejs/node/blob/main/doc/api/process.md#a-note-on-process-io
if (process.stdout._handle != null) {
// @ts-ignore
process.stdout._handle.setBlocking(true)
}
if (process.stderr._handle != null) {
// @ts-ignore
process.stderr._handle.setBlocking(true)
}
// rust needs stdout to be blocking, otherwise it will throw an error (on macOS at least) when writing a lot of data (logs) to it
// see https://github.com/napi-rs/napi-rs/issues/1630
// and https://github.com/nodejs/node/blob/main/doc/api/process.md#a-note-on-process-io
if (process.stdout._handle != null) {
// @ts-ignore
process.stdout._handle.setBlocking(true)
}
if (process.stderr._handle != null) {
// @ts-ignore
process.stderr._handle.setBlocking(true)
}
pendingBindings = new Promise(async (resolve, _reject) => {
......
......@@ -1061,8 +1061,8 @@ importers:
specifier: 0.22.6
version: 0.22.6
'@vercel/turbopack-ecmascript-runtime':
specifier: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2
version: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2(react-refresh@0.12.0)(webpack@5.86.0)'
specifier: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3
version: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3(react-refresh@0.12.0)(webpack@5.86.0)'
acorn:
specifier: 8.5.0
version: 8.5.0
......@@ -26805,9 +26805,9 @@ packages:
/zwitch@2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
'@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2(react-refresh@0.12.0)(webpack@5.86.0)':
resolution: {tarball: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2}
id: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.2'
'@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3(react-refresh@0.12.0)(webpack@5.86.0)':
resolution: {tarball: https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3}
id: '@gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-231013.3'
name: '@vercel/turbopack-ecmascript-runtime'
version: 0.0.0
dependencies:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment