Unverified Commit 322c0ee5 authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

add nft bundle to SsrChunk (#72305)

A small PR to start inserting nft on the critical path. This is only enabled in production builds and implements the bare minimum to get NFT outputs by inserting it into the server assets.

Followed up by another PR and inserts more hooks / flags elsewhere in the APIs.
parent e0ce7da7
...@@ -12,6 +12,7 @@ pub mod global_module_id_strategy; ...@@ -12,6 +12,7 @@ pub mod global_module_id_strategy;
mod instrumentation; mod instrumentation;
mod loadable_manifest; mod loadable_manifest;
mod middleware; mod middleware;
mod nft_json;
mod pages; mod pages;
pub mod paths; pub mod paths;
pub mod project; pub mod project;
......
use anyhow::{bail, Result};
use serde_json::json;
use turbo_tasks::{RcStr, ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{DiskFileSystem, File, FileSystem, FileSystemPath, VirtualFileSystem};
use turbopack_core::{
asset::{Asset, AssetContent},
ident::AssetIdent,
output::OutputAsset,
reference::all_assets_from_entries,
};
/// A json file that produces references to all files that are needed by the given module
/// at runtime. This will include, for example, node native modules, unanalyzable packages,
/// client side chunks, etc.
///
/// With this file, users can determine the minimum set of files that are needed alongside
/// their bundle.
#[turbo_tasks::value(shared)]
pub struct NftJsonAsset {
/// The chunk for which the asset is being generated
chunk: Vc<Box<dyn OutputAsset>>,
output_fs: Vc<DiskFileSystem>,
project_fs: Vc<DiskFileSystem>,
client_fs: Vc<Box<dyn FileSystem>>,
/// Additional assets to include in the nft json. This can be used to manually collect assets
/// that are known to be required but are not in the graph yet, for whatever reason.
///
/// An example of this is the two-phase approach used by the `ClientReferenceManifest` in
/// next.js.
additional_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
}
#[turbo_tasks::value_impl]
impl NftJsonAsset {
#[turbo_tasks::function]
pub fn new(
chunk: Vc<Box<dyn OutputAsset>>,
output_fs: Vc<DiskFileSystem>,
project_fs: Vc<DiskFileSystem>,
client_fs: Vc<Box<dyn FileSystem>>,
additional_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
) -> Vc<Self> {
NftJsonAsset {
chunk,
output_fs,
project_fs,
client_fs,
additional_assets,
}
.cell()
}
}
#[turbo_tasks::value(transparent)]
pub struct OutputSpecifier(Option<RcStr>);
#[turbo_tasks::value_impl]
impl NftJsonAsset {
#[turbo_tasks::function]
async fn ident_in_project_fs(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
let this = self.await?;
let project_fs = this.project_fs.await?;
let output_fs = this.output_fs.await?;
let nft_folder = self.ident().path().parent().await?;
if let Some(subdir) = output_fs.root.strip_prefix(&*project_fs.root) {
Ok(this
.project_fs
.root()
.join(subdir.into())
.join(nft_folder.path.clone()))
} else {
// TODO: what are the implications of this?
bail!("output fs not inside project fs");
}
}
#[turbo_tasks::function]
async fn ident_in_client_fs(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
Ok(self
.await?
.client_fs
.root()
.join(self.ident().path().parent().await?.path.clone()))
}
#[turbo_tasks::function]
async fn get_output_specifier(
self: Vc<Self>,
path: Vc<FileSystemPath>,
) -> Result<Vc<OutputSpecifier>> {
let this = self.await?;
let path_fs = path.fs().resolve().await?;
let path_ref = path.await?;
let nft_folder = self.ident().path().parent().await?;
if path_fs == Vc::upcast(this.output_fs.resolve().await?) {
// e.g. a referenced chunk
return Ok(Vc::cell(Some(
nft_folder.get_relative_path_to(&path_ref).unwrap(),
)));
} else if path_fs == Vc::upcast(this.project_fs.resolve().await?) {
return Ok(Vc::cell(Some(
self.ident_in_project_fs()
.await?
.get_relative_path_to(&path_ref)
.unwrap(),
)));
} else if path_fs == Vc::upcast(this.client_fs.resolve().await?) {
return Ok(Vc::cell(Some(
self.ident_in_client_fs()
.await?
.get_relative_path_to(&path_ref)
.unwrap()
.replace("/_next/", "/.next/")
.into(),
)));
}
if let Some(path_fs) = Vc::try_resolve_downcast_type::<VirtualFileSystem>(path_fs).await? {
if path_fs.await?.name == "externals" || path_fs.await?.name == "traced" {
return Ok(Vc::cell(Some(
self.ident_in_project_fs()
.await?
.get_relative_path_to(
&*this.project_fs.root().join(path_ref.path.clone()).await?,
)
.unwrap(),
)));
}
}
println!("Unknown filesystem for {}", path.to_string().await?);
Ok(Vc::cell(None))
}
}
#[turbo_tasks::value_impl]
impl OutputAsset for NftJsonAsset {
#[turbo_tasks::function]
async fn ident(&self) -> Result<Vc<AssetIdent>> {
let path = self.chunk.ident().path().await?;
Ok(AssetIdent::from_path(
path.fs
.root()
.join(format!("{}.nft.json", path.path).into()),
))
}
}
#[turbo_tasks::value_impl]
impl Asset for NftJsonAsset {
#[turbo_tasks::function]
async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
let this = &*self.await?;
let mut result = Vec::new();
let chunk = this.chunk.to_resolved().await?;
let entries = this
.additional_assets
.iter()
.copied()
.chain(std::iter::once(chunk))
.collect();
for referenced_chunk in all_assets_from_entries(Vc::cell(entries)).await? {
if referenced_chunk.ident().path().await?.extension_ref() == Some("map") {
continue;
}
if chunk == referenced_chunk.to_resolved().await? {
continue;
}
let specifier = self
.get_output_specifier(referenced_chunk.ident().path())
.await?;
if let Some(specifier) = &*specifier {
result.push(specifier.clone());
}
}
result.sort();
result.dedup();
let json = json!({
"version": 1,
"files": result
});
Ok(AssetContent::file(File::from(json.to_string()).into()))
}
}
...@@ -50,7 +50,7 @@ use turbopack_core::{ ...@@ -50,7 +50,7 @@ use turbopack_core::{
file_source::FileSource, file_source::FileSource,
ident::AssetIdent, ident::AssetIdent,
module::{Module, Modules}, module::{Module, Modules},
output::{OutputAsset, OutputAssets}, output::{OptionOutputAsset, OutputAsset, OutputAssets},
reference_type::{EcmaScriptModulesReferenceSubType, EntryReferenceSubType, ReferenceType}, reference_type::{EcmaScriptModulesReferenceSubType, EntryReferenceSubType, ReferenceType},
resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern}, resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern},
source::Source, source::Source,
...@@ -66,6 +66,7 @@ use crate::{ ...@@ -66,6 +66,7 @@ use crate::{
}, },
font::create_font_manifest, font::create_font_manifest,
loadable_manifest::create_react_loadable_manifest, loadable_manifest::create_react_loadable_manifest,
nft_json::NftJsonAsset,
paths::{ paths::{
all_paths_in_root, all_server_paths, get_js_paths_from_root, get_paths_from_root, all_paths_in_root, all_server_paths, get_js_paths_from_root, get_paths_from_root,
get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings, get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings,
...@@ -879,9 +880,32 @@ impl PageEndpoint { ...@@ -879,9 +880,32 @@ impl PageEndpoint {
) )
.await?; .await?;
let nft = if this
.pages_project
.project()
.next_mode()
.await?
.is_production()
{
ResolvedVc::cell(Some(ResolvedVc::upcast(
NftJsonAsset::new(
*ssr_entry_chunk,
this.pages_project.project().output_fs(),
this.pages_project.project().project_fs(),
this.pages_project.project().client_fs(),
vec![],
)
.to_resolved()
.await?,
)))
} else {
ResolvedVc::cell(None)
};
Ok(SsrChunk::NodeJs { Ok(SsrChunk::NodeJs {
entry: ssr_entry_chunk, entry: ssr_entry_chunk,
dynamic_import_entries, dynamic_import_entries,
nft,
} }
.cell()) .cell())
} }
...@@ -1097,10 +1121,14 @@ impl PageEndpoint { ...@@ -1097,10 +1121,14 @@ impl PageEndpoint {
SsrChunk::NodeJs { SsrChunk::NodeJs {
entry, entry,
dynamic_import_entries, dynamic_import_entries,
nft,
} => { } => {
let pages_manifest = self.pages_manifest(*entry).to_resolved().await?; let pages_manifest = self.pages_manifest(*entry).to_resolved().await?;
server_assets.push(pages_manifest); server_assets.push(pages_manifest);
server_assets.push(entry); server_assets.push(entry);
if let Some(nft) = &*nft.await? {
server_assets.push(*nft);
}
let loadable_manifest_output = self.react_loadable_manifest(dynamic_import_entries); let loadable_manifest_output = self.react_loadable_manifest(dynamic_import_entries);
server_assets.extend(loadable_manifest_output.await?.iter().copied()); server_assets.extend(loadable_manifest_output.await?.iter().copied());
...@@ -1373,6 +1401,7 @@ pub enum SsrChunk { ...@@ -1373,6 +1401,7 @@ pub enum SsrChunk {
NodeJs { NodeJs {
entry: ResolvedVc<Box<dyn OutputAsset>>, entry: ResolvedVc<Box<dyn OutputAsset>>,
dynamic_import_entries: Vc<DynamicImportedChunks>, dynamic_import_entries: Vc<DynamicImportedChunks>,
nft: ResolvedVc<OptionOutputAsset>,
}, },
Edge { Edge {
files: Vc<OutputAssets>, files: Vc<OutputAssets>,
......
...@@ -539,7 +539,7 @@ impl Project { ...@@ -539,7 +539,7 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn project_fs(&self) -> Vc<DiskFileSystem> { pub fn project_fs(&self) -> Vc<DiskFileSystem> {
DiskFileSystem::new( DiskFileSystem::new(
PROJECT_FILESYSTEM_NAME.into(), PROJECT_FILESYSTEM_NAME.into(),
self.root_path.clone(), self.root_path.clone(),
...@@ -548,7 +548,7 @@ impl Project { ...@@ -548,7 +548,7 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn client_fs(self: Vc<Self>) -> Vc<Box<dyn FileSystem>> { pub fn client_fs(self: Vc<Self>) -> Vc<Box<dyn FileSystem>> {
let virtual_fs = VirtualFileSystem::new(); let virtual_fs = VirtualFileSystem::new();
Vc::upcast(virtual_fs) Vc::upcast(virtual_fs)
} }
......
...@@ -5,7 +5,7 @@ use super::{DirectoryContent, FileContent, FileMeta, FileSystem, FileSystemPath, ...@@ -5,7 +5,7 @@ use super::{DirectoryContent, FileContent, FileMeta, FileSystem, FileSystemPath,
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct VirtualFileSystem { pub struct VirtualFileSystem {
name: RcStr, pub name: RcStr,
} }
impl VirtualFileSystem { impl VirtualFileSystem {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment