Unverified Commit c8ba12f2 authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

Turbopack: NFT followups (#72517)

Closes PACK-3421

Address PR feedback from https://github.com/vercel/next.js/pull/72305

When working on this, I uncovered another bug which will be handled in a
different PR.

Changes:
- Don't rely on `DiskFileSystem` but only via file paths and
`is_inside()`
- Put the `RebasedAsset` for externals on the `[project]/` fs (this way,
they aren't emitted either), i.e. don't perform any rebasing at all and
just reference the files where they are

---------
Co-authored-by: 's avatarNiklas Mischkulnig <4586894+mischnic@users.noreply.github.com>
parent bc492870
...@@ -1400,10 +1400,8 @@ impl AppEndpoint { ...@@ -1400,10 +1400,8 @@ impl AppEndpoint {
{ {
server_assets.insert(ResolvedVc::upcast( server_assets.insert(ResolvedVc::upcast(
NftJsonAsset::new( NftJsonAsset::new(
this.app_project.project(),
*rsc_chunk, *rsc_chunk,
this.app_project.project().output_fs(),
this.app_project.project().project_fs(),
this.app_project.project().client_fs(),
client_reference_manifest.iter().map(|m| **m).collect(), client_reference_manifest.iter().map(|m| **m).collect(),
) )
.to_resolved() .to_resolved()
......
...@@ -225,15 +225,9 @@ impl InstrumentationEndpoint { ...@@ -225,15 +225,9 @@ impl InstrumentationEndpoint {
let mut output_assets = vec![chunk]; let mut output_assets = vec![chunk];
if this.project.next_mode().await?.is_production() { if this.project.next_mode().await?.is_production() {
output_assets.push(ResolvedVc::upcast( output_assets.push(ResolvedVc::upcast(
NftJsonAsset::new( NftJsonAsset::new(this.project, *chunk, vec![])
*chunk, .to_resolved()
this.project.output_fs(), .await?,
this.project.project_fs(),
this.project.client_fs(),
vec![],
)
.to_resolved()
.await?,
)); ));
} }
Ok(Vc::cell(output_assets)) Ok(Vc::cell(output_assets))
......
use std::collections::BTreeSet;
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use serde_json::json; use serde_json::json;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::{DiskFileSystem, File, FileSystem, FileSystemPath, VirtualFileSystem}; use turbo_tasks_fs::{File, FileSystem, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
ident::AssetIdent, ident::AssetIdent,
...@@ -10,6 +12,8 @@ use turbopack_core::{ ...@@ -10,6 +12,8 @@ use turbopack_core::{
reference::all_assets_from_entries, reference::all_assets_from_entries,
}; };
use crate::project::Project;
/// A json file that produces references to all files that are needed by the given module /// A json file that produces references to all files that are needed by the given module
/// at runtime. This will include, for example, node native modules, unanalyzable packages, /// at runtime. This will include, for example, node native modules, unanalyzable packages,
/// client side chunks, etc. /// client side chunks, etc.
...@@ -18,11 +22,9 @@ use turbopack_core::{ ...@@ -18,11 +22,9 @@ use turbopack_core::{
/// their bundle. /// their bundle.
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct NftJsonAsset { pub struct NftJsonAsset {
project: ResolvedVc<Project>,
/// The chunk for which the asset is being generated /// The chunk for which the asset is being generated
chunk: Vc<Box<dyn OutputAsset>>, chunk: ResolvedVc<Box<dyn OutputAsset>>,
output_fs: Vc<DiskFileSystem>,
project_fs: Vc<DiskFileSystem>,
client_fs: Vc<Box<dyn FileSystem>>,
/// Additional assets to include in the nft json. This can be used to manually collect assets /// Additional assets to include in the nft json. This can be used to manually collect assets
/// that are known to be required but are not in the graph yet, for whatever reason. /// that are known to be required but are not in the graph yet, for whatever reason.
/// ///
...@@ -35,21 +37,44 @@ pub struct NftJsonAsset { ...@@ -35,21 +37,44 @@ pub struct NftJsonAsset {
impl NftJsonAsset { impl NftJsonAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new( pub fn new(
chunk: Vc<Box<dyn OutputAsset>>, project: ResolvedVc<Project>,
output_fs: Vc<DiskFileSystem>, chunk: ResolvedVc<Box<dyn OutputAsset>>,
project_fs: Vc<DiskFileSystem>,
client_fs: Vc<Box<dyn FileSystem>>,
additional_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>, additional_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
) -> Vc<Self> { ) -> Vc<Self> {
NftJsonAsset { NftJsonAsset {
chunk, chunk,
output_fs, project,
project_fs,
client_fs,
additional_assets, additional_assets,
} }
.cell() .cell()
} }
#[turbo_tasks::function]
fn output_root(&self) -> Vc<FileSystemPath> {
self.project.output_fs().root()
}
#[turbo_tasks::function]
fn project_root(&self) -> Vc<FileSystemPath> {
self.project.project_fs().root()
}
#[turbo_tasks::function]
fn client_root(&self) -> Vc<FileSystemPath> {
self.project.client_fs().root()
}
#[turbo_tasks::function]
fn project_path(&self) -> Vc<FileSystemPath> {
self.project.project_path()
}
#[turbo_tasks::function]
async fn dist_dir(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("/{}/", self.project.dist_dir().await?).into(),
))
}
} }
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
...@@ -58,31 +83,22 @@ pub struct OutputSpecifier(Option<RcStr>); ...@@ -58,31 +83,22 @@ pub struct OutputSpecifier(Option<RcStr>);
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl NftJsonAsset { impl NftJsonAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn ident_in_project_fs(self: Vc<Self>) -> Result<Vc<FileSystemPath>> { async fn ident_folder(self: Vc<Self>) -> Vc<FileSystemPath> {
let this = self.await?; self.ident().path().parent()
let project_fs = this.project_fs.await?;
let output_fs = this.output_fs.await?;
let nft_folder = self.ident().path().parent().await?;
if let Some(subdir) = output_fs.root().strip_prefix(&**project_fs.root()) {
Ok(this
.project_fs
.root()
.join(subdir.into())
.join(nft_folder.path.clone()))
} else {
// TODO: what are the implications of this?
bail!("output fs not inside project fs");
}
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn ident_in_client_fs(self: Vc<Self>) -> Result<Vc<FileSystemPath>> { async fn ident_folder_in_project_fs(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
Ok(self Ok(self
.await? .project_path()
.client_fs .join(self.ident_folder().await?.path.clone()))
.root() }
.join(self.ident().path().parent().await?.path.clone()))
#[turbo_tasks::function]
async fn ident_folder_in_client_fs(self: Vc<Self>) -> Result<Vc<FileSystemPath>> {
Ok(self
.client_root()
.join(self.ident_folder().await?.path.clone()))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -90,49 +106,45 @@ impl NftJsonAsset { ...@@ -90,49 +106,45 @@ impl NftJsonAsset {
self: Vc<Self>, self: Vc<Self>,
path: Vc<FileSystemPath>, path: Vc<FileSystemPath>,
) -> Result<Vc<OutputSpecifier>> { ) -> Result<Vc<OutputSpecifier>> {
let this = self.await?;
let path_fs = path.fs().resolve().await?;
let path_ref = path.await?; let path_ref = path.await?;
let nft_folder = self.ident().path().parent().await?;
if path_fs == Vc::upcast(this.output_fs.resolve().await?) { // include assets in the outputs such as referenced chunks
// e.g. a referenced chunk if path_ref.is_inside_ref(&*(self.output_root().await?)) {
return Ok(Vc::cell(Some( return Ok(Vc::cell(Some(
nft_folder.get_relative_path_to(&path_ref).unwrap(), self.ident_folder()
.await?
.get_relative_path_to(&path_ref)
.unwrap(),
))); )));
} else if path_fs == Vc::upcast(this.project_fs.resolve().await?) { }
// include assets in the project root such as images and traced references (externals)
if path_ref.is_inside_ref(&*(self.project_root().await?)) {
return Ok(Vc::cell(Some( return Ok(Vc::cell(Some(
self.ident_in_project_fs() self.ident_folder_in_project_fs()
.await? .await?
.get_relative_path_to(&path_ref) .get_relative_path_to(&path_ref)
.unwrap(), .unwrap(),
))); )));
} else if path_fs == Vc::upcast(this.client_fs.resolve().await?) { }
// assets that are needed on the client side such as fonts and icons
if path_ref.is_inside_ref(&*(self.client_root().await?)) {
return Ok(Vc::cell(Some( return Ok(Vc::cell(Some(
self.ident_in_client_fs() self.ident_folder_in_client_fs()
.await? .await?
.get_relative_path_to(&path_ref) .get_relative_path_to(&path_ref)
.unwrap() .unwrap()
.replace("/_next/", "/.next/") .replace("/_next/", &self.dist_dir().await?)
.into(), .into(),
))); )));
} }
if let Some(path_fs) = Vc::try_resolve_downcast_type::<VirtualFileSystem>(path_fs).await? { // Make this an error for now, this should effectively be unreachable
if path_fs.await?.name == "externals" || path_fs.await?.name == "traced" { bail!(
return Ok(Vc::cell(Some( "NftJsonAsset: cannot handle filepath {}",
self.ident_in_project_fs() path.to_string().await?
.await? );
.get_relative_path_to(
&*this.project_fs.root().join(path_ref.path.clone()).await?,
)
.unwrap(),
)));
}
}
println!("Unknown filesystem for {}", path.to_string().await?);
Ok(Vc::cell(None))
} }
} }
...@@ -154,7 +166,7 @@ impl Asset for NftJsonAsset { ...@@ -154,7 +166,7 @@ impl Asset for NftJsonAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> { async fn content(self: Vc<Self>) -> Result<Vc<AssetContent>> {
let this = &*self.await?; let this = &*self.await?;
let mut result = Vec::new(); let mut result = BTreeSet::new();
let chunk = this.chunk.to_resolved().await?; let chunk = this.chunk.to_resolved().await?;
let entries = this let entries = this
...@@ -176,12 +188,10 @@ impl Asset for NftJsonAsset { ...@@ -176,12 +188,10 @@ impl Asset for NftJsonAsset {
.get_output_specifier(referenced_chunk.ident().path()) .get_output_specifier(referenced_chunk.ident().path())
.await?; .await?;
if let Some(specifier) = &*specifier { if let Some(specifier) = &*specifier {
result.push(specifier.clone()); result.insert(specifier.clone());
} }
} }
result.sort();
result.dedup();
let json = json!({ let json = json!({
"version": 1, "version": 1,
"files": result "files": result
......
...@@ -882,7 +882,7 @@ impl PageEndpoint { ...@@ -882,7 +882,7 @@ impl PageEndpoint {
) )
.await?; .await?;
let nft = if this let server_asset_trace_file = if this
.pages_project .pages_project
.project() .project()
.next_mode() .next_mode()
...@@ -890,15 +890,9 @@ impl PageEndpoint { ...@@ -890,15 +890,9 @@ impl PageEndpoint {
.is_production() .is_production()
{ {
ResolvedVc::cell(Some(ResolvedVc::upcast( ResolvedVc::cell(Some(ResolvedVc::upcast(
NftJsonAsset::new( NftJsonAsset::new(this.pages_project.project(), *ssr_entry_chunk, vec![])
*ssr_entry_chunk, .to_resolved()
this.pages_project.project().output_fs(), .await?,
this.pages_project.project().project_fs(),
this.pages_project.project().client_fs(),
vec![],
)
.to_resolved()
.await?,
))) )))
} else { } else {
ResolvedVc::cell(None) ResolvedVc::cell(None)
...@@ -907,7 +901,7 @@ impl PageEndpoint { ...@@ -907,7 +901,7 @@ impl PageEndpoint {
Ok(SsrChunk::NodeJs { Ok(SsrChunk::NodeJs {
entry: ssr_entry_chunk, entry: ssr_entry_chunk,
dynamic_import_entries, dynamic_import_entries,
nft, server_asset_trace_file,
} }
.cell()) .cell())
} }
...@@ -1109,11 +1103,11 @@ impl PageEndpoint { ...@@ -1109,11 +1103,11 @@ impl PageEndpoint {
SsrChunk::NodeJs { SsrChunk::NodeJs {
entry, entry,
dynamic_import_entries, dynamic_import_entries,
nft, server_asset_trace_file,
} => { } => {
server_assets.push(entry); server_assets.push(entry);
if let Some(nft) = &*nft.await? { if let Some(server_asset_trace_file) = &*server_asset_trace_file.await? {
server_assets.push(*nft); server_assets.push(*server_asset_trace_file);
} }
if emit_manifests { if emit_manifests {
...@@ -1396,7 +1390,7 @@ pub enum SsrChunk { ...@@ -1396,7 +1390,7 @@ pub enum SsrChunk {
NodeJs { NodeJs {
entry: ResolvedVc<Box<dyn OutputAsset>>, entry: ResolvedVc<Box<dyn OutputAsset>>,
dynamic_import_entries: Vc<DynamicImportedChunks>, dynamic_import_entries: Vc<DynamicImportedChunks>,
nft: ResolvedVc<OptionOutputAsset>, server_asset_trace_file: ResolvedVc<OptionOutputAsset>,
}, },
Edge { Edge {
files: Vc<OutputAssets>, files: Vc<OutputAssets>,
......
...@@ -6,7 +6,6 @@ use futures::future::try_join_all; ...@@ -6,7 +6,6 @@ use futures::future::try_join_all;
use turbo_tasks::{ use turbo_tasks::{
FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc, FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc,
}; };
use turbo_tasks_fs::{FileSystem, VirtualFileSystem};
use super::{ use super::{
availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItemInfo, availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItemInfo,
...@@ -154,7 +153,12 @@ pub async fn make_chunk_group( ...@@ -154,7 +153,12 @@ pub async fn make_chunk_group(
.clone_value(); .clone_value();
let rebased_modules = try_join_all(traced_modules.into_iter().map(|module| { let rebased_modules = try_join_all(traced_modules.into_iter().map(|module| {
RebasedAsset::new(*module, module.ident().path().root(), traced_fs().root()).to_resolved() RebasedAsset::new(
*module,
module.ident().path().root(),
module.ident().path().root(),
)
.to_resolved()
})) }))
.await?; .await?;
...@@ -198,12 +202,6 @@ pub async fn make_chunk_group( ...@@ -198,12 +202,6 @@ pub async fn make_chunk_group(
}) })
} }
// Without this wrapper, VirtualFileSystem::new_with_name always returns a new filesystem
#[turbo_tasks::function]
fn traced_fs() -> Vc<VirtualFileSystem> {
VirtualFileSystem::new_with_name("traced".into())
}
async fn references_to_output_assets( async fn references_to_output_assets(
references: FxIndexSet<Vc<Box<dyn ModuleReference>>>, references: FxIndexSet<Vc<Box<dyn ModuleReference>>>,
) -> Result<Vc<OutputAssets>> { ) -> Result<Vc<OutputAssets>> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment