Unverified Commit ba182aa1 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Cache bundle analysis per output file (#100475)

## Summary

Analyze each emitted and traced file once in a cached task that returns
its filename, chunk membership, and chunk parts with normalized sources
and compressed sizes. Routes previously made 3–5 task calls per shared
file, and task-graph bookkeeping for those calls dominated analysis.
Splitting, compression, and membership enumeration run as plain helpers
under that task, since each is read once.

Measured on v0/chat with `next build --analyze` (#99962), analysis phase
only: new tasks went from 269k to 242k, calls into existing tasks from
8.49M to 5.07M, and median CPU time from 150s to 124s. All 455 data
artifacts were byte-identical across 12 runs.

Supersedes #100486.

## Verification

- `cargo test -p turbopack-analyze --test split_chunk` and `cargo test
-p next-api --lib analyze` passed.
- Rebuilt the native binding with `release-with-debug`, installed it in
v0/chat, and compared `next build --analyze` output against the previous
binary across alternating runs.

<!-- NEXT_JS_LLM -->
parent 62e889a5
mod groups; mod groups;
mod membership; mod membership;
mod output_file;
use std::{borrow::Cow, io::Write}; use std::io::Write;
use anyhow::Result; use anyhow::Result;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use byteorder::{BE, WriteBytesExt}; use byteorder::{BE, WriteBytesExt};
use either::Either;
use membership::output_chunk_modules;
use next_core::app_structure::FileSystemPathVec; use next_core::app_structure::FileSystemPathVec;
use output_file::{AnalyzedChunkPart, analyze_output_file, analyze_traced_file};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use serde::Serialize; use serde::Serialize;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
FxIndexSet, JoinIterExt, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, ValueToString, FxIndexSet, JoinIterExt, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt,
ValueToStringRef, Vc, ValueToString, ValueToStringRef, Vc,
}; };
use turbo_tasks_fs::{ use turbo_tasks_fs::{
File, FileContent, FileSystemPath, File, FileContent, FileSystemPath,
rope::{Rope, RopeBuilder}, rope::{Rope, RopeBuilder},
}; };
use turbo_tasks_hash::hash_xxh3_hash64; use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_analyze::split_chunk::{split_output_asset_into_parts, split_traced_file_into_parts};
use turbopack_browser::ecmascript::EcmascriptBrowserChunk;
use turbopack_core::{ use turbopack_core::{
SOURCE_URL_PROTOCOL,
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::{ChunkingType, TracedMode}, chunk::{ChunkingType, TracedMode},
module::Module, module::Module,
...@@ -254,11 +251,12 @@ struct ModulesDataHeader { ...@@ -254,11 +251,12 @@ struct ModulesDataHeader {
pub traced_module_dependencies: EdgesDataReference, pub traced_module_dependencies: EdgesDataReference,
} }
#[derive(Clone, Copy, Serialize)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, NonLocalValue, Encode, Decode, Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
enum AnalyzeOutputFileCoverage { enum AnalyzeOutputFileCoverage {
Exact, Exact,
Unsupported, Unsupported,
#[default]
NotAChunk, NotAChunk,
} }
...@@ -687,7 +685,8 @@ pub async fn analyze_output_assets( ...@@ -687,7 +685,8 @@ pub async fn analyze_output_assets(
}, },
}; };
let route_entries = route_entries.await?.iter().cloned().collect(); let route_entries = route_entries.await?.iter().cloned().collect();
let module_index = analyze_module_index(module_graph).await?; let module_index_vc = analyze_module_index(module_graph);
let module_index = module_index_vc.await?;
let mut builder = let mut builder =
AnalyzeDataBuilder::new(route_entries, module_index.module_index_hash.clone()); AnalyzeDataBuilder::new(route_entries, module_index.module_index_hash.clone());
...@@ -696,78 +695,63 @@ pub async fn analyze_output_assets( ...@@ -696,78 +695,63 @@ pub async fn analyze_output_assets(
let mut candidates = Vec::new(); let mut candidates = Vec::new();
let mut browser_chunks = FxHashSet::default(); let mut browser_chunks = FxHashSet::default();
let prefix = format!("{SOURCE_URL_PROTOCOL}///"); let output_assets = output_assets.await?;
let traced_files = traced_files.await?;
// Process the output assets and extract chunk parts. // One call per file: per-route fan-out over shared assets dominates analysis
// Also creates sources for the chunk parts. // through task-graph bookkeeping, not the work behind each call.
for asset in output_assets let files = output_assets
.await?
.iter() .iter()
.copied() .map(|&asset| analyze_output_file(*asset, module_index_vc))
.map(Either::Left) .chain(
.chain(traced_files.await?.iter().cloned().map(Either::Right)) traced_files
{ .iter()
let file_system_path = match &asset { .map(|path| analyze_traced_file(path.clone())),
Either::Left(asset) => Either::Left(asset.path().await?), )
Either::Right(path) => Either::Right(path), .try_join()
}; .await?;
let path = match &file_system_path {
Either::Left(path) => &path.path,
Either::Right(path) => &path.path,
};
if path.ends_with(".map") || path.ends_with(".nft.json") {
// Skip source maps.
continue;
}
let filename = match &file_system_path {
Either::Left(path) => path.to_string_ref().await?,
Either::Right(path) => path.to_string_ref().await?,
};
let is_client = filename.starts_with("[client-fs]/"); for file in files.iter().filter_map(|file| (**file).as_ref()) {
let is_client = file.filename.starts_with("[client-fs]/");
let output_file_index = builder.add_output_file(AnalyzeOutputFile { let output_file_index = builder.add_output_file(AnalyzeOutputFile {
filename: filename.clone(), filename: file.filename.clone(),
}); });
if let Either::Left(asset) = &asset { if let Some(emitted) = &file.emitted {
let membership = &emitted.membership;
asset_indices asset_indices
.entry(*asset) .entry(emitted.asset)
.or_default() .or_default()
.push(output_file_index); .push(output_file_index);
if ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(*asset).is_some() { if emitted.is_browser_chunk {
browser_chunks.insert(output_file_index); browser_chunks.insert(output_file_index);
} }
let (indices, coverage, loads, async_loaders) = candidates.extend(
output_chunk_modules(*asset, &filename, &module_index, output_file_index).await?; membership
candidates.extend(loads); .load_targets
let file = &mut builder.output_files[output_file_index as usize]; .iter()
file.module_indices.extend(indices); .map(|load| ChunkLoadCandidate {
file.module_indices.sort_unstable(); source: output_file_index,
file.module_indices.dedup(); target: load.target,
file.async_loader_indices.extend(async_loaders); trigger_module_index: load.trigger_module_index,
file.async_loader_indices.sort_unstable(); }),
file.async_loader_indices.dedup(); );
file.module_coverage = coverage; let output = &mut builder.output_files[output_file_index as usize];
output.module_indices.extend(&membership.module_indices);
output.module_indices.sort_unstable();
output.module_indices.dedup();
output
.async_loader_indices
.extend(&membership.async_loader_indices);
output.async_loader_indices.sort_unstable();
output.async_loader_indices.dedup();
output.module_coverage = membership.coverage;
} }
let chunk_parts = match asset { for &AnalyzedChunkPart {
Either::Left(asset) => split_output_asset_into_parts(*asset).await?, ref source,
Either::Right(path) => split_traced_file_into_parts(path).await?, size,
}; compressed_size,
for chunk_part in &chunk_parts { } in &file.chunk_parts
let decoded_source = urlencoding::decode(&chunk_part.source)?; {
let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) { let source_index = builder.ensure_source(source).1;
Cow::Borrowed(stripped)
} else if decoded_source.starts_with('[') && decoded_source.contains("]/") {
decoded_source
} else {
Cow::Owned(format!(
"[project]/{}",
decoded_source.trim_start_matches("../")
))
};
let source_index = builder.ensure_source(&source).1;
let size = chunk_part.real_size + chunk_part.unaccounted_size;
let compressed_size = chunk_part.get_compressed_size().await?.unwrap_or(size);
summary.size += u64::from(size); summary.size += u64::from(size);
summary.compressed_size += u64::from(compressed_size); summary.compressed_size += u64::from(compressed_size);
if is_client { if is_client {
......
...@@ -2,9 +2,10 @@ ...@@ -2,9 +2,10 @@
//! snapshot's ordered module index. Unknown wrappers remain unsupported. //! snapshot's ordered module index. Unknown wrappers remain unsupported.
use anyhow::{Context, Result, anyhow}; use anyhow::{Context, Result, anyhow};
use bincode::{Decode, Encode};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ResolvedVc, ValueToString, Vc}; use turbo_tasks::{FxIndexSet, NonLocalValue, ResolvedVc, ValueToString, Vc};
use turbopack_browser::ecmascript::{ use turbopack_browser::ecmascript::{
EcmascriptBrowserChunk, EcmascriptBrowserRuntimeChunk, EcmascriptBrowserSingleEntryChunk, EcmascriptBrowserChunk, EcmascriptBrowserRuntimeChunk, EcmascriptBrowserSingleEntryChunk,
EcmascriptBrowserWorkerEntrypoint, EcmascriptBrowserWorkerEntrypoint,
...@@ -23,7 +24,22 @@ use turbopack_ecmascript::{ ...@@ -23,7 +24,22 @@ use turbopack_ecmascript::{
}; };
use turbopack_nodejs::{EcmascriptBuildNodeChunk, EcmascriptBuildNodeRuntimeChunk}; use turbopack_nodejs::{EcmascriptBuildNodeChunk, EcmascriptBuildNodeRuntimeChunk};
use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage, ChunkLoadCandidate}; use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage};
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct ChunkLoadTarget {
pub target: ResolvedVc<Box<dyn OutputAsset>>,
/// Index of the dynamically imported target module in the analyzer's shared module index.
pub trigger_module_index: u32,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct OutputChunkModules {
pub module_indices: Vec<u32>,
pub coverage: AnalyzeOutputFileCoverage,
pub load_targets: Vec<ChunkLoadTarget>,
pub async_loader_indices: Vec<u32>,
}
async fn insert_chunk_item( async fn insert_chunk_item(
module: Vc<Box<dyn Module>>, module: Vc<Box<dyn Module>>,
...@@ -78,8 +94,7 @@ fn insert_chunk_item_ident( ...@@ -78,8 +94,7 @@ fn insert_chunk_item_ident(
async fn chunk_item_load_candidates( async fn chunk_item_load_candidates(
item: ResolvedVc<Box<dyn turbopack_ecmascript::chunk::EcmascriptChunkItem>>, item: ResolvedVc<Box<dyn turbopack_ecmascript::chunk::EcmascriptChunkItem>>,
index: &AnalyzeModuleIndex, index: &AnalyzeModuleIndex,
source: u32, ) -> Result<Vec<ChunkLoadTarget>> {
) -> Result<Vec<ChunkLoadCandidate>> {
let module = item.module().to_resolved().await?; let module = item.module().to_resolved().await?;
let trigger = if let Some(loader) = ResolvedVc::try_downcast_type::<AsyncLoaderModule>(module) { let trigger = if let Some(loader) = ResolvedVc::try_downcast_type::<AsyncLoaderModule>(module) {
loader.await?.inner loader.await?.inner
...@@ -101,8 +116,7 @@ async fn chunk_item_load_candidates( ...@@ -101,8 +116,7 @@ async fn chunk_item_load_candidates(
Ok(assets Ok(assets
.iter() .iter()
.copied() .copied()
.map(|target| ChunkLoadCandidate { .map(|target| ChunkLoadTarget {
source,
target, target,
trigger_module_index, trigger_module_index,
}) })
...@@ -112,22 +126,16 @@ async fn chunk_item_load_candidates( ...@@ -112,22 +126,16 @@ async fn chunk_item_load_candidates(
/// Enumerate known browser/Node.js JS and CSS chunk items; unknown wrappers remain unsupported. /// Enumerate known browser/Node.js JS and CSS chunk items; unknown wrappers remain unsupported.
pub(super) async fn output_chunk_modules( pub(super) async fn output_chunk_modules(
asset: ResolvedVc<Box<dyn OutputAsset>>, asset: ResolvedVc<Box<dyn OutputAsset>>,
filename: &str, filename: &RcStr,
module_index: &AnalyzeModuleIndex, module_index: &AnalyzeModuleIndex,
output_file_index: u32, ) -> Result<OutputChunkModules> {
) -> Result<(
Vec<u32>,
AnalyzeOutputFileCoverage,
Vec<ChunkLoadCandidate>,
Vec<u32>,
)> {
if ResolvedVc::try_downcast_type::<EcmascriptBrowserRuntimeChunk>(asset).is_some() if ResolvedVc::try_downcast_type::<EcmascriptBrowserRuntimeChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBuildNodeRuntimeChunk>(asset).is_some() || ResolvedVc::try_downcast_type::<EcmascriptBuildNodeRuntimeChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<SingleFileEcmascriptOutput>(asset).is_some() || ResolvedVc::try_downcast_type::<SingleFileEcmascriptOutput>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBrowserSingleEntryChunk>(asset).is_some() || ResolvedVc::try_downcast_type::<EcmascriptBrowserSingleEntryChunk>(asset).is_some()
|| ResolvedVc::try_downcast_type::<EcmascriptBrowserWorkerEntrypoint>(asset).is_some() || ResolvedVc::try_downcast_type::<EcmascriptBrowserWorkerEntrypoint>(asset).is_some()
{ {
return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![], vec![])); return Ok(OutputChunkModules::default());
} }
let mut indices = FxIndexSet::default(); let mut indices = FxIndexSet::default();
let mut async_loaders = FxIndexSet::default(); let mut async_loaders = FxIndexSet::default();
...@@ -157,14 +165,8 @@ pub(super) async fn output_chunk_modules( ...@@ -157,14 +165,8 @@ pub(super) async fn output_chunk_modules(
) )
.await .await
.with_context(|| format!("Analyzing output {filename}"))?; .with_context(|| format!("Analyzing output {filename}"))?;
candidates.extend( candidates
chunk_item_load_candidates( .extend(chunk_item_load_candidates(item.chunk_item, module_index).await?);
item.chunk_item,
module_index,
output_file_index,
)
.await?,
);
} }
EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => { EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => {
for item in &batch.await?.chunk_items { for item in &batch.await?.chunk_items {
...@@ -177,12 +179,7 @@ pub(super) async fn output_chunk_modules( ...@@ -177,12 +179,7 @@ pub(super) async fn output_chunk_modules(
.await .await
.with_context(|| format!("Analyzing output {filename}"))?; .with_context(|| format!("Analyzing output {filename}"))?;
candidates.extend( candidates.extend(
chunk_item_load_candidates( chunk_item_load_candidates(item.chunk_item, module_index).await?,
item.chunk_item,
module_index,
output_file_index,
)
.await?,
); );
} }
} }
...@@ -204,19 +201,19 @@ pub(super) async fn output_chunk_modules( ...@@ -204,19 +201,19 @@ pub(super) async fn output_chunk_modules(
// Other emitted JS/CSS wrappers cannot enumerate their module members. // Other emitted JS/CSS wrappers cannot enumerate their module members.
enumerated = false; enumerated = false;
} else { } else {
return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![], vec![])); return Ok(OutputChunkModules::default());
} }
let coverage = if enumerated { let coverage = if enumerated {
AnalyzeOutputFileCoverage::Exact AnalyzeOutputFileCoverage::Exact
} else { } else {
AnalyzeOutputFileCoverage::Unsupported AnalyzeOutputFileCoverage::Unsupported
}; };
Ok(( Ok(OutputChunkModules {
indices.into_iter().collect(), module_indices: indices.into_iter().collect(),
coverage, coverage,
candidates, load_targets: candidates,
async_loaders.into_iter().collect(), async_loader_indices: async_loaders.into_iter().collect(),
)) })
} }
#[cfg(test)] #[cfg(test)]
......
//! One emitted or traced file's analysis, cached per file and shared by every
//! route that includes it.
use anyhow::Result;
use bincode::{Decode, Encode};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ResolvedVc, ValueToStringRef, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_analyze::split_chunk::{
ChunkPart, split_output_asset_into_parts, split_traced_file_into_parts,
};
use turbopack_browser::ecmascript::EcmascriptBrowserChunk;
use turbopack_core::{SOURCE_URL_PROTOCOL, output::OutputAsset};
use crate::analyze::{
AnalyzeModuleIndex,
membership::{OutputChunkModules, output_chunk_modules},
};
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct AnalyzedChunkPart {
/// Normalized source path, ready for the route's source tree.
pub source: RcStr,
pub size: u32,
pub compressed_size: u32,
}
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct EmittedChunk {
pub asset: ResolvedVc<Box<dyn OutputAsset>>,
pub is_browser_chunk: bool,
pub membership: OutputChunkModules,
}
#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]
pub(super) struct AnalyzedOutputFile {
pub filename: RcStr,
pub chunk_parts: Vec<AnalyzedChunkPart>,
/// `None` for traced files.
pub emitted: Option<EmittedChunk>,
}
/// `None` for source maps and trace manifests, which are not reported.
#[turbo_tasks::value(transparent)]
pub(super) struct OptionAnalyzedOutputFile(Option<AnalyzedOutputFile>);
#[turbo_tasks::function]
pub(super) async fn analyze_output_file(
asset: ResolvedVc<Box<dyn OutputAsset>>,
module_index: Vc<AnalyzeModuleIndex>,
) -> Result<Vc<OptionAnalyzedOutputFile>> {
let path = asset.path().await?;
if !is_reported_file(&path) {
return Ok(Vc::cell(None));
}
let filename = path.to_string_ref().await?;
let membership = output_chunk_modules(asset, &filename, &*module_index.await?).await?;
let chunk_parts = analyzed_chunk_parts(&split_output_asset_into_parts(*asset).await?).await?;
Ok(Vc::cell(Some(AnalyzedOutputFile {
filename,
chunk_parts,
emitted: Some(EmittedChunk {
asset,
is_browser_chunk: ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(asset)
.is_some(),
membership,
}),
})))
}
#[turbo_tasks::function]
pub(super) async fn analyze_traced_file(
path: FileSystemPath,
) -> Result<Vc<OptionAnalyzedOutputFile>> {
if !is_reported_file(&path) {
return Ok(Vc::cell(None));
}
Ok(Vc::cell(Some(AnalyzedOutputFile {
filename: path.to_string_ref().await?,
chunk_parts: analyzed_chunk_parts(&split_traced_file_into_parts(path).await?).await?,
emitted: None,
})))
}
fn is_reported_file(path: &FileSystemPath) -> bool {
!(path.path.ends_with(".map") || path.path.ends_with(".nft.json"))
}
async fn analyzed_chunk_parts(chunk_parts: &[ChunkPart]) -> Result<Vec<AnalyzedChunkPart>> {
let mut analyzed = Vec::with_capacity(chunk_parts.len());
for part in chunk_parts {
let size = part.real_size + part.unaccounted_size;
analyzed.push(AnalyzedChunkPart {
source: normalize_source(&part.source)?,
size,
compressed_size: part.get_compressed_size().await?.unwrap_or(size),
});
}
Ok(analyzed)
}
fn normalize_source(raw: &str) -> Result<RcStr> {
let decoded = urlencoding::decode(raw)?;
Ok(
if let Some(path) = decoded
.strip_prefix(SOURCE_URL_PROTOCOL.as_str())
.and_then(|rest| rest.strip_prefix("///"))
{
path.into()
} else if decoded.starts_with('[') && decoded.contains("]/") {
decoded.as_ref().into()
} else {
format!("[project]/{}", decoded.trim_start_matches("../")).into()
},
)
}
...@@ -60,19 +60,14 @@ impl ChunkPart { ...@@ -60,19 +60,14 @@ impl ChunkPart {
} }
} }
#[turbo_tasks::value(transparent)] pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vec<ChunkPart>> {
#[derive(Debug)]
pub struct ChunkParts(Vec<ChunkPart>);
#[turbo_tasks::function]
pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vc<ChunkParts>> {
let source = FileSource::new(path.clone()); let source = FileSource::new(path.clone());
let content = source.content().await?; let content = source.content().await?;
let AssetContent::File(file_content) = &*content else { let AssetContent::File(file_content) = &*content else {
return Ok(Vc::cell(vec![])); return Ok(vec![]);
}; };
let FileContent::Content(content) = &*file_content.await? else { let FileContent::Content(content) = &*file_content.await? else {
return Ok(Vc::cell(vec![])); return Ok(vec![]);
}; };
let content = content.content(); let content = content.content();
let lines_vc = file_content.lines().to_resolved().await?; let lines_vc = file_content.lines().to_resolved().await?;
...@@ -80,16 +75,15 @@ pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vc<Chu ...@@ -80,16 +75,15 @@ pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vc<Chu
self_mapped(path.to_string_ref().await?, content, lines_vc).await self_mapped(path.to_string_ref().await?, content, lines_vc).await
} }
#[turbo_tasks::function]
pub async fn split_output_asset_into_parts( pub async fn split_output_asset_into_parts(
asset: Vc<Box<dyn OutputAsset>>, asset: Vc<Box<dyn OutputAsset>>,
) -> Result<Vc<ChunkParts>> { ) -> Result<Vec<ChunkPart>> {
let content = asset.content().await?; let content = asset.content().await?;
let AssetContent::File(file_content) = &*content else { let AssetContent::File(file_content) = &*content else {
return Ok(Vc::cell(vec![])); return Ok(vec![]);
}; };
let FileContent::Content(content) = &*file_content.await? else { let FileContent::Content(content) = &*file_content.await? else {
return Ok(Vc::cell(vec![])); return Ok(vec![]);
}; };
let content = content.content(); let content = content.content();
let lines_vc = file_content.lines().to_resolved().await?; let lines_vc = file_content.lines().to_resolved().await?;
...@@ -397,37 +391,37 @@ pub async fn split_output_asset_into_parts( ...@@ -397,37 +391,37 @@ pub async fn split_output_asset_into_parts(
} }
} }
Ok(Vc::cell(chunk_parts.into_values().collect())) Ok(chunk_parts.into_values().collect())
} }
pub async fn self_mapped( pub async fn self_mapped(
path: RcStr, path: RcStr,
content: &Rope, content: &Rope,
lines: ResolvedVc<FileLinesContent>, lines: ResolvedVc<FileLinesContent>,
) -> Result<Vc<ChunkParts>> { ) -> Result<Vec<ChunkPart>> {
let len = content.len().try_into().unwrap_or(u32::MAX); let len = content.len().try_into().unwrap_or(u32::MAX);
Ok(Vc::cell(vec![ChunkPart { Ok(vec![ChunkPart {
source: path, source: path,
real_size: len, real_size: len,
unaccounted_size: 0, unaccounted_size: 0,
ranges: vec![], ranges: vec![],
lines, lines,
}])) }])
} }
async fn unaccounted( async fn unaccounted(
asset: Vc<Box<dyn OutputAsset>>, asset: Vc<Box<dyn OutputAsset>>,
content: &Rope, content: &Rope,
lines: ResolvedVc<FileLinesContent>, lines: ResolvedVc<FileLinesContent>,
) -> Result<Vc<ChunkParts>> { ) -> Result<Vec<ChunkPart>> {
let len = content.len().try_into().unwrap_or(u32::MAX); let len = content.len().try_into().unwrap_or(u32::MAX);
Ok(Vc::cell(vec![ChunkPart { Ok(vec![ChunkPart {
source: asset.path().to_string().owned().await?, source: asset.path().to_string().owned().await?,
real_size: 0, real_size: 0,
unaccounted_size: len, unaccounted_size: len,
ranges: vec![], ranges: vec![],
lines, lines,
}])) }])
} }
fn append_content_between( fn append_content_between(
......
...@@ -5,12 +5,12 @@ ...@@ -5,12 +5,12 @@
use anyhow::Result; use anyhow::Result;
use serde_json::json; use serde_json::json;
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::{OperationVc, ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
File, FileContent, FileSystem, FileSystemPath, VirtualFileSystem, rope::Rope, File, FileContent, FileSystem, FileSystemPath, VirtualFileSystem, rope::Rope,
}; };
use turbo_tasks_testing::{Registration, register, run_once}; use turbo_tasks_testing::{Registration, register, run_once};
use turbopack_analyze::split_chunk::{ChunkPartRange, ChunkParts, split_output_asset_into_parts}; use turbopack_analyze::split_chunk::{ChunkPart, ChunkPartRange, split_output_asset_into_parts};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
code_builder::{Code, CodeBuilder}, code_builder::{Code, CodeBuilder},
...@@ -55,13 +55,10 @@ async fn split_chunk() { ...@@ -55,13 +55,10 @@ async fn split_chunk() {
} }
.resolved_cell(); .resolved_cell();
#[turbo_tasks::function(operation, root)] let result = split_parts_operation(asset)
fn split_parts_operation(asset: ResolvedVc<TestAsset>) -> Vc<ChunkParts> { .read_strongly_consistent()
split_output_asset_into_parts(Vc::upcast(*asset)) .await?;
} let parts = &result.parts;
let parts_op = split_parts_operation(asset);
let parts = parts_op.read_strongly_consistent().await?;
assert_eq!(parts.len(), 2); assert_eq!(parts.len(), 2);
...@@ -96,27 +93,7 @@ async fn split_chunk() { ...@@ -96,27 +93,7 @@ async fn split_chunk() {
}] }]
); );
#[turbo_tasks::function(operation, root)] assert_eq!(result.compressed_sizes, [Some(43), Some(28)]);
async fn compressed_size_operation(
parts: OperationVc<ChunkParts>,
index: usize,
) -> Result<Vc<u32>> {
Ok(Vc::cell(
parts.connect().await?[index]
.get_compressed_size()
.await?
.unwrap(),
))
}
let compressed_size_0 = compressed_size_operation(parts_op, 0)
.read_strongly_consistent()
.await?;
let compressed_size_1 = compressed_size_operation(parts_op, 1)
.read_strongly_consistent()
.await?;
assert_eq!(*compressed_size_0, 43);
assert_eq!(*compressed_size_1, 28);
anyhow::Ok(()) anyhow::Ok(())
}) })
...@@ -124,6 +101,26 @@ async fn split_chunk() { ...@@ -124,6 +101,26 @@ async fn split_chunk() {
.unwrap() .unwrap()
} }
#[turbo_tasks::value]
struct SplitResult {
parts: Vec<ChunkPart>,
compressed_sizes: Vec<Option<u32>>,
}
#[turbo_tasks::function(operation, root)]
async fn split_parts_operation(asset: ResolvedVc<TestAsset>) -> Result<Vc<SplitResult>> {
let parts = split_output_asset_into_parts(Vc::upcast(*asset)).await?;
let mut compressed_sizes = Vec::with_capacity(parts.len());
for part in &parts {
compressed_sizes.push(part.get_compressed_size().await?);
}
Ok(SplitResult {
parts,
compressed_sizes,
}
.cell())
}
#[turbo_tasks::value] #[turbo_tasks::value]
struct TestAsset { struct TestAsset {
code: ResolvedVc<Code>, code: ResolvedVc<Code>,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment