Unverified Commit ae64f6cf authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: make routes hashes mainfest more incremental (#87107)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the PR.
- Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to understand the PR)
- When linking to a Slack thread, you might want to share details of the conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->
parent be56ce3a
......@@ -6,7 +6,7 @@ use turbo_tasks_fs::{FileContent, FileSystemPath};
use turbo_tasks_hash::{DeterministicHash, Xxh3Hash64Hasher};
use turbopack_core::{
asset::{Asset, AssetContent},
module::Module,
module::{Module, Modules},
module_graph::{GraphTraversalAction, ModuleGraph},
output::{
ExpandOutputAssetsInput, OutputAsset, OutputAssets, OutputAssetsReference,
......@@ -16,7 +16,7 @@ use turbopack_core::{
use crate::{
project::Project,
route::{Endpoint, EndpointGroup},
route::{Endpoint, EndpointGroup, Endpoints},
};
#[turbo_tasks::value(shared)]
......@@ -45,11 +45,58 @@ impl EndpointHashes {
}
#[turbo_tasks::function]
pub async fn endpoint_hashes(
pub async fn endpoint_outputs(endpoint: Vc<Box<dyn Endpoint>>) -> Result<Vc<OutputAssets>> {
Ok(*endpoint.output().await?.output_assets)
}
#[turbo_tasks::function]
pub async fn endpoints_outputs(endpoints: Vc<Endpoints>) -> Result<Vc<OutputAssets>> {
let endpoints = endpoints.await?;
let all_outputs = endpoints
.iter()
.map(async |endpoint| Ok(endpoint.output().await?.output_assets.await?))
.try_join()
.await?;
let set = all_outputs
.into_iter()
.flatten()
.copied()
.collect::<FxIndexSet<_>>();
Ok(Vc::cell(set.into_iter().collect()))
}
#[turbo_tasks::function]
pub async fn outputs_hash(outputs: Vc<OutputAssets>) -> Result<Vc<u64>> {
let output_assets = expand_output_assets(
outputs
.await?
.into_iter()
.map(|asset| ExpandOutputAssetsInput::Asset(*asset)),
true,
)
.await?;
let outputs_hashes = output_assets
.iter()
.map(|asset| asset.content().hash())
.try_join()
.await?;
let outputs_hash = {
let mut hasher = Xxh3Hash64Hasher::new();
for hash in outputs_hashes.iter() {
hash.deterministic_hash(&mut hasher);
}
hasher.finish()
};
Ok(Vc::cell(outputs_hash))
}
#[turbo_tasks::function]
pub async fn endpoint_entry_modules(
base_module_graph: Vc<ModuleGraph>,
module_graph: Vc<ModuleGraph>,
endpoint: Vc<Box<dyn Endpoint>>,
) -> Result<Vc<EndpointHashes>> {
) -> Result<Vc<Modules>> {
let entries = endpoint.entries();
let additional_entries = endpoint.additional_entries(base_module_graph);
let modules = entries
......@@ -58,14 +105,46 @@ pub async fn endpoint_hashes(
.chain(additional_entries.await?.into_iter())
.flat_map(|e| e.entries())
.collect::<FxIndexSet<_>>();
let outputs = endpoint.output();
Ok(Vc::cell(modules.into_iter().collect()))
}
#[turbo_tasks::function]
pub async fn endpoints_entry_modules(
base_module_graph: Vc<ModuleGraph>,
endpoints: Vc<Endpoints>,
) -> Result<Vc<Modules>> {
let endpoints = endpoints.await?;
let entries_and_additional_entries = endpoints
.iter()
.map(async |endpoint| {
let entries = endpoint.entries();
let additional_entries = endpoint.additional_entries(base_module_graph);
Ok((entries.await?, additional_entries.await?))
})
.try_join()
.await?;
let modules = entries_and_additional_entries
.into_iter()
.flat_map(|(entries, additional_entries)| {
entries
.into_iter()
.chain(additional_entries.into_iter())
.flat_map(|e| e.entries())
})
.collect::<FxIndexSet<_>>();
Ok(Vc::cell(modules.into_iter().collect()))
}
#[turbo_tasks::function]
pub async fn sources_hash(module_graph: Vc<ModuleGraph>, modules: Vc<Modules>) -> Result<Vc<u64>> {
let modules = modules.await?;
let mut all_modules = FxIndexSet::default();
let module_graph = module_graph.await?;
module_graph.traverse_nodes_dfs(
modules,
modules.into_iter().copied(),
&mut all_modules,
|module, all_modules| {
all_modules.insert(*module);
......@@ -84,22 +163,6 @@ pub async fn endpoint_hashes(
.try_join()
.await?;
let output_assets = expand_output_assets(
outputs
.await?
.output_assets
.await?
.into_iter()
.map(|asset| ExpandOutputAssetsInput::Asset(*asset)),
true,
)
.await?;
let outputs_hashes = output_assets
.iter()
.map(|asset| asset.content().hash())
.try_join()
.await?;
let sources_hash = {
let mut hasher = Xxh3Hash64Hasher::new();
for source in sources.iter() {
......@@ -107,19 +170,8 @@ pub async fn endpoint_hashes(
}
hasher.finish()
};
let outputs_hash = {
let mut hasher = Xxh3Hash64Hasher::new();
for hash in outputs_hashes.iter() {
hash.deterministic_hash(&mut hasher);
}
hasher.finish()
};
Ok(EndpointHashes {
sources_hash,
outputs_hash,
}
.cell())
Ok(Vc::cell(sources_hash))
}
#[derive(Serialize)]
......@@ -161,43 +213,45 @@ impl Asset for RoutesHashesManifestAsset {
let entrypoint_groups = self.project.get_all_endpoint_groups(false).await?;
for (key, EndpointGroup { primary, .. }) in entrypoint_groups {
if let &[entry] = &primary.as_slice() {
entrypoint_hashes.insert(
key.as_str(),
endpoint_hashes(base_module_graph, full_module_graph, *entry.endpoint),
);
let entry = if let &[entry] = &primary.as_slice() {
(
sources_hash(
full_module_graph,
endpoint_entry_modules(base_module_graph, *entry.endpoint),
),
outputs_hash(endpoint_outputs(*entry.endpoint)),
)
} else {
let hashes = primary
.iter()
.map(|entry| {
endpoint_hashes(base_module_graph, full_module_graph, *entry.endpoint)
})
.try_join()
.await?;
let hashes = EndpointHashes::merge(
primary
.iter()
.map(|page| page.sub_name.clone())
.zip(hashes.iter())
.map(|(k, v)| (k, &**v)),
let endpoints = Vc::cell(primary.iter().map(|entry| entry.endpoint).collect());
(
sources_hash(
full_module_graph,
endpoints_entry_modules(base_module_graph, endpoints),
),
outputs_hash(endpoints_outputs(endpoints)),
)
.cell();
entrypoint_hashes.insert(key.as_str(), hashes);
}
};
entrypoint_hashes.insert(key.as_str(), entry);
}
let entrypoint_hashes_values = entrypoint_hashes.values().copied().try_join().await?;
let entrypoint_hashes_values = entrypoint_hashes
.values()
.map(async |(sources_hash, outputs_hash)| {
Ok((sources_hash.await?, outputs_hash.await?))
})
.try_join()
.await?;
let manifest = serde_json::to_string_pretty(&RoutesHashesManifest {
routes: entrypoint_hashes
.into_keys()
.zip(entrypoint_hashes_values.into_iter())
.map(|(k, v)| {
.map(|(k, (sources_hash, outputs_hash))| {
(
k,
EndpointHashStrings {
sources_hash: format!("{:016x}", v.sources_hash),
outputs_hash: format!("{:016x}", v.outputs_hash),
sources_hash: format!("{:016x}", *sources_hash),
outputs_hash: format!("{:016x}", *outputs_hash),
},
)
})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment