Unverified Commit 91bdad8f authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: use file_content.hash() consistently (#90078)

So that we don't hash asset contents multiple times.

`hash_xxh3_hash64(file.content())`  isn't a turbotask, but `file.hash()` is cached
parent 2a9e7560
......@@ -34,7 +34,7 @@ async fn server_path(
) -> Result<Vc<OptionServerPath>> {
Ok(Vc::cell(
if let Some(path) = node_root.get_path_to(&*asset.path().await?) {
let content_hash = *asset.content().file_content().hash().await?;
let content_hash = *asset.content().hash().await?;
Some(ServerPath {
path: RcStr::from(path),
content_hash,
......
......@@ -2,12 +2,11 @@
//!
//! See `next/src/build/webpack/loaders/next-metadata-image-loader`
use anyhow::{Result, bail};
use anyhow::{Context, Result};
use indoc::formatdoc;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{File, FileContent, FileSystemPath};
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::{
asset::AssetContent,
context::AssetContext,
......@@ -24,23 +23,6 @@ use turbopack_ecmascript::{
use crate::next_app::AppPage;
async fn hash_file_content(path: FileSystemPath) -> Result<u64> {
let original_file_content = path.read().await?;
Ok(match &*original_file_content {
FileContent::Content(content) => {
let content = content.content().to_bytes();
hash_xxh3_hash64(&*content)
}
FileContent::NotFound => {
bail!(
"metadata file not found: {}",
&path.value_to_string().await?
);
}
})
}
async fn dynamic_image_metadata_with_generator_source(
path: FileSystemPath,
ty: RcStr,
......@@ -51,7 +33,13 @@ async fn dynamic_image_metadata_with_generator_source(
let stem = stem.unwrap_or_default();
let ext = path.extension();
let hash_query = format!("?{:x}", hash_file_content(path.clone()).await?);
let hash_query = format!(
"?{:x}",
path.read()
.content_hash()
.await?
.context("metadata file not found")?
);
let use_numeric_sizes = ty == "twitter" || ty == "openGraph";
let sizes = if use_numeric_sizes {
......@@ -125,7 +113,13 @@ async fn dynamic_image_metadata_without_generator_source(
let stem = stem.unwrap_or_default();
let ext = path.extension();
let hash_query = format!("?{:x}", hash_file_content(path.clone()).await?);
let hash_query = format!(
"?{:x}",
path.read()
.content_hash()
.await?
.context("metadata file not found")?
);
let use_numeric_sizes = ty == "twitter" || ty == "openGraph";
let sizes = if use_numeric_sizes {
......
......@@ -2364,8 +2364,18 @@ impl FileContent {
}
#[turbo_tasks::function]
pub async fn hash(self: Vc<Self>) -> Result<Vc<u64>> {
Ok(Vc::cell(hash_xxh3_hash64(&self.await?)))
pub async fn hash(&self) -> Result<Vc<u64>> {
Ok(Vc::cell(hash_xxh3_hash64(self)))
}
/// Compared to [FileContent::hash], this hashes only the bytes of the file content and nothing
/// else. If there is no file content, it returns `None`.
#[turbo_tasks::function]
pub async fn content_hash(&self) -> Result<Vc<Option<u64>>> {
match self {
FileContent::Content(file) => Ok(Vc::cell(Some(hash_xxh3_hash64(&file.content)))),
FileContent::NotFound => Ok(Vc::cell(None)),
}
}
}
......
......@@ -893,6 +893,7 @@ mod test {
};
use anyhow::Result;
use turbo_tasks_hash::hash_xxh3_hash64;
use super::{InnerRope, Rope, RopeBuilder, RopeElem};
......@@ -1078,6 +1079,21 @@ mod test {
assert_ne!(a, b);
}
#[test]
fn hash_structure_invariance() {
let shared = Rope::from("def");
let a = Rope::new(vec!["abc".into(), shared.clone().into(), "ggg".into()]);
let b = Rope::new(vec![
"ab".into(),
"c".into(),
shared.into(),
"g".into(),
"gg".into(),
]);
assert_eq!(hash_xxh3_hash64(a), hash_xxh3_hash64(b));
}
#[test]
fn iteration() {
let shared = Rope::from("def");
......
......@@ -7,9 +7,9 @@ use turbo_tasks::{
trace::TraceRawVcs,
};
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{DeterministicHash, hash_xxh3_hash64};
use turbo_tasks_hash::{DeterministicHash, encode_hex};
use turbopack_core::{
asset::{Asset, AssetContent},
asset::Asset,
chunk::{
AssetSuffix, Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule,
ChunkingConfig, ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset,
......@@ -577,20 +577,15 @@ impl ChunkingContext for BrowserChunkingContext {
let Some(asset) = asset else {
bail!("chunk_path requires an asset when content hashing is enabled");
};
let content = asset.content().await?;
if let AssetContent::File(file) = &*content {
let hash = hash_xxh3_hash64(&file.await?);
let length = length as usize;
if let Some(prefix) = prefix {
format!("{prefix}-{hash:0length$x}{extension}").into()
} else {
format!("{hash:0length$x}{extension}").into()
}
let hash = asset.content().content_hash().await?.context(
"chunk_path requires an asset with file content when content hashing is \
enabled",
)?;
let length = length as usize;
if let Some(prefix) = prefix {
format!("{prefix}-{hash:0length$x}{extension}").into()
} else {
bail!(
"chunk_path requires an asset with file content when content hashing is \
enabled"
);
format!("{hash:0length$x}{extension}").into()
}
}
};
......@@ -646,12 +641,13 @@ impl ChunkingContext for BrowserChunkingContext {
#[turbo_tasks::function]
async fn asset_path(
&self,
content_hash: RcStr,
content_hash: Vc<u64>,
original_asset_ident: Vc<AssetIdent>,
tag: Option<RcStr>,
) -> Result<Vc<FileSystemPath>> {
let source_path = original_asset_ident.path().await?;
let basename = source_path.file_name();
let content_hash = encode_hex(*content_hash.await?);
let asset_path = match source_path.extension_ref() {
Some(ext) => format!(
"{basename}.{content_hash}.{ext}",
......
......@@ -117,4 +117,14 @@ impl AssetContent {
}
}
}
/// Compared to [AssetContent::hash], this hashes only the bytes of the file content and nothing
/// else. If there is no file content, it returns `None`.
#[turbo_tasks::function]
pub async fn content_hash(&self) -> Result<Vc<Option<u64>>> {
match self {
AssetContent::File(content) => Ok(content.content_hash()),
AssetContent::Redirect { .. } => Ok(Vc::cell(None)),
}
}
}
......@@ -350,7 +350,7 @@ pub trait ChunkingContext {
#[turbo_tasks::function]
fn asset_path(
self: Vc<Self>,
content_hash: RcStr,
content_hash: Vc<u64>,
original_asset_ident: Vc<AssetIdent>,
tag: Option<RcStr>,
) -> Vc<FileSystemPath>;
......
use std::sync::Arc;
use anyhow::{Result, anyhow};
use anyhow::{Context, Result, anyhow};
use turbo_rcstr::RcStr;
use turbo_tasks::{
IntoTraitRef, NonLocalValue, OperationValue, ReadRef, ResolvedVc, State, TraitRef, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs,
};
use turbo_tasks_fs::{FileContent, LinkType};
use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64};
use turbo_tasks_hash::encode_hex;
use crate::asset::AssetContent;
......@@ -232,16 +232,12 @@ impl FileHashVersion {
/// Computes a new [`Vc<FileHashVersion>`] from a path.
pub async fn compute(asset_content: &AssetContent) -> Result<Vc<Self>> {
match asset_content {
AssetContent::File(file_vc) => match &*file_vc.await? {
FileContent::Content(file) => {
let hash = hash_xxh3_hash64(file.content());
let hex_hash = encode_hex(hash);
Ok(Self::cell(FileHashVersion {
hash: hex_hash.into(),
}))
}
FileContent::NotFound => Err(anyhow!("file not found")),
},
AssetContent::File(file_vc) => {
let hash = file_vc.content_hash().await?.context("file not found")?;
Ok(Self::cell(FileHashVersion {
hash: encode_hex(hash).into(),
}))
}
AssetContent::Redirect { .. } => Err(anyhow!("not a file")),
}
}
......
......@@ -3,6 +3,7 @@ use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexMap, ResolvedVc, TryJoinIterExt, Upcast, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::encode_hex;
use turbopack_core::{
asset::Asset,
chunk::{
......@@ -454,12 +455,13 @@ impl ChunkingContext for NodeJsChunkingContext {
#[turbo_tasks::function]
async fn asset_path(
&self,
content_hash: RcStr,
content_hash: Vc<u64>,
original_asset_ident: Vc<AssetIdent>,
tag: Option<RcStr>,
) -> Result<Vc<FileSystemPath>> {
let source_path = original_asset_ident.path().await?;
let basename = source_path.file_name();
let content_hash = encode_hex(*content_hash.await?);
let asset_path = match source_path.extension_ref() {
Some(ext) => format!(
"{basename}.{content_hash}.{ext}",
......
use anyhow::Result;
use anyhow::{Context, Result};
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{FileContent, FileSystemPath};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{
asset::{Asset, AssetContent},
chunk::ChunkingContext,
......@@ -39,18 +39,11 @@ impl OutputAsset for StaticOutputAsset {
#[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> {
let content = self.source.content();
let content_hash = if let AssetContent::File(file) = &*content.await? {
if let FileContent::Content(file) = &*file.await? {
turbo_tasks_hash::hash_xxh3_hash64(file.content())
} else {
anyhow::bail!("StaticAsset::path: not found")
}
} else {
anyhow::bail!("StaticAsset::path: unsupported file content")
};
let content_hash_b16 = turbo_tasks_hash::encode_hex(content_hash);
let content_hash = content.content_hash().await?.context(
"Missing content when trying to generate the content hash for StaticOutputAsset",
)?;
Ok(self.chunking_context.asset_path(
content_hash_b16.into(),
Vc::cell(content_hash),
self.source.ident(),
self.tag.clone(),
))
......
......@@ -9,10 +9,9 @@
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
use anyhow::Result;
use anyhow::{Context, Result};
use turbo_rcstr::RcStr;
use turbo_tasks::Vc;
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::asset::Asset;
pub(crate) mod analysis;
......@@ -24,8 +23,10 @@ pub mod source;
#[turbo_tasks::function]
pub async fn wasm_edge_var_name(asset: Vc<Box<dyn Asset>>) -> Result<Vc<RcStr>> {
let content = asset.content().file_content().await?;
Ok(Vc::cell(
format!("wasm_{:08x}", hash_xxh3_hash64(content)).into(),
))
let hash = asset
.content()
.content_hash()
.await?
.context("Missing content when trying to generate the content hash for a WASM asset")?;
Ok(Vc::cell(format!("wasm_{:08x}", hash).into()))
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment