Unverified Commit 91bdad8f authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: use file_content.hash() consistently (#90078)

So that we don't hash asset contents multiple times.

`hash_xxh3_hash64(file.content())`  isn't a turbotask, but `file.hash()` is cached
parent 2a9e7560
...@@ -34,7 +34,7 @@ async fn server_path( ...@@ -34,7 +34,7 @@ async fn server_path(
) -> Result<Vc<OptionServerPath>> { ) -> Result<Vc<OptionServerPath>> {
Ok(Vc::cell( Ok(Vc::cell(
if let Some(path) = node_root.get_path_to(&*asset.path().await?) { if let Some(path) = node_root.get_path_to(&*asset.path().await?) {
let content_hash = *asset.content().file_content().hash().await?; let content_hash = *asset.content().hash().await?;
Some(ServerPath { Some(ServerPath {
path: RcStr::from(path), path: RcStr::from(path),
content_hash, content_hash,
......
...@@ -2,12 +2,11 @@ ...@@ -2,12 +2,11 @@
//! //!
//! See `next/src/build/webpack/loaders/next-metadata-image-loader` //! See `next/src/build/webpack/loaders/next-metadata-image-loader`
use anyhow::{Result, bail}; use anyhow::{Context, Result};
use indoc::formatdoc; use indoc::formatdoc;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{File, FileContent, FileSystemPath}; use turbo_tasks_fs::{File, FileContent, FileSystemPath};
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::{ use turbopack_core::{
asset::AssetContent, asset::AssetContent,
context::AssetContext, context::AssetContext,
...@@ -24,23 +23,6 @@ use turbopack_ecmascript::{ ...@@ -24,23 +23,6 @@ use turbopack_ecmascript::{
use crate::next_app::AppPage; use crate::next_app::AppPage;
async fn hash_file_content(path: FileSystemPath) -> Result<u64> {
let original_file_content = path.read().await?;
Ok(match &*original_file_content {
FileContent::Content(content) => {
let content = content.content().to_bytes();
hash_xxh3_hash64(&*content)
}
FileContent::NotFound => {
bail!(
"metadata file not found: {}",
&path.value_to_string().await?
);
}
})
}
async fn dynamic_image_metadata_with_generator_source( async fn dynamic_image_metadata_with_generator_source(
path: FileSystemPath, path: FileSystemPath,
ty: RcStr, ty: RcStr,
...@@ -51,7 +33,13 @@ async fn dynamic_image_metadata_with_generator_source( ...@@ -51,7 +33,13 @@ async fn dynamic_image_metadata_with_generator_source(
let stem = stem.unwrap_or_default(); let stem = stem.unwrap_or_default();
let ext = path.extension(); let ext = path.extension();
let hash_query = format!("?{:x}", hash_file_content(path.clone()).await?); let hash_query = format!(
"?{:x}",
path.read()
.content_hash()
.await?
.context("metadata file not found")?
);
let use_numeric_sizes = ty == "twitter" || ty == "openGraph"; let use_numeric_sizes = ty == "twitter" || ty == "openGraph";
let sizes = if use_numeric_sizes { let sizes = if use_numeric_sizes {
...@@ -125,7 +113,13 @@ async fn dynamic_image_metadata_without_generator_source( ...@@ -125,7 +113,13 @@ async fn dynamic_image_metadata_without_generator_source(
let stem = stem.unwrap_or_default(); let stem = stem.unwrap_or_default();
let ext = path.extension(); let ext = path.extension();
let hash_query = format!("?{:x}", hash_file_content(path.clone()).await?); let hash_query = format!(
"?{:x}",
path.read()
.content_hash()
.await?
.context("metadata file not found")?
);
let use_numeric_sizes = ty == "twitter" || ty == "openGraph"; let use_numeric_sizes = ty == "twitter" || ty == "openGraph";
let sizes = if use_numeric_sizes { let sizes = if use_numeric_sizes {
......
...@@ -2364,8 +2364,18 @@ impl FileContent { ...@@ -2364,8 +2364,18 @@ impl FileContent {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn hash(self: Vc<Self>) -> Result<Vc<u64>> { pub async fn hash(&self) -> Result<Vc<u64>> {
Ok(Vc::cell(hash_xxh3_hash64(&self.await?))) Ok(Vc::cell(hash_xxh3_hash64(self)))
}
/// Compared to [FileContent::hash], this hashes only the bytes of the file content and nothing
/// else. If there is no file content, it returns `None`.
#[turbo_tasks::function]
pub async fn content_hash(&self) -> Result<Vc<Option<u64>>> {
match self {
FileContent::Content(file) => Ok(Vc::cell(Some(hash_xxh3_hash64(&file.content)))),
FileContent::NotFound => Ok(Vc::cell(None)),
}
} }
} }
......
...@@ -893,6 +893,7 @@ mod test { ...@@ -893,6 +893,7 @@ mod test {
}; };
use anyhow::Result; use anyhow::Result;
use turbo_tasks_hash::hash_xxh3_hash64;
use super::{InnerRope, Rope, RopeBuilder, RopeElem}; use super::{InnerRope, Rope, RopeBuilder, RopeElem};
...@@ -1078,6 +1079,21 @@ mod test { ...@@ -1078,6 +1079,21 @@ mod test {
assert_ne!(a, b); assert_ne!(a, b);
} }
#[test]
fn hash_structure_invariance() {
let shared = Rope::from("def");
let a = Rope::new(vec!["abc".into(), shared.clone().into(), "ggg".into()]);
let b = Rope::new(vec![
"ab".into(),
"c".into(),
shared.into(),
"g".into(),
"gg".into(),
]);
assert_eq!(hash_xxh3_hash64(a), hash_xxh3_hash64(b));
}
#[test] #[test]
fn iteration() { fn iteration() {
let shared = Rope::from("def"); let shared = Rope::from("def");
......
...@@ -7,9 +7,9 @@ use turbo_tasks::{ ...@@ -7,9 +7,9 @@ use turbo_tasks::{
trace::TraceRawVcs, trace::TraceRawVcs,
}; };
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{DeterministicHash, hash_xxh3_hash64}; use turbo_tasks_hash::{DeterministicHash, encode_hex};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::Asset,
chunk::{ chunk::{
AssetSuffix, Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, AssetSuffix, Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule,
ChunkingConfig, ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset, ChunkingConfig, ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset,
...@@ -577,21 +577,16 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -577,21 +577,16 @@ impl ChunkingContext for BrowserChunkingContext {
let Some(asset) = asset else { let Some(asset) = asset else {
bail!("chunk_path requires an asset when content hashing is enabled"); bail!("chunk_path requires an asset when content hashing is enabled");
}; };
let content = asset.content().await?; let hash = asset.content().content_hash().await?.context(
if let AssetContent::File(file) = &*content { "chunk_path requires an asset with file content when content hashing is \
let hash = hash_xxh3_hash64(&file.await?); enabled",
)?;
let length = length as usize; let length = length as usize;
if let Some(prefix) = prefix { if let Some(prefix) = prefix {
format!("{prefix}-{hash:0length$x}{extension}").into() format!("{prefix}-{hash:0length$x}{extension}").into()
} else { } else {
format!("{hash:0length$x}{extension}").into() format!("{hash:0length$x}{extension}").into()
} }
} else {
bail!(
"chunk_path requires an asset with file content when content hashing is \
enabled"
);
}
} }
}; };
Ok(chunk_root_path.join(&name)?.cell()) Ok(chunk_root_path.join(&name)?.cell())
...@@ -646,12 +641,13 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -646,12 +641,13 @@ impl ChunkingContext for BrowserChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn asset_path( async fn asset_path(
&self, &self,
content_hash: RcStr, content_hash: Vc<u64>,
original_asset_ident: Vc<AssetIdent>, original_asset_ident: Vc<AssetIdent>,
tag: Option<RcStr>, tag: Option<RcStr>,
) -> Result<Vc<FileSystemPath>> { ) -> Result<Vc<FileSystemPath>> {
let source_path = original_asset_ident.path().await?; let source_path = original_asset_ident.path().await?;
let basename = source_path.file_name(); let basename = source_path.file_name();
let content_hash = encode_hex(*content_hash.await?);
let asset_path = match source_path.extension_ref() { let asset_path = match source_path.extension_ref() {
Some(ext) => format!( Some(ext) => format!(
"{basename}.{content_hash}.{ext}", "{basename}.{content_hash}.{ext}",
......
...@@ -117,4 +117,14 @@ impl AssetContent { ...@@ -117,4 +117,14 @@ impl AssetContent {
} }
} }
} }
/// Compared to [AssetContent::hash], this hashes only the bytes of the file content and nothing
/// else. If there is no file content, it returns `None`.
#[turbo_tasks::function]
pub async fn content_hash(&self) -> Result<Vc<Option<u64>>> {
match self {
AssetContent::File(content) => Ok(content.content_hash()),
AssetContent::Redirect { .. } => Ok(Vc::cell(None)),
}
}
} }
...@@ -350,7 +350,7 @@ pub trait ChunkingContext { ...@@ -350,7 +350,7 @@ pub trait ChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
fn asset_path( fn asset_path(
self: Vc<Self>, self: Vc<Self>,
content_hash: RcStr, content_hash: Vc<u64>,
original_asset_ident: Vc<AssetIdent>, original_asset_ident: Vc<AssetIdent>,
tag: Option<RcStr>, tag: Option<RcStr>,
) -> Vc<FileSystemPath>; ) -> Vc<FileSystemPath>;
......
use std::sync::Arc; use std::sync::Arc;
use anyhow::{Result, anyhow}; use anyhow::{Context, Result, anyhow};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
IntoTraitRef, NonLocalValue, OperationValue, ReadRef, ResolvedVc, State, TraitRef, Vc, IntoTraitRef, NonLocalValue, OperationValue, ReadRef, ResolvedVc, State, TraitRef, Vc,
debug::ValueDebugFormat, trace::TraceRawVcs, debug::ValueDebugFormat, trace::TraceRawVcs,
}; };
use turbo_tasks_fs::{FileContent, LinkType}; use turbo_tasks_fs::{FileContent, LinkType};
use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64}; use turbo_tasks_hash::encode_hex;
use crate::asset::AssetContent; use crate::asset::AssetContent;
...@@ -232,16 +232,12 @@ impl FileHashVersion { ...@@ -232,16 +232,12 @@ impl FileHashVersion {
/// Computes a new [`Vc<FileHashVersion>`] from a path. /// Computes a new [`Vc<FileHashVersion>`] from a path.
pub async fn compute(asset_content: &AssetContent) -> Result<Vc<Self>> { pub async fn compute(asset_content: &AssetContent) -> Result<Vc<Self>> {
match asset_content { match asset_content {
AssetContent::File(file_vc) => match &*file_vc.await? { AssetContent::File(file_vc) => {
FileContent::Content(file) => { let hash = file_vc.content_hash().await?.context("file not found")?;
let hash = hash_xxh3_hash64(file.content());
let hex_hash = encode_hex(hash);
Ok(Self::cell(FileHashVersion { Ok(Self::cell(FileHashVersion {
hash: hex_hash.into(), hash: encode_hex(hash).into(),
})) }))
} }
FileContent::NotFound => Err(anyhow!("file not found")),
},
AssetContent::Redirect { .. } => Err(anyhow!("not a file")), AssetContent::Redirect { .. } => Err(anyhow!("not a file")),
} }
} }
......
...@@ -3,6 +3,7 @@ use tracing::Instrument; ...@@ -3,6 +3,7 @@ use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexMap, ResolvedVc, TryJoinIterExt, Upcast, ValueToString, Vc}; use turbo_tasks::{FxIndexMap, ResolvedVc, TryJoinIterExt, Upcast, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::encode_hex;
use turbopack_core::{ use turbopack_core::{
asset::Asset, asset::Asset,
chunk::{ chunk::{
...@@ -454,12 +455,13 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -454,12 +455,13 @@ impl ChunkingContext for NodeJsChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn asset_path( async fn asset_path(
&self, &self,
content_hash: RcStr, content_hash: Vc<u64>,
original_asset_ident: Vc<AssetIdent>, original_asset_ident: Vc<AssetIdent>,
tag: Option<RcStr>, tag: Option<RcStr>,
) -> Result<Vc<FileSystemPath>> { ) -> Result<Vc<FileSystemPath>> {
let source_path = original_asset_ident.path().await?; let source_path = original_asset_ident.path().await?;
let basename = source_path.file_name(); let basename = source_path.file_name();
let content_hash = encode_hex(*content_hash.await?);
let asset_path = match source_path.extension_ref() { let asset_path = match source_path.extension_ref() {
Some(ext) => format!( Some(ext) => format!(
"{basename}.{content_hash}.{ext}", "{basename}.{content_hash}.{ext}",
......
use anyhow::Result; use anyhow::{Context, Result};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{FileContent, FileSystemPath}; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
chunk::ChunkingContext, chunk::ChunkingContext,
...@@ -39,18 +39,11 @@ impl OutputAsset for StaticOutputAsset { ...@@ -39,18 +39,11 @@ impl OutputAsset for StaticOutputAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn path(&self) -> Result<Vc<FileSystemPath>> { async fn path(&self) -> Result<Vc<FileSystemPath>> {
let content = self.source.content(); let content = self.source.content();
let content_hash = if let AssetContent::File(file) = &*content.await? { let content_hash = content.content_hash().await?.context(
if let FileContent::Content(file) = &*file.await? { "Missing content when trying to generate the content hash for StaticOutputAsset",
turbo_tasks_hash::hash_xxh3_hash64(file.content()) )?;
} else {
anyhow::bail!("StaticAsset::path: not found")
}
} else {
anyhow::bail!("StaticAsset::path: unsupported file content")
};
let content_hash_b16 = turbo_tasks_hash::encode_hex(content_hash);
Ok(self.chunking_context.asset_path( Ok(self.chunking_context.asset_path(
content_hash_b16.into(), Vc::cell(content_hash),
self.source.ident(), self.source.ident(),
self.tag.clone(), self.tag.clone(),
)) ))
......
...@@ -9,10 +9,9 @@ ...@@ -9,10 +9,9 @@
#![feature(arbitrary_self_types)] #![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
use anyhow::Result; use anyhow::{Context, Result};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::Vc; use turbo_tasks::Vc;
use turbo_tasks_hash::hash_xxh3_hash64;
use turbopack_core::asset::Asset; use turbopack_core::asset::Asset;
pub(crate) mod analysis; pub(crate) mod analysis;
...@@ -24,8 +23,10 @@ pub mod source; ...@@ -24,8 +23,10 @@ pub mod source;
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn wasm_edge_var_name(asset: Vc<Box<dyn Asset>>) -> Result<Vc<RcStr>> { pub async fn wasm_edge_var_name(asset: Vc<Box<dyn Asset>>) -> Result<Vc<RcStr>> {
let content = asset.content().file_content().await?; let hash = asset
Ok(Vc::cell( .content()
format!("wasm_{:08x}", hash_xxh3_hash64(content)).into(), .content_hash()
)) .await?
.context("Missing content when trying to generate the content hash for a WASM asset")?;
Ok(Vc::cell(format!("wasm_{:08x}", hash).into()))
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment