Unverified Commit 0b3da788 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Adapter: STATIC_FILE.immutableHash (#90045)

Disabled for now

Expose a hash for each file that should be treated as immutable

For https://github.com/nextjs/adapter-vercel/pull/25

Part of PACK-6802
parent 4c6c31ad
......@@ -21,6 +21,7 @@ pub mod operation;
mod pages;
pub mod paths;
pub mod project;
pub mod project_asset_hashes_manifest;
pub mod route;
pub mod routes_hashes_manifest;
mod server_actions;
......
......@@ -1171,6 +1171,12 @@ impl Project {
Vc::cell(self.is_persistent_caching_enabled)
}
#[turbo_tasks::function]
pub(super) fn emit_client_hashes(&self) -> Vc<bool> {
// TODO
Vc::cell(false)
}
#[turbo_tasks::function]
pub(super) fn next_mode(&self) -> Vc<NextMode> {
*self.mode
......
use anyhow::Result;
use serde::{
Serializer,
ser::{Error, SerializeMap},
};
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexSet, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
use turbo_tasks_fs::{File, FileContent, FileSystemPath};
use turbo_tasks_hash::HashAlgorithm;
use turbopack_core::{
asset::{Asset, AssetContent},
output::{
ExpandOutputAssetsInput, OutputAsset, OutputAssets, OutputAssetsReference,
expand_output_assets,
},
};
use crate::{
project::Project,
route::{Endpoint, EndpointGroup, Endpoints},
};
#[turbo_tasks::value]
struct AssetHashesManifestAsset {
output_path: FileSystemPath,
project: ResolvedVc<Project>,
asset_root: FileSystemPath,
}
#[turbo_tasks::value_impl]
impl AssetHashesManifestAsset {
#[turbo_tasks::function]
pub fn new(
output_path: FileSystemPath,
project: ResolvedVc<Project>,
asset_root: FileSystemPath,
) -> Vc<Self> {
Self {
output_path,
project,
asset_root,
}
.cell()
}
}
#[turbo_tasks::value_impl]
impl OutputAssetsReference for AssetHashesManifestAsset {}
#[turbo_tasks::value_impl]
impl OutputAsset for AssetHashesManifestAsset {
#[turbo_tasks::function]
async fn path(&self) -> Vc<FileSystemPath> {
self.output_path.clone().cell()
}
}
#[turbo_tasks::function]
pub async fn endpoint_outputs(endpoint: Vc<Box<dyn Endpoint>>) -> Result<Vc<OutputAssets>> {
Ok(*endpoint.output().await?.output_assets)
}
#[turbo_tasks::function]
pub async fn endpoints_outputs(endpoints: Vc<Endpoints>) -> Result<Vc<OutputAssets>> {
let endpoints = endpoints.await?;
let all_outputs = endpoints
.iter()
.map(async |endpoint| endpoint.output().await?.output_assets.await)
.try_join()
.await?;
let set = all_outputs
.into_iter()
.flatten()
.copied()
.collect::<FxIndexSet<_>>();
Ok(Vc::cell(set.into_iter().collect()))
}
#[turbo_tasks::value(transparent)]
pub struct OutputAssetsWithPaths(Vec<(ResolvedVc<Box<dyn OutputAsset>>, RcStr)>);
#[turbo_tasks::function]
pub async fn expand_outputs(
project: Vc<Project>,
root: FileSystemPath,
) -> Result<Vc<OutputAssetsWithPaths>> {
let entrypoint_groups = project.get_all_endpoint_groups(false).await?;
let output_assets = entrypoint_groups
.iter()
.map(|(_, EndpointGroup { primary, .. })| {
if let &[entry] = &primary.as_slice() {
endpoint_outputs(*entry.endpoint)
} else {
let endpoints = Vc::cell(primary.iter().map(|entry| entry.endpoint).collect());
endpoints_outputs(endpoints)
}
})
.collect::<Vec<_>>();
let output_assets = expand_output_assets(
output_assets
.iter()
.try_join()
.await?
.into_iter()
.flatten()
.map(|asset| ExpandOutputAssetsInput::Asset(*asset)),
true,
)
.await?;
let mut output_assets = output_assets
.into_iter()
.map(async |asset| {
if let Some(path) = root.get_path_to(&*asset.path().await?) {
Ok(Some((asset, RcStr::from(path))))
} else {
Ok(None)
}
})
.try_flat_join()
.await?;
// Shared JS assets aren't duplicated here, but we have some duplicate OutputAssets with the
// same path, e.g. a static image which exists twice, once with the server and then also with
// the client chunking context.
output_assets.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
output_assets.dedup_by(|(_, a), (_, b)| a == b);
Ok(Vc::cell(output_assets))
}
#[turbo_tasks::value_impl]
impl Asset for AssetHashesManifestAsset {
#[turbo_tasks::function(root)]
async fn content(&self) -> Result<Vc<AssetContent>> {
let output_assets = expand_outputs(*self.project, self.asset_root.clone()).await?;
let asset_paths = output_assets
.into_iter()
.map(async |(asset, path)| {
Ok((
path,
asset.content_hash(HashAlgorithm::Xxh3Hash128Hex).await?,
))
})
.try_join()
.await?;
struct Manifest<'a> {
asset_paths: &'a Vec<(&'a RcStr, ReadRef<Option<RcStr>>)>,
}
impl serde::Serialize for Manifest<'_> {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut map = serializer.serialize_map(Some(self.asset_paths.len()))?;
for (path, content_hash) in self.asset_paths {
map.serialize_entry(
path,
if let Some(content_hash) = content_hash.as_ref() {
content_hash
} else {
return Err(S::Error::custom("asset content hash failed"));
},
)?;
}
map.end()
}
}
let json = serde_json::to_string(&Manifest {
asset_paths: &asset_paths,
})?;
Ok(AssetContent::file(
FileContent::Content(File::from(json)).cell(),
))
}
}
#[turbo_tasks::function]
pub async fn immutable_hashes_manifest_asset_if_enabled(
project: ResolvedVc<Project>,
) -> Result<Vc<OutputAssets>> {
if *project.emit_client_hashes().await? {
let path = project
.node_root()
.await?
.join("immutable-static-hashes.json")?;
let asset = AssetHashesManifestAsset::new(
path,
*project,
project.client_relative_path().owned().await?,
)
.to_resolved()
.await?;
Ok(Vc::cell(vec![ResolvedVc::upcast(asset)]))
} else {
Ok(OutputAssets::empty())
}
}
......@@ -28,6 +28,7 @@ use next_api::{
DebugBuildPaths, DefineEnv, DraftModeOptions, HmrTarget, PartialProjectOptions, Project,
ProjectContainer, ProjectOptions, WatchOptions,
},
project_asset_hashes_manifest::immutable_hashes_manifest_asset_if_enabled,
route::{Endpoint, EndpointGroupKey, Route},
routes_hashes_manifest::routes_hashes_manifest_asset_if_enabled,
};
......@@ -1680,6 +1681,8 @@ async fn output_assets_operation(
let whole_app_module_graphs = project.whole_app_module_graphs();
let nft = next_server_nft_assets(project).await?;
let routes_hashes_manifest = routes_hashes_manifest_asset_if_enabled(project).await?;
let immutable_hashes_manifest_asset =
immutable_hashes_manifest_asset_if_enabled(project).await?;
whole_app_module_graphs.as_side_effect().await?;
......@@ -1688,6 +1691,7 @@ async fn output_assets_operation(
.into_iter()
.chain(nft.iter().copied())
.chain(routes_hashes_manifest.iter().copied())
.chain(immutable_hashes_manifest_asset.iter().copied())
.collect(),
))
}
......
......@@ -253,6 +253,11 @@ export interface AdapterOutput {
filePath: string
pathname: string
type: AdapterOutputType.STATIC_FILE
/**
* If this static file is immutable (because its filename contains a content hash), then this
* field contains the untruncated content hash.
*/
immutableHash: string | undefined
}
/**
......@@ -478,19 +483,32 @@ export async function handleBuildComplete({
pathname,
filePath: path.join(configOutDir, file),
type: AdapterOutputType.STATIC_FILE,
immutableHash: undefined,
} satisfies AdapterOutput['STATIC_FILE'])
}
} else {
const staticFiles = await recursiveReadDir(path.join(distDir, 'static'))
const clientHashes: Record<string, string> | undefined =
bundler === Bundler.Turbopack && config.experimental.immutableAssetToken
? JSON.parse(
await fs.readFile(
path.join(distDir, 'immutable-static-hashes.json'),
'utf8'
)
)
: undefined
for (const file of staticFiles) {
const pathname = path.posix.join('/_next/static', file)
const filePath = path.join(distDir, 'static', file)
const id = path.join('static', file)
outputs.staticFiles.push({
type: AdapterOutputType.STATIC_FILE,
id: path.join('static', file),
id,
pathname,
filePath,
immutableHash: clientHashes?.[id],
})
}
......@@ -838,6 +856,7 @@ export async function handleBuildComplete({
pagesDistDir,
`${normalizePagePath(localePage)}.html`
),
immutableHash: undefined,
} satisfies AdapterOutput['STATIC_FILE']
outputs.staticFiles.push(localeOutput)
......@@ -848,6 +867,7 @@ export async function handleBuildComplete({
pathname: `${localePage}.rsc`,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
}
......@@ -857,6 +877,7 @@ export async function handleBuildComplete({
pathname: route,
type: AdapterOutputType.STATIC_FILE,
filePath: pageFile.replace(/\.js$/, '.html'),
immutableHash: undefined,
} satisfies AdapterOutput['STATIC_FILE']
outputs.staticFiles.push(staticOutput)
......@@ -867,6 +888,7 @@ export async function handleBuildComplete({
pathname: `${route}.rsc`,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
}
......@@ -929,6 +951,7 @@ export async function handleBuildComplete({
pathname: rscPage,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
}
......@@ -960,6 +983,7 @@ export async function handleBuildComplete({
pathname: `${localePage}.rsc`,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
}
......@@ -1321,6 +1345,7 @@ export async function handleBuildComplete({
pathname: route,
type: AdapterOutputType.STATIC_FILE,
filePath: staticMetadataFilePath,
immutableHash: undefined,
})
continue
}
......@@ -1445,6 +1470,7 @@ export async function handleBuildComplete({
pathname: rscPage,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
......@@ -1632,6 +1658,7 @@ export async function handleBuildComplete({
pathname: rscPage,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
......@@ -1739,6 +1766,7 @@ export async function handleBuildComplete({
pathname: rscPage,
type: AdapterOutputType.STATIC_FILE,
filePath: rscFallbackPath,
immutableHash: undefined,
})
}
......@@ -1793,6 +1821,7 @@ export async function handleBuildComplete({
id: currentDocPath,
type: AdapterOutputType.STATIC_FILE,
filePath: currentFilePath,
immutableHash: undefined,
})
}
}
......
......@@ -23,6 +23,7 @@ import { Telemetry } from '../../telemetry/storage'
import { eventAnalyzeCompleted } from '../../telemetry/events'
import { traceGlobals } from '../../trace/shared'
import type { RoutesManifest } from '..'
import { Bundler } from '../../lib/bundler'
export type AnalyzeOptions = {
dir: string
......@@ -45,6 +46,7 @@ export default async function analyze({
const config: NextConfigComplete = await loadConfig(PHASE_ANALYZE, dir, {
silent: false,
reactProductionProfiling,
bundler: Bundler.Turbopack,
})
process.env.NEXT_DEPLOYMENT_ID = config.deploymentId || ''
......
......@@ -975,6 +975,7 @@ export default async function build(
({ key: a }, { key: b }) => a.localeCompare(b)
)
},
bundler,
}),
turborepoAccessTraceResult
)
......
......@@ -26,6 +26,7 @@ import type {
RawEntrypoints,
TurbopackResult,
} from '../swc/types'
import { Bundler } from '../../lib/bundler'
export async function turbopackBuild(): Promise<{
duration: number
......@@ -130,6 +131,7 @@ export async function turbopackBuild(): Promise<{
debugPrerender: NextBuildContext.debugPrerender,
reactProductionProfiling:
NextBuildContext.reactProductionProfiling,
bundler: Bundler.Turbopack,
})
await workerConfig.experimental.onBeforeDeferredEntries?.()
......@@ -279,6 +281,7 @@ export async function workerMain(workerData: {
{
debugPrerender: NextBuildContext.debugPrerender,
reactProductionProfiling: NextBuildContext.reactProductionProfiling,
bundler: Bundler.Turbopack,
}
)
NextBuildContext.config = config
......
......@@ -44,6 +44,7 @@ import origDebug from 'next/dist/compiled/debug'
import { Telemetry } from '../../telemetry/storage'
import { durationToString, hrtimeToSeconds } from '../duration-to-string'
import { installBindings } from '../swc/install-bindings'
import { Bundler } from '../../lib/bundler'
const debug = origDebug('next:build:webpack-build')
......@@ -425,6 +426,7 @@ export async function workerMain(workerData: {
{
debugPrerender: NextBuildContext.debugPrerender,
reactProductionProfiling: NextBuildContext.reactProductionProfiling,
bundler: process.env.NEXT_RSPACK ? Bundler.Rspack : Bundler.Webpack,
}
))
await installBindings(config.experimental?.useWasmBinary)
......
......@@ -382,6 +382,7 @@ export const experimentalSchema = {
lockDistDir: z.boolean().optional(),
hideLogsAfterAbort: z.boolean().optional(),
runtimeServerDeploymentId: z.boolean().optional(),
immutableAssetToken: z.string().optional(),
devCacheControlNoCache: z.boolean().optional(),
deferredEntries: z.array(z.string()).optional(),
onBeforeDeferredEntries: z.function().returns(z.promise(z.void())).optional(),
......
......@@ -952,6 +952,12 @@ export interface ExperimentalConfig {
*/
runtimeServerDeploymentId?: boolean
/**
* A different token to use for static assets (as opposed to config.deploymentId) which
* doesn't have to be unique per deployment.
*/
immutableAssetToken?: string
/**
* Use 'no-cache' instead of 'no-store' in the Cache-Control header for development.
* This allows conditional requests to the server, which can help with development
......@@ -1811,6 +1817,7 @@ export interface NextConfigRuntime {
| 'maxPostponedStateSize'
| 'devCacheControlNoCache'
| 'exposeTestingApiInProductionBuild'
| 'immutableAssetToken'
> & {
// Pick on @internal fields generates invalid .d.ts files
/** @internal */
......@@ -1875,6 +1882,7 @@ export function getNextConfigRuntime(
maxPostponedStateSize: ex.maxPostponedStateSize,
devCacheControlNoCache: ex.devCacheControlNoCache,
exposeTestingApiInProductionBuild: ex.exposeTestingApiInProductionBuild,
immutableAssetToken: ex.immutableAssetToken,
trustHostHeader: ex.trustHostHeader,
isExperimentalCompile: ex.isExperimentalCompile,
......
......@@ -45,6 +45,7 @@ import { djb2Hash } from '../shared/lib/hash'
import type { NextAdapter } from '../build/adapter/build-complete'
import { HardDeprecatedConfigError } from '../shared/lib/errors/hard-deprecated-config-error'
import { NextInstanceErrorState } from './mcp/tools/next-instance-error-state'
import { Bundler } from '../lib/bundler'
export { normalizeConfig } from './config-shared'
export type { DomainLocale, NextConfig } from './config-shared'
......@@ -1478,6 +1479,7 @@ type LoadConfigOptions = {
) => void
reactProductionProfiling?: boolean
debugPrerender?: boolean
bundler?: Bundler
}
export default async function loadConfig(
......@@ -1506,6 +1508,7 @@ export default async function loadConfig(
reportExperimentalFeatures,
reactProductionProfiling,
debugPrerender,
bundler,
}: LoadConfigOptions = {}
): Promise<NextConfigComplete> {
// Generate cache key based on parameters that affect config output
......@@ -1726,6 +1729,16 @@ export default async function loadConfig(
)
}
if (
// TODO enable once everything is merged
(true || bundler !== Bundler.Turbopack) &&
userConfig.experimental?.immutableAssetToken
) {
// Silently ignore that flag for Webpack/Rspack since the server code assumes that all files
// in `static/chunks` are always immutable without checking the manifest.
userConfig.experimental.immutableAssetToken = undefined
}
if (reactProductionProfiling) {
userConfig.reactProductionProfiling = reactProductionProfiling
}
......
......@@ -20,6 +20,12 @@ pub trait Asset {
fn versioned_content(self: Vc<Self>) -> Result<Vc<Box<dyn VersionedContent>>> {
Ok(Vc::upcast(VersionedAssetContent::new(self.content())))
}
/// Hash of the content of the [Asset]
#[turbo_tasks::function]
fn content_hash(self: Vc<Self>, algorithm: HashAlgorithm) -> Vc<Option<RcStr>> {
self.content().content_hash(algorithm)
}
}
#[turbo_tasks::value(shared)]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment