Unverified Commit 505771a2 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Use Next.js version as Turbopack persistent cache versioning key (#93605)

### What?

The Turbopack persistent cache directory under
`.next/cache/turbopack/<version>/` is now versioned by the Next.js
package version concatenated with the git short SHA, instead of `git
describe`.

Example: `v16.0.1-canary.13-94e9fa67`

### Why?

The previous version key came from `VERGEN_GIT_DESCRIBE` (`git describe
--match 'v[0-9]' --dirty`), which depends on having an annotated `v*`
tag reachable from the build commit. That ties the cache version to
local git tag state and breaks down for builds against forks, shallow
clones, or branches without a recent matching tag.

The Next.js package version from `packages/next/package.json` is the
value users actually care about for cache compatibility, and it is
already plumbed through to the Rust side as
`ProjectOptions::next_version` (sourced from
`process.env.__NEXT_VERSION`). Combining it with the git short SHA keeps
per-commit cache uniqueness while removing the dependency on git tags.

### How?

**Rust (`crates/next-napi-bindings`)**

- `build.rs`: also emit `VERGEN_GIT_SHA` (short form) via
`vergen_gitcl`. `VERGEN_GIT_DESCRIBE` is still emitted and is still used
for the bug-report URL / panic log.
- `src/next_api/turbopack_ctx.rs`:
- New `cache_describe(next_version)` returns
`format!("v{next_version}-{}", env!("VERGEN_GIT_SHA"))`.
- `git_version_info` now takes the pre-built `describe` string instead
of reading `VERGEN_GIT_DESCRIBE` itself. `dirty` handling is unchanged
(still derived from `VERGEN_GIT_DIRTY`, still suppressed in CI), so
`handle_db_versioning`'s dirty-repo behavior is preserved.
- `create_turbo_tasks` takes `next_version: &str` and threads it
through.
- `src/next_api/project.rs`:
- `project_new` passes `options.next_version` into `create_turbo_tasks`.
No new field is needed on `NapiProjectOptions` — the `nextVersion` field
already exists and is already populated by all three call sites.
- `turbopack_database_compact` (the napi `databaseCompact` function used
by `next-post-build`) gains a `next_version: String` parameter so
post-build compaction targets the same versioned directory.

**JS (`packages/next`)**

- `src/build/swc/types.ts`, `generated-native.d.ts`, `index.ts`: update
the `databaseCompact(path, nextVersion)` signature on both the native
and WASM-fallback paths.
- `src/cli/next-post-build.ts`: passes `process.env.__NEXT_VERSION`
through to `bindings.turbo.databaseCompact`.

`turbopack/crates/turbopack-cli` (the standalone Turbopack CLI) is
intentionally left on `VERGEN_GIT_DESCRIBE` — it is not tied to a
Next.js package version.

---------
Co-authored-by: 's avatarv-work-app[bot] <262237222+v-work-app[bot]@users.noreply.github.com>
Co-authored-by: 's avatarClaude <noreply@anthropic.com>
Co-authored-by: 's avatarTobias Koppers <sokra@users.noreply.github.com>
parent 766b7cbf
...@@ -5,7 +5,6 @@ use serde_json::Value; ...@@ -5,7 +5,6 @@ use serde_json::Value;
fn main() -> anyhow::Result<()> { fn main() -> anyhow::Result<()> {
println!("cargo:rerun-if-env-changed=CI"); println!("cargo:rerun-if-env-changed=CI");
println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS"); println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS");
let is_ci = env::var("CI").is_ok_and(|value| !value.is_empty());
let is_macos_target = env::var("CARGO_CFG_TARGET_OS").is_ok_and(|value| value == "macos"); let is_macos_target = env::var("CARGO_CFG_TARGET_OS").is_ok_and(|value| value == "macos");
let nextjs_version = { let nextjs_version = {
...@@ -50,11 +49,7 @@ fn main() -> anyhow::Result<()> { ...@@ -50,11 +49,7 @@ fn main() -> anyhow::Result<()> {
// commit hash as a version is okay. // commit hash as a version is okay.
let git = vergen_gitcl::GitclBuilder::default() let git = vergen_gitcl::GitclBuilder::default()
.dirty(/* include_untracked */ true) .dirty(/* include_untracked */ true)
.describe( .sha(/* short */ true)
/* tags */ true,
/* dirty */ !is_ci, // suppress the dirty suffix in CI
/* matches */ Some("v[0-9]*"), // find the last version tag
)
.build()?; .build()?;
vergen_gitcl::Emitter::default() vergen_gitcl::Emitter::default()
.fail_on_error() .fail_on_error()
......
...@@ -568,6 +568,7 @@ pub fn project_new( ...@@ -568,6 +568,7 @@ pub fn project_new(
let skip_compaction = turbo_engine_options.skip_compaction.unwrap_or(false); let skip_compaction = turbo_engine_options.skip_compaction.unwrap_or(false);
let turbo_tasks = create_turbo_tasks( let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir), PathBuf::from(&options.dist_dir),
&options.next_version,
options.is_persistent_caching_enabled, options.is_persistent_caching_enabled,
memory_limit, memory_limit,
dependency_tracking, dependency_tracking,
...@@ -2594,8 +2595,9 @@ pub async fn project_get_all_compilation_issues( ...@@ -2594,8 +2595,9 @@ pub async fn project_get_all_compilation_issues(
/// ///
/// The `path` should point to the `<distDir>/cache/turbopack` directory. /// The `path` should point to the `<distDir>/cache/turbopack` directory.
#[napi] #[napi]
pub async fn turbopack_database_compact(path: String) -> napi::Result<()> { pub async fn turbopack_database_compact(path: String, next_version: String) -> napi::Result<()> {
let version_info = crate::next_api::turbopack_ctx::git_version_info(); let describe = crate::next_api::turbopack_ctx::cache_describe(&next_version);
let version_info = crate::next_api::turbopack_ctx::git_version_info(&describe);
let is_ci = std::env::var("CI").is_ok_and(|v| !v.is_empty()); let is_ci = std::env::var("CI").is_ok_and(|v| !v.is_empty());
turbo_tasks_backend::compact_database(&PathBuf::from(path), &version_info, is_ci) turbo_tasks_backend::compact_database(&PathBuf::from(path), &version_info, is_ci)
.map_err(|e| napi::Error::from_reason(format!("Database compaction failed: {e}")))?; .map_err(|e| napi::Error::from_reason(format!("Database compaction failed: {e}")))?;
......
...@@ -207,13 +207,19 @@ impl NapiNextTurbopackCallbacks { ...@@ -207,13 +207,19 @@ impl NapiNextTurbopackCallbacks {
} }
} }
/// Returns version info derived from compile-time git metadata. /// Returns the cache version `describe` string for the given Next.js version, of the form
/// `v<next_version>-<git_short_sha>` (e.g. `v16.0.1-canary.13-94e9fa6`).
pub fn cache_describe(next_version: &str) -> String {
format!("v{next_version}-{}", env!("VERGEN_GIT_SHA"))
}
/// Returns version info derived from the supplied Next.js version and compile-time git metadata.
/// ///
/// The `dirty` flag is only set when not running in CI (`CI` env var unset at build time) and the /// The `dirty` flag is only set when not running in CI (`CI` env var unset at build time) and the
/// working tree was dirty at build time. /// working tree was dirty at build time.
pub fn git_version_info() -> GitVersionInfo<'static> { pub fn git_version_info(describe: &str) -> GitVersionInfo<'_> {
GitVersionInfo { GitVersionInfo {
describe: env!("VERGEN_GIT_DESCRIBE"), describe,
dirty: option_env!("CI").is_none_or(|value| value.is_empty()) dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true", && env!("VERGEN_GIT_DIRTY") == "true",
} }
...@@ -221,6 +227,7 @@ pub fn git_version_info() -> GitVersionInfo<'static> { ...@@ -221,6 +227,7 @@ pub fn git_version_info() -> GitVersionInfo<'static> {
pub fn create_turbo_tasks( pub fn create_turbo_tasks(
output_path: PathBuf, output_path: PathBuf,
next_version: &str,
persistent_caching: bool, persistent_caching: bool,
_memory_limit: usize, _memory_limit: usize,
dependency_tracking: bool, dependency_tracking: bool,
...@@ -229,7 +236,8 @@ pub fn create_turbo_tasks( ...@@ -229,7 +236,8 @@ pub fn create_turbo_tasks(
skip_compaction: bool, skip_compaction: bool,
) -> Result<NextTurboTasks> { ) -> Result<NextTurboTasks> {
Ok(if persistent_caching { Ok(if persistent_caching {
let version_info = git_version_info(); let describe = cache_describe(next_version);
let version_info = git_version_info(&describe);
let (backing_storage, cache_state) = turbo_backing_storage( let (backing_storage, cache_state) = turbo_backing_storage(
&output_path.join("cache/turbopack"), &output_path.join("cache/turbopack"),
&version_info, &version_info,
...@@ -393,7 +401,7 @@ pub fn log_internal_error_and_inform(internal_error: &anyhow::Error) { ...@@ -393,7 +401,7 @@ pub fn log_internal_error_and_inform(internal_error: &anyhow::Error) {
); );
let version_str = format!( let version_str = format!(
"Turbopack version: `{}`\nNext.js version: `{}`", "Turbopack version: `{}`\nNext.js version: `{}`",
env!("VERGEN_GIT_DESCRIBE"), env!("VERGEN_GIT_SHA"),
env!("NEXTJS_VERSION") env!("NEXTJS_VERSION")
); );
let bug_report_url = format!( let bug_report_url = format!(
......
...@@ -491,7 +491,10 @@ export declare function projectGetAllCompilationIssues(project: { ...@@ -491,7 +491,10 @@ export declare function projectGetAllCompilationIssues(project: {
* *
* The `path` should point to the `<distDir>/cache/turbopack` directory. * The `path` should point to the `<distDir>/cache/turbopack` directory.
*/ */
export declare function turbopackDatabaseCompact(path: string): Promise<void> export declare function turbopackDatabaseCompact(
path: string,
nextVersion: string
): Promise<void>
/** /**
* A version of [`NapiNextTurbopackCallbacks`] that can accepted as an argument to a napi function. * A version of [`NapiNextTurbopackCallbacks`] that can accepted as an argument to a napi function.
* *
......
...@@ -1410,7 +1410,7 @@ async function loadWasm(importPath = '') { ...@@ -1410,7 +1410,7 @@ async function loadWasm(importPath = '') {
`Only WebAssembly (WASM) bindings were loaded, and Turbopack requires native bindings.` `Only WebAssembly (WASM) bindings were loaded, and Turbopack requires native bindings.`
) )
}, },
databaseCompact(_path: string): Promise<void> { databaseCompact(_path: string, _nextVersion: string): Promise<void> {
throw new Error( throw new Error(
'Turbopack database compaction is not supported on this platform' 'Turbopack database compaction is not supported on this platform'
) )
...@@ -1659,8 +1659,11 @@ function loadNative(importPath?: string): Binding { ...@@ -1659,8 +1659,11 @@ function loadNative(importPath?: string): Binding {
queryTraceSpans(handle, options) { queryTraceSpans(handle, options) {
return (customBindings ?? bindings).queryTraceSpans(handle, options) return (customBindings ?? bindings).queryTraceSpans(handle, options)
}, },
databaseCompact(dbPath: string) { databaseCompact(dbPath: string, dbNextVersion: string) {
return (customBindings ?? bindings).turbopackDatabaseCompact(dbPath) return (customBindings ?? bindings).turbopackDatabaseCompact(
dbPath,
dbNextVersion
)
}, },
}, },
mdx: { mdx: {
......
...@@ -40,7 +40,7 @@ export interface Binding { ...@@ -40,7 +40,7 @@ export interface Binding {
handle: TraceServerHandle, handle: TraceServerHandle,
options: TraceQueryOptions options: TraceQueryOptions
): TraceQueryResult ): TraceQueryResult
databaseCompact(path: string): Promise<void> databaseCompact(path: string, nextVersion: string): Promise<void>
nextBuild?: any nextBuild?: any
} }
......
...@@ -38,7 +38,10 @@ const nextPostBuild = async ( ...@@ -38,7 +38,10 @@ const nextPostBuild = async (
} }
const bindings = await loadBindings(config.experimental?.useWasmBinary) const bindings = await loadBindings(config.experimental?.useWasmBinary)
await bindings.turbo.databaseCompact(cachePath) await bindings.turbo.databaseCompact(
cachePath,
process.env.__NEXT_VERSION as string
)
console.log('Turbopack database compaction complete.') console.log('Turbopack database compaction complete.')
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment