Unverified Commit 190df615 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and next-api…

fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and next-api crates to fix stale git lock files (#76773)

Both `shadow-rs` and `vergen-gitcl` call out to the git CLI, which takes optional locks to update the index: https://git-scm.com/docs/git-status#_background_refresh

This is why we sometimes find ourselves with stale `.git/index.lock` files.

Using a lightly patched version of https://github.com/martinpitt/fatrace (to look up parent process names) shows this:

![Screenshot 2025-03-03 at 5.39.53 PM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/HAZVitxRNnZz8QMiPn4a/64a88f82-47bc-4d80-b49a-c5349e946411.png)

`shadow-rs` has a `git2` feature (enabled by default), but that doesn't prevent it from running the git CLI. Instead it runs *both* and overwrites some values.

`vergen-git2` only uses `libgit2` and does not invoke the CLI. This fixes things because `libgit2` disables `GIT_STATUS_OPT_UPDATE_INDEX` by default.

Additionally, after looking through both codebases, I've decided I like `vergen`'s implementation a bit more than `shadow-rs`'s.

Supposedly, `git`'s command-line should support a `GIT_OPTIONAL_LOCKS=0` environment variable to disable this behavior, but I'm still finding the lock written to when trying to use that. I'm still investigating if I can use something like this to contribute a proper fix upstream to these crates.
parent 48c58434
This diff is collapsed.
......@@ -426,6 +426,8 @@ unicode-segmentation = "1.10.1"
unsize = "1.1.0"
url = "2.2.2"
urlencoding = "2.1.2"
vergen = { version = "9.0.4", features = ["cargo"] }
vergen-git2 = { version = "1.0.5", features = ["cargo"] }
webbrowser = "0.8.7"
[patch.crates-io]
......
......@@ -66,14 +66,13 @@ rand = { workspace = true }
rustc-hash = { workspace = true }
serde = "1"
serde_json = "1"
shadow-rs = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
tracing-chrome = "0.5.0"
url = { workspace = true }
urlencoding = { workspace = true }
once_cell = { workspace = true }
dashmap = "6.1.0"
dashmap = { workspace = true }
swc_core = { workspace = true, features = [
"base_concurrent",
......@@ -139,11 +138,11 @@ turbo-tasks-malloc = { workspace = true, default-features = false }
tokio = { workspace = true, features = ["full"] }
[build-dependencies]
anyhow = { workspace = true }
napi-build = "2"
serde = { workspace = true }
serde_json = { workspace = true }
# It is not a mistake this dependency is specified in dep / build-dep both.
shadow-rs = { workspace = true }
vergen-git2 = { workspace = true }
# build-dependencies for the native, non-wasm32 build
[target.'cfg(not(target_arch = "wasm32"))'.build-dependencies]
......
use std::fs;
use std::{env, process::Command, str};
extern crate napi_build;
fn main() {
fn main() -> anyhow::Result<()> {
println!("cargo:rerun-if-env-changed=CI");
let is_ci = env::var("CI").is_ok_and(|value| !value.is_empty());
// Generates, stores build-time information as static values.
// There are some places relying on correct values for this (i.e telemetry),
// So failing build if this fails.
shadow_rs::ShadowBuilder::builder()
.build()
.expect("Should able to generate build time information");
let cargo = vergen_git2::CargoBuilder::default()
.target_triple(true)
.build()?;
// We use the git dirty state to disable persistent caching (persistent caching relies on a
// commit hash to be safe). One tradeoff of this is that we must invalidate the rust build more
// often.
//
// This invalidates the build if any untracked files change. That's sufficient for the case
// where we transition from dirty to clean.
//
// There's an edge-case here where the repository could be newly dirty, but we can't know
// because our build hasn't been invalidated, since the untracked files weren't untracked last
// time we ran. That will cause us to incorrectly report ourselves as clean.
//
// However, in practice that shouldn't be much of an issue: If no other dependency of this
// top-level crate has changed (which would've triggered our rebuild), then the resulting binary
// must be equivalent to a clean build anyways. Therefore, persistent caching using the HEAD
// commit hash as a version is okay.
let git = vergen_git2::Git2Builder::default()
.dirty(/* include_untracked */ true)
.describe(
/* tags */ true,
/* dirty */ !is_ci, // suppress the dirty suffix in CI
/* matches */ Some("v[0-9]*"), // find the last version tag
)
.build()?;
vergen_git2::Emitter::default()
.add_instructions(&cargo)?
.add_instructions(&git)?
.fail_on_error()
.emit()?;
let git_head = fs::read_to_string("../../.git/HEAD").unwrap_or_default();
if !git_head.is_empty() && !git_head.starts_with("ref: ") {
println!("cargo:warning=git version {}", git_head);
match Command::new("git").args(["rev-parse", "HEAD"]).output() {
Ok(out) if out.status.success() => println!(
"cargo:warning=git HEAD: {}",
str::from_utf8(&out.stdout).unwrap()
),
_ => println!("cargo:warning=`git rev-parse HEAD` failed"),
}
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
......@@ -36,4 +70,6 @@ fn main() {
#[cfg(not(target_arch = "wasm32"))]
turbo_tasks_build::generate_register();
Ok(())
}
......@@ -61,9 +61,6 @@ pub mod turbo_trace_server;
pub mod turbopack;
pub mod util;
// Declare build-time information variables generated in build.rs
shadow_rs::shadow!(build);
#[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
#[global_allocator]
static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
......
......@@ -13,7 +13,8 @@ use turbo_tasks::{
TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc,
};
use turbo_tasks_backend::{
default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage,
default_backing_storage, noop_backing_storage, DefaultBackingStorage, GitVersionInfo,
NoopBackingStorage,
};
use turbo_tasks_fs::FileContent;
use turbopack_core::{
......@@ -130,26 +131,10 @@ pub fn create_turbo_tasks(
dependency_tracking: bool,
) -> Result<NextTurboTasks> {
Ok(if persistent_caching {
let dirty_suffix = if crate::build::GIT_CLEAN
|| option_env!("CI").is_some_and(|value| !value.is_empty())
{
""
} else {
"-dirty"
};
#[allow(
clippy::const_is_empty,
reason = "LAST_TAG might be empty if the tag can't be determined"
)]
let version_info = if crate::build::LAST_TAG.is_empty() {
format!("{}{}", crate::build::SHORT_COMMIT, dirty_suffix)
} else {
format!(
"{}-{}{}",
crate::build::LAST_TAG,
crate::build::SHORT_COMMIT,
dirty_suffix
)
let version_info = GitVersionInfo {
describe: env!("VERGEN_GIT_DESCRIBE"),
dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true",
};
NextTurboTasks::PersistentCaching(TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new(
......
......@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) {
}
#[napi]
pub fn get_target_triple() -> String {
crate::build::BUILD_TARGET.to_string()
pub fn get_target_triple() -> &'static str {
env!("VERGEN_CARGO_TARGET_TRIPLE")
}
pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {
......
......@@ -24,7 +24,6 @@ regex = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
shadow-rs = { workspace = true }
swc_core = { workspace = true }
tracing = { workspace = true }
turbo-rcstr = { workspace = true }
......@@ -43,6 +42,6 @@ turbopack-node = { workspace = true }
turbopack-nodejs = { workspace = true }
[build-dependencies]
# It is not a mistake this dependency is specified in dep / build-dep both.
shadow-rs = { workspace = true }
anyhow = { workspace = true }
turbo-tasks-build = { workspace = true }
vergen = { workspace = true }
use turbo_tasks_build::generate_register;
fn main() {
fn main() -> anyhow::Result<()> {
// Generates, stores build-time information as static values.
// There are some places relying on correct values for this (i.e telemetry),
// So failing build if this fails.
shadow_rs::ShadowBuilder::builder()
.build_pattern(shadow_rs::BuildPattern::Lazy)
.build()
.expect("Should able to generate build time information");
let cargo = vergen::CargoBuilder::default()
.target_triple(true)
.build()?;
vergen::Emitter::default()
.add_instructions(&cargo)?
.fail_on_error()
.emit()?;
generate_register();
Ok(())
}
......@@ -23,9 +23,6 @@ mod server_actions;
mod versioned_content_map;
mod webpack_stats;
// Declare build-time information variables generated in build.rs
shadow_rs::shadow!(build);
pub fn register() {
next_core::register();
turbopack_nodejs::register();
......
......@@ -68,7 +68,6 @@ use turbopack_nodejs::NodeJsChunkingContext;
use crate::{
app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME},
build,
empty::EmptyEndpoint,
entrypoints::Entrypoints,
instrumentation::InstrumentationEndpoint,
......@@ -1069,7 +1068,7 @@ impl Project {
// First, emit an event for the binary target triple.
// This is different to webpack-config; when this is being called,
// it is always using SWC so we don't check swc here.
emit_event(build::BUILD_TARGET, true);
emit_event(env!("VERGEN_CARGO_TARGET_TRIPLE"), true);
// Go over jsconfig and report enabled features.
let compiler_options = self.js_config().compiler_options().await?;
......
......@@ -16,25 +16,5 @@ workspace = true
next-core = { workspace = true }
turbopack-core = { workspace = true }
#turbopack-binding = { workspace = true, features = [
# "__turbo_tasks",
# "__turbo_tasks_memory",
# "__turbo_tasks_env",
# "__turbo_tasks_fs",
# "__turbo_tasks_memory",
# "__turbopack",
# "__turbopack_nodejs",
# "__turbopack_core",
# "__turbopack_browser",
# "__turbopack_ecmascript",
# "__turbopack_ecmascript_runtime",
# "__turbopack_env",
# "__turbopack_node",
#] }
[build-dependencies]
turbo-tasks-build = { workspace = true }
vergen = { version = "7.3.2", default-features = false, features = [
"cargo",
"build",
] }
use turbo_tasks_build::generate_register;
use vergen::{vergen, Config};
fn main() {
generate_register();
// Attempt to collect some build time env values but will skip if there are any
// errors.
let _ = vergen(Config::default());
}
......@@ -7,24 +7,31 @@ use std::{
use anyhow::Result;
/// Information gathered by `vergen_git2` in the top-level binary crate and passed down. This
/// information must be computed in the top-level crate for cargo incremental compilation to work
/// correctly.
///
/// See `crates/napi/build.rs` for details.
pub struct GitVersionInfo<'a> {
/// Output of `git describe --match 'v[0-9]' --dirty`.
pub describe: &'a str,
/// Is the git repository dirty? Always forced to `false` when the `CI` environment variable is
/// set and non-empty.
pub dirty: bool,
}
/// Specifies many databases that have a different version than the current one are retained.
/// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory,
/// the current one and two older/newer ones.
const MAX_OTHER_DB_VERSIONS: usize = 2;
pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<PathBuf> {
pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> Result<PathBuf> {
if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
return Ok(base_path.join(version));
}
// Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a
// dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable
// versioning and always use the same database.
let (version_info, git_dirty) = if let Some(version_info) = version_info.strip_suffix("-dirty")
{
(version_info, true)
} else {
(version_info, false)
};
let ignore_dirty = env::var("TURBO_ENGINE_IGNORE_DIRTY").ok().is_some();
let disabled_versioning = env::var("TURBO_ENGINE_DISABLE_VERSIONING").ok().is_some();
let version = if disabled_versioning {
......@@ -33,14 +40,14 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<Path
caching database might be required."
);
Some("unversioned")
} else if !git_dirty {
Some(version_info)
} else if !version_info.dirty {
Some(version_info.describe)
} else if ignore_dirty {
println!(
"WARNING: The git repository is dirty, but Persistent Caching is still enabled. \
Manual removal of the persistent caching database might be required."
);
Some(version_info)
Some(version_info.describe)
} else {
println!(
"WARNING: The git repository is dirty: Persistent Caching is disabled. Use \
......
......@@ -15,13 +15,14 @@ use std::path::Path;
use anyhow::Result;
pub use self::{
backend::{BackendOptions, StorageMode, TurboTasksBackend},
kv_backing_storage::KeyValueDatabaseBackingStorage,
};
use crate::database::{
db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
};
pub use crate::{
backend::{BackendOptions, StorageMode, TurboTasksBackend},
database::db_versioning::GitVersionInfo,
kv_backing_storage::KeyValueDatabaseBackingStorage,
};
#[cfg(feature = "lmdb")]
pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
......@@ -35,7 +36,10 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
>;
#[cfg(feature = "lmdb")]
pub fn lmdb_backing_storage(path: &Path, version_info: &str) -> Result<LmdbBackingStorage> {
pub fn lmdb_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<LmdbBackingStorage> {
use crate::database::{
fresh_db_optimization::{is_fresh, FreshDbOptimization},
read_transaction_cache::ReadTransactionCache,
......@@ -53,7 +57,10 @@ pub fn lmdb_backing_storage(path: &Path, version_info: &str) -> Result<LmdbBacki
pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
pub fn turbo_backing_storage(path: &Path, version_info: &str) -> Result<TurboBackingStorage> {
pub fn turbo_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<TurboBackingStorage> {
let path = handle_db_versioning(path, version_info)?;
let database = TurboKeyValueDatabase::new(path)?;
Ok(KeyValueDatabaseBackingStorage::new(database))
......@@ -69,7 +76,10 @@ pub fn noop_backing_storage() -> NoopBackingStorage {
pub type DefaultBackingStorage = LmdbBackingStorage;
#[cfg(feature = "lmdb")]
pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<DefaultBackingStorage> {
lmdb_backing_storage(path, version_info)
}
......@@ -77,6 +87,9 @@ pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<Defaul
pub type DefaultBackingStorage = TurboBackingStorage;
#[cfg(not(feature = "lmdb"))]
pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<DefaultBackingStorage> {
turbo_backing_storage(path, version_info)
}
......@@ -12,7 +12,10 @@
turbo_tasks_backend::BackendOptions::default(),
turbo_tasks_backend::default_backing_storage(
path.as_path(),
"test"
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
dirty: false,
},
).unwrap()
)
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment