Unverified Commit ee5c7d39 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Revert "fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and…

Revert "fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and next-api crates to fix stale git lock files" (#76879)

Reverts vercel/next.js#76773

> Looks like we broke compilation for our linux swc builds for musl/gnu
parent 190df615
This diff is collapsed.
...@@ -426,8 +426,6 @@ unicode-segmentation = "1.10.1" ...@@ -426,8 +426,6 @@ unicode-segmentation = "1.10.1"
unsize = "1.1.0" unsize = "1.1.0"
url = "2.2.2" url = "2.2.2"
urlencoding = "2.1.2" urlencoding = "2.1.2"
vergen = { version = "9.0.4", features = ["cargo"] }
vergen-git2 = { version = "1.0.5", features = ["cargo"] }
webbrowser = "0.8.7" webbrowser = "0.8.7"
[patch.crates-io] [patch.crates-io]
......
...@@ -66,13 +66,14 @@ rand = { workspace = true } ...@@ -66,13 +66,14 @@ rand = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = "1" serde = "1"
serde_json = "1" serde_json = "1"
shadow-rs = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
tracing-subscriber = { workspace = true } tracing-subscriber = { workspace = true }
tracing-chrome = "0.5.0" tracing-chrome = "0.5.0"
url = { workspace = true } url = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
once_cell = { workspace = true } once_cell = { workspace = true }
dashmap = { workspace = true } dashmap = "6.1.0"
swc_core = { workspace = true, features = [ swc_core = { workspace = true, features = [
"base_concurrent", "base_concurrent",
...@@ -138,11 +139,11 @@ turbo-tasks-malloc = { workspace = true, default-features = false } ...@@ -138,11 +139,11 @@ turbo-tasks-malloc = { workspace = true, default-features = false }
tokio = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["full"] }
[build-dependencies] [build-dependencies]
anyhow = { workspace = true }
napi-build = "2" napi-build = "2"
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
vergen-git2 = { workspace = true } # It is not a mistake this dependency is specified in dep / build-dep both.
shadow-rs = { workspace = true }
# build-dependencies for the native, non-wasm32 build # build-dependencies for the native, non-wasm32 build
[target.'cfg(not(target_arch = "wasm32"))'.build-dependencies] [target.'cfg(not(target_arch = "wasm32"))'.build-dependencies]
......
use std::{env, process::Command, str}; use std::fs;
extern crate napi_build; extern crate napi_build;
fn main() -> anyhow::Result<()> { fn main() {
println!("cargo:rerun-if-env-changed=CI");
let is_ci = env::var("CI").is_ok_and(|value| !value.is_empty());
// Generates, stores build-time information as static values. // Generates, stores build-time information as static values.
// There are some places relying on correct values for this (i.e telemetry), // There are some places relying on correct values for this (i.e telemetry),
// So failing build if this fails. // So failing build if this fails.
let cargo = vergen_git2::CargoBuilder::default() shadow_rs::ShadowBuilder::builder()
.target_triple(true) .build()
.build()?; .expect("Should able to generate build time information");
// We use the git dirty state to disable persistent caching (persistent caching relies on a
// commit hash to be safe). One tradeoff of this is that we must invalidate the rust build more
// often.
//
// This invalidates the build if any untracked files change. That's sufficient for the case
// where we transition from dirty to clean.
//
// There's an edge-case here where the repository could be newly dirty, but we can't know
// because our build hasn't been invalidated, since the untracked files weren't untracked last
// time we ran. That will cause us to incorrectly report ourselves as clean.
//
// However, in practice that shouldn't be much of an issue: If no other dependency of this
// top-level crate has changed (which would've triggered our rebuild), then the resulting binary
// must be equivalent to a clean build anyways. Therefore, persistent caching using the HEAD
// commit hash as a version is okay.
let git = vergen_git2::Git2Builder::default()
.dirty(/* include_untracked */ true)
.describe(
/* tags */ true,
/* dirty */ !is_ci, // suppress the dirty suffix in CI
/* matches */ Some("v[0-9]*"), // find the last version tag
)
.build()?;
vergen_git2::Emitter::default()
.add_instructions(&cargo)?
.add_instructions(&git)?
.fail_on_error()
.emit()?;
match Command::new("git").args(["rev-parse", "HEAD"]).output() { let git_head = fs::read_to_string("../../.git/HEAD").unwrap_or_default();
Ok(out) if out.status.success() => println!( if !git_head.is_empty() && !git_head.starts_with("ref: ") {
"cargo:warning=git HEAD: {}", println!("cargo:warning=git version {}", git_head);
str::from_utf8(&out.stdout).unwrap()
),
_ => println!("cargo:warning=`git rev-parse HEAD` failed"),
} }
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
...@@ -70,6 +36,4 @@ fn main() -> anyhow::Result<()> { ...@@ -70,6 +36,4 @@ fn main() -> anyhow::Result<()> {
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_arch = "wasm32"))]
turbo_tasks_build::generate_register(); turbo_tasks_build::generate_register();
Ok(())
} }
...@@ -61,6 +61,9 @@ pub mod turbo_trace_server; ...@@ -61,6 +61,9 @@ pub mod turbo_trace_server;
pub mod turbopack; pub mod turbopack;
pub mod util; pub mod util;
// Declare build-time information variables generated in build.rs
shadow_rs::shadow!(build);
#[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))] #[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
#[global_allocator] #[global_allocator]
static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc; static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
......
...@@ -13,8 +13,7 @@ use turbo_tasks::{ ...@@ -13,8 +13,7 @@ use turbo_tasks::{
TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc, TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc,
}; };
use turbo_tasks_backend::{ use turbo_tasks_backend::{
default_backing_storage, noop_backing_storage, DefaultBackingStorage, GitVersionInfo, default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage,
NoopBackingStorage,
}; };
use turbo_tasks_fs::FileContent; use turbo_tasks_fs::FileContent;
use turbopack_core::{ use turbopack_core::{
...@@ -131,10 +130,26 @@ pub fn create_turbo_tasks( ...@@ -131,10 +130,26 @@ pub fn create_turbo_tasks(
dependency_tracking: bool, dependency_tracking: bool,
) -> Result<NextTurboTasks> { ) -> Result<NextTurboTasks> {
Ok(if persistent_caching { Ok(if persistent_caching {
let version_info = GitVersionInfo { let dirty_suffix = if crate::build::GIT_CLEAN
describe: env!("VERGEN_GIT_DESCRIBE"), || option_env!("CI").is_some_and(|value| !value.is_empty())
dirty: option_env!("CI").is_none_or(|value| value.is_empty()) {
&& env!("VERGEN_GIT_DIRTY") == "true", ""
} else {
"-dirty"
};
#[allow(
clippy::const_is_empty,
reason = "LAST_TAG might be empty if the tag can't be determined"
)]
let version_info = if crate::build::LAST_TAG.is_empty() {
format!("{}{}", crate::build::SHORT_COMMIT, dirty_suffix)
} else {
format!(
"{}-{}{}",
crate::build::LAST_TAG,
crate::build::SHORT_COMMIT,
dirty_suffix
)
}; };
NextTurboTasks::PersistentCaching(TurboTasks::new( NextTurboTasks::PersistentCaching(TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new( turbo_tasks_backend::TurboTasksBackend::new(
......
...@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) { ...@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) {
} }
#[napi] #[napi]
pub fn get_target_triple() -> &'static str { pub fn get_target_triple() -> String {
env!("VERGEN_CARGO_TARGET_TRIPLE") crate::build::BUILD_TARGET.to_string()
} }
pub trait MapErr<T>: Into<Result<T, anyhow::Error>> { pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {
......
...@@ -24,6 +24,7 @@ regex = { workspace = true } ...@@ -24,6 +24,7 @@ regex = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
shadow-rs = { workspace = true }
swc_core = { workspace = true } swc_core = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
...@@ -42,6 +43,6 @@ turbopack-node = { workspace = true } ...@@ -42,6 +43,6 @@ turbopack-node = { workspace = true }
turbopack-nodejs = { workspace = true } turbopack-nodejs = { workspace = true }
[build-dependencies] [build-dependencies]
anyhow = { workspace = true } # It is not a mistake this dependency is specified in dep / build-dep both.
shadow-rs = { workspace = true }
turbo-tasks-build = { workspace = true } turbo-tasks-build = { workspace = true }
vergen = { workspace = true }
use turbo_tasks_build::generate_register; use turbo_tasks_build::generate_register;
fn main() -> anyhow::Result<()> { fn main() {
// Generates, stores build-time information as static values. // Generates, stores build-time information as static values.
// There are some places relying on correct values for this (i.e telemetry), // There are some places relying on correct values for this (i.e telemetry),
// So failing build if this fails. // So failing build if this fails.
let cargo = vergen::CargoBuilder::default() shadow_rs::ShadowBuilder::builder()
.target_triple(true) .build_pattern(shadow_rs::BuildPattern::Lazy)
.build()?; .build()
vergen::Emitter::default() .expect("Should able to generate build time information");
.add_instructions(&cargo)?
.fail_on_error()
.emit()?;
generate_register(); generate_register();
Ok(())
} }
...@@ -23,6 +23,9 @@ mod server_actions; ...@@ -23,6 +23,9 @@ mod server_actions;
mod versioned_content_map; mod versioned_content_map;
mod webpack_stats; mod webpack_stats;
// Declare build-time information variables generated in build.rs
shadow_rs::shadow!(build);
pub fn register() { pub fn register() {
next_core::register(); next_core::register();
turbopack_nodejs::register(); turbopack_nodejs::register();
......
...@@ -68,6 +68,7 @@ use turbopack_nodejs::NodeJsChunkingContext; ...@@ -68,6 +68,7 @@ use turbopack_nodejs::NodeJsChunkingContext;
use crate::{ use crate::{
app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME}, app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME},
build,
empty::EmptyEndpoint, empty::EmptyEndpoint,
entrypoints::Entrypoints, entrypoints::Entrypoints,
instrumentation::InstrumentationEndpoint, instrumentation::InstrumentationEndpoint,
...@@ -1068,7 +1069,7 @@ impl Project { ...@@ -1068,7 +1069,7 @@ impl Project {
// First, emit an event for the binary target triple. // First, emit an event for the binary target triple.
// This is different to webpack-config; when this is being called, // This is different to webpack-config; when this is being called,
// it is always using SWC so we don't check swc here. // it is always using SWC so we don't check swc here.
emit_event(env!("VERGEN_CARGO_TARGET_TRIPLE"), true); emit_event(build::BUILD_TARGET, true);
// Go over jsconfig and report enabled features. // Go over jsconfig and report enabled features.
let compiler_options = self.js_config().compiler_options().await?; let compiler_options = self.js_config().compiler_options().await?;
......
...@@ -16,5 +16,25 @@ workspace = true ...@@ -16,5 +16,25 @@ workspace = true
next-core = { workspace = true } next-core = { workspace = true }
turbopack-core = { workspace = true } turbopack-core = { workspace = true }
#turbopack-binding = { workspace = true, features = [
# "__turbo_tasks",
# "__turbo_tasks_memory",
# "__turbo_tasks_env",
# "__turbo_tasks_fs",
# "__turbo_tasks_memory",
# "__turbopack",
# "__turbopack_nodejs",
# "__turbopack_core",
# "__turbopack_browser",
# "__turbopack_ecmascript",
# "__turbopack_ecmascript_runtime",
# "__turbopack_env",
# "__turbopack_node",
#] }
[build-dependencies] [build-dependencies]
turbo-tasks-build = { workspace = true } turbo-tasks-build = { workspace = true }
vergen = { version = "7.3.2", default-features = false, features = [
"cargo",
"build",
] }
use turbo_tasks_build::generate_register; use turbo_tasks_build::generate_register;
use vergen::{vergen, Config};
fn main() { fn main() {
generate_register(); generate_register();
// Attempt to collect some build time env values but will skip if there are any
// errors.
let _ = vergen(Config::default());
} }
...@@ -7,31 +7,24 @@ use std::{ ...@@ -7,31 +7,24 @@ use std::{
use anyhow::Result; use anyhow::Result;
/// Information gathered by `vergen_git2` in the top-level binary crate and passed down. This
/// information must be computed in the top-level crate for cargo incremental compilation to work
/// correctly.
///
/// See `crates/napi/build.rs` for details.
pub struct GitVersionInfo<'a> {
/// Output of `git describe --match 'v[0-9]' --dirty`.
pub describe: &'a str,
/// Is the git repository dirty? Always forced to `false` when the `CI` environment variable is
/// set and non-empty.
pub dirty: bool,
}
/// Specifies many databases that have a different version than the current one are retained. /// Specifies many databases that have a different version than the current one are retained.
/// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory, /// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory,
/// the current one and two older/newer ones. /// the current one and two older/newer ones.
const MAX_OTHER_DB_VERSIONS: usize = 2; const MAX_OTHER_DB_VERSIONS: usize = 2;
pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> Result<PathBuf> { pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<PathBuf> {
if let Ok(version) = env::var("TURBO_ENGINE_VERSION") { if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
return Ok(base_path.join(version)); return Ok(base_path.join(version));
} }
// Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a // Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a
// dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable // dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable
// versioning and always use the same database. // versioning and always use the same database.
let (version_info, git_dirty) = if let Some(version_info) = version_info.strip_suffix("-dirty")
{
(version_info, true)
} else {
(version_info, false)
};
let ignore_dirty = env::var("TURBO_ENGINE_IGNORE_DIRTY").ok().is_some(); let ignore_dirty = env::var("TURBO_ENGINE_IGNORE_DIRTY").ok().is_some();
let disabled_versioning = env::var("TURBO_ENGINE_DISABLE_VERSIONING").ok().is_some(); let disabled_versioning = env::var("TURBO_ENGINE_DISABLE_VERSIONING").ok().is_some();
let version = if disabled_versioning { let version = if disabled_versioning {
...@@ -40,14 +33,14 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> ...@@ -40,14 +33,14 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) ->
caching database might be required." caching database might be required."
); );
Some("unversioned") Some("unversioned")
} else if !version_info.dirty { } else if !git_dirty {
Some(version_info.describe) Some(version_info)
} else if ignore_dirty { } else if ignore_dirty {
println!( println!(
"WARNING: The git repository is dirty, but Persistent Caching is still enabled. \ "WARNING: The git repository is dirty, but Persistent Caching is still enabled. \
Manual removal of the persistent caching database might be required." Manual removal of the persistent caching database might be required."
); );
Some(version_info.describe) Some(version_info)
} else { } else {
println!( println!(
"WARNING: The git repository is dirty: Persistent Caching is disabled. Use \ "WARNING: The git repository is dirty: Persistent Caching is disabled. Use \
......
...@@ -15,14 +15,13 @@ use std::path::Path; ...@@ -15,14 +15,13 @@ use std::path::Path;
use anyhow::Result; use anyhow::Result;
use crate::database::{ pub use self::{
db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
};
pub use crate::{
backend::{BackendOptions, StorageMode, TurboTasksBackend}, backend::{BackendOptions, StorageMode, TurboTasksBackend},
database::db_versioning::GitVersionInfo,
kv_backing_storage::KeyValueDatabaseBackingStorage, kv_backing_storage::KeyValueDatabaseBackingStorage,
}; };
use crate::database::{
db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
};
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage< pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
...@@ -36,10 +35,7 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage< ...@@ -36,10 +35,7 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
>; >;
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
pub fn lmdb_backing_storage( pub fn lmdb_backing_storage(path: &Path, version_info: &str) -> Result<LmdbBackingStorage> {
path: &Path,
version_info: &GitVersionInfo,
) -> Result<LmdbBackingStorage> {
use crate::database::{ use crate::database::{
fresh_db_optimization::{is_fresh, FreshDbOptimization}, fresh_db_optimization::{is_fresh, FreshDbOptimization},
read_transaction_cache::ReadTransactionCache, read_transaction_cache::ReadTransactionCache,
...@@ -57,10 +53,7 @@ pub fn lmdb_backing_storage( ...@@ -57,10 +53,7 @@ pub fn lmdb_backing_storage(
pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>; pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
pub fn turbo_backing_storage( pub fn turbo_backing_storage(path: &Path, version_info: &str) -> Result<TurboBackingStorage> {
path: &Path,
version_info: &GitVersionInfo,
) -> Result<TurboBackingStorage> {
let path = handle_db_versioning(path, version_info)?; let path = handle_db_versioning(path, version_info)?;
let database = TurboKeyValueDatabase::new(path)?; let database = TurboKeyValueDatabase::new(path)?;
Ok(KeyValueDatabaseBackingStorage::new(database)) Ok(KeyValueDatabaseBackingStorage::new(database))
...@@ -76,10 +69,7 @@ pub fn noop_backing_storage() -> NoopBackingStorage { ...@@ -76,10 +69,7 @@ pub fn noop_backing_storage() -> NoopBackingStorage {
pub type DefaultBackingStorage = LmdbBackingStorage; pub type DefaultBackingStorage = LmdbBackingStorage;
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
pub fn default_backing_storage( pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
path: &Path,
version_info: &GitVersionInfo,
) -> Result<DefaultBackingStorage> {
lmdb_backing_storage(path, version_info) lmdb_backing_storage(path, version_info)
} }
...@@ -87,9 +77,6 @@ pub fn default_backing_storage( ...@@ -87,9 +77,6 @@ pub fn default_backing_storage(
pub type DefaultBackingStorage = TurboBackingStorage; pub type DefaultBackingStorage = TurboBackingStorage;
#[cfg(not(feature = "lmdb"))] #[cfg(not(feature = "lmdb"))]
pub fn default_backing_storage( pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
path: &Path,
version_info: &GitVersionInfo,
) -> Result<DefaultBackingStorage> {
turbo_backing_storage(path, version_info) turbo_backing_storage(path, version_info)
} }
...@@ -12,10 +12,7 @@ ...@@ -12,10 +12,7 @@
turbo_tasks_backend::BackendOptions::default(), turbo_tasks_backend::BackendOptions::default(),
turbo_tasks_backend::default_backing_storage( turbo_tasks_backend::default_backing_storage(
path.as_path(), path.as_path(),
&turbo_tasks_backend::GitVersionInfo { "test"
describe: "test-unversioned",
dirty: false,
},
).unwrap() ).unwrap()
) )
) )
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment