Unverified Commit ee5c7d39 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Revert "fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and…

Revert "fix(turbopack): Use vergen-git2 instead of shadow-rs for napi and next-api crates to fix stale git lock files" (#76879)

Reverts vercel/next.js#76773

> Looks like we broke compilation for our linux swc builds for musl/gnu
parent 190df615
This diff is collapsed.
......@@ -426,8 +426,6 @@ unicode-segmentation = "1.10.1"
unsize = "1.1.0"
url = "2.2.2"
urlencoding = "2.1.2"
vergen = { version = "9.0.4", features = ["cargo"] }
vergen-git2 = { version = "1.0.5", features = ["cargo"] }
webbrowser = "0.8.7"
[patch.crates-io]
......
......@@ -66,13 +66,14 @@ rand = { workspace = true }
rustc-hash = { workspace = true }
serde = "1"
serde_json = "1"
shadow-rs = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
tracing-chrome = "0.5.0"
url = { workspace = true }
urlencoding = { workspace = true }
once_cell = { workspace = true }
dashmap = { workspace = true }
dashmap = "6.1.0"
swc_core = { workspace = true, features = [
"base_concurrent",
......@@ -138,11 +139,11 @@ turbo-tasks-malloc = { workspace = true, default-features = false }
tokio = { workspace = true, features = ["full"] }
[build-dependencies]
anyhow = { workspace = true }
napi-build = "2"
serde = { workspace = true }
serde_json = { workspace = true }
vergen-git2 = { workspace = true }
# It is not a mistake this dependency is specified in dep / build-dep both.
shadow-rs = { workspace = true }
# build-dependencies for the native, non-wasm32 build
[target.'cfg(not(target_arch = "wasm32"))'.build-dependencies]
......
use std::{env, process::Command, str};
use std::fs;
extern crate napi_build;
fn main() -> anyhow::Result<()> {
println!("cargo:rerun-if-env-changed=CI");
let is_ci = env::var("CI").is_ok_and(|value| !value.is_empty());
fn main() {
// Generates, stores build-time information as static values.
// There are some places relying on correct values for this (i.e telemetry),
// So failing build if this fails.
let cargo = vergen_git2::CargoBuilder::default()
.target_triple(true)
.build()?;
// We use the git dirty state to disable persistent caching (persistent caching relies on a
// commit hash to be safe). One tradeoff of this is that we must invalidate the rust build more
// often.
//
// This invalidates the build if any untracked files change. That's sufficient for the case
// where we transition from dirty to clean.
//
// There's an edge-case here where the repository could be newly dirty, but we can't know
// because our build hasn't been invalidated, since the untracked files weren't untracked last
// time we ran. That will cause us to incorrectly report ourselves as clean.
//
// However, in practice that shouldn't be much of an issue: If no other dependency of this
// top-level crate has changed (which would've triggered our rebuild), then the resulting binary
// must be equivalent to a clean build anyways. Therefore, persistent caching using the HEAD
// commit hash as a version is okay.
let git = vergen_git2::Git2Builder::default()
.dirty(/* include_untracked */ true)
.describe(
/* tags */ true,
/* dirty */ !is_ci, // suppress the dirty suffix in CI
/* matches */ Some("v[0-9]*"), // find the last version tag
)
.build()?;
vergen_git2::Emitter::default()
.add_instructions(&cargo)?
.add_instructions(&git)?
.fail_on_error()
.emit()?;
shadow_rs::ShadowBuilder::builder()
.build()
.expect("Should able to generate build time information");
match Command::new("git").args(["rev-parse", "HEAD"]).output() {
Ok(out) if out.status.success() => println!(
"cargo:warning=git HEAD: {}",
str::from_utf8(&out.stdout).unwrap()
),
_ => println!("cargo:warning=`git rev-parse HEAD` failed"),
let git_head = fs::read_to_string("../../.git/HEAD").unwrap_or_default();
if !git_head.is_empty() && !git_head.starts_with("ref: ") {
println!("cargo:warning=git version {}", git_head);
}
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
......@@ -70,6 +36,4 @@ fn main() -> anyhow::Result<()> {
#[cfg(not(target_arch = "wasm32"))]
turbo_tasks_build::generate_register();
Ok(())
}
......@@ -61,6 +61,9 @@ pub mod turbo_trace_server;
pub mod turbopack;
pub mod util;
// Declare build-time information variables generated in build.rs
shadow_rs::shadow!(build);
#[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
#[global_allocator]
static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
......
......@@ -13,8 +13,7 @@ use turbo_tasks::{
TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc,
};
use turbo_tasks_backend::{
default_backing_storage, noop_backing_storage, DefaultBackingStorage, GitVersionInfo,
NoopBackingStorage,
default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage,
};
use turbo_tasks_fs::FileContent;
use turbopack_core::{
......@@ -131,10 +130,26 @@ pub fn create_turbo_tasks(
dependency_tracking: bool,
) -> Result<NextTurboTasks> {
Ok(if persistent_caching {
let version_info = GitVersionInfo {
describe: env!("VERGEN_GIT_DESCRIBE"),
dirty: option_env!("CI").is_none_or(|value| value.is_empty())
&& env!("VERGEN_GIT_DIRTY") == "true",
let dirty_suffix = if crate::build::GIT_CLEAN
|| option_env!("CI").is_some_and(|value| !value.is_empty())
{
""
} else {
"-dirty"
};
#[allow(
clippy::const_is_empty,
reason = "LAST_TAG might be empty if the tag can't be determined"
)]
let version_info = if crate::build::LAST_TAG.is_empty() {
format!("{}{}", crate::build::SHORT_COMMIT, dirty_suffix)
} else {
format!(
"{}-{}{}",
crate::build::LAST_TAG,
crate::build::SHORT_COMMIT,
dirty_suffix
)
};
NextTurboTasks::PersistentCaching(TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new(
......
......@@ -127,8 +127,8 @@ pub fn log_internal_error_and_inform(err_info: &str) {
}
#[napi]
pub fn get_target_triple() -> &'static str {
env!("VERGEN_CARGO_TARGET_TRIPLE")
pub fn get_target_triple() -> String {
crate::build::BUILD_TARGET.to_string()
}
pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {
......
......@@ -24,6 +24,7 @@ regex = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
shadow-rs = { workspace = true }
swc_core = { workspace = true }
tracing = { workspace = true }
turbo-rcstr = { workspace = true }
......@@ -42,6 +43,6 @@ turbopack-node = { workspace = true }
turbopack-nodejs = { workspace = true }
[build-dependencies]
anyhow = { workspace = true }
# It is not a mistake this dependency is specified in dep / build-dep both.
shadow-rs = { workspace = true }
turbo-tasks-build = { workspace = true }
vergen = { workspace = true }
use turbo_tasks_build::generate_register;
fn main() -> anyhow::Result<()> {
fn main() {
// Generates, stores build-time information as static values.
// There are some places relying on correct values for this (i.e telemetry),
// So failing build if this fails.
let cargo = vergen::CargoBuilder::default()
.target_triple(true)
.build()?;
vergen::Emitter::default()
.add_instructions(&cargo)?
.fail_on_error()
.emit()?;
shadow_rs::ShadowBuilder::builder()
.build_pattern(shadow_rs::BuildPattern::Lazy)
.build()
.expect("Should able to generate build time information");
generate_register();
Ok(())
}
......@@ -23,6 +23,9 @@ mod server_actions;
mod versioned_content_map;
mod webpack_stats;
// Declare build-time information variables generated in build.rs
shadow_rs::shadow!(build);
pub fn register() {
next_core::register();
turbopack_nodejs::register();
......
......@@ -68,6 +68,7 @@ use turbopack_nodejs::NodeJsChunkingContext;
use crate::{
app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME},
build,
empty::EmptyEndpoint,
entrypoints::Entrypoints,
instrumentation::InstrumentationEndpoint,
......@@ -1068,7 +1069,7 @@ impl Project {
// First, emit an event for the binary target triple.
// This is different to webpack-config; when this is being called,
// it is always using SWC so we don't check swc here.
emit_event(env!("VERGEN_CARGO_TARGET_TRIPLE"), true);
emit_event(build::BUILD_TARGET, true);
// Go over jsconfig and report enabled features.
let compiler_options = self.js_config().compiler_options().await?;
......
......@@ -16,5 +16,25 @@ workspace = true
next-core = { workspace = true }
turbopack-core = { workspace = true }
#turbopack-binding = { workspace = true, features = [
# "__turbo_tasks",
# "__turbo_tasks_memory",
# "__turbo_tasks_env",
# "__turbo_tasks_fs",
# "__turbo_tasks_memory",
# "__turbopack",
# "__turbopack_nodejs",
# "__turbopack_core",
# "__turbopack_browser",
# "__turbopack_ecmascript",
# "__turbopack_ecmascript_runtime",
# "__turbopack_env",
# "__turbopack_node",
#] }
[build-dependencies]
turbo-tasks-build = { workspace = true }
vergen = { version = "7.3.2", default-features = false, features = [
"cargo",
"build",
] }
use turbo_tasks_build::generate_register;
use vergen::{vergen, Config};
fn main() {
generate_register();
// Attempt to collect some build time env values but will skip if there are any
// errors.
let _ = vergen(Config::default());
}
......@@ -7,31 +7,24 @@ use std::{
use anyhow::Result;
/// Information gathered by `vergen_git2` in the top-level binary crate and passed down. This
/// information must be computed in the top-level crate for cargo incremental compilation to work
/// correctly.
///
/// See `crates/napi/build.rs` for details.
pub struct GitVersionInfo<'a> {
/// Output of `git describe --match 'v[0-9]' --dirty`.
pub describe: &'a str,
/// Is the git repository dirty? Always forced to `false` when the `CI` environment variable is
/// set and non-empty.
pub dirty: bool,
}
/// Specifies many databases that have a different version than the current one are retained.
/// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory,
/// the current one and two older/newer ones.
const MAX_OTHER_DB_VERSIONS: usize = 2;
pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> Result<PathBuf> {
pub fn handle_db_versioning(base_path: &Path, version_info: &str) -> Result<PathBuf> {
if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
return Ok(base_path.join(version));
}
// Database versioning. Pass `TURBO_ENGINE_IGNORE_DIRTY` at runtime to ignore a
// dirty git repository. Pass `TURBO_ENGINE_DISABLE_VERSIONING` at runtime to disable
// versioning and always use the same database.
let (version_info, git_dirty) = if let Some(version_info) = version_info.strip_suffix("-dirty")
{
(version_info, true)
} else {
(version_info, false)
};
let ignore_dirty = env::var("TURBO_ENGINE_IGNORE_DIRTY").ok().is_some();
let disabled_versioning = env::var("TURBO_ENGINE_DISABLE_VERSIONING").ok().is_some();
let version = if disabled_versioning {
......@@ -40,14 +33,14 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) ->
caching database might be required."
);
Some("unversioned")
} else if !version_info.dirty {
Some(version_info.describe)
} else if !git_dirty {
Some(version_info)
} else if ignore_dirty {
println!(
"WARNING: The git repository is dirty, but Persistent Caching is still enabled. \
Manual removal of the persistent caching database might be required."
);
Some(version_info.describe)
Some(version_info)
} else {
println!(
"WARNING: The git repository is dirty: Persistent Caching is disabled. Use \
......
......@@ -15,14 +15,13 @@ use std::path::Path;
use anyhow::Result;
use crate::database::{
db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
};
pub use crate::{
pub use self::{
backend::{BackendOptions, StorageMode, TurboTasksBackend},
database::db_versioning::GitVersionInfo,
kv_backing_storage::KeyValueDatabaseBackingStorage,
};
use crate::database::{
db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
};
#[cfg(feature = "lmdb")]
pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
......@@ -36,10 +35,7 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
>;
#[cfg(feature = "lmdb")]
pub fn lmdb_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<LmdbBackingStorage> {
pub fn lmdb_backing_storage(path: &Path, version_info: &str) -> Result<LmdbBackingStorage> {
use crate::database::{
fresh_db_optimization::{is_fresh, FreshDbOptimization},
read_transaction_cache::ReadTransactionCache,
......@@ -57,10 +53,7 @@ pub fn lmdb_backing_storage(
pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
pub fn turbo_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<TurboBackingStorage> {
pub fn turbo_backing_storage(path: &Path, version_info: &str) -> Result<TurboBackingStorage> {
let path = handle_db_versioning(path, version_info)?;
let database = TurboKeyValueDatabase::new(path)?;
Ok(KeyValueDatabaseBackingStorage::new(database))
......@@ -76,10 +69,7 @@ pub fn noop_backing_storage() -> NoopBackingStorage {
pub type DefaultBackingStorage = LmdbBackingStorage;
#[cfg(feature = "lmdb")]
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<DefaultBackingStorage> {
pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
lmdb_backing_storage(path, version_info)
}
......@@ -87,9 +77,6 @@ pub fn default_backing_storage(
pub type DefaultBackingStorage = TurboBackingStorage;
#[cfg(not(feature = "lmdb"))]
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
) -> Result<DefaultBackingStorage> {
pub fn default_backing_storage(path: &Path, version_info: &str) -> Result<DefaultBackingStorage> {
turbo_backing_storage(path, version_info)
}
......@@ -12,10 +12,7 @@
turbo_tasks_backend::BackendOptions::default(),
turbo_tasks_backend::default_backing_storage(
path.as_path(),
&turbo_tasks_backend::GitVersionInfo {
describe: "test-unversioned",
dirty: false,
},
"test"
).unwrap()
)
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment