Unverified Commit 9ff3f848 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: only keep old database version if not on CI (#79964)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the PR.
- Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to understand the PR)
- When linking to a Slack thread, you might want to share details of the conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->
parent baa71a54
......@@ -222,6 +222,8 @@ pub struct NapiTurboEngineOptions {
pub memory_limit: Option<f64>,
/// Track dependencies between tasks. If false, any change during build will error.
pub dependency_tracking: Option<bool>,
/// Whether the project is running in a CI environment.
pub is_ci: Option<bool>,
}
impl From<NapiWatchOptions> for WatchOptions {
......@@ -394,11 +396,13 @@ pub async fn project_new(
.unwrap_or(usize::MAX);
let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default();
let dependency_tracking = turbo_engine_options.dependency_tracking.unwrap_or(true);
let is_ci = turbo_engine_options.is_ci.unwrap_or(false);
let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir),
persistent_caching,
memory_limit,
dependency_tracking,
is_ci,
)?;
turbo_tasks.send_compilation_event(Arc::new(DiagnosticEvent::new(
......
......@@ -156,6 +156,7 @@ pub fn create_turbo_tasks(
persistent_caching: bool,
_memory_limit: usize,
dependency_tracking: bool,
is_ci: bool,
) -> Result<NextTurboTasks> {
Ok(if persistent_caching {
let version_info = GitVersionInfo {
......@@ -174,7 +175,11 @@ pub fn create_turbo_tasks(
dependency_tracking,
..Default::default()
},
default_backing_storage(&output_path.join("cache/turbopack"), &version_info)?,
default_backing_storage(
&output_path.join("cache/turbopack"),
&version_info,
is_ci,
)?,
),
))
} else {
......
......@@ -193,6 +193,8 @@ export interface NapiTurboEngineOptions {
memoryLimit?: number
/** Track dependencies between tasks. If false, any change during build will error. */
dependencyTracking?: boolean
/** Whether the project is running in a CI environment. */
isCi?: boolean
}
export declare function projectNew(
options: NapiProjectOptions,
......
......@@ -20,6 +20,7 @@ import { hasCustomExportOutput } from '../../export/utils'
import { Telemetry } from '../../telemetry/storage'
import { setGlobal } from '../../trace'
import * as Log from '../output/log'
import { isCI } from '../../server/ci-info'
export async function turbopackBuild(): Promise<{
duration: number
......@@ -86,6 +87,7 @@ export async function turbopackBuild(): Promise<{
persistentCaching,
memoryLimit: config.experimental?.turbopackMemoryLimit,
dependencyTracking: persistentCaching,
isCi: isCI,
}
)
try {
......
......@@ -22,10 +22,14 @@ pub struct GitVersionInfo<'a> {
/// Specifies many databases that have a different version than the current one are retained.
/// For example if MAX_OTHER_DB_VERSIONS is 2, there can be at most 3 databases in the directory,
/// the current one and two older/newer ones.
/// the current one and two older/newer ones. On CI it never keeps any other versions.
const MAX_OTHER_DB_VERSIONS: usize = 2;
pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) -> Result<PathBuf> {
pub fn handle_db_versioning(
base_path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<PathBuf> {
if let Ok(version) = env::var("TURBO_ENGINE_VERSION") {
return Ok(base_path.join(version));
}
......@@ -59,6 +63,8 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) ->
if let Some(version) = version {
path = base_path.join(version);
let max_other_db_versions = if is_ci { 0 } else { MAX_OTHER_DB_VERSIONS };
// Remove old databases if needed
if let Ok(read_dir) = read_dir(base_path) {
let old_dbs = read_dir
......@@ -75,7 +81,7 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) ->
Some(entry.path())
})
.collect::<Vec<_>>();
if old_dbs.len() > MAX_OTHER_DB_VERSIONS {
if old_dbs.len() > max_other_db_versions {
let mut old_dbs = old_dbs
.iter()
.map(|p| {
......@@ -90,7 +96,7 @@ pub fn handle_db_versioning(base_path: &Path, version_info: &GitVersionInfo) ->
})
.collect::<Vec<_>>();
old_dbs.sort_by_key(|(_, age)| *age);
for (p, _) in old_dbs.into_iter().skip(MAX_OTHER_DB_VERSIONS) {
for (p, _) in old_dbs.into_iter().skip(max_other_db_versions) {
let _ = remove_dir_all(p);
}
}
......
......@@ -39,6 +39,7 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
pub fn lmdb_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<LmdbBackingStorage> {
use crate::database::{
fresh_db_optimization::{FreshDbOptimization, is_fresh},
......@@ -46,7 +47,7 @@ pub fn lmdb_backing_storage(
startup_cache::StartupCacheLayer,
};
let path = handle_db_versioning(path, version_info)?;
let path = handle_db_versioning(path, version_info, is_ci)?;
let fresh_db = is_fresh(&path);
let database = crate::database::lmdb::LmbdKeyValueDatabase::new(&path)?;
let database = FreshDbOptimization::new(database, fresh_db);
......@@ -60,8 +61,9 @@ pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatab
pub fn turbo_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<TurboBackingStorage> {
let path = handle_db_versioning(path, version_info)?;
let path = handle_db_versioning(path, version_info, is_ci)?;
let database = TurboKeyValueDatabase::new(path)?;
Ok(KeyValueDatabaseBackingStorage::new(database))
}
......@@ -79,8 +81,9 @@ pub type DefaultBackingStorage = LmdbBackingStorage;
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<DefaultBackingStorage> {
lmdb_backing_storage(path, version_info)
lmdb_backing_storage(path, version_info, is_ci)
}
#[cfg(not(feature = "lmdb"))]
......@@ -90,6 +93,7 @@ pub type DefaultBackingStorage = TurboBackingStorage;
pub fn default_backing_storage(
path: &Path,
version_info: &GitVersionInfo,
is_ci: bool,
) -> Result<DefaultBackingStorage> {
turbo_backing_storage(path, version_info)
turbo_backing_storage(path, version_info, is_ci)
}
......@@ -16,6 +16,7 @@
describe: "test-unversioned",
dirty: false,
},
false
).unwrap()
)
)
......
......@@ -16,6 +16,7 @@
describe: "test-unversioned",
dirty: false,
},
false
).unwrap()
)
)
......
......@@ -284,6 +284,7 @@ fn node_file_trace_persistent(#[case] input: CaseInput) {
describe: "test-unversioned",
dirty: false,
},
false,
)
.unwrap(),
))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment