Unverified Commit 3f01d2fc authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] add persistentCaching config option (#70904)

### What?

add experimental option to enable persistent caching
parent 556d34ef
......@@ -37,8 +37,6 @@ __internal_dhat-heap = ["dhat"]
# effectively does nothing.
__internal_dhat-ad-hoc = ["dhat"]
new-backend = ["dep:turbo-tasks-backend"]
# Enable specific tls features per-target.
[target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
next-core = { workspace = true, features = ["native-tls"] }
......@@ -107,7 +105,7 @@ lightningcss-napi = { workspace = true }
tokio = { workspace = true, features = ["full"] }
turbo-tasks = { workspace = true }
turbo-tasks-memory = { workspace = true }
turbo-tasks-backend = { workspace = true, optional = true }
turbo-tasks-backend = { workspace = true }
turbo-tasks-fs = { workspace = true }
next-api = { workspace = true }
next-build = { workspace = true }
......
......@@ -22,7 +22,7 @@ use rand::Rng;
use tokio::{io::AsyncWriteExt, time::Instant};
use tracing::Instrument;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, TurboTasks, UpdateInfo, Vc};
use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, UpdateInfo, Vc};
use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
use turbopack_core::{
diagnostics::PlainDiagnostic,
......@@ -44,7 +44,7 @@ use super::{
endpoint::ExternalEndpoint,
utils::{
create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue,
NextBackend, RootTask, TurbopackResult, VcArc,
NextTurboTasks, RootTask, TurbopackResult, VcArc,
},
};
use crate::register;
......@@ -188,6 +188,8 @@ pub struct NapiDefineEnv {
#[napi(object)]
pub struct NapiTurboEngineOptions {
/// Use the new backend with persistent caching enabled.
pub persistent_caching: Option<bool>,
/// An upper bound of memory that turbopack will attempt to stay under.
pub memory_limit: Option<f64>,
}
......@@ -272,7 +274,7 @@ impl From<NapiDefineEnv> for DefineEnv {
}
pub struct ProjectInstance {
turbo_tasks: Arc<TurboTasks<NextBackend>>,
turbo_tasks: NextTurboTasks,
container: Vc<ProjectContainer>,
exit_receiver: tokio::sync::Mutex<Option<ExitReceiver>>,
}
......@@ -338,13 +340,20 @@ pub async fn project_new(
.memory_limit
.map(|m| m as usize)
.unwrap_or(usize::MAX);
let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?;
#[cfg(not(feature = "new-backend"))]
{
let persistent_caching = turbo_engine_options.persistent_caching.unwrap_or_default();
let turbo_tasks = create_turbo_tasks(
PathBuf::from(&options.dist_dir),
persistent_caching,
memory_limit,
)?;
if !persistent_caching {
use std::io::Write;
let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
if let Some(stats_path) = stats_path {
let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
let Some(backend) = turbo_tasks.memory_backend() else {
return Err(anyhow!("task statistics require a memory backend").into());
};
let task_stats = backend.task_statistics().enable().clone();
exit.on_exit(async move {
tokio::task::spawn_blocking(move || {
let mut file = std::fs::File::create(&stats_path)
......@@ -498,11 +507,7 @@ struct NapiRoute {
}
impl NapiRoute {
fn from_route(
pathname: String,
value: Route,
turbo_tasks: &Arc<TurboTasks<NextBackend>>,
) -> Self {
fn from_route(pathname: String, value: Route, turbo_tasks: &NextTurboTasks) -> Self {
let convert_endpoint = |endpoint: Vc<Box<dyn Endpoint>>| {
Some(External::new(ExternalEndpoint(VcArc::new(
turbo_tasks.clone(),
......@@ -566,10 +571,7 @@ struct NapiMiddleware {
}
impl NapiMiddleware {
fn from_middleware(
value: &Middleware,
turbo_tasks: &Arc<TurboTasks<NextBackend>>,
) -> Result<Self> {
fn from_middleware(value: &Middleware, turbo_tasks: &NextTurboTasks) -> Result<Self> {
Ok(NapiMiddleware {
endpoint: External::new(ExternalEndpoint(VcArc::new(
turbo_tasks.clone(),
......@@ -586,10 +588,7 @@ struct NapiInstrumentation {
}
impl NapiInstrumentation {
fn from_instrumentation(
value: &Instrumentation,
turbo_tasks: &Arc<TurboTasks<NextBackend>>,
) -> Result<Self> {
fn from_instrumentation(value: &Instrumentation, turbo_tasks: &NextTurboTasks) -> Result<Self> {
Ok(NapiInstrumentation {
node_js: External::new(ExternalEndpoint(VcArc::new(
turbo_tasks.clone(),
......
use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc};
use std::{
collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc, time::Duration,
};
use anyhow::{anyhow, Context, Result};
use napi::{
......@@ -7,7 +9,9 @@ use napi::{
JsFunction, JsObject, JsUnknown, NapiRaw, NapiValue, Status,
};
use serde::Serialize;
use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc};
use turbo_tasks::{
trace::TraceRawVcs, ReadRef, TaskId, TryJoinIterExt, TurboTasks, UpdateInfo, Vc,
};
use turbo_tasks_fs::FileContent;
use turbopack_core::{
diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic},
......@@ -18,41 +22,136 @@ use turbopack_core::{
use crate::util::log_internal_error_and_inform;
#[cfg(not(feature = "new-backend"))]
pub type NextBackend = turbo_tasks_memory::MemoryBackend;
#[cfg(feature = "new-backend")]
pub type NextBackend = turbo_tasks_backend::TurboTasksBackend;
#[derive(Clone)]
pub enum NextTurboTasks {
Memory(Arc<TurboTasks<turbo_tasks_memory::MemoryBackend>>),
PersistentCaching(Arc<TurboTasks<turbo_tasks_backend::TurboTasksBackend>>),
}
impl NextTurboTasks {
pub fn dispose_root_task(&self, task: TaskId) {
match self {
NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.dispose_root_task(task),
NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.dispose_root_task(task),
}
}
pub fn spawn_root_task<T, F, Fut>(&self, functor: F) -> TaskId
where
T: Send,
F: Fn() -> Fut + Send + Sync + Clone + 'static,
Fut: Future<Output = Result<Vc<T>>> + Send,
{
match self {
NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.spawn_root_task(functor),
NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.spawn_root_task(functor),
}
}
pub async fn run_once<T: TraceRawVcs + Send + 'static>(
&self,
future: impl Future<Output = Result<T>> + Send + 'static,
) -> Result<T> {
match self {
NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.run_once(future).await,
NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.run_once(future).await,
}
}
pub fn spawn_once_task<T, Fut>(&self, future: Fut) -> TaskId
where
T: Send,
Fut: Future<Output = Result<Vc<T>>> + Send + 'static,
{
match self {
NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.spawn_once_task(future),
NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.spawn_once_task(future),
}
}
pub async fn aggregated_update_info(
&self,
aggregation: Duration,
timeout: Duration,
) -> Option<UpdateInfo> {
match self {
NextTurboTasks::Memory(turbo_tasks) => {
turbo_tasks
.aggregated_update_info(aggregation, timeout)
.await
}
NextTurboTasks::PersistentCaching(turbo_tasks) => {
turbo_tasks
.aggregated_update_info(aggregation, timeout)
.await
}
}
}
pub async fn get_or_wait_aggregated_update_info(&self, aggregation: Duration) -> UpdateInfo {
match self {
NextTurboTasks::Memory(turbo_tasks) => {
turbo_tasks
.get_or_wait_aggregated_update_info(aggregation)
.await
}
NextTurboTasks::PersistentCaching(turbo_tasks) => {
turbo_tasks
.get_or_wait_aggregated_update_info(aggregation)
.await
}
}
}
pub fn memory_backend(&self) -> Option<&turbo_tasks_memory::MemoryBackend> {
match self {
NextTurboTasks::Memory(turbo_tasks) => Some(turbo_tasks.backend()),
NextTurboTasks::PersistentCaching(_) => None,
}
}
pub async fn stop_and_wait(&self) {
match self {
NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.stop_and_wait().await,
NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.stop_and_wait().await,
}
}
}
#[allow(unused_variables, reason = "feature-gated")]
pub fn create_turbo_tasks(
output_path: PathBuf,
persistent_caching: bool,
memory_limit: usize,
) -> Result<Arc<TurboTasks<NextBackend>>> {
#[cfg(not(feature = "new-backend"))]
let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit));
#[cfg(feature = "new-backend")]
let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new(
turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?,
)));
Ok(backend)
) -> Result<NextTurboTasks> {
Ok(if persistent_caching {
NextTurboTasks::PersistentCaching(TurboTasks::new(
turbo_tasks_backend::TurboTasksBackend::new(Arc::new(
turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?,
)),
))
} else {
NextTurboTasks::Memory(TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(
memory_limit,
)))
})
}
/// A helper type to hold both a Vc operation and the TurboTasks root process.
/// Without this, we'd need to pass both individually all over the place
#[derive(Clone)]
pub struct VcArc<T> {
turbo_tasks: Arc<TurboTasks<NextBackend>>,
turbo_tasks: NextTurboTasks,
/// The Vc. Must be resolved, otherwise you are referencing an inactive
/// operation.
vc: T,
}
impl<T> VcArc<T> {
pub fn new(turbo_tasks: Arc<TurboTasks<NextBackend>>, vc: T) -> Self {
pub fn new(turbo_tasks: NextTurboTasks, vc: T) -> Self {
Self { turbo_tasks, vc }
}
pub fn turbo_tasks(&self) -> &Arc<TurboTasks<NextBackend>> {
pub fn turbo_tasks(&self) -> &NextTurboTasks {
&self.turbo_tasks
}
}
......@@ -75,7 +174,7 @@ pub fn serde_enum_to_string<T: Serialize>(value: &T) -> Result<String> {
/// The root of our turbopack computation.
pub struct RootTask {
#[allow(dead_code)]
turbo_tasks: Arc<TurboTasks<NextBackend>>,
turbo_tasks: NextTurboTasks,
#[allow(dead_code)]
task_id: Option<TaskId>,
}
......@@ -319,7 +418,7 @@ impl<T: ToNapiValue> ToNapiValue for TurbopackResult<T> {
}
pub fn subscribe<T: 'static + Send + Sync, F: Future<Output = Result<T>> + Send, V: ToNapiValue>(
turbo_tasks: Arc<TurboTasks<NextBackend>>,
turbo_tasks: NextTurboTasks,
func: JsFunction,
handler: impl 'static + Sync + Send + Clone + Fn() -> F,
mapper: impl 'static + Sync + Send + FnMut(ThreadSafeCallContext<T>) -> napi::Result<Vec<V>>,
......
......@@ -2,46 +2,52 @@ use std::{path::PathBuf, sync::Arc};
use napi::bindgen_prelude::*;
use node_file_trace::{start, Args};
use turbo_tasks::TurboTasks;
use turbopack::{
module_options::{EcmascriptOptionsContext, ModuleOptionsContext},
resolve_options_context::ResolveOptionsContext,
};
use crate::next_api::utils::{self, NextBackend};
use crate::next_api::utils::{self, NextTurboTasks};
#[napi]
pub fn create_turbo_tasks(
output_path: String,
persistent_caching: bool,
memory_limit: Option<i64>,
) -> External<Arc<TurboTasks<NextBackend>>> {
) -> External<NextTurboTasks> {
let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX);
let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit)
.expect("Failed to create TurboTasks");
let turbo_tasks =
utils::create_turbo_tasks(PathBuf::from(&output_path), persistent_caching, limit)
.expect("Failed to create TurboTasks");
External::new_with_size_hint(turbo_tasks, limit)
}
#[napi]
pub async fn run_turbo_tracing(
options: Buffer,
turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
turbo_tasks: External<NextTurboTasks>,
) -> napi::Result<Vec<String>> {
let args: Args = serde_json::from_slice(options.as_ref())?;
let files = start(
Arc::new(args),
turbo_tasks.clone(),
Some(ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_types: true,
..Default::default()
},
enable_mdx: true,
let args = Arc::new(args);
let module_options = Some(ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_types: true,
..Default::default()
}),
Some(ResolveOptionsContext {
..Default::default()
}),
)
.await?;
},
enable_mdx: true,
..Default::default()
});
let resolve_options = Some(ResolveOptionsContext {
..Default::default()
});
let files = match &*turbo_tasks {
NextTurboTasks::Memory(turbo_tasks) => {
start(args, turbo_tasks.clone(), module_options, resolve_options).await?
}
NextTurboTasks::PersistentCaching(turbo_tasks) => {
start(args, turbo_tasks.clone(), module_options, resolve_options).await?
}
};
Ok(files.into_iter().map(|f| f.to_string()).collect())
}
......@@ -27,7 +27,7 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
import isError from '../lib/is-error'
import type { NodeFileTraceReasons } from '@vercel/nft'
import type { RoutesUsingEdgeRuntime } from './utils'
import type { ExternalObject, TurboTasks } from './swc/generated-native'
import type { ExternalObject, NextTurboTasks } from './swc/generated-native'
const debug = debugOriginal('next:build:build-traces')
......@@ -108,7 +108,7 @@ export async function collectBuildTraces({
}) {
const startTime = Date.now()
debug('starting build traces')
let turboTasksForTrace: ExternalObject<TurboTasks>
let turboTasksForTrace: ExternalObject<NextTurboTasks>
let bindings = await loadBindings()
const runTurbotrace = async function () {
......@@ -120,6 +120,7 @@ export async function collectBuildTraces({
let turbotraceFiles: string[] | undefined
turboTasksForTrace = bindings.turbo.createTurboTasks(
distDir,
false,
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
......
......@@ -1409,6 +1409,7 @@ export default async function build(
browserslistQuery: supportedBrowsers.join(', '),
},
{
persistentCaching: config.experimental.turbo?.persistentCaching,
memoryLimit: config.experimental.turbo?.memoryLimit,
}
)
......
......@@ -4,6 +4,7 @@ import type { TurbopackResult } from './types'
export type TurboTasks = { readonly __tag: unique symbol }
export type ExternalEndpoint = { readonly __tag: unique symbol }
export type NextTurboTasks = { readonly __tag: unique symbol }
export type RefCell = { readonly __tag: unique symbol }
export type NapiRouteHas = {
type: string
......@@ -172,6 +173,8 @@ export interface NapiDefineEnv {
nodejs: Array<NapiEnvVar>
}
export interface NapiTurboEngineOptions {
/** Use the new backend with persistent caching enabled. */
persistentCaching?: boolean
/** An upper bound of memory that turbopack will attempt to stay under. */
memoryLimit?: number
}
......@@ -364,11 +367,12 @@ export interface NapiRewrite {
}
export function createTurboTasks(
outputPath: string,
persistentCaching: boolean,
memoryLimit?: number | undefined | null
): ExternalObject<TurboTasks>
): ExternalObject<NextTurboTasks>
export function runTurboTracing(
options: Buffer,
turboTasks: ExternalObject<TurboTasks>
turboTasks: ExternalObject<NextTurboTasks>
): Promise<Array<string>>
export function getTargetTriple(): string
export function initHeapProfiler(): ExternalObject<RefCell>
......
......@@ -26,7 +26,7 @@ import type {
ExternalObject,
NapiPartialProjectOptions,
NapiProjectOptions,
TurboTasks,
NextTurboTasks,
} from './generated-native'
import type {
Binding,
......@@ -1055,8 +1055,9 @@ async function loadWasm(importPath = '') {
},
createTurboTasks: function (
_outputPath: string,
_persistentCaching: boolean,
_memoryLimit?: number | undefined
): ExternalObject<TurboTasks> {
): ExternalObject<NextTurboTasks> {
throw new Error(
'`turbo.createTurboTasks` is not supported by the wasm bindings.'
)
......@@ -1228,7 +1229,7 @@ function loadNative(importPath?: string) {
initHeapProfiler: bindings.initHeapProfiler,
teardownHeapProfiler: bindings.teardownHeapProfiler,
turbo: {
startTrace(options = {}, turboTasks: ExternalObject<TurboTasks>) {
startTrace(options = {}, turboTasks: ExternalObject<NextTurboTasks>) {
initHeapProfiler()
return (customBindings ?? bindings).runTurboTracing(
toBuffer({ exact: true, ...options }),
......@@ -1237,9 +1238,14 @@ function loadNative(importPath?: string) {
},
createTurboTasks(
outputPath: string,
persistentCaching: boolean,
memoryLimit?: number
): ExternalObject<TurboTasks> {
return bindings.createTurboTasks(outputPath, memoryLimit)
): ExternalObject<NextTurboTasks> {
return bindings.createTurboTasks(
outputPath,
persistentCaching,
memoryLimit
)
},
createProject: bindingToApi(customBindings ?? bindings, false),
startTurbopackTraceServer(traceFilePath) {
......
import type { NextConfigComplete } from '../../server/config-shared'
import type { __ApiPreviewProps } from '../../server/api-utils'
import type { ExternalObject, RefCell, TurboTasks } from './generated-native'
import type {
ExternalObject,
NextTurboTasks,
RefCell,
} from './generated-native'
export interface Binding {
isWasm: boolean
turbo: {
startTrace(options: any, turboTasks: ExternalObject<TurboTasks>): any
startTrace(options: any, turboTasks: ExternalObject<NextTurboTasks>): any
createTurboTasks(
outputPath: string,
persistentCaching: boolean,
memoryLimit?: number
): ExternalObject<TurboTasks>
): ExternalObject<NextTurboTasks>
createProject(
options: ProjectOptions,
turboEngineOptions?: TurboEngineOptions
......@@ -107,6 +112,11 @@ export type TurbopackResult<T = {}> = T & {
}
export interface TurboEngineOptions {
/**
* Use the new backend with persistent caching enabled.
*/
persistentCaching?: boolean
/**
* An upper bound of memory that turbopack will attempt to stay under.
*/
......
......@@ -391,6 +391,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
resolveExtensions: z.array(z.string()).optional(),
useSwcCss: z.boolean().optional(),
treeShaking: z.boolean().optional(),
persistentCaching: z.boolean().optional(),
memoryLimit: z.number().optional(),
moduleIdStrategy: z.enum(['named', 'deterministic']).optional(),
})
......
......@@ -154,6 +154,11 @@ export interface ExperimentalTurboOptions {
*/
memoryLimit?: number
/**
* Enable persistent caching for the turbopack dev server and build.
*/
persistentCaching?: boolean
/**
* Enable tree shaking for the turbopack dev server and build.
*/
......
......@@ -176,6 +176,7 @@ export async function createHotReloaderTurbopack(
browserslistQuery: supportedBrowsers.join(', '),
},
{
persistentCaching: opts.nextConfig.experimental.turbo?.persistentCaching,
memoryLimit: opts.nextConfig.experimental.turbo?.memoryLimit,
}
)
......
......@@ -1529,8 +1529,9 @@ impl Backend for TurboTasksBackend {
) -> TaskId {
self.0.create_transient_task(task_type)
}
fn dispose_root_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi<Self>) {
todo!()
// TODO implement
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment