Unverified Commit 6de83774 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] store project options in state (#69658)

### What?

We can't pass the whole config (including next config) as argument to `ProjectContainer::new` as this would result in a new project every time the config is different. (It's always different due to some random values).

So to leverage persistent caching we need to remove the argument from `ProjectContainer::new` (so it's always the same project) and push the config into a `State`. This way we "modify" the project and can benefit from caching for the unchanged values.
parent 158a7400
...@@ -331,11 +331,12 @@ pub async fn project_new( ...@@ -331,11 +331,12 @@ pub async fn project_new(
.unwrap(); .unwrap();
}); });
} }
let options = options.into(); let options: ProjectOptions = options.into();
let container = turbo_tasks let container = turbo_tasks
.run_once(async move { .run_once(async move {
let project = ProjectContainer::new(options); let project = ProjectContainer::new("next.js".into(), options.dev);
let project = project.resolve().await?; let project = project.resolve().await?;
project.initialize(options).await?;
Ok(project) Ok(project)
}) })
.await .await
...@@ -365,9 +366,6 @@ pub async fn project_new( ...@@ -365,9 +366,6 @@ pub async fn project_new(
/// - https://github.com/oven-sh/bun/blob/06a9aa80c38b08b3148bfeabe560/src/install/install.zig#L3038 /// - https://github.com/oven-sh/bun/blob/06a9aa80c38b08b3148bfeabe560/src/install/install.zig#L3038
#[tracing::instrument] #[tracing::instrument]
async fn benchmark_file_io(directory: Vc<FileSystemPath>) -> Result<Vc<Completion>> { async fn benchmark_file_io(directory: Vc<FileSystemPath>) -> Result<Vc<Completion>> {
let temp_path =
directory.join(format!("tmp_file_io_benchmark_{:x}", rand::random::<u128>()).into());
// try to get the real file path on disk so that we can use it with tokio // try to get the real file path on disk so that we can use it with tokio
let fs = Vc::try_resolve_downcast_type::<DiskFileSystem>(directory.fs()) let fs = Vc::try_resolve_downcast_type::<DiskFileSystem>(directory.fs())
.await? .await?
...@@ -375,7 +373,12 @@ async fn benchmark_file_io(directory: Vc<FileSystemPath>) -> Result<Vc<Completio ...@@ -375,7 +373,12 @@ async fn benchmark_file_io(directory: Vc<FileSystemPath>) -> Result<Vc<Completio
"expected node_root to be a DiskFileSystem, cannot benchmark" "expected node_root to be a DiskFileSystem, cannot benchmark"
))? ))?
.await?; .await?;
let temp_path = fs.to_sys_path(temp_path).await?;
let directory = fs.to_sys_path(directory).await?;
let temp_path = directory.join(format!(
"tmp_file_io_benchmark_{:x}",
rand::random::<u128>()
));
let mut random_buffer = [0u8; 512]; let mut random_buffer = [0u8; 512];
rand::thread_rng().fill(&mut random_buffer[..]); rand::thread_rng().fill(&mut random_buffer[..]);
...@@ -404,7 +407,7 @@ async fn benchmark_file_io(directory: Vc<FileSystemPath>) -> Result<Vc<Completio ...@@ -404,7 +407,7 @@ async fn benchmark_file_io(directory: Vc<FileSystemPath>) -> Result<Vc<Completio
println!( println!(
"Slow filesystem detected. If {} is a network drive, consider moving it to a local \ "Slow filesystem detected. If {} is a network drive, consider moving it to a local \
folder. If you have an antivirus enabled, consider excluding your project directory.", folder. If you have an antivirus enabled, consider excluding your project directory.",
fs.to_sys_path(directory).await?.to_string_lossy(), directory.to_string_lossy(),
); );
} }
......
use std::path::MAIN_SEPARATOR; use std::path::MAIN_SEPARATOR;
use anyhow::{bail, Result}; use anyhow::{bail, Context, Result};
use indexmap::{indexmap, map::Entry, IndexMap}; use indexmap::{indexmap, map::Entry, IndexMap};
use next_core::{ use next_core::{
all_assets_from_entries, all_assets_from_entries,
...@@ -26,8 +26,8 @@ use turbo_tasks::{ ...@@ -26,8 +26,8 @@ use turbo_tasks::{
debug::ValueDebugFormat, debug::ValueDebugFormat,
graph::{AdjacencyMap, GraphTraversal}, graph::{AdjacencyMap, GraphTraversal},
trace::TraceRawVcs, trace::TraceRawVcs,
Completion, Completions, IntoTraitRef, RcStr, State, TaskInput, TraitRef, TransientInstance, Completion, Completions, IntoTraitRef, RcStr, ReadRef, State, TaskInput, TraitRef,
TryFlatJoinIterExt, Value, Vc, TransientInstance, TryFlatJoinIterExt, Value, Vc,
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem};
...@@ -173,25 +173,46 @@ pub struct Instrumentation { ...@@ -173,25 +173,46 @@ pub struct Instrumentation {
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct ProjectContainer { pub struct ProjectContainer {
options_state: State<ProjectOptions>, name: RcStr,
options_state: State<Option<ProjectOptions>>,
versioned_content_map: Option<Vc<VersionedContentMap>>, versioned_content_map: Option<Vc<VersionedContentMap>>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ProjectContainer { impl ProjectContainer {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(options: ProjectOptions) -> Vc<Self> { pub fn new(name: RcStr, dev: bool) -> Vc<Self> {
ProjectContainer { ProjectContainer {
name,
// we only need to enable versioning in dev mode, since build // we only need to enable versioning in dev mode, since build
// is assumed to be operating over a static snapshot // is assumed to be operating over a static snapshot
versioned_content_map: options.dev.then(VersionedContentMap::new), versioned_content_map: dev.then(VersionedContentMap::new),
options_state: State::new(options), options_state: State::new(None),
} }
.cell() .cell()
} }
}
#[turbo_tasks::function] impl ProjectContainer {
pub fn update(&self, options: PartialProjectOptions) -> Vc<()> { #[tracing::instrument(level = "info", name = "initialize project", skip_all)]
pub async fn initialize(self: Vc<Self>, options: ProjectOptions) -> Result<()> {
self.await?.options_state.set(Some(options));
let project = self.project();
project
.project_fs()
.strongly_consistent()
.await?
.start_watching_with_invalidation_reason()?;
project
.output_fs()
.strongly_consistent()
.await?
.invalidate_with_reason();
Ok(())
}
#[tracing::instrument(level = "info", name = "update project", skip_all)]
pub async fn update(self: Vc<Self>, options: PartialProjectOptions) -> Result<()> {
let PartialProjectOptions { let PartialProjectOptions {
root_path, root_path,
project_path, project_path,
...@@ -206,7 +227,13 @@ impl ProjectContainer { ...@@ -206,7 +227,13 @@ impl ProjectContainer {
preview_props, preview_props,
} = options; } = options;
let mut new_options = self.options_state.get().clone(); let this = self.await?;
let mut new_options = this
.options_state
.get()
.clone()
.context("ProjectContainer need to be initialized with initialize()")?;
if let Some(root_path) = root_path { if let Some(root_path) = root_path {
new_options.root_path = root_path; new_options.root_path = root_path;
...@@ -244,11 +271,28 @@ impl ProjectContainer { ...@@ -244,11 +271,28 @@ impl ProjectContainer {
// TODO: Handle mode switch, should prevent mode being switched. // TODO: Handle mode switch, should prevent mode being switched.
self.options_state.set(new_options); let project = self.project();
let prev_project_fs = project.project_fs().strongly_consistent().await?;
let prev_output_fs = project.output_fs().strongly_consistent().await?;
this.options_state.set(Some(new_options));
let project_fs = project.project_fs().strongly_consistent().await?;
let output_fs = project.output_fs().strongly_consistent().await?;
if !ReadRef::ptr_eq(&prev_project_fs, &project_fs) {
// TODO stop watching: prev_project_fs.stop_watching()?;
project_fs.start_watching_with_invalidation_reason()?;
}
if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) {
prev_output_fs.invalidate_with_reason();
}
Default::default() Ok(())
} }
}
#[turbo_tasks::value_impl]
impl ProjectContainer {
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn project(self: Vc<Self>) -> Result<Vc<Project>> { pub async fn project(self: Vc<Self>) -> Result<Vc<Project>> {
let this = self.await?; let this = self.await?;
...@@ -266,6 +310,9 @@ impl ProjectContainer { ...@@ -266,6 +310,9 @@ impl ProjectContainer {
let preview_props; let preview_props;
{ {
let options = this.options_state.get(); let options = this.options_state.get();
let options = options
.as_ref()
.context("ProjectContainer need to be initialized with initialize()")?;
env_map = Vc::cell(options.env.iter().cloned().collect()); env_map = Vc::cell(options.env.iter().cloned().collect());
define_env = ProjectDefineEnv { define_env = ProjectDefineEnv {
client: Vc::cell(options.define_env.client.iter().cloned().collect()), client: Vc::cell(options.define_env.client.iter().cloned().collect()),
...@@ -462,7 +509,7 @@ impl Project { ...@@ -462,7 +509,7 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn project_fs(self: Vc<Self>) -> Result<Vc<Box<dyn FileSystem>>> { async fn project_fs(self: Vc<Self>) -> Result<Vc<DiskFileSystem>> {
let this = self.await?; let this = self.await?;
let disk_fs = DiskFileSystem::new( let disk_fs = DiskFileSystem::new(
PROJECT_FILESYSTEM_NAME.into(), PROJECT_FILESYSTEM_NAME.into(),
...@@ -472,7 +519,7 @@ impl Project { ...@@ -472,7 +519,7 @@ impl Project {
if this.watch { if this.watch {
disk_fs.await?.start_watching_with_invalidation_reason()?; disk_fs.await?.start_watching_with_invalidation_reason()?;
} }
Ok(Vc::upcast(disk_fs)) Ok(disk_fs)
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -482,10 +529,10 @@ impl Project { ...@@ -482,10 +529,10 @@ impl Project {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn output_fs(self: Vc<Self>) -> Result<Vc<Box<dyn FileSystem>>> { pub async fn output_fs(self: Vc<Self>) -> Result<Vc<DiskFileSystem>> {
let this = self.await?; let this = self.await?;
let disk_fs = DiskFileSystem::new("output".into(), this.project_path.clone(), vec![]); let disk_fs = DiskFileSystem::new("output".into(), this.project_path.clone(), vec![]);
Ok(Vc::upcast(disk_fs)) Ok(disk_fs)
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -38,7 +38,12 @@ pub async fn main_inner( ...@@ -38,7 +38,12 @@ pub async fn main_inner(
} }
let project = tt let project = tt
.run_once(async { Ok(ProjectContainer::new(options)) }) .run_once(async {
let project = ProjectContainer::new("next-build-test".into(), options.dev);
let project = project.resolve().await?;
project.initialize(options).await?;
Ok(project)
})
.await?; .await?;
tracing::info!("collecting endpoints"); tracing::info!("collecting endpoints");
......
...@@ -192,7 +192,7 @@ async fn create_fs(name: &str, root: &str, watch: bool) -> Result<Vc<Box<dyn Fil ...@@ -192,7 +192,7 @@ async fn create_fs(name: &str, root: &str, watch: bool) -> Result<Vc<Box<dyn Fil
if watch { if watch {
fs.await?.start_watching()?; fs.await?.start_watching()?;
} else { } else {
fs.await?.invalidate(); fs.await?.invalidate_with_reason();
} }
Ok(Vc::upcast(fs)) Ok(Vc::upcast(fs))
} }
......
use std::fmt::{Display, Formatter}; use std::{
fmt::{Display, Formatter},
hash::Hash,
};
use indexmap::IndexSet; use indexmap::IndexSet;
use turbo_tasks::{util::StaticOrArc, InvalidationReason, InvalidationReasonKind, RcStr}; use turbo_tasks::{util::StaticOrArc, InvalidationReason, InvalidationReasonKind, RcStr};
...@@ -51,6 +54,7 @@ impl InvalidationReasonKind for WatchChangeKind { ...@@ -51,6 +54,7 @@ impl InvalidationReasonKind for WatchChangeKind {
#[derive(PartialEq, Eq, Hash)] #[derive(PartialEq, Eq, Hash)]
pub struct WatchStart { pub struct WatchStart {
pub name: RcStr, pub name: RcStr,
pub path: RcStr,
} }
impl InvalidationReason for WatchStart { impl InvalidationReason for WatchStart {
...@@ -61,7 +65,7 @@ impl InvalidationReason for WatchStart { ...@@ -61,7 +65,7 @@ impl InvalidationReason for WatchStart {
impl Display for WatchStart { impl Display for WatchStart {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} started watching", self.name) write!(f, "started watching {} in {}", self.path, self.name)
} }
} }
...@@ -77,15 +81,13 @@ impl InvalidationReasonKind for WatchStartKind { ...@@ -77,15 +81,13 @@ impl InvalidationReasonKind for WatchStartKind {
reasons: &IndexSet<StaticOrArc<dyn InvalidationReason>>, reasons: &IndexSet<StaticOrArc<dyn InvalidationReason>>,
f: &mut Formatter<'_>, f: &mut Formatter<'_>,
) -> std::fmt::Result { ) -> std::fmt::Result {
let example = reasons[0].as_any().downcast_ref::<WatchStart>().unwrap();
write!( write!(
f, f,
"{} directories started watching (e. g. {})", "{} items started watching (e. g. {} in {})",
reasons.len(), reasons.len(),
reasons[0] example.path,
.as_any() example.name
.downcast_ref::<WatchStart>()
.unwrap()
.name
) )
} }
} }
...@@ -128,3 +130,46 @@ impl InvalidationReasonKind for WriteKind { ...@@ -128,3 +130,46 @@ impl InvalidationReasonKind for WriteKind {
) )
} }
} }
/// Invalidation was caused by a invalidate operation on the filesystem
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct InvalidateFilesystem {
pub path: RcStr,
}
impl InvalidationReason for InvalidateFilesystem {
fn kind(&self) -> Option<StaticOrArc<dyn InvalidationReasonKind>> {
Some(StaticOrArc::Static(&INVALIDATE_FILESYSTEM_KIND))
}
}
impl Display for InvalidateFilesystem {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{} in filesystem invalidated", self.path)
}
}
/// Invalidation kind for [InvalidateFilesystem]
#[derive(PartialEq, Eq, Hash)]
struct InvalidateFilesystemKind;
static INVALIDATE_FILESYSTEM_KIND: InvalidateFilesystemKind = InvalidateFilesystemKind;
impl InvalidationReasonKind for InvalidateFilesystemKind {
fn fmt(
&self,
reasons: &IndexSet<StaticOrArc<dyn InvalidationReason>>,
f: &mut Formatter<'_>,
) -> std::fmt::Result {
write!(
f,
"{} items in filesystem invalidated ({}, ...)",
reasons.len(),
reasons[0]
.as_any()
.downcast_ref::<InvalidateFilesystem>()
.unwrap()
.path
)
}
}
...@@ -43,6 +43,7 @@ use auto_hash_map::AutoMap; ...@@ -43,6 +43,7 @@ use auto_hash_map::AutoMap;
use bitflags::bitflags; use bitflags::bitflags;
use dunce::simplified; use dunce::simplified;
use glob::Glob; use glob::Glob;
use invalidation::InvalidateFilesystem;
use invalidator_map::InvalidatorMap; use invalidator_map::InvalidatorMap;
use jsonc_parser::{parse_to_serde_value, ParseOptions}; use jsonc_parser::{parse_to_serde_value, ParseOptions};
use mime::Mime; use mime::Mime;
...@@ -57,8 +58,7 @@ use tokio::{ ...@@ -57,8 +58,7 @@ use tokio::{
}; };
use tracing::Instrument; use tracing::Instrument;
use turbo_tasks::{ use turbo_tasks::{
mark_stateful, trace::TraceRawVcs, Completion, InvalidationReason, Invalidator, RcStr, ReadRef, mark_stateful, trace::TraceRawVcs, Completion, Invalidator, RcStr, ReadRef, ValueToString, Vc,
ValueToString, Vc,
}; };
use turbo_tasks_hash::{hash_xxh3_hash64, DeterministicHash, DeterministicHasher}; use turbo_tasks_hash::{hash_xxh3_hash64, DeterministicHash, DeterministicHasher};
use util::{extract_disk_access, join_path, normalize_path, sys_to_unix, unix_to_sys}; use util::{extract_disk_access, join_path, normalize_path, sys_to_unix, unix_to_sys};
...@@ -165,6 +165,7 @@ impl DiskFileSystem { ...@@ -165,6 +165,7 @@ impl DiskFileSystem {
} }
pub fn invalidate(&self) { pub fn invalidate(&self) {
let _span = tracing::info_span!("invalidate filesystem", path = &*self.root).entered();
for (_, invalidators) in take(&mut *self.invalidator_map.lock().unwrap()).into_iter() { for (_, invalidators) in take(&mut *self.invalidator_map.lock().unwrap()).into_iter() {
invalidators.into_iter().for_each(|i| i.invalidate()); invalidators.into_iter().for_each(|i| i.invalidate());
} }
...@@ -173,13 +174,17 @@ impl DiskFileSystem { ...@@ -173,13 +174,17 @@ impl DiskFileSystem {
} }
} }
pub fn invalidate_with_reason<T: InvalidationReason + Clone>(&self, reason: T) { pub fn invalidate_with_reason(&self) {
for (_, invalidators) in take(&mut *self.invalidator_map.lock().unwrap()).into_iter() { let _span = tracing::info_span!("invalidate filesystem", path = &*self.root).entered();
for (path, invalidators) in take(&mut *self.invalidator_map.lock().unwrap()).into_iter() {
let reason = InvalidateFilesystem { path: path.into() };
invalidators invalidators
.into_iter() .into_iter()
.for_each(|i| i.invalidate_with_reason(reason.clone())); .for_each(|i| i.invalidate_with_reason(reason.clone()));
} }
for (_, invalidators) in take(&mut *self.dir_invalidator_map.lock().unwrap()).into_iter() { for (path, invalidators) in take(&mut *self.dir_invalidator_map.lock().unwrap()).into_iter()
{
let reason = InvalidateFilesystem { path: path.into() };
invalidators invalidators
.into_iter() .into_iter()
.for_each(|i| i.invalidate_with_reason(reason.clone())); .for_each(|i| i.invalidate_with_reason(reason.clone()));
...@@ -195,6 +200,7 @@ impl DiskFileSystem { ...@@ -195,6 +200,7 @@ impl DiskFileSystem {
} }
fn start_watching_internal(&self, report_invalidation_reason: bool) -> Result<()> { fn start_watching_internal(&self, report_invalidation_reason: bool) -> Result<()> {
let _span = tracing::info_span!("start filesystem watching", path = &*self.root).entered();
let invalidator_map = self.invalidator_map.clone(); let invalidator_map = self.invalidator_map.clone();
let dir_invalidator_map = self.dir_invalidator_map.clone(); let dir_invalidator_map = self.dir_invalidator_map.clone();
let root_path = self.root_path().to_path_buf(); let root_path = self.root_path().to_path_buf();
......
...@@ -153,15 +153,25 @@ impl DiskWatcher { ...@@ -153,15 +153,25 @@ impl DiskWatcher {
// We need to invalidate all reads that happened before watching // We need to invalidate all reads that happened before watching
// Best is to start_watching before starting to read // Best is to start_watching before starting to read
for invalidator in take(&mut *invalidator_map.lock().unwrap())
.into_iter()
.chain(take(&mut *dir_invalidator_map.lock().unwrap()).into_iter())
.flat_map(|(_, invalidators)| invalidators.into_iter())
{ {
if report_invalidation_reason.is_some() { let _span = tracing::info_span!("invalidate filesystem").entered();
invalidator.invalidate_with_reason(WatchStart { name: name.clone() }) for (path, invalidators) in take(&mut *invalidator_map.lock().unwrap())
} else { .into_iter()
invalidator.invalidate(); .chain(take(&mut *dir_invalidator_map.lock().unwrap()).into_iter())
{
if report_invalidation_reason.is_some() {
let path: RcStr = path.into();
for invalidator in invalidators {
invalidator.invalidate_with_reason(WatchStart {
name: name.clone(),
path: path.clone(),
})
}
} else {
for invalidator in invalidators {
invalidator.invalidate();
}
}
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment