Unverified Commit d1ec45e9 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] Preparations for persisting (#70468)

### What?

parallel filesystem invalidation

use turbo_tasks spawn blocking
parent cc3eb6f0
......@@ -5290,9 +5290,9 @@ checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9"
[[package]]
name = "rayon"
version = "1.7.0"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
......@@ -5300,14 +5300,12 @@ dependencies = [
[[package]]
name = "rayon-core"
version = "1.11.0"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]]
......@@ -8672,6 +8670,7 @@ dependencies = [
"mime",
"notify",
"parking_lot",
"rayon",
"rstest",
"serde",
"serde_bytes",
......
......@@ -179,6 +179,7 @@ proc-macro2 = "1.0.79"
qstring = "0.7.2"
quote = "1.0.23"
rand = "0.8.5"
rayon = "1.10.0"
regex = "1.10.6"
rstest = "0.16.0"
rustc-hash = "1.1.0"
......
......@@ -216,21 +216,19 @@ impl ProjectContainer {
impl ProjectContainer {
#[tracing::instrument(level = "info", name = "initialize project", skip_all)]
pub async fn initialize(self: Vc<Self>, options: ProjectOptions) -> Result<()> {
let poll_interval = options.watch.poll_interval;
let watch = options.watch;
self.await?.options_state.set(Some(options));
let project = self.project();
project
.project_fs()
.strongly_consistent()
.await?
.start_watching_with_invalidation_reason(poll_interval)?;
project
.output_fs()
.strongly_consistent()
.await?
.invalidate_with_reason();
let project_fs = project.project_fs().strongly_consistent().await?;
if watch.enable {
project_fs.start_watching_with_invalidation_reason(watch.poll_interval)?;
} else {
project_fs.invalidate_with_reason();
}
let output_fs = project.output_fs().strongly_consistent().await?;
output_fs.invalidate_with_reason();
Ok(())
}
......@@ -293,20 +291,23 @@ impl ProjectContainer {
}
// TODO: Handle mode switch, should prevent mode being switched.
let watch = new_options.watch;
let project = self.project();
let prev_project_fs = project.project_fs().strongly_consistent().await?;
let prev_output_fs = project.output_fs().strongly_consistent().await?;
let poll_interval = new_options.watch.poll_interval;
this.options_state.set(Some(new_options));
let project_fs = project.project_fs().strongly_consistent().await?;
let output_fs = project.output_fs().strongly_consistent().await?;
if !ReadRef::ptr_eq(&prev_project_fs, &project_fs) {
// TODO stop watching: prev_project_fs.stop_watching()?;
project_fs.start_watching_with_invalidation_reason(poll_interval)?;
if watch.enable {
// TODO stop watching: prev_project_fs.stop_watching()?;
project_fs.start_watching_with_invalidation_reason(watch.poll_interval)?;
} else {
project_fs.invalidate_with_reason();
}
}
if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) {
prev_output_fs.invalidate_with_reason();
......@@ -533,18 +534,12 @@ impl Project {
}
#[turbo_tasks::function]
async fn project_fs(&self) -> Result<Vc<DiskFileSystem>> {
let disk_fs = DiskFileSystem::new(
fn project_fs(&self) -> Vc<DiskFileSystem> {
DiskFileSystem::new(
PROJECT_FILESYSTEM_NAME.into(),
self.root_path.clone(),
vec![],
);
if self.watch.enable {
disk_fs
.await?
.start_watching_with_invalidation_reason(self.watch.poll_interval)?;
}
Ok(disk_fs)
)
}
#[turbo_tasks::function]
......
......@@ -51,7 +51,7 @@ pub struct AppDirModules {
pub default: Option<Vc<FileSystemPath>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub route: Option<Vc<FileSystemPath>>,
#[serde(skip_serializing_if = "Metadata::is_empty")]
#[serde(skip_serializing_if = "Metadata::is_empty", default)]
pub metadata: Metadata,
}
......@@ -137,13 +137,13 @@ impl From<MetadataWithAltItem> for MetadataItem {
/// Metadata file that can be placed in any segment of the app directory.
#[derive(Default, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, TraceRawVcs)]
pub struct Metadata {
#[serde(skip_serializing_if = "Vec::is_empty")]
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub icon: Vec<MetadataWithAltItem>,
#[serde(skip_serializing_if = "Vec::is_empty")]
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub apple: Vec<MetadataWithAltItem>,
#[serde(skip_serializing_if = "Vec::is_empty")]
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub twitter: Vec<MetadataWithAltItem>,
#[serde(skip_serializing_if = "Vec::is_empty")]
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub open_graph: Vec<MetadataWithAltItem>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sitemap: Option<MetadataItem>,
......
......@@ -57,7 +57,7 @@ impl Default for MiddlewaresManifest {
Serialize,
Deserialize,
)]
#[serde(rename_all = "camelCase")]
#[serde(rename_all = "camelCase", default)]
pub struct MiddlewareMatcher {
// When skipped next.js with fill that during merging.
#[serde(skip_serializing_if = "Option::is_none")]
......@@ -308,3 +308,39 @@ pub struct ClientBuildManifest<'a> {
#[serde(flatten)]
pub pages: HashMap<RcStr, Vec<&'a str>>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_middleware_matcher_serialization() {
let matchers = vec![
MiddlewareMatcher {
regexp: None,
locale: false,
has: None,
missing: None,
original_source: "".into(),
},
MiddlewareMatcher {
regexp: Some(".*".into()),
locale: true,
has: Some(vec![RouteHas::Query {
key: "foo".into(),
value: None,
}]),
missing: Some(vec![RouteHas::Query {
key: "bar".into(),
value: Some("value".into()),
}]),
original_source: "source".into(),
},
];
let serialized = serde_json::to_string(&matchers).unwrap();
let deserialized: Vec<MiddlewareMatcher> = serde_json::from_str(&serialized).unwrap();
assert_eq!(matchers, deserialized);
}
}
......@@ -1492,7 +1492,15 @@ export default async function build(
'Building'
)
const promises: Promise<any>[] = []
const sema = new Sema(10)
// Concurrency will start at INITIAL_CONCURRENCY and
// slowly ramp up to CONCURRENCY by increasing the
// concurrency by 1 every time a task is completed.
const INITIAL_CONCURRENCY = 5
const CONCURRENCY = 10
const sema = new Sema(INITIAL_CONCURRENCY)
let remainingRampup = CONCURRENCY - INITIAL_CONCURRENCY
const enqueue = (fn: () => Promise<void>) => {
promises.push(
(async () => {
......@@ -1501,6 +1509,10 @@ export default async function build(
await fn()
} finally {
sema.release()
if (remainingRampup > 0) {
remainingRampup--
sema.release()
}
progress()
}
})()
......
......@@ -39,6 +39,7 @@ jsonc-parser = { version = "0.21.0", features = ["serde"] }
mime = { workspace = true }
notify = { workspace = true }
parking_lot = { workspace = true }
rayon = { workspace = true }
serde = { workspace = true, features = ["rc"] }
serde_bytes = { workspace = true }
serde_json = { workspace = true }
......
......@@ -51,7 +51,7 @@ impl InvalidationReasonKind for WatchChangeKind {
/// Invalidation was caused by a directory starting to watch from which was read
/// before.
#[derive(PartialEq, Eq, Hash)]
#[derive(PartialEq, Eq, Hash, Clone)]
pub struct WatchStart {
pub name: RcStr,
pub path: RcStr,
......
......@@ -48,6 +48,7 @@ use invalidation::InvalidateFilesystem;
use invalidator_map::InvalidatorMap;
use jsonc_parser::{parse_to_serde_value, ParseOptions};
use mime::Mime;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use read_glob::read_glob;
pub use read_glob::ReadGlobResult;
use serde::{Deserialize, Serialize};
......@@ -273,30 +274,43 @@ impl DiskFileSystem {
pub fn invalidate(&self) {
let _span = tracing::info_span!("invalidate filesystem", path = &*self.root).entered();
for (_, invalidators) in take(&mut *self.invalidator_map.lock().unwrap()).into_iter() {
invalidators.into_iter().for_each(|i| i.invalidate());
}
for (_, invalidators) in take(&mut *self.dir_invalidator_map.lock().unwrap()).into_iter() {
invalidators.into_iter().for_each(|i| i.invalidate());
}
let span = tracing::Span::current();
let handle = tokio::runtime::Handle::current();
let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap());
let iter = invalidator_map
.into_par_iter()
.chain(dir_invalidator_map.into_par_iter())
.flat_map(|(_, invalidators)| invalidators.into_par_iter());
iter.for_each(|i| {
let _span = span.clone().entered();
let _guard = handle.enter();
i.invalidate()
});
self.serialization_invalidator.invalidate();
}
pub fn invalidate_with_reason(&self) {
let _span = tracing::info_span!("invalidate filesystem", path = &*self.root).entered();
for (path, invalidators) in take(&mut *self.invalidator_map.lock().unwrap()).into_iter() {
let reason = InvalidateFilesystem { path: path.into() };
invalidators
.into_iter()
.for_each(|i| i.invalidate_with_reason(reason.clone()));
}
for (path, invalidators) in take(&mut *self.dir_invalidator_map.lock().unwrap()).into_iter()
{
let reason = InvalidateFilesystem { path: path.into() };
invalidators
.into_iter()
.for_each(|i| i.invalidate_with_reason(reason.clone()));
}
let span = tracing::Span::current();
let handle = tokio::runtime::Handle::current();
let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap());
let iter = invalidator_map
.into_par_iter()
.chain(dir_invalidator_map.into_par_iter())
.flat_map(|(path, invalidators)| {
let _span = span.clone().entered();
let reason = InvalidateFilesystem { path: path.into() };
invalidators
.into_par_iter()
.map(move |i| (reason.clone(), i))
});
iter.for_each(|(reason, invalidator)| {
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate_with_reason(reason)
});
self.serialization_invalidator.invalidate();
}
......@@ -333,6 +347,7 @@ impl DiskFileSystem {
invalidator_map,
dir_invalidator_map,
poll_interval,
self.serialization_invalidator.clone(),
)?;
self.serialization_invalidator.invalidate();
......@@ -441,11 +456,11 @@ impl DiskFileSystem {
// we use the sync std function here as it's a lot faster (600%) in
// node-file-trace
let read_dir = match retry_blocking(
&full_path,
tracing::info_span!("read directory", path = display(full_path.display())),
|path| std::fs::read_dir(path),
)
let read_dir = match retry_blocking(&full_path, |path| {
let _span =
tracing::info_span!("read directory", path = display(path.display())).entered();
std::fs::read_dir(path)
})
.await
{
Ok(dir) => dir,
......@@ -810,26 +825,25 @@ impl FileSystem for DiskFileSystem {
} else {
PathBuf::from(unix_to_sys(target).as_ref())
};
retry_blocking(
&target_path,
tracing::info_span!("write symlink", path = display(full_path.display())),
move |target_path| {
// we use the sync std method here because `symlink` is fast
// if we put it into a task, it will be slower
#[cfg(not(target_family = "windows"))]
{
std::os::unix::fs::symlink(target_path, &full_path)
}
#[cfg(target_family = "windows")]
{
if link_type.contains(LinkType::DIRECTORY) {
std::os::windows::fs::symlink_dir(target_path, &full_path)
} else {
std::os::windows::fs::symlink_file(target_path, &full_path)
}
retry_blocking(&target_path, move |target_path| {
let _span =
tracing::info_span!("write symlink", path = display(target_path.display()))
.entered();
// we use the sync std method here because `symlink` is fast
// if we put it into a task, it will be slower
#[cfg(not(target_family = "windows"))]
{
std::os::unix::fs::symlink(target_path, &full_path)
}
#[cfg(target_family = "windows")]
{
if link_type.contains(LinkType::DIRECTORY) {
std::os::windows::fs::symlink_dir(target_path, &full_path)
} else {
std::os::windows::fs::symlink_file(target_path, &full_path)
}
},
)
}
})
.await
.with_context(|| format!("create symlink to {}", target))?;
}
......
use std::{future::Future, io, io::ErrorKind, path::Path, thread::sleep, time::Duration};
use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy};
use tokio::task::spawn_blocking;
const MAX_RETRY_ATTEMPTS: usize = 10;
......@@ -23,21 +22,14 @@ where
}
}
pub(crate) async fn retry_blocking<R, F>(
path: impl AsRef<Path>,
span: tracing::Span,
func: F,
) -> io::Result<R>
pub(crate) async fn retry_blocking<R, F>(path: impl AsRef<Path>, func: F) -> io::Result<R>
where
F: Fn(&Path) -> io::Result<R> + Send + 'static,
R: Send + 'static,
{
let path = path.as_ref().to_owned();
let current_span = tracing::Span::current();
asyncify(move || {
let _entered = current_span.entered();
let _entered = span.entered();
turbo_tasks::spawn_blocking(move || {
let mut attempt = 1;
loop {
......@@ -84,15 +76,3 @@ impl ErrorHandler<io::Error> for FsRetryHandler {
}
}
}
// stolen from tokio::fs
async fn asyncify<F, T>(f: F) -> io::Result<T>
where
F: FnOnce() -> io::Result<T> + Send + 'static,
T: Send + 'static,
{
match spawn_blocking(f).await {
Ok(res) => res,
Err(_) => Err(io::Error::new(ErrorKind::Other, "background task failed")),
}
}
......@@ -14,10 +14,11 @@ use notify::{
event::{MetadataKind, ModifyKind, RenameMode},
Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher,
};
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use tracing::instrument;
use turbo_tasks::{spawn_thread, Invalidator, RcStr};
use turbo_tasks::{spawn_thread, Invalidator, RcStr, SerializationInvalidator};
use crate::{
format_absolute_fs_path,
......@@ -143,6 +144,7 @@ impl DiskWatcher {
invalidator_map: Arc<InvalidatorMap>,
dir_invalidator_map: Arc<InvalidatorMap>,
poll_interval: Option<Duration>,
serialization_invalidator: SerializationInvalidator,
) -> Result<()> {
let mut watcher_guard = self.watcher.lock().unwrap();
if watcher_guard.is_some() {
......@@ -179,24 +181,41 @@ impl DiskWatcher {
// Best is to start_watching before starting to read
{
let _span = tracing::info_span!("invalidate filesystem").entered();
for (path, invalidators) in take(&mut *invalidator_map.lock().unwrap())
.into_iter()
.chain(take(&mut *dir_invalidator_map.lock().unwrap()).into_iter())
{
if report_invalidation_reason.is_some() {
let path: RcStr = path.into();
for invalidator in invalidators {
invalidator.invalidate_with_reason(WatchStart {
name: name.clone(),
path: path.clone(),
})
}
} else {
for invalidator in invalidators {
invalidator.invalidate();
}
}
let span = tracing::Span::current();
let invalidator_map = take(&mut *invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *dir_invalidator_map.lock().unwrap());
let iter = invalidator_map
.into_par_iter()
.chain(dir_invalidator_map.into_par_iter());
let handle = tokio::runtime::Handle::current();
if report_invalidation_reason.is_some() {
iter.flat_map(|(path, invalidators)| {
let _span = span.clone().entered();
let reason = WatchStart {
name: name.clone(),
path: path.into(),
};
invalidators
.into_par_iter()
.map(move |i| (reason.clone(), i))
})
.for_each(|(reason, invalidator)| {
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate_with_reason(reason)
});
} else {
iter.flat_map(|(_, invalidators)| {
let _span = span.clone().entered();
invalidators.into_par_iter().map(move |i| i)
})
.for_each(|invalidator| {
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate()
});
}
serialization_invalidator.invalidate();
}
watcher_guard.replace(watcher);
......
use anyhow::Result;
use crate::{self as turbo_tasks, RawVc, TryJoinIterExt, Vc};
/// Just an empty type, but it's never equal to itself.
/// [`Vc<Completion>`] can be used as return value instead of `()`
......@@ -74,7 +76,9 @@ impl Completions {
self.0
.iter()
.map(|&c| async move {
c.await?;
// Wraps the completion in a new completion. This makes it cheaper to restore
// since it doesn't need to restore the original task resp task chain.
wrap(c).await?;
Ok(())
})
.try_join()
......@@ -83,3 +87,9 @@ impl Completions {
}
}
}
#[turbo_tasks::function]
pub async fn wrap(completion: Vc<Completion>) -> Result<Vc<Completion>> {
completion.await?;
Ok(Completion::new())
}
......@@ -8,6 +8,7 @@ use tokio::runtime::Handle;
use crate::{manager::with_turbo_tasks, trace::TraceRawVcs, TaskId, TurboTasksApi};
#[derive(Clone)]
pub struct SerializationInvalidator {
task: TaskId,
turbo_tasks: Weak<dyn TurboTasksApi>,
......
......@@ -9,7 +9,7 @@ type VersionTraitRef = TraitRef<Box<dyn Version>>;
/// The version of a [`EcmascriptDevChunkListContent`].
///
/// [`EcmascriptDevChunkListContent`]: super::content::EcmascriptDevChunkListContent
#[turbo_tasks::value(shared)]
#[turbo_tasks::value(serialization = "none", shared)]
pub(super) struct EcmascriptDevChunkListVersion {
/// A map from chunk path to its version.
#[turbo_tasks(trace_ignore)]
......
use std::sync::Arc;
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use turbo_tasks::{
debug::ValueDebugFormat, trace::TraceRawVcs, IntoTraitRef, RcStr, ReadRef, State, TraitRef, Vc,
};
......@@ -168,7 +167,7 @@ impl Version for NotFoundVersion {
}
/// Describes an update to a versioned object.
#[turbo_tasks::value(shared)]
#[turbo_tasks::value(serialization = "none", shared)]
#[derive(Debug)]
pub enum Update {
/// The asset can't be meaningfully updated while the app is running, so the
......@@ -187,7 +186,7 @@ pub enum Update {
}
/// A total update to a versioned object.
#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat, Serialize, Deserialize)]
#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat)]
pub struct TotalUpdate {
/// The version this update will bring the object to.
#[turbo_tasks(trace_ignore)]
......@@ -195,7 +194,7 @@ pub struct TotalUpdate {
}
/// A partial update to a versioned object.
#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat, Serialize, Deserialize)]
#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat)]
pub struct PartialUpdate {
/// The version this update will bring the object to.
#[turbo_tasks(trace_ignore)]
......@@ -241,7 +240,7 @@ impl Version for FileHashVersion {
}
}
#[turbo_tasks::value]
#[turbo_tasks::value(serialization = "none")]
pub struct VersionState {
#[turbo_tasks(trace_ignore)]
version: State<TraitRef<Box<dyn Version>>>,
......
......@@ -18,7 +18,7 @@ flate2 = { version = "1.0.28" }
indexmap = { workspace = true, features = ["serde"] }
itertools = { workspace = true }
postcard = { workspace = true }
rayon = "1"
rayon = { workspace = true }
rustc-demangle = "0.1"
serde = { workspace = true }
serde_json = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment