Unverified Commit bf2ad52d authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Batch ipc info messages to improve performance. (#78522)

Batch the dependency messages sent from our nodejs worker process.

Some large projects using tailwind may report thousands of dependencies which can cause the node process to hang due to memory pressure.  We are addressing the memory pressure issue independently in #78462, but by reducing the number of small packets we can also improve performance.
parent 071e3170
...@@ -95,6 +95,8 @@ function createIpc<TIncoming, TOutgoing>( ...@@ -95,6 +95,8 @@ function createIpc<TIncoming, TOutgoing>(
process.exit(0); process.exit(0);
}); });
// TODO(lukesandberg): some of the messages being sent are very large and contain lots
// of redundant information. Consider adding gzip compression to our stream.
function send(message: any): Promise<void> { function send(message: any): Promise<void> {
// Reserve 4 bytes for our length prefix, we will over-write after encoding. // Reserve 4 bytes for our length prefix, we will over-write after encoding.
const packet = Buffer.from("0000" + JSON.stringify(message), "utf8"); const packet = Buffer.from("0000" + JSON.stringify(message), "utf8");
......
...@@ -87,6 +87,10 @@ export default async function transform( ...@@ -87,6 +87,10 @@ export default async function transform(
}); });
const assets = []; const assets = [];
const filePaths: string[] =[];
const buildFilePaths: string[] = [];
const directories: Array<[string, string]> = [];
for (const msg of messages) { for (const msg of messages) {
switch (msg.type) { switch (msg.type) {
case "asset": case "asset":
...@@ -104,36 +108,28 @@ export default async function transform( ...@@ -104,36 +108,28 @@ export default async function transform(
break; break;
case "dependency": case "dependency":
case "missing-dependency": case "missing-dependency":
ipc.sendInfo({ filePaths.push(toPath(msg.file));
type: "fileDependency",
path: toPath(msg.file),
});
break; break;
case "build-dependency": case "build-dependency":
ipc.sendInfo({ buildFilePaths.push(toPath(msg.file));
type: "buildDependency",
path: toPath(msg.file),
});
break; break;
case "dir-dependency": case "dir-dependency":
ipc.sendInfo({ directories.push([toPath(msg.dir), msg.glob]);
type: "dirDependency",
path: toPath(msg.dir),
glob: msg.glob,
});
break; break;
case "context-dependency": case "context-dependency":
ipc.sendInfo({ directories.push([toPath(msg.dir), "**"]);
type: "dirDependency",
path: toPath(msg.file),
glob: "**",
});
break; break;
default: default:
// TODO: do we need to do anything here? // TODO: do we need to do anything here?
break; break;
} }
} }
ipc.sendInfo({
type: "dependencies",
filePaths,
directories,
buildFilePaths,
});
return { return {
css, css,
map: sourceMap ? JSON.stringify(map) : undefined, map: sourceMap ? JSON.stringify(map) : undefined,
......
...@@ -19,21 +19,11 @@ import { type StructuredError } from "src/ipc"; ...@@ -19,21 +19,11 @@ import { type StructuredError } from "src/ipc";
export type IpcInfoMessage = export type IpcInfoMessage =
| { | {
type: "fileDependency"; type: 'dependencies'
path: string; envVariables?: string[]
} directories?: Array<[string, string]>
| { filePaths?: string[],
type: "buildDependency"; buildFilePaths?: string[],
path: string;
}
| {
type: "dirDependency";
path: string;
glob: string;
}
| {
type: "envDependency";
name: string;
} }
| { | {
type: "emittedError"; type: "emittedError";
...@@ -364,7 +354,8 @@ const transform = ( ...@@ -364,7 +354,8 @@ const transform = (
// TODO: do we need to handle this? // TODO: do we need to handle this?
break break
} }
// TODO(lukesandberg): should we batch these and flush lazily?
// turbopack just collects these and reports them when finishing the task.
ipc.sendInfo({ ipc.sendInfo({
type: "log", type: "log",
time: Date.now(), time: Date.now(),
...@@ -469,25 +460,15 @@ const transform = ( ...@@ -469,25 +460,15 @@ const transform = (
}, },
}, },
(err, result) => { (err, result) => {
for (const envVar of readEnvVars) {
ipc.sendInfo({ ipc.sendInfo({
type: "envDependency", type: 'dependencies',
name: envVar, envVariables: Array.from(readEnvVars),
filePaths: result.fileDependencies.map(toPath),
directories: result.contextDependencies.map((dep) => [
toPath(dep),
'**',
]),
}); });
}
for (const dep of result.contextDependencies) {
ipc.sendInfo({
type: "dirDependency",
path: toPath(dep),
glob: "**",
});
}
for (const dep of result.fileDependencies) {
ipc.sendInfo({
type: "fileDependency",
path: toPath(dep),
});
}
if (err) return reject(err); if (err) return reject(err);
if (!result.result) return reject(new Error("No result from loaders")); if (!result.result) return reject(new Error("No result from loaders"));
const [source, map] = result.result; const [source, map] = result.result;
......
...@@ -3,6 +3,7 @@ use std::mem::take; ...@@ -3,6 +3,7 @@ use std::mem::take;
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use async_trait::async_trait; use async_trait::async_trait;
use either::Either; use either::Either;
use futures::try_join;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{json, Value as JsonValue}; use serde_json::{json, Value as JsonValue};
use serde_with::serde_as; use serde_with::serde_as;
...@@ -348,18 +349,18 @@ pub struct LogInfo { ...@@ -348,18 +349,18 @@ pub struct LogInfo {
#[derive(Deserialize, Debug)] #[derive(Deserialize, Debug)]
#[serde(tag = "type", rename_all = "camelCase")] #[serde(tag = "type", rename_all = "camelCase")]
pub enum InfoMessage { pub enum InfoMessage {
FileDependency { // Sent to inform Turbopack about the dependencies of the task.
path: RcStr, // All fields are `default` since it is ok for the client to
}, // simply omit instead of sending empty arrays.
BuildDependency { Dependencies {
path: RcStr, #[serde(default)]
}, env_variables: Vec<RcStr>,
DirDependency { #[serde(default)]
path: RcStr, file_paths: Vec<RcStr>,
glob: RcStr, #[serde(default)]
}, directories: Vec<(RcStr, RcStr)>,
EnvDependency { #[serde(default)]
name: RcStr, build_file_paths: Vec<RcStr>,
}, },
EmittedError { EmittedError {
severity: IssueSeverity, severity: IssueSeverity,
...@@ -472,29 +473,60 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -472,29 +473,60 @@ impl EvaluateContext for WebpackLoaderContext {
pool: &NodeJsPool, pool: &NodeJsPool,
) -> Result<()> { ) -> Result<()> {
match data { match data {
InfoMessage::FileDependency { path } => { InfoMessage::Dependencies {
// TODO We might miss some changes that happened during execution env_variables,
// Read dependencies to make them a dependencies of this task. This task will file_paths,
// execute again when they change. directories,
self.cwd.join(path).read().await?; build_file_paths,
} } => {
InfoMessage::BuildDependency { path } => { // Track dependencies of the loader task
// TODO We might miss some changes that happened during execution // TODO: Because these are reported _after_ the loader actually read the dependency
// there is a race condition where we may miss updates that race
// with the loader execution.
// Track all the subscriptions in parallel, since certain loaders like tailwind
// might add thousands of subscriptions.
let env_subscriptions = env_variables
.iter()
.map(|e| self.env.read(e.clone()))
.try_join();
let file_subscriptions = file_paths
.iter()
.map(|p| self.cwd.join(p.clone()).read())
.try_join();
let directory_subscriptions = directories
.iter()
.map(|(dir, glob)| {
// TODO: there is some redundancy between `dir_dependency` and what
// `read_glob` does, Introduce a new read_glob
// option that will track all files the way
// `dir_dependency` does but in a single traversal.
dir_dependency(
self.cwd
.join(dir.clone())
.read_glob(Glob::new(glob.clone()), false),
)
})
.try_join();
let build_paths = build_file_paths
.iter()
.map(|path| self.cwd.join(path.clone()).to_resolved())
.try_join();
let (resolved_build_paths, ..) = try_join!(
build_paths,
env_subscriptions,
file_subscriptions,
directory_subscriptions
)?;
for build_path in resolved_build_paths {
BuildDependencyIssue { BuildDependencyIssue {
context_ident: self.context_ident_for_issue, context_ident: self.context_ident_for_issue,
path: self.cwd.join(path).to_resolved().await?, path: build_path,
} }
.resolved_cell() .resolved_cell()
.emit(); .emit();
} }
InfoMessage::DirDependency { path, glob } => {
// TODO We might miss some changes that happened during execution
// Read dependencies to make them a dependencies of this task. This task will
// execute again when they change.
dir_dependency(self.cwd.join(path).read_glob(Glob::new(glob), false)).await?;
}
InfoMessage::EnvDependency { name } => {
self.env.read(name).await?;
} }
InfoMessage::EmittedError { error, severity } => { InfoMessage::EmittedError { error, severity } => {
EvaluateEmittedErrorIssue { EvaluateEmittedErrorIssue {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment