Unverified Commit 03a65fd7 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Share logic across our two transforms (#78680)

This reduces a small bit of duplication between our two transforms (`toPath` and `fromPath`) and fixes a small bug in `postcss` where we didn't register the environment variables it read as dependencies which could cause persistence issues.

Finally, batch `log` messages sent by webpack transforms, the turbopack side doesn't read them until the end anyway and we can reduce overhead a touch by sending them in a single batch.
parent 928acc7a
...@@ -69,8 +69,8 @@ macro_rules! embed_directory_internal { ...@@ -69,8 +69,8 @@ macro_rules! embed_directory_internal {
// make sure the types the `include_dir!` proc macro refers to are in scope // make sure the types the `include_dir!` proc macro refers to are in scope
use turbo_tasks_fs::embed::include_dir; use turbo_tasks_fs::embed::include_dir;
static dir: include_dir::Dir<'static> = turbo_tasks_fs::embed::include_dir!($path); static DIR: include_dir::Dir<'static> = turbo_tasks_fs::embed::include_dir!($path);
turbo_tasks_fs::embed::directory_from_include_dir($name.into(), &dir) turbo_tasks_fs::embed::directory_from_include_dir($name.into(), &DIR)
}}; }};
} }
...@@ -11,13 +11,13 @@ export type StructuredError = { ...@@ -11,13 +11,13 @@ export type StructuredError = {
cause: StructuredError | undefined cause: StructuredError | undefined
} }
export function structuredError(e: Error): StructuredError { export function structuredError(e: unknown): StructuredError {
e = getProperError(e) e = getProperError(e)
return { return {
name: e.name, name: e.name,
message: e.message, message: e.message,
stack: typeof e.stack === 'string' ? parseStackTrace(e.stack!) : [], stack: typeof e.stack === 'string' ? parseStackTrace(e.stack) : [],
cause: e.cause ? structuredError(getProperError(e.cause)) : undefined, cause: e.cause ? structuredError(getProperError(e.cause)) : undefined,
} }
} }
...@@ -34,7 +34,7 @@ type State = ...@@ -34,7 +34,7 @@ type State =
export type Ipc<TIncoming, TOutgoing> = { export type Ipc<TIncoming, TOutgoing> = {
recv(): Promise<TIncoming> recv(): Promise<TIncoming>
send(message: TOutgoing): Promise<void> send(message: TOutgoing): Promise<void>
sendError(error: Error): Promise<never> sendError(error: Error | string): Promise<never>
sendReady(): Promise<void> sendReady(): Promise<void>
} }
...@@ -121,7 +121,7 @@ function createIpc<TIncoming, TOutgoing>( ...@@ -121,7 +121,7 @@ function createIpc<TIncoming, TOutgoing>(
// TODO(lukesandberg): some of the messages being sent are very large and contain lots // TODO(lukesandberg): some of the messages being sent are very large and contain lots
// of redundant information. Consider adding gzip compression to our stream. // of redundant information. Consider adding gzip compression to our stream.
function doSend(message: any): Promise<void> { function doSend(message: string): Promise<void> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// Reserve 4 bytes for our length prefix, we will over-write after encoding. // Reserve 4 bytes for our length prefix, we will over-write after encoding.
const packet = Buffer.from('0000' + message, 'utf8') const packet = Buffer.from('0000' + message, 'utf8')
......
...@@ -10,25 +10,11 @@ import type { Processor } from 'postcss' ...@@ -10,25 +10,11 @@ import type { Processor } from 'postcss'
import postcss from '@vercel/turbopack/postcss' import postcss from '@vercel/turbopack/postcss'
// @ts-ignore // @ts-ignore
import importedConfig from 'CONFIG' import importedConfig from 'CONFIG'
import { relative, isAbsolute, sep } from 'path' import { getReadEnvVariables, toPath, type TransformIpc } from './transforms'
import type { Ipc } from '../ipc/evaluate'
import type { IpcInfoMessage, IpcRequestMessage } from './webpack-loaders'
const contextDir = process.cwd()
function toPath(file: string) {
const relPath = relative(contextDir, file)
if (isAbsolute(relPath)) {
throw new Error(
`Cannot depend on path (${file}) outside of root directory (${contextDir})`
)
}
return sep !== '/' ? relPath.replaceAll(sep, '/') : relPath
}
let processor: Processor | undefined let processor: Processor | undefined
export const init = async (ipc: Ipc<IpcInfoMessage, IpcRequestMessage>) => { export const init = async (ipc: TransformIpc) => {
let config = importedConfig let config = importedConfig
if (typeof config === 'function') { if (typeof config === 'function') {
config = await config({ env: 'development' }) config = await config({ env: 'development' })
...@@ -76,7 +62,7 @@ export const init = async (ipc: Ipc<IpcInfoMessage, IpcRequestMessage>) => { ...@@ -76,7 +62,7 @@ export const init = async (ipc: Ipc<IpcInfoMessage, IpcRequestMessage>) => {
} }
export default async function transform( export default async function transform(
ipc: Ipc<IpcInfoMessage, IpcRequestMessage>, ipc: TransformIpc,
cssContent: string, cssContent: string,
name: string, name: string,
sourceMap: boolean sourceMap: boolean
...@@ -134,6 +120,7 @@ export default async function transform( ...@@ -134,6 +120,7 @@ export default async function transform(
filePaths, filePaths,
directories, directories,
buildFilePaths, buildFilePaths,
envVariables: getReadEnvVariables(),
}) })
return { return {
css, css,
......
/**
* Shared utilities for our 2 transform implementations.
*/
import type { Ipc } from '../ipc/evaluate'
import { relative, isAbsolute, join, sep } from 'path'
import { type StructuredError } from '../ipc'
import { type StackFrame } from '../compiled/stacktrace-parser'
export type IpcInfoMessage =
| {
type: 'dependencies'
envVariables?: string[]
directories?: Array<[string, string]>
filePaths?: string[]
buildFilePaths?: string[]
}
| {
type: 'emittedError'
severity: 'warning' | 'error'
error: StructuredError
}
| {
type: 'log'
logs: Array<{
time: number
logType: string
args: any[]
trace?: StackFrame[]
}>
}
export type IpcRequestMessage = {
type: 'resolve'
options: any
lookupPath: string
request: string
}
export type TransformIpc = Ipc<IpcInfoMessage, IpcRequestMessage>
const contextDir = process.cwd()
export const toPath = (file: string) => {
const relPath = relative(contextDir, file)
if (isAbsolute(relPath)) {
throw new Error(
`Cannot depend on path (${file}) outside of root directory (${contextDir})`
)
}
return sep !== '/' ? relPath.replaceAll(sep, '/') : relPath
}
export const fromPath = (path: string) => {
return join(contextDir, sep !== '/' ? path.replaceAll('/', sep) : path)
}
// Patch process.env to track which env vars are read
const originalEnv = process.env
const readEnvVars = new Set<string>()
process.env = new Proxy(originalEnv, {
get(target, prop) {
if (typeof prop === 'string') {
// We register the env var as dependency on the
// current transform and all future transforms
// since the env var might be cached in module scope
// and influence them all
readEnvVars.add(prop)
}
return Reflect.get(target, prop)
},
})
export function getReadEnvVariables(): string[] {
return Array.from(readEnvVars)
}
...@@ -3,19 +3,18 @@ declare const __turbopack_external_require__: { ...@@ -3,19 +3,18 @@ declare const __turbopack_external_require__: {
} & ((id: string, thunk: () => any, esm?: boolean) => any) } & ((id: string, thunk: () => any, esm?: boolean) => any)
import type { Ipc } from '../ipc/evaluate' import type { Ipc } from '../ipc/evaluate'
import { import { dirname, resolve as pathResolve } from 'path'
relative,
isAbsolute,
join,
sep,
dirname,
resolve as pathResolve,
} from 'path'
import { import {
StackFrame, StackFrame,
parse as parseStackTrace, parse as parseStackTrace,
} from '../compiled/stacktrace-parser' } from '../compiled/stacktrace-parser'
import { type StructuredError } from 'src/ipc' import { structuredError, type StructuredError } from '../ipc'
import {
fromPath,
getReadEnvVariables,
toPath,
type TransformIpc,
} from './transforms'
export type IpcInfoMessage = export type IpcInfoMessage =
| { | {
...@@ -32,10 +31,12 @@ export type IpcInfoMessage = ...@@ -32,10 +31,12 @@ export type IpcInfoMessage =
} }
| { | {
type: 'log' type: 'log'
time: number logs: Array<{
logType: string time: number
args: any[] logType: string
trace?: StackFrame[] args: any[]
trace?: StackFrame[]
}>
} }
export type IpcRequestMessage = { export type IpcRequestMessage = {
...@@ -57,18 +58,6 @@ const { ...@@ -57,18 +58,6 @@ const {
}: typeof import('loader-runner') = require('@vercel/turbopack/loader-runner') }: typeof import('loader-runner') = require('@vercel/turbopack/loader-runner')
const contextDir = process.cwd() const contextDir = process.cwd()
const toPath = (file: string) => {
const relPath = relative(contextDir, file)
if (isAbsolute(relPath)) {
throw new Error(
`Cannot depend on path (${file}) outside of root directory (${contextDir})`
)
}
return sep !== '/' ? relPath.replaceAll(sep, '/') : relPath
}
const fromPath = (path: string) => {
return join(contextDir, sep !== '/' ? path.replaceAll('/', sep) : path)
}
const LogType = Object.freeze({ const LogType = Object.freeze({
error: 'error', error: 'error',
...@@ -155,24 +144,8 @@ type ResolveOptions = { ...@@ -155,24 +144,8 @@ type ResolveOptions = {
importFields?: string[] importFields?: string[]
} }
// Patch process.env to track which env vars are read
const originalEnv = process.env
const readEnvVars = new Set<string>()
process.env = new Proxy(originalEnv, {
get(target, prop) {
if (typeof prop === 'string' && !readEnvVars.has(prop)) {
// We register the env var as dependency on the
// current transform and all future transforms
// since the env var might be cached in module scope
// and influence them all
readEnvVars.add(prop)
}
return Reflect.get(target, prop)
},
})
const transform = ( const transform = (
ipc: Ipc<IpcInfoMessage, IpcRequestMessage>, ipc: TransformIpc,
content: string | { binary: string }, content: string | { binary: string },
name: string, name: string,
query: string, query: string,
...@@ -187,6 +160,13 @@ const transform = ( ...@@ -187,6 +160,13 @@ const transform = (
typeof loader === 'string' ? { loader, options: {} } : loader typeof loader === 'string' ? { loader, options: {} } : loader
) )
const logs: Array<{
time: number
logType: string
args: unknown[]
trace: StackFrame[] | undefined
}> = []
runLoaders( runLoaders(
{ {
resource: resource + query, resource: resource + query,
...@@ -337,7 +317,7 @@ const transform = ( ...@@ -337,7 +317,7 @@ const transform = (
emitError: makeErrorEmitter('error', ipc), emitError: makeErrorEmitter('error', ipc),
getLogger(name: unknown) { getLogger(name: unknown) {
const logFn = (logType: string, ...args: unknown[]) => { const logFn = (logType: string, ...args: unknown[]) => {
let trace let trace: StackFrame[] | undefined
switch (logType) { switch (logType) {
case LogType.warn: case LogType.warn:
case LogType.error: case LogType.error:
...@@ -354,10 +334,8 @@ const transform = ( ...@@ -354,10 +334,8 @@ const transform = (
// TODO: do we need to handle this? // TODO: do we need to handle this?
break break
} }
// TODO(lukesandberg): should we batch these and flush lazily? // Batch logs messages to be sent at the end
// turbopack just collects these and reports them when finishing the task. logs.push({
ipc.sendInfo({
type: 'log',
time: Date.now(), time: Date.now(),
logType, logType,
args, args,
...@@ -464,9 +442,13 @@ const transform = ( ...@@ -464,9 +442,13 @@ const transform = (
}, },
}, },
(err, result) => { (err, result) => {
if (logs.length) {
ipc.sendInfo({ type: 'log', logs: logs })
logs.length = 0
}
ipc.sendInfo({ ipc.sendInfo({
type: 'dependencies', type: 'dependencies',
envVariables: Array.from(readEnvVars), envVariables: getReadEnvVariables(),
filePaths: result.fileDependencies.map(toPath), filePaths: result.fileDependencies.map(toPath),
directories: result.contextDependencies.map((dep) => [ directories: result.contextDependencies.map((dep) => [
toPath(dep), toPath(dep),
...@@ -502,20 +484,7 @@ function makeErrorEmitter( ...@@ -502,20 +484,7 @@ function makeErrorEmitter(
ipc.sendInfo({ ipc.sendInfo({
type: 'emittedError', type: 'emittedError',
severity: severity, severity: severity,
error: error: structuredError(error),
error instanceof Error
? {
name: error.name,
message: error.message,
stack: error.stack ? parseStackTrace(error.stack) : [],
cause: undefined,
}
: {
name: 'Error',
message: error,
stack: [],
cause: undefined,
},
}) })
} }
} }
...@@ -33,7 +33,7 @@ use super::{ ...@@ -33,7 +33,7 @@ use super::{
webpack::WebpackLoaderContext, webpack::WebpackLoaderContext,
}; };
use crate::{ use crate::{
embed_js::embed_file, execution_context::ExecutionContext, embed_js::embed_file_path, execution_context::ExecutionContext,
transforms::webpack::evaluate_webpack_loader, transforms::webpack::evaluate_webpack_loader,
}; };
...@@ -410,15 +410,9 @@ async fn postcss_executor( ...@@ -410,15 +410,9 @@ async fn postcss_executor(
.await?; .await?;
Ok(asset_context.process( Ok(asset_context.process(
Vc::upcast(VirtualSource::new( Vc::upcast(FileSource::new(embed_file_path(
postcss_config_path.join("transform.ts".into()), "transforms/postcss.ts".into(),
AssetContent::File( ))),
embed_file("transforms/postcss.ts".into())
.to_resolved()
.await?,
)
.cell(),
)),
Value::new(ReferenceType::Internal(ResolvedVc::cell(fxindexmap! { Value::new(ReferenceType::Internal(ResolvedVc::cell(fxindexmap! {
"CONFIG".into() => config_asset "CONFIG".into() => config_asset
}))), }))),
......
...@@ -378,7 +378,9 @@ pub enum InfoMessage { ...@@ -378,7 +378,9 @@ pub enum InfoMessage {
severity: IssueSeverity, severity: IssueSeverity,
error: StructuredError, error: StructuredError,
}, },
Log(LogInfo), Log {
logs: Vec<LogInfo>,
},
} }
#[derive(Debug, Clone, TaskInput, Hash, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)] #[derive(Debug, Clone, TaskInput, Hash, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)]
...@@ -546,8 +548,8 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -546,8 +548,8 @@ impl EvaluateContext for WebpackLoaderContext {
.resolved_cell() .resolved_cell()
.emit(); .emit();
} }
InfoMessage::Log(log) => { InfoMessage::Log { logs } => {
state.push(log); state.extend(logs);
} }
} }
Ok(()) Ok(())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment