Unverified Commit 73469204 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Fix webpack loaders if source file exists in an additional root (#100744)

The webpack runner needs to compute the absolute path to the loader
implementation, and it cannot compute it relative to the cwd if we're
using additional roots.

Adds this struct:

```
/// An argument passed to the evaluated JavaScript.
#[turbo_tasks::task_input]
#[derive(Clone, Hash, Debug, PartialEq, Eq, Encode, Decode)]
pub enum EvaluateArg {
    Vc(ResolvedVc<JsonValue>),
    /// Passed as a string: A path relative to [`EvaluateContext::cwd`] when both are on the same
    /// filesystem, otherwise an absolute system path (e.g. for an additional root).
    ///
    /// This is converted when the evaluation runs, so that machine-specific system paths aren't
    /// stored in task inputs or cells.
    Path(FileSystemPath),
}
```

The `Path` variant allows us to defer conversion to an absolute system
path until after we've crossed the turbo-tasks boundary, right before we
call the webpack loader.
parent 7d91feca
...@@ -2,6 +2,10 @@ const fs = require('node:fs') ...@@ -2,6 +2,10 @@ const fs = require('node:fs')
const path = require('node:path') const path = require('node:path')
module.exports = async function (source) { module.exports = async function (source) {
if (!fs.existsSync(this.resourcePath)) {
throw new Error(`Expected resourcePath to exist: ${this.resourcePath}`)
}
const linkedPackage = fs.realpathSync(path.join(this.rootContext, 'linked')) const linkedPackage = fs.realpathSync(path.join(this.rootContext, 'linked'))
const siblingRelative = '../../node_modules/sibling/index.js' const siblingRelative = '../../node_modules/sibling/index.js'
const siblingAbsolute = path.resolve(linkedPackage, siblingRelative) const siblingAbsolute = path.resolve(linkedPackage, siblingRelative)
......
import value from './input.loader' import value from './input.loader'
import linkedValue from '../../linked/value.loader'
export default function Page() { export default function Page() {
return <p id="loader-value">{value}</p> return (
<>
<p id="loader-value">{value}</p>
<p id="linked-loader-value">{linkedValue}</p>
</>
)
} }
import '../../linked/styles.css'
export default function Page() {
return <p id="postcss-from" />
}
const fs = require('node:fs')
module.exports = {
plugins: [
{
// Replaces `from-placeholder` with whether PostCSS was given the path of
// an existing file in `from`, including for files in an additional root.
postcssPlugin: 'check-from',
Declaration(decl, { result }) {
if (decl.value === "'from-placeholder'") {
const { from } = result.opts
decl.value = JSON.stringify(
from && fs.existsSync(from) ? 'from-exists' : `from-missing:${from}`
)
}
},
},
],
}
...@@ -73,6 +73,22 @@ describe('turbopack additional roots', () => { ...@@ -73,6 +73,22 @@ describe('turbopack additional roots', () => {
expect(await browser.elementByCss('#loader-value').text()).toBe('processed') expect(await browser.elementByCss('#loader-value').text()).toBe('processed')
}) })
it('runs a webpack loader on a file in an additional root', async () => {
const browser = await next.browser('/loader')
expect(await browser.elementByCss('#linked-loader-value').text()).toBe(
'processed'
)
})
it('passes the path of a file in an additional root to PostCSS', async () => {
const browser = await next.browser('/postcss')
expect(
await browser.eval(
`getComputedStyle(document.querySelector('#postcss-from'), '::after').content`
)
).toBe('"from-exists"')
})
it('uses a placeholder for import.meta.url in an additional root', async () => { it('uses a placeholder for import.meta.url in an additional root', async () => {
const $ = await next.render$('/root-info') const $ = await next.render$('/root-info')
expect(JSON.parse($('#server-root-info').text())).toEqual({ expect(JSON.parse($('#server-root-info').text())).toEqual({
......
use std::{iter, process::ExitStatus, time::Duration}; use std::{iter, process::ExitStatus, time::Duration};
use anyhow::{Result, bail}; use anyhow::{Context, Result, anyhow, bail};
use async_trait::async_trait; use async_trait::async_trait;
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use bytes::Bytes; use bytes::Bytes;
...@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; ...@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Completion, FxIndexMap, OperationVc, PrettyPrintError, ResolvedVc, TryJoinIterExt, Vc, Completion, FxIndexMap, OperationVc, PrettyPrintError, ReadRef, ResolvedVc, TryJoinIterExt, Vc,
duration_span, fxindexmap, parallel::available_parallelism, duration_span, fxindexmap, parallel::available_parallelism,
resolve_strongly_consistent_and_take_and_apply_effects, resolve_strongly_consistent_and_take_and_apply_effects,
}; };
...@@ -45,11 +45,58 @@ use crate::{ ...@@ -45,11 +45,58 @@ use crate::{
source_map::StructuredError, source_map::StructuredError,
}; };
/// An argument passed to the evaluated JavaScript.
#[turbo_tasks::task_input]
#[derive(Clone, Hash, Debug, PartialEq, Eq, Encode, Decode)]
pub enum EvaluateArg {
Vc(ResolvedVc<JsonValue>),
/// Passed as a string: A path relative to [`EvaluateContext::cwd`] when both are on the same
/// filesystem, otherwise an absolute system path (e.g. for an additional root).
///
/// This is converted when the evaluation runs, so that machine-specific system paths aren't
/// stored in task inputs or cells.
Path(FileSystemPath),
}
impl EvaluateArg {
async fn to_json(&self, cwd: &FileSystemPath) -> Result<EvaluatedArg> {
Ok(match self {
Self::Vc(value) => EvaluatedArg::ReadRef(value.await?),
Self::Path(path) => {
let path = if let Some(relative) = cwd.get_relative_path_to(path) {
// prefer to give the loader a relative path if we can, to reduce the risk of
// the webpack loader storing a absolute path in the output.
//
// this uses a unix-style `/`-separated path, but Node.js file APIs support both
// `/` and `\` on Windows.
relative.into_owned()
} else {
// if the path is on another root, we must give the loader an absolute path
to_sys_path(path.clone())
.await?
.with_context(|| format!("{path} is not on a disk filesystem"))?
.into_os_string()
.into_string()
.map_err(|path| anyhow!("{path:?} is not valid UTF-8"))?
};
EvaluatedArg::Owned(JsonValue::String(path))
}
})
}
}
#[derive(Serialize)]
#[serde(untagged)]
enum EvaluatedArg {
ReadRef(ReadRef<JsonValue>),
Owned(JsonValue),
}
#[derive(Serialize)] #[derive(Serialize)]
#[serde(tag = "type", rename_all = "camelCase")] #[serde(tag = "type", rename_all = "camelCase")]
enum EvalJavaScriptOutgoingMessage<'a> { enum EvalJavaScriptOutgoingMessage<'a> {
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
Evaluate { args: Vec<&'a JsonValue> }, Evaluate { args: &'a [EvaluatedArg] },
Result { Result {
id: u64, id: u64,
data: Option<JsonValue>, data: Option<JsonValue>,
...@@ -360,7 +407,7 @@ pub trait EvaluateContext { ...@@ -360,7 +407,7 @@ pub trait EvaluateContext {
fn keep_alive(&self) -> bool { fn keep_alive(&self) -> bool {
false false
} }
fn args(&self) -> &[ResolvedVc<JsonValue>]; fn args(&self) -> &[EvaluateArg];
fn cwd(&self) -> Vc<FileSystemPath>; fn cwd(&self) -> Vc<FileSystemPath>;
/// The source that issues about this evaluation are attributed to. /// The source that issues about this evaluation are attributed to.
fn context_source_for_issue(&self) -> ResolvedVc<Box<dyn Source>>; fn context_source_for_issue(&self) -> ResolvedVc<Box<dyn Source>>;
...@@ -405,7 +452,13 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result<V ...@@ -405,7 +452,13 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result<V
// node.js code. // node.js code.
let pool = pool_op.read_strongly_consistent().await?; let pool = pool_op.read_strongly_consistent().await?;
let args = evaluate_context.args().iter().try_join().await?; let cwd = evaluate_context.cwd().owned().await?;
let args = evaluate_context
.args()
.iter()
.map(|arg| arg.to_json(&cwd))
.try_join()
.await?;
// Assume this is a one-off operation, so we can kill the process // Assume this is a one-off operation, so we can kill the process
// TODO use a better way to decide that. // TODO use a better way to decide that.
let kill = !evaluate_context.keep_alive(); let kill = !evaluate_context.keep_alive();
...@@ -420,9 +473,7 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result<V ...@@ -420,9 +473,7 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result<V
let mut operation = pool.operation().await?; let mut operation = pool.operation().await?;
operation operation
.send(Bytes::from(serde_json::to_vec( .send(Bytes::from(serde_json::to_vec(
&EvalJavaScriptOutgoingMessage::Evaluate { &EvalJavaScriptOutgoingMessage::Evaluate { args: &args },
args: args.iter().map(|v| &**v).collect(),
},
)?)) )?))
.await?; .await?;
Ok(operation) Ok(operation)
...@@ -583,7 +634,7 @@ pub async fn evaluate( ...@@ -583,7 +634,7 @@ pub async fn evaluate(
context_source_for_issue: ResolvedVc<Box<dyn Source>>, context_source_for_issue: ResolvedVc<Box<dyn Source>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
args: Vec<ResolvedVc<JsonValue>>, args: Vec<EvaluateArg>,
additional_invalidation: ResolvedVc<Completion>, additional_invalidation: ResolvedVc<Completion>,
debug: bool, debug: bool,
) -> Result<Vc<Option<RcStr>>> { ) -> Result<Vc<Option<RcStr>>> {
...@@ -697,7 +748,7 @@ struct BasicEvaluateContext { ...@@ -697,7 +748,7 @@ struct BasicEvaluateContext {
context_source_for_issue: ResolvedVc<Box<dyn Source>>, context_source_for_issue: ResolvedVc<Box<dyn Source>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
args: Vec<ResolvedVc<JsonValue>>, args: Vec<EvaluateArg>,
additional_invalidation: ResolvedVc<Completion>, additional_invalidation: ResolvedVc<Completion>,
debug: bool, debug: bool,
} }
...@@ -722,7 +773,7 @@ impl EvaluateContext for BasicEvaluateContext { ...@@ -722,7 +773,7 @@ impl EvaluateContext for BasicEvaluateContext {
) )
} }
fn args(&self) -> &[ResolvedVc<serde_json::Value>] { fn args(&self) -> &[EvaluateArg] {
&self.args &self.args
} }
......
...@@ -7,7 +7,8 @@ use turbo_tasks::{ ...@@ -7,7 +7,8 @@ use turbo_tasks::{
Completion, Completions, ResolvedVc, TryFlatJoinIterExt, Vc, fxindexmap, turbofmt, Completion, Completions, ResolvedVc, TryFlatJoinIterExt, Vc, fxindexmap, turbofmt,
}; };
use turbo_tasks_fs::{ use turbo_tasks_fs::{
File, FileContent, FileSystemEntryType, FileSystemPath, json::parse_json_with_source_context, DiskFileSystem, File, FileContent, FileSystemEntryType, FileSystemPath,
json::parse_json_with_source_context,
}; };
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -27,7 +28,7 @@ use turbopack_ecmascript::runtime_functions::TURBOPACK_EXTERNAL_IMPORT; ...@@ -27,7 +28,7 @@ use turbopack_ecmascript::runtime_functions::TURBOPACK_EXTERNAL_IMPORT;
use crate::{ use crate::{
embed_js::embed_file_path, embed_js::embed_file_path,
evaluate::get_evaluate_entries, evaluate::{EvaluateArg, get_evaluate_entries},
execution_context::ExecutionContext, execution_context::ExecutionContext,
transforms::{ transforms::{
util::{EmittedAsset, emitted_assets_to_virtual_sources}, util::{EmittedAsset, emitted_assets_to_virtual_sources},
...@@ -541,15 +542,13 @@ impl PostCssTransformedAsset { ...@@ -541,15 +542,13 @@ impl PostCssTransformedAsset {
let source_ident = self.source.ident().await?; let source_ident = self.source.ident().await?;
// We need to get a path relative to the project because the postcss loader let css_path =
// runs with the project as the current working directory. if ResolvedVc::try_downcast_type::<DiskFileSystem>(source_ident.path.fs).is_some() {
let css_path = if let Some(css_path) = project_path.get_relative_path_to(&source_ident.path) EvaluateArg::Path(source_ident.path.clone())
{ } else {
css_path.into_owned() // This shouldn't be an error since it can happen on virtual assets
} else { EvaluateArg::Vc(ResolvedVc::cell(serde_json::Value::from("")))
// This shouldn't be an error since it can happen on virtual assets };
"".into()
};
let config_value = evaluate_webpack_loader(WebpackLoaderContext { let config_value = evaluate_webpack_loader(WebpackLoaderContext {
entries, entries,
...@@ -563,9 +562,9 @@ impl PostCssTransformedAsset { ...@@ -563,9 +562,9 @@ impl PostCssTransformedAsset {
resolve_options_context: None, resolve_options_context: None,
asset_context: self.asset_context, asset_context: self.asset_context,
args: vec![ args: vec![
ResolvedVc::cell(content.into()), EvaluateArg::Vc(ResolvedVc::cell(content.into())),
ResolvedVc::cell(css_path.into()), css_path,
ResolvedVc::cell(source_map.into()), EvaluateArg::Vc(ResolvedVc::cell(source_map.into())),
], ],
additional_invalidation: config_changed, additional_invalidation: config_changed,
loader_names: vec![turbo_rcstr::rcstr!("postcss")], loader_names: vec![turbo_rcstr::rcstr!("postcss")],
......
...@@ -64,8 +64,8 @@ use crate::{ ...@@ -64,8 +64,8 @@ use crate::{
debug::should_debug, debug::should_debug,
embed_js::embed_file_path, embed_js::embed_file_path,
evaluate::{ evaluate::{
EnvVarTracking, EvaluateContext, EvaluateEntries, EvaluatePool, EvaluationIssue, EnvVarTracking, EvaluateArg, EvaluateContext, EvaluateEntries, EvaluatePool,
custom_evaluate, get_evaluate_entries, get_evaluate_pool, EvaluationIssue, custom_evaluate, get_evaluate_entries, get_evaluate_pool,
}, },
execution_context::ExecutionContext, execution_context::ExecutionContext,
format::FormattingMode, format::FormattingMode,
...@@ -502,13 +502,6 @@ impl WebpackLoadersProcessedAsset { ...@@ -502,13 +502,6 @@ impl WebpackLoadersProcessedAsset {
let source_ident = self.source.ident().await?; let source_ident = self.source.ident().await?;
let resource_fs_path = &source_ident.path; let resource_fs_path = &source_ident.path;
let Some(resource_path) = project_path.get_relative_path_to(resource_fs_path) else {
bail!(
"Resource path \"{}\" needs to be on project filesystem \"{}\"",
resource_fs_path,
project_path
);
};
let loader_names: Vec<RcStr> = loaders.iter().map(|l| l.loader.clone()).collect(); let loader_names: Vec<RcStr> = loaders.iter().map(|l| l.loader.clone()).collect();
// TODO pass this into additional_invalidation again once we have proper pool handling // TODO pass this into additional_invalidation again once we have proper pool handling
...@@ -536,14 +529,14 @@ impl WebpackLoadersProcessedAsset { ...@@ -536,14 +529,14 @@ impl WebpackLoadersProcessedAsset {
resolve_options_context: Some(transform.resolve_options_context), resolve_options_context: Some(transform.resolve_options_context),
asset_context: self.asset_context, asset_context: self.asset_context,
args: vec![ args: vec![
ResolvedVc::cell(content), EvaluateArg::Vc(ResolvedVc::cell(content)),
EvaluateArg::Path(resource_fs_path.clone()),
// We need to pass the query string to the loader // We need to pass the query string to the loader
ResolvedVc::cell(resource_path.to_string().into()), EvaluateArg::Vc(ResolvedVc::cell(source_ident.query.to_string().into())),
ResolvedVc::cell(self.source.ident().await?.query.to_string().into()), EvaluateArg::Vc(ResolvedVc::cell(json!(*loaders))),
ResolvedVc::cell(json!(*loaders)), EvaluateArg::Vc(ResolvedVc::cell(transform.target.await?.to_string().into())),
ResolvedVc::cell(transform.target.await?.to_string().into()), EvaluateArg::Vc(ResolvedVc::cell(transform.mode.to_string().into())),
ResolvedVc::cell(transform.mode.to_string().into()), EvaluateArg::Vc(ResolvedVc::cell(transform.source_maps.into())),
ResolvedVc::cell(transform.source_maps.into()),
], ],
additional_invalidation: Completion::immutable().to_resolved().await?, additional_invalidation: Completion::immutable().to_resolved().await?,
loader_names, loader_names,
...@@ -737,7 +730,7 @@ pub struct WebpackLoaderContext { ...@@ -737,7 +730,7 @@ pub struct WebpackLoaderContext {
pub evaluate_context: ResolvedVc<Box<dyn AssetContext>>, pub evaluate_context: ResolvedVc<Box<dyn AssetContext>>,
pub resolve_options_context: Option<ResolvedVc<ResolveOptionsContext>>, pub resolve_options_context: Option<ResolvedVc<ResolveOptionsContext>>,
pub asset_context: ResolvedVc<Box<dyn AssetContext>>, pub asset_context: ResolvedVc<Box<dyn AssetContext>>,
pub args: Vec<ResolvedVc<JsonValue>>, pub args: Vec<EvaluateArg>,
pub additional_invalidation: ResolvedVc<Completion>, pub additional_invalidation: ResolvedVc<Completion>,
/// Names of the loaders being applied to the source, in pipeline order. /// Names of the loaders being applied to the source, in pipeline order.
/// Used to enrich error messages and issue details so users know which /// Used to enrich error messages and issue details so users know which
...@@ -792,7 +785,7 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -792,7 +785,7 @@ impl EvaluateContext for WebpackLoaderContext {
) )
} }
fn args(&self) -> &[ResolvedVc<serde_json::Value>] { fn args(&self) -> &[EvaluateArg] {
&self.args &self.args
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment