Unverified Commit 73469204 authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Fix webpack loaders if source file exists in an additional root (#100744)

The webpack runner needs to compute the absolute path to the loader
implementation, and it cannot compute it relative to the cwd if we're
using additional roots.

Adds this struct:

```
/// An argument passed to the evaluated JavaScript.
#[turbo_tasks::task_input]
#[derive(Clone, Hash, Debug, PartialEq, Eq, Encode, Decode)]
pub enum EvaluateArg {
    Vc(ResolvedVc<JsonValue>),
    /// Passed as a string: A path relative to [`EvaluateContext::cwd`] when both are on the same
    /// filesystem, otherwise an absolute system path (e.g. for an additional root).
    ///
    /// This is converted when the evaluation runs, so that machine-specific system paths aren't
    /// stored in task inputs or cells.
    Path(FileSystemPath),
}
```

The `Path` variant allows us to defer conversion to an absolute system
path until after we've crossed the turbo-tasks boundary, right before we
call the webpack loader.
parent 7d91feca
......@@ -2,6 +2,10 @@ const fs = require('node:fs')
const path = require('node:path')
module.exports = async function (source) {
if (!fs.existsSync(this.resourcePath)) {
throw new Error(`Expected resourcePath to exist: ${this.resourcePath}`)
}
const linkedPackage = fs.realpathSync(path.join(this.rootContext, 'linked'))
const siblingRelative = '../../node_modules/sibling/index.js'
const siblingAbsolute = path.resolve(linkedPackage, siblingRelative)
......
import value from './input.loader'
import linkedValue from '../../linked/value.loader'
export default function Page() {
return <p id="loader-value">{value}</p>
return (
<>
<p id="loader-value">{value}</p>
<p id="linked-loader-value">{linkedValue}</p>
</>
)
}
import '../../linked/styles.css'
export default function Page() {
return <p id="postcss-from" />
}
const fs = require('node:fs')
module.exports = {
plugins: [
{
// Replaces `from-placeholder` with whether PostCSS was given the path of
// an existing file in `from`, including for files in an additional root.
postcssPlugin: 'check-from',
Declaration(decl, { result }) {
if (decl.value === "'from-placeholder'") {
const { from } = result.opts
decl.value = JSON.stringify(
from && fs.existsSync(from) ? 'from-exists' : `from-missing:${from}`
)
}
},
},
],
}
......@@ -73,6 +73,22 @@ describe('turbopack additional roots', () => {
expect(await browser.elementByCss('#loader-value').text()).toBe('processed')
})
it('runs a webpack loader on a file in an additional root', async () => {
const browser = await next.browser('/loader')
expect(await browser.elementByCss('#linked-loader-value').text()).toBe(
'processed'
)
})
it('passes the path of a file in an additional root to PostCSS', async () => {
const browser = await next.browser('/postcss')
expect(
await browser.eval(
`getComputedStyle(document.querySelector('#postcss-from'), '::after').content`
)
).toBe('"from-exists"')
})
it('uses a placeholder for import.meta.url in an additional root', async () => {
const $ = await next.render$('/root-info')
expect(JSON.parse($('#server-root-info').text())).toEqual({
......
use std::{iter, process::ExitStatus, time::Duration};
use anyhow::{Result, bail};
use anyhow::{Context, Result, anyhow, bail};
use async_trait::async_trait;
use bincode::{Decode, Encode};
use bytes::Bytes;
......@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value as JsonValue;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, FxIndexMap, OperationVc, PrettyPrintError, ResolvedVc, TryJoinIterExt, Vc,
Completion, FxIndexMap, OperationVc, PrettyPrintError, ReadRef, ResolvedVc, TryJoinIterExt, Vc,
duration_span, fxindexmap, parallel::available_parallelism,
resolve_strongly_consistent_and_take_and_apply_effects,
};
......@@ -45,11 +45,58 @@ use crate::{
source_map::StructuredError,
};
/// An argument passed to the evaluated JavaScript.
#[turbo_tasks::task_input]
#[derive(Clone, Hash, Debug, PartialEq, Eq, Encode, Decode)]
pub enum EvaluateArg {
Vc(ResolvedVc<JsonValue>),
/// Passed as a string: A path relative to [`EvaluateContext::cwd`] when both are on the same
/// filesystem, otherwise an absolute system path (e.g. for an additional root).
///
/// This is converted when the evaluation runs, so that machine-specific system paths aren't
/// stored in task inputs or cells.
Path(FileSystemPath),
}
impl EvaluateArg {
async fn to_json(&self, cwd: &FileSystemPath) -> Result<EvaluatedArg> {
Ok(match self {
Self::Vc(value) => EvaluatedArg::ReadRef(value.await?),
Self::Path(path) => {
let path = if let Some(relative) = cwd.get_relative_path_to(path) {
// prefer to give the loader a relative path if we can, to reduce the risk of
// the webpack loader storing a absolute path in the output.
//
// this uses a unix-style `/`-separated path, but Node.js file APIs support both
// `/` and `\` on Windows.
relative.into_owned()
} else {
// if the path is on another root, we must give the loader an absolute path
to_sys_path(path.clone())
.await?
.with_context(|| format!("{path} is not on a disk filesystem"))?
.into_os_string()
.into_string()
.map_err(|path| anyhow!("{path:?} is not valid UTF-8"))?
};
EvaluatedArg::Owned(JsonValue::String(path))
}
})
}
}
#[derive(Serialize)]
#[serde(untagged)]
enum EvaluatedArg {
ReadRef(ReadRef<JsonValue>),
Owned(JsonValue),
}
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "camelCase")]
enum EvalJavaScriptOutgoingMessage<'a> {
#[serde(rename_all = "camelCase")]
Evaluate { args: Vec<&'a JsonValue> },
Evaluate { args: &'a [EvaluatedArg] },
Result {
id: u64,
data: Option<JsonValue>,
......@@ -360,7 +407,7 @@ pub trait EvaluateContext {
fn keep_alive(&self) -> bool {
false
}
fn args(&self) -> &[ResolvedVc<JsonValue>];
fn args(&self) -> &[EvaluateArg];
fn cwd(&self) -> Vc<FileSystemPath>;
/// The source that issues about this evaluation are attributed to.
fn context_source_for_issue(&self) -> ResolvedVc<Box<dyn Source>>;
......@@ -405,7 +452,13 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result<V
// node.js code.
let pool = pool_op.read_strongly_consistent().await?;
let args = evaluate_context.args().iter().try_join().await?;
let cwd = evaluate_context.cwd().owned().await?;
let args = evaluate_context
.args()
.iter()
.map(|arg| arg.to_json(&cwd))
.try_join()
.await?;
// Assume this is a one-off operation, so we can kill the process
// TODO use a better way to decide that.
let kill = !evaluate_context.keep_alive();
......@@ -420,9 +473,7 @@ pub async fn custom_evaluate(evaluate_context: impl EvaluateContext) -> Result<V
let mut operation = pool.operation().await?;
operation
.send(Bytes::from(serde_json::to_vec(
&EvalJavaScriptOutgoingMessage::Evaluate {
args: args.iter().map(|v| &**v).collect(),
},
&EvalJavaScriptOutgoingMessage::Evaluate { args: &args },
)?))
.await?;
Ok(operation)
......@@ -583,7 +634,7 @@ pub async fn evaluate(
context_source_for_issue: ResolvedVc<Box<dyn Source>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>,
args: Vec<ResolvedVc<JsonValue>>,
args: Vec<EvaluateArg>,
additional_invalidation: ResolvedVc<Completion>,
debug: bool,
) -> Result<Vc<Option<RcStr>>> {
......@@ -697,7 +748,7 @@ struct BasicEvaluateContext {
context_source_for_issue: ResolvedVc<Box<dyn Source>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>,
args: Vec<ResolvedVc<JsonValue>>,
args: Vec<EvaluateArg>,
additional_invalidation: ResolvedVc<Completion>,
debug: bool,
}
......@@ -722,7 +773,7 @@ impl EvaluateContext for BasicEvaluateContext {
)
}
fn args(&self) -> &[ResolvedVc<serde_json::Value>] {
fn args(&self) -> &[EvaluateArg] {
&self.args
}
......
......@@ -7,7 +7,8 @@ use turbo_tasks::{
Completion, Completions, ResolvedVc, TryFlatJoinIterExt, Vc, fxindexmap, turbofmt,
};
use turbo_tasks_fs::{
File, FileContent, FileSystemEntryType, FileSystemPath, json::parse_json_with_source_context,
DiskFileSystem, File, FileContent, FileSystemEntryType, FileSystemPath,
json::parse_json_with_source_context,
};
use turbopack_core::{
asset::{Asset, AssetContent},
......@@ -27,7 +28,7 @@ use turbopack_ecmascript::runtime_functions::TURBOPACK_EXTERNAL_IMPORT;
use crate::{
embed_js::embed_file_path,
evaluate::get_evaluate_entries,
evaluate::{EvaluateArg, get_evaluate_entries},
execution_context::ExecutionContext,
transforms::{
util::{EmittedAsset, emitted_assets_to_virtual_sources},
......@@ -541,15 +542,13 @@ impl PostCssTransformedAsset {
let source_ident = self.source.ident().await?;
// We need to get a path relative to the project because the postcss loader
// runs with the project as the current working directory.
let css_path = if let Some(css_path) = project_path.get_relative_path_to(&source_ident.path)
{
css_path.into_owned()
} else {
// This shouldn't be an error since it can happen on virtual assets
"".into()
};
let css_path =
if ResolvedVc::try_downcast_type::<DiskFileSystem>(source_ident.path.fs).is_some() {
EvaluateArg::Path(source_ident.path.clone())
} else {
// This shouldn't be an error since it can happen on virtual assets
EvaluateArg::Vc(ResolvedVc::cell(serde_json::Value::from("")))
};
let config_value = evaluate_webpack_loader(WebpackLoaderContext {
entries,
......@@ -563,9 +562,9 @@ impl PostCssTransformedAsset {
resolve_options_context: None,
asset_context: self.asset_context,
args: vec![
ResolvedVc::cell(content.into()),
ResolvedVc::cell(css_path.into()),
ResolvedVc::cell(source_map.into()),
EvaluateArg::Vc(ResolvedVc::cell(content.into())),
css_path,
EvaluateArg::Vc(ResolvedVc::cell(source_map.into())),
],
additional_invalidation: config_changed,
loader_names: vec![turbo_rcstr::rcstr!("postcss")],
......
......@@ -64,8 +64,8 @@ use crate::{
debug::should_debug,
embed_js::embed_file_path,
evaluate::{
EnvVarTracking, EvaluateContext, EvaluateEntries, EvaluatePool, EvaluationIssue,
custom_evaluate, get_evaluate_entries, get_evaluate_pool,
EnvVarTracking, EvaluateArg, EvaluateContext, EvaluateEntries, EvaluatePool,
EvaluationIssue, custom_evaluate, get_evaluate_entries, get_evaluate_pool,
},
execution_context::ExecutionContext,
format::FormattingMode,
......@@ -502,13 +502,6 @@ impl WebpackLoadersProcessedAsset {
let source_ident = self.source.ident().await?;
let resource_fs_path = &source_ident.path;
let Some(resource_path) = project_path.get_relative_path_to(resource_fs_path) else {
bail!(
"Resource path \"{}\" needs to be on project filesystem \"{}\"",
resource_fs_path,
project_path
);
};
let loader_names: Vec<RcStr> = loaders.iter().map(|l| l.loader.clone()).collect();
// TODO pass this into additional_invalidation again once we have proper pool handling
......@@ -536,14 +529,14 @@ impl WebpackLoadersProcessedAsset {
resolve_options_context: Some(transform.resolve_options_context),
asset_context: self.asset_context,
args: vec![
ResolvedVc::cell(content),
EvaluateArg::Vc(ResolvedVc::cell(content)),
EvaluateArg::Path(resource_fs_path.clone()),
// We need to pass the query string to the loader
ResolvedVc::cell(resource_path.to_string().into()),
ResolvedVc::cell(self.source.ident().await?.query.to_string().into()),
ResolvedVc::cell(json!(*loaders)),
ResolvedVc::cell(transform.target.await?.to_string().into()),
ResolvedVc::cell(transform.mode.to_string().into()),
ResolvedVc::cell(transform.source_maps.into()),
EvaluateArg::Vc(ResolvedVc::cell(source_ident.query.to_string().into())),
EvaluateArg::Vc(ResolvedVc::cell(json!(*loaders))),
EvaluateArg::Vc(ResolvedVc::cell(transform.target.await?.to_string().into())),
EvaluateArg::Vc(ResolvedVc::cell(transform.mode.to_string().into())),
EvaluateArg::Vc(ResolvedVc::cell(transform.source_maps.into())),
],
additional_invalidation: Completion::immutable().to_resolved().await?,
loader_names,
......@@ -737,7 +730,7 @@ pub struct WebpackLoaderContext {
pub evaluate_context: ResolvedVc<Box<dyn AssetContext>>,
pub resolve_options_context: Option<ResolvedVc<ResolveOptionsContext>>,
pub asset_context: ResolvedVc<Box<dyn AssetContext>>,
pub args: Vec<ResolvedVc<JsonValue>>,
pub args: Vec<EvaluateArg>,
pub additional_invalidation: ResolvedVc<Completion>,
/// Names of the loaders being applied to the source, in pipeline order.
/// Used to enrich error messages and issue details so users know which
......@@ -792,7 +785,7 @@ impl EvaluateContext for WebpackLoaderContext {
)
}
fn args(&self) -> &[ResolvedVc<serde_json::Value>] {
fn args(&self) -> &[EvaluateArg] {
&self.args
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment