Unverified Commit f80a4b4d authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: loader change invalidation (#95732)

We apparently never invalidated when the loader source code changed.
This now uses the same mechanism we already use for watching
postcss/tailwind config files

This also required changing the tracing mode of configs (and now
loaders) to only care about imports, not FS file references to prevent
infinite loops.
parent 86d3d61e
...@@ -73,12 +73,10 @@ pub(crate) async fn require_hook_modules( ...@@ -73,12 +73,10 @@ pub(crate) async fn require_hook_modules(
/// graph. Include the renderer as an explicit Pages trace entry so that its runtime closure is /// graph. Include the renderer as an explicit Pages trace entry so that its runtime closure is
/// available when the endpoint initializes. /// available when the endpoint initializes.
#[turbo_tasks::function] #[turbo_tasks::function]
pub(crate) async fn pages_renderer_modules(project_path: FileSystemPath) -> Result<Vc<Modules>> { pub(crate) async fn pages_renderer_modules(
let asset_context = Vc::upcast(externals_tracing_module_context( project_path: FileSystemPath,
get_tracing_compile_time_info(), asset_context: Vc<Box<dyn AssetContext>>,
false, ) -> Result<Vc<Modules>> {
None,
));
let next_resolve_origin = Vc::upcast(PlainResolveOrigin::new( let next_resolve_origin = Vc::upcast(PlainResolveOrigin::new(
asset_context, asset_context,
get_next_package(project_path).await?.join("_")?, get_next_package(project_path).await?.join("_")?,
...@@ -365,8 +363,9 @@ impl ServerNftJsonAsset { ...@@ -365,8 +363,9 @@ impl ServerNftJsonAsset {
let asset_context = Vc::upcast(externals_tracing_module_context( let asset_context = Vc::upcast(externals_tracing_module_context(
get_tracing_compile_time_info(), get_tracing_compile_time_info(),
false, /* resolve_typescript */ false,
Some((self.project.project_root_path().owned().await?, prune)), /* prune */ Some((self.project.project_root_path().owned().await?, prune)),
/* trace_file_references */ true,
)); ));
let project_path = self.project.project_path().owned().await?; let project_path = self.project.project_path().owned().await?;
......
...@@ -2819,8 +2819,12 @@ impl Project { ...@@ -2819,8 +2819,12 @@ impl Project {
.cache_handlers(project_path.clone()) .cache_handlers(project_path.clone())
.await?; .await?;
let asset_context = let asset_context = externals_tracing_module_context(
externals_tracing_module_context(get_tracing_compile_time_info(), false, None); get_tracing_compile_time_info(),
/* resolve_typescript */ false,
/* prune */ None,
/* trace_file_references */ true,
);
Ok(Vc::cell( Ok(Vc::cell(
cache_handler cache_handler
...@@ -2847,15 +2851,17 @@ impl Project { ...@@ -2847,15 +2851,17 @@ impl Project {
pub async fn pages_traced_modules(self: Vc<Self>) -> Result<Vc<Modules>> { pub async fn pages_traced_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
let asset_context = Vc::upcast(externals_tracing_module_context( let asset_context = Vc::upcast(externals_tracing_module_context(
get_tracing_compile_time_info(), get_tracing_compile_time_info(),
false, /* resolve_typescript */ false,
None, /* prune */ None,
/* trace_file_references */ true,
)); ));
let hook_modules = require_hook_modules(self.project_path().owned().await?, asset_context) let hook_modules = require_hook_modules(self.project_path().owned().await?, asset_context)
.owned() .owned()
.await?; .await?;
let renderer_modules = pages_renderer_modules(self.project_path().owned().await?) let renderer_modules =
.owned() pages_renderer_modules(self.project_path().owned().await?, asset_context)
.await?; .owned()
.await?;
Ok(Vc::cell( Ok(Vc::cell(
self.additional_traced_modules() self.additional_traced_modules()
......
...@@ -989,7 +989,7 @@ function bindingToApi( ...@@ -989,7 +989,7 @@ function bindingToApi(
const turbopack = { ...nextConfigSerializable.turbopack } const turbopack = { ...nextConfigSerializable.turbopack }
if (turbopack.rules) { if (turbopack.rules) {
turbopack.rules = serializeTurbopackRules(turbopack.rules) turbopack.rules = serializeTurbopackRules(turbopack.rules, projectPath)
} }
// Serialize ignoreIssue rules: convert RegExp to {source, flags} // Serialize ignoreIssue rules: convert RegExp to {source, flags}
...@@ -1129,7 +1129,8 @@ function bindingToApi( ...@@ -1129,7 +1129,8 @@ function bindingToApi(
// Note: Returns an updated `turbopackRules` with serialized conditions. Does not mutate in-place. // Note: Returns an updated `turbopackRules` with serialized conditions. Does not mutate in-place.
function serializeTurbopackRules( function serializeTurbopackRules(
turbopackRules: Record<string, TurbopackRuleConfigCollection> turbopackRules: Record<string, TurbopackRuleConfigCollection>,
projectPath: string
): Record<string, any> { ): Record<string, any> {
const serializedRules: Record<string, any> = {} const serializedRules: Record<string, any> = {}
for (const [glob, rule] of Object.entries(turbopackRules)) { for (const [glob, rule] of Object.entries(turbopackRules)) {
...@@ -1141,8 +1142,7 @@ function bindingToApi( ...@@ -1141,8 +1142,7 @@ function bindingToApi(
) { ) {
return serializeConfigItem(item as TurbopackRuleConfigItem, glob) return serializeConfigItem(item as TurbopackRuleConfigItem, glob)
} else { } else {
checkLoaderItem(item as TurbopackLoaderItem, glob) return serializeLoaderItem(item as TurbopackLoaderItem, glob)
return item
} }
}) })
} else { } else {
...@@ -1158,8 +1158,9 @@ function bindingToApi( ...@@ -1158,8 +1158,9 @@ function bindingToApi(
): any { ): any {
if (!rule) return rule if (!rule) return rule
if (rule.loaders) { if (rule.loaders) {
for (const item of rule.loaders) { rule = {
checkLoaderItem(item, glob) ...rule,
loaders: rule.loaders.map((item) => serializeLoaderItem(item, glob)),
} }
} }
let serializedRule: any = rule let serializedRule: any = rule
...@@ -1172,20 +1173,37 @@ function bindingToApi( ...@@ -1172,20 +1173,37 @@ function bindingToApi(
return serializedRule return serializedRule
} }
function checkLoaderItem(loaderItem: TurbopackLoaderItem, glob: string) { function serializeLoaderItem(
if ( loaderItem: TurbopackLoaderItem,
typeof loaderItem !== 'string' && glob: string
!(require('util') as typeof import('util')).isDeepStrictEqual( ): TurbopackLoaderItem {
loaderItem, if (typeof loaderItem === 'string') {
JSON.parse(JSON.stringify(loaderItem)) return serializeLoader(loaderItem)
) } else {
) { if (
throw new Error( !(require('util') as typeof import('util')).isDeepStrictEqual(
`loader ${loaderItem.loader} for match "${glob}" does not have serializable options. ` + loaderItem,
'Ensure that options passed are plain JavaScript objects and values.' JSON.parse(JSON.stringify(loaderItem))
) )
) {
throw new Error(
`loader ${loaderItem.loader} for match "${glob}" does not have serializable options. ` +
'Ensure that options passed are plain JavaScript objects and values.'
)
}
return {
...loaderItem,
loader: serializeLoader(loaderItem.loader),
}
} }
} }
// Webpack loader specifiers can be absolute paths, we need it to be relative for turbopack.
function serializeLoader(loader: string) {
return path.isAbsolute(loader)
? normalizePathOnWindows('./' + path.relative(projectPath, loader))
: loader
}
} }
function napiEntrypointsToRawEntrypoints( function napiEntrypointsToRawEntrypoints(
......
...@@ -251,23 +251,18 @@ for (const cacheEnabled of [false, true]) { ...@@ -251,23 +251,18 @@ for (const cacheEnabled of [false, true]) {
checkChanged: makeTextCheck('/add-me', 'hello world'), checkChanged: makeTextCheck('/add-me', 'hello world'),
largeCacheIncrease: true, largeCacheIncrease: true,
}, },
// TODO fix this case with Turbopack 'loader change': {
...(isTurbopack async checkInitial() {
? {} await textCheck('/loader', 'hello world')
: { await textCheck('/loader/client', 'hello world')
'loader change': { },
async checkInitial() { withChange: makeFileEdit('my-loader.js'),
await textCheck('/loader', 'hello world') async checkChanged() {
await textCheck('/loader/client', 'hello world') await textCheck('/loader', 'hello filesystem cache')
}, await textCheck('/loader/client', 'hello filesystem cache')
withChange: makeFileEdit('my-loader.js'), },
async checkChanged() { fullInvalidation: !isTurbopack,
await textCheck('/loader', 'hello filesystem cache') },
await textCheck('/loader/client', 'hello filesystem cache')
},
fullInvalidation: !isTurbopack,
},
}),
'next config change': { 'next config change': {
async checkInitial() { async checkInitial() {
await textCheck('/next-config', 'hello world') await textCheck('/next-config', 'hello world')
......
...@@ -10,7 +10,7 @@ const nextConfig = { ...@@ -10,7 +10,7 @@ const nextConfig = {
'./app/**/page.{jsx,tsx}': { './app/**/page.{jsx,tsx}': {
loaders: ['./my-timestamp-loader.js'], loaders: ['./my-timestamp-loader.js'],
}, },
'./app/loader/page.tsx': { './app/loader/**/page.tsx': {
loaders: ['./my-loader.js'], loaders: ['./my-loader.js'],
}, },
'./pages/pages.tsx': { './pages/pages.tsx': {
......
...@@ -127,7 +127,7 @@ describe('webpack-loader-parse-error (development)', () => { ...@@ -127,7 +127,7 @@ describe('webpack-loader-parse-error (development)', () => {
Parsing ecmascript source code failed Parsing ecmascript source code failed
Generated code of loaders [broken-js-loader.js] transform of file content of app/data.broken.js: Generated code of loaders [./broken-js-loader.js] transform of file content of app/data.broken.js:
./app/data.broken.js:3:46 ./app/data.broken.js:3:46
1 | // Generated by broken-js-loader 1 | // Generated by broken-js-loader
2 | export default function Page() { 2 | export default function Page() {
...@@ -180,7 +180,7 @@ describe('webpack-loader-parse-error (development)', () => { ...@@ -180,7 +180,7 @@ describe('webpack-loader-parse-error (development)', () => {
Unexpected token CurlyBracketBlock Unexpected token CurlyBracketBlock
Generated code of PostCSS transform of loaders [broken-css-loader.js] transform of file content of app/css-page/styles.broken.css: Generated code of PostCSS transform of loaders [./broken-css-loader.js] transform of file content of app/css-page/styles.broken.css:
./app/css-page/styles.broken.css:5:15 ./app/css-page/styles.broken.css:5:15
3 | color: red 3 | color: red
4 | @@@ THIS IS NOT VALID CSS @@@; 4 | @@@ THIS IS NOT VALID CSS @@@;
...@@ -232,7 +232,7 @@ describe('webpack-loader-parse-error (production)', () => { ...@@ -232,7 +232,7 @@ describe('webpack-loader-parse-error (production)', () => {
Parsing ecmascript source code failed Parsing ecmascript source code failed
Generated code of loaders [broken-js-loader.js] transform of file content of app/data.broken.js: Generated code of loaders [./broken-js-loader.js] transform of file content of app/data.broken.js:
./app/data.broken.js:3:46 ./app/data.broken.js:3:46
1 | // Generated by broken-js-loader 1 | // Generated by broken-js-loader
2 | export default function Page() { 2 | export default function Page() {
...@@ -263,7 +263,7 @@ describe('webpack-loader-parse-error (production)', () => { ...@@ -263,7 +263,7 @@ describe('webpack-loader-parse-error (production)', () => {
Unexpected token CurlyBracketBlock Unexpected token CurlyBracketBlock
Generated code of PostCSS transform of loaders [broken-css-loader.js] transform of file content of app/css-page/styles.broken.css: Generated code of PostCSS transform of loaders [./broken-css-loader.js] transform of file content of app/css-page/styles.broken.css:
./app/css-page/styles.broken.css:5:15 ./app/css-page/styles.broken.css:5:15
3 | color: red 3 | color: red
4 | @@@ THIS IS NOT VALID CSS @@@; 4 | @@@ THIS IS NOT VALID CSS @@@;
......
...@@ -2,6 +2,7 @@ use std::hash::{Hash, Hasher}; ...@@ -2,6 +2,7 @@ use std::hash::{Hash, Hasher};
use bincode::{Decode, Encode}; use bincode::{Decode, Encode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, OperationValue, TaskInput, trace::TraceRawVcs}; use turbo_tasks::{NonLocalValue, OperationValue, TaskInput, trace::TraceRawVcs};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
...@@ -19,7 +20,7 @@ use turbo_tasks_fs::FileSystemPath; ...@@ -19,7 +20,7 @@ use turbo_tasks_fs::FileSystemPath;
Decode, Decode,
)] )]
pub struct WebpackLoaderItem { pub struct WebpackLoaderItem {
pub loader: turbo_rcstr::RcStr, pub loader: RcStr,
#[serde(default)] #[serde(default)]
#[bincode(with = "turbo_bincode::serde_self_describing")] #[bincode(with = "turbo_bincode::serde_self_describing")]
pub options: serde_json::Map<String, serde_json::Value>, pub options: serde_json::Map<String, serde_json::Value>,
......
...@@ -12,8 +12,8 @@ use serde_with::serde_as; ...@@ -12,8 +12,8 @@ use serde_with::serde_as;
use tracing::Instrument; use tracing::Instrument;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Completion, OperationVc, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, ValueToStringRef, Completion, Completions, OperationVc, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt,
Vc, trace::TraceRawVcs, ValueToString, ValueToStringRef, Vc, trace::TraceRawVcs,
}; };
use turbo_tasks_env::ProcessEnv; use turbo_tasks_env::ProcessEnv;
use turbo_tasks_fs::{ use turbo_tasks_fs::{
...@@ -24,6 +24,7 @@ use turbo_tasks_fs::{ ...@@ -24,6 +24,7 @@ use turbo_tasks_fs::{
}; };
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
changed::any_source_content_changed_of_module,
chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAsset}, chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAsset},
context::{AssetContext, ProcessResult}, context::{AssetContext, ProcessResult},
file_source::FileSource, file_source::FileSource,
...@@ -102,6 +103,7 @@ pub struct WebpackLoaders { ...@@ -102,6 +103,7 @@ pub struct WebpackLoaders {
rename_as: Option<RcStr>, rename_as: Option<RcStr>,
resolve_options_context: ResolvedVc<ResolveOptionsContext>, resolve_options_context: ResolvedVc<ResolveOptionsContext>,
source_maps: bool, source_maps: bool,
config_tracing_context: ResolvedVc<Box<dyn AssetContext>>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -116,6 +118,7 @@ impl WebpackLoaders { ...@@ -116,6 +118,7 @@ impl WebpackLoaders {
rename_as: Option<RcStr>, rename_as: Option<RcStr>,
resolve_options_context: ResolvedVc<ResolveOptionsContext>, resolve_options_context: ResolvedVc<ResolveOptionsContext>,
source_maps: bool, source_maps: bool,
config_tracing_context: ResolvedVc<Box<dyn AssetContext>>,
) -> Vc<Self> { ) -> Vc<Self> {
WebpackLoaders { WebpackLoaders {
evaluate_context, evaluate_context,
...@@ -126,6 +129,7 @@ impl WebpackLoaders { ...@@ -126,6 +129,7 @@ impl WebpackLoaders {
rename_as, rename_as,
resolve_options_context, resolve_options_context,
source_maps, source_maps,
config_tracing_context,
} }
.cell() .cell()
} }
...@@ -258,6 +262,48 @@ async fn build_files_changed( ...@@ -258,6 +262,48 @@ async fn build_files_changed(
Ok(Completion::new()) Ok(Completion::new())
} }
/// Returns a completion that changes when any of the loader modules (or their
/// transitive dependencies) change. This makes editing a loader file invalidate
/// the cached transform of every module processed by that loader.
///
/// Loaders are resolved relative to the resource's directory, matching webpack
/// semantics (`paths: [contextDir, resourceDir]` in `webpack-loaders.ts`).
/// Loaders that fail to resolve are skipped so that the transform degrades to
/// the previous (non-invalidating) behavior instead of breaking the build.
#[turbo_tasks::function]
async fn loaders_changed(
loaders: Vc<WebpackLoaderItems>,
project_path: FileSystemPath,
asset_context: Vc<Box<dyn AssetContext>>,
resolve_options_context: Vc<ResolveOptionsContext>,
) -> Result<Vc<Completion>> {
let options = resolve_options(project_path.clone(), resolve_options_context);
let origin_path = project_path.clone().join("_")?;
let completions = loaders
.await?
.iter()
.map(async |loader| {
let result = asset_context
.resolve_asset(
origin_path.clone(),
Request::parse(Pattern::Constant(loader.loader.clone())),
options,
ReferenceType::Loader,
)
.await?;
result
.primary_modules_raw_iter()
.map(|m| any_source_content_changed_of_module(*m).to_resolved())
.try_join()
.await
})
.try_flat_join()
.await?;
Ok(Vc::<Completions>::cell(completions).completed())
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl WebpackLoadersProcessedAsset { impl WebpackLoadersProcessedAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -338,6 +384,18 @@ impl WebpackLoadersProcessedAsset { ...@@ -338,6 +384,18 @@ impl WebpackLoadersProcessedAsset {
); );
}; };
let loader_names: Vec<RcStr> = loaders.iter().map(|l| l.loader.clone()).collect(); let loader_names: Vec<RcStr> = loaders.iter().map(|l| l.loader.clone()).collect();
// Invalidate the transform when a loader module (or any of its
// transitive dependencies) changes.
let loaders_changed = loaders_changed(
*transform.loaders,
project_path.clone(),
*transform.config_tracing_context,
*transform.resolve_options_context,
)
.to_resolved()
.await?;
let config_value = evaluate_webpack_loader(WebpackLoaderContext { let config_value = evaluate_webpack_loader(WebpackLoaderContext {
entries, entries,
cwd: project_path.clone(), cwd: project_path.clone(),
...@@ -359,7 +417,7 @@ impl WebpackLoadersProcessedAsset { ...@@ -359,7 +417,7 @@ impl WebpackLoadersProcessedAsset {
ResolvedVc::cell(transform.mode.to_string().into()), ResolvedVc::cell(transform.mode.to_string().into()),
ResolvedVc::cell(transform.source_maps.into()), ResolvedVc::cell(transform.source_maps.into()),
], ],
additional_invalidation: Completion::immutable().to_resolved().await?, additional_invalidation: loaders_changed,
loader_names, loader_names,
}) })
.await?; .await?;
......
...@@ -139,7 +139,9 @@ pub async fn config_tracing_module_context( ...@@ -139,7 +139,9 @@ pub async fn config_tracing_module_context(
) )
.cell() .cell()
.await?, .await?,
true, /* resolve_typescript */ true,
None, /* prune */ None,
// Don't trace fs.readFile in configs and webpack loaders. This can cause infinite loops
false,
))) )))
} }
...@@ -65,7 +65,7 @@ use turbopack_static::{css::StaticUrlCssModule, ecma::StaticUrlJsModule}; ...@@ -65,7 +65,7 @@ use turbopack_static::{css::StaticUrlCssModule, ecma::StaticUrlJsModule};
use turbopack_wasm::{module_asset::WebAssemblyModuleAsset, source::WebAssemblySource}; use turbopack_wasm::{module_asset::WebAssemblyModuleAsset, source::WebAssemblySource};
use crate::{ use crate::{
evaluate_context::node_evaluate_asset_context, evaluate_context::{config_tracing_module_context, node_evaluate_asset_context},
module_options::{ module_options::{
CssOptionsContext, CustomModuleType, EcmascriptOptionsContext, TypescriptTransformOptions, CssOptionsContext, CustomModuleType, EcmascriptOptionsContext, TypescriptTransformOptions,
package_import_map_from_context, package_import_map_from_import_mapping, package_import_map_from_context, package_import_map_from_import_mapping,
...@@ -688,6 +688,7 @@ async fn process_default_internal( ...@@ -688,6 +688,7 @@ async fn process_default_internal(
rename_as.clone(), rename_as.clone(),
*resolve_options_context, *resolve_options_context,
source_maps, source_maps,
config_tracing_module_context(*execution_context),
) )
.to_resolved() .to_resolved()
.await?; .await?;
...@@ -871,6 +872,7 @@ pub async fn externals_tracing_module_context( ...@@ -871,6 +872,7 @@ pub async fn externals_tracing_module_context(
compile_time_info: Vc<CompileTimeInfo>, compile_time_info: Vc<CompileTimeInfo>,
resolve_typescript: bool, resolve_typescript: bool,
prune: Option<(FileSystemPath, ResolvedVc<Glob>)>, prune: Option<(FileSystemPath, ResolvedVc<Glob>)>,
trace_file_references: bool,
) -> Result<Vc<ModuleAssetContext>> { ) -> Result<Vc<ModuleAssetContext>> {
let mut extensions = vec![rcstr!(".js"), rcstr!(".node"), rcstr!(".json")]; let mut extensions = vec![rcstr!(".js"), rcstr!(".node"), rcstr!(".json")];
if resolve_typescript { if resolve_typescript {
...@@ -924,7 +926,10 @@ pub async fn externals_tracing_module_context( ...@@ -924,7 +926,10 @@ pub async fn externals_tracing_module_context(
// Environment is not passed in order to avoid downleveling JS / CSS for // Environment is not passed in order to avoid downleveling JS / CSS for
// node-file-trace. // node-file-trace.
environment: None, environment: None,
analyze_mode: AnalyzeMode::tracing(), analyze_mode: AnalyzeMode {
is_codegen: false,
trace_file_references,
},
module_rules: prune_rules, module_rules: prune_rules,
// Disable tree shaking. Even side-effect-free imports need to be traced, as they will // Disable tree shaking. Even side-effect-free imports need to be traced, as they will
// execute at runtime. // execute at runtime.
...@@ -1080,8 +1085,9 @@ impl AssetContext for ModuleAssetContext { ...@@ -1080,8 +1085,9 @@ impl AssetContext for ModuleAssetContext {
let origin = PlainResolveOrigin::new( let origin = PlainResolveOrigin::new(
Vc::upcast(externals_tracing_module_context( Vc::upcast(externals_tracing_module_context(
*options.compile_time_info, *options.compile_time_info,
false, /* resolve_typescript */ false,
None, /* prune */ None,
/* trace_file_references */ true,
)), )),
// If target is specified, a symlink will be created to // If target is specified, a symlink will be created to
// make the folder // make the folder
......
...@@ -506,6 +506,7 @@ impl ModuleOptions { ...@@ -506,6 +506,7 @@ impl ModuleOptions {
rule.rename_as.clone(), rule.rename_as.clone(),
resolve_options_context, resolve_options_context,
matches!(ecmascript_source_maps, SourceMapsType::Full), matches!(ecmascript_source_maps, SourceMapsType::Full),
config_tracing_module_context(*execution_context),
) )
.to_resolved() .to_resolved()
.await?, .await?,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment