Unverified Commit 6779a5c0 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: error when importing Typescript in node_modules (#83990)

Webpack treats TS files in node_modules as Javascript files (i.e. you get a JS parse error)

Turbopack previously did a similar thing (though it parsed as Typescript, but then never ran the TS AST transform).
This never worked properly, it failed due to some unreachable assertion in the swc transforms.

Now it fails with a proper error ~~, though too many of them~~:

```
./node_modules/.pnpm/pkg@file+pkg/node_modules/pkg/index.ts
Missing module type
The module type effect must be applied before adding Ecmascript transforms


./node_modules/.pnpm/pkg@file+pkg/node_modules/pkg/index.ts
Unknown module type
This module doesn't have an associated type. Use a known file extension, or register a loader for it.

Read more: https://nextjs.org/docs/app/api-reference/next-config-js/turbo#webpack-loaders
    at ignore-listed frames
```

Additionally: don't run styled-jsx/styled-components/relay/... in node_modules in the server. This aligns it what we already had configured for the client context
parent 9cf9183e
......@@ -32,7 +32,10 @@ pub async fn get_next_client_transforms_rules(
let modularize_imports_config = &next_config.modularize_imports().await?;
let enable_mdx_rs = next_config.mdx_rs().await?.is_some();
rules.push(get_next_lint_transform_rule(enable_mdx_rs));
if !foreign_code {
rules.push(get_next_lint_transform_rule(enable_mdx_rs));
}
if !modularize_imports_config.is_empty() {
rules.push(get_next_modularize_imports_rule(
......
......@@ -559,14 +559,16 @@ pub async fn get_server_module_options_context(
.flatten()
.collect();
// Custom ecma transform rules selectively being applied depends on the server
// context type.
let styled_components_transform_rule =
get_styled_components_transform_rule(next_config).await?;
// It's important the client's browserlist config is used for styled-jsx, otherwise we transpile
// the CSS to be compatible with Node.js 20.
let styled_jsx_transform_rule =
get_styled_jsx_transform_rule(next_config, client_environment.runtime_versions()).await?;
// Only relevant for pages, not routes/etc.
let page_transform_rules: Vec<ModuleRule> = vec![
get_styled_components_transform_rule(next_config).await?,
// It's important the client's browserlist config is used for styled-jsx, otherwise we
// transpile the CSS to be compatible with Node.js 20.
get_styled_jsx_transform_rule(next_config, client_environment.runtime_versions()).await?,
]
.into_iter()
.flatten()
.collect();
let source_maps = if *next_config.server_source_maps().await? {
SourceMapsType::Full
......@@ -608,23 +610,16 @@ pub async fn get_server_module_options_context(
let module_options_context = match ty {
ServerContextType::Pages { .. } | ServerContextType::PagesApi { .. } => {
let mut custom_source_transform_rules: Vec<ModuleRule> =
vec![styled_components_transform_rule, styled_jsx_transform_rule]
.into_iter()
.flatten()
.collect();
next_server_rules.extend(page_transform_rules);
if let ServerContextType::Pages { .. } = ty {
custom_source_transform_rules.push(
next_server_rules.push(
get_next_react_server_components_transform_rule(next_config, false, None)
.await?,
);
}
next_server_rules.extend(custom_source_transform_rules.iter().cloned());
next_server_rules.extend(source_transform_rules);
foreign_next_server_rules.extend(custom_source_transform_rules);
foreign_next_server_rules.extend(internal_custom_rules);
let url_rewrite_behavior = Some(
......@@ -690,21 +685,13 @@ pub async fn get_server_module_options_context(
}
}
ServerContextType::AppSSR { app_dir, .. } => {
let mut custom_source_transform_rules: Vec<ModuleRule> =
vec![styled_components_transform_rule, styled_jsx_transform_rule]
.into_iter()
.flatten()
.collect();
foreign_next_server_rules.extend(custom_source_transform_rules.iter().cloned());
foreign_next_server_rules.extend(internal_custom_rules);
custom_source_transform_rules.push(
next_server_rules.extend(page_transform_rules.clone());
next_server_rules.push(
get_next_react_server_components_transform_rule(next_config, false, Some(app_dir))
.await?,
);
next_server_rules.extend(custom_source_transform_rules.clone());
next_server_rules.extend(source_transform_rules);
let foreign_code_module_options_context = ModuleOptionsContext {
......@@ -755,33 +742,29 @@ pub async fn get_server_module_options_context(
ecmascript_client_reference_transition_name,
..
} => {
let mut custom_source_transform_rules: Vec<ModuleRule> =
vec![styled_components_transform_rule, styled_jsx_transform_rule]
.into_iter()
.flatten()
.collect();
if let Some(ecmascript_client_reference_transition_name) =
ecmascript_client_reference_transition_name
{
custom_source_transform_rules.push(get_ecma_transform_rule(
Box::new(ClientDirectiveTransformer::new(
ecmascript_client_reference_transition_name,
)),
enable_mdx_rs.is_some(),
EcmascriptTransformStage::Preprocess,
));
}
next_server_rules.extend(page_transform_rules);
let client_directive_transformer = ecmascript_client_reference_transition_name.map(
|ecmascript_client_reference_transition_name| {
get_ecma_transform_rule(
Box::new(ClientDirectiveTransformer::new(
ecmascript_client_reference_transition_name,
)),
enable_mdx_rs.is_some(),
EcmascriptTransformStage::Preprocess,
)
},
);
foreign_next_server_rules.extend(custom_source_transform_rules.iter().cloned());
foreign_next_server_rules.extend(client_directive_transformer.clone());
foreign_next_server_rules.extend(internal_custom_rules);
custom_source_transform_rules.push(
next_server_rules.extend(client_directive_transformer.clone());
next_server_rules.push(
get_next_react_server_components_transform_rule(next_config, true, Some(app_dir))
.await?,
);
next_server_rules.extend(custom_source_transform_rules.clone());
next_server_rules.extend(source_transform_rules);
let foreign_code_module_options_context = ModuleOptionsContext {
......@@ -909,35 +892,28 @@ pub async fn get_server_module_options_context(
app_dir,
ecmascript_client_reference_transition_name,
} => {
let mut custom_source_transform_rules: Vec<ModuleRule> =
vec![styled_components_transform_rule, styled_jsx_transform_rule]
.into_iter()
.flatten()
.collect();
if let Some(ecmascript_client_reference_transition_name) =
ecmascript_client_reference_transition_name
{
custom_source_transform_rules.push(get_ecma_transform_rule(
Box::new(ClientDirectiveTransformer::new(
ecmascript_client_reference_transition_name,
)),
enable_mdx_rs.is_some(),
EcmascriptTransformStage::Preprocess,
));
} else {
custom_source_transform_rules.push(get_ecma_transform_rule(
Box::new(ClientDisallowedDirectiveTransformer::new(
"next/dist/client/use-client-disallowed.js".to_string(),
)),
enable_mdx_rs.is_some(),
EcmascriptTransformStage::Preprocess,
));
}
custom_source_transform_rules.push(
let custom_source_transform_rules: Vec<ModuleRule> = vec![
if let Some(ecmascript_client_reference_transition_name) =
ecmascript_client_reference_transition_name
{
get_ecma_transform_rule(
Box::new(ClientDirectiveTransformer::new(
ecmascript_client_reference_transition_name,
)),
enable_mdx_rs.is_some(),
EcmascriptTransformStage::Preprocess,
)
} else {
get_ecma_transform_rule(
Box::new(ClientDisallowedDirectiveTransformer::new(
"next/dist/client/use-client-disallowed.js".to_string(),
)),
enable_mdx_rs.is_some(),
EcmascriptTransformStage::Preprocess,
)
},
get_next_react_server_components_transform_rule(next_config, true, app_dir).await?,
);
];
internal_custom_rules.extend(custom_source_transform_rules.iter().cloned());
......
......@@ -39,7 +39,9 @@ pub async fn get_next_server_transforms_rules(
let modularize_imports_config = &next_config.modularize_imports().await?;
let mdx_rs = next_config.mdx_rs().await?.is_some();
rules.push(get_next_lint_transform_rule(mdx_rs));
if !foreign_code {
rules.push(get_next_lint_transform_rule(mdx_rs));
}
if !modularize_imports_config.is_empty() {
rules.push(get_next_modularize_imports_rule(
......
import * as React from 'react'
export default function Root({ children }) {
return (
<html>
<head></head>
<body>{children}</body>
</html>
)
}
import { Foo } from 'pkg/index.ts'
export default function Page() {
return <main>Hello {Foo}</main>
}
import { nextTestSetup } from 'e2e-utils'
describe('transpile-packages-typescript-foreign', () => {
describe('without transpilePackages', () => {
const { next, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
skipStart: true,
dependencies: {
pkg: `file:./pkg`,
},
})
if (skipped) {
return
}
it('should fail', async () => {
try {
await next.start()
await next.render('/')
} catch (e) {}
if (process.env.IS_TURBOPACK_TEST) {
expect(next.cliOutput).toContain(`pkg/index.ts
Unknown module type
This module doesn't have an associated type`)
expect(
next.cliOutput.match(/Unknown module type/g).length
).toBeLessThanOrEqual(1)
expect(
next.cliOutput.match(/Missing module type/g).length
).toBeLessThanOrEqual(1)
} else {
expect(next.cliOutput).toContain(`pkg/index.ts
Module parse failed: Unexpected token`)
}
})
})
describe('with transpilePackages', () => {
const { next, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
dependencies: {
pkg: `file:./pkg`,
},
nextConfig: {
transpilePackages: ['pkg'],
},
})
if (skipped) {
return
}
it('should work', async () => {
const $ = await next.render$('/')
expect($('main').text()).toEqual('Hello 123')
})
})
})
import type { Cents } from './cents'
export type Type = Cents
export const Foo = 123
......@@ -4,10 +4,7 @@ use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_hash::hash_xxh3_hash64;
use super::ModuleId;
use crate::{
ident::AssetIdent,
issue::{IssueExt, StyledString, module::ModuleIssue},
};
use crate::ident::AssetIdent;
#[turbo_tasks::value_trait]
pub trait ModuleIdStrategy {
......@@ -54,32 +51,17 @@ impl ModuleIdStrategy for GlobalModuleIdStrategy {
}
let ident_string = ident.to_string().await?;
if !ident_string.ends_with("[app-client] (ecmascript, next/dynamic entry)") {
if ident_string.ends_with("[app-client] (ecmascript, next/dynamic entry)") {
// TODO: This shouldn't happen, but is a temporary workaround to ignore next/dynamic
// imports of a server component from another server component.
// TODO(PACK-4879): should this be a debug_assert! a panic!? a bail!?
ModuleIssue {
ident,
title: StyledString::Text(
format!("ModuleId not found for ident: {ident_string:?}").into(),
)
.resolved_cell(),
description: StyledString::Text(
format!("ModuleId not found for ident: {ident_string:?}").into(),
)
.resolved_cell(),
source: None,
}
.resolved_cell()
.emit();
return Ok(ModuleId::String(
hash_xxh3_hash64(ident.to_string().await?)
.to_string()
.into(),
)
.cell());
}
Ok(ModuleId::String(
hash_xxh3_hash64(ident.to_string().await?)
.to_string()
.into(),
)
.cell())
bail!("ModuleId not found for ident: {ident_string:?}");
}
}
use anyhow::Result;
use turbo_rcstr::rcstr;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath;
......@@ -10,7 +10,7 @@ use crate::{
source::Source,
};
#[turbo_tasks::value(shared)]
#[turbo_tasks::value]
pub struct ModuleIssue {
pub ident: ResolvedVc<AssetIdent>,
pub title: ResolvedVc<StyledString>,
......@@ -18,6 +18,24 @@ pub struct ModuleIssue {
// TODO(PACK-4879): make this mandatory and drop `ident`
pub source: Option<IssueSource>,
}
#[turbo_tasks::value_impl]
impl ModuleIssue {
#[turbo_tasks::function]
pub fn new(
ident: ResolvedVc<AssetIdent>,
title: RcStr,
description: RcStr,
source: Option<IssueSource>,
) -> Vc<Self> {
ModuleIssue {
ident,
title: StyledString::Text(title).resolved_cell(),
description: StyledString::Text(description).resolved_cell(),
source,
}
.cell()
}
}
#[turbo_tasks::value_impl]
impl Issue for ModuleIssue {
......
......@@ -6,7 +6,10 @@ use turbo_tasks::{ResolvedVc, TransientInstance, TryJoinIterExt, ValueToString,
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{
ModuleAssetContext,
module_options::{CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext},
module_options::{
CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext,
TypescriptTransformOptions,
},
};
use turbopack_cli_utils::issue::{ConsoleUi, LogOptions};
use turbopack_core::{
......@@ -80,6 +83,9 @@ async fn node_file_trace_operation(
CompileTimeInfo::new(environment),
ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_typescript_transform: Some(
TypescriptTransformOptions::default().resolved_cell(),
),
..Default::default()
},
css: CssOptionsContext {
......
......@@ -23,7 +23,7 @@ use turbopack::{
ecmascript::{EcmascriptInputTransform, TreeShakingMode, chunk::EcmascriptChunkType},
module_options::{
EcmascriptOptionsContext, JsxTransformOptions, ModuleOptionsContext, ModuleRule,
ModuleRuleEffect, RuleCondition,
ModuleRuleEffect, RuleCondition, TypescriptTransformOptions,
},
};
use turbopack_browser::BrowserChunkingContext;
......@@ -340,6 +340,9 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
compile_time_info,
ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_typescript_transform: Some(
TypescriptTransformOptions::default().resolved_cell(),
),
enable_jsx: Some(JsxTransformOptions::resolved_cell(JsxTransformOptions {
development: true,
..Default::default()
......
......@@ -34,7 +34,10 @@ use turbo_tasks_backend::TurboTasksBackend;
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{
ModuleAssetContext, emit_with_completion_operation,
module_options::{CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext},
module_options::{
CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext,
TypescriptTransformOptions,
},
};
use turbopack_core::{
compile_time_info::CompileTimeInfo,
......@@ -357,6 +360,9 @@ async fn node_file_trace_operation(
CompileTimeInfo::new(environment),
ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_typescript_transform: Some(
TypescriptTransformOptions::default().resolved_cell(),
),
enable_types: true,
..Default::default()
},
......
......@@ -15,7 +15,10 @@ use turbo_tasks_backend::TurboTasksBackend;
use turbo_tasks_fs::{DiskFileSystem, FileSystem};
use turbopack::{
ModuleAssetContext,
module_options::{CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext},
module_options::{
CssOptionsContext, EcmascriptOptionsContext, ModuleOptionsContext,
TypescriptTransformOptions,
},
};
use turbopack_core::{
chunk::ChunkingType,
......@@ -195,6 +198,9 @@ async fn node_file_trace_operation(package_root: RcStr, input: RcStr) -> Result<
CompileTimeInfo::new(environment),
ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_typescript_transform: Some(
TypescriptTransformOptions::default().resolved_cell(),
),
..Default::default()
},
css: CssOptionsContext {
......
......@@ -37,7 +37,7 @@ use turbopack_core::{
compile_time_info::CompileTimeInfo,
context::{AssetContext, ProcessResult},
ident::Layer,
issue::{IssueExt, IssueSource, StyledString, module::ModuleIssue},
issue::{IssueExt, IssueSource, module::ModuleIssue},
module::Module,
node_addon_module::NodeAddonModule,
output::OutputAsset,
......@@ -70,7 +70,9 @@ use turbopack_static::{css::StaticUrlCssModule, ecma::StaticUrlJsModule};
use turbopack_wasm::{module_asset::WebAssemblyModuleAsset, source::WebAssemblySource};
use self::transition::{Transition, TransitionOptions};
use crate::module_options::{CssOptionsContext, CustomModuleType, EcmascriptOptionsContext};
use crate::module_options::{
CssOptionsContext, CustomModuleType, EcmascriptOptionsContext, TypescriptTransformOptions,
};
async fn apply_module_type(
source: ResolvedVc<Box<dyn Source>>,
......@@ -620,34 +622,32 @@ async fn process_default_internal(
options,
}),
Some(module_type) => {
ModuleIssue {
ident,
title: StyledString::Text(rcstr!("Invalid module type"))
.resolved_cell(),
description: StyledString::Text(rcstr!(
ModuleIssue::new(
*ident,
rcstr!("Invalid module type"),
rcstr!(
"The module type must be Ecmascript or Typescript to add \
Ecmascript transforms"
))
.resolved_cell(),
source: Some(IssueSource::from_source_only(current_source)),
}
.resolved_cell()
),
Some(IssueSource::from_source_only(current_source)),
)
.to_resolved()
.await?
.emit();
Some(module_type)
}
None => {
ModuleIssue {
ident,
title: StyledString::Text(rcstr!("Missing module type"))
.resolved_cell(),
description: StyledString::Text(rcstr!(
ModuleIssue::new(
*ident,
rcstr!("Missing module type"),
rcstr!(
"The module type effect must be applied before adding \
Ecmascript transforms"
))
.resolved_cell(),
source: Some(IssueSource::from_source_only(current_source)),
}
.resolved_cell()
),
Some(IssueSource::from_source_only(current_source)),
)
.to_resolved()
.await?
.emit();
None
}
......@@ -703,6 +703,9 @@ pub async fn externals_tracing_module_context(
// are actually representative of what Turbopack does.
ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_typescript_transform: Some(
TypescriptTransformOptions::default().resolved_cell(),
),
// enable_types should not be enabled here. It gets set automatically when a TS file
// is encountered.
source_maps: SourceMapsType::None,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment