Unverified Commit c2dfe402 authored by Dale Bustad's avatar Dale Bustad Committed by GitHub

Babel fast mode (#23760)

@timneutkens it'd be great to get your input.

These changes introduce a new Babel loader that eliminates much of the existing overhead, resulting in better HMR speeds. 
 Multithreading is still in flight, and may be omitted if speed improvements end up being negligible.  For now, the new loader is hidden behind an `experimental` flag.

Items to be completed before this PR is ready to merge:

- [x] reconfigure `ncc` to precompile the parts of `@babel/core` and `@babel/traverse` that we're accessing directly
- [x] change `@babel/core/...` imports to `ncc`ed version
- [x] ~~measure multithreading (not currently pushed) functionality, and include the functionality depending on the results~~ I'll open a separate PR for this
- [x] ensure TypeScript is happy with all imports as final step (`--no-verify` was used to bypass)

There will be two follow-up PRs:
- loader support for projects with custom `.babelrc`
- multithreaded loader (should the change we warranted after measurement)
parent bce78ba8
import nextBabelPreset from '../preset'
import { createConfigItem, loadOptions } from 'next/dist/compiled/babel/core'
import loadConfig from 'next/dist/compiled/babel/core-lib-config'
import { NextBabelLoaderOptions, NextJsLoaderContext } from './types'
import { consumeIterator } from './util'
function getPlugins(
loaderOptions: NextBabelLoaderOptions,
source: string,
filename: string
) {
const { hasReactRefresh, isServer, development, pagesDir } = loaderOptions
const isPageFile = filename.startsWith(pagesDir)
const applyCommonJsItem =
source.indexOf('module.exports') === -1
? null
: createConfigItem(require('../plugins/commonjs'), { type: 'plugin' })
const reactRefreshItem = hasReactRefresh
? createConfigItem(
[require('react-refresh/babel'), { skipEnvCheck: true }],
{ type: 'plugin' }
)
: null
const noAnonymousDefaultExportItem =
hasReactRefresh && !isServer
? createConfigItem(
[require('../plugins/no-anonymous-default-export'), {}],
{ type: 'plugin' }
)
: null
const pageConfigItem =
!isServer && isPageFile
? createConfigItem([require('../plugins/next-page-config')], {
type: 'plugin',
})
: null
const disallowExportAllItem =
!isServer && isPageFile
? createConfigItem(
[require('../plugins/next-page-disallow-re-export-all-exports')],
{ type: 'plugin' }
)
: null
const transformDefineItem = createConfigItem(
[
require.resolve('next/dist/compiled/babel/plugin-transform-define'),
{
'process.env.NODE_ENV': development ? 'development' : 'production',
'typeof window': isServer ? 'undefined' : 'object',
'process.browser': isServer ? false : true,
},
'next-js-transform-define-instance',
],
{ type: 'plugin' }
)
const nextSsgItem =
!isServer && isPageFile
? createConfigItem([require.resolve('../plugins/next-ssg-transform')], {
type: 'plugin',
})
: null
return [
noAnonymousDefaultExportItem,
reactRefreshItem,
pageConfigItem,
disallowExportAllItem,
applyCommonJsItem,
transformDefineItem,
nextSsgItem,
].filter(Boolean)
}
function getOverrides(overrides = []) {
const commonJsItem = createConfigItem(
require('next/dist/compiled/babel/plugin-transform-modules-commonjs'),
{ type: 'plugin' }
)
return [
...overrides,
{
test: [
/next[\\/]dist[\\/]next-server[\\/]lib/,
/next[\\/]dist[\\/]client/,
/next[\\/]dist[\\/]pages/,
],
plugins: [commonJsItem],
},
]
}
const configs = new Map()
export default function getConfig(
this: NextJsLoaderContext,
{
source,
loaderOptions,
inputSourceMap,
target,
filename,
}: {
source: string
loaderOptions: NextBabelLoaderOptions
inputSourceMap?: object | null
target: string
filename: string
}
) {
const {
presets = [],
isServer,
pagesDir,
development,
hasReactRefresh,
hasJsxRuntime,
} = loaderOptions
const configKey = `${isServer ? 'server' : 'client'}:${filename}`
if (configs.has(configKey)) {
return configs.get(configKey)
}
const nextPresetItem = createConfigItem(nextBabelPreset, { type: 'preset' })
let options = {
babelrc: false,
cloneInputAst: false,
filename,
inputSourceMap: inputSourceMap || undefined,
// Set the default sourcemap behavior based on Webpack's mapping flag,
// but allow users to override if they want.
sourceMaps:
loaderOptions.sourceMaps === undefined
? inputSourceMap
: loaderOptions.sourceMaps,
// Ensure that Webpack will get a full absolute path in the sourcemap
// so that it can properly map the module back to its internal cached
// modules.
sourceFileName: filename,
plugins: getPlugins(loaderOptions, source, filename),
presets: [...presets, nextPresetItem],
overrides: getOverrides(loaderOptions.overrides),
caller: {
name: 'next-babel-turbo-loader',
supportsStaticESM: true,
supportsDynamicImport: true,
// Provide plugins with insight into webpack target.
// https://github.com/babel/babel-loader/issues/787
target: target,
// Webpack 5 supports TLA behind a flag. We enable it by default
// for Babel, and then webpack will throw an error if the experimental
// flag isn't enabled.
supportsTopLevelAwait: true,
isServer,
pagesDir,
development,
hasReactRefresh,
hasJsxRuntime,
...loaderOptions.caller,
},
} as any
Object.defineProperty(options.caller, 'onWarning', {
enumerable: false,
writable: false,
value: (reason: any) => {
if (!(reason instanceof Error)) {
reason = new Error(reason)
}
this.emitWarning(reason)
},
})
const loadedOptions = loadOptions(options)
const config = consumeIterator(loadConfig(loadedOptions))
configs.set(configKey, config)
return config
}
import { inspect } from 'util'
import { getOptions } from 'next/dist/compiled/loader-utils'
import { trace } from '../../../telemetry/trace'
import { Span } from '../../../telemetry/trace'
import transform from './transform'
import { NextJsLoaderContext } from './types'
async function nextBabelLoader(
this: NextJsLoaderContext,
parentTrace: Span,
inputSource: string,
inputSourceMap: object | null | undefined
) {
const filename = this.resourcePath
const target = this.target
const loaderOptions = parentTrace
.traceChild('get-options')
.traceFn(() => getOptions(this))
const loaderSpanInner = parentTrace.traceChild('next-babel-turbo-transform')
const {
code: transformedSource,
map: outputSourceMap,
} = loaderSpanInner.traceFn(() =>
transform.call(
this,
inputSource,
inputSourceMap,
loaderOptions,
filename,
target,
loaderSpanInner
)
)
return [transformedSource, outputSourceMap]
}
const nextBabelLoaderOuter = function nextBabelLoaderOuter(
this: NextJsLoaderContext,
inputSource: string,
inputSourceMap: object | null | undefined
) {
const callback = this.async()
const loaderSpan = trace('next-babel-turbo-loader', this.currentTraceSpan?.id)
loaderSpan
.traceAsyncFn(() =>
nextBabelLoader.call(this, loaderSpan, inputSource, inputSourceMap)
)
.then(
([transformedSource, outputSourceMap]) =>
callback?.(null, transformedSource, outputSourceMap || inputSourceMap),
(err) => {
console.error(
`Problem encountered in next-babel-turbo-loader. \n${inspect(err)}`
)
callback?.(err)
}
)
}
export default nextBabelLoaderOuter
/*
* Partially adapted from @babel/core (MIT license).
*/
import traverse from 'next/dist/compiled/babel/traverse'
import generate from 'next/dist/compiled/babel/generator'
import normalizeFile from 'next/dist/compiled/babel/core-lib-normalize-file'
import normalizeOpts from 'next/dist/compiled/babel/core-lib-normalize-opts'
import loadBlockHoistPlugin from 'next/dist/compiled/babel/core-lib-block-hoist-plugin'
import PluginPass from 'next/dist/compiled/babel/core-lib-plugin-pass'
import getConfig from './get-config'
import { consumeIterator } from './util'
import { Span } from '../../../telemetry/trace'
import { NextJsLoaderContext } from './types'
function getTraversalParams(file: any, pluginPairs: any[]) {
const passPairs = []
const passes = []
const visitors = []
for (const plugin of pluginPairs.concat(loadBlockHoistPlugin())) {
const pass = new PluginPass(file, plugin.key, plugin.options)
passPairs.push([plugin, pass])
passes.push(pass)
visitors.push(plugin.visitor)
}
return { passPairs, passes, visitors }
}
function invokePluginPre(file: any, passPairs: any[]) {
for (const [{ pre }, pass] of passPairs) {
if (pre) {
pre.call(pass, file)
}
}
}
function invokePluginPost(file: any, passPairs: any[]) {
for (const [{ post }, pass] of passPairs) {
if (post) {
post.call(pass, file)
}
}
}
function transformAstPass(file: any, pluginPairs: any[], parentSpan: Span) {
const { passPairs, passes, visitors } = getTraversalParams(file, pluginPairs)
invokePluginPre(file, passPairs)
const visitor = traverse.visitors.merge(
visitors,
passes,
// @ts-ignore - the exported types are incorrect here
file.opts.wrapPluginVisitorMethod
)
parentSpan
.traceChild('babel-turbo-traverse')
.traceFn(() => traverse(file.ast, visitor, file.scope))
invokePluginPost(file, passPairs)
}
function transformAst(file: any, babelConfig: any, parentSpan: Span) {
for (const pluginPairs of babelConfig.passes) {
transformAstPass(file, pluginPairs, parentSpan)
}
}
export default function transform(
this: NextJsLoaderContext,
source: string,
inputSourceMap: object | null | undefined,
loaderOptions: any,
filename: string,
target: string,
parentSpan: Span
) {
const getConfigSpan = parentSpan.traceChild('babel-turbo-get-config')
const babelConfig = getConfig.call(this, {
source,
loaderOptions,
inputSourceMap,
target,
filename,
})
getConfigSpan.stop()
const normalizeSpan = parentSpan.traceChild('babel-turbo-normalize-file')
const file = consumeIterator(
normalizeFile(babelConfig.passes, normalizeOpts(babelConfig), source)
)
normalizeSpan.stop()
const transformSpan = parentSpan.traceChild('babel-turbo-transform')
transformAst(file, babelConfig, transformSpan)
transformSpan.stop()
const generateSpan = parentSpan.traceChild('babel-turbo-generate')
const { code, map } = generate(file.ast, file.opts.generatorOpts, file.code)
generateSpan.stop()
return { code, map }
}
import { loader } from 'next/dist/compiled/webpack/webpack'
import { Span } from '../../../telemetry/trace'
export interface NextJsLoaderContext extends loader.LoaderContext {
currentTraceSpan?: Span
}
export interface NextBabelLoaderOptions {
hasJsxRuntime: boolean
hasReactRefresh: boolean
isServer: boolean
development: boolean
pagesDir: string
presets: any[]
sourceMaps?: any[]
overrides: any
caller: any
}
export function consumeIterator(iter: Iterator<any>) {
while (true) {
const { value, done } = iter.next()
if (done) {
return value
}
}
}
...@@ -58,7 +58,7 @@ function supportsStaticESM(caller: any): boolean { ...@@ -58,7 +58,7 @@ function supportsStaticESM(caller: any): boolean {
return !!caller?.supportsStaticESM return !!caller?.supportsStaticESM
} }
module.exports = ( export default (
api: any, api: any,
options: NextBabelPresetOptions = {} options: NextBabelPresetOptions = {}
): BabelPreset => { ): BabelPreset => {
...@@ -78,7 +78,10 @@ module.exports = ( ...@@ -78,7 +78,10 @@ module.exports = (
const isProduction = !(isTest || isDevelopment) const isProduction = !(isTest || isDevelopment)
const isBabelLoader = api.caller( const isBabelLoader = api.caller(
(caller: any) => !!caller && caller.name === 'babel-loader' (caller: any) =>
!!caller &&
(caller.name === 'babel-loader' ||
caller.name === 'next-babel-turbo-loader')
) )
const useJsxRuntime = const useJsxRuntime =
......
...@@ -235,9 +235,13 @@ export default async function getBaseWebpackConfig( ...@@ -235,9 +235,13 @@ export default async function getBaseWebpackConfig(
semver.gte(reactVersion!, '17.0.0-rc.1') semver.gte(reactVersion!, '17.0.0-rc.1')
const distDir = path.join(dir, config.distDir) const distDir = path.join(dir, config.distDir)
const babelLoader = config.experimental.turboMode
? 'next/dist/build/babel/loader'
: 'next-babel-loader'
const defaultLoaders = { const defaultLoaders = {
babel: { babel: {
loader: 'next-babel-loader', loader: babelLoader,
options: { options: {
isServer, isServer,
distDir, distDir,
......
...@@ -8,6 +8,34 @@ function core() { ...@@ -8,6 +8,34 @@ function core() {
return require('@babel/core') return require('@babel/core')
} }
function coreLibConfig() {
return require('@babel/core/lib/config')
}
function coreLibNormalizeFile() {
return require('@babel/core/lib/transformation/normalize-file')
}
function coreLibNormalizeOpts() {
return require('@babel/core/lib/transformation/normalize-opts')
}
function coreLibBlockHoistPlugin() {
return require('@babel/core/lib/transformation/block-hoist-plugin')
}
function coreLibPluginPass() {
return require('@babel/core/lib/transformation/plugin-pass')
}
function traverse() {
return require('@babel/traverse')
}
function generator() {
return require('@babel/generator')
}
function pluginProposalClassProperties() { function pluginProposalClassProperties() {
return require('@babel/plugin-proposal-class-properties') return require('@babel/plugin-proposal-class-properties')
} }
...@@ -67,6 +95,12 @@ function presetTypescript() { ...@@ -67,6 +95,12 @@ function presetTypescript() {
module.exports = { module.exports = {
codeFrame, codeFrame,
core, core,
coreLibConfig,
coreLibNormalizeFile,
coreLibNormalizeOpts,
coreLibBlockHoistPlugin,
coreLibPluginPass,
generator,
pluginProposalClassProperties, pluginProposalClassProperties,
pluginProposalExportNamespaceFrom, pluginProposalExportNamespaceFrom,
pluginProposalNumericSeparator, pluginProposalNumericSeparator,
...@@ -81,4 +115,5 @@ module.exports = { ...@@ -81,4 +115,5 @@ module.exports = {
presetEnv, presetEnv,
presetReact, presetReact,
presetTypescript, presetTypescript,
traverse,
} }
module.exports = require('./bundle').coreLibBlockHoistPlugin()
module.exports = require('./bundle').coreLibConfig()
module.exports = require('./bundle').coreLibNormalizeFile()
module.exports = require('./bundle').coreLibNormalizeOpts()
module.exports = require('./bundle').coreLibPluginPass()
module.exports = require('./bundle').generator()
module.exports = require('./bundle').traverse()
This diff is collapsed.
module.exports = require('./bundle').coreLibBlockHoistPlugin()
module.exports = require('./bundle').coreLibConfig()
module.exports = require('./bundle').coreLibNormalizeFile()
module.exports = require('./bundle').coreLibNormalizeOpts()
module.exports = require('./bundle').coreLibPluginPass()
module.exports = require('./bundle').generator()
module.exports = require('./bundle').traverse()
This diff is collapsed.
...@@ -59,6 +59,7 @@ export type NextConfig = { [key: string]: any } & { ...@@ -59,6 +59,7 @@ export type NextConfig = { [key: string]: any } & {
validator?: string validator?: string
skipValidation?: boolean skipValidation?: boolean
} }
turboMode: boolean
} }
} }
...@@ -113,6 +114,7 @@ export const defaultConfig: NextConfig = { ...@@ -113,6 +114,7 @@ export const defaultConfig: NextConfig = {
stats: false, stats: false,
externalDir: false, externalDir: false,
serialWebpackBuild: false, serialWebpackBuild: false,
turboMode: false,
}, },
future: { future: {
strictPostcssConfiguration: false, strictPostcssConfiguration: false,
......
...@@ -132,6 +132,7 @@ ...@@ -132,6 +132,7 @@
"devDependencies": { "devDependencies": {
"@ampproject/toolbox-optimizer": "2.7.1-alpha.0", "@ampproject/toolbox-optimizer": "2.7.1-alpha.0",
"@babel/code-frame": "7.12.11", "@babel/code-frame": "7.12.11",
"@babel/generator": "^7.12.10",
"@babel/core": "7.12.10", "@babel/core": "7.12.10",
"@babel/plugin-proposal-class-properties": "7.12.1", "@babel/plugin-proposal-class-properties": "7.12.1",
"@babel/plugin-proposal-export-namespace-from": "7.12.1", "@babel/plugin-proposal-export-namespace-from": "7.12.1",
...@@ -145,6 +146,7 @@ ...@@ -145,6 +146,7 @@
"@babel/preset-env": "7.12.11", "@babel/preset-env": "7.12.11",
"@babel/preset-react": "7.12.10", "@babel/preset-react": "7.12.10",
"@babel/preset-typescript": "7.12.7", "@babel/preset-typescript": "7.12.7",
"@babel/traverse": "^7.12.10",
"@babel/types": "7.12.12", "@babel/types": "7.12.12",
"@next/polyfill-nomodule": "10.1.4-canary.2", "@next/polyfill-nomodule": "10.1.4-canary.2",
"@taskr/clear": "1.1.0", "@taskr/clear": "1.1.0",
...@@ -178,6 +180,7 @@ ...@@ -178,6 +180,7 @@
"@types/send": "0.14.4", "@types/send": "0.14.4",
"@types/styled-jsx": "2.2.8", "@types/styled-jsx": "2.2.8",
"@types/text-table": "0.2.1", "@types/text-table": "0.2.1",
"@types/webpack": "5.28.0",
"@types/webpack-sources": "0.1.5", "@types/webpack-sources": "0.1.5",
"@vercel/ncc": "0.27.0", "@vercel/ncc": "0.27.0",
"amphtml-validator": "1.0.33", "amphtml-validator": "1.0.33",
......
...@@ -114,8 +114,9 @@ export async function ncc_async_sema(task, opts) { ...@@ -114,8 +114,9 @@ export async function ncc_async_sema(task, opts) {
// eslint-disable-next-line camelcase // eslint-disable-next-line camelcase
export async function ncc_babel_bundle(task, opts) { export async function ncc_babel_bundle(task, opts) {
const bundleExternals = { ...externals } const bundleExternals = { ...externals }
for (const pkg of Object.keys(babelBundlePackages)) for (const pkg of Object.keys(babelBundlePackages)) {
delete bundleExternals[pkg] delete bundleExternals[pkg]
}
await task await task
.source(opts.src || 'bundles/babel/bundle.js') .source(opts.src || 'bundles/babel/bundle.js')
.ncc({ .ncc({
...@@ -128,7 +129,18 @@ export async function ncc_babel_bundle(task, opts) { ...@@ -128,7 +129,18 @@ export async function ncc_babel_bundle(task, opts) {
const babelBundlePackages = { const babelBundlePackages = {
'code-frame': 'next/dist/compiled/babel/code-frame', 'code-frame': 'next/dist/compiled/babel/code-frame',
'@babel/generator': 'next/dist/compiled/babel/generator',
'@babel/traverse': 'next/dist/compiled/babel/traverse',
'@babel/core': 'next/dist/compiled/babel/core', '@babel/core': 'next/dist/compiled/babel/core',
'@babel/core/lib/config': 'next/dist/compiled/babel/core-lib-config',
'@babel/core/lib/transformation/normalize-file':
'next/dist/compiled/babel/core-lib-normalize-config',
'@babel/core/lib/transformation/normalize-opts':
'next/dist/compiled/babel/core-lib-normalize-opts',
'@babel/core/lib/transformation/block-hoist-plugin':
'next/dist/compiled/babel/core-lib-block-hoisting-plugin',
'@babel/core/lib/transformation/plugin-pass':
'next/dist/compiled/babel/core-lib-plugin-pass',
'@babel/plugin-proposal-class-properties': '@babel/plugin-proposal-class-properties':
'next/dist/compiled/babel/plugin-proposal-class-properties', 'next/dist/compiled/babel/plugin-proposal-class-properties',
'@babel/plugin-proposal-export-namespace-from': '@babel/plugin-proposal-export-namespace-from':
......
...@@ -48,12 +48,30 @@ declare module 'next/dist/compiled/arg/index.js' { ...@@ -48,12 +48,30 @@ declare module 'next/dist/compiled/arg/index.js' {
declare module 'next/dist/compiled/babel/code-frame' { declare module 'next/dist/compiled/babel/code-frame' {
export * from '@babel/code-frame' export * from '@babel/code-frame'
} }
declare module 'next/dist/compiled/babel/traverse' {
import traverse from '@babel/traverse'
export default traverse
export * from '@babel/traverse'
}
declare module 'next/dist/compiled/babel/generator' {
import generate from '@babel/generator'
export default generate
export * from '@babel/generator'
}
declare module 'next/dist/compiled/babel/preset-env' { declare module 'next/dist/compiled/babel/preset-env' {
export default any const anyType: any
export default anyType
} }
declare module 'next/dist/compiled/babel/core' { declare module 'next/dist/compiled/babel/core' {
export * from '@babel/core' export * from '@babel/core'
} }
declare module 'next/dist/compiled/babel/core-lib-config'
declare module 'next/dist/compiled/babel/core-lib-normalize-file'
declare module 'next/dist/compiled/babel/core-lib-normalize-opts'
declare module 'next/dist/compiled/babel/core-lib-block-hoist-plugin'
declare module 'next/dist/compiled/babel/core-lib-plugin-pass'
declare module 'next/dist/compiled/ci-info' { declare module 'next/dist/compiled/ci-info' {
import m from 'ci-info' import m from 'ci-info'
export = m export = m
......
...@@ -28,14 +28,14 @@ declare module 'next/dist/compiled/loader-utils' ...@@ -28,14 +28,14 @@ declare module 'next/dist/compiled/loader-utils'
declare module 'next/dist/compiled/webpack/webpack' { declare module 'next/dist/compiled/webpack/webpack' {
import webpackSources from 'webpack-sources' import webpackSources from 'webpack-sources'
import webpack from 'webpack' import webpack, { loader } from 'webpack'
export let isWebpack5: boolean export let isWebpack5: boolean
export function init(useWebpack5: boolean): void export function init(useWebpack5: boolean): void
export let BasicEvaluatedExpression: any export let BasicEvaluatedExpression: any
export let GraphHelpers: any export let GraphHelpers: any
export function onWebpackInit(cb: () => void): void export function onWebpackInit(cb: () => void): void
export let sources: typeof webpackSources export let sources: typeof webpackSources
export { webpack } export { webpack, loader }
} }
declare module 'webpack' { declare module 'webpack' {
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment