Unverified Commit 7d664d5b authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add stats config for SWC (#28883)

* Add stats config for SWC

* Remove stats for deprecated mode

* Include native binaries in local linking

* Expose minify in next-swc

* Build next-swc binaries

* Only upload changed binary artifact

* Build next-swc binaries

* Expose minify in js binding

* Use new minify function in terser plugin

* Build next-swc binaries

* Build next-swc binaries

* Build next-swc binaries

* Pass pagesDir option to SWC for next/dynamic

* Build next-swc binaries

* Add spans for next-trace-entrypoints-plugin

* Fix linting
Co-authored-by: 's avatarMaia Teegarden <dev@padmaia.rocks>
Co-authored-by: 's avatarpadmaia <padmaia@users.noreply.github.com>
Co-authored-by: 's avatartimneutkens <timneutkens@users.noreply.github.com>
parent 0e29a2d0
...@@ -93,6 +93,10 @@ module.exports = (actionInfo) => { ...@@ -93,6 +93,10 @@ module.exports = (actionInfo) => {
if (!pkgData.dependencies || !pkgData.dependencies[pkg]) continue if (!pkgData.dependencies || !pkgData.dependencies[pkg]) continue
pkgData.dependencies[pkg] = packedPkgPath pkgData.dependencies[pkg] = packedPkgPath
} }
// make sure native binaries are included in local linking
if (pkg === 'next') {
pkgData.files.push('native')
}
await fs.writeFile( await fs.writeFile(
pkgDataPath, pkgDataPath,
JSON.stringify(pkgData, null, 2), JSON.stringify(pkgData, null, 2),
......
...@@ -29,7 +29,13 @@ DEALINGS IN THE SOFTWARE. ...@@ -29,7 +29,13 @@ DEALINGS IN THE SOFTWARE.
import { getOptions } from 'next/dist/compiled/loader-utils' import { getOptions } from 'next/dist/compiled/loader-utils'
import { transform } from '../../swc' import { transform } from '../../swc'
function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) { function getSWCOptions({
isTypeScript,
isServer,
development,
isPageFile,
pagesDir,
}) {
const jsc = { const jsc = {
parser: { parser: {
syntax: isTypeScript ? 'typescript' : 'ecmascript', syntax: isTypeScript ? 'typescript' : 'ecmascript',
...@@ -54,6 +60,7 @@ function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) { ...@@ -54,6 +60,7 @@ function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) {
jsc, jsc,
// Disables getStaticProps/getServerSideProps tree shaking on the server compilation for pages // Disables getStaticProps/getServerSideProps tree shaking on the server compilation for pages
disableNextSsg: true, disableNextSsg: true,
pagesDir,
env: { env: {
targets: { targets: {
// Targets the current version of Node.js // Targets the current version of Node.js
...@@ -64,7 +71,11 @@ function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) { ...@@ -64,7 +71,11 @@ function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) {
} else { } else {
// Matches default @babel/preset-env behavior // Matches default @babel/preset-env behavior
jsc.target = 'es5' jsc.target = 'es5'
return { disableNextSsg: !isPageFile, jsc } return {
disableNextSsg: !isPageFile,
pagesDir,
jsc,
}
} }
} }
...@@ -80,6 +91,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) { ...@@ -80,6 +91,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) {
const isPageFile = filename.startsWith(pagesDir) const isPageFile = filename.startsWith(pagesDir)
const swcOptions = getSWCOptions({ const swcOptions = getSWCOptions({
pagesDir,
isTypeScript, isTypeScript,
isServer: isServer, isServer: isServer,
isPageFile, isPageFile,
......
...@@ -6,6 +6,8 @@ import { ...@@ -6,6 +6,8 @@ import {
sources, sources,
} from 'next/dist/compiled/webpack/webpack' } from 'next/dist/compiled/webpack/webpack'
import { TRACE_OUTPUT_VERSION } from '../../../shared/lib/constants' import { TRACE_OUTPUT_VERSION } from '../../../shared/lib/constants'
import { spans } from './profiling-plugin'
import { Span } from '../../../trace'
const PLUGIN_NAME = 'TraceEntryPointsPlugin' const PLUGIN_NAME = 'TraceEntryPointsPlugin'
const TRACE_IGNORES = [ const TRACE_IGNORES = [
...@@ -43,116 +45,109 @@ export class TraceEntryPointsPlugin implements webpack.Plugin { ...@@ -43,116 +45,109 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
// Here we output all traced assets and webpack chunks to a // Here we output all traced assets and webpack chunks to a
// ${page}.js.nft.json file // ${page}.js.nft.json file
createTraceAssets(compilation: any, assets: any) { createTraceAssets(compilation: any, assets: any, span: Span) {
const outputPath = compilation.outputOptions.path const outputPath = compilation.outputOptions.path
for (const entrypoint of compilation.entrypoints.values()) { const nodeFileTraceSpan = span.traceChild('create-trace-assets')
const entryFiles = new Set<string>() nodeFileTraceSpan.traceFn(() => {
for (const entrypoint of compilation.entrypoints.values()) {
const entryFiles = new Set<string>()
for (const chunk of entrypoint for (const chunk of entrypoint
.getEntrypointChunk() .getEntrypointChunk()
.getAllReferencedChunks()) { .getAllReferencedChunks()) {
for (const file of chunk.files) { for (const file of chunk.files) {
entryFiles.add(nodePath.join(outputPath, file)) entryFiles.add(nodePath.join(outputPath, file))
} }
for (const file of chunk.auxiliaryFiles) { for (const file of chunk.auxiliaryFiles) {
entryFiles.add(nodePath.join(outputPath, file)) entryFiles.add(nodePath.join(outputPath, file))
}
} }
} // don't include the entry itself in the trace
// don't include the entry itself in the trace entryFiles.delete(
entryFiles.delete( nodePath.join(
nodePath.join( outputPath,
outputPath, `${isWebpack5 ? '../' : ''}${entrypoint.name}.js`
`${isWebpack5 ? '../' : ''}${entrypoint.name}.js` )
)
const traceOutputName = `${isWebpack5 ? '../' : ''}${
entrypoint.name
}.js.nft.json`
const traceOutputPath = nodePath.dirname(
nodePath.join(outputPath, traceOutputName)
) )
)
const traceOutputName = `${isWebpack5 ? '../' : ''}${
entrypoint.name
}.js.nft.json`
const traceOutputPath = nodePath.dirname(
nodePath.join(outputPath, traceOutputName)
)
assets[traceOutputName] = new sources.RawSource(
JSON.stringify({
version: TRACE_OUTPUT_VERSION,
files: [
...entryFiles,
...(this.entryTraces.get(entrypoint.name) || []),
].map((file) => {
return nodePath.relative(traceOutputPath, file).replace(/\\/g, '/')
}),
})
)
}
}
apply(compiler: webpack.Compiler) { assets[traceOutputName] = new sources.RawSource(
if (isWebpack5) { JSON.stringify({
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => { version: TRACE_OUTPUT_VERSION,
// @ts-ignore TODO: Remove ignore when webpack 5 is stable files: [
compilation.hooks.processAssets.tap( ...entryFiles,
{ ...(this.entryTraces.get(entrypoint.name) || []),
name: PLUGIN_NAME, ].map((file) => {
// @ts-ignore TODO: Remove ignore when webpack 5 is stable return nodePath
stage: webpack.Compilation.PROCESS_ASSETS_STAGE_SUMMARIZE, .relative(traceOutputPath, file)
}, .replace(/\\/g, '/')
(assets: any) => { }),
this.createTraceAssets(compilation, assets) })
}
) )
}) }
} else { })
compiler.hooks.emit.tap(PLUGIN_NAME, (compilation: any) => { }
this.createTraceAssets(compilation, compilation.assets)
}) tapfinishModules(
} compilation: webpack.compilation.Compilation,
traceEntrypointsPluginSpan: Span
) {
compilation.hooks.finishModules.tapAsync(
PLUGIN_NAME,
async (_stats: any, callback: any) => {
const finishModulesSpan =
traceEntrypointsPluginSpan.traceChild('finish-modules')
await finishModulesSpan
.traceAsyncFn(async () => {
// we create entry -> module maps so that we can
// look them up faster instead of having to iterate
// over the compilation modules list
const entryNameMap = new Map<string, string>()
const entryModMap = new Map<string, any>()
const additionalEntries = new Map<string, Map<string, any>>()
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
compilation.hooks.finishModules.tapAsync(
PLUGIN_NAME,
async (_stats: any, callback: any) => {
// we create entry -> module maps so that we can
// look them up faster instead of having to iterate
// over the compilation modules list
const entryNameMap = new Map<string, string>()
const entryModMap = new Map<string, any>()
const additionalEntries = new Map<string, Map<string, any>>()
try {
const depModMap = new Map<string, any>() const depModMap = new Map<string, any>()
compilation.entries.forEach((entry) => { finishModulesSpan.traceChild('get-entries').traceFn(() => {
const name = entry.name || entry.options?.name compilation.entries.forEach((entry) => {
const name = entry.name || entry.options?.name
if (name?.replace(/\\/g, '/').startsWith('pages/')) {
for (const dep of entry.dependencies) { if (name?.replace(/\\/g, '/').startsWith('pages/')) {
if (!dep) continue for (const dep of entry.dependencies) {
const entryMod = getModuleFromDependency(compilation, dep) if (!dep) continue
const entryMod = getModuleFromDependency(compilation, dep)
if (entryMod && entryMod.resource) {
if ( if (entryMod && entryMod.resource) {
entryMod.resource.replace(/\\/g, '/').includes('pages/') if (
) { entryMod.resource.replace(/\\/g, '/').includes('pages/')
entryNameMap.set(entryMod.resource, name) ) {
entryModMap.set(entryMod.resource, entryMod) entryNameMap.set(entryMod.resource, name)
} else { entryModMap.set(entryMod.resource, entryMod)
let curMap = additionalEntries.get(name) } else {
let curMap = additionalEntries.get(name)
if (!curMap) {
curMap = new Map() if (!curMap) {
additionalEntries.set(name, curMap) curMap = new Map()
additionalEntries.set(name, curMap)
}
curMap.set(entryMod.resource, entryMod)
} }
curMap.set(entryMod.resource, entryMod)
} }
} }
} }
} })
}) })
// TODO: investigate allowing non-sync fs calls in node-file-trace // TODO: investigate allowing non-sync fs calls in node-file-trace
// for better performance // for better performance
const readFile = (path: string) => { const readFile = (path: string, _span: Span) => {
// return span.traceChild('read-file', { path }).traceFn(() => {
const mod = depModMap.get(path) || entryModMap.get(path) const mod = depModMap.get(path) || entryModMap.get(path)
// map the transpiled source when available to avoid // map the transpiled source when available to avoid
...@@ -171,8 +166,10 @@ export class TraceEntryPointsPlugin implements webpack.Plugin { ...@@ -171,8 +166,10 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
} }
throw e throw e
} }
// })
} }
const readlink = (path: string) => { const readlink = (path: string, _span: Span) => {
// return span.traceChild('read-link', { path }).traceFn(() => {
try { try {
return compilation.inputFileSystem.readlinkSync(path) return compilation.inputFileSystem.readlinkSync(path)
} catch (e) { } catch (e) {
...@@ -185,8 +182,10 @@ export class TraceEntryPointsPlugin implements webpack.Plugin { ...@@ -185,8 +182,10 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
} }
return null return null
} }
// })
} }
const stat = (path: string) => { const stat = (path: string, _span: Span) => {
// return span.traceChild('stat', { path }).traceFn(() => {
try { try {
return compilation.inputFileSystem.statSync(path) return compilation.inputFileSystem.statSync(path)
} catch (e) { } catch (e) {
...@@ -195,89 +194,154 @@ export class TraceEntryPointsPlugin implements webpack.Plugin { ...@@ -195,89 +194,154 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
} }
throw e throw e
} }
// })
} }
const nftCache = {} const nftCache = {}
const entryPaths = Array.from(entryModMap.keys()) const entryPaths = Array.from(entryModMap.keys())
for (const entry of entryPaths) { for (const entry of entryPaths) {
depModMap.clear() const entrySpan = finishModulesSpan.traceChild('entry', { entry })
const entryMod = entryModMap.get(entry) await entrySpan.traceAsyncFn(async () => {
// TODO: investigate caching, will require ensuring no traced depModMap.clear()
// files in the cache have changed, we could potentially hash const entryMod = entryModMap.get(entry)
// all traced files and only leverage the cache if the hashes // TODO: investigate caching, will require ensuring no traced
// match // files in the cache have changed, we could potentially hash
// const cachedTraces = entryMod.buildInfo?.cachedNextEntryTrace // all traced files and only leverage the cache if the hashes
// match
// Use cached trace if available and trace version matches // const cachedTraces = entryMod.buildInfo?.cachedNextEntryTrace
// if (
// isWebpack5 && // Use cached trace if available and trace version matches
// cachedTraces && // if (
// cachedTraces.version === TRACE_OUTPUT_VERSION // isWebpack5 &&
// ) { // cachedTraces &&
// this.entryTraces.set( // cachedTraces.version === TRACE_OUTPUT_VERSION
// entryNameMap.get(entry)!, // ) {
// cachedTraces.tracedDeps // this.entryTraces.set(
// ) // entryNameMap.get(entry)!,
// continue // cachedTraces.tracedDeps
// } // )
const collectDependencies = (mod: any) => { // continue
if (!mod || !mod.dependencies) return // }
const collectDependencies = (mod: any, span: Span) => {
for (const dep of mod.dependencies) { const childSpan = span.traceChild('collect-dependencies', {
const depMod = getModuleFromDependency(compilation, dep) resource: mod.resource,
})
if (depMod?.resource && !depModMap.get(depMod.resource)) { return childSpan.traceFn(() => {
depModMap.set(depMod.resource, depMod) if (!mod || !mod.dependencies) return
collectDependencies(depMod)
} for (const dep of mod.dependencies) {
const depMod = getModuleFromDependency(compilation, dep)
if (depMod?.resource && !depModMap.get(depMod.resource)) {
depModMap.set(depMod.resource, depMod)
collectDependencies(depMod, childSpan)
}
}
})
} }
} collectDependencies(entryMod, entrySpan)
collectDependencies(entryMod)
const toTrace: string[] = [entry, ...depModMap.keys()] const toTrace: string[] = [entry, ...depModMap.keys()]
const entryName = entryNameMap.get(entry)! const entryName = entryNameMap.get(entry)!
const curExtraEntries = additionalEntries.get(entryName) const curExtraEntries = additionalEntries.get(entryName)
if (curExtraEntries) { if (curExtraEntries) {
toTrace.push(...curExtraEntries.keys()) toTrace.push(...curExtraEntries.keys())
} }
const root = nodePath.parse(process.cwd()).root
const result = await nodeFileTrace(toTrace, {
base: root,
cache: nftCache,
processCwd: this.appDir,
readFile,
readlink,
stat,
ignore: [...TRACE_IGNORES, ...this.excludeFiles],
mixedModules: true,
})
const tracedDeps: string[] = []
for (const file of result.fileList) { const root = nodePath.parse(process.cwd()).root
if (result.reasons[file].type === 'initial') { const fileTraceSpan = entrySpan.traceChild('node-file-trace')
continue const result = await fileTraceSpan.traceAsyncFn(() =>
nodeFileTrace(toTrace, {
base: root,
cache: nftCache,
processCwd: this.appDir,
readFile: (path) => readFile(path, fileTraceSpan),
readlink: (path) => readlink(path, fileTraceSpan),
stat: (path) => stat(path, fileTraceSpan),
ignore: [...TRACE_IGNORES, ...this.excludeFiles],
mixedModules: true,
})
)
const tracedDeps: string[] = []
for (const file of result.fileList) {
if (result.reasons[file].type === 'initial') {
continue
}
tracedDeps.push(nodePath.join(root, file))
} }
tracedDeps.push(nodePath.join(root, file))
}
// entryMod.buildInfo.cachedNextEntryTrace = { // entryMod.buildInfo.cachedNextEntryTrace = {
// version: TRACE_OUTPUT_VERSION, // version: TRACE_OUTPUT_VERSION,
// tracedDeps, // tracedDeps,
// } // }
this.entryTraces.set(entryName, tracedDeps) this.entryTraces.set(entryName, tracedDeps)
})
} }
})
.then(
() => callback(),
(err) => callback(err)
)
}
)
}
callback() apply(compiler: webpack.Compiler) {
} catch (err) { if (isWebpack5) {
callback(err) compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
} const compilerSpan = spans.get(compiler)!
} const traceEntrypointsPluginSpan = compilerSpan.traceChild(
) 'next-trace-entrypoint-plugin'
}) )
traceEntrypointsPluginSpan.traceFn(() => {
// @ts-ignore TODO: Remove ignore when webpack 5 is stable
compilation.hooks.processAssets.tap(
{
name: PLUGIN_NAME,
// @ts-ignore TODO: Remove ignore when webpack 5 is stable
stage: webpack.Compilation.PROCESS_ASSETS_STAGE_SUMMARIZE,
},
(assets: any) => {
this.createTraceAssets(
compilation,
assets,
traceEntrypointsPluginSpan
)
}
)
this.tapfinishModules(compilation, traceEntrypointsPluginSpan)
})
})
} else {
compiler.hooks.emit.tap(PLUGIN_NAME, (compilation: any) => {
const compilerSpan = spans.get(compiler)!
const traceEntrypointsPluginSpan = compilerSpan.traceChild(
'next-trace-entrypoint-plugin'
)
traceEntrypointsPluginSpan.traceFn(() => {
this.createTraceAssets(
compilation,
compilation.assets,
traceEntrypointsPluginSpan
)
})
})
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
const compilerSpan = spans.get(compiler)!
const traceEntrypointsPluginSpan = compilerSpan.traceChild(
'next-trace-entrypoint-plugin'
)
traceEntrypointsPluginSpan.traceFn(() =>
this.tapfinishModules(compilation, traceEntrypointsPluginSpan)
)
})
}
} }
} }
...@@ -121,15 +121,22 @@ module.exports = { ...@@ -121,15 +121,22 @@ module.exports = {
}, },
}, },
{ {
title: 'Webpack 4 Mode', title: 'Default Build with SWC',
diff: 'onOutputChange', diff: 'onOutputChange',
diffConfigFiles: [ diffConfigFiles: [
{
path: 'pages/image.js',
content: imagePageData,
},
{ {
path: 'next.config.js', path: 'next.config.js',
content: ` content: `
module.exports = { module.exports = {
generateBuildId: () => 'BUILD_ID', generateBuildId: () => 'BUILD_ID',
webpack5: false, experimental: {
swcLoader: true,
swcMinify: true,
},
webpack(config) { webpack(config) {
config.optimization.minimize = false config.optimization.minimize = false
config.optimization.minimizer = undefined config.optimization.minimizer = undefined
...@@ -139,14 +146,22 @@ module.exports = { ...@@ -139,14 +146,22 @@ module.exports = {
`, `,
}, },
], ],
// renames to apply to make file names deterministic
renames, renames,
configFiles: [ configFiles: [
{
path: 'pages/image.js',
content: imagePageData,
},
{ {
path: 'next.config.js', path: 'next.config.js',
content: ` content: `
module.exports = { module.exports = {
generateBuildId: () => 'BUILD_ID', experimental: {
webpack5: false swcLoader: true,
swcMinify: true
},
generateBuildId: () => 'BUILD_ID'
} }
`, `,
}, },
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment