Unverified Commit 7d664d5b authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add stats config for SWC (#28883)

* Add stats config for SWC

* Remove stats for deprecated mode

* Include native binaries in local linking

* Expose minify in next-swc

* Build next-swc binaries

* Only upload changed binary artifact

* Build next-swc binaries

* Expose minify in js binding

* Use new minify function in terser plugin

* Build next-swc binaries

* Build next-swc binaries

* Build next-swc binaries

* Pass pagesDir option to SWC for next/dynamic

* Build next-swc binaries

* Add spans for next-trace-entrypoints-plugin

* Fix linting
Co-authored-by: 's avatarMaia Teegarden <dev@padmaia.rocks>
Co-authored-by: 's avatarpadmaia <padmaia@users.noreply.github.com>
Co-authored-by: 's avatartimneutkens <timneutkens@users.noreply.github.com>
parent 0e29a2d0
......@@ -93,6 +93,10 @@ module.exports = (actionInfo) => {
if (!pkgData.dependencies || !pkgData.dependencies[pkg]) continue
pkgData.dependencies[pkg] = packedPkgPath
}
// make sure native binaries are included in local linking
if (pkg === 'next') {
pkgData.files.push('native')
}
await fs.writeFile(
pkgDataPath,
JSON.stringify(pkgData, null, 2),
......
......@@ -29,7 +29,13 @@ DEALINGS IN THE SOFTWARE.
import { getOptions } from 'next/dist/compiled/loader-utils'
import { transform } from '../../swc'
function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) {
function getSWCOptions({
isTypeScript,
isServer,
development,
isPageFile,
pagesDir,
}) {
const jsc = {
parser: {
syntax: isTypeScript ? 'typescript' : 'ecmascript',
......@@ -54,6 +60,7 @@ function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) {
jsc,
// Disables getStaticProps/getServerSideProps tree shaking on the server compilation for pages
disableNextSsg: true,
pagesDir,
env: {
targets: {
// Targets the current version of Node.js
......@@ -64,7 +71,11 @@ function getSWCOptions({ isTypeScript, isServer, development, isPageFile }) {
} else {
// Matches default @babel/preset-env behavior
jsc.target = 'es5'
return { disableNextSsg: !isPageFile, jsc }
return {
disableNextSsg: !isPageFile,
pagesDir,
jsc,
}
}
}
......@@ -80,6 +91,7 @@ async function loaderTransform(parentTrace, source, inputSourceMap) {
const isPageFile = filename.startsWith(pagesDir)
const swcOptions = getSWCOptions({
pagesDir,
isTypeScript,
isServer: isServer,
isPageFile,
......
......@@ -6,6 +6,8 @@ import {
sources,
} from 'next/dist/compiled/webpack/webpack'
import { TRACE_OUTPUT_VERSION } from '../../../shared/lib/constants'
import { spans } from './profiling-plugin'
import { Span } from '../../../trace'
const PLUGIN_NAME = 'TraceEntryPointsPlugin'
const TRACE_IGNORES = [
......@@ -43,116 +45,109 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
// Here we output all traced assets and webpack chunks to a
// ${page}.js.nft.json file
createTraceAssets(compilation: any, assets: any) {
createTraceAssets(compilation: any, assets: any, span: Span) {
const outputPath = compilation.outputOptions.path
for (const entrypoint of compilation.entrypoints.values()) {
const entryFiles = new Set<string>()
const nodeFileTraceSpan = span.traceChild('create-trace-assets')
nodeFileTraceSpan.traceFn(() => {
for (const entrypoint of compilation.entrypoints.values()) {
const entryFiles = new Set<string>()
for (const chunk of entrypoint
.getEntrypointChunk()
.getAllReferencedChunks()) {
for (const file of chunk.files) {
entryFiles.add(nodePath.join(outputPath, file))
}
for (const file of chunk.auxiliaryFiles) {
entryFiles.add(nodePath.join(outputPath, file))
for (const chunk of entrypoint
.getEntrypointChunk()
.getAllReferencedChunks()) {
for (const file of chunk.files) {
entryFiles.add(nodePath.join(outputPath, file))
}
for (const file of chunk.auxiliaryFiles) {
entryFiles.add(nodePath.join(outputPath, file))
}
}
}
// don't include the entry itself in the trace
entryFiles.delete(
nodePath.join(
outputPath,
`${isWebpack5 ? '../' : ''}${entrypoint.name}.js`
// don't include the entry itself in the trace
entryFiles.delete(
nodePath.join(
outputPath,
`${isWebpack5 ? '../' : ''}${entrypoint.name}.js`
)
)
const traceOutputName = `${isWebpack5 ? '../' : ''}${
entrypoint.name
}.js.nft.json`
const traceOutputPath = nodePath.dirname(
nodePath.join(outputPath, traceOutputName)
)
)
const traceOutputName = `${isWebpack5 ? '../' : ''}${
entrypoint.name
}.js.nft.json`
const traceOutputPath = nodePath.dirname(
nodePath.join(outputPath, traceOutputName)
)
assets[traceOutputName] = new sources.RawSource(
JSON.stringify({
version: TRACE_OUTPUT_VERSION,
files: [
...entryFiles,
...(this.entryTraces.get(entrypoint.name) || []),
].map((file) => {
return nodePath.relative(traceOutputPath, file).replace(/\\/g, '/')
}),
})
)
}
}
apply(compiler: webpack.Compiler) {
if (isWebpack5) {
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
// @ts-ignore TODO: Remove ignore when webpack 5 is stable
compilation.hooks.processAssets.tap(
{
name: PLUGIN_NAME,
// @ts-ignore TODO: Remove ignore when webpack 5 is stable
stage: webpack.Compilation.PROCESS_ASSETS_STAGE_SUMMARIZE,
},
(assets: any) => {
this.createTraceAssets(compilation, assets)
}
assets[traceOutputName] = new sources.RawSource(
JSON.stringify({
version: TRACE_OUTPUT_VERSION,
files: [
...entryFiles,
...(this.entryTraces.get(entrypoint.name) || []),
].map((file) => {
return nodePath
.relative(traceOutputPath, file)
.replace(/\\/g, '/')
}),
})
)
})
} else {
compiler.hooks.emit.tap(PLUGIN_NAME, (compilation: any) => {
this.createTraceAssets(compilation, compilation.assets)
})
}
}
})
}
tapfinishModules(
compilation: webpack.compilation.Compilation,
traceEntrypointsPluginSpan: Span
) {
compilation.hooks.finishModules.tapAsync(
PLUGIN_NAME,
async (_stats: any, callback: any) => {
const finishModulesSpan =
traceEntrypointsPluginSpan.traceChild('finish-modules')
await finishModulesSpan
.traceAsyncFn(async () => {
// we create entry -> module maps so that we can
// look them up faster instead of having to iterate
// over the compilation modules list
const entryNameMap = new Map<string, string>()
const entryModMap = new Map<string, any>()
const additionalEntries = new Map<string, Map<string, any>>()
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
compilation.hooks.finishModules.tapAsync(
PLUGIN_NAME,
async (_stats: any, callback: any) => {
// we create entry -> module maps so that we can
// look them up faster instead of having to iterate
// over the compilation modules list
const entryNameMap = new Map<string, string>()
const entryModMap = new Map<string, any>()
const additionalEntries = new Map<string, Map<string, any>>()
try {
const depModMap = new Map<string, any>()
compilation.entries.forEach((entry) => {
const name = entry.name || entry.options?.name
if (name?.replace(/\\/g, '/').startsWith('pages/')) {
for (const dep of entry.dependencies) {
if (!dep) continue
const entryMod = getModuleFromDependency(compilation, dep)
if (entryMod && entryMod.resource) {
if (
entryMod.resource.replace(/\\/g, '/').includes('pages/')
) {
entryNameMap.set(entryMod.resource, name)
entryModMap.set(entryMod.resource, entryMod)
} else {
let curMap = additionalEntries.get(name)
if (!curMap) {
curMap = new Map()
additionalEntries.set(name, curMap)
finishModulesSpan.traceChild('get-entries').traceFn(() => {
compilation.entries.forEach((entry) => {
const name = entry.name || entry.options?.name
if (name?.replace(/\\/g, '/').startsWith('pages/')) {
for (const dep of entry.dependencies) {
if (!dep) continue
const entryMod = getModuleFromDependency(compilation, dep)
if (entryMod && entryMod.resource) {
if (
entryMod.resource.replace(/\\/g, '/').includes('pages/')
) {
entryNameMap.set(entryMod.resource, name)
entryModMap.set(entryMod.resource, entryMod)
} else {
let curMap = additionalEntries.get(name)
if (!curMap) {
curMap = new Map()
additionalEntries.set(name, curMap)
}
curMap.set(entryMod.resource, entryMod)
}
curMap.set(entryMod.resource, entryMod)
}
}
}
}
})
})
// TODO: investigate allowing non-sync fs calls in node-file-trace
// for better performance
const readFile = (path: string) => {
const readFile = (path: string, _span: Span) => {
// return span.traceChild('read-file', { path }).traceFn(() => {
const mod = depModMap.get(path) || entryModMap.get(path)
// map the transpiled source when available to avoid
......@@ -171,8 +166,10 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
}
throw e
}
// })
}
const readlink = (path: string) => {
const readlink = (path: string, _span: Span) => {
// return span.traceChild('read-link', { path }).traceFn(() => {
try {
return compilation.inputFileSystem.readlinkSync(path)
} catch (e) {
......@@ -185,8 +182,10 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
}
return null
}
// })
}
const stat = (path: string) => {
const stat = (path: string, _span: Span) => {
// return span.traceChild('stat', { path }).traceFn(() => {
try {
return compilation.inputFileSystem.statSync(path)
} catch (e) {
......@@ -195,89 +194,154 @@ export class TraceEntryPointsPlugin implements webpack.Plugin {
}
throw e
}
// })
}
const nftCache = {}
const entryPaths = Array.from(entryModMap.keys())
for (const entry of entryPaths) {
depModMap.clear()
const entryMod = entryModMap.get(entry)
// TODO: investigate caching, will require ensuring no traced
// files in the cache have changed, we could potentially hash
// all traced files and only leverage the cache if the hashes
// match
// const cachedTraces = entryMod.buildInfo?.cachedNextEntryTrace
// Use cached trace if available and trace version matches
// if (
// isWebpack5 &&
// cachedTraces &&
// cachedTraces.version === TRACE_OUTPUT_VERSION
// ) {
// this.entryTraces.set(
// entryNameMap.get(entry)!,
// cachedTraces.tracedDeps
// )
// continue
// }
const collectDependencies = (mod: any) => {
if (!mod || !mod.dependencies) return
for (const dep of mod.dependencies) {
const depMod = getModuleFromDependency(compilation, dep)
if (depMod?.resource && !depModMap.get(depMod.resource)) {
depModMap.set(depMod.resource, depMod)
collectDependencies(depMod)
}
const entrySpan = finishModulesSpan.traceChild('entry', { entry })
await entrySpan.traceAsyncFn(async () => {
depModMap.clear()
const entryMod = entryModMap.get(entry)
// TODO: investigate caching, will require ensuring no traced
// files in the cache have changed, we could potentially hash
// all traced files and only leverage the cache if the hashes
// match
// const cachedTraces = entryMod.buildInfo?.cachedNextEntryTrace
// Use cached trace if available and trace version matches
// if (
// isWebpack5 &&
// cachedTraces &&
// cachedTraces.version === TRACE_OUTPUT_VERSION
// ) {
// this.entryTraces.set(
// entryNameMap.get(entry)!,
// cachedTraces.tracedDeps
// )
// continue
// }
const collectDependencies = (mod: any, span: Span) => {
const childSpan = span.traceChild('collect-dependencies', {
resource: mod.resource,
})
return childSpan.traceFn(() => {
if (!mod || !mod.dependencies) return
for (const dep of mod.dependencies) {
const depMod = getModuleFromDependency(compilation, dep)
if (depMod?.resource && !depModMap.get(depMod.resource)) {
depModMap.set(depMod.resource, depMod)
collectDependencies(depMod, childSpan)
}
}
})
}
}
collectDependencies(entryMod)
collectDependencies(entryMod, entrySpan)
const toTrace: string[] = [entry, ...depModMap.keys()]
const toTrace: string[] = [entry, ...depModMap.keys()]
const entryName = entryNameMap.get(entry)!
const curExtraEntries = additionalEntries.get(entryName)
const entryName = entryNameMap.get(entry)!
const curExtraEntries = additionalEntries.get(entryName)
if (curExtraEntries) {
toTrace.push(...curExtraEntries.keys())
}
const root = nodePath.parse(process.cwd()).root
const result = await nodeFileTrace(toTrace, {
base: root,
cache: nftCache,
processCwd: this.appDir,
readFile,
readlink,
stat,
ignore: [...TRACE_IGNORES, ...this.excludeFiles],
mixedModules: true,
})
const tracedDeps: string[] = []
if (curExtraEntries) {
toTrace.push(...curExtraEntries.keys())
}
for (const file of result.fileList) {
if (result.reasons[file].type === 'initial') {
continue
const root = nodePath.parse(process.cwd()).root
const fileTraceSpan = entrySpan.traceChild('node-file-trace')
const result = await fileTraceSpan.traceAsyncFn(() =>
nodeFileTrace(toTrace, {
base: root,
cache: nftCache,
processCwd: this.appDir,
readFile: (path) => readFile(path, fileTraceSpan),
readlink: (path) => readlink(path, fileTraceSpan),
stat: (path) => stat(path, fileTraceSpan),
ignore: [...TRACE_IGNORES, ...this.excludeFiles],
mixedModules: true,
})
)
const tracedDeps: string[] = []
for (const file of result.fileList) {
if (result.reasons[file].type === 'initial') {
continue
}
tracedDeps.push(nodePath.join(root, file))
}
tracedDeps.push(nodePath.join(root, file))
}
// entryMod.buildInfo.cachedNextEntryTrace = {
// version: TRACE_OUTPUT_VERSION,
// tracedDeps,
// }
this.entryTraces.set(entryName, tracedDeps)
// entryMod.buildInfo.cachedNextEntryTrace = {
// version: TRACE_OUTPUT_VERSION,
// tracedDeps,
// }
this.entryTraces.set(entryName, tracedDeps)
})
}
})
.then(
() => callback(),
(err) => callback(err)
)
}
)
}
callback()
} catch (err) {
callback(err)
}
}
)
})
apply(compiler: webpack.Compiler) {
if (isWebpack5) {
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
const compilerSpan = spans.get(compiler)!
const traceEntrypointsPluginSpan = compilerSpan.traceChild(
'next-trace-entrypoint-plugin'
)
traceEntrypointsPluginSpan.traceFn(() => {
// @ts-ignore TODO: Remove ignore when webpack 5 is stable
compilation.hooks.processAssets.tap(
{
name: PLUGIN_NAME,
// @ts-ignore TODO: Remove ignore when webpack 5 is stable
stage: webpack.Compilation.PROCESS_ASSETS_STAGE_SUMMARIZE,
},
(assets: any) => {
this.createTraceAssets(
compilation,
assets,
traceEntrypointsPluginSpan
)
}
)
this.tapfinishModules(compilation, traceEntrypointsPluginSpan)
})
})
} else {
compiler.hooks.emit.tap(PLUGIN_NAME, (compilation: any) => {
const compilerSpan = spans.get(compiler)!
const traceEntrypointsPluginSpan = compilerSpan.traceChild(
'next-trace-entrypoint-plugin'
)
traceEntrypointsPluginSpan.traceFn(() => {
this.createTraceAssets(
compilation,
compilation.assets,
traceEntrypointsPluginSpan
)
})
})
compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => {
const compilerSpan = spans.get(compiler)!
const traceEntrypointsPluginSpan = compilerSpan.traceChild(
'next-trace-entrypoint-plugin'
)
traceEntrypointsPluginSpan.traceFn(() =>
this.tapfinishModules(compilation, traceEntrypointsPluginSpan)
)
})
}
}
}
......@@ -121,15 +121,22 @@ module.exports = {
},
},
{
title: 'Webpack 4 Mode',
title: 'Default Build with SWC',
diff: 'onOutputChange',
diffConfigFiles: [
{
path: 'pages/image.js',
content: imagePageData,
},
{
path: 'next.config.js',
content: `
module.exports = {
generateBuildId: () => 'BUILD_ID',
webpack5: false,
experimental: {
swcLoader: true,
swcMinify: true,
},
webpack(config) {
config.optimization.minimize = false
config.optimization.minimizer = undefined
......@@ -139,14 +146,22 @@ module.exports = {
`,
},
],
// renames to apply to make file names deterministic
renames,
configFiles: [
{
path: 'pages/image.js',
content: imagePageData,
},
{
path: 'next.config.js',
content: `
module.exports = {
generateBuildId: () => 'BUILD_ID',
webpack5: false
experimental: {
swcLoader: true,
swcMinify: true
},
generateBuildId: () => 'BUILD_ID'
}
`,
},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment