Unverified Commit c90e03d9 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Add pack/unpack scripts from nextpack (#68471)

This brings over the utility scripts `pnpm pack-next` and `pnpm
unpack-next path/to/app` from Nextpack, along with the documentation,
which has been added to `contributing/core/developing.md`.
parent 5f0c5f91
......@@ -4,6 +4,7 @@ dist
.next
target
packages/next/wasm/@next
tarballs/
# dependencies
node_modules
......
......@@ -43,3 +43,38 @@ To develop locally:
For instructions on how to build a project with your local version of the CLI,
see **[Developing Using Your Local Version of Next.js](./developing-using-local-app.md)** as linking the package is not sufficient to develop locally.
## Testing a local Next.js version on an application
Since Turbopack doesn't support symlinks when pointing outside of the workspace directory, it can be difficult to develop against a local Next.js version. Neither `pnpm link` nor `file:` imports quite cut it. An alternative is to pack the Next.js version you want to test into a tarball and add it to the pnpm overrides of your test application. The following script will do it for you:
```bash
pnpm pack-next --release && pnpm unpack-next path/to/project
```
Or without running the build:
```bash
pnpm pack-next --no-build --release && pnpm unpack-next path/to/project
```
### Explanation of the scripts
```bash
# Generate a tarball of the Next.js version you want to test
$ pnpm pack-next
# If you need to build in release mode:
$ pnpm pack-next --release
# You can also pass any cargo argument to the script
# To skip the `pnpm i` and `pnpm build` steps in next.js (e. g. if you are running `pnpm dev`)
$ pnpm pack-next --no-build
```
Afterwards, you'll need to unpack the tarball into your test project. You can either manually edit the `package.json` to point to the new tarballs (see the stdout from `pack-next` script), or you can automatically unpack it with:
```bash
# Unpack the tarballs generated with pack-next into project's node_modules
$ pnpm unpack-next path/to/project
```
......@@ -12,6 +12,7 @@
"build": "turbo run build --remote-cache-timeout 60 --summarize true",
"lerna": "lerna",
"dev": "turbo run dev --parallel",
"pack-next": "node scripts/pack-next.cjs",
"test-types": "tsc",
"test-unit": "jest test/unit/ packages/next/ packages/font",
"test-dev": "cross-env NEXT_TEST_MODE=dev pnpm testheadless",
......@@ -58,6 +59,7 @@
"prepare": "husky",
"sync-react": "node ./scripts/sync-react.js",
"update-google-fonts": "node ./scripts/update-google-fonts.js",
"unpack-next": "node scripts/unpack-next.cjs",
"swc-build-native": "pnpm --filter=@next/swc build-native"
},
"devDependencies": {
......
#!/usr/bin/env node
const {
booleanArg,
exec,
execAsyncWithOutput,
packageFiles,
} = require('./pack-util.cjs')
const fs = require('fs')
const fsPromises = require('fs/promises')
const args = process.argv.slice(2)
const CWD = process.cwd()
const TARBALLS = `${CWD}/tarballs`
const NEXT_PACKAGES = `${CWD}/packages`
const noBuild = booleanArg(args, '--no-build')
;(async () => {
const { globby } = await import('globby')
// the debuginfo on macos is much smaller, so we don't typically need to strip
const DEFAULT_PACK_NEXT_COMPRESS =
process.platform === 'darwin' ? 'none' : 'strip'
const PACK_NEXT_COMPRESS =
process.env.PACK_NEXT_COMPRESS || DEFAULT_PACK_NEXT_COMPRESS
fs.mkdirSync(TARBALLS, { recursive: true })
if (!noBuild) {
exec('Install Next.js build dependencies', 'pnpm i')
exec('Build Next.js', 'pnpm run build')
}
if (PACK_NEXT_COMPRESS !== 'strip') {
// HACK: delete any pre-existing binaries to force napi-rs to rewrite it
let binaries = await nextSwcBinaries()
await Promise.all(binaries.map((bin) => fsPromises.rm(bin)))
}
exec('Build native modules', 'pnpm run swc-build-native')
const NEXT_TARBALL = `${TARBALLS}/next.tar`
const NEXT_SWC_TARBALL = `${TARBALLS}/next-swc.tar`
const NEXT_MDX_TARBALL = `${TARBALLS}/next-mdx.tar`
const NEXT_ENV_TARBALL = `${TARBALLS}/next-env.tar`
const NEXT_BA_TARBALL = `${TARBALLS}/next-bundle-analyzer.tar`
async function nextSwcBinaries() {
return await globby([`${NEXT_PACKAGES}/next-swc/native/*.node`])
}
// We use neither:
// * npm pack, as it doesn't include native modules in the tarball
// * pnpm pack, as it tries to include target directories and compress them,
// which takes forever.
// Instead, we generate non-compressed tarballs.
async function packWithTar(packagePath, tarballPath, extraArgs = []) {
const paths = await packageFiles(packagePath)
const command = [
'tar',
'-c',
// https://apple.stackexchange.com/a/444073
...(process.platform === 'darwin' ? ['--no-mac-metadata'] : []),
'-f',
tarballPath,
...extraArgs,
'--',
...paths.map((p) => `./${p}`),
]
await execAsyncWithOutput(`Pack ${packagePath}`, command, {
cwd: packagePath,
})
}
// Special-case logic for packing next-swc.
//
// pnpm emits `ERR_FS_FILE_TOO_LARGE` if the tarfile is >2GiB due to limits
// in libuv (https://github.com/libuv/libuv/pull/1501). This is common with
// next-swc due to the large amount of debugging symbols. We can fix this one
// of two ways: strip or compression.
//
// We default to stripping (usually faster), but on Linux, we can compress
// instead with objcopy, keeping debug symbols intact. This is controlled by
// `PACK_NEXT_COMPRESS`.
async function packNextSwc() {
const packagePath = `${NEXT_PACKAGES}/next-swc`
switch (PACK_NEXT_COMPRESS) {
case 'strip':
await execAsyncWithOutput('Stripping next-swc native binary', [
'strip',
...(process.platform === 'darwin' ? ['-x', '-'] : ['--']),
await nextSwcBinaries(),
])
await packWithTar(packagePath, NEXT_SWC_TARBALL)
break
case 'objcopy-zstd':
if (process.platform !== 'linux') {
throw new Error('objcopy-zstd is only supported on Linux')
}
await Promise.all(
(await nextSwcBinaries()).map((bin) =>
execAsyncWithOutput(
'Compressing debug symbols in next-swc native binary',
['objcopy', '--compress-debug-sections=zstd', '--', bin]
)
)
)
await packWithTar(packagePath, NEXT_SWC_TARBALL)
break
case 'none':
await packWithTar(packagePath, NEXT_SWC_TARBALL)
break
default:
throw new Error(
"PACK_NEXT_COMPRESS must be one of 'strip', 'objcopy-zstd', or 'none'"
)
}
}
// build all tarfiles in parallel
await Promise.all([
packNextSwc(),
...[
[`${NEXT_PACKAGES}/next`, NEXT_TARBALL],
[`${NEXT_PACKAGES}/next-mdx`, NEXT_MDX_TARBALL],
[`${NEXT_PACKAGES}/next-env`, NEXT_ENV_TARBALL],
[`${NEXT_PACKAGES}/next-bundle-analyzer`, NEXT_BA_TARBALL],
].map(([packagePath, tarballPath]) =>
packWithTar(packagePath, tarballPath)
),
])
console.log('Add the following overrides to your workspace package.json:')
console.log(` "pnpm": {`)
console.log(` "overrides": {`)
console.log(` "next": ${JSON.stringify(`file:${NEXT_TARBALL}`)},`)
console.log(
` "@next/mdx": ${JSON.stringify(`file:${NEXT_MDX_TARBALL}`)},`
)
console.log(
` "@next/env": ${JSON.stringify(`file:${NEXT_ENV_TARBALL}`)},`
)
console.log(
` "@next/bundle-analyzer": ${JSON.stringify(
`file:${NEXT_BA_TARBALL}`
)}`
)
console.log(` }`)
console.log(` }`)
console.log()
console.log('Add the following dependencies to your workspace package.json:')
console.log(` "dependencies": {`)
console.log(` "@next/swc": ${JSON.stringify(`file:${NEXT_SWC_TARBALL}`)},`)
console.log(` ...`)
console.log(` }`)
console.log()
})()
const { execSync, execFileSync, spawn } = require('child_process')
const { existsSync } = require('fs')
const { join } = require('path')
function exec(title, command, opts) {
if (Array.isArray(command)) {
logCommand(title, command)
return execFileSync(command[0], command.slice(1), {
stdio: 'inherit',
...opts,
})
} else {
logCommand(title, command)
return execSync(command, {
stdio: 'inherit',
...opts,
})
}
}
exports.exec = exec
function execAsyncWithOutput(title, command, opts) {
logCommand(title, command)
const proc = spawn(command[0], command.slice(1), {
encoding: 'utf8',
stdio: ['inherit', 'pipe', 'pipe'],
...opts,
})
const stdout = []
proc.stdout.on('data', (data) => {
process.stdout.write(data)
stdout.push(data)
})
const stderr = []
proc.stderr.on('data', (data) => {
process.stderr.write(data)
stderr.push(data)
})
return new Promise((resolve, reject) => {
proc.on('exit', (code) => {
if (code === 0) {
return resolve({
stdout: Buffer.concat(stdout),
stderr: Buffer.concat(stderr),
})
}
const err = new Error(
`Command failed with exit code ${code}: ${prettyCommand(command)}`
)
err.code = code
err.stdout = Buffer.concat(stdout)
err.stderr = Buffer.concat(stderr)
reject(err)
})
})
}
exports.execAsyncWithOutput = execAsyncWithOutput
function prettyCommand(command) {
if (Array.isArray(command)) command = command.join(' ')
return command.replace(/ -- .*/, ' -- …')
}
function logCommand(title, command) {
if (command) {
const pretty = prettyCommand(command)
console.log(`\n\x1b[1;4m${title}\x1b[0m\n> \x1b[1m${pretty}\x1b[0m\n`)
} else {
console.log(`\n\x1b[1;4m${title}\x1b[0m\n`)
}
}
exports.logCommand = logCommand
function booleanArg(args, name) {
const index = args.indexOf(name)
if (index === -1) return false
args.splice(index, 1)
return true
}
exports.booleanArg = booleanArg
const DEFAULT_GLOBS = ['**', '!target', '!node_modules', '!crates', '!.turbo']
const FORCED_GLOBS = ['package.json', 'README*', 'LICENSE*', 'LICENCE*']
async function packageFiles(path) {
const { globby } = await import('globby')
const { files = DEFAULT_GLOBS, main, bin } = require(`${path}/package.json`)
const allFiles = files.concat(
FORCED_GLOBS,
main ?? [],
Object.values(bin ?? {})
)
const isGlob = (f) => f.includes('*') || f.startsWith('!')
const simpleFiles = allFiles
.filter((f) => !isGlob(f) && existsSync(join(path, f)))
.map((f) => f.replace(/^\.\//, ''))
const globFiles = allFiles.filter(isGlob)
const globbedFiles = await globby(globFiles, { cwd: path })
const packageFiles = [...globbedFiles, ...simpleFiles].sort()
const set = new Set()
return packageFiles.filter((f) => {
if (set.has(f)) return false
// We add the full path, but check for parent directories too.
// This catches the case where the whole directory is added and then a single file from the directory.
// The sorting before ensures that the directory comes before the files inside of the directory.
set.add(f)
while (f.includes('/')) {
f = f.replace(/\/[^/]+$/, '')
if (set.has(f)) return false
}
return true
})
}
exports.packageFiles = packageFiles
#!/usr/bin/env node
const { exec } = require('./pack-util.cjs')
const fs = require('fs')
const path = require('path')
const CWD = process.cwd()
const TARBALLS = `${CWD}/tarballs`
const PROJECT_DIR = path.resolve(process.argv[2])
function realPathIfAny(path) {
try {
return fs.realpathSync(path)
} catch {
return null
}
}
const packages = {
next: realPathIfAny(`${PROJECT_DIR}/node_modules/next`),
'next-swc': realPathIfAny(`${PROJECT_DIR}/node_modules/@next/swc`),
'next-mdx': realPathIfAny(`${PROJECT_DIR}/node_modules/@next/mdx`),
'next-bundle-analyzer': realPathIfAny(
`${PROJECT_DIR}/node_modules/@next/bundle-anlyzer`
),
}
for (const [key, path] of Object.entries(packages)) {
if (!path) continue
exec(`Unpack ${key}`, `tar -xf '${TARBALLS}/${key}.tar' -C '${path}'`)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment