Unverified Commit cdd77ac4 authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add deployable tarballs to pack-next (#93963)

### What?

Adds a `--deployable-tar` option to `pack-next` that writes generated
tarballs into the target project and patches package references to use
relative `file:` paths.

### Why?

Projects that need to deploy with locally packed Next.js packages should
be able to include those tarballs inside the project directory instead
of referencing tarballs from the Next.js checkout.

### How?

Resolves the target project package.json location, writes tarballs to a
sibling `tarballs/` directory, and maps package override paths to
project-relative references when patching package.json. Existing `--tar`
behavior continues to write to the repository tarballs directory.

### Verification

- `git diff --check`
- `git diff --cached --check`
- `pnpm prettier --with-node-modules --ignore-path .prettierignore
--write scripts/pack-next.ts scripts/pack-utils/patch-package-json.ts
contributing/core/testing.md contributing/core/developing.md`
- `npx eslint --config eslint.config.mjs --fix scripts/pack-next.ts
scripts/pack-utils/patch-package-json.ts contributing/core/testing.md
contributing/core/developing.md` (Markdown files were ignored because no
matching ESLint config was supplied)
- `pnpm pack-next --project ../sandbox/nextjs-duplicated-html-repro/
--deployable-tar`

<!-- NEXT_JS_LLM_PR -->
parent 4e3eb513
......@@ -129,6 +129,12 @@ Or without running the build:
pnpm pack-next --no-js-build --tar && pnpm unpack-next path/to/project
```
To create tarballs inside the project directory with deployable relative `file:` references:
```bash
pnpm pack-next --project path/to/project --deployable-tar
```
Without going through a tarball (only works if you've added the overrides from `pack-next`):
```bash
......
......@@ -214,6 +214,15 @@ On Linux, this generates stripped `@next/swc` binaries to avoid exceeding 2 GiB,
known to cause problems with `pnpm`](https://github.com/libuv/libuv/pull/1501). That behavior can be
overridden with `--compress objcopy-zstd` on Linux (which is slower, but retains debuginfo).
To create tarballs that can be deployed with a project, use:
```bash
pnpm pack-next --project ~/my-project/ --deployable-tar
```
This writes the tarballs to a `tarballs` directory next to the patched project `package.json` and
uses relative `file:` references so the tarballs can be included with the project.
These tarballs can be extracted directly into a project's `node_modules` directory (bypassing the
package manager) by using:
......
// This script must be run with tsx
import fs from 'node:fs/promises'
import path from 'node:path'
import yargs from 'yargs'
import { hideBin } from 'yargs/helpers'
import { default as patchPackageJson } from './pack-utils/patch-package-json.js'
import patchPackageJson, {
findPackageJsonPath,
} from './pack-utils/patch-package-json.js'
import buildNative from './build-native.js'
import {
......@@ -14,13 +17,14 @@ import {
packageFiles,
} from './pack-util.js'
const TARBALLS = `${NEXT_DIR}/tarballs`
const TARBALLS_DIRNAME = 'tarballs'
const TARBALLS = path.join(NEXT_DIR, TARBALLS_DIRNAME)
const NEXT_PACKAGES = `${NEXT_DIR}/packages`
const NEXT_TARBALL = `${TARBALLS}/next.tar`
const NEXT_SWC_TARBALL = `${TARBALLS}/next-swc.tar`
const NEXT_MDX_TARBALL = `${TARBALLS}/next-mdx.tar`
const NEXT_ENV_TARBALL = `${TARBALLS}/next-env.tar`
const NEXT_BA_TARBALL = `${TARBALLS}/next-bundle-analyzer.tar`
const NEXT_TARBALL = 'next.tar'
const NEXT_SWC_TARBALL = 'next-swc.tar'
const NEXT_MDX_TARBALL = 'next-mdx.tar'
const NEXT_ENV_TARBALL = 'next-env.tar'
const NEXT_BA_TARBALL = 'next-bundle-analyzer.tar'
type CompressOpt = 'none' | 'strip' | 'objcopy-zlib' | 'objcopy-zstd'
......@@ -41,12 +45,18 @@ const cliOptions = yargs(hideBin(process.argv))
type: 'boolean',
describe: 'Create tarballs instead of direct reflinks',
})
.option('deployable-tar', {
type: 'boolean',
describe:
'Create tarballs in the target project directory and patch package.json ' +
'with relative file references. Requires `--project` to be set.',
})
.option('compress', {
describe:
'How compress the binary, useful on platforms where tarballs can ' +
'exceed 2 GiB, which causes ERR_FS_FILE_TOO_LARGE with pnpm. Defaults ' +
'to "strip" on Linux, otherwise defaults to "none". Requires `--tar` ' +
'to be set.',
'or `--deployable-tar` to be set.',
choices: [
'none',
'strip',
......@@ -57,13 +67,28 @@ const cliOptions = yargs(hideBin(process.argv))
})
.check((opts) => {
const compress = opts.compress
if (!opts.tar && (compress ?? 'none') !== 'none') {
throw new Error('--compress is only valid in combination with --tar')
const shouldCreateTarballs = opts.tar || opts.deployableTar
if (opts.tar && opts.deployableTar) {
throw new Error(
'--deployable-tar creates tarballs and cannot be combined with --tar'
)
}
if (opts.deployableTar && opts.project == null) {
throw new Error('--deployable-tar requires --project')
}
if (!shouldCreateTarballs && (compress ?? 'none') !== 'none') {
throw new Error(
'--compress is only valid in combination with --tar or --deployable-tar'
)
}
return true
})
.middleware((opts) => {
if (opts.tar && process.platform === 'linux' && opts.compress == null) {
if (
(opts.tar || opts.deployableTar) &&
process.platform === 'linux' &&
opts.compress == null
) {
opts.compress = 'strip'
}
})
......@@ -78,12 +103,24 @@ interface PackageFiles {
}
async function main(): Promise<void> {
const shouldCreateTarballs = cliOptions.tar || cliOptions.deployableTar
const packageJsonPath = cliOptions.deployableTar
? await findPackageJsonPath(cliOptions.project!)
: null
const packageJsonDir =
packageJsonPath == null ? null : path.dirname(packageJsonPath)
const tarballsDir =
packageJsonDir == null
? TARBALLS
: path.join(packageJsonDir, TARBALLS_DIRNAME)
const tarballFiles = getTarballFiles(tarballsDir)
if (cliOptions.jsBuild) {
exec('Install Next.js build dependencies', 'pnpm i')
exec('Build Next.js', 'pnpm run build')
}
if (cliOptions.tar && cliOptions.compress !== 'strip') {
if (shouldCreateTarballs && cliOptions.compress !== 'strip') {
// HACK: delete any pre-existing binaries to force napi-rs to rewrite it
// We must do this as pre-existing could've been stripped.
let binaries = await nextSwcBinaries()
......@@ -92,24 +129,30 @@ async function main(): Promise<void> {
await buildNative(cliOptions._ as string[])
if (cliOptions.tar) {
await fs.mkdir(TARBALLS, { recursive: true })
if (shouldCreateTarballs) {
await fs.mkdir(tarballsDir, { recursive: true })
// build all tarfiles in parallel
await Promise.all([
packNextSwcWithTar(cliOptions.compress ?? 'none'),
packNextSwcWithTar(
cliOptions.compress ?? 'none',
tarballFiles.nextSwcFile
),
...[
[`${NEXT_PACKAGES}/next`, NEXT_TARBALL],
[`${NEXT_PACKAGES}/next-mdx`, NEXT_MDX_TARBALL],
[`${NEXT_PACKAGES}/next-env`, NEXT_ENV_TARBALL],
[`${NEXT_PACKAGES}/next-bundle-analyzer`, NEXT_BA_TARBALL],
[`${NEXT_PACKAGES}/next`, tarballFiles.nextFile],
[`${NEXT_PACKAGES}/next-mdx`, tarballFiles.nextMdxFile],
[`${NEXT_PACKAGES}/next-env`, tarballFiles.nextEnvFile],
[`${NEXT_PACKAGES}/next-bundle-analyzer`, tarballFiles.nextBaFile],
].map(([packagePath, tarballPath]) =>
packWithTar(packagePath, tarballPath)
),
])
}
const packageFiles = getPackageFiles(cliOptions.tar)
const packageFiles = getPackageFiles(
shouldCreateTarballs ? tarballFiles : null,
cliOptions.deployableTar ? packageJsonDir : null
)
if (cliOptions.project != null) {
const patchedPath = await patchPackageJson(cliOptions.project, {
......@@ -120,6 +163,9 @@ async function main(): Promise<void> {
nextSwcTarball: packageFiles.nextSwcFile,
})
console.log(`Patched ${patchedPath}`)
if (cliOptions.deployableTar) {
console.log(`Wrote tarballs to ${tarballsDir}`)
}
} else {
console.log('Add the following overrides to your workspace package.json:')
console.log(` "pnpm": {`)
......@@ -203,7 +249,10 @@ async function packWithTar(
// We default to stripping (usually faster), but on Linux, we can compress
// instead with objcopy, keeping debug symbols intact. This is controlled by
// `PACK_NEXT_COMPRESS`.
async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
async function packNextSwcWithTar(
compress: CompressOpt,
tarballPath: string
): Promise<void> {
const packagePath = `${NEXT_PACKAGES}/next-swc`
switch (compress) {
case 'strip':
......@@ -212,7 +261,7 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
...(process.platform === 'darwin' ? ['-x', '-'] : ['--']),
...(await nextSwcBinaries()),
])
await packWithTar(packagePath, NEXT_SWC_TARBALL)
await packWithTar(packagePath, tarballPath)
break
case 'objcopy-zstd':
case 'objcopy-zlib':
......@@ -226,10 +275,10 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
)
)
)
await packWithTar(packagePath, NEXT_SWC_TARBALL)
await packWithTar(packagePath, tarballPath)
break
case 'none':
await packWithTar(packagePath, NEXT_SWC_TARBALL)
await packWithTar(packagePath, tarballPath)
break
default:
// should never happen, yargs enforces the `choices` array
......@@ -237,15 +286,26 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
}
}
function getPackageFiles(shouldCreateTarballs?: boolean): PackageFiles {
if (shouldCreateTarballs) {
return {
nextFile: NEXT_TARBALL,
nextMdxFile: NEXT_MDX_TARBALL,
nextEnvFile: NEXT_ENV_TARBALL,
nextBaFile: NEXT_BA_TARBALL,
nextSwcFile: NEXT_SWC_TARBALL,
}
function getTarballFiles(tarballsDir: string): PackageFiles {
return {
nextFile: path.join(tarballsDir, NEXT_TARBALL),
nextMdxFile: path.join(tarballsDir, NEXT_MDX_TARBALL),
nextEnvFile: path.join(tarballsDir, NEXT_ENV_TARBALL),
nextBaFile: path.join(tarballsDir, NEXT_BA_TARBALL),
nextSwcFile: path.join(tarballsDir, NEXT_SWC_TARBALL),
}
}
function getPackageFiles(
tarballFiles: PackageFiles | null,
packageJsonDir: string | null
): PackageFiles {
if (tarballFiles != null) {
return packageJsonDir == null
? tarballFiles
: mapPackageFiles(tarballFiles, (filePath) =>
toProjectRelativePath(packageJsonDir, filePath)
)
}
return {
......@@ -256,3 +316,25 @@ function getPackageFiles(shouldCreateTarballs?: boolean): PackageFiles {
nextSwcFile: `${NEXT_PACKAGES}/next-swc`,
}
}
function mapPackageFiles(
packageFiles: PackageFiles,
mapFile: (filePath: string) => string
): PackageFiles {
return {
nextFile: mapFile(packageFiles.nextFile),
nextMdxFile: mapFile(packageFiles.nextMdxFile),
nextEnvFile: mapFile(packageFiles.nextEnvFile),
nextBaFile: mapFile(packageFiles.nextBaFile),
nextSwcFile: mapFile(packageFiles.nextSwcFile),
}
}
function toProjectRelativePath(fromDir: string, filePath: string): string {
const relativePath = path
.relative(fromDir, filePath)
.split(path.sep)
.join(path.posix.sep)
return relativePath.startsWith('.') ? relativePath : `./${relativePath}`
}
......@@ -36,11 +36,7 @@ export default async function patchPackageJson(
paths: DependencyPaths
): Promise<string> {
try {
const root = await findWorkspaceRoot(targetProjectPath)
const packageJsonPath = root
? path.join(root, 'package.json')
: path.join(targetProjectPath, 'package.json')
const packageJsonPath = await findPackageJsonPath(targetProjectPath)
const packageJsonValue = await readJsonValue(packageJsonPath)
await patchWorkspacePackageJsonMap(paths, packageJsonValue)
await writeJsonValue(packageJsonPath, packageJsonValue)
......@@ -51,6 +47,15 @@ export default async function patchPackageJson(
}
}
export async function findPackageJsonPath(
targetProjectPath: string
): Promise<string> {
const root = await findWorkspaceRoot(targetProjectPath)
return root
? path.join(root, 'package.json')
: path.join(targetProjectPath, 'package.json')
}
async function readJsonValue(filePath: string): Promise<PackageJson> {
try {
const content = await fs.readFile(filePath, 'utf8')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment