Unverified Commit cdd77ac4 authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Add deployable tarballs to pack-next (#93963)

### What?

Adds a `--deployable-tar` option to `pack-next` that writes generated
tarballs into the target project and patches package references to use
relative `file:` paths.

### Why?

Projects that need to deploy with locally packed Next.js packages should
be able to include those tarballs inside the project directory instead
of referencing tarballs from the Next.js checkout.

### How?

Resolves the target project package.json location, writes tarballs to a
sibling `tarballs/` directory, and maps package override paths to
project-relative references when patching package.json. Existing `--tar`
behavior continues to write to the repository tarballs directory.

### Verification

- `git diff --check`
- `git diff --cached --check`
- `pnpm prettier --with-node-modules --ignore-path .prettierignore
--write scripts/pack-next.ts scripts/pack-utils/patch-package-json.ts
contributing/core/testing.md contributing/core/developing.md`
- `npx eslint --config eslint.config.mjs --fix scripts/pack-next.ts
scripts/pack-utils/patch-package-json.ts contributing/core/testing.md
contributing/core/developing.md` (Markdown files were ignored because no
matching ESLint config was supplied)
- `pnpm pack-next --project ../sandbox/nextjs-duplicated-html-repro/
--deployable-tar`

<!-- NEXT_JS_LLM_PR -->
parent 4e3eb513
...@@ -129,6 +129,12 @@ Or without running the build: ...@@ -129,6 +129,12 @@ Or without running the build:
pnpm pack-next --no-js-build --tar && pnpm unpack-next path/to/project pnpm pack-next --no-js-build --tar && pnpm unpack-next path/to/project
``` ```
To create tarballs inside the project directory with deployable relative `file:` references:
```bash
pnpm pack-next --project path/to/project --deployable-tar
```
Without going through a tarball (only works if you've added the overrides from `pack-next`): Without going through a tarball (only works if you've added the overrides from `pack-next`):
```bash ```bash
......
...@@ -214,6 +214,15 @@ On Linux, this generates stripped `@next/swc` binaries to avoid exceeding 2 GiB, ...@@ -214,6 +214,15 @@ On Linux, this generates stripped `@next/swc` binaries to avoid exceeding 2 GiB,
known to cause problems with `pnpm`](https://github.com/libuv/libuv/pull/1501). That behavior can be known to cause problems with `pnpm`](https://github.com/libuv/libuv/pull/1501). That behavior can be
overridden with `--compress objcopy-zstd` on Linux (which is slower, but retains debuginfo). overridden with `--compress objcopy-zstd` on Linux (which is slower, but retains debuginfo).
To create tarballs that can be deployed with a project, use:
```bash
pnpm pack-next --project ~/my-project/ --deployable-tar
```
This writes the tarballs to a `tarballs` directory next to the patched project `package.json` and
uses relative `file:` references so the tarballs can be included with the project.
These tarballs can be extracted directly into a project's `node_modules` directory (bypassing the These tarballs can be extracted directly into a project's `node_modules` directory (bypassing the
package manager) by using: package manager) by using:
......
// This script must be run with tsx // This script must be run with tsx
import fs from 'node:fs/promises' import fs from 'node:fs/promises'
import path from 'node:path'
import yargs from 'yargs' import yargs from 'yargs'
import { hideBin } from 'yargs/helpers' import { hideBin } from 'yargs/helpers'
import { default as patchPackageJson } from './pack-utils/patch-package-json.js' import patchPackageJson, {
findPackageJsonPath,
} from './pack-utils/patch-package-json.js'
import buildNative from './build-native.js' import buildNative from './build-native.js'
import { import {
...@@ -14,13 +17,14 @@ import { ...@@ -14,13 +17,14 @@ import {
packageFiles, packageFiles,
} from './pack-util.js' } from './pack-util.js'
const TARBALLS = `${NEXT_DIR}/tarballs` const TARBALLS_DIRNAME = 'tarballs'
const TARBALLS = path.join(NEXT_DIR, TARBALLS_DIRNAME)
const NEXT_PACKAGES = `${NEXT_DIR}/packages` const NEXT_PACKAGES = `${NEXT_DIR}/packages`
const NEXT_TARBALL = `${TARBALLS}/next.tar` const NEXT_TARBALL = 'next.tar'
const NEXT_SWC_TARBALL = `${TARBALLS}/next-swc.tar` const NEXT_SWC_TARBALL = 'next-swc.tar'
const NEXT_MDX_TARBALL = `${TARBALLS}/next-mdx.tar` const NEXT_MDX_TARBALL = 'next-mdx.tar'
const NEXT_ENV_TARBALL = `${TARBALLS}/next-env.tar` const NEXT_ENV_TARBALL = 'next-env.tar'
const NEXT_BA_TARBALL = `${TARBALLS}/next-bundle-analyzer.tar` const NEXT_BA_TARBALL = 'next-bundle-analyzer.tar'
type CompressOpt = 'none' | 'strip' | 'objcopy-zlib' | 'objcopy-zstd' type CompressOpt = 'none' | 'strip' | 'objcopy-zlib' | 'objcopy-zstd'
...@@ -41,12 +45,18 @@ const cliOptions = yargs(hideBin(process.argv)) ...@@ -41,12 +45,18 @@ const cliOptions = yargs(hideBin(process.argv))
type: 'boolean', type: 'boolean',
describe: 'Create tarballs instead of direct reflinks', describe: 'Create tarballs instead of direct reflinks',
}) })
.option('deployable-tar', {
type: 'boolean',
describe:
'Create tarballs in the target project directory and patch package.json ' +
'with relative file references. Requires `--project` to be set.',
})
.option('compress', { .option('compress', {
describe: describe:
'How compress the binary, useful on platforms where tarballs can ' + 'How compress the binary, useful on platforms where tarballs can ' +
'exceed 2 GiB, which causes ERR_FS_FILE_TOO_LARGE with pnpm. Defaults ' + 'exceed 2 GiB, which causes ERR_FS_FILE_TOO_LARGE with pnpm. Defaults ' +
'to "strip" on Linux, otherwise defaults to "none". Requires `--tar` ' + 'to "strip" on Linux, otherwise defaults to "none". Requires `--tar` ' +
'to be set.', 'or `--deployable-tar` to be set.',
choices: [ choices: [
'none', 'none',
'strip', 'strip',
...@@ -57,13 +67,28 @@ const cliOptions = yargs(hideBin(process.argv)) ...@@ -57,13 +67,28 @@ const cliOptions = yargs(hideBin(process.argv))
}) })
.check((opts) => { .check((opts) => {
const compress = opts.compress const compress = opts.compress
if (!opts.tar && (compress ?? 'none') !== 'none') { const shouldCreateTarballs = opts.tar || opts.deployableTar
throw new Error('--compress is only valid in combination with --tar') if (opts.tar && opts.deployableTar) {
throw new Error(
'--deployable-tar creates tarballs and cannot be combined with --tar'
)
}
if (opts.deployableTar && opts.project == null) {
throw new Error('--deployable-tar requires --project')
}
if (!shouldCreateTarballs && (compress ?? 'none') !== 'none') {
throw new Error(
'--compress is only valid in combination with --tar or --deployable-tar'
)
} }
return true return true
}) })
.middleware((opts) => { .middleware((opts) => {
if (opts.tar && process.platform === 'linux' && opts.compress == null) { if (
(opts.tar || opts.deployableTar) &&
process.platform === 'linux' &&
opts.compress == null
) {
opts.compress = 'strip' opts.compress = 'strip'
} }
}) })
...@@ -78,12 +103,24 @@ interface PackageFiles { ...@@ -78,12 +103,24 @@ interface PackageFiles {
} }
async function main(): Promise<void> { async function main(): Promise<void> {
const shouldCreateTarballs = cliOptions.tar || cliOptions.deployableTar
const packageJsonPath = cliOptions.deployableTar
? await findPackageJsonPath(cliOptions.project!)
: null
const packageJsonDir =
packageJsonPath == null ? null : path.dirname(packageJsonPath)
const tarballsDir =
packageJsonDir == null
? TARBALLS
: path.join(packageJsonDir, TARBALLS_DIRNAME)
const tarballFiles = getTarballFiles(tarballsDir)
if (cliOptions.jsBuild) { if (cliOptions.jsBuild) {
exec('Install Next.js build dependencies', 'pnpm i') exec('Install Next.js build dependencies', 'pnpm i')
exec('Build Next.js', 'pnpm run build') exec('Build Next.js', 'pnpm run build')
} }
if (cliOptions.tar && cliOptions.compress !== 'strip') { if (shouldCreateTarballs && cliOptions.compress !== 'strip') {
// HACK: delete any pre-existing binaries to force napi-rs to rewrite it // HACK: delete any pre-existing binaries to force napi-rs to rewrite it
// We must do this as pre-existing could've been stripped. // We must do this as pre-existing could've been stripped.
let binaries = await nextSwcBinaries() let binaries = await nextSwcBinaries()
...@@ -92,24 +129,30 @@ async function main(): Promise<void> { ...@@ -92,24 +129,30 @@ async function main(): Promise<void> {
await buildNative(cliOptions._ as string[]) await buildNative(cliOptions._ as string[])
if (cliOptions.tar) { if (shouldCreateTarballs) {
await fs.mkdir(TARBALLS, { recursive: true }) await fs.mkdir(tarballsDir, { recursive: true })
// build all tarfiles in parallel // build all tarfiles in parallel
await Promise.all([ await Promise.all([
packNextSwcWithTar(cliOptions.compress ?? 'none'), packNextSwcWithTar(
cliOptions.compress ?? 'none',
tarballFiles.nextSwcFile
),
...[ ...[
[`${NEXT_PACKAGES}/next`, NEXT_TARBALL], [`${NEXT_PACKAGES}/next`, tarballFiles.nextFile],
[`${NEXT_PACKAGES}/next-mdx`, NEXT_MDX_TARBALL], [`${NEXT_PACKAGES}/next-mdx`, tarballFiles.nextMdxFile],
[`${NEXT_PACKAGES}/next-env`, NEXT_ENV_TARBALL], [`${NEXT_PACKAGES}/next-env`, tarballFiles.nextEnvFile],
[`${NEXT_PACKAGES}/next-bundle-analyzer`, NEXT_BA_TARBALL], [`${NEXT_PACKAGES}/next-bundle-analyzer`, tarballFiles.nextBaFile],
].map(([packagePath, tarballPath]) => ].map(([packagePath, tarballPath]) =>
packWithTar(packagePath, tarballPath) packWithTar(packagePath, tarballPath)
), ),
]) ])
} }
const packageFiles = getPackageFiles(cliOptions.tar) const packageFiles = getPackageFiles(
shouldCreateTarballs ? tarballFiles : null,
cliOptions.deployableTar ? packageJsonDir : null
)
if (cliOptions.project != null) { if (cliOptions.project != null) {
const patchedPath = await patchPackageJson(cliOptions.project, { const patchedPath = await patchPackageJson(cliOptions.project, {
...@@ -120,6 +163,9 @@ async function main(): Promise<void> { ...@@ -120,6 +163,9 @@ async function main(): Promise<void> {
nextSwcTarball: packageFiles.nextSwcFile, nextSwcTarball: packageFiles.nextSwcFile,
}) })
console.log(`Patched ${patchedPath}`) console.log(`Patched ${patchedPath}`)
if (cliOptions.deployableTar) {
console.log(`Wrote tarballs to ${tarballsDir}`)
}
} else { } else {
console.log('Add the following overrides to your workspace package.json:') console.log('Add the following overrides to your workspace package.json:')
console.log(` "pnpm": {`) console.log(` "pnpm": {`)
...@@ -203,7 +249,10 @@ async function packWithTar( ...@@ -203,7 +249,10 @@ async function packWithTar(
// We default to stripping (usually faster), but on Linux, we can compress // We default to stripping (usually faster), but on Linux, we can compress
// instead with objcopy, keeping debug symbols intact. This is controlled by // instead with objcopy, keeping debug symbols intact. This is controlled by
// `PACK_NEXT_COMPRESS`. // `PACK_NEXT_COMPRESS`.
async function packNextSwcWithTar(compress: CompressOpt): Promise<void> { async function packNextSwcWithTar(
compress: CompressOpt,
tarballPath: string
): Promise<void> {
const packagePath = `${NEXT_PACKAGES}/next-swc` const packagePath = `${NEXT_PACKAGES}/next-swc`
switch (compress) { switch (compress) {
case 'strip': case 'strip':
...@@ -212,7 +261,7 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> { ...@@ -212,7 +261,7 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
...(process.platform === 'darwin' ? ['-x', '-'] : ['--']), ...(process.platform === 'darwin' ? ['-x', '-'] : ['--']),
...(await nextSwcBinaries()), ...(await nextSwcBinaries()),
]) ])
await packWithTar(packagePath, NEXT_SWC_TARBALL) await packWithTar(packagePath, tarballPath)
break break
case 'objcopy-zstd': case 'objcopy-zstd':
case 'objcopy-zlib': case 'objcopy-zlib':
...@@ -226,10 +275,10 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> { ...@@ -226,10 +275,10 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
) )
) )
) )
await packWithTar(packagePath, NEXT_SWC_TARBALL) await packWithTar(packagePath, tarballPath)
break break
case 'none': case 'none':
await packWithTar(packagePath, NEXT_SWC_TARBALL) await packWithTar(packagePath, tarballPath)
break break
default: default:
// should never happen, yargs enforces the `choices` array // should never happen, yargs enforces the `choices` array
...@@ -237,15 +286,26 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> { ...@@ -237,15 +286,26 @@ async function packNextSwcWithTar(compress: CompressOpt): Promise<void> {
} }
} }
function getPackageFiles(shouldCreateTarballs?: boolean): PackageFiles { function getTarballFiles(tarballsDir: string): PackageFiles {
if (shouldCreateTarballs) { return {
return { nextFile: path.join(tarballsDir, NEXT_TARBALL),
nextFile: NEXT_TARBALL, nextMdxFile: path.join(tarballsDir, NEXT_MDX_TARBALL),
nextMdxFile: NEXT_MDX_TARBALL, nextEnvFile: path.join(tarballsDir, NEXT_ENV_TARBALL),
nextEnvFile: NEXT_ENV_TARBALL, nextBaFile: path.join(tarballsDir, NEXT_BA_TARBALL),
nextBaFile: NEXT_BA_TARBALL, nextSwcFile: path.join(tarballsDir, NEXT_SWC_TARBALL),
nextSwcFile: NEXT_SWC_TARBALL, }
} }
function getPackageFiles(
tarballFiles: PackageFiles | null,
packageJsonDir: string | null
): PackageFiles {
if (tarballFiles != null) {
return packageJsonDir == null
? tarballFiles
: mapPackageFiles(tarballFiles, (filePath) =>
toProjectRelativePath(packageJsonDir, filePath)
)
} }
return { return {
...@@ -256,3 +316,25 @@ function getPackageFiles(shouldCreateTarballs?: boolean): PackageFiles { ...@@ -256,3 +316,25 @@ function getPackageFiles(shouldCreateTarballs?: boolean): PackageFiles {
nextSwcFile: `${NEXT_PACKAGES}/next-swc`, nextSwcFile: `${NEXT_PACKAGES}/next-swc`,
} }
} }
function mapPackageFiles(
packageFiles: PackageFiles,
mapFile: (filePath: string) => string
): PackageFiles {
return {
nextFile: mapFile(packageFiles.nextFile),
nextMdxFile: mapFile(packageFiles.nextMdxFile),
nextEnvFile: mapFile(packageFiles.nextEnvFile),
nextBaFile: mapFile(packageFiles.nextBaFile),
nextSwcFile: mapFile(packageFiles.nextSwcFile),
}
}
function toProjectRelativePath(fromDir: string, filePath: string): string {
const relativePath = path
.relative(fromDir, filePath)
.split(path.sep)
.join(path.posix.sep)
return relativePath.startsWith('.') ? relativePath : `./${relativePath}`
}
...@@ -36,11 +36,7 @@ export default async function patchPackageJson( ...@@ -36,11 +36,7 @@ export default async function patchPackageJson(
paths: DependencyPaths paths: DependencyPaths
): Promise<string> { ): Promise<string> {
try { try {
const root = await findWorkspaceRoot(targetProjectPath) const packageJsonPath = await findPackageJsonPath(targetProjectPath)
const packageJsonPath = root
? path.join(root, 'package.json')
: path.join(targetProjectPath, 'package.json')
const packageJsonValue = await readJsonValue(packageJsonPath) const packageJsonValue = await readJsonValue(packageJsonPath)
await patchWorkspacePackageJsonMap(paths, packageJsonValue) await patchWorkspacePackageJsonMap(paths, packageJsonValue)
await writeJsonValue(packageJsonPath, packageJsonValue) await writeJsonValue(packageJsonPath, packageJsonValue)
...@@ -51,6 +47,15 @@ export default async function patchPackageJson( ...@@ -51,6 +47,15 @@ export default async function patchPackageJson(
} }
} }
export async function findPackageJsonPath(
targetProjectPath: string
): Promise<string> {
const root = await findWorkspaceRoot(targetProjectPath)
return root
? path.join(root, 'package.json')
: path.join(targetProjectPath, 'package.json')
}
async function readJsonValue(filePath: string): Promise<PackageJson> { async function readJsonValue(filePath: string): Promise<PackageJson> {
try { try {
const content = await fs.readFile(filePath, 'utf8') const content = await fs.readFile(filePath, 'utf8')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment