Apply publish step optimizations (#43620)

Follow-up to https://github.com/vercel/next.js/pull/32337 this removes
the un-necessary step where we fetch all of the tags which requires
pulling a lot of un-necessary git history inflating cache size and
publish times.

The only reason these tags were needing to be fetched is due to an issue
in how the `actions/checkout` step works
(https://github.com/actions/checkout/issues/882).

This reduces the publish times by at least 4 minutes by removing the
tags fetching step
https://github.com/vercel/next.js/actions/runs/3598569786/jobs/6061449995#step:16:14

As a further optimization this adds concurrency to the `npm publish`
calls themselves to hopefully reduce time spent there as well.
parent 2063ff33
...@@ -84,11 +84,10 @@ jobs: ...@@ -84,11 +84,10 @@ jobs:
- run: pnpm install - run: pnpm install
- run: pnpm run build - run: pnpm run build
- run: node run-tests.js --timings --write-timings -g 1/1 - run: node run-tests.js --timings --write-timings -g 1/1
- run: node ./scripts/fetch-tags.mjs ${{ github.sha }}
- id: check-release - id: check-release
run: | run: |
if [[ $(git describe --exact-match 2> /dev/null || :) = v* ]]; if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
then then
echo "::set-output name=IS_RELEASE::true" echo "::set-output name=IS_RELEASE::true"
else else
...@@ -893,7 +892,7 @@ jobs: ...@@ -893,7 +892,7 @@ jobs:
- run: npm i -g pnpm@${PNPM_VERSION} - run: npm i -g pnpm@${PNPM_VERSION}
- run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc - run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
- run: ./scripts/publish-native.js $GITHUB_REF - run: ./scripts/publish-native.js
- run: ./scripts/publish-release.js - run: ./scripts/publish-release.js
testDeployE2E: testDeployE2E:
......
import { execSync } from 'child_process' const { execSync } = require('child_process')
import execa from 'execa'
;(async () => { const checkIsRelease = async () => {
let commitId = process.argv[2] || '' let commitId = process.argv[2] || ''
// parse only the last string which should be version if // parse only the last string which should be version if
...@@ -14,21 +14,13 @@ import execa from 'execa' ...@@ -14,21 +14,13 @@ import execa from 'execa'
const versionString = commitMsg.split(' ').pop().trim() const versionString = commitMsg.split(' ').pop().trim()
const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/ const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/
console.log({ commitId, commitMsg, versionString })
if (publishMsgRegex.test(versionString)) { if (publishMsgRegex.test(versionString)) {
console.log('publish commit, fetching tags') console.log(versionString)
process.exit(0)
const result = await execa(
'git',
['fetch', '--depth=1', 'origin', '+refs/tags/*:refs/tags/*'],
{
stdio: ['ignore', 'inherit', 'inherit'],
}
)
process.exit(result.exitCode)
} else { } else {
console.log('not publish commit') console.log('not publish commit', { commitId, commitMsg, versionString })
process.exit(1)
} }
})() }
checkIsRelease()
#!/usr/bin/env node #!/usr/bin/env node
const path = require('path') const path = require('path')
const { readFile, readdir, writeFile } = require('fs/promises') const execa = require('execa')
const { copy } = require('fs-extra') const { copy } = require('fs-extra')
const { execSync } = require('child_process') const { Sema } = require('async-sema')
const { readFile, readdir, writeFile } = require('fs/promises')
const cwd = process.cwd() const cwd = process.cwd()
;(async function () { ;(async function () {
try { try {
const publishSema = new Sema(2)
let version = JSON.parse( let version = JSON.parse(
await readFile(path.join(cwd, 'lerna.json')) await readFile(path.join(cwd, 'lerna.json'))
).version ).version
let gitref = process.argv.slice(2)[0]
// Copy binaries to package folders, update version, and publish // Copy binaries to package folders, update version, and publish
let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm') let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
...@@ -20,93 +22,116 @@ const cwd = process.cwd() ...@@ -20,93 +22,116 @@ const cwd = process.cwd()
(name) => !name.startsWith('.') (name) => !name.startsWith('.')
) )
for (let platform of platforms) { await Promise.all(
try { platforms.map(async (platform) => {
let binaryName = `next-swc.${platform}.node` await publishSema.acquire()
await copy(
path.join(cwd, 'packages/next-swc/native', binaryName),
path.join(nativePackagesDir, platform, binaryName)
)
let pkg = JSON.parse(
await readFile(path.join(nativePackagesDir, platform, 'package.json'))
)
pkg.version = version
await writeFile(
path.join(nativePackagesDir, platform, 'package.json'),
JSON.stringify(pkg, null, 2)
)
execSync(
`npm publish ${path.join(
nativePackagesDir,
platform
)} --access public ${
gitref.includes('canary') ? ' --tag canary' : ''
}`
)
} catch (err) {
// don't block publishing other versions on single platform error
console.error(`Failed to publish`, platform)
if ( try {
err.message && let binaryName = `next-swc.${platform}.node`
err.message.includes( await copy(
'You cannot publish over the previously published versions' path.join(cwd, 'packages/next-swc/native', binaryName),
path.join(nativePackagesDir, platform, binaryName)
)
let pkg = JSON.parse(
await readFile(
path.join(nativePackagesDir, platform, 'package.json')
)
) )
) { pkg.version = version
console.error('Ignoring already published error', platform) await writeFile(
} else { path.join(nativePackagesDir, platform, 'package.json'),
// throw err JSON.stringify(pkg, null, 2)
)
await execa(
`npm`,
[
`publish`,
`${path.join(nativePackagesDir, platform)}`,
`--access`,
`public`,
...(version.includes('canary') ? ['--tag', 'canary'] : []),
],
{ stdio: 'inherit' }
)
} catch (err) {
// don't block publishing other versions on single platform error
console.error(`Failed to publish`, platform, err)
if (
err.message &&
err.message.includes(
'You cannot publish over the previously published versions'
)
) {
console.error('Ignoring already published error', platform, err)
} else {
// throw err
}
} finally {
publishSema.release()
} }
} // lerna publish in next step sill fail if git status is not clean
// lerna publish in next step sill fail if git status is not clean await execa(
execSync( `git`,
`git update-index --skip-worktree ${path.join( [
nativePackagesDir, 'update-index',
platform, '--skip-worktree',
'package.json' `${path.join(nativePackagesDir, platform, 'package.json')}`,
)}` ],
) { stdio: 'inherit' }
} )
})
)
// Update name/version of wasm packages and publish // Update name/version of wasm packages and publish
let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm') let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm')
for (let wasmTarget of ['web', 'nodejs']) {
let wasmPkg = JSON.parse(
await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
)
wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
wasmPkg.version = version
await writeFile( await Promise.all(
path.join(wasmDir, `pkg-${wasmTarget}/package.json`), ['web', 'nodejs'].map(async (wasmTarget) => {
JSON.stringify(wasmPkg, null, 2) await publishSema.acquire()
)
try { let wasmPkg = JSON.parse(
execSync( await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
`npm publish ${path.join( )
wasmDir, wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
`pkg-${wasmTarget}` wasmPkg.version = version
)} --access public ${
gitref.includes('canary') ? ' --tag canary' : '' await writeFile(
}` path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
JSON.stringify(wasmPkg, null, 2)
) )
} catch (err) {
// don't block publishing other versions on single platform error
console.error(`Failed to publish`, wasmTarget)
if ( try {
err.message && await execa(
err.message.includes( `npm`,
'You cannot publish over the previously published versions' [
'publish',
`${path.join(wasmDir, `pkg-${wasmTarget}`)}`,
'--access',
'public',
...(version.includes('canary') ? ['--tag', 'canary'] : []),
],
{ stdio: 'inherit' }
) )
) { } catch (err) {
console.error('Ignoring already published error', wasmTarget) // don't block publishing other versions on single platform error
} else { console.error(`Failed to publish`, wasmTarget, err)
// throw err
if (
err.message &&
err.message.includes(
'You cannot publish over the previously published versions'
)
) {
console.error('Ignoring already published error', wasmTarget)
} else {
// throw err
}
} finally {
publishSema.release()
} }
} })
} )
// Update optional dependencies versions // Update optional dependencies versions
let nextPkg = JSON.parse( let nextPkg = JSON.parse(
...@@ -122,7 +147,13 @@ const cwd = process.cwd() ...@@ -122,7 +147,13 @@ const cwd = process.cwd()
JSON.stringify(nextPkg, null, 2) JSON.stringify(nextPkg, null, 2)
) )
// lerna publish in next step will fail if git status is not clean // lerna publish in next step will fail if git status is not clean
execSync('git update-index --skip-worktree packages/next/package.json') await execa(
'git',
['update-index', '--skip-worktree', 'packages/next/package.json'],
{
stdio: 'inherit',
}
)
} catch (err) { } catch (err) {
console.error(err) console.error(err)
process.exit(1) process.exit(1)
......
...@@ -2,22 +2,20 @@ ...@@ -2,22 +2,20 @@
// @ts-check // @ts-check
const path = require('path') const path = require('path')
const { readdir } = require('fs/promises') const execa = require('execa')
const { Sema } = require('async-sema')
const { execSync } = require('child_process') const { execSync } = require('child_process')
const { readJson } = require('fs-extra') const { readJson, readdir } = require('fs-extra')
const cwd = process.cwd() const cwd = process.cwd()
;(async function () { ;(async function () {
let isCanary = true let isCanary = true
if (!process.env.NPM_TOKEN) {
console.log('No NPM_TOKEN, exiting...')
return
}
try { try {
const tagOutput = execSync('git describe --exact-match').toString() const tagOutput = execSync(
`node ${path.join(__dirname, 'check-is-release.js')}`
).toString()
console.log(tagOutput) console.log(tagOutput)
if (tagOutput.trim().startsWith('v')) { if (tagOutput.trim().startsWith('v')) {
...@@ -34,15 +32,28 @@ const cwd = process.cwd() ...@@ -34,15 +32,28 @@ const cwd = process.cwd()
} }
console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`) console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`)
if (!process.env.NPM_TOKEN) {
console.log('No NPM_TOKEN, exiting...')
return
}
const packagesDir = path.join(cwd, 'packages') const packagesDir = path.join(cwd, 'packages')
const packageDirs = await readdir(packagesDir) const packageDirs = await readdir(packagesDir)
const publishSema = new Sema(2)
const publish = async (pkg, retry = 0) => { const publish = async (pkg, retry = 0) => {
try { try {
execSync( await publishSema.acquire()
`npm publish ${path.join(packagesDir, pkg)} --access public${ await execa(
isCanary ? ' --tag canary' : '' `npm`,
}` [
'publish',
`${path.join(packagesDir, pkg)}`,
'--access',
'public',
...(isCanary ? ['--tag', 'canary'] : []),
],
{ stdio: 'inherit' }
) )
} catch (err) { } catch (err) {
console.error(`Failed to publish ${pkg}`, err) console.error(`Failed to publish ${pkg}`, err)
...@@ -66,18 +77,22 @@ const cwd = process.cwd() ...@@ -66,18 +77,22 @@ const cwd = process.cwd()
await publish(pkg, retry + 1) await publish(pkg, retry + 1)
} }
throw err throw err
} finally {
publishSema.release()
} }
} }
for (const packageDir of packageDirs) { await Promise.all(
const pkgJson = await readJson( packageDirs.map(async (packageDir) => {
path.join(packagesDir, packageDir, 'package.json') const pkgJson = await readJson(
) path.join(packagesDir, packageDir, 'package.json')
)
if (pkgJson.private) { if (pkgJson.private) {
console.log(`Skipping private package ${packageDir}`) console.log(`Skipping private package ${packageDir}`)
continue return
} }
await publish(packageDir) await publish(packageDir)
} })
)
})() })()
#!/bin/bash #!/bin/bash
git describe --exact-match if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
then
if [[ ! $? -eq 0 ]];then echo "Publish occurred, running release stats..."
echo "Nothing to publish, exiting.." else
touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt echo "Not publish commit, exiting..."
exit 0; touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
exit 0;
fi fi
if [[ -z "$NPM_TOKEN" ]];then if [[ -z "$NPM_TOKEN" ]];then
...@@ -13,6 +14,5 @@ if [[ -z "$NPM_TOKEN" ]];then ...@@ -13,6 +14,5 @@ if [[ -z "$NPM_TOKEN" ]];then
exit 0; exit 0;
fi fi
echo "Publish occurred, running release stats..."
echo "Waiting 30 seconds to allow publish to finalize" echo "Waiting 30 seconds to allow publish to finalize"
sleep 30 sleep 30
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment