Unverified Commit f993ea3d authored by Alexander Lyon's avatar Alexander Lyon Committed by GitHub

add next-build-test binary / library (#64121)

This adds a new target in the next-swc package designed as an entrypoint
for collecting memory benches. Later PRs in the stack add a script to
run the benches and finally a manual-triggered CI step
parent dfe7fc03
This diff is collapsed.
......@@ -6,6 +6,7 @@ members = [
"packages/next-swc/crates/napi",
"packages/next-swc/crates/wasm",
"packages/next-swc/crates/next-api",
"packages/next-swc/crates/next-build-test",
"packages/next-swc/crates/next-build",
"packages/next-swc/crates/next-core",
"packages/next-swc/crates/next-custom-transforms",
......@@ -22,6 +23,11 @@ debug-assertions = false
[profile.dev.build-override]
opt-level = 3
[profile.release-with-debug]
inherits = "release"
debug = true
lto = "thin"
[workspace.dependencies]
# Workspace crates
next-api = { path = "packages/next-swc/crates/next-api" }
......@@ -42,6 +48,7 @@ turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbop
turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240521.2" }
# [TODO]: need to refactor embed_directory! macro usage in next-core
turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240521.2" }
turbo-tasks-malloc = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240521.2" }
# General Deps
......
[package]
name = "next-build-test"
version = "0.1.0"
description = "TBD"
license = "MPL-2.0"
edition = "2021"
autobenches = false
[lints]
workspace = true
[dependencies]
next-core = { workspace = true, features = ["rustls-tls"] }
next-api = { workspace = true }
serde_json = { workspace = true }
anyhow = { workspace = true }
tokio = { workspace = true, features = ["full"] }
turbo-tasks-malloc = { workspace = true, default-features = false }
turbopack-binding = { workspace = true, features = [
"__turbo_tasks",
"__turbo_tasks_memory",
"__turbo_tasks_env",
"__turbo_tasks_fs",
"__turbo_tasks_memory",
"__turbopack",
"__turbopack_nodejs",
"__turbopack_core",
"__turbopack_browser",
"__turbopack_ecmascript",
"__turbopack_ecmascript_runtime",
"__turbopack_env",
"__turbopack_node",
] }
turbo-tasks = { workspace = true }
rand = { workspace = true, features = ["small_rng"] }
num_cpus = "1.16.0"
tokio-stream = "0.1.15"
tracing = "0.1"
tracing-subscriber = "0.3"
futures-util = "0.3.30"
[build-dependencies]
turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
[dev-dependencies]
iai-callgrind = "0.10.2"
tempdir = "0.3.7"
# next-build-test
This binary lets you sidestep all of the node bundling and run a turbo build
against a raw rust binary. It does _not_ do everything nextjs does, but it
is an ok approximation.
## Getting started
You will need a project_options file that points to some nextjs repo that has
its dependencies installed. The easiest way to do that is to run a nextjs
build using a modified binary that produces one out for you or to run the
`generate` command and tweak it manually. We cannot bundle one in the repo,
since it needs fs-specific paths and env vars.
You can run the binary with the `generate` flag to build one for you.
```sh
cargo run -- generate /path/to/project > project_options.json
cargo run -- run
```
## Flags
The `run` command supports 4 flags:
- `strategy` can be one of sequential, concurrent, or parallel. defines how
work is structured
- `factor` defined how many pages should be built at once. defaults to num_cpus
- `limit` defines the highest number of pages to build. the pages are
shuffled deterministically. defaults to 1 page
- `pages` a comma separated list of routes to run. queues that precise set in
the order specified
fn main() {
turbopack_binding::turbo::tasks_build::generate_register();
}
{
"compilerOptions": {
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedIndexedAccess": false,
"skipLibCheck": true,
"strict": true,
"lib": ["lib.dom.d.ts", "lib.dom.iterable.d.ts", "lib.esnext.d.ts"],
"module": 99,
"target": 8,
"moduleResolution": 2,
"incremental": true,
"noEmit": true,
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": 1,
"plugins": [
{
"name": "typescript-plugin-css-modules",
"options": {
"goToDefinition": true
}
},
{
"name": "next"
}
],
"allowJs": true,
"paths": {},
"tsBuildInfoFile": "/tmp/ignore",
"strictNullChecks": true,
"pathsBasePath": "/tmp/ignore"
}
}
{
"env": {},
"webpack": {},
"eslint": {
"ignoreDuringBuilds": false
},
"typescript": {
"ignoreBuildErrors": false,
"tsconfigPath": "tsconfig.json"
},
"distDir": ".next",
"cleanDistDir": true,
"assetPrefix": "",
"cacheMaxMemorySize": 52428800,
"configOrigin": "next.config.mjs",
"useFileSystemPublicRoutes": true,
"generateBuildId": null,
"generateEtags": true,
"pageExtensions": ["jsx", "js", "tsx", "ts", "mdx", "md"],
"poweredByHeader": true,
"compress": true,
"analyticsId": "",
"images": {
"deviceSizes": [640, 750, 828, 1080, 1200, 1920, 2048, 3840],
"imageSizes": [16, 32, 48, 64, 96, 128, 256, 384],
"path": "/_next/image",
"loader": "default",
"loaderFile": "",
"domains": [],
"disableStaticImages": false,
"minimumCacheTTL": 60,
"formats": ["image/avif", "image/webp"],
"dangerouslyAllowSVG": false,
"contentSecurityPolicy": "script-src 'none'; frame-src 'none'; sandbox;",
"contentDispositionType": "inline",
"remotePatterns": [],
"unoptimized": false
},
"devIndicators": {
"buildActivity": true,
"buildActivityPosition": "bottom-right"
},
"onDemandEntries": {
"maxInactiveAge": 60000,
"pagesBufferLength": 5
},
"amp": {
"canonicalBase": ""
},
"basePath": "",
"sassOptions": {},
"trailingSlash": false,
"i18n": null,
"productionBrowserSourceMaps": false,
"optimizeFonts": true,
"excludeDefaultMomentLocales": true,
"serverRuntimeConfig": {},
"publicRuntimeConfig": {},
"reactProductionProfiling": false,
"reactStrictMode": true,
"httpAgentOptions": {
"keepAlive": true
},
"outputFileTracing": true,
"staticPageGenerationTimeout": 60,
"swcMinify": true,
"modularizeImports": {},
"experimental": {
"prerenderEarlyExit": false,
"serverMinification": true,
"serverSourceMaps": false,
"linkNoTouchStart": false,
"caseSensitiveRoutes": false,
"clientRouterFilter": true,
"clientRouterFilterRedirects": false,
"fetchCacheKeyPrefix": "",
"middlewarePrefetch": "flexible",
"optimisticClientCache": true,
"manualClientBasePath": false,
"cpus": 2,
"memoryBasedWorkersCount": false,
"isrFlushToDisk": true,
"workerThreads": false,
"optimizeCss": false,
"nextScriptWorkers": false,
"scrollRestoration": false,
"externalDir": false,
"disableOptimizedLoading": false,
"gzipSize": true,
"craCompat": false,
"esmExternals": true,
"fullySpecified": false,
"outputFileTracingRoot": "/tmp/ignore",
"swcTraceProfiling": false,
"forceSwcTransforms": false,
"largePageDataBytes": 128000,
"adjustFontFallbacks": false,
"adjustFontFallbacksWithSizeAdjust": false,
"turbo": {
"rules": {
"*.mdx": {
"loaders": ["turbopack-mdx-loader"],
"as": "*.tsx"
}
},
"resolveAlias": {
"fs": {
"browser": "./turbopack/empty.js"
},
"cookie": {
"browser": "./turbopack/empty.js"
},
"http": {
"browser": "./turbopack/empty.js"
},
"https": {
"browser": "./turbopack/empty.js"
},
"node-fetch": {
"browser": "./turbopack/empty.js"
}
}
},
"typedRoutes": false,
"instrumentationHook": true,
"bundlePagesExternals": false,
"parallelServerCompiles": false,
"parallelServerBuildTraces": false,
"ppr": false,
"missingSuspenseWithCSRBailout": true,
"optimizeServerReact": true,
"useEarlyImport": false,
"serverComponentsExternalPackages": [],
"useLightningcss": true,
"optimizePackageImports": []
},
"configFile": "/tmp/ignore",
"configFileName": "next.config.mjs",
"transpilePackages": [],
"_originalRewrites": {
"beforeFiles": [],
"afterFiles": [],
"fallback": []
},
"_originalRedirects": [],
"exportPathMap": {}
}
#![feature(future_join)]
#![feature(min_specialization)]
#![feature(arbitrary_self_types)]
use std::str::FromStr;
use anyhow::{Context, Result};
use futures_util::{StreamExt, TryStreamExt};
use next_api::{
project::{ProjectContainer, ProjectOptions},
route::{Endpoint, Route},
};
pub async fn main_inner(
strat: Strategy,
factor: usize,
limit: usize,
files: Option<Vec<String>>,
) -> Result<()> {
register();
let mut file = std::fs::File::open("project_options.json").with_context(|| {
let path = std::env::current_dir()
.unwrap()
.join("project_options.json");
format!("loading file at {}", path.display())
})?;
let options: ProjectOptions = serde_json::from_reader(&mut file).unwrap();
let project = ProjectContainer::new(options);
tracing::info!("collecting endpoints");
let entrypoints = project.entrypoints().await?;
let routes = if let Some(files) = files {
tracing::info!("builing only the files:");
for file in &files {
tracing::info!(" {}", file);
}
// filter out the files that are not in the list
// we expect this to be small so linear search OK
Box::new(
entrypoints
.routes
.clone()
.into_iter()
.filter(move |(name, _)| files.iter().any(|f| f == name)),
) as Box<dyn Iterator<Item = _> + Send + Sync>
} else {
Box::new(shuffle(entrypoints.routes.clone().into_iter()))
};
let count = render_routes(routes, strat, factor, limit).await;
tracing::info!("rendered {} pages", count);
Ok(())
}
pub fn register() {
next_api::register();
include!(concat!(env!("OUT_DIR"), "/register.rs"));
}
#[derive(PartialEq, Copy, Clone)]
pub enum Strategy {
Sequential,
Concurrent,
Parallel,
}
impl std::fmt::Display for Strategy {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Strategy::Sequential => write!(f, "sequential"),
Strategy::Concurrent => write!(f, "concurrent"),
Strategy::Parallel => write!(f, "parallel"),
}
}
}
impl FromStr for Strategy {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
match s {
"sequential" => Ok(Strategy::Sequential),
"concurrent" => Ok(Strategy::Concurrent),
"parallel" => Ok(Strategy::Parallel),
_ => Err(anyhow::anyhow!("invalid strategy")),
}
}
}
pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
use rand::{seq::SliceRandom, SeedableRng};
let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
let mut input = items.collect::<Vec<_>>();
input.shuffle(&mut rng);
input.into_iter()
}
pub async fn render_routes(
routes: impl Iterator<Item = (String, Route)>,
strategy: Strategy,
factor: usize,
limit: usize,
) -> usize {
tracing::info!(
"rendering routes with {} parallel and strat {}",
factor,
strategy
);
let stream = tokio_stream::iter(routes)
.map(move |(name, route)| {
let fut = async move {
tracing::info!("{name}");
match route {
Route::Page {
html_endpoint,
data_endpoint: _,
} => {
html_endpoint.write_to_disk().await?;
}
Route::PageApi { endpoint } => {
endpoint.write_to_disk().await?;
}
Route::AppPage(routes) => {
for route in routes {
route.html_endpoint.write_to_disk().await?;
}
}
Route::AppRoute {
original_name: _,
endpoint,
} => {
endpoint.write_to_disk().await?;
}
Route::Conflict => {
tracing::info!("WARN: conflict {}", name);
}
}
Ok::<_, anyhow::Error>(())
};
async move {
match strategy {
Strategy::Parallel => tokio::task::spawn(fut).await.unwrap(),
_ => fut.await,
}
}
})
.take(limit)
.buffer_unordered(factor)
.try_collect::<Vec<_>>()
.await
.unwrap();
stream.len()
}
use std::{convert::Infallible, str::FromStr};
use next_api::project::{DefineEnv, ProjectOptions};
use next_build_test::{main_inner, Strategy};
use turbo_tasks::TurboTasks;
use turbo_tasks_malloc::TurboMalloc;
use turbopack_binding::turbo::tasks_memory::MemoryBackend;
enum Cmd {
Run,
Generate,
}
impl FromStr for Cmd {
type Err = Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"run" => Ok(Cmd::Run),
"generate" => Ok(Cmd::Generate),
_ => panic!("invalid command, please use 'run' or 'generate'"),
}
}
}
fn main() {
tracing_subscriber::fmt::init();
let cmd = std::env::args()
.nth(1)
.map(|s| Cmd::from_str(&s))
.unwrap_or(Ok(Cmd::Run))
.unwrap();
match cmd {
Cmd::Run => {
let strat = std::env::args()
.nth(2)
.map(|s| Strategy::from_str(&s))
.transpose()
.unwrap()
.unwrap_or(Strategy::Sequential);
let mut factor = std::env::args()
.nth(3)
.map(|s| s.parse().unwrap())
.unwrap_or(num_cpus::get());
let limit = std::env::args()
.nth(4)
.map(|s| s.parse().unwrap())
.unwrap_or(1);
let files = std::env::args()
.nth(5)
.map(|f| f.split(',').map(ToOwned::to_owned).collect());
if strat == Strategy::Sequential {
factor = 1;
}
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.on_thread_stop(|| {
TurboMalloc::thread_stop();
tracing::debug!("threads stopped");
})
.build()
.unwrap()
.block_on(async {
let tt = TurboTasks::new(MemoryBackend::new(usize::MAX));
let x = tt.run_once(main_inner(strat, factor, limit, files)).await;
tracing::debug!("done");
x
})
.unwrap();
}
Cmd::Generate => {
let project_path = std::env::args().nth(2).unwrap_or(".".to_string());
let current_dir = std::env::current_dir().unwrap();
let absolute_dir = current_dir.join(project_path);
let canonical_path = std::fs::canonicalize(absolute_dir).unwrap();
let options = ProjectOptions {
build_id: "test".to_owned(),
define_env: DefineEnv {
client: vec![],
edge: vec![],
nodejs: vec![],
},
dev: true,
encryption_key: "deadbeef".to_string(),
env: vec![],
js_config: include_str!("../jsConfig.json").to_string(),
next_config: include_str!("../nextConfig.json").to_string(),
preview_props: next_api::project::DraftModeOptions {
preview_mode_encryption_key: "deadbeef".to_string(),
preview_mode_id: "test".to_string(),
preview_mode_signing_key: "deadbeef".to_string(),
},
project_path: canonical_path.to_string_lossy().to_string(),
root_path: "/".to_string(),
watch: false,
};
let json = serde_json::to_string_pretty(&options).unwrap();
println!("{}", json);
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment