Unverified Commit a9323da2 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Stop pinning compiled chunk source on `EcmascriptBuildNodeChunkVersion`.

 (#93807)

## What

Stop pinning compiled chunk source on `EcmascriptBuildNodeChunkVersion`.

The struct previously held a `Vec<ReadRef<CodeAndIds>>` for every module
in the chunk, even though the HMR update path only needs the bytes for
the *changed* subset and the bytes are already on disk. That field was
also forcing the upstream `CodeAndIds` / `BatchGroupCodeAndIds` tasks to
stay `serialization = "skip"`, so warm restarts had to re-walk every
module and re-hash its source to rebuild `entries_hashes`.

This change mirrors the browser-side pattern: a new
`EcmascriptBuildNodeChunkContentEntries` task lives on the chunk content
and holds `ResolvedVc<Code>` + `ResolvedVc<u64>` per module. The version
struct shrinks to `{ chunk_path, minify_type, entries_hashes }`, drops
`serialization = "skip"`, and switches `chunk_path` from `String` to
`RcStr`.
`update_ecmascript_node_chunk_content` now resolves entries lazily, only
when an added or modified module actually needs its code shipped.

## Why

Two wins, both for dev sessions starting against a warm filesystem
cache:

- **Memory.** The version no longer transitively pins every module's
compiled `Rope` in heap — those bytes can stay on disk until HMR
actually needs them.
- **Warm-restart CPU.** `entries_hashes` is sourced from the per-module
`Code::source_code_hash()` task (already cached) and the version itself
now round-trips through the persistent cache, so we don't re-hash
anything on warm start.

The HMR payload shape is unchanged.

## Perf
This should speed up warm builds a bit but the major benefit is not
recomputing node outputs and keeping them in ram

measuring v0 after loading the main route
Branch: Cold: 12.3G Warm: 7.34G
Canary: Cold 12.3G Warm: 8.5G

The trace file confirms the recomputations are gone and the heap
measurements confirm we trimming ~1.1g of ram

Using the devlow benchmarks i was able to confirm a possible small
progression

```
# canary
chat dev startup build=warm: root page = 23.96 s (from root page/start)
chat dev startup build=warm: root page = 21.21 s (from root page/start)
chat dev startup build=warm: root page = 22.70 s (from root page/start)

# branch
chat dev startup build=warm: root page = 20.94 s (from root page/start)
chat dev startup build=warm: root page = 19.43 s (from root page/start)
chat dev startup build=warm: root page = 23.49 s (from root page/start)

```


## Tests

I added a new integration test to ensure we don't accidentally regress
here. Which confirms that 'clean warm builds' run nothing and 'clean
warm dev sessions' just set up HMR session infra
<!-- NEXT_JS_LLM_PR -->
parent f7dee1b8
...@@ -10606,6 +10606,7 @@ version = "0.1.0" ...@@ -10606,6 +10606,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bincode 2.0.1", "bincode 2.0.1",
"either",
"indoc", "indoc",
"serde", "serde",
"serde_json", "serde_json",
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
.vscode .vscode
e2e/**/tsconfig.json e2e/**/tsconfig.json
# See PACK-6932: we need to submit this so that builds are more consistent
!e2e/filesystem-cache/tsconfig.json
production/**/tsconfig.json production/**/tsconfig.json
development/**/tsconfig.json development/**/tsconfig.json
......
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": false,
"noEmit": true,
"incremental": true,
"module": "esnext",
"esModuleInterop": true,
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "react-jsx",
"plugins": [
{
"name": "next"
}
],
"strictNullChecks": true
},
"include": [
"next-env.d.ts",
".next/types/**/*.ts",
".next/dev/types/**/*.ts",
"**/*.mts",
"**/*.ts",
"**/*.tsx"
],
"exclude": ["node_modules", "**/*.test.ts", "**/*.test.tsx"]
}
import { nextTestSetup, isNextDev } from 'e2e-utils'
import { waitFor } from 'next-test-utils'
import fs from 'fs/promises'
import path from 'path'
// Regression guard for warm-cache work: after one cold cycle, run a warm
// cycle and snapshot the set of turbo-tasks functions that had any cache
// miss. If a task previously cached (e.g. via a persistable entries map)
// regresses to recomputing on warm restart, this snapshot will change and
// fail the test.
//
// When the snapshot moves:
// - FEWER entries → that's a win. Run with `-u` and commit the new snapshot.
// - MORE entries → something is recomputing on warm start. Investigate
// before updating.
//
// Dev and start (build) snapshots are kept separate because the warm graphs
// differ: dev brings up HMR/Fast Refresh infra that build doesn't.
interface TaskFunctionStatistics {
cache_hit: number
cache_miss: number
}
type TaskStats = Record<string, TaskFunctionStatistics>
const STATS_RELATIVE_PATH = '.next/warm-restart-task-stats.json'
;(process.env.IS_TURBOPACK_TEST ? describe : describe.skip)(
'warm-restart task statistics',
() => {
const env = [
'ENABLE_CACHING=1',
'TURBO_ENGINE_IGNORE_DIRTY=1',
'TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS=1000',
`NEXT_TURBOPACK_TASK_STATISTICS=${STATS_RELATIVE_PATH}`,
// The task-statistics file is written by an `on_exit` handler in the
// napi binding. In dev that handler only runs if the child process
// gets a chance to clean up (i.e. SIGTERM, not SIGKILL). The parent
// `next dev` process gives the child 100ms by default before
// escalating to SIGKILL — bump that so the on-exit handler can flush.
'NEXT_EXIT_TIMEOUT_MS=30000',
].join(' ')
const { skipped, next } = nextTestSetup({
files: __dirname,
skipDeployment: true,
packageJson: {
packageManager: 'npm@10.9.2',
scripts: {
build: `${env} next build`,
dev: `${env} next dev`,
start: 'next start',
},
},
installCommand: 'npm i',
buildCommand: 'npm run build',
startCommand: isNextDev ? 'npm run dev' : 'npm run start',
})
if (skipped) {
return
}
beforeAll(() => {
// No file edits in this test — skip HMR debounce.
;(next as any).handleDevWatchDelayBeforeChange = () => {}
;(next as any).handleDevWatchDelayAfterChange = () => {}
})
async function stop() {
if (isNextDev) {
// Persistent cache snapshot is on a 1s idle timer; give it room.
await waitFor(3000)
// SIGTERM (not the harness default SIGKILL) so the dev server gets
// to run its cleanup, which is what flushes the task-stats file.
await next.stop('SIGTERM')
} else {
await next.stop()
}
}
async function readMissedTaskNames(): Promise<string[]> {
const absPath = path.join(next.testDir, STATS_RELATIVE_PATH)
const raw = await fs.readFile(absPath, 'utf8')
const stats = JSON.parse(raw) as TaskStats
const misses: string[] = []
for (const [name, s] of Object.entries(stats)) {
if (s.cache_miss > 0) misses.push(name)
}
// Turbopack sorts on the Rust side, but sort again here to be robust
// against the JSON parser's object iteration order.
misses.sort()
return misses
}
if (isNextDev) {
it('snapshot of tasks with cache misses on warm dev restart', async () => {
// Cycle 1: the harness already auto-started; perform a request so the
// SSR/Node chunk work runs and gets persisted on shutdown.
async function runDevCycle() {
const browser = await next.browser('/')
// Wait for actual rendered content before declaring "ready".
await browser.elementByCss('p')
// Settle: the dev server keeps doing background work after first
// paint (HMR socket handshake, tail compilation). A fixed sleep
// is the least-bad option here since next.js doesn't expose
// turbopack's internal idle signal. If this proves flaky in CI,
// raise the value.
await waitFor(2000)
await browser.close()
}
await runDevCycle()
await stop()
// Cycle 2: warm.
await next.start()
await runDevCycle()
await stop()
const missed = await readMissedTaskNames()
expect(missed).toMatchInlineSnapshot(`
[
"<dyn turbopack_core::version::VersionedContent>::update",
"<turbopack_browser::ecmascript::list::content::EcmascriptDevChunkListContent as dyn turbopack_core::version::VersionedContent>::update",
"<turbopack_nodejs::ecmascript::node::content::EcmascriptBuildNodeChunkContent as dyn turbopack_core::version::VersionedContent>::update",
"next_api::project::Project::hmr_update",
"next_api::project::Project::hmr_version_state",
"next_napi_bindings::next_api::project::hmr_update_with_issues_operation",
"next_napi_bindings::next_api::project::project_hmr_update_operation",
"turbopack_core::version::VersionState::get",
]
`)
}, 180_000)
} else {
it('snapshot of tasks with cache misses on warm build', async () => {
// In start mode the harness auto-runs `next build` then `next start`.
// The build is cycle 1. We stop the server (we don't care about it)
// and run a second build manually for the warm measurement.
await stop()
const result = await (next as any).build()
if (result.exitCode !== 0) {
throw new Error(`next build exited with ${result.exitCode}`)
}
const missed = await readMissedTaskNames()
expect(missed).toMatchInlineSnapshot(`[]`)
}, 240_000)
}
}
)
...@@ -71,9 +71,19 @@ impl Serialize for TaskStatistics { ...@@ -71,9 +71,19 @@ impl Serialize for TaskStatistics {
where where
S: Serializer, S: Serializer,
{ {
let mut map = serializer.serialize_map(Some(self.inner.len()))?; // Sort by `global_name` so the emitted JSON is deterministic — the
for entry in &self.inner { // underlying `FxDashMap` has unspecified iteration order. The map is
map.serialize_entry(entry.key().ty.global_name, entry.value())?; // small (~1500 entries in practice), so the sort cost is negligible
// and not worth optimizing.
let mut entries: Vec<_> = self
.inner
.iter()
.map(|e| (e.key().ty.global_name, e.value().clone()))
.collect();
entries.sort_unstable_by_key(|(name, _)| *name);
let mut map = serializer.serialize_map(Some(entries.len()))?;
for (name, stats) in &entries {
map.serialize_entry(name, stats)?;
} }
map.end() map.end()
} }
......
...@@ -13,11 +13,15 @@ use turbopack_core::{ ...@@ -13,11 +13,15 @@ use turbopack_core::{
source_map::{GenerateSourceMap, SourceMapAsset}, source_map::{GenerateSourceMap, SourceMapAsset},
version::{MergeableVersionedContent, Version, VersionedContent, VersionedContentMerger}, version::{MergeableVersionedContent, Version, VersionedContent, VersionedContentMerger},
}; };
use turbopack_ecmascript::{chunk::EcmascriptChunkContent, minify::minify, utils::StringifyJs}; use turbopack_ecmascript::{
chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries},
minify::minify,
utils::StringifyJs,
};
use super::{ use super::{
chunk::EcmascriptBrowserChunk, content_entry::EcmascriptBrowserChunkContentEntries, chunk::EcmascriptBrowserChunk, merged::merger::EcmascriptBrowserChunkContentMerger,
merged::merger::EcmascriptBrowserChunkContentMerger, version::EcmascriptBrowserChunkVersion, version::EcmascriptBrowserChunkVersion,
}; };
use crate::{ use crate::{
BrowserChunkingContext, BrowserChunkingContext,
...@@ -51,8 +55,8 @@ impl EcmascriptBrowserChunkContent { ...@@ -51,8 +55,8 @@ impl EcmascriptBrowserChunkContent {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn entries(&self) -> Vc<EcmascriptBrowserChunkContentEntries> { pub fn entries(&self) -> Vc<EcmascriptChunkContentEntries> {
EcmascriptBrowserChunkContentEntries::new(*self.content) EcmascriptChunkContentEntries::new(*self.content)
} }
} }
......
...@@ -196,7 +196,10 @@ impl VersionedContent for EcmascriptDevChunkListContent { ...@@ -196,7 +196,10 @@ impl VersionedContent for EcmascriptDevChunkListContent {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn update(self: Vc<Self>, from_version: Vc<Box<dyn Version>>) -> Vc<Update> { async fn update(
update_chunk_list(self, from_version) self: ResolvedVc<Self>,
from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> {
update_chunk_list(self, from_version).await
} }
} }
...@@ -47,9 +47,8 @@ impl ChunkListUpdate<'_> { ...@@ -47,9 +47,8 @@ impl ChunkListUpdate<'_> {
} }
/// Computes the update of a chunk list from one version to another. /// Computes the update of a chunk list from one version to another.
#[turbo_tasks::function]
pub(super) async fn update_chunk_list( pub(super) async fn update_chunk_list(
content: Vc<EcmascriptDevChunkListContent>, content: ResolvedVc<EcmascriptDevChunkListContent>,
from_version: ResolvedVc<Box<dyn Version>>, from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> { ) -> Result<Vc<Update>> {
let to_version = content.version(); let to_version = content.version();
......
pub(crate) mod chunk; pub(crate) mod chunk;
pub(crate) mod content; pub(crate) mod content;
pub(crate) mod content_entry;
pub(crate) mod evaluate; pub(crate) mod evaluate;
pub(crate) mod list; pub(crate) mod list;
pub(crate) mod merged; pub(crate) mod merged;
......
use anyhow::Result; use anyhow::Result;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, Vc, turbobail}; use turbo_tasks::{FxIndexMap, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64}; use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::{chunk::ModuleId, version::Version}; use turbopack_core::{chunk::ModuleId, version::Version};
use turbopack_ecmascript::chunk::EcmascriptChunkContent; use turbopack_ecmascript::chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries};
use super::content_entry::EcmascriptBrowserChunkContentEntries;
#[turbo_tasks::value(serialization = "skip")] #[turbo_tasks::value(serialization = "skip")]
pub(super) struct EcmascriptBrowserChunkVersion { pub(super) struct EcmascriptBrowserChunkVersion {
...@@ -29,12 +27,14 @@ impl EcmascriptBrowserChunkVersion { ...@@ -29,12 +27,14 @@ impl EcmascriptBrowserChunkVersion {
} else { } else {
turbobail!("chunk path {chunk_path} is not in client root {output_root}"); turbobail!("chunk path {chunk_path} is not in client root {output_root}");
}; };
let entries = EcmascriptBrowserChunkContentEntries::new(content).await?; let entries_hashes = EcmascriptChunkContentEntries::new(content)
let mut entries_hashes = .await?
FxIndexMap::with_capacity_and_hasher(entries.len(), Default::default()); .iter()
for (id, entry) in entries.iter() { .map(async |(id, entry)| Ok((id.clone(), *entry.hash.await?)))
entries_hashes.insert(id.clone(), *entry.hash.await?); .try_join()
} .await?
.into_iter()
.collect();
Ok(EcmascriptBrowserChunkVersion { Ok(EcmascriptBrowserChunkVersion {
chunk_path: chunk_path.to_string(), chunk_path: chunk_path.to_string(),
entries_hashes, entries_hashes,
......
...@@ -5,7 +5,8 @@ use turbopack_core::{ ...@@ -5,7 +5,8 @@ use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItemExt, ModuleId}, chunk::{AsyncModuleInfo, ChunkItemExt, ModuleId},
code_builder::Code, code_builder::Code,
}; };
use turbopack_ecmascript::chunk::{
use crate::chunk::{
EcmascriptChunkContent, EcmascriptChunkItem, EcmascriptChunkItemExt, EcmascriptChunkContent, EcmascriptChunkItem, EcmascriptChunkItemExt,
EcmascriptChunkItemOrBatchWithAsyncInfo, EcmascriptChunkItemWithAsyncInfo, EcmascriptChunkItemOrBatchWithAsyncInfo, EcmascriptChunkItemWithAsyncInfo,
}; };
...@@ -18,18 +19,18 @@ use turbopack_ecmascript::chunk::{ ...@@ -18,18 +19,18 @@ use turbopack_ecmascript::chunk::{
/// computing updates. /// computing updates.
#[turbo_tasks::value] #[turbo_tasks::value]
#[derive(Debug)] #[derive(Debug)]
pub struct EcmascriptDevChunkContentEntry { pub struct EcmascriptChunkContentEntry {
pub code: ResolvedVc<Code>, pub code: ResolvedVc<Code>,
pub hash: ResolvedVc<u64>, pub hash: ResolvedVc<u64>,
} }
impl EcmascriptDevChunkContentEntry { impl EcmascriptChunkContentEntry {
pub async fn new( pub async fn new(
chunk_item: ResolvedVc<Box<dyn EcmascriptChunkItem>>, chunk_item: ResolvedVc<Box<dyn EcmascriptChunkItem>>,
async_module_info: Option<Vc<AsyncModuleInfo>>, async_module_info: Option<Vc<AsyncModuleInfo>>,
) -> Result<Self> { ) -> Result<Self> {
let code = chunk_item.code(async_module_info).to_resolved().await?; let code = chunk_item.code(async_module_info).to_resolved().await?;
Ok(EcmascriptDevChunkContentEntry { Ok(EcmascriptChunkContentEntry {
code, code,
hash: code.source_code_hash().to_resolved().await?, hash: code.source_code_hash().to_resolved().await?,
}) })
...@@ -37,17 +38,16 @@ impl EcmascriptDevChunkContentEntry { ...@@ -37,17 +38,16 @@ impl EcmascriptDevChunkContentEntry {
} }
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct EcmascriptBrowserChunkContentEntries( pub struct EcmascriptChunkContentEntries(
#[bincode(with = "turbo_bincode::indexmap")] #[bincode(with = "turbo_bincode::indexmap")] FxIndexMap<ModuleId, EcmascriptChunkContentEntry>,
FxIndexMap<ModuleId, EcmascriptDevChunkContentEntry>,
); );
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl EcmascriptBrowserChunkContentEntries { impl EcmascriptChunkContentEntries {
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn new( pub async fn new(
chunk_content: Vc<EcmascriptChunkContent>, chunk_content: Vc<EcmascriptChunkContent>,
) -> Result<Vc<EcmascriptBrowserChunkContentEntries>> { ) -> Result<Vc<EcmascriptChunkContentEntries>> {
let chunk_content = chunk_content.await?; let chunk_content = chunk_content.await?;
let entries: FxIndexMap<_, _> = chunk_content let entries: FxIndexMap<_, _> = chunk_content
...@@ -62,11 +62,8 @@ impl EcmascriptBrowserChunkContentEntries { ...@@ -62,11 +62,8 @@ impl EcmascriptBrowserChunkContentEntries {
}, },
) => Either::Left(std::iter::once(( ) => Either::Left(std::iter::once((
chunk_item.id().await?, chunk_item.id().await?,
EcmascriptDevChunkContentEntry::new( EcmascriptChunkContentEntry::new(chunk_item, async_info.map(|info| *info))
chunk_item, .await?,
async_info.map(|info| *info),
)
.await?,
))), ))),
EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => { EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => {
let batch = batch.await?; let batch = batch.await?;
...@@ -77,7 +74,7 @@ impl EcmascriptBrowserChunkContentEntries { ...@@ -77,7 +74,7 @@ impl EcmascriptBrowserChunkContentEntries {
.map(|item| async move { .map(|item| async move {
Ok(( Ok((
item.chunk_item.id().await?, item.chunk_item.id().await?,
EcmascriptDevChunkContentEntry::new( EcmascriptChunkContentEntry::new(
item.chunk_item, item.chunk_item,
item.async_info.map(|info| *info), item.async_info.map(|info| *info),
) )
......
...@@ -2,6 +2,7 @@ pub(crate) mod batch; ...@@ -2,6 +2,7 @@ pub(crate) mod batch;
pub(crate) mod chunk_type; pub(crate) mod chunk_type;
pub(crate) mod code_and_ids; pub(crate) mod code_and_ids;
pub(crate) mod content; pub(crate) mod content;
pub(crate) mod content_entry;
pub(crate) mod data; pub(crate) mod data;
pub(crate) mod item; pub(crate) mod item;
pub(crate) mod placeable; pub(crate) mod placeable;
...@@ -31,6 +32,7 @@ pub use self::{ ...@@ -31,6 +32,7 @@ pub use self::{
chunk_type::EcmascriptChunkType, chunk_type::EcmascriptChunkType,
code_and_ids::{BatchGroupCodeAndIds, CodeAndIds, batch_group_code_and_ids, item_code_and_ids}, code_and_ids::{BatchGroupCodeAndIds, CodeAndIds, batch_group_code_and_ids, item_code_and_ids},
content::EcmascriptChunkContent, content::EcmascriptChunkContent,
content_entry::{EcmascriptChunkContentEntries, EcmascriptChunkContentEntry},
data::EcmascriptChunkData, data::EcmascriptChunkData,
item::{ item::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemExt, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemExt,
......
...@@ -21,6 +21,7 @@ workspace = true ...@@ -21,6 +21,7 @@ workspace = true
[dependencies] [dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
bincode = { workspace = true } bincode = { workspace = true }
either = { workspace = true }
indoc = { workspace = true } indoc = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
......
...@@ -9,7 +9,11 @@ use turbopack_core::{ ...@@ -9,7 +9,11 @@ use turbopack_core::{
source_map::{GenerateSourceMap, SourceMapAsset}, source_map::{GenerateSourceMap, SourceMapAsset},
version::{Update, Version, VersionedContent}, version::{Update, Version, VersionedContent},
}; };
use turbopack_ecmascript::{chunk::EcmascriptChunkContent, minify::minify, utils::StringifyJs}; use turbopack_ecmascript::{
chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries},
minify::minify,
utils::StringifyJs,
};
use super::{ use super::{
chunk::EcmascriptBuildNodeChunk, update::update_node_chunk, chunk::EcmascriptBuildNodeChunk, update::update_node_chunk,
...@@ -42,6 +46,11 @@ impl EcmascriptBuildNodeChunkContent { ...@@ -42,6 +46,11 @@ impl EcmascriptBuildNodeChunkContent {
} }
.cell() .cell()
} }
#[turbo_tasks::function]
pub(crate) fn entries(&self) -> Vc<EcmascriptChunkContentEntries> {
EcmascriptChunkContentEntries::new(*self.content)
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
......
...@@ -105,8 +105,8 @@ pub(super) async fn update_node_chunk( ...@@ -105,8 +105,8 @@ pub(super) async fn update_node_chunk(
return Ok(Update::None); return Ok(Update::None);
} }
let chunk_path = &to.chunk_path; let chunk_path = to.chunk_path.as_str();
let chunk_update = update_ecmascript_node_chunk_content(&to, &from).await?; let chunk_update = update_ecmascript_node_chunk_content(content, &to, &from).await?;
let mut merged_update = EcmascriptMergedUpdate::default(); let mut merged_update = EcmascriptMergedUpdate::default();
...@@ -171,6 +171,7 @@ enum NodeChunkUpdate { ...@@ -171,6 +171,7 @@ enum NodeChunkUpdate {
} }
async fn update_ecmascript_node_chunk_content( async fn update_ecmascript_node_chunk_content(
content: Vc<EcmascriptBuildNodeChunkContent>,
to: &ReadRef<EcmascriptBuildNodeChunkVersion>, to: &ReadRef<EcmascriptBuildNodeChunkVersion>,
from: &ReadRef<EcmascriptBuildNodeChunkVersion>, from: &ReadRef<EcmascriptBuildNodeChunkVersion>,
) -> Result<NodeChunkUpdate> { ) -> Result<NodeChunkUpdate> {
...@@ -178,22 +179,23 @@ async fn update_ecmascript_node_chunk_content( ...@@ -178,22 +179,23 @@ async fn update_ecmascript_node_chunk_content(
let mut modified = FxIndexMap::default(); let mut modified = FxIndexMap::default();
let mut deleted = FxIndexMap::default(); let mut deleted = FxIndexMap::default();
// Build a map of module_id -> Vc<Code> for the "to" version // Lazily resolve the entries map only when we actually need to ship code
let mut to_entries: FxIndexMap<ModuleId, Vc<Code>> = FxIndexMap::default(); // bytes for an added or modified module. For chunks that only have deletions
for item in &to.chunk_items { // (or no changes that need code beyond hashes), this avoids materializing
for (id, code) in item { // any `Vc<Code>`.
// Convert ReadRef<Code> to Vc<Code> let mut entries_ref = None;
to_entries.insert(id.clone(), ReadRef::cell(code.clone()));
}
}
// Check for deleted and modified modules // Check for deleted and modified modules
for (id, from_hash) in &from.entries_hashes { for (id, from_hash) in &from.entries_hashes {
if let Some(to_hash) = to.entries_hashes.get(id) { if let Some(to_hash) = to.entries_hashes.get(id) {
if *to_hash != *from_hash { if *to_hash != *from_hash {
// Module was modified // Module was modified
if let Some(code) = to_entries.get(id) { let entries = match &entries_ref {
modified.insert(id.clone(), *code); Some(entries) => entries,
None => entries_ref.insert(content.entries().await?),
};
if let Some(entry) = entries.get(id) {
modified.insert(id.clone(), *entry.code);
} }
} }
} else { } else {
...@@ -204,10 +206,14 @@ async fn update_ecmascript_node_chunk_content( ...@@ -204,10 +206,14 @@ async fn update_ecmascript_node_chunk_content(
// Check for added modules // Check for added modules
for (id, _hash) in &to.entries_hashes { for (id, _hash) in &to.entries_hashes {
if !from.entries_hashes.contains_key(id) if !from.entries_hashes.contains_key(id) {
&& let Some(code) = to_entries.get(id) let entries = match &entries_ref {
{ Some(entries) => entries,
added.insert(id.clone(), *code); None => entries_ref.insert(content.entries().await?),
};
if let Some(entry) = entries.get(id) {
added.insert(id.clone(), *entry.code);
}
} }
} }
......
use anyhow::Result; use anyhow::Result;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, ReadRef, Vc, turbobail}; use turbo_tasks::{FxIndexMap, TryJoinIterExt, Vc, turbobail};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64}; use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::{ use turbopack_core::{
chunk::{MinifyType, ModuleId}, chunk::{MinifyType, ModuleId},
version::Version, version::Version,
}; };
use turbopack_ecmascript::chunk::{CodeAndIds, EcmascriptChunkContent}; use turbopack_ecmascript::chunk::{EcmascriptChunkContent, EcmascriptChunkContentEntries};
#[turbo_tasks::value(serialization = "skip")] #[turbo_tasks::value(serialization = "skip")]
pub(super) struct EcmascriptBuildNodeChunkVersion { pub(super) struct EcmascriptBuildNodeChunkVersion {
pub(super) chunk_path: String, pub(super) chunk_path: RcStr,
pub(super) chunk_items: Vec<ReadRef<CodeAndIds>>,
pub(super) minify_type: MinifyType, pub(super) minify_type: MinifyType,
pub(super) entries_hashes: FxIndexMap<ModuleId, u64>, pub(super) entries_hashes: FxIndexMap<ModuleId, u64>,
} }
...@@ -33,24 +32,17 @@ impl EcmascriptBuildNodeChunkVersion { ...@@ -33,24 +32,17 @@ impl EcmascriptBuildNodeChunkVersion {
} else { } else {
turbobail!("chunk path {chunk_path} is not in client root {output_root}"); turbobail!("chunk path {chunk_path} is not in client root {output_root}");
}; };
let chunk_items = content.await?.chunk_item_code_and_ids().await?; let entries_hashes = EcmascriptChunkContentEntries::new(content)
.await?
// Compute per-module hashes for fine-grained HMR tracking .iter()
let mut entries_hashes = FxIndexMap::default(); .map(async |(id, entry)| Ok((id.clone(), *entry.hash.await?)))
for item in &chunk_items { .try_join()
for (module_id, code) in item { .await?
let mut hasher = Xxh3Hash64Hasher::new(); .into_iter()
let source = code.source_code(); .collect();
hasher.write_ref(source);
let hash = hasher.finish();
entries_hashes.insert(module_id.clone(), hash);
}
}
Ok(EcmascriptBuildNodeChunkVersion { Ok(EcmascriptBuildNodeChunkVersion {
chunk_path: chunk_path.to_string(), chunk_path: chunk_path.into(),
chunk_items,
minify_type, minify_type,
entries_hashes, entries_hashes,
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment