Unverified Commit e169a26f authored by Sam Poder's avatar Sam Poder Committed by GitHub

[turbopack] Sort modules in files by path not ID (#95065)

Sorting-by-ID was flagged as bad for g-zipping:

> This sorts by chunk item id.
> Tobias has been saying that this is bad, because it gzips worse
> We should be sort by path, so that similar modules stay together. E.g.
if you have many similar icon modules
> Sorting by chunk item results in random order

Switched to sorting by path!
parent ae7ab812
......@@ -169,7 +169,7 @@ describe('ReactRefreshLogBox', () => {
"stack": [
"module evaluation index.js (3:7)",
"module evaluation pages/index.js (1:1)",
"module evaluation pages/index.js (1:1)",
"module evaluation index.js (9:16)",
"<FIXME-next-dist-dir>",
],
}
......
......@@ -118,10 +118,13 @@ impl EcmascriptBrowserChunkContent {
)?;
let content = this.content.await?;
let mut chunk_items = content.chunk_item_code_and_ids().await?;
chunk_items.sort_by_key(|item| item.first().map(|(id, _)| id.clone()));
let mut chunk_items = content.chunk_item_code_module_ids_and_paths().await?;
// Sort items by their module path so that similar modules stay
// together so that the chunks gzips better.
chunk_items
.sort_by_key(|item| item.first().map(|(id, _, path)| (path.clone(), id.clone())));
for item in &chunk_items {
for (id, item_code) in &**item {
for (id, item_code, _) in &**item {
write!(code, "\n{}, ", StringifyJs(id))?;
code.push_code(item_code);
write!(code, ",")?;
......
use anyhow::Result;
use rustc_hash::FxHashMap;
use smallvec::{SmallVec, smallvec};
use turbo_tasks::{ReadRef, TryJoinIterExt, Vc};
use turbo_rcstr::RcStr;
use turbo_tasks::{ReadRef, TryJoinIterExt, ValueToString, Vc};
use turbopack_core::{
chunk::{ChunkItemExt, ModuleId},
chunk::{ChunkItem, ChunkItemExt, ModuleId},
code_builder::Code,
};
......@@ -13,23 +14,28 @@ use crate::chunk::{
};
#[turbo_tasks::value(transparent, serialization = "skip")]
pub struct CodeAndIds(SmallVec<[(ModuleId, ReadRef<Code>); 1]>);
pub struct CodeModuleIdsAndPaths(SmallVec<[(ModuleId, ReadRef<Code>, RcStr); 1]>);
#[turbo_tasks::value(transparent, serialization = "skip")]
pub struct BatchGroupCodeAndIds(
FxHashMap<EcmascriptChunkItemOrBatchWithAsyncInfo, ReadRef<CodeAndIds>>,
pub struct BatchGroupCodeModuleIdsAndPaths(
FxHashMap<EcmascriptChunkItemOrBatchWithAsyncInfo, ReadRef<CodeModuleIdsAndPaths>>,
);
#[turbo_tasks::function]
pub async fn batch_group_code_and_ids(
pub async fn batch_group_code_module_ids_and_paths(
batch_group: Vc<EcmascriptChunkItemBatchGroup>,
) -> Result<Vc<BatchGroupCodeAndIds>> {
) -> Result<Vc<BatchGroupCodeModuleIdsAndPaths>> {
Ok(Vc::cell(
batch_group
.await?
.items
.iter()
.map(async |item| Ok((item.clone(), item_code_and_ids(item.clone()).await?)))
.map(async |item| {
Ok((
item.clone(),
item_code_module_ids_and_paths(item.clone()).await?,
))
})
.try_join()
.await?
.into_iter()
......@@ -38,9 +44,9 @@ pub async fn batch_group_code_and_ids(
}
#[turbo_tasks::function]
pub async fn item_code_and_ids(
pub async fn item_code_module_ids_and_paths(
item: EcmascriptChunkItemOrBatchWithAsyncInfo,
) -> Result<Vc<CodeAndIds>> {
) -> Result<Vc<CodeModuleIdsAndPaths>> {
Ok(Vc::cell(match item {
EcmascriptChunkItemOrBatchWithAsyncInfo::ChunkItem(EcmascriptChunkItemWithAsyncInfo {
chunk_item,
......@@ -49,7 +55,8 @@ pub async fn item_code_and_ids(
}) => {
let id = chunk_item.id().await?;
let code = chunk_item.code(async_info.map(|info| *info));
smallvec![(id, code.await?)]
let path = chunk_item.asset_ident().to_string().owned().await?;
smallvec![(id, code.await?, path)]
}
EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => batch
.await?
......@@ -61,6 +68,7 @@ pub async fn item_code_and_ids(
item.chunk_item
.code(item.async_info.map(|info| *info))
.await?,
item.chunk_item.asset_ident().to_string().owned().await?,
))
})
.try_join()
......
......@@ -6,9 +6,9 @@ use turbo_tasks::{ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::chunk::{ChunkItem, ChunkItems, batch_info};
use crate::chunk::{
CodeAndIds,
CodeModuleIdsAndPaths,
batch::{EcmascriptChunkItemBatchGroup, EcmascriptChunkItemOrBatchWithAsyncInfo},
batch_group_code_and_ids, item_code_and_ids,
batch_group_code_module_ids_and_paths, item_code_module_ids_and_paths,
};
#[turbo_tasks::value(shared)]
......@@ -49,12 +49,14 @@ impl EcmascriptChunkContent {
}
impl EcmascriptChunkContent {
pub async fn chunk_item_code_and_ids(&self) -> Result<Vec<ReadRef<CodeAndIds>>> {
pub async fn chunk_item_code_module_ids_and_paths(
&self,
) -> Result<Vec<ReadRef<CodeModuleIdsAndPaths>>> {
batch_info(
&self.batch_groups,
&self.chunk_items,
|batch| batch_group_code_and_ids(batch).into_future(),
|item| item_code_and_ids(item.clone()).into_future(),
|batch| batch_group_code_module_ids_and_paths(batch).into_future(),
|item| item_code_module_ids_and_paths(item.clone()).into_future(),
)
.await
}
......
pub(crate) mod batch;
pub(crate) mod chunk_type;
pub(crate) mod code_and_ids;
pub(crate) mod code_module_ids_and_paths;
pub(crate) mod content;
pub(crate) mod content_entry;
pub(crate) mod data;
......@@ -30,7 +30,10 @@ pub use self::{
EcmascriptChunkItemOrBatchWithAsyncInfo,
},
chunk_type::EcmascriptChunkType,
code_and_ids::{BatchGroupCodeAndIds, CodeAndIds, batch_group_code_and_ids, item_code_and_ids},
code_module_ids_and_paths::{
BatchGroupCodeModuleIdsAndPaths, CodeModuleIdsAndPaths,
batch_group_code_module_ids_and_paths, item_code_module_ids_and_paths,
},
content::EcmascriptChunkContent,
content_entry::{EcmascriptChunkContentEntries, EcmascriptChunkContentEntry},
data::EcmascriptChunkData,
......
......@@ -68,9 +68,11 @@ impl EcmascriptBuildNodeChunkContent {
write!(code, "module.exports = [")?;
let content = self.content.await?;
let chunk_items = content.chunk_item_code_and_ids().await?;
let mut chunk_items = content.chunk_item_code_module_ids_and_paths().await?;
chunk_items
.sort_by_key(|item| item.first().map(|(id, _, path)| (path.clone(), id.clone())));
for item in &chunk_items {
for (id, item_code) in &**item {
for (id, item_code, _) in &**item {
write!(code, "\n{}, ", StringifyJs(id))?;
code.push_code(item_code);
write!(code, ",")?;
......
module.exports = [
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s([
"foo",
()=>foo
]);
function foo(value) {
console.assert(value);
}
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
......@@ -23,6 +12,17 @@ var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbo
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$basic$2f$async_chunk_build$2f$input$2f$node_modules$2f$foo$2f$index$2e$js__$5b$test$5d$__$28$ecmascript$29$__["foo"])(true);
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$basic$2f$async_chunk_build$2f$input$2f$node_modules$2f$bar$2f$index$2e$js__$5b$test$5d$__$28$ecmascript$29$__["bar"])(true);
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s([
"foo",
()=>foo
]);
function foo(value) {
console.assert(value);
}
}),
];
//# sourceMappingURL=1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js.map
\ No newline at end of file
......@@ -2,6 +2,6 @@
"version": 3,
"sources": [],
"sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js"],"sourcesContent":["export function foo(value) {\n console.assert(value);\n}\n"],"names":["foo","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}},
{"offset": {"line": 15, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js"],"sourcesContent":["import { foo } from 'foo'\nimport { bar } from 'bar'\nimport './shared'\n\nfoo(true)\nbar(true)\n"],"names":[],"mappings":";AAAA;AACA;AACA;;;;AAEA,IAAA,sOAAG,EAAC;AACJ,IAAA,sOAAG,EAAC"}}]
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js"],"sourcesContent":["import { foo } from 'foo'\nimport { bar } from 'bar'\nimport './shared'\n\nfoo(true)\nbar(true)\n"],"names":[],"mappings":";AAAA;AACA;AACA;;;;AAEA,IAAA,sOAAG,EAAC;AACJ,IAAA,sOAAG,EAAC"}},
{"offset": {"line": 17, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js"],"sourcesContent":["export function foo(value) {\n console.assert(value);\n}\n"],"names":["foo","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}}]
}
\ No newline at end of file
module.exports = [
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s([
"bar",
()=>bar
]);
function bar(value) {
console.assert(value);
}
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js [test] (ecmascript)", ((__turbopack_context__, module, exports) => {
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript, async loader)", ((__turbopack_context__) => {
// shared package
__turbopack_context__.v((parentImport) => {
return Promise.all([
"output/1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js"
].map((chunk) => __turbopack_context__.l(chunk))).then(() => {
return parentImport("[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)");
});
});
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
......@@ -27,15 +22,20 @@ __turbopack_context__.A("[project]/turbopack/crates/turbopack-tests/tests/snapsh
foo(true);
});
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript, async loader)", ((__turbopack_context__) => {
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.v((parentImport) => {
return Promise.all([
"output/1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js"
].map((chunk) => __turbopack_context__.l(chunk))).then(() => {
return parentImport("[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)");
});
});
__turbopack_context__.s([
"bar",
()=>bar
]);
function bar(value) {
console.assert(value);
}
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js [test] (ecmascript)", ((__turbopack_context__, module, exports) => {
// shared package
}),
];
......
......@@ -2,7 +2,7 @@
"version": 3,
"sources": [],
"sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js"],"sourcesContent":["export function bar(value) {\n console.assert(value);\n}\n"],"names":["bar","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}},
{"offset": {"line": 14, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js"],"sourcesContent":["// shared package\n"],"names":[],"mappings":"AAAA,iBAAiB"}},
{"offset": {"line": 19, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js"],"sourcesContent":["import { bar } from 'bar'\nimport './shared'\n\nbar(true)\n\nimport('./import').then(({ foo }) => {\n foo(true)\n})\n"],"names":["then","foo"],"mappings":";AAAA;AACA;;;AAEA,IAAA,sOAAG,EAAC;AAEJ,+JAAmBA,IAAI,CAAC,CAAC,EAAEC,GAAG,EAAE;IAC9BA,IAAI;AACN"}}]
{"offset": {"line": 14, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js"],"sourcesContent":["import { bar } from 'bar'\nimport './shared'\n\nbar(true)\n\nimport('./import').then(({ foo }) => {\n foo(true)\n})\n"],"names":["then","foo"],"mappings":";AAAA;AACA;;;AAEA,IAAA,sOAAG,EAAC;AAEJ,+JAAmBA,IAAI,CAAC,CAAC,EAAEC,GAAG,EAAE;IAC9BA,IAAI;AACN"}},
{"offset": {"line": 27, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js"],"sourcesContent":["export function bar(value) {\n console.assert(value);\n}\n"],"names":["bar","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}},
{"offset": {"line": 37, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js"],"sourcesContent":["// shared package\n"],"names":[],"mappings":"AAAA,iBAAiB"}}]
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment