Unverified Commit e169a26f authored by Sam Poder's avatar Sam Poder Committed by GitHub

[turbopack] Sort modules in files by path not ID (#95065)

Sorting-by-ID was flagged as bad for g-zipping:

> This sorts by chunk item id.
> Tobias has been saying that this is bad, because it gzips worse
> We should be sort by path, so that similar modules stay together. E.g.
if you have many similar icon modules
> Sorting by chunk item results in random order

Switched to sorting by path!
parent ae7ab812
...@@ -169,7 +169,7 @@ describe('ReactRefreshLogBox', () => { ...@@ -169,7 +169,7 @@ describe('ReactRefreshLogBox', () => {
"stack": [ "stack": [
"module evaluation index.js (3:7)", "module evaluation index.js (3:7)",
"module evaluation pages/index.js (1:1)", "module evaluation pages/index.js (1:1)",
"module evaluation pages/index.js (1:1)", "module evaluation index.js (9:16)",
"<FIXME-next-dist-dir>", "<FIXME-next-dist-dir>",
], ],
} }
......
...@@ -118,10 +118,13 @@ impl EcmascriptBrowserChunkContent { ...@@ -118,10 +118,13 @@ impl EcmascriptBrowserChunkContent {
)?; )?;
let content = this.content.await?; let content = this.content.await?;
let mut chunk_items = content.chunk_item_code_and_ids().await?; let mut chunk_items = content.chunk_item_code_module_ids_and_paths().await?;
chunk_items.sort_by_key(|item| item.first().map(|(id, _)| id.clone())); // Sort items by their module path so that similar modules stay
// together so that the chunks gzips better.
chunk_items
.sort_by_key(|item| item.first().map(|(id, _, path)| (path.clone(), id.clone())));
for item in &chunk_items { for item in &chunk_items {
for (id, item_code) in &**item { for (id, item_code, _) in &**item {
write!(code, "\n{}, ", StringifyJs(id))?; write!(code, "\n{}, ", StringifyJs(id))?;
code.push_code(item_code); code.push_code(item_code);
write!(code, ",")?; write!(code, ",")?;
......
use anyhow::Result; use anyhow::Result;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use smallvec::{SmallVec, smallvec}; use smallvec::{SmallVec, smallvec};
use turbo_tasks::{ReadRef, TryJoinIterExt, Vc}; use turbo_rcstr::RcStr;
use turbo_tasks::{ReadRef, TryJoinIterExt, ValueToString, Vc};
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkItemExt, ModuleId}, chunk::{ChunkItem, ChunkItemExt, ModuleId},
code_builder::Code, code_builder::Code,
}; };
...@@ -13,23 +14,28 @@ use crate::chunk::{ ...@@ -13,23 +14,28 @@ use crate::chunk::{
}; };
#[turbo_tasks::value(transparent, serialization = "skip")] #[turbo_tasks::value(transparent, serialization = "skip")]
pub struct CodeAndIds(SmallVec<[(ModuleId, ReadRef<Code>); 1]>); pub struct CodeModuleIdsAndPaths(SmallVec<[(ModuleId, ReadRef<Code>, RcStr); 1]>);
#[turbo_tasks::value(transparent, serialization = "skip")] #[turbo_tasks::value(transparent, serialization = "skip")]
pub struct BatchGroupCodeAndIds( pub struct BatchGroupCodeModuleIdsAndPaths(
FxHashMap<EcmascriptChunkItemOrBatchWithAsyncInfo, ReadRef<CodeAndIds>>, FxHashMap<EcmascriptChunkItemOrBatchWithAsyncInfo, ReadRef<CodeModuleIdsAndPaths>>,
); );
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn batch_group_code_and_ids( pub async fn batch_group_code_module_ids_and_paths(
batch_group: Vc<EcmascriptChunkItemBatchGroup>, batch_group: Vc<EcmascriptChunkItemBatchGroup>,
) -> Result<Vc<BatchGroupCodeAndIds>> { ) -> Result<Vc<BatchGroupCodeModuleIdsAndPaths>> {
Ok(Vc::cell( Ok(Vc::cell(
batch_group batch_group
.await? .await?
.items .items
.iter() .iter()
.map(async |item| Ok((item.clone(), item_code_and_ids(item.clone()).await?))) .map(async |item| {
Ok((
item.clone(),
item_code_module_ids_and_paths(item.clone()).await?,
))
})
.try_join() .try_join()
.await? .await?
.into_iter() .into_iter()
...@@ -38,9 +44,9 @@ pub async fn batch_group_code_and_ids( ...@@ -38,9 +44,9 @@ pub async fn batch_group_code_and_ids(
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn item_code_and_ids( pub async fn item_code_module_ids_and_paths(
item: EcmascriptChunkItemOrBatchWithAsyncInfo, item: EcmascriptChunkItemOrBatchWithAsyncInfo,
) -> Result<Vc<CodeAndIds>> { ) -> Result<Vc<CodeModuleIdsAndPaths>> {
Ok(Vc::cell(match item { Ok(Vc::cell(match item {
EcmascriptChunkItemOrBatchWithAsyncInfo::ChunkItem(EcmascriptChunkItemWithAsyncInfo { EcmascriptChunkItemOrBatchWithAsyncInfo::ChunkItem(EcmascriptChunkItemWithAsyncInfo {
chunk_item, chunk_item,
...@@ -49,7 +55,8 @@ pub async fn item_code_and_ids( ...@@ -49,7 +55,8 @@ pub async fn item_code_and_ids(
}) => { }) => {
let id = chunk_item.id().await?; let id = chunk_item.id().await?;
let code = chunk_item.code(async_info.map(|info| *info)); let code = chunk_item.code(async_info.map(|info| *info));
smallvec![(id, code.await?)] let path = chunk_item.asset_ident().to_string().owned().await?;
smallvec![(id, code.await?, path)]
} }
EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => batch EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => batch
.await? .await?
...@@ -61,6 +68,7 @@ pub async fn item_code_and_ids( ...@@ -61,6 +68,7 @@ pub async fn item_code_and_ids(
item.chunk_item item.chunk_item
.code(item.async_info.map(|info| *info)) .code(item.async_info.map(|info| *info))
.await?, .await?,
item.chunk_item.asset_ident().to_string().owned().await?,
)) ))
}) })
.try_join() .try_join()
......
...@@ -6,9 +6,9 @@ use turbo_tasks::{ReadRef, ResolvedVc, TryJoinIterExt, Vc}; ...@@ -6,9 +6,9 @@ use turbo_tasks::{ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::chunk::{ChunkItem, ChunkItems, batch_info}; use turbopack_core::chunk::{ChunkItem, ChunkItems, batch_info};
use crate::chunk::{ use crate::chunk::{
CodeAndIds, CodeModuleIdsAndPaths,
batch::{EcmascriptChunkItemBatchGroup, EcmascriptChunkItemOrBatchWithAsyncInfo}, batch::{EcmascriptChunkItemBatchGroup, EcmascriptChunkItemOrBatchWithAsyncInfo},
batch_group_code_and_ids, item_code_and_ids, batch_group_code_module_ids_and_paths, item_code_module_ids_and_paths,
}; };
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
...@@ -49,12 +49,14 @@ impl EcmascriptChunkContent { ...@@ -49,12 +49,14 @@ impl EcmascriptChunkContent {
} }
impl EcmascriptChunkContent { impl EcmascriptChunkContent {
pub async fn chunk_item_code_and_ids(&self) -> Result<Vec<ReadRef<CodeAndIds>>> { pub async fn chunk_item_code_module_ids_and_paths(
&self,
) -> Result<Vec<ReadRef<CodeModuleIdsAndPaths>>> {
batch_info( batch_info(
&self.batch_groups, &self.batch_groups,
&self.chunk_items, &self.chunk_items,
|batch| batch_group_code_and_ids(batch).into_future(), |batch| batch_group_code_module_ids_and_paths(batch).into_future(),
|item| item_code_and_ids(item.clone()).into_future(), |item| item_code_module_ids_and_paths(item.clone()).into_future(),
) )
.await .await
} }
......
pub(crate) mod batch; pub(crate) mod batch;
pub(crate) mod chunk_type; pub(crate) mod chunk_type;
pub(crate) mod code_and_ids; pub(crate) mod code_module_ids_and_paths;
pub(crate) mod content; pub(crate) mod content;
pub(crate) mod content_entry; pub(crate) mod content_entry;
pub(crate) mod data; pub(crate) mod data;
...@@ -30,7 +30,10 @@ pub use self::{ ...@@ -30,7 +30,10 @@ pub use self::{
EcmascriptChunkItemOrBatchWithAsyncInfo, EcmascriptChunkItemOrBatchWithAsyncInfo,
}, },
chunk_type::EcmascriptChunkType, chunk_type::EcmascriptChunkType,
code_and_ids::{BatchGroupCodeAndIds, CodeAndIds, batch_group_code_and_ids, item_code_and_ids}, code_module_ids_and_paths::{
BatchGroupCodeModuleIdsAndPaths, CodeModuleIdsAndPaths,
batch_group_code_module_ids_and_paths, item_code_module_ids_and_paths,
},
content::EcmascriptChunkContent, content::EcmascriptChunkContent,
content_entry::{EcmascriptChunkContentEntries, EcmascriptChunkContentEntry}, content_entry::{EcmascriptChunkContentEntries, EcmascriptChunkContentEntry},
data::EcmascriptChunkData, data::EcmascriptChunkData,
......
...@@ -68,9 +68,11 @@ impl EcmascriptBuildNodeChunkContent { ...@@ -68,9 +68,11 @@ impl EcmascriptBuildNodeChunkContent {
write!(code, "module.exports = [")?; write!(code, "module.exports = [")?;
let content = self.content.await?; let content = self.content.await?;
let chunk_items = content.chunk_item_code_and_ids().await?; let mut chunk_items = content.chunk_item_code_module_ids_and_paths().await?;
chunk_items
.sort_by_key(|item| item.first().map(|(id, _, path)| (path.clone(), id.clone())));
for item in &chunk_items { for item in &chunk_items {
for (id, item_code) in &**item { for (id, item_code, _) in &**item {
write!(code, "\n{}, ", StringifyJs(id))?; write!(code, "\n{}, ", StringifyJs(id))?;
code.push_code(item_code); code.push_code(item_code);
write!(code, ",")?; write!(code, ",")?;
......
module.exports = [ module.exports = [
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s([
"foo",
()=>foo
]);
function foo(value) {
console.assert(value);
}
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)", ((__turbopack_context__) => { "[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict"; "use strict";
...@@ -23,6 +12,17 @@ var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbo ...@@ -23,6 +12,17 @@ var __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbo
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$basic$2f$async_chunk_build$2f$input$2f$node_modules$2f$foo$2f$index$2e$js__$5b$test$5d$__$28$ecmascript$29$__["foo"])(true); (0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$basic$2f$async_chunk_build$2f$input$2f$node_modules$2f$foo$2f$index$2e$js__$5b$test$5d$__$28$ecmascript$29$__["foo"])(true);
(0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$basic$2f$async_chunk_build$2f$input$2f$node_modules$2f$bar$2f$index$2e$js__$5b$test$5d$__$28$ecmascript$29$__["bar"])(true); (0, __TURBOPACK__imported__module__$5b$project$5d2f$turbopack$2f$crates$2f$turbopack$2d$tests$2f$tests$2f$snapshot$2f$basic$2f$async_chunk_build$2f$input$2f$node_modules$2f$bar$2f$index$2e$js__$5b$test$5d$__$28$ecmascript$29$__["bar"])(true);
}), }),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s([
"foo",
()=>foo
]);
function foo(value) {
console.assert(value);
}
}),
]; ];
//# sourceMappingURL=1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js.map //# sourceMappingURL=1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js.map
\ No newline at end of file
...@@ -2,6 +2,6 @@ ...@@ -2,6 +2,6 @@
"version": 3, "version": 3,
"sources": [], "sources": [],
"sections": [ "sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js"],"sourcesContent":["export function foo(value) {\n console.assert(value);\n}\n"],"names":["foo","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}}, {"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js"],"sourcesContent":["import { foo } from 'foo'\nimport { bar } from 'bar'\nimport './shared'\n\nfoo(true)\nbar(true)\n"],"names":[],"mappings":";AAAA;AACA;AACA;;;;AAEA,IAAA,sOAAG,EAAC;AACJ,IAAA,sOAAG,EAAC"}},
{"offset": {"line": 15, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js"],"sourcesContent":["import { foo } from 'foo'\nimport { bar } from 'bar'\nimport './shared'\n\nfoo(true)\nbar(true)\n"],"names":[],"mappings":";AAAA;AACA;AACA;;;;AAEA,IAAA,sOAAG,EAAC;AACJ,IAAA,sOAAG,EAAC"}}] {"offset": {"line": 17, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/foo/index.js"],"sourcesContent":["export function foo(value) {\n console.assert(value);\n}\n"],"names":["foo","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}}]
} }
\ No newline at end of file
module.exports = [ module.exports = [
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js [test] (ecmascript)", ((__turbopack_context__) => { "[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript, async loader)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.s([
"bar",
()=>bar
]);
function bar(value) {
console.assert(value);
}
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js [test] (ecmascript)", ((__turbopack_context__, module, exports) => {
// shared package __turbopack_context__.v((parentImport) => {
return Promise.all([
"output/1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js"
].map((chunk) => __turbopack_context__.l(chunk))).then(() => {
return parentImport("[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)");
});
});
}), }),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js [test] (ecmascript)", ((__turbopack_context__) => { "[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict"; "use strict";
...@@ -27,15 +22,20 @@ __turbopack_context__.A("[project]/turbopack/crates/turbopack-tests/tests/snapsh ...@@ -27,15 +22,20 @@ __turbopack_context__.A("[project]/turbopack/crates/turbopack-tests/tests/snapsh
foo(true); foo(true);
}); });
}), }),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript, async loader)", ((__turbopack_context__) => { "[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
__turbopack_context__.v((parentImport) => { __turbopack_context__.s([
return Promise.all([ "bar",
"output/1do3_crates_turbopack-tests_tests_snapshot_basic_async_chunk_build_input_0aldyki._.js" ()=>bar
].map((chunk) => __turbopack_context__.l(chunk))).then(() => { ]);
return parentImport("[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/import.js [test] (ecmascript)"); function bar(value) {
}); console.assert(value);
}); }
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js [test] (ecmascript)", ((__turbopack_context__, module, exports) => {
// shared package
}), }),
]; ];
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
"version": 3, "version": 3,
"sources": [], "sources": [],
"sections": [ "sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js"],"sourcesContent":["export function bar(value) {\n console.assert(value);\n}\n"],"names":["bar","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}}, {"offset": {"line": 14, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js"],"sourcesContent":["import { bar } from 'bar'\nimport './shared'\n\nbar(true)\n\nimport('./import').then(({ foo }) => {\n foo(true)\n})\n"],"names":["then","foo"],"mappings":";AAAA;AACA;;;AAEA,IAAA,sOAAG,EAAC;AAEJ,+JAAmBA,IAAI,CAAC,CAAC,EAAEC,GAAG,EAAE;IAC9BA,IAAI;AACN"}},
{"offset": {"line": 14, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js"],"sourcesContent":["// shared package\n"],"names":[],"mappings":"AAAA,iBAAiB"}}, {"offset": {"line": 27, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/node_modules/bar/index.js"],"sourcesContent":["export function bar(value) {\n console.assert(value);\n}\n"],"names":["bar","value","console","assert"],"mappings":";;;;AAAO,SAASA,IAAIC,KAAK;IACvBC,QAAQC,MAAM,CAACF;AACjB","ignoreList":[0]}},
{"offset": {"line": 19, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/index.js"],"sourcesContent":["import { bar } from 'bar'\nimport './shared'\n\nbar(true)\n\nimport('./import').then(({ foo }) => {\n foo(true)\n})\n"],"names":["then","foo"],"mappings":";AAAA;AACA;;;AAEA,IAAA,sOAAG,EAAC;AAEJ,+JAAmBA,IAAI,CAAC,CAAC,EAAEC,GAAG,EAAE;IAC9BA,IAAI;AACN"}}] {"offset": {"line": 37, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic/async_chunk_build/input/shared.js"],"sourcesContent":["// shared package\n"],"names":[],"mappings":"AAAA,iBAAiB"}}]
} }
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment