Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
e431f9e6
Unverified
Commit
e431f9e6
authored
Jan 24, 2025
by
Niklas Mischkulnig
Committed by
GitHub
Jan 24, 2025
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Turbopack: migrate global module ids to single-graph (#73752)
parent
bc3e597a
Show whitespace changes
Inline
Side-by-side
Showing
24 changed files
with
331 additions
and
383 deletions
+331
-383
Cargo.lock
Cargo.lock
+4
-0
Cargo.toml
crates/next-api/Cargo.toml
+1
-0
global_module_id_strategy.rs
crates/next-api/src/global_module_id_strategy.rs
+0
-88
lib.rs
crates/next-api/src/lib.rs
+0
-1
pages.rs
crates/next-api/src/pages.rs
+7
-2
project.rs
crates/next-api/src/project.rs
+20
-11
next_config.rs
crates/next-core/src/next_config.rs
+3
-3
server_component_module.rs
...core/src/next_server_component/server_component_module.rs
+2
-24
index.test.ts
test/production/pages-dir/production/test/index.test.ts
+13
-10
module_id_strategies.rs
...k/crates/turbopack-core/src/chunk/module_id_strategies.rs
+37
-16
file_source.rs
turbopack/crates/turbopack-core/src/file_source.rs
+9
-2
module.rs
turbopack/crates/turbopack-core/src/module.rs
+0
-4
mod.rs
turbopack/crates/turbopack-core/src/module_graph/mod.rs
+35
-0
chunk_item.rs
...crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs
+3
-3
module.rs
...ack/crates/turbopack-ecmascript/src/async_chunk/module.rs
+2
-2
global_module_id_strategy.rs
...tes/turbopack-ecmascript/src/global_module_id_strategy.rs
+0
-207
lib.rs
turbopack/crates/turbopack-ecmascript/src/lib.rs
+13
-1
loader_item.rs
...k/crates/turbopack-ecmascript/src/manifest/loader_item.rs
+3
-3
chunk_item.rs
.../crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs
+19
-5
side_effect_module.rs
...turbopack-ecmascript/src/tree_shake/side_effect_module.rs
+2
-1
utils.rs
turbopack/crates/turbopack-ecmascript/src/utils.rs
+17
-0
Cargo.toml
turbopack/crates/turbopack/Cargo.toml
+3
-0
global_module_ids.rs
turbopack/crates/turbopack/src/global_module_ids.rs
+137
-0
lib.rs
turbopack/crates/turbopack/src/lib.rs
+1
-0
No files found.
Cargo.lock
View file @
e431f9e6
...
@@ -4412,6 +4412,7 @@ dependencies = [
...
@@ -4412,6 +4412,7 @@ dependencies = [
"turbo-tasks-build",
"turbo-tasks-build",
"turbo-tasks-env",
"turbo-tasks-env",
"turbo-tasks-fs",
"turbo-tasks-fs",
"turbo-tasks-hash",
"turbo-tasks-memory",
"turbo-tasks-memory",
"turbopack",
"turbopack",
"turbopack-browser",
"turbopack-browser",
...
@@ -9377,8 +9378,10 @@ dependencies = [
...
@@ -9377,8 +9378,10 @@ dependencies = [
"regex",
"regex",
"rstest",
"rstest",
"rstest_reuse",
"rstest_reuse",
"rustc-hash 1.1.0",
"serde",
"serde",
"serde_json",
"serde_json",
"smallvec",
"tokio",
"tokio",
"tracing",
"tracing",
"turbo-rcstr",
"turbo-rcstr",
...
@@ -9386,6 +9389,7 @@ dependencies = [
...
@@ -9386,6 +9389,7 @@ dependencies = [
"turbo-tasks-build",
"turbo-tasks-build",
"turbo-tasks-env",
"turbo-tasks-env",
"turbo-tasks-fs",
"turbo-tasks-fs",
"turbo-tasks-hash",
"turbo-tasks-malloc",
"turbo-tasks-malloc",
"turbo-tasks-memory",
"turbo-tasks-memory",
"turbopack-core",
"turbopack-core",
...
...
crates/next-api/Cargo.toml
View file @
e431f9e6
...
@@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true }
...
@@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks
=
{
workspace
=
true
}
turbo-tasks
=
{
workspace
=
true
}
turbo-tasks-env
=
{
workspace
=
true
}
turbo-tasks-env
=
{
workspace
=
true
}
turbo-tasks-fs
=
{
workspace
=
true
}
turbo-tasks-fs
=
{
workspace
=
true
}
turbo-tasks-hash
=
{
workspace
=
true
}
turbo-tasks-memory
=
{
workspace
=
true
}
turbo-tasks-memory
=
{
workspace
=
true
}
turbopack
=
{
workspace
=
true
}
turbopack
=
{
workspace
=
true
}
turbopack-browser
=
{
workspace
=
true
}
turbopack-browser
=
{
workspace
=
true
}
...
...
crates/next-api/src/global_module_id_strategy.rs
deleted
100644 → 0
View file @
bc3e597a
use
anyhow
::
Result
;
use
turbo_tasks
::
Vc
;
use
turbopack_core
::
chunk
::
module_id_strategies
::{
GlobalModuleIdStrategy
,
ModuleIdStrategy
};
use
turbopack_ecmascript
::
global_module_id_strategy
::{
children_modules_idents
,
merge_preprocessed_module_ids
,
PreprocessedChildrenIdents
,
};
use
crate
::{
project
::
Project
,
route
::{
Endpoint
,
Route
},
};
#[turbo_tasks
::
value]
pub
struct
GlobalModuleIdStrategyBuilder
;
// NOTE(LichuAcu) To access all entrypoints, we need to access an instance of `Project`, but
// `Project` is not available in `turbopack-core`, so we need need this
// `GlobalModuleIdStrategyBuilder` in `next-api`.
#[turbo_tasks
::
value_impl]
impl
GlobalModuleIdStrategyBuilder
{
#[turbo_tasks
::
function]
pub
async
fn
build
(
project
:
Vc
<
Project
>
)
->
Result
<
Vc
<
Box
<
dyn
ModuleIdStrategy
>>>
{
let
mut
preprocessed_module_ids
=
Vec
::
new
();
preprocessed_module_ids
.push
(
children_modules_idents
(
project
.client_main_modules
()));
let
entrypoints
=
project
.entrypoints
()
.await
?
;
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
entrypoints
.pages_error_endpoint
));
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
entrypoints
.pages_app_endpoint
));
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
entrypoints
.pages_document_endpoint
));
if
let
Some
(
middleware
)
=
&
entrypoints
.middleware
{
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
middleware
.endpoint
));
}
if
let
Some
(
instrumentation
)
=
&
entrypoints
.instrumentation
{
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
instrumentation
.node_js
));
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
instrumentation
.edge
));
}
for
(
_
,
route
)
in
entrypoints
.routes
.iter
()
{
match
route
{
Route
::
Page
{
html_endpoint
,
data_endpoint
,
}
=>
{
preprocessed_module_ids
.push
(
preprocess_module_ids
(
**
html_endpoint
));
preprocessed_module_ids
.push
(
preprocess_module_ids
(
**
data_endpoint
));
}
Route
::
PageApi
{
endpoint
}
=>
{
preprocessed_module_ids
.push
(
preprocess_module_ids
(
**
endpoint
));
}
Route
::
AppPage
(
page_routes
)
=>
{
for
page_route
in
page_routes
{
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
page_route
.html_endpoint
));
preprocessed_module_ids
.push
(
preprocess_module_ids
(
*
page_route
.rsc_endpoint
));
}
}
Route
::
AppRoute
{
original_name
:
_
,
endpoint
,
}
=>
{
preprocessed_module_ids
.push
(
preprocess_module_ids
(
**
endpoint
));
}
Route
::
Conflict
=>
{
tracing
::
info!
(
"WARN: conflict"
);
}
}
}
let
module_id_map
=
merge_preprocessed_module_ids
(
preprocessed_module_ids
)
.await
?
;
Ok
(
Vc
::
upcast
(
GlobalModuleIdStrategy
::
new
(
module_id_map
)
.await
?
,
))
}
}
// NOTE(LichuAcu) We can't move this function to `turbopack-core` because we need access to
// `Endpoint`, which is not available there.
#[turbo_tasks
::
function]
fn
preprocess_module_ids
(
endpoint
:
Vc
<
Box
<
dyn
Endpoint
>>
)
->
Vc
<
PreprocessedChildrenIdents
>
{
let
root_modules
=
endpoint
.root_modules
();
children_modules_idents
(
root_modules
)
}
crates/next-api/src/lib.rs
View file @
e431f9e6
...
@@ -9,7 +9,6 @@ mod dynamic_imports;
...
@@ -9,7 +9,6 @@ mod dynamic_imports;
mod
empty
;
mod
empty
;
pub
mod
entrypoints
;
pub
mod
entrypoints
;
mod
font
;
mod
font
;
pub
mod
global_module_id_strategy
;
mod
instrumentation
;
mod
instrumentation
;
mod
loadable_manifest
;
mod
loadable_manifest
;
mod
middleware
;
mod
middleware
;
...
...
crates/next-api/src/pages.rs
View file @
e431f9e6
...
@@ -86,6 +86,11 @@ pub struct PagesProject {
...
@@ -86,6 +86,11 @@ pub struct PagesProject {
project
:
ResolvedVc
<
Project
>
,
project
:
ResolvedVc
<
Project
>
,
}
}
#[turbo_tasks
::
function]
fn
client_layer
()
->
Vc
<
RcStr
>
{
Vc
::
cell
(
"client"
.into
())
}
#[turbo_tasks
::
value_impl]
#[turbo_tasks
::
value_impl]
impl
PagesProject
{
impl
PagesProject
{
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
...
@@ -312,7 +317,7 @@ impl PagesProject {
...
@@ -312,7 +317,7 @@ impl PagesProject {
self
.project
()
.client_compile_time_info
(),
self
.project
()
.client_compile_time_info
(),
self
.client_module_options_context
(),
self
.client_module_options_context
(),
self
.client_resolve_options_context
(),
self
.client_resolve_options_context
(),
Vc
::
cell
(
"client"
.into
()
),
client_layer
(
),
)
)
}
}
...
@@ -351,7 +356,7 @@ impl PagesProject {
...
@@ -351,7 +356,7 @@ impl PagesProject {
self
.project
()
.client_compile_time_info
(),
self
.project
()
.client_compile_time_info
(),
self
.client_module_options_context
(),
self
.client_module_options_context
(),
self
.client_resolve_options_context
(),
self
.client_resolve_options_context
(),
Vc
::
cell
(
"client"
.into
()
),
client_layer
(
),
))
))
}
}
...
...
crates/next-api/src/project.rs
View file @
e431f9e6
...
@@ -34,7 +34,8 @@ use turbo_tasks::{
...
@@ -34,7 +34,8 @@ use turbo_tasks::{
use
turbo_tasks_env
::{
EnvMap
,
ProcessEnv
};
use
turbo_tasks_env
::{
EnvMap
,
ProcessEnv
};
use
turbo_tasks_fs
::{
DiskFileSystem
,
FileSystem
,
FileSystemPath
,
VirtualFileSystem
};
use
turbo_tasks_fs
::{
DiskFileSystem
,
FileSystem
,
FileSystemPath
,
VirtualFileSystem
};
use
turbopack
::{
use
turbopack
::{
evaluate_context
::
node_build_environment
,
transition
::
TransitionOptions
,
ModuleAssetContext
,
evaluate_context
::
node_build_environment
,
global_module_ids
::
get_global_module_id_strategy
,
transition
::
TransitionOptions
,
ModuleAssetContext
,
};
};
use
turbopack_core
::{
use
turbopack_core
::{
changed
::
content_changed
,
changed
::
content_changed
,
...
@@ -68,7 +69,6 @@ use crate::{
...
@@ -68,7 +69,6 @@ use crate::{
build
,
build
,
empty
::
EmptyEndpoint
,
empty
::
EmptyEndpoint
,
entrypoints
::
Entrypoints
,
entrypoints
::
Entrypoints
,
global_module_id_strategy
::
GlobalModuleIdStrategyBuilder
,
instrumentation
::
InstrumentationEndpoint
,
instrumentation
::
InstrumentationEndpoint
,
middleware
::
MiddlewareEndpoint
,
middleware
::
MiddlewareEndpoint
,
pages
::
PagesProject
,
pages
::
PagesProject
,
...
@@ -1576,16 +1576,25 @@ impl Project {
...
@@ -1576,16 +1576,25 @@ impl Project {
/// Gets the module id strategy for the project.
/// Gets the module id strategy for the project.
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
pub
async
fn
module_id_strategy
(
self
:
Vc
<
Self
>
)
->
Result
<
Vc
<
Box
<
dyn
ModuleIdStrategy
>>>
{
pub
async
fn
module_id_strategy
(
self
:
Vc
<
Self
>
)
->
Result
<
Vc
<
Box
<
dyn
ModuleIdStrategy
>>>
{
let
module_id_strategy
=
self
.next_config
()
.module_id_strategy_config
();
let
module_id_strategy
=
if
let
Some
(
module_id_strategy
)
=
match
*
module_id_strategy
.await
?
{
&*
self
.next_config
()
.module_id_strategy_config
()
.await
?
Some
(
ModuleIdStrategyConfig
::
Named
)
=>
Ok
(
Vc
::
upcast
(
DevModuleIdStrategy
::
new
())),
{
Some
(
ModuleIdStrategyConfig
::
Deterministic
)
=>
{
*
module_id_strategy
Ok
(
Vc
::
upcast
(
GlobalModuleIdStrategyBuilder
::
build
(
self
)))
}
else
{
}
match
*
self
.next_mode
()
.await
?
{
None
=>
match
*
self
.next_mode
()
.await
?
{
NextMode
::
Development
=>
ModuleIdStrategyConfig
::
Named
,
NextMode
::
Development
=>
Ok
(
Vc
::
upcast
(
DevModuleIdStrategy
::
new
())),
NextMode
::
Build
=>
ModuleIdStrategyConfig
::
Deterministic
,
NextMode
::
Build
=>
Ok
(
Vc
::
upcast
(
DevModuleIdStrategy
::
new
())),
}
},
};
match
module_id_strategy
{
ModuleIdStrategyConfig
::
Named
=>
Ok
(
Vc
::
upcast
(
DevModuleIdStrategy
::
new
())),
ModuleIdStrategyConfig
::
Deterministic
=>
{
let
module_graphs
=
self
.whole_app_module_graphs
()
.await
?
;
Ok
(
Vc
::
upcast
(
get_global_module_id_strategy
(
*
module_graphs
.full
,
)))
}
}
}
}
}
}
}
...
...
crates/next-core/src/next_config.rs
View file @
e431f9e6
...
@@ -567,7 +567,7 @@ pub enum LoaderItem {
...
@@ -567,7 +567,7 @@ pub enum LoaderItem {
}
}
#[turbo_tasks
::
value(operation)]
#[turbo_tasks
::
value(operation)]
#[derive(Clone,
Debug)]
#[derive(C
opy,
C
lone,
Debug)]
#[serde(rename_all
=
"camelCase"
)]
#[serde(rename_all
=
"camelCase"
)]
pub
enum
ModuleIdStrategy
{
pub
enum
ModuleIdStrategy
{
Named
,
Named
,
...
@@ -1483,11 +1483,11 @@ impl NextConfig {
...
@@ -1483,11 +1483,11 @@ impl NextConfig {
.experimental
.experimental
.turbo
.turbo
.as_ref
()
.as_ref
()
.and_then
(|
t
|
t
.module_id_strategy
.as_ref
()
)
.and_then
(|
t
|
t
.module_id_strategy
)
else
{
else
{
return
Vc
::
cell
(
None
);
return
Vc
::
cell
(
None
);
};
};
Vc
::
cell
(
Some
(
module_id_strategy
.clone
()
))
Vc
::
cell
(
Some
(
module_id_strategy
))
}
}
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
...
...
crates/next-core/src/next_server_component/server_component_module.rs
View file @
e431f9e6
...
@@ -9,7 +9,7 @@ use turbopack_core::{
...
@@ -9,7 +9,7 @@ use turbopack_core::{
asset
::{
Asset
,
AssetContent
},
asset
::{
Asset
,
AssetContent
},
chunk
::{
ChunkItem
,
ChunkItemExt
,
ChunkType
,
ChunkableModule
,
ChunkingContext
},
chunk
::{
ChunkItem
,
ChunkItemExt
,
ChunkType
,
ChunkableModule
,
ChunkingContext
},
ident
::
AssetIdent
,
ident
::
AssetIdent
,
module
::
{
Module
,
Modules
}
,
module
::
Module
,
module_graph
::
ModuleGraph
,
module_graph
::
ModuleGraph
,
reference
::
ModuleReferences
,
reference
::
ModuleReferences
,
};
};
...
@@ -18,17 +18,11 @@ use turbopack_ecmascript::{
...
@@ -18,17 +18,11 @@ use turbopack_ecmascript::{
EcmascriptChunkItem
,
EcmascriptChunkItemContent
,
EcmascriptChunkPlaceable
,
EcmascriptChunkItem
,
EcmascriptChunkItemContent
,
EcmascriptChunkPlaceable
,
EcmascriptChunkType
,
EcmascriptExports
,
EcmascriptChunkType
,
EcmascriptExports
,
},
},
references
::{
references
::
esm
::{
EsmExport
,
EsmExports
},
esm
::{
EsmExport
,
EsmExports
},
external_module
::
IncludeIdentModule
,
},
utils
::
StringifyJs
,
utils
::
StringifyJs
,
};
};
use
super
::
server_component_reference
::
NextServerComponentModuleReference
;
use
super
::
server_component_reference
::
NextServerComponentModuleReference
;
use
crate
::
next_app
::
app_client_references_chunks
::{
client_modules_modifier
,
ssr_modules_modifier
,
};
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
fn
modifier
()
->
Vc
<
RcStr
>
{
fn
modifier
()
->
Vc
<
RcStr
>
{
...
@@ -68,22 +62,6 @@ impl Module for NextServerComponentModule {
...
@@ -68,22 +62,6 @@ impl Module for NextServerComponentModule {
.await
?
,
.await
?
,
)]))
)]))
}
}
#[turbo_tasks
::
function]
async
fn
additional_layers_modules
(
self
:
Vc
<
Self
>
)
->
Result
<
Vc
<
Modules
>>
{
let
base_ident
=
self
.ident
();
let
ssr_entry_module
=
ResolvedVc
::
upcast
(
IncludeIdentModule
::
new
(
base_ident
.with_modifier
(
ssr_modules_modifier
()))
.to_resolved
()
.await
?
,
);
let
client_entry_module
=
ResolvedVc
::
upcast
(
IncludeIdentModule
::
new
(
base_ident
.with_modifier
(
client_modules_modifier
()))
.to_resolved
()
.await
?
,
);
Ok
(
Vc
::
cell
(
vec!
[
ssr_entry_module
,
client_entry_module
]))
}
}
}
#[turbo_tasks
::
value_impl]
#[turbo_tasks
::
value_impl]
...
...
test/production/pages-dir/production/test/index.test.ts
View file @
e431f9e6
...
@@ -580,16 +580,19 @@ describe('Production Usage', () => {
...
@@ -580,16 +580,19 @@ describe('Production Usage', () => {
const
resources
:
Set
<
string
>
=
new
Set
()
const
resources
:
Set
<
string
>
=
new
Set
()
const
manifestKey
=
Object
.
keys
(
reactLoadableManifest
).
find
((
item
)
=>
{
let
manifestKey
:
string
return
item
if
(
process
.
env
.
TURBOPACK
)
{
// the key is an arbitrary and changing number for Turbopack prod, but each page has its own manifest
expect
(
Object
.
keys
(
reactLoadableManifest
).
length
).
toBe
(
1
)
manifestKey
=
Object
.
keys
(
reactLoadableManifest
)[
0
]
expect
(
manifestKey
).
toBeString
()
}
else
{
manifestKey
=
Object
.
keys
(
reactLoadableManifest
).
find
((
item
)
=>
item
.
replace
(
/
\\
/g
,
'/'
)
.
replace
(
/
\\
/g
,
'/'
)
.
endsWith
(
.
endsWith
(
'dynamic/css.js -> ../../components/dynamic-css/with-css'
)
process
.
env
.
TURBOPACK
?
'components/dynamic-css/with-css.js [client] (ecmascript, next/dynamic entry)'
:
'dynamic/css.js -> ../../components/dynamic-css/with-css'
)
)
})
}
expect
(
manifestKey
).
toBeString
()
// test dynamic chunk
// test dynamic chunk
reactLoadableManifest
[
manifestKey
].
files
.
forEach
((
f
)
=>
{
reactLoadableManifest
[
manifestKey
].
files
.
forEach
((
f
)
=>
{
...
...
turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs
View file @
e431f9e6
use
anyhow
::
Result
;
use
anyhow
::
{
bail
,
Result
}
;
use
turbo_rcstr
::
RcStr
;
use
rustc_hash
::
FxHashMap
;
use
turbo_tasks
::{
FxIndexMap
,
ResolvedVc
,
ValueToString
,
Vc
};
use
turbo_tasks
::{
ResolvedVc
,
ValueToString
,
Vc
};
use
turbo_tasks_hash
::
hash_xxh3_hash64
;
use
turbo_tasks_hash
::
hash_xxh3_hash64
;
use
super
::
ModuleId
;
use
super
::
ModuleId
;
use
crate
::
ident
::
AssetIdent
;
use
crate
::{
ident
::
AssetIdent
,
issue
::{
module
::
ModuleIssue
,
IssueExt
,
StyledString
},
};
#[turbo_tasks
::
value_trait]
#[turbo_tasks
::
value_trait]
pub
trait
ModuleIdStrategy
{
pub
trait
ModuleIdStrategy
{
...
@@ -32,25 +35,43 @@ impl ModuleIdStrategy for DevModuleIdStrategy {
...
@@ -32,25 +35,43 @@ impl ModuleIdStrategy for DevModuleIdStrategy {
}
}
}
}
#[turbo_tasks
::
value]
#[turbo_tasks
::
value
(shared)
]
pub
struct
GlobalModuleIdStrategy
{
pub
struct
GlobalModuleIdStrategy
{
module_id_map
:
FxIndexMap
<
RcStr
,
ModuleId
>
,
pub
module_id_map
:
FxHashMap
<
ResolvedVc
<
AssetIdent
>
,
u64
>
,
}
impl
GlobalModuleIdStrategy
{
pub
async
fn
new
(
module_id_map
:
FxIndexMap
<
RcStr
,
ModuleId
>
)
->
Result
<
Vc
<
Self
>>
{
Ok
(
GlobalModuleIdStrategy
{
module_id_map
}
.cell
())
}
}
}
#[turbo_tasks
::
value_impl]
#[turbo_tasks
::
value_impl]
impl
ModuleIdStrategy
for
GlobalModuleIdStrategy
{
impl
ModuleIdStrategy
for
GlobalModuleIdStrategy
{
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
async
fn
get_module_id
(
&
self
,
ident
:
Vc
<
AssetIdent
>
)
->
Result
<
Vc
<
ModuleId
>>
{
async
fn
get_module_id
(
&
self
,
ident
:
ResolvedVc
<
AssetIdent
>
)
->
Result
<
Vc
<
ModuleId
>>
{
let
ident_string
=
ident
.to_string
()
.await
?
.clone_value
();
if
let
Some
(
module_id
)
=
self
.module_id_map
.get
(
&
ident
)
{
if
let
Some
(
module_id
)
=
self
.module_id_map
.get
(
&
ident_string
)
{
const
JS_MAX_SAFE_INTEGER
:
u64
=
(
1u64
<<
53
)
-
1
;
return
Ok
(
module_id
.clone
()
.cell
());
if
*
module_id
>
JS_MAX_SAFE_INTEGER
{
bail!
(
"Numeric module id is too large: {}"
,
module_id
);
}
return
Ok
(
ModuleId
::
Number
(
*
module_id
)
.cell
());
}
let
ident_string
=
ident
.to_string
()
.await
?
;
if
!
ident_string
.ends_with
(
"[app-client] (ecmascript, next/dynamic entry)"
)
{
// TODO: This shouldn't happen, but is a temporary workaround to ignore next/dynamic
// imports of a server component from another server component.
ModuleIssue
{
ident
,
title
:
StyledString
::
Text
(
format!
(
"ModuleId not found for ident: {:?}"
,
ident_string
)
.into
(),
)
.resolved_cell
(),
description
:
StyledString
::
Text
(
format!
(
"ModuleId not found for ident: {:?}"
,
ident_string
)
.into
(),
)
.resolved_cell
(),
}
.resolved_cell
()
.emit
();
}
}
Ok
(
ModuleId
::
String
(
Ok
(
ModuleId
::
String
(
hash_xxh3_hash64
(
ident
.to_string
()
.await
?
)
hash_xxh3_hash64
(
ident
.to_string
()
.await
?
)
.to_string
()
.to_string
()
...
...
turbopack/crates/turbopack-core/src/file_source.rs
View file @
e431f9e6
...
@@ -28,8 +28,15 @@ impl FileSource {
...
@@ -28,8 +28,15 @@ impl FileSource {
}
}
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
pub
fn
new_with_query
(
path
:
ResolvedVc
<
FileSystemPath
>
,
query
:
ResolvedVc
<
RcStr
>
)
->
Vc
<
Self
>
{
pub
async
fn
new_with_query
(
Self
::
cell
(
FileSource
{
path
,
query
})
path
:
ResolvedVc
<
FileSystemPath
>
,
query
:
ResolvedVc
<
RcStr
>
,
)
->
Result
<
Vc
<
Self
>>
{
if
query
.await
?
.is_empty
()
{
Ok
(
Self
::
new
(
*
path
))
}
else
{
Ok
(
Self
::
cell
(
FileSource
{
path
,
query
}))
}
}
}
}
}
...
...
turbopack/crates/turbopack-core/src/module.rs
View file @
e431f9e6
...
@@ -16,10 +16,6 @@ pub trait Module: Asset {
...
@@ -16,10 +16,6 @@ pub trait Module: Asset {
ModuleReferences
::
empty
()
ModuleReferences
::
empty
()
}
}
fn
additional_layers_modules
(
self
:
Vc
<
Self
>
)
->
Vc
<
Modules
>
{
Vc
::
cell
(
vec!
[])
}
/// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc.
/// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc.
fn
is_self_async
(
self
:
Vc
<
Self
>
)
->
Vc
<
bool
>
{
fn
is_self_async
(
self
:
Vc
<
Self
>
)
->
Vc
<
bool
>
{
Vc
::
cell
(
false
)
Vc
::
cell
(
false
)
...
...
turbopack/crates/turbopack-core/src/module_graph/mod.rs
View file @
e431f9e6
...
@@ -731,6 +731,41 @@ impl ModuleGraph {
...
@@ -731,6 +731,41 @@ impl ModuleGraph {
Ok
(())
Ok
(())
}
}
/// Traverses all edges exactly once (in an unspecified order) and calls the visitor with the
/// edge source and target.
///
/// This means that target nodes can be revisited (once per incoming edge).
///
/// * `visitor` - Called before visiting the children of a node.
/// - Receives (originating &SingleModuleGraphNode, edge &ChunkingType), target
/// &SingleModuleGraphNode
pub
async
fn
traverse_all_edges_unordered
(
&
self
,
mut
visitor
:
impl
FnMut
(
(
&
'_
SingleModuleGraphModuleNode
,
&
'_
ChunkingType
),
&
'_
SingleModuleGraphModuleNode
,
)
->
Result
<
()
>
,
)
->
Result
<
()
>
{
let
graphs
=
self
.get_graphs
()
.await
?
;
for
graph
in
&
graphs
{
let
graph
=
&
graph
.graph
;
for
edge
in
graph
.edge_references
()
{
let
source
=
match
graph
.node_weight
(
edge
.source
())
.unwrap
()
{
SingleModuleGraphNode
::
Module
(
node
)
=>
node
,
SingleModuleGraphNode
::
VisitedModule
{
..
}
=>
unreachable!
(),
};
let
target
=
match
graph
.node_weight
(
edge
.target
())
.unwrap
()
{
SingleModuleGraphNode
::
Module
(
node
)
=>
node
,
SingleModuleGraphNode
::
VisitedModule
{
idx
}
=>
get_node!
(
graphs
,
idx
),
};
visitor
((
source
,
edge
.weight
()),
target
)
?
;
}
}
Ok
(())
}
/// Traverses all reachable edges in topological order. The preorder visitor can be used to
/// Traverses all reachable edges in topological order. The preorder visitor can be used to
/// forward state down the graph, and to skip subgraphs
/// forward state down the graph, and to skip subgraphs
///
///
...
...
turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs
View file @
e431f9e6
...
@@ -19,7 +19,7 @@ use crate::{
...
@@ -19,7 +19,7 @@ use crate::{
data
::
EcmascriptChunkData
,
EcmascriptChunkItem
,
EcmascriptChunkItemContent
,
data
::
EcmascriptChunkData
,
EcmascriptChunkItem
,
EcmascriptChunkItemContent
,
EcmascriptChunkPlaceable
,
EcmascriptChunkType
,
EcmascriptChunkPlaceable
,
EcmascriptChunkType
,
},
},
utils
::
StringifyJs
,
utils
::
{
StringifyJs
,
StringifyModuleId
}
,
};
};
#[turbo_tasks
::
value(shared)]
#[turbo_tasks
::
value(shared)]
...
@@ -102,7 +102,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
...
@@ -102,7 +102,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
}});
}});
}});
}});
"
#
,
"
#
,
id
=
Stringify
Js
(
id
),
id
=
Stringify
ModuleId
(
id
),
}
}
}
}
(
Some
(
id
),
false
)
=>
{
(
Some
(
id
),
false
)
=>
{
...
@@ -115,7 +115,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
...
@@ -115,7 +115,7 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
}});
}});
"
#
,
"
#
,
chunks
=
StringifyJs
(
&
chunks_data
),
chunks
=
StringifyJs
(
&
chunks_data
),
id
=
Stringify
Js
(
id
),
id
=
Stringify
ModuleId
(
id
),
}
}
}
}
(
None
,
true
)
=>
{
(
None
,
true
)
=>
{
...
...
turbopack/crates/turbopack-ecmascript/src/async_chunk/module.rs
View file @
e431f9e6
...
@@ -13,7 +13,7 @@ use turbopack_core::{
...
@@ -13,7 +13,7 @@ use turbopack_core::{
use
crate
::
async_chunk
::
chunk_item
::
AsyncLoaderChunkItem
;
use
crate
::
async_chunk
::
chunk_item
::
AsyncLoaderChunkItem
;
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
fn
modifier
()
->
Vc
<
RcStr
>
{
fn
async_loader_
modifier
()
->
Vc
<
RcStr
>
{
Vc
::
cell
(
"async loader"
.into
())
Vc
::
cell
(
"async loader"
.into
())
}
}
...
@@ -43,7 +43,7 @@ impl AsyncLoaderModule {
...
@@ -43,7 +43,7 @@ impl AsyncLoaderModule {
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
pub
fn
asset_ident_for
(
module
:
Vc
<
Box
<
dyn
ChunkableModule
>>
)
->
Vc
<
AssetIdent
>
{
pub
fn
asset_ident_for
(
module
:
Vc
<
Box
<
dyn
ChunkableModule
>>
)
->
Vc
<
AssetIdent
>
{
module
.ident
()
.with_modifier
(
modifier
())
module
.ident
()
.with_modifier
(
async_loader_
modifier
())
}
}
}
}
...
...
turbopack/crates/turbopack-ecmascript/src/global_module_id_strategy.rs
deleted
100644 → 0
View file @
bc3e597a
use
anyhow
::
Result
;
use
turbo_rcstr
::
RcStr
;
use
turbo_tasks
::{
graph
::{
AdjacencyMap
,
GraphTraversal
},
FxIndexMap
,
FxIndexSet
,
ResolvedVc
,
TryJoinIterExt
,
ValueToString
,
Vc
,
};
use
turbo_tasks_hash
::
hash_xxh3_hash64
;
use
turbopack_core
::{
chunk
::
ModuleId
,
module
::{
Module
,
Modules
},
reference
::
ModuleReference
,
};
use
crate
::
references
::
esm
::
EsmAsyncAssetReference
;
#[turbo_tasks
::
value]
pub
struct
PreprocessedChildrenIdents
{
// ident.to_string() -> full hash
// We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids`
// if this endpoint did not change.
modules_idents
:
FxIndexMap
<
RcStr
,
u64
>
,
}
#[derive(Clone,
Hash)]
#[turbo_tasks
::
value(shared)]
pub
enum
ReferencedModule
{
Module
(
ResolvedVc
<
Box
<
dyn
Module
>>
),
AsyncLoaderModule
(
ResolvedVc
<
Box
<
dyn
Module
>>
),
}
impl
ReferencedModule
{
fn
module
(
&
self
)
->
Vc
<
Box
<
dyn
Module
>>
{
match
*
self
{
ReferencedModule
::
Module
(
module
)
=>
*
module
,
ReferencedModule
::
AsyncLoaderModule
(
module
)
=>
*
module
,
}
}
}
#[turbo_tasks
::
value(transparent)]
pub
struct
ReferencedModules
(
Vec
<
ResolvedVc
<
ReferencedModule
>>
);
#[turbo_tasks
::
function]
async
fn
referenced_modules
(
module
:
Vc
<
Box
<
dyn
Module
>>
)
->
Result
<
Vc
<
ReferencedModules
>>
{
let
references
=
module
.references
()
.await
?
;
// TODO(LichuAcu): Reduce type complexity
#[allow(clippy
::
type_complexity)]
type
ModulesAndAsyncLoaders
=
Vec
<
(
Vec
<
ResolvedVc
<
Box
<
dyn
Module
>>>
,
Option
<
ResolvedVc
<
Box
<
dyn
Module
>>>
,
)
>
;
let
modules_and_async_loaders
:
ModulesAndAsyncLoaders
=
references
.iter
()
.map
(|
reference
|
async
move
{
let
async_loader
=
if
ResolvedVc
::
try_downcast_type
::
<
EsmAsyncAssetReference
>
(
*
reference
)
.await
?
.is_some
()
{
*
reference
.resolve_reference
()
.resolve
()
.await
?
.first_module
()
.await
?
}
else
{
None
};
let
modules
=
reference
.resolve_reference
()
.resolve
()
.await
?
.primary_modules
()
.await
?
.clone_value
();
Ok
((
modules
,
async_loader
))
})
.try_join
()
.await
?
;
let
mut
set
=
FxIndexSet
::
default
();
let
mut
modules
=
Vec
::
new
();
for
(
module_list
,
async_loader
)
in
modules_and_async_loaders
{
for
module
in
module_list
{
if
set
.insert
(
module
)
{
modules
.push
(
ReferencedModule
::
Module
(
module
)
.resolved_cell
());
}
}
if
let
Some
(
async_loader_module
)
=
async_loader
{
if
set
.insert
(
async_loader_module
)
{
modules
.push
(
ReferencedModule
::
AsyncLoaderModule
(
async_loader_module
)
.resolved_cell
());
}
}
}
Ok
(
Vc
::
cell
(
modules
))
}
pub
async
fn
get_children_modules
(
parent
:
ResolvedVc
<
ReferencedModule
>
,
)
->
Result
<
impl
Iterator
<
Item
=
ResolvedVc
<
ReferencedModule
>>
+
Send
>
{
let
parent_module
=
parent
.await
?
.module
();
let
mut
modules
=
referenced_modules
(
parent_module
)
.await
?
.clone_value
();
for
module
in
parent_module
.additional_layers_modules
()
.await
?
{
modules
.push
(
ReferencedModule
::
Module
(
*
module
)
.resolved_cell
());
}
Ok
(
modules
.into_iter
())
}
// NOTE(LichuAcu) Called on endpoint.root_modules(). It would probably be better if this was called
// directly on `Endpoint`, but such struct is not available in turbopack-core. The whole function
// could be moved to `next-api`, but it would require adding turbo-tasks-hash to `next-api`,
// making it heavier.
#[turbo_tasks
::
function]
pub
async
fn
children_modules_idents
(
root_modules
:
Vc
<
Modules
>
,
)
->
Result
<
Vc
<
PreprocessedChildrenIdents
>>
{
let
children_modules_iter
=
AdjacencyMap
::
new
()
.skip_duplicates
()
.visit
(
root_modules
.await
?
.iter
()
.map
(|
module
|
ReferencedModule
::
Module
(
*
module
)
.resolved_cell
())
.collect
::
<
Vec
<
_
>>
(),
get_children_modules
,
)
.await
.completed
()
?
.into_inner
()
.into_reverse_topological
();
// module_id -> full hash
let
mut
modules_idents
=
FxIndexMap
::
default
();
for
child_module
in
children_modules_iter
{
match
*
child_module
.await
?
{
ReferencedModule
::
Module
(
module
)
=>
{
let
module_ident
=
module
.ident
();
let
ident_str
=
module_ident
.to_string
()
.await
?
.clone_value
();
let
hash
=
hash_xxh3_hash64
(
&
ident_str
);
modules_idents
.insert
(
ident_str
,
hash
);
}
ReferencedModule
::
AsyncLoaderModule
(
async_loader_module
)
=>
{
let
loader_ident
=
async_loader_module
.ident
()
.with_modifier
(
Vc
::
cell
(
"async loader"
.into
()));
let
loader_ident_str
=
loader_ident
.to_string
()
.await
?
.clone_value
();
let
loader_hash
=
hash_xxh3_hash64
(
&
loader_ident_str
);
modules_idents
.insert
(
loader_ident_str
,
loader_hash
);
let
loaded_client_ident
=
async_loader_module
.ident
()
.with_layer
(
Vc
::
cell
(
"app-client"
.into
()));
let
loaded_client_ident_str
=
loaded_client_ident
.to_string
()
.await
?
.clone_value
();
let
loaded_client_hash
=
hash_xxh3_hash64
(
&
loaded_client_ident_str
);
modules_idents
.insert
(
loaded_client_ident_str
,
loaded_client_hash
);
}
}
}
Ok
(
PreprocessedChildrenIdents
{
modules_idents
}
.cell
())
}
const
JS_MAX_SAFE_INTEGER
:
u64
=
(
1u64
<<
53
)
-
1
;
// Note(LichuAcu): This could be split into two functions: one that merges the preprocessed module
// ids and another that generates the final, optimized module ids. Thoughts?
pub
async
fn
merge_preprocessed_module_ids
(
preprocessed_module_ids
:
Vec
<
Vc
<
PreprocessedChildrenIdents
>>
,
)
->
Result
<
FxIndexMap
<
RcStr
,
ModuleId
>>
{
let
mut
merged_module_ids
=
FxIndexMap
::
default
();
for
preprocessed_module_ids
in
preprocessed_module_ids
{
for
(
module_ident
,
full_hash
)
in
&
preprocessed_module_ids
.await
?
.modules_idents
{
merged_module_ids
.insert
(
module_ident
.clone
(),
*
full_hash
);
}
}
// 5% fill rate, as done in Webpack
// https://github.com/webpack/webpack/blob/27cf3e59f5f289dfc4d76b7a1df2edbc4e651589/lib/ids/IdHelpers.js#L366-L405
let
optimal_range
=
merged_module_ids
.len
()
*
20
;
let
digit_mask
=
std
::
cmp
::
min
(
10u64
.pow
((
optimal_range
as
f64
)
.log10
()
.ceil
()
as
u32
),
JS_MAX_SAFE_INTEGER
,
);
let
mut
module_id_map
=
FxIndexMap
::
default
();
let
mut
used_ids
=
FxIndexSet
::
default
();
for
(
module_ident
,
full_hash
)
in
merged_module_ids
.iter
()
{
let
mut
trimmed_hash
=
full_hash
%
digit_mask
;
let
mut
i
=
1
;
while
used_ids
.contains
(
&
trimmed_hash
)
{
// If the id is already used, seek to find another available id.
trimmed_hash
=
hash_xxh3_hash64
(
full_hash
+
i
)
%
digit_mask
;
i
+=
1
;
}
used_ids
.insert
(
trimmed_hash
);
module_id_map
.insert
(
module_ident
.clone
(),
ModuleId
::
Number
(
trimmed_hash
));
}
Ok
(
module_id_map
)
}
turbopack/crates/turbopack-ecmascript/src/lib.rs
View file @
e431f9e6
...
@@ -16,7 +16,6 @@ pub mod chunk;
...
@@ -16,7 +16,6 @@ pub mod chunk;
pub
mod
chunk_group_files_asset
;
pub
mod
chunk_group_files_asset
;
pub
mod
code_gen
;
pub
mod
code_gen
;
mod
errors
;
mod
errors
;
pub
mod
global_module_id_strategy
;
pub
mod
magic_identifier
;
pub
mod
magic_identifier
;
pub
mod
manifest
;
pub
mod
manifest
;
pub
mod
minify
;
pub
mod
minify
;
...
@@ -253,6 +252,19 @@ pub struct EcmascriptModuleAsset {
...
@@ -253,6 +252,19 @@ pub struct EcmascriptModuleAsset {
#[turbo_tasks(debug_ignore)]
#[turbo_tasks(debug_ignore)]
last_successful_parse
:
turbo_tasks
::
TransientState
<
ReadRef
<
ParseResult
>>
,
last_successful_parse
:
turbo_tasks
::
TransientState
<
ReadRef
<
ParseResult
>>
,
}
}
impl
core
::
fmt
::
Debug
for
EcmascriptModuleAsset
{
fn
fmt
(
&
self
,
f
:
&
mut
core
::
fmt
::
Formatter
)
->
core
::
fmt
::
Result
{
f
.debug_struct
(
"EcmascriptModuleAsset"
)
.field
(
"source"
,
&
self
.source
)
.field
(
"asset_context"
,
&
self
.asset_context
)
.field
(
"ty"
,
&
self
.ty
)
.field
(
"transforms"
,
&
self
.transforms
)
.field
(
"options"
,
&
self
.options
)
.field
(
"compile_time_info"
,
&
self
.compile_time_info
)
.field
(
"inner_assets"
,
&
self
.inner_assets
)
.finish
()
}
}
#[turbo_tasks
::
value_trait]
#[turbo_tasks
::
value_trait]
pub
trait
EcmascriptParsable
{
pub
trait
EcmascriptParsable
{
...
...
turbopack/crates/turbopack-ecmascript/src/manifest/loader_item.rs
View file @
e431f9e6
...
@@ -20,7 +20,7 @@ use crate::{
...
@@ -20,7 +20,7 @@ use crate::{
data
::
EcmascriptChunkData
,
EcmascriptChunkItem
,
EcmascriptChunkItemContent
,
data
::
EcmascriptChunkData
,
EcmascriptChunkItem
,
EcmascriptChunkItemContent
,
EcmascriptChunkPlaceable
,
EcmascriptChunkType
,
EcmascriptChunkPlaceable
,
EcmascriptChunkType
,
},
},
utils
::
StringifyJs
,
utils
::
{
StringifyJs
,
StringifyModuleId
}
,
};
};
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
...
@@ -197,8 +197,8 @@ impl EcmascriptChunkItem for ManifestLoaderChunkItem {
...
@@ -197,8 +197,8 @@ impl EcmascriptChunkItem for ManifestLoaderChunkItem {
.map
(|
chunk_data
|
EcmascriptChunkData
::
new
(
chunk_data
))
.map
(|
chunk_data
|
EcmascriptChunkData
::
new
(
chunk_data
))
.collect
::
<
Vec
<
_
>>
()
.collect
::
<
Vec
<
_
>>
()
),
),
item_id
=
Stringify
Js
(
item_id
),
item_id
=
Stringify
ModuleId
(
item_id
),
dynamic_id
=
Stringify
Js
(
dynamic_id
),
dynamic_id
=
Stringify
ModuleId
(
dynamic_id
),
)
?
;
)
?
;
Ok
(
EcmascriptChunkItemContent
{
Ok
(
EcmascriptChunkItemContent
{
...
...
turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs
View file @
e431f9e6
use
anyhow
::
Result
;
use
anyhow
::
Result
;
use
turbo_tasks
::{
ResolvedVc
,
ValueDefault
,
V
alueToString
,
V
c
};
use
turbo_tasks
::{
ResolvedVc
,
ValueDefault
,
Vc
};
use
turbo_tasks_fs
::
rope
::
RopeBuilder
;
use
turbo_tasks_fs
::
rope
::
RopeBuilder
;
use
turbopack_core
::{
use
turbopack_core
::{
chunk
::{
AsyncModuleInfo
,
ChunkItem
,
ChunkType
,
ChunkingContext
},
chunk
::{
AsyncModuleInfo
,
ChunkItem
,
ChunkItemExt
,
ChunkType
,
ChunkableModule
,
ChunkingContext
,
},
ident
::
AssetIdent
,
ident
::
AssetIdent
,
module
::
Module
,
module
::
Module
,
module_graph
::
ModuleGraph
,
module_graph
::
ModuleGraph
,
...
@@ -16,7 +18,7 @@ use crate::{
...
@@ -16,7 +18,7 @@ use crate::{
},
},
references
::
async_module
::
AsyncModuleOptions
,
references
::
async_module
::
AsyncModuleOptions
,
tree_shake
::
side_effect_module
::
SideEffectsModule
,
tree_shake
::
side_effect_module
::
SideEffectsModule
,
utils
::
Stringify
Js
,
utils
::
Stringify
ModuleId
,
EcmascriptModuleContent
,
EcmascriptModuleContent
,
};
};
...
@@ -113,6 +115,7 @@ impl ChunkItem for EcmascriptModulePartChunkItem {
...
@@ -113,6 +115,7 @@ impl ChunkItem for EcmascriptModulePartChunkItem {
#[turbo_tasks
::
value(shared)]
#[turbo_tasks
::
value(shared)]
pub
(
super
)
struct
SideEffectsModuleChunkItem
{
pub
(
super
)
struct
SideEffectsModuleChunkItem
{
pub
module
:
ResolvedVc
<
SideEffectsModule
>
,
pub
module
:
ResolvedVc
<
SideEffectsModule
>
,
pub
module_graph
:
ResolvedVc
<
ModuleGraph
>
,
pub
chunking_context
:
ResolvedVc
<
Box
<
dyn
ChunkingContext
>>
,
pub
chunking_context
:
ResolvedVc
<
Box
<
dyn
ChunkingContext
>>
,
}
}
...
@@ -167,7 +170,12 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
...
@@ -167,7 +170,12 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
format!
(
format!
(
"{}__turbopack_import__({});
\n
"
,
"{}__turbopack_import__({});
\n
"
,
if
need_await
{
"await "
}
else
{
""
},
if
need_await
{
"await "
}
else
{
""
},
StringifyJs
(
&*
side_effect
.ident
()
.to_string
()
.await
?
)
StringifyModuleId
(
&*
side_effect
.as_chunk_item
(
*
self
.module_graph
,
*
self
.chunking_context
)
.id
()
.await
?
)
)
)
.as_bytes
(),
.as_bytes
(),
);
);
...
@@ -176,7 +184,13 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
...
@@ -176,7 +184,13 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
code
.push_bytes
(
code
.push_bytes
(
format!
(
format!
(
"__turbopack_export_namespace__(__turbopack_import__({}));
\n
"
,
"__turbopack_export_namespace__(__turbopack_import__({}));
\n
"
,
StringifyJs
(
&*
module
.resolved_as
.ident
()
.to_string
()
.await
?
)
StringifyModuleId
(
&*
module
.resolved_as
.as_chunk_item
(
*
self
.module_graph
,
*
self
.chunking_context
)
.id
()
.await
?
)
)
)
.as_bytes
(),
.as_bytes
(),
);
);
...
...
turbopack/crates/turbopack-ecmascript/src/tree_shake/side_effect_module.rs
View file @
e431f9e6
...
@@ -133,12 +133,13 @@ impl ChunkableModule for SideEffectsModule {
...
@@ -133,12 +133,13 @@ impl ChunkableModule for SideEffectsModule {
#[turbo_tasks
::
function]
#[turbo_tasks
::
function]
async
fn
as_chunk_item
(
async
fn
as_chunk_item
(
self
:
Vc
<
Self
>
,
self
:
Vc
<
Self
>
,
_
module_graph
:
ResolvedVc
<
ModuleGraph
>
,
module_graph
:
ResolvedVc
<
ModuleGraph
>
,
chunking_context
:
ResolvedVc
<
Box
<
dyn
ChunkingContext
>>
,
chunking_context
:
ResolvedVc
<
Box
<
dyn
ChunkingContext
>>
,
)
->
Result
<
Vc
<
Box
<
dyn
turbopack_core
::
chunk
::
ChunkItem
>>>
{
)
->
Result
<
Vc
<
Box
<
dyn
turbopack_core
::
chunk
::
ChunkItem
>>>
{
Ok
(
Vc
::
upcast
(
Ok
(
Vc
::
upcast
(
SideEffectsModuleChunkItem
{
SideEffectsModuleChunkItem
{
module
:
self
.to_resolved
()
.await
?
,
module
:
self
.to_resolved
()
.await
?
,
module_graph
,
chunking_context
,
chunking_context
,
}
}
.cell
(),
.cell
(),
...
...
turbopack/crates/turbopack-ecmascript/src/utils.rs
View file @
e431f9e6
...
@@ -78,6 +78,23 @@ pub fn module_id_to_lit(module_id: &ModuleId) -> Expr {
...
@@ -78,6 +78,23 @@ pub fn module_id_to_lit(module_id: &ModuleId) -> Expr {
})
})
}
}
pub
struct
StringifyModuleId
<
'a
>
(
pub
&
'a
ModuleId
);
impl
std
::
fmt
::
Display
for
StringifyModuleId
<
'_
>
{
fn
fmt
(
&
self
,
f
:
&
mut
std
::
fmt
::
Formatter
<
'_
>
)
->
std
::
fmt
::
Result
{
match
self
.
0
{
ModuleId
::
Number
(
n
)
=>
{
if
*
n
<=
JS_MAX_SAFE_INTEGER
{
n
.fmt
(
f
)
}
else
{
write!
(
f
,
"
\
"
{
n
}
\
""
)
}
}
ModuleId
::
String
(
s
)
=>
StringifyJs
(
s
)
.fmt
(
f
),
}
}
}
pub
struct
StringifyJs
<
'a
,
T
>
(
pub
&
'a
T
)
pub
struct
StringifyJs
<
'a
,
T
>
(
pub
&
'a
T
)
where
where
T
:
?
Sized
;
T
:
?
Sized
;
...
...
turbopack/crates/turbopack/Cargo.toml
View file @
e431f9e6
...
@@ -21,6 +21,8 @@ anyhow = { workspace = true }
...
@@ -21,6 +21,8 @@ anyhow = { workspace = true }
indexmap
=
{
workspace
=
true
,
features
=
["serde"]
}
indexmap
=
{
workspace
=
true
,
features
=
["serde"]
}
lazy_static
=
{
workspace
=
true
}
lazy_static
=
{
workspace
=
true
}
regex
=
{
workspace
=
true
}
regex
=
{
workspace
=
true
}
rustc-hash
=
{
workspace
=
true
}
smallvec
=
{
workspace
=
true
}
serde
=
{
workspace
=
true
}
serde
=
{
workspace
=
true
}
serde_json
=
{
workspace
=
true
}
serde_json
=
{
workspace
=
true
}
tokio
=
{
workspace
=
true
}
tokio
=
{
workspace
=
true
}
...
@@ -30,6 +32,7 @@ turbo-rcstr = { workspace = true }
...
@@ -30,6 +32,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks
=
{
workspace
=
true
}
turbo-tasks
=
{
workspace
=
true
}
turbo-tasks-env
=
{
workspace
=
true
}
turbo-tasks-env
=
{
workspace
=
true
}
turbo-tasks-fs
=
{
workspace
=
true
}
turbo-tasks-fs
=
{
workspace
=
true
}
turbo-tasks-hash
=
{
workspace
=
true
}
turbopack-core
=
{
workspace
=
true
}
turbopack-core
=
{
workspace
=
true
}
turbopack-css
=
{
workspace
=
true
}
turbopack-css
=
{
workspace
=
true
}
turbopack-ecmascript
=
{
workspace
=
true
}
turbopack-ecmascript
=
{
workspace
=
true
}
...
...
turbopack/crates/turbopack/src/global_module_ids.rs
0 → 100644
View file @
e431f9e6
use
anyhow
::{
Context
,
Result
};
use
rustc_hash
::
FxHashMap
;
use
smallvec
::
SmallVec
;
use
tracing
::
Instrument
;
use
turbo_rcstr
::
RcStr
;
use
turbo_tasks
::{
ReadRef
,
ResolvedVc
,
TryJoinIterExt
,
ValueToString
,
Vc
};
use
turbo_tasks_hash
::
hash_xxh3_hash64
;
use
turbopack_core
::{
chunk
::{
module_id_strategies
::
GlobalModuleIdStrategy
,
ChunkableModule
,
ChunkingType
},
ident
::
AssetIdent
,
module
::
Module
,
module_graph
::
ModuleGraph
,
};
use
turbopack_ecmascript
::
async_chunk
::
module
::
AsyncLoaderModule
;
#[turbo_tasks
::
function]
pub
async
fn
get_global_module_id_strategy
(
module_graph
:
ResolvedVc
<
ModuleGraph
>
,
)
->
Result
<
Vc
<
GlobalModuleIdStrategy
>>
{
let
span
=
tracing
::
info_span!
(
"compute module id map"
);
async
move
{
let
module_graph
=
module_graph
.await
?
;
let
graphs
=
module_graph
.graphs
.iter
()
.try_join
()
.await
?
;
// All modules in the graph
let
module_idents
=
graphs
.iter
()
.flat_map
(|
graph
|
graph
.iter_nodes
())
.map
(|
m
|
m
.module
.ident
());
// And additionally, all the modules that are inserted by chunking (i.e. async loaders)
let
mut
async_idents
=
vec!
[];
module_graph
.traverse_all_edges_unordered
(|
parent
,
current
|
{
if
let
(
_
,
&
ChunkingType
::
Async
)
=
parent
{
let
module
=
ResolvedVc
::
try_sidecast_sync
::
<
Box
<
dyn
ChunkableModule
>>
(
current
.module
)
.context
(
"expected chunkable module for async reference"
)
?
;
async_idents
.push
(
AsyncLoaderModule
::
asset_ident_for
(
*
module
));
}
Ok
(())
})
.await
?
;
let
mut
module_id_map
=
module_idents
.chain
(
async_idents
.into_iter
())
.map
(|
ident
|
async
move
{
let
ident
=
ident
.to_resolved
()
.await
?
;
let
ident_str
=
ident
.to_string
()
.await
?
;
let
hash
=
hash_xxh3_hash64
(
&
ident_str
);
Ok
((
ident
,
(
ident_str
,
hash
)))
})
.try_join
()
.await
?
.into_iter
()
.collect
::
<
FxHashMap
<
_
,
_
>>
();
finalize_module_ids
(
&
mut
module_id_map
);
Ok
(
GlobalModuleIdStrategy
{
module_id_map
:
module_id_map
.into_iter
()
.map
(|(
ident
,
(
_
,
hash
))|
(
ident
,
hash
))
.collect
(),
}
.cell
())
}
.instrument
(
span
)
.await
}
const
JS_MAX_SAFE_INTEGER
:
u64
=
(
1u64
<<
53
)
-
1
;
/// Shorten hashes and handle any collisions.
fn
finalize_module_ids
(
merged_module_ids
:
&
mut
FxHashMap
<
ResolvedVc
<
AssetIdent
>
,
(
ReadRef
<
RcStr
>
,
u64
)
>
,
)
{
// 5% fill rate, as done in Webpack
// https://github.com/webpack/webpack/blob/27cf3e59f5f289dfc4d76b7a1df2edbc4e651589/lib/ids/IdHelpers.js#L366-L405
let
optimal_range
=
merged_module_ids
.len
()
*
20
;
let
digit_mask
=
std
::
cmp
::
min
(
10u64
.pow
((
optimal_range
as
f64
)
.log10
()
.ceil
()
as
u32
),
JS_MAX_SAFE_INTEGER
,
);
let
mut
used_ids
=
FxHashMap
::
<
u64
,
SmallVec
<
[(
ResolvedVc
<
AssetIdent
>
,
ReadRef
<
RcStr
>
);
1
]
>>
::
default
();
// Run in multiple passes, to not depend on the order of the `merged_module_ids` (i.e. the order
// of imports). Hashes could still change if modules are added or removed.
// Find pass: shorten hashes, potentially causing (more) collisions
for
(
ident
,
(
ident_str
,
full_hash
))
in
merged_module_ids
.iter_mut
()
{
let
first_pass_hash
=
*
full_hash
%
digit_mask
;
used_ids
.entry
(
first_pass_hash
)
.or_default
()
.push
((
*
ident
,
ident_str
.clone
()));
*
full_hash
=
first_pass_hash
;
}
// Filter conflicts
let
mut
conflicting_hashes
=
used_ids
.iter
()
.filter
(|(
_
,
list
)|
(
list
.len
()
>
1
))
.map
(|(
hash
,
_
)|
*
hash
)
.collect
::
<
Vec
<
_
>>
();
conflicting_hashes
.sort
();
// Second pass over the conflicts to resolve them
for
hash
in
conflicting_hashes
.into_iter
()
{
let
list
=
used_ids
.get_mut
(
&
hash
)
.unwrap
();
// Take the vector but keep the (empty) entry, so that the "contains_key" check below works
let
mut
list
=
std
::
mem
::
take
(
list
);
list
.sort_by
(|
a
,
b
|
a
.
1
.cmp
(
&
b
.
1
));
// Skip the first one, one module can keep the original hash
for
(
ident
,
_
)
in
list
.into_iter
()
.skip
(
1
)
{
let
hash
=
&
mut
merged_module_ids
.get_mut
(
&
ident
)
.unwrap
()
.
1
;
// the original algorithm since all that runs in deterministic order now
let
mut
i
=
1
;
let
mut
trimmed_hash
;
loop
{
// If the id is already used, find the next available hash.
trimmed_hash
=
hash_xxh3_hash64
((
*
hash
,
i
))
%
digit_mask
;
if
!
used_ids
.contains_key
(
&
trimmed_hash
)
{
break
;
}
i
+=
1
;
}
// At this point, we don't care about the values anymore, just the keys
used_ids
.entry
(
trimmed_hash
)
.or_default
();
*
hash
=
trimmed_hash
;
}
}
}
turbopack/crates/turbopack/src/lib.rs
View file @
e431f9e6
...
@@ -8,6 +8,7 @@
...
@@ -8,6 +8,7 @@
#
!
[
feature
(
arbitrary_self_types_pointers
)]
#
!
[
feature
(
arbitrary_self_types_pointers
)]
pub
mod
evaluate_context
;
pub
mod
evaluate_context
;
pub
mod
global_module_ids
;
mod
graph
;
mod
graph
;
pub
mod
module_options
;
pub
mod
module_options
;
pub
mod
transition
;
pub
mod
transition
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment