Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
8d841c64
Commit
8d841c64
authored
Mar 05, 2026
by
Benjamin Woodruff
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
some cleanup
parent
6a63d1d7
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
27 additions
and
31 deletions
+27
-31
mod.rs
...opack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs
+19
-17
tx_cache.rs
.../crates/turbo-tasks-backend/src/database/lmdb/tx_cache.rs
+8
-14
No files found.
turbopack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs
View file @
8d841c64
use
std
::{
fs
::
create_dir_all
,
marker
::
PhantomData
,
path
::
Path
,
thread
::
available_parallelism
};
mod
extended_key
;
mod
tx_cache
;
use
std
::{
fs
::
create_dir_all
,
path
::
Path
,
thread
::
available_parallelism
};
use
anyhow
::{
Context
,
Result
};
use
anyhow
::{
Context
,
Result
};
use
lmdb
::{
use
lmdb
::{
...
@@ -7,26 +10,22 @@ use lmdb::{
...
@@ -7,26 +10,22 @@ use lmdb::{
use
crate
::
database
::{
use
crate
::
database
::{
key_value_database
::{
KeySpace
,
KeyValueDatabase
},
key_value_database
::{
KeySpace
,
KeyValueDatabase
},
lmdb
::
tx_cache
::{
ReadTxGuard
,
TxCache
,
WriteTxGuard
},
write_batch
::{
BaseWriteBatch
,
SerialWriteBatch
,
WriteBatch
,
WriteBuffer
},
write_batch
::{
BaseWriteBatch
,
SerialWriteBatch
,
WriteBatch
,
WriteBuffer
},
};
};
mod
extended_key
;
mod
read_tx_cache
;
pub
struct
LmdbValueBuffer
<
'l
>
{
pub
struct
LmdbValueBuffer
<
'l
>
{
ptr
:
*
const
u8
,
ptr
:
*
const
u8
,
len
:
usize
,
len
:
usize
,
_guard
:
read_tx_cache
::
ReadTxGuard
<
'l
>
,
_guard
:
ReadTxGuard
<
'l
>
,
_marker
:
PhantomData
<&
'l
LmbdKeyValueDatabase
>
,
}
}
impl
<
'l
>
LmdbValueBuffer
<
'l
>
{
impl
<
'l
>
LmdbValueBuffer
<
'l
>
{
fn
from_raw_parts
(
ptr
:
*
const
u8
,
len
:
usize
,
guard
:
read_tx_cache
::
ReadTxGuard
<
'l
>
)
->
Self
{
fn
from_raw_parts
(
ptr
:
*
const
u8
,
len
:
usize
,
guard
:
ReadTxGuard
<
'l
>
)
->
Self
{
Self
{
Self
{
ptr
,
ptr
,
len
,
len
,
_guard
:
guard
,
_guard
:
guard
,
_marker
:
PhantomData
,
}
}
}
}
}
}
...
@@ -40,8 +39,9 @@ impl AsRef<[u8]> for LmdbValueBuffer<'_> {
...
@@ -40,8 +39,9 @@ impl AsRef<[u8]> for LmdbValueBuffer<'_> {
}
}
pub
struct
LmbdKeyValueDatabase
{
pub
struct
LmbdKeyValueDatabase
{
// Safety: this cache must be dropped before `env`.
// Safety: this cache references data stored inside of `env`'s internals and must be dropped
read_tx_cache
:
read_tx_cache
::
ReadTxCache
,
// before `env`.
tx_cache
:
TxCache
,
env
:
Environment
,
env
:
Environment
,
infra_db
:
Database
,
infra_db
:
Database
,
data_db
:
Database
,
data_db
:
Database
,
...
@@ -58,13 +58,15 @@ impl LmbdKeyValueDatabase {
...
@@ -58,13 +58,15 @@ impl LmbdKeyValueDatabase {
#[cfg(not(target_arch
=
"x86"
))]
#[cfg(not(target_arch
=
"x86"
))]
const
MAP_SIZE
:
usize
=
40
*
1024
*
1024
*
1024
;
const
MAP_SIZE
:
usize
=
40
*
1024
*
1024
*
1024
;
let
max_read_tx_per_generation
=
available_parallelism
()
.map_or
(
4
,
|
v
|
v
.get
())
.max
(
8
);
let
env
=
Environment
::
new
()
let
env
=
Environment
::
new
()
.set_flags
(
.set_flags
(
EnvironmentFlags
::
WRITE_MAP
EnvironmentFlags
::
WRITE_MAP
|
EnvironmentFlags
::
NO_META_SYNC
|
EnvironmentFlags
::
NO_META_SYNC
// NO_TLS is needed for memory safety in `tx_cache`
|
EnvironmentFlags
::
NO_TLS
,
|
EnvironmentFlags
::
NO_TLS
,
)
)
.set_max_readers
(
(
available_parallelism
()
.map_or
(
16
,
|
v
|
v
.get
())
*
8
)
as
u32
)
.set_max_readers
(
max_read_tx_per_generation
as
u32
*
8
)
.set_max_dbs
(
4
)
.set_max_dbs
(
4
)
.set_map_size
(
MAP_SIZE
)
.set_map_size
(
MAP_SIZE
)
.open
(
path
)
?
;
.open
(
path
)
?
;
...
@@ -73,7 +75,7 @@ impl LmbdKeyValueDatabase {
...
@@ -73,7 +75,7 @@ impl LmbdKeyValueDatabase {
let
meta_db
=
env
.create_db
(
Some
(
"meta"
),
DatabaseFlags
::
INTEGER_KEY
)
?
;
let
meta_db
=
env
.create_db
(
Some
(
"meta"
),
DatabaseFlags
::
INTEGER_KEY
)
?
;
let
task_cache_db
=
env
.create_db
(
Some
(
"task_cache"
),
DatabaseFlags
::
empty
())
?
;
let
task_cache_db
=
env
.create_db
(
Some
(
"task_cache"
),
DatabaseFlags
::
empty
())
?
;
Ok
(
LmbdKeyValueDatabase
{
Ok
(
LmbdKeyValueDatabase
{
read_tx_cache
:
read_tx_cache
::
ReadTxCache
::
new
(
&
env
),
tx_cache
:
TxCache
::
new
(
&
env
,
max_read_tx_per_generation
),
env
,
env
,
infra_db
,
infra_db
,
data_db
,
data_db
,
...
@@ -91,12 +93,12 @@ impl LmbdKeyValueDatabase {
...
@@ -91,12 +93,12 @@ impl LmbdKeyValueDatabase {
}
}
}
}
fn
get_read_tx
(
&
self
)
->
read_tx_cache
::
ReadTxGuard
<
'_
>
{
fn
get_read_tx
(
&
self
)
->
ReadTxGuard
<
'_
>
{
self
.
read_
tx_cache
.get_read_tx
()
self
.tx_cache
.get_read_tx
()
}
}
fn
begin_write
(
&
self
)
->
read_tx_cache
::
WriteTxGuard
<
'_
>
{
fn
begin_write
(
&
self
)
->
WriteTxGuard
<
'_
>
{
self
.
read_
tx_cache
.begin_write
(
&
self
.env
)
self
.tx_cache
.begin_write
(
&
self
.env
)
}
}
fn
get_from_tx
<
'tx
>
(
fn
get_from_tx
<
'tx
>
(
...
@@ -163,7 +165,7 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
...
@@ -163,7 +165,7 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
pub
struct
LmbdWriteBatch
<
'l
>
{
pub
struct
LmbdWriteBatch
<
'l
>
{
tx
:
RwTransaction
<
'l
>
,
tx
:
RwTransaction
<
'l
>
,
_write_guard
:
Option
<
read_tx_cache
::
WriteTxGuard
<
'l
>>
,
_write_guard
:
Option
<
WriteTxGuard
<
'l
>>
,
this
:
&
'l
LmbdKeyValueDatabase
,
this
:
&
'l
LmbdKeyValueDatabase
,
}
}
...
...
turbopack/crates/turbo-tasks-backend/src/database/lmdb/
read_
tx_cache.rs
→
turbopack/crates/turbo-tasks-backend/src/database/lmdb/tx_cache.rs
View file @
8d841c64
...
@@ -6,7 +6,6 @@ use std::{
...
@@ -6,7 +6,6 @@ use std::{
atomic
::{
AtomicU64
,
Ordering
},
atomic
::{
AtomicU64
,
Ordering
},
mpsc
::{
Receiver
,
SyncSender
,
sync_channel
},
mpsc
::{
Receiver
,
SyncSender
,
sync_channel
},
},
},
thread
::
available_parallelism
,
};
};
use
lmdb
::{
Environment
,
RoTransaction
};
use
lmdb
::{
Environment
,
RoTransaction
};
...
@@ -16,12 +15,6 @@ pub struct SendRoTransaction<'tx> {
...
@@ -16,12 +15,6 @@ pub struct SendRoTransaction<'tx> {
generation
:
u64
,
generation
:
u64
,
}
}
impl
<
'tx
>
SendRoTransaction
<
'tx
>
{
fn
generation
(
&
self
)
->
u64
{
self
.generation
}
}
impl
<
'tx
>
Deref
for
SendRoTransaction
<
'tx
>
{
impl
<
'tx
>
Deref
for
SendRoTransaction
<
'tx
>
{
type
Target
=
RoTransaction
<
'tx
>
;
type
Target
=
RoTransaction
<
'tx
>
;
...
@@ -50,7 +43,7 @@ impl QueueState {
...
@@ -50,7 +43,7 @@ impl QueueState {
let
tx
=
env
.begin_ro_txn
()
.unwrap_or_else
(|
err
|
{
let
tx
=
env
.begin_ro_txn
()
.unwrap_or_else
(|
err
|
{
panic!
(
"failed to begin LMDB read transaction: {err}"
);
panic!
(
"failed to begin LMDB read transaction: {err}"
);
});
});
// Safety: these cached transactions are owned by `
Read
TxCache`, which is dropped
// Safety: these cached transactions are owned by `TxCache`, which is dropped
// before the LMDB environment field in the parent database type.
// before the LMDB environment field in the parent database type.
let
tx
=
unsafe
{
transmute
::
<
RoTransaction
<
'_
>
,
RoTransaction
<
'static
>>
(
tx
)
};
let
tx
=
unsafe
{
transmute
::
<
RoTransaction
<
'_
>
,
RoTransaction
<
'static
>>
(
tx
)
};
sender
sender
...
@@ -76,7 +69,7 @@ impl Drop for ReadTxGuardInner<'_> {
...
@@ -76,7 +69,7 @@ impl Drop for ReadTxGuardInner<'_> {
let
Some
(
lease
)
=
self
.lease
.take
()
else
{
let
Some
(
lease
)
=
self
.lease
.take
()
else
{
return
;
return
;
};
};
if
lease
.generation
()
!=
self
.generation
.load
(
Ordering
::
Acquire
)
{
if
lease
.generation
!=
self
.generation
.load
(
Ordering
::
Acquire
)
{
return
;
return
;
}
}
...
@@ -132,15 +125,14 @@ impl Drop for WriteTxGuard<'_> {
...
@@ -132,15 +125,14 @@ impl Drop for WriteTxGuard<'_> {
}
}
}
}
pub
struct
Read
TxCache
{
pub
struct
TxCache
{
generation
:
AtomicU64
,
generation
:
AtomicU64
,
queue
:
RwLock
<
QueueState
>
,
queue
:
RwLock
<
QueueState
>
,
capacity
:
usize
,
capacity
:
usize
,
}
}
impl
ReadTxCache
{
impl
TxCache
{
pub
fn
new
(
env
:
&
Environment
)
->
Self
{
pub
fn
new
(
env
:
&
Environment
,
capacity
:
usize
)
->
Self
{
let
capacity
=
available_parallelism
()
.map_or
(
1
,
|
v
|
v
.get
());
Self
{
Self
{
generation
:
AtomicU64
::
new
(
0
),
generation
:
AtomicU64
::
new
(
0
),
queue
:
RwLock
::
new
(
QueueState
::
new
(
env
,
capacity
,
0
)),
queue
:
RwLock
::
new
(
QueueState
::
new
(
env
,
capacity
,
0
)),
...
@@ -159,6 +151,8 @@ impl ReadTxCache {
...
@@ -159,6 +151,8 @@ impl ReadTxCache {
}
}
}
}
/// NOTE: Callers should avoid acquiring multiple guards at once. If a single thread tries to
/// acquire more than `capacity` read transactions, it could deadlock.
pub
fn
get_read_tx
<
'db
>
(
&
'db
self
)
->
ReadTxGuard
<
'db
>
{
pub
fn
get_read_tx
<
'db
>
(
&
'db
self
)
->
ReadTxGuard
<
'db
>
{
loop
{
loop
{
let
(
sender
,
receiver
)
=
{
let
(
sender
,
receiver
)
=
{
...
@@ -174,7 +168,7 @@ impl ReadTxCache {
...
@@ -174,7 +168,7 @@ impl ReadTxCache {
}
}
};
};
if
lease
.generation
()
!=
self
.generation
.load
(
Ordering
::
Acquire
)
{
if
lease
.generation
!=
self
.generation
.load
(
Ordering
::
Acquire
)
{
continue
;
continue
;
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment