Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
749c94eb
Commit
749c94eb
authored
Mar 05, 2026
by
Benjamin Woodruff
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Some manual cleanup
parent
8d841c64
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
73 additions
and
69 deletions
+73
-69
Cargo.lock
Cargo.lock
+0
-1
Cargo.toml
turbopack/crates/turbo-tasks-backend/Cargo.toml
+0
-1
mod.rs
...opack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs
+15
-29
tx_cache.rs
.../crates/turbo-tasks-backend/src/database/lmdb/tx_cache.rs
+58
-38
No files found.
Cargo.lock
View file @
749c94eb
...
@@ -9594,7 +9594,6 @@ dependencies = [
...
@@ -9594,7 +9594,6 @@ dependencies = [
"serde_path_to_error",
"serde_path_to_error",
"smallvec",
"smallvec",
"tempfile",
"tempfile",
"thread_local",
"tokio",
"tokio",
"tracing",
"tracing",
"turbo-bincode",
"turbo-bincode",
...
...
turbopack/crates/turbo-tasks-backend/Cargo.toml
View file @
749c94eb
...
@@ -56,7 +56,6 @@ serde_path_to_error = { workspace = true }
...
@@ -56,7 +56,6 @@ serde_path_to_error = { workspace = true }
smallvec
=
{
workspace
=
true
}
smallvec
=
{
workspace
=
true
}
tokio
=
{
workspace
=
true
}
tokio
=
{
workspace
=
true
}
tracing
=
{
workspace
=
true
}
tracing
=
{
workspace
=
true
}
thread_local
=
{
workspace
=
true
}
turbo-bincode
=
{
workspace
=
true
}
turbo-bincode
=
{
workspace
=
true
}
turbo-persistence
=
{
workspace
=
true
}
turbo-persistence
=
{
workspace
=
true
}
turbo-rcstr
=
{
workspace
=
true
}
turbo-rcstr
=
{
workspace
=
true
}
...
...
turbopack/crates/turbo-tasks-backend/src/database/lmdb/mod.rs
View file @
749c94eb
...
@@ -4,24 +4,22 @@ mod tx_cache;
...
@@ -4,24 +4,22 @@ mod tx_cache;
use
std
::{
fs
::
create_dir_all
,
path
::
Path
,
thread
::
available_parallelism
};
use
std
::{
fs
::
create_dir_all
,
path
::
Path
,
thread
::
available_parallelism
};
use
anyhow
::{
Context
,
Result
};
use
anyhow
::{
Context
,
Result
};
use
lmdb
::{
use
lmdb
::{
Database
,
DatabaseFlags
,
Environment
,
EnvironmentFlags
,
Transaction
,
WriteFlags
};
Database
,
DatabaseFlags
,
Environment
,
EnvironmentFlags
,
RwTransaction
,
Transaction
,
WriteFlags
,
};
use
crate
::
database
::{
use
crate
::
database
::{
key_value_database
::{
KeySpace
,
KeyValueDatabase
},
key_value_database
::{
KeySpace
,
KeyValueDatabase
},
lmdb
::
tx_cache
::{
R
eadTxGuard
,
TxCache
,
WriteTxGuard
},
lmdb
::
tx_cache
::{
R
oTransactionGuard
,
RwTransactionGuard
,
TxCache
},
write_batch
::{
BaseWriteBatch
,
SerialWriteBatch
,
WriteBatch
,
WriteBuffer
},
write_batch
::{
BaseWriteBatch
,
SerialWriteBatch
,
WriteBatch
,
WriteBuffer
},
};
};
pub
struct
LmdbValueBuffer
<
'l
>
{
pub
struct
LmdbValueBuffer
<
'l
>
{
ptr
:
*
const
u8
,
ptr
:
*
const
u8
,
len
:
usize
,
len
:
usize
,
_guard
:
R
eadTx
Guard
<
'l
>
,
_guard
:
R
oTransaction
Guard
<
'l
>
,
}
}
impl
<
'l
>
LmdbValueBuffer
<
'l
>
{
impl
<
'l
>
LmdbValueBuffer
<
'l
>
{
fn
from_raw_parts
(
ptr
:
*
const
u8
,
len
:
usize
,
guard
:
R
eadTx
Guard
<
'l
>
)
->
Self
{
fn
from_raw_parts
(
ptr
:
*
const
u8
,
len
:
usize
,
guard
:
R
oTransaction
Guard
<
'l
>
)
->
Self
{
Self
{
Self
{
ptr
,
ptr
,
len
,
len
,
...
@@ -93,12 +91,12 @@ impl LmbdKeyValueDatabase {
...
@@ -93,12 +91,12 @@ impl LmbdKeyValueDatabase {
}
}
}
}
fn
get_read_tx
(
&
self
)
->
R
eadTx
Guard
<
'_
>
{
fn
get_read_tx
(
&
self
)
->
R
oTransaction
Guard
<
'_
>
{
self
.tx_cache
.get_read_tx
()
self
.tx_cache
.get_read_tx
()
}
}
fn
begin_write
(
&
self
)
->
WriteTxGuard
<
'_
>
{
fn
get_write_tx
(
&
self
)
->
Result
<
RwTransactionGuard
<
'_
>
>
{
self
.tx_cache
.
begin_write
(
&
self
.env
)
self
.tx_cache
.
get_write_tx
(
&
self
.env
)
}
}
fn
get_from_tx
<
'tx
>
(
fn
get_from_tx
<
'tx
>
(
...
@@ -108,7 +106,7 @@ impl LmbdKeyValueDatabase {
...
@@ -108,7 +106,7 @@ impl LmbdKeyValueDatabase {
)
->
Result
<
Option
<&
'tx
[
u8
]
>>
{
)
->
Result
<
Option
<&
'tx
[
u8
]
>>
{
match
extended_key
::
get
(
tx
,
db
,
key
)
{
match
extended_key
::
get
(
tx
,
db
,
key
)
{
Ok
(
result
)
=>
Ok
(
Some
(
result
)),
Ok
(
result
)
=>
Ok
(
Some
(
result
)),
Err
(
err
)
if
err
==
lmdb
::
Error
::
NotFound
=>
Ok
(
None
),
Err
(
lmdb
::
Error
::
NotFound
)
=>
Ok
(
None
),
Err
(
err
)
=>
Err
(
err
.into
()),
Err
(
err
)
=>
Err
(
err
.into
()),
}
}
}
}
...
@@ -154,18 +152,15 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
...
@@ -154,18 +152,15 @@ impl KeyValueDatabase for LmbdKeyValueDatabase {
fn
write_batch
(
fn
write_batch
(
&
self
,
&
self
,
)
->
Result
<
WriteBatch
<
'_
,
Self
::
SerialWriteBatch
<
'_
>
,
Self
::
ConcurrentWriteBatch
<
'_
>>>
{
)
->
Result
<
WriteBatch
<
'_
,
Self
::
SerialWriteBatch
<
'_
>
,
Self
::
ConcurrentWriteBatch
<
'_
>>>
{
let
write_guard
=
self
.begin_write
();
Ok
(
WriteBatch
::
serial
(
LmbdWriteBatch
{
Ok
(
WriteBatch
::
serial
(
LmbdWriteBatch
{
tx
:
self
.env
.begin_rw_txn
()
?
,
write_tx_guard
:
self
.get_write_tx
()
?
,
_write_guard
:
Some
(
write_guard
),
this
:
self
,
this
:
self
,
}))
}))
}
}
}
}
pub
struct
LmbdWriteBatch
<
'l
>
{
pub
struct
LmbdWriteBatch
<
'l
>
{
tx
:
RwTransaction
<
'l
>
,
write_tx_guard
:
RwTransactionGuard
<
'l
>
,
_write_guard
:
Option
<
WriteTxGuard
<
'l
>>
,
this
:
&
'l
LmbdKeyValueDatabase
,
this
:
&
'l
LmbdKeyValueDatabase
,
}
}
...
@@ -180,21 +175,12 @@ impl<'a> BaseWriteBatch<'a> for LmbdWriteBatch<'a> {
...
@@ -180,21 +175,12 @@ impl<'a> BaseWriteBatch<'a> for LmbdWriteBatch<'a> {
where
where
'a
:
'l
,
'a
:
'l
,
{
{
LmbdKeyValueDatabase
::
get_from_tx
(
&
self
.tx
,
self
.this
.db
(
key_space
),
key
)
LmbdKeyValueDatabase
::
get_from_tx
(
&
*
self
.write_tx_guard
,
self
.this
.db
(
key_space
),
key
)
}
}
fn
commit
(
self
)
->
Result
<
()
>
{
fn
commit
(
self
)
->
Result
<
()
>
{
let
Self
{
let
Self
{
write_tx_guard
,
..
}
=
self
;
tx
,
write_tx_guard
.commit
()
mut
_write_guard
,
..
}
=
self
;
tx
.commit
()
?
;
_write_guard
.take
()
.expect
(
"write guard is always present"
)
.commit
();
Ok
(())
}
}
}
}
...
@@ -206,7 +192,7 @@ impl<'a> SerialWriteBatch<'a> for LmbdWriteBatch<'a> {
...
@@ -206,7 +192,7 @@ impl<'a> SerialWriteBatch<'a> for LmbdWriteBatch<'a> {
value
:
WriteBuffer
<
'_
>
,
value
:
WriteBuffer
<
'_
>
,
)
->
Result
<
()
>
{
)
->
Result
<
()
>
{
extended_key
::
put
(
extended_key
::
put
(
&
mut
self
.
tx
,
&
mut
self
.
write_tx_guard
,
self
.this
.db
(
key_space
),
self
.this
.db
(
key_space
),
&
key
,
&
key
,
&
value
,
&
value
,
...
@@ -217,7 +203,7 @@ impl<'a> SerialWriteBatch<'a> for LmbdWriteBatch<'a> {
...
@@ -217,7 +203,7 @@ impl<'a> SerialWriteBatch<'a> for LmbdWriteBatch<'a> {
fn
delete
(
&
mut
self
,
key_space
:
KeySpace
,
key
:
WriteBuffer
<
'_
>
)
->
Result
<
()
>
{
fn
delete
(
&
mut
self
,
key_space
:
KeySpace
,
key
:
WriteBuffer
<
'_
>
)
->
Result
<
()
>
{
extended_key
::
delete
(
extended_key
::
delete
(
&
mut
self
.
tx
,
&
mut
self
.
write_tx_guard
,
self
.this
.db
(
key_space
),
self
.this
.db
(
key_space
),
&
key
,
&
key
,
WriteFlags
::
empty
(),
WriteFlags
::
empty
(),
...
...
turbopack/crates/turbo-tasks-backend/src/database/lmdb/tx_cache.rs
View file @
749c94eb
use
std
::{
use
std
::{
mem
::
transmute
,
mem
::
transmute
,
ops
::
Deref
,
ops
::{
Deref
,
DerefMut
},
rc
::
Rc
,
sync
::{
sync
::{
Arc
,
Mutex
,
PoisonError
,
RwLock
,
RwLockWriteGuard
,
Arc
,
Mutex
,
PoisonError
,
RwLock
,
RwLockWriteGuard
,
atomic
::{
AtomicU64
,
Ordering
},
atomic
::{
AtomicU64
,
Ordering
},
...
@@ -8,7 +9,8 @@ use std::{
...
@@ -8,7 +9,8 @@ use std::{
},
},
};
};
use
lmdb
::{
Environment
,
RoTransaction
};
use
anyhow
::
Result
;
use
lmdb
::{
Environment
,
RoTransaction
,
RwTransaction
,
Transaction
};
pub
struct
SendRoTransaction
<
'tx
>
{
pub
struct
SendRoTransaction
<
'tx
>
{
tx
:
RoTransaction
<
'tx
>
,
tx
:
RoTransaction
<
'tx
>
,
...
@@ -58,13 +60,14 @@ impl QueueState {
...
@@ -58,13 +60,14 @@ impl QueueState {
}
}
}
}
struct
ReadTxGuardInner
<
'db
>
{
struct
RoTransactionGuardInner
<
'db
>
{
// Option<> is so that the Drop impl can take ownership of the transaction
lease
:
Option
<
SendRoTransaction
<
'db
>>
,
lease
:
Option
<
SendRoTransaction
<
'db
>>
,
sender
:
SyncSender
<
SendRoTransaction
<
'static
>>
,
sender
:
SyncSender
<
SendRoTransaction
<
'static
>>
,
generation
:
&
'db
AtomicU64
,
generation
:
&
'db
AtomicU64
,
}
}
impl
Drop
for
R
eadTx
GuardInner
<
'_
>
{
impl
Drop
for
R
oTransaction
GuardInner
<
'_
>
{
fn
drop
(
&
mut
self
)
{
fn
drop
(
&
mut
self
)
{
let
Some
(
lease
)
=
self
.lease
.take
()
else
{
let
Some
(
lease
)
=
self
.lease
.take
()
else
{
return
;
return
;
...
@@ -82,46 +85,59 @@ impl Drop for ReadTxGuardInner<'_> {
...
@@ -82,46 +85,59 @@ impl Drop for ReadTxGuardInner<'_> {
}
}
#[derive(Clone)]
#[derive(Clone)]
pub
struct
R
eadTx
Guard
<
'db
>
{
pub
struct
R
oTransaction
Guard
<
'db
>
{
inner
:
Arc
<
ReadTx
GuardInner
<
'db
>>
,
inner
:
Rc
<
RoTransaction
GuardInner
<
'db
>>
,
}
}
impl
<
'db
>
Deref
for
R
eadTx
Guard
<
'db
>
{
impl
<
'db
>
Deref
for
R
oTransaction
Guard
<
'db
>
{
type
Target
=
SendRoTransaction
<
'db
>
;
type
Target
=
SendRoTransaction
<
'db
>
;
fn
deref
(
&
self
)
->
&
Self
::
Target
{
fn
deref
(
&
self
)
->
&
Self
::
Target
{
self
.inner
self
.inner.lease
.as_ref
()
.unwrap
()
.lease
.as_ref
()
.expect
(
"read transaction lease is always present"
)
}
}
}
}
pub
struct
WriteTxGuard
<
'db
>
{
pub
struct
RwTransactionGuard
<
'db
>
{
env
:
&
'db
Environment
,
tx
:
RwTransaction
<
'db
>
,
generation
:
u64
,
_queue_guard
:
RebuildQueueOnDrop
<
'db
>
,
capacity
:
usize
,
queue
:
Option
<
RwLockWriteGuard
<
'db
,
QueueState
>>
,
}
}
impl
WriteTxGuard
<
'_
>
{
impl
<
'db
>
Deref
for
RwTransactionGuard
<
'db
>
{
fn
rebuild_queue
(
&
mut
self
)
{
type
Target
=
RwTransaction
<
'db
>
;
if
let
Some
(
queue
)
=
self
.queue
.as_mut
()
{
**
queue
=
QueueState
::
new
(
self
.env
,
self
.capacity
,
self
.generation
);
fn
deref
(
&
self
)
->
&
Self
::
Target
{
}
&
self
.tx
}
}
impl
DerefMut
for
RwTransactionGuard
<
'_
>
{
fn
deref_mut
(
&
mut
self
)
->
&
mut
Self
::
Target
{
&
mut
self
.tx
}
}
}
pub
fn
commit
(
mut
self
)
{
impl
RwTransactionGuard
<
'_
>
{
self
.rebuild_queue
();
// We can't use the method provided by Deref<Target = RwTransaction<'_>> here because this
self
.queue
=
None
;
// consumes the guard
pub
fn
commit
(
self
)
->
Result
<
()
>
{
self
.tx
.commit
()
?
;
Ok
(())
}
}
}
}
impl
Drop
for
WriteTxGuard
<
'_
>
{
struct
RebuildQueueOnDrop
<
'db
>
{
queue
:
RwLockWriteGuard
<
'db
,
QueueState
>
,
env
:
&
'db
Environment
,
generation
:
u64
,
capacity
:
usize
,
}
impl
Drop
for
RebuildQueueOnDrop
<
'_
>
{
fn
drop
(
&
mut
self
)
{
fn
drop
(
&
mut
self
)
{
if
self
.queue
.is_some
()
{
// We might avoid some allocation overhead if we used RoTransaction::reset and
self
.rebuild_queue
();
// InactiveTransaction::renew, but we don't care much about perf here.
}
// https://docs.rs/lmdb/latest/lmdb/struct.RoTransaction.html#method.reset
*
self
.queue
=
QueueState
::
new
(
self
.env
,
self
.capacity
,
self
.generation
);
}
}
}
}
...
@@ -140,20 +156,24 @@ impl TxCache {
...
@@ -140,20 +156,24 @@ impl TxCache {
}
}
}
}
pub
fn
begin_write
<
'db
>
(
&
'db
self
,
env
:
&
'db
Environment
)
->
WriteTxGuard
<
'db
>
{
pub
fn
get_write_tx
<
'db
>
(
&
'db
self
,
env
:
&
'db
Environment
)
->
Result
<
RwTransactionGuard
<
'db
>
>
{
let
generation
=
self
.generation
.fetch_add
(
1
,
Ordering
::
AcqRel
)
+
1
;
let
generation
=
self
.generation
.fetch_add
(
1
,
Ordering
::
AcqRel
)
+
1
;
let
queue
=
self
.queue
.write
()
.unwrap_or_else
(
PoisonError
::
into_inner
);
let
queue
=
self
.queue
.write
()
.unwrap_or_else
(
PoisonError
::
into_inner
);
WriteTxGuard
{
let
tx
=
env
.begin_rw_txn
()
?
;
env
,
Ok
(
RwTransactionGuard
{
generation
,
tx
,
capacity
:
self
.capacity
,
_queue_guard
:
RebuildQueueOnDrop
{
queue
:
Some
(
queue
),
queue
,
}
env
,
generation
,
capacity
:
self
.capacity
,
},
})
}
}
/// NOTE: Callers should avoid acquiring multiple guards at once. If a single thread tries to
/// NOTE: Callers should avoid acquiring multiple guards at once. If a single thread tries to
/// acquire more than `capacity` read transactions, it could deadlock.
/// acquire more than `capacity` read transactions, it could deadlock.
pub
fn
get_read_tx
<
'db
>
(
&
'db
self
)
->
R
eadTx
Guard
<
'db
>
{
pub
fn
get_read_tx
<
'db
>
(
&
'db
self
)
->
R
oTransaction
Guard
<
'db
>
{
loop
{
loop
{
let
(
sender
,
receiver
)
=
{
let
(
sender
,
receiver
)
=
{
let
queue
=
self
.queue
.read
()
.unwrap_or_else
(
PoisonError
::
into_inner
);
let
queue
=
self
.queue
.read
()
.unwrap_or_else
(
PoisonError
::
into_inner
);
...
@@ -177,8 +197,8 @@ impl TxCache {
...
@@ -177,8 +197,8 @@ impl TxCache {
let
lease
=
let
lease
=
unsafe
{
transmute
::
<
SendRoTransaction
<
'static
>
,
SendRoTransaction
<
'db
>>
(
lease
)
};
unsafe
{
transmute
::
<
SendRoTransaction
<
'static
>
,
SendRoTransaction
<
'db
>>
(
lease
)
};
return
R
eadTx
Guard
{
return
R
oTransaction
Guard
{
inner
:
Arc
::
new
(
ReadTx
GuardInner
{
inner
:
Rc
::
new
(
RoTransaction
GuardInner
{
lease
:
Some
(
lease
),
lease
:
Some
(
lease
),
sender
,
sender
,
generation
:
&
self
.generation
,
generation
:
&
self
.generation
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment