diff --git a/bin/reth/src/db/diff.rs b/bin/reth/src/db/diff.rs index 1fff88e08d26..7170f15303d7 100644 --- a/bin/reth/src/db/diff.rs +++ b/bin/reth/src/db/diff.rs @@ -17,7 +17,7 @@ use clap::Parser; use reth_db::{ cursor::DbCursorRO, database::Database, open_db_read_only, table::Table, transaction::DbTx, AccountChangeSet, AccountHistory, AccountsTrie, BlockBodyIndices, BlockOmmers, - BlockWithdrawals, Bytecodes, CanonicalHeaders, DatabaseEnvRO, HashedAccount, HashedStorage, + BlockWithdrawals, Bytecodes, CanonicalHeaders, DatabaseEnv, HashedAccount, HashedStorage, HeaderNumbers, HeaderTD, Headers, PlainAccountState, PlainStorageState, PruneCheckpoints, Receipts, StorageChangeSet, StorageHistory, StoragesTrie, SyncStage, SyncStageProgress, Tables, TransactionBlock, Transactions, TxHashNumber, TxSenders, @@ -58,7 +58,7 @@ impl Command { /// /// The discrepancies and extra elements, along with a brief summary of the diff results are /// then written to a file in the output directory. - pub fn execute(self, tool: &DbTool<'_, DatabaseEnvRO>) -> eyre::Result<()> { + pub fn execute(self, tool: &DbTool<'_, DatabaseEnv>) -> eyre::Result<()> { // open second db let second_db_path: PathBuf = self.secondary_datadir.join("db").into(); let second_db = open_db_read_only(&second_db_path, self.second_db.log_level)?; diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 854efbe9f928..b6c04051adda 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -2,7 +2,7 @@ use super::tui::DbListTUI; use crate::utils::{DbTool, ListFilter}; use clap::Parser; use eyre::WrapErr; -use reth_db::{database::Database, table::Table, DatabaseEnvRO, RawValue, TableViewer, Tables}; +use reth_db::{database::Database, table::Table, DatabaseEnv, RawValue, TableViewer, Tables}; use reth_primitives::hex; use std::cell::RefCell; use tracing::error; @@ -50,7 +50,7 @@ pub struct Command { impl Command { /// Execute `db list` command - pub fn execute(self, tool: &DbTool<'_, DatabaseEnvRO>) -> eyre::Result<()> { + pub fn execute(self, tool: &DbTool<'_, DatabaseEnv>) -> eyre::Result<()> { self.table.view(&ListTableViewer { tool, args: &self }) } @@ -81,7 +81,7 @@ impl Command { } struct ListTableViewer<'a> { - tool: &'a DbTool<'a, DatabaseEnvRO>, + tool: &'a DbTool<'a, DatabaseEnv>, args: &'a Command, } diff --git a/bin/reth/src/db/snapshots/bench.rs b/bin/reth/src/db/snapshots/bench.rs index 47c5ec2fa077..2505b23d4015 100644 --- a/bin/reth/src/db/snapshots/bench.rs +++ b/bin/reth/src/db/snapshots/bench.rs @@ -1,4 +1,4 @@ -use reth_db::DatabaseEnvRO; +use reth_db::DatabaseEnv; use reth_primitives::{ snapshot::{Compression, Filters}, ChainSpec, SnapshotSegment, @@ -16,7 +16,7 @@ pub(crate) enum BenchKind { pub(crate) fn bench( bench_kind: BenchKind, - db: (DatabaseEnvRO, Arc), + db: (DatabaseEnv, Arc), segment: SnapshotSegment, filters: Filters, compression: Compression, @@ -25,7 +25,7 @@ pub(crate) fn bench( ) -> eyre::Result<()> where F1: FnMut() -> eyre::Result, - F2: Fn(DatabaseProviderRO<'_, DatabaseEnvRO>) -> eyre::Result, + F2: Fn(DatabaseProviderRO<'_, DatabaseEnv>) -> eyre::Result, R: Debug + PartialEq, { let (db, chain) = db; diff --git a/bin/reth/src/db/snapshots/mod.rs b/bin/reth/src/db/snapshots/mod.rs index 80f0813c539d..1113d7086830 100644 --- a/bin/reth/src/db/snapshots/mod.rs +++ b/bin/reth/src/db/snapshots/mod.rs @@ -1,6 +1,6 @@ use clap::Parser; use itertools::Itertools; -use reth_db::{open_db_read_only, DatabaseEnvRO}; +use reth_db::{open_db_read_only, DatabaseEnv}; use reth_interfaces::db::LogLevel; use reth_primitives::{ snapshot::{Compression, InclusionFilter, PerfectHashingFunction}, @@ -71,22 +71,21 @@ impl Command { if !self.only_bench { for ((mode, compression), phf) in all_combinations.clone() { match mode { - SnapshotSegment::Headers => self - .generate_headers_snapshot::( - &provider, - *compression, - InclusionFilter::Cuckoo, - *phf, - )?, + SnapshotSegment::Headers => self.generate_headers_snapshot::( + &provider, + *compression, + InclusionFilter::Cuckoo, + *phf, + )?, SnapshotSegment::Transactions => self - .generate_transactions_snapshot::( + .generate_transactions_snapshot::( &provider, *compression, InclusionFilter::Cuckoo, *phf, )?, SnapshotSegment::Receipts => self - .generate_receipts_snapshot::( + .generate_receipts_snapshot::( &provider, *compression, InclusionFilter::Cuckoo, diff --git a/crates/consensus/beacon/src/engine/sync.rs b/crates/consensus/beacon/src/engine/sync.rs index 0d95412f2e49..07780d4330e1 100644 --- a/crates/consensus/beacon/src/engine/sync.rs +++ b/crates/consensus/beacon/src/engine/sync.rs @@ -395,7 +395,7 @@ mod tests { use assert_matches::assert_matches; use futures::poll; use reth_db::{ - mdbx::{Env, WriteMap}, + mdbx::DatabaseEnv, test_utils::{create_test_rw_db, TempDatabase}, }; use reth_interfaces::{p2p::either::EitherDownloader, test_utils::TestFullBlockClient}; @@ -449,7 +449,7 @@ mod tests { } /// Builds the pipeline. - fn build(self, chain_spec: Arc) -> Pipeline>>> { + fn build(self, chain_spec: Arc) -> Pipeline>> { reth_tracing::init_test_tracing(); let db = create_test_rw_db(); diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 1154005c6b0b..161d87e415ed 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -8,8 +8,7 @@ use crate::{ }; use reth_interfaces::db::LogLevel; use reth_libmdbx::{ - DatabaseFlags, Environment, EnvironmentFlags, EnvironmentKind, Geometry, Mode, PageSize, - SyncMode, RO, RW, + DatabaseFlags, Environment, EnvironmentFlags, Geometry, Mode, PageSize, SyncMode, RO, RW, }; use std::{ops::Deref, path::Path}; use tx::Tx; @@ -25,7 +24,7 @@ const DEFAULT_MAX_READERS: u64 = 32_000; /// Environment used when opening a MDBX environment. RO/RW. #[derive(Debug)] -pub enum EnvKind { +pub enum DatabaseEnvKind { /// Read-only MDBX environment. RO, /// Read-write MDBX environment. @@ -34,19 +33,19 @@ pub enum EnvKind { /// Wrapper for the libmdbx environment. #[derive(Debug)] -pub struct Env { +pub struct DatabaseEnv { /// Libmdbx-sys environment. - pub inner: Environment, + pub inner: Environment, /// Whether to record metrics or not. with_metrics: bool, } -impl<'a, E: EnvironmentKind> DatabaseGAT<'a> for Env { - type TX = tx::Tx<'a, RO, E>; - type TXMut = tx::Tx<'a, RW, E>; +impl<'a> DatabaseGAT<'a> for DatabaseEnv { + type TX = tx::Tx<'a, RO>; + type TXMut = tx::Tx<'a, RW>; } -impl Database for Env { +impl Database for DatabaseEnv { fn tx(&self) -> Result<>::TX, DatabaseError> { Ok(Tx::new_with_metrics( self.inner.begin_ro_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, @@ -62,21 +61,26 @@ impl Database for Env { } } -impl Env { +impl DatabaseEnv { /// Opens the database at the specified path with the given `EnvKind`. /// - /// It does not create the tables, for that call [`Env::create_tables`]. + /// It does not create the tables, for that call [`DatabaseEnv::create_tables`]. pub fn open( path: &Path, - kind: EnvKind, + kind: DatabaseEnvKind, log_level: Option, - ) -> Result, DatabaseError> { + ) -> Result { + let mut inner_env = Environment::builder(); + let mode = match kind { - EnvKind::RO => Mode::ReadOnly, - EnvKind::RW => Mode::ReadWrite { sync_mode: SyncMode::Durable }, + DatabaseEnvKind::RO => Mode::ReadOnly, + DatabaseEnvKind::RW => { + // enable writemap mode in RW mode + inner_env.write_map(); + Mode::ReadWrite { sync_mode: SyncMode::Durable } + } }; - let mut inner_env = Environment::builder(); inner_env.set_max_dbs(Tables::ALL.len()); inner_env.set_geometry(Geometry { // Maximum database size of 4 terabytes @@ -124,7 +128,7 @@ impl Env { } } - let env = Env { + let env = DatabaseEnv { inner: inner_env.open(path).map_err(|e| DatabaseError::Open(e.into()))?, with_metrics: false, }; @@ -158,8 +162,8 @@ impl Env { } } -impl Deref for Env { - type Target = Environment; +impl Deref for DatabaseEnv { + type Target = Environment; fn deref(&self) -> &Self::Target { &self.inner @@ -180,13 +184,12 @@ mod tests { AccountChangeSet, }; use reth_interfaces::db::{DatabaseWriteError, DatabaseWriteOperation}; - use reth_libmdbx::{NoWriteMap, WriteMap}; use reth_primitives::{Account, Address, Header, IntegerList, StorageEntry, B256, U256}; use std::{path::Path, str::FromStr, sync::Arc}; use tempfile::TempDir; /// Create database for testing - fn create_test_db(kind: EnvKind) -> Arc> { + fn create_test_db(kind: DatabaseEnvKind) -> Arc { Arc::new(create_test_db_with_path( kind, &tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(), @@ -194,8 +197,8 @@ mod tests { } /// Create database for testing with specified path - fn create_test_db_with_path(kind: EnvKind, path: &Path) -> Env { - let env = Env::::open(path, kind, None).expect(ERROR_DB_CREATION); + fn create_test_db_with_path(kind: DatabaseEnvKind, path: &Path) -> DatabaseEnv { + let env = DatabaseEnv::open(path, kind, None).expect(ERROR_DB_CREATION); env.create_tables().expect(ERROR_TABLE_CREATION); env } @@ -212,12 +215,12 @@ mod tests { #[test] fn db_creation() { - create_test_db::(EnvKind::RW); + create_test_db(DatabaseEnvKind::RW); } #[test] fn db_manual_put_get() { - let env = create_test_db::(EnvKind::RW); + let env = create_test_db(DatabaseEnvKind::RW); let value = Header::default(); let key = 1u64; @@ -236,7 +239,7 @@ mod tests { #[test] fn db_cursor_walk() { - let env = create_test_db::(EnvKind::RW); + let env = create_test_db(DatabaseEnvKind::RW); let value = Header::default(); let key = 1u64; @@ -261,7 +264,7 @@ mod tests { #[test] fn db_cursor_walk_range() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT (0, 0), (1, 0), (2, 0), (3, 0) let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -325,7 +328,7 @@ mod tests { #[test] fn db_cursor_walk_range_on_dup_table() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let address0 = Address::ZERO; let address1 = Address::with_last_byte(1); @@ -367,7 +370,7 @@ mod tests { #[allow(clippy::reversed_empty_ranges)] #[test] fn db_cursor_walk_range_invalid() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT (0, 0), (1, 0), (2, 0), (3, 0) let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -395,7 +398,7 @@ mod tests { #[test] fn db_walker() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT (0, 0), (1, 0), (3, 0) let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -425,7 +428,7 @@ mod tests { #[test] fn db_reverse_walker() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT (0, 0), (1, 0), (3, 0) let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -455,7 +458,7 @@ mod tests { #[test] fn db_walk_back() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT (0, 0), (1, 0), (3, 0) let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -494,7 +497,7 @@ mod tests { #[test] fn db_cursor_seek_exact_or_previous_key() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -520,7 +523,7 @@ mod tests { #[test] fn db_cursor_insert() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -563,7 +566,7 @@ mod tests { #[test] fn db_cursor_insert_dup() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let tx = db.tx_mut().expect(ERROR_INIT_TX); let mut dup_cursor = tx.cursor_dup_write::().unwrap(); @@ -581,7 +584,7 @@ mod tests { #[test] fn db_cursor_delete_current_non_existent() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let tx = db.tx_mut().expect(ERROR_INIT_TX); let key1 = Address::with_last_byte(1); @@ -609,7 +612,7 @@ mod tests { #[test] fn db_cursor_insert_wherever_cursor_is() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let tx = db.tx_mut().expect(ERROR_INIT_TX); // PUT @@ -642,7 +645,7 @@ mod tests { #[test] fn db_cursor_append() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -669,7 +672,7 @@ mod tests { #[test] fn db_cursor_append_failure() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); // PUT let tx = db.tx_mut().expect(ERROR_INIT_TX); @@ -706,7 +709,7 @@ mod tests { #[test] fn db_cursor_upsert() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let tx = db.tx_mut().expect(ERROR_INIT_TX); let mut cursor = tx.cursor_write::().unwrap(); @@ -741,7 +744,7 @@ mod tests { #[test] fn db_cursor_dupsort_append() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let transition_id = 2; @@ -810,28 +813,28 @@ mod tests { .expect(ERROR_ETH_ADDRESS); { - let env = create_test_db_with_path::(EnvKind::RW, &path); + let env = create_test_db_with_path(DatabaseEnvKind::RW, &path); // PUT let result = env.update(|tx| { tx.put::(key, value).expect(ERROR_PUT); 200 }); - assert!(result.expect(ERROR_RETURN_VALUE) == 200); + assert_eq!(result.expect(ERROR_RETURN_VALUE), 200); } - let env = Env::::open(&path, EnvKind::RO, None).expect(ERROR_DB_CREATION); + let env = DatabaseEnv::open(&path, DatabaseEnvKind::RO, None).expect(ERROR_DB_CREATION); // GET let result = env.view(|tx| tx.get::(key).expect(ERROR_GET)).expect(ERROR_GET); - assert!(result == Some(value)) + assert_eq!(result, Some(value)) } #[test] fn db_dup_sort() { - let env = create_test_db::(EnvKind::RW); + let env = create_test_db(DatabaseEnvKind::RW); let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047") .expect(ERROR_ETH_ADDRESS); @@ -875,7 +878,7 @@ mod tests { #[test] fn db_iterate_over_all_dup_values() { - let env = create_test_db::(EnvKind::RW); + let env = create_test_db(DatabaseEnvKind::RW); let key1 = Address::from_str("0x1111111111111111111111111111111111111111") .expect(ERROR_ETH_ADDRESS); let key2 = Address::from_str("0x2222222222222222222222222222222222222222") @@ -921,7 +924,7 @@ mod tests { #[test] fn dup_value_with_same_subkey() { - let env = create_test_db::(EnvKind::RW); + let env = create_test_db(DatabaseEnvKind::RW); let key1 = Address::new([0x11; 20]); let key2 = Address::new([0x22; 20]); @@ -964,7 +967,7 @@ mod tests { #[test] fn db_sharded_key() { - let db: Arc> = create_test_db(EnvKind::RW); + let db: Arc = create_test_db(DatabaseEnvKind::RW); let real_key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047").unwrap(); for i in 1..5 { diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 689a0cd76267..78c52172d591 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -12,14 +12,14 @@ use crate::{ }; use parking_lot::RwLock; use reth_interfaces::db::{DatabaseWriteError, DatabaseWriteOperation}; -use reth_libmdbx::{ffi::DBI, EnvironmentKind, Transaction, TransactionKind, WriteFlags, RW}; +use reth_libmdbx::{ffi::DBI, Transaction, TransactionKind, WriteFlags, RW}; use std::{marker::PhantomData, str::FromStr, sync::Arc, time::Instant}; /// Wrapper for the libmdbx transaction. #[derive(Debug)] -pub struct Tx<'a, K: TransactionKind, E: EnvironmentKind> { +pub struct Tx<'a, K: TransactionKind> { /// Libmdbx-sys transaction. - pub inner: Transaction<'a, K, E>, + pub inner: Transaction<'a, K>, /// Database table handle cache. pub(crate) db_handles: Arc; NUM_TABLES]>>, /// Handler for metrics with its own [Drop] implementation for cases when the transaction isn't @@ -29,9 +29,9 @@ pub struct Tx<'a, K: TransactionKind, E: EnvironmentKind> { metrics_handler: Option>, } -impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> { +impl<'env, K: TransactionKind> Tx<'env, K> { /// Creates new `Tx` object with a `RO` or `RW` transaction. - pub fn new<'a>(inner: Transaction<'a, K, E>) -> Self + pub fn new<'a>(inner: Transaction<'a, K>) -> Self where 'a: 'env, { @@ -39,7 +39,7 @@ impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> { } /// Creates new `Tx` object with a `RO` or `RW` transaction and optionally enables metrics. - pub fn new_with_metrics<'a>(inner: Transaction<'a, K, E>, with_metrics: bool) -> Self + pub fn new_with_metrics<'a>(inner: Transaction<'a, K>, with_metrics: bool) -> Self where 'a: 'env, { @@ -128,7 +128,7 @@ impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> { &self, operation: Operation, value_size: Option, - f: impl FnOnce(&Transaction<'_, K, E>) -> R, + f: impl FnOnce(&Transaction<'_, K>) -> R, ) -> R { if self.metrics_handler.is_some() { OperationMetrics::record(T::NAME, operation, value_size, || f(&self.inner)) @@ -173,19 +173,19 @@ impl Drop for MetricsHandler { } } -impl<'a, K: TransactionKind, E: EnvironmentKind> DbTxGAT<'a> for Tx<'_, K, E> { +impl<'a, K: TransactionKind> DbTxGAT<'a> for Tx<'_, K> { type Cursor = Cursor<'a, K, T>; type DupCursor = Cursor<'a, K, T>; } -impl<'a, K: TransactionKind, E: EnvironmentKind> DbTxMutGAT<'a> for Tx<'_, K, E> { +impl<'a, K: TransactionKind> DbTxMutGAT<'a> for Tx<'_, K> { type CursorMut = Cursor<'a, RW, T>; type DupCursorMut = Cursor<'a, RW, T>; } -impl TableImporter for Tx<'_, RW, E> {} +impl TableImporter for Tx<'_, RW> {} -impl DbTx for Tx<'_, K, E> { +impl DbTx for Tx<'_, K> { fn get(&self, key: T::Key) -> Result::Value>, DatabaseError> { self.execute_with_operation_metric::(Operation::Get, None, |tx| { tx.get(self.get_dbi::()?, key.encode().as_ref()) @@ -229,7 +229,7 @@ impl DbTx for Tx<'_, K, E> { } } -impl DbTxMut for Tx<'_, RW, E> { +impl DbTxMut for Tx<'_, RW> { fn put(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { let key = key.encode(); let value = value.compress(); diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index a1511fb09ed6..250177dfb182 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -87,15 +87,7 @@ pub use tables::*; pub use utils::is_database_empty; #[cfg(feature = "mdbx")] -use mdbx::{Env, EnvKind, NoWriteMap, WriteMap}; - -#[cfg(feature = "mdbx")] -/// Alias type for the database environment in use. Read/Write mode. -pub type DatabaseEnv = Env; - -#[cfg(feature = "mdbx")] -/// Alias type for the database engine in use. Read only mode. -pub type DatabaseEnvRO = Env; +pub use mdbx::{DatabaseEnv, DatabaseEnvKind}; use eyre::WrapErr; use reth_interfaces::db::LogLevel; @@ -120,7 +112,7 @@ pub fn init_db>(path: P, log_level: Option) -> eyre::Re } #[cfg(feature = "mdbx")] { - let db = DatabaseEnv::open(rpath, EnvKind::RW, log_level)?; + let db = DatabaseEnv::open(rpath, DatabaseEnvKind::RW, log_level)?; db.create_tables()?; Ok(db) } @@ -131,10 +123,10 @@ pub fn init_db>(path: P, log_level: Option) -> eyre::Re } /// Opens up an existing database. Read only mode. It doesn't create it or create tables if missing. -pub fn open_db_read_only(path: &Path, log_level: Option) -> eyre::Result { +pub fn open_db_read_only(path: &Path, log_level: Option) -> eyre::Result { #[cfg(feature = "mdbx")] { - Env::::open(path, EnvKind::RO, log_level) + DatabaseEnv::open(path, DatabaseEnvKind::RO, log_level) .with_context(|| format!("Could not open database at path: {}", path.display())) } #[cfg(not(feature = "mdbx"))] @@ -143,12 +135,12 @@ pub fn open_db_read_only(path: &Path, log_level: Option) -> eyre::Resu } } -/// Opens up an existing database. Read/Write mode. It doesn't create it or create tables if -/// missing. +/// Opens up an existing database. Read/Write mode with WriteMap enabled. It doesn't create it or +/// create tables if missing. pub fn open_db(path: &Path, log_level: Option) -> eyre::Result { #[cfg(feature = "mdbx")] { - Env::::open(path, EnvKind::RW, log_level) + DatabaseEnv::open(path, DatabaseEnvKind::RW, log_level) .with_context(|| format!("Could not open database at path: {}", path.display())) } #[cfg(not(feature = "mdbx"))] @@ -234,7 +226,7 @@ pub mod test_utils { } /// Create read only database for testing - pub fn create_test_ro_db() -> Arc> { + pub fn create_test_ro_db() -> Arc> { let path = tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(); { init_db(path.as_path(), None).expect(ERROR_DB_CREATION); diff --git a/crates/storage/libmdbx-rs/benches/utils.rs b/crates/storage/libmdbx-rs/benches/utils.rs index 7388e1f676c3..bf24b4866ae4 100644 --- a/crates/storage/libmdbx-rs/benches/utils.rs +++ b/crates/storage/libmdbx-rs/benches/utils.rs @@ -1,4 +1,4 @@ -use reth_libmdbx::{Environment, NoWriteMap, WriteFlags}; +use reth_libmdbx::{Environment, WriteFlags}; use tempfile::{tempdir, TempDir}; pub fn get_key(n: u32) -> String { @@ -9,7 +9,7 @@ pub fn get_data(n: u32) -> String { format!("data{n}") } -pub fn setup_bench_db(num_rows: u32) -> (TempDir, Environment) { +pub fn setup_bench_db(num_rows: u32) -> (TempDir, Environment) { let dir = tempdir().unwrap(); let env = Environment::builder().open(dir.path()).unwrap(); diff --git a/crates/storage/libmdbx-rs/src/cursor.rs b/crates/storage/libmdbx-rs/src/cursor.rs index d9dd917591eb..dfbdcc52c989 100644 --- a/crates/storage/libmdbx-rs/src/cursor.rs +++ b/crates/storage/libmdbx-rs/src/cursor.rs @@ -3,7 +3,7 @@ use crate::{ flags::*, mdbx_try_optional, transaction::{TransactionKind, TransactionPtr, RW}, - EnvironmentKind, TableObject, Transaction, + TableObject, Transaction, }; use ffi::{ MDBX_cursor_op, MDBX_FIRST, MDBX_FIRST_DUP, MDBX_GET_BOTH, MDBX_GET_BOTH_RANGE, @@ -28,10 +28,7 @@ impl<'txn, K> Cursor<'txn, K> where K: TransactionKind, { - pub(crate) fn new( - txn: &'txn Transaction<'_, K, E>, - dbi: ffi::MDBX_dbi, - ) -> Result { + pub(crate) fn new(txn: &'txn Transaction<'_, K>, dbi: ffi::MDBX_dbi) -> Result { let mut cursor: *mut ffi::MDBX_cursor = ptr::null_mut(); let txn = txn.txn_ptr(); unsafe { diff --git a/crates/storage/libmdbx-rs/src/database.rs b/crates/storage/libmdbx-rs/src/database.rs index 8609c4e49b3e..f0f48951aff1 100644 --- a/crates/storage/libmdbx-rs/src/database.rs +++ b/crates/storage/libmdbx-rs/src/database.rs @@ -1,5 +1,4 @@ use crate::{ - environment::EnvironmentKind, error::{mdbx_result, Result}, transaction::TransactionKind, Transaction, @@ -21,8 +20,8 @@ impl<'txn> Database<'txn> { /// /// Prefer using `Environment::open_db`, `Environment::create_db`, `TransactionExt::open_db`, /// or `RwTransaction::create_db`. - pub(crate) fn new<'env, K: TransactionKind, E: EnvironmentKind>( - txn: &'txn Transaction<'env, K, E>, + pub(crate) fn new<'env, K: TransactionKind>( + txn: &'txn Transaction<'env, K>, name: Option<&str>, flags: MDBX_db_flags_t, ) -> Result { diff --git a/crates/storage/libmdbx-rs/src/environment.rs b/crates/storage/libmdbx-rs/src/environment.rs index e8ace3115f67..b0338826b0f5 100644 --- a/crates/storage/libmdbx-rs/src/environment.rs +++ b/crates/storage/libmdbx-rs/src/environment.rs @@ -11,7 +11,6 @@ use std::{ ffi::CString, fmt, fmt::Debug, - marker::PhantomData, mem, ops::{Bound, RangeBounds}, path::Path, @@ -21,32 +20,41 @@ use std::{ time::Duration, }; -mod private { - use super::*; - - pub trait Sealed {} - - impl Sealed for NoWriteMap {} - impl Sealed for WriteMap {} -} - -pub trait EnvironmentKind: private::Sealed + Debug + 'static { - const EXTRA_FLAGS: ffi::MDBX_env_flags_t; +/// Determines how data is mapped into memory +/// +/// It only takes affect when the environment is opened. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum EnvironmentKind { + /// Open the environment in default mode, without WRITEMAP. + #[default] + Default, + /// Open the environment as mdbx-WRITEMAP. + /// Use a writeable memory map unless the environment is opened as MDBX_RDONLY + /// ([Mode::ReadOnly]). + /// + /// All data will be mapped into memory in the read-write mode [Mode::ReadWrite]. This offers a + /// significant performance benefit, since the data will be modified directly in mapped + /// memory and then flushed to disk by single system call, without any memory management + /// nor copying. + /// + /// This mode is incompatible with nested transactions. + WriteMap, } -#[derive(Debug)] -#[non_exhaustive] -pub struct NoWriteMap; - -#[derive(Debug)] -#[non_exhaustive] -pub struct WriteMap; +impl EnvironmentKind { + /// Returns true if the environment was opened as WRITEMAP. + #[inline] + pub const fn is_write_map(&self) -> bool { + matches!(self, EnvironmentKind::WriteMap) + } -impl EnvironmentKind for NoWriteMap { - const EXTRA_FLAGS: ffi::MDBX_env_flags_t = ffi::MDBX_ENV_DEFAULTS; -} -impl EnvironmentKind for WriteMap { - const EXTRA_FLAGS: ffi::MDBX_env_flags_t = ffi::MDBX_WRITEMAP; + /// Additional flags required when opening the environment. + pub(crate) fn extra_flags(&self) -> ffi::MDBX_env_flags_t { + match self { + EnvironmentKind::Default => ffi::MDBX_ENV_DEFAULTS, + EnvironmentKind::WriteMap => ffi::MDBX_WRITEMAP, + } + } } #[derive(Copy, Clone, Debug)] @@ -66,20 +74,13 @@ pub(crate) enum TxnManagerMessage { } /// An environment supports multiple databases, all residing in the same shared-memory map. -pub struct Environment -where - E: EnvironmentKind, -{ +pub struct Environment { inner: EnvironmentInner, - _marker: PhantomData, } -impl Environment -where - E: EnvironmentKind, -{ +impl Environment { /// Creates a new builder for specifying options for opening an MDBX environment. - pub fn builder() -> EnvironmentBuilder { + pub fn builder() -> EnvironmentBuilder { EnvironmentBuilder { flags: EnvironmentFlags::default(), max_readers: None, @@ -92,10 +93,20 @@ where spill_min_denominator: None, geometry: None, log_level: None, - _marker: PhantomData, + kind: Default::default(), } } + /// Returns true if the environment was opened as WRITEMAP. + pub fn is_write_map(&self) -> bool { + self.inner.env_kind.is_write_map() + } + + /// Returns the kind of the environment. + pub fn env_kind(&self) -> EnvironmentKind { + self.inner.env_kind + } + /// Returns the manager that handles transaction messages. /// /// Requires [Mode::ReadWrite] and returns None otherwise. @@ -115,13 +126,13 @@ where /// Create a read-only transaction for use with the environment. #[inline] - pub fn begin_ro_txn(&self) -> Result> { + pub fn begin_ro_txn(&self) -> Result> { Transaction::new(self) } /// Create a read-write transaction for use with the environment. This method will block while /// there are any other read-write transactions open on the environment. - pub fn begin_rw_txn(&self) -> Result> { + pub fn begin_rw_txn(&self) -> Result> { let sender = self.txn_manager().ok_or(Error::Access)?; let txn = loop { let (tx, rx) = sync_channel(0); @@ -183,9 +194,8 @@ where /// /// ``` /// # use reth_libmdbx::Environment; - /// # use reth_libmdbx::NoWriteMap; /// let dir = tempfile::tempdir().unwrap(); - /// let env = Environment::::builder().open(dir.path()).unwrap(); + /// let env = Environment::builder().open(dir.path()).unwrap(); /// let info = env.info().unwrap(); /// let stat = env.stat().unwrap(); /// let freelist = env.freelist().unwrap(); @@ -228,6 +238,7 @@ where /// The env is opened via [mdbx_env_create](ffi::mdbx_env_create) and closed when this type drops. struct EnvironmentInner { env: *mut ffi::MDBX_env, + env_kind: EnvironmentKind, txn_manager: Option>, } @@ -347,15 +358,12 @@ impl Info { } } -unsafe impl Send for Environment where E: EnvironmentKind {} -unsafe impl Sync for Environment where E: EnvironmentKind {} +unsafe impl Send for Environment {} +unsafe impl Sync for Environment {} -impl fmt::Debug for Environment -where - E: EnvironmentKind, -{ +impl fmt::Debug for Environment { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Environment").finish_non_exhaustive() + f.debug_struct("Environment").field("kind", &self.inner.env_kind).finish_non_exhaustive() } } @@ -385,10 +393,7 @@ impl Default for Geometry { /// Options for opening or creating an environment. #[derive(Debug, Clone)] -pub struct EnvironmentBuilder -where - E: EnvironmentKind, -{ +pub struct EnvironmentBuilder { flags: EnvironmentFlags, max_readers: Option, max_dbs: Option, @@ -400,17 +405,14 @@ where spill_min_denominator: Option, geometry: Option, Option)>>, log_level: Option, - _marker: PhantomData, + kind: EnvironmentKind, } -impl EnvironmentBuilder -where - E: EnvironmentKind, -{ +impl EnvironmentBuilder { /// Open an environment. /// /// Database files will be opened with 644 permissions. - pub fn open(&self, path: &Path) -> Result> { + pub fn open(&self, path: &Path) -> Result { self.open_with_permissions(path, 0o644) } @@ -421,7 +423,7 @@ where &self, path: &Path, mode: ffi::mdbx_mode_t, - ) -> Result> { + ) -> Result { let mut env: *mut ffi::MDBX_env = ptr::null_mut(); unsafe { if let Some(log_level) = self.log_level { @@ -505,7 +507,7 @@ where mdbx_result(ffi::mdbx_env_open( env, path.as_ptr(), - self.flags.make_flags() | E::EXTRA_FLAGS, + self.flags.make_flags() | self.kind.extra_flags(), mode, ))?; @@ -517,7 +519,7 @@ where } } - let mut env = EnvironmentInner { env, txn_manager: None }; + let mut env = EnvironmentInner { env, txn_manager: None, env_kind: self.kind }; if let Mode::ReadWrite { .. } = self.flags.mode { let (tx, rx) = std::sync::mpsc::sync_channel(0); @@ -562,7 +564,20 @@ where env.txn_manager = Some(tx); } - Ok(Environment { inner: env, _marker: Default::default() }) + Ok(Environment { inner: env }) + } + + /// Configures how this environment will be opened. + pub fn set_kind(&mut self, kind: EnvironmentKind) -> &mut Self { + self.kind = kind; + self + } + + /// Opens the environment with mdbx WRITEMAP + /// + /// See also [EnvironmentKind] + pub fn write_map(&mut self) -> &mut Self { + self.set_kind(EnvironmentKind::WriteMap) } /// Sets the provided options in the environment. diff --git a/crates/storage/libmdbx-rs/src/error.rs b/crates/storage/libmdbx-rs/src/error.rs index ebe5a43a3d43..274fcc47f7a6 100644 --- a/crates/storage/libmdbx-rs/src/error.rs +++ b/crates/storage/libmdbx-rs/src/error.rs @@ -55,6 +55,7 @@ pub enum Error { Access, TooLarge, DecodeErrorLenDiff, + NestedTransactionsUnsupportedWithWriteMap, Other(i32), } diff --git a/crates/storage/libmdbx-rs/src/flags.rs b/crates/storage/libmdbx-rs/src/flags.rs index df9f817de109..e464a3b20a05 100644 --- a/crates/storage/libmdbx-rs/src/flags.rs +++ b/crates/storage/libmdbx-rs/src/flags.rs @@ -28,11 +28,11 @@ pub enum SyncMode { /// are recycled the MVCC snapshots corresponding to previous "steady" transactions (see /// below). /// - /// With [crate::WriteMap] the [SyncMode::SafeNoSync] instructs MDBX to use asynchronous - /// mmap-flushes to disk. Asynchronous mmap-flushes means that actually all writes will - /// scheduled and performed by operation system on it own manner, i.e. unordered. - /// MDBX itself just notify operating system that it would be nice to write data to disk, but - /// no more. + /// With [crate::EnvironmentKind::WriteMap] the [SyncMode::SafeNoSync] instructs MDBX to use + /// asynchronous mmap-flushes to disk. Asynchronous mmap-flushes means that actually all + /// writes will scheduled and performed by operation system on it own manner, i.e. + /// unordered. MDBX itself just notify operating system that it would be nice to write data + /// to disk, but no more. /// /// Depending on the platform and hardware, with [SyncMode::SafeNoSync] you may get a multiple /// increase of write performance, even 10 times or more. @@ -70,17 +70,18 @@ pub enum SyncMode { /// you may get a multiple increase of write performance, even 100 times or more. /// /// If the filesystem preserves write order (which is rare and never provided unless explicitly - /// noted) and the [WriteMap](crate::WriteMap) and [EnvironmentFlags::liforeclaim] flags are - /// not used, then a system crash can't corrupt the database, but you can lose the last - /// transactions, if at least one buffer is not yet flushed to disk. The risk is governed - /// by how often the system flushes dirty buffers to disk and how often - /// [Environment::sync()](crate::Environment::sync) is called. So, transactions exhibit ACI - /// (atomicity, consistency, isolation) properties and only lose D (durability). - /// I.e. database integrity is maintained, but a system crash may undo the final transactions. + /// noted) and the [WriteMap](crate::EnvironmentKind::WriteMap) and + /// [EnvironmentFlags::liforeclaim] flags are not used, then a system crash can't corrupt + /// the database, but you can lose the last transactions, if at least one buffer is not yet + /// flushed to disk. The risk is governed by how often the system flushes dirty buffers to + /// disk and how often [Environment::sync()](crate::Environment::sync) is called. So, + /// transactions exhibit ACI (atomicity, consistency, isolation) properties and only lose D + /// (durability). I.e. database integrity is maintained, but a system crash may undo the + /// final transactions. /// /// Otherwise, if the filesystem not preserves write order (which is typically) or - /// [WriteMap](crate::WriteMap) or [EnvironmentFlags::liforeclaim] flags are used, you should - /// expect the corrupted database after a system crash. + /// [WriteMap](crate::EnvironmentKind::WriteMap) or [EnvironmentFlags::liforeclaim] flags are + /// used, you should expect the corrupted database after a system crash. /// /// So, most important thing about [SyncMode::UtterlyNoSync]: /// - A system crash immediately after commit the write transaction high likely lead to diff --git a/crates/storage/libmdbx-rs/src/lib.rs b/crates/storage/libmdbx-rs/src/lib.rs index cad77bc6f691..14e20de72188 100644 --- a/crates/storage/libmdbx-rs/src/lib.rs +++ b/crates/storage/libmdbx-rs/src/lib.rs @@ -15,8 +15,7 @@ pub use crate::{ cursor::{Cursor, Iter, IterDup}, database::Database, environment::{ - Environment, EnvironmentBuilder, EnvironmentKind, Geometry, Info, NoWriteMap, PageSize, - Stat, WriteMap, + Environment, EnvironmentBuilder, EnvironmentKind, Geometry, Info, PageSize, Stat, }, error::{Error, Result}, flags::*, @@ -40,8 +39,6 @@ mod test_utils { use byteorder::{ByteOrder, LittleEndian}; use tempfile::tempdir; - type Environment = crate::Environment; - /// Regression test for https://github.com/danburkert/lmdb-rs/issues/21. /// This test reliably segfaults when run against lmbdb compiled with opt level -O3 and newer /// GCC compilers. diff --git a/crates/storage/libmdbx-rs/src/transaction.rs b/crates/storage/libmdbx-rs/src/transaction.rs index bd8cbae8f03f..ddea959125bf 100644 --- a/crates/storage/libmdbx-rs/src/transaction.rs +++ b/crates/storage/libmdbx-rs/src/transaction.rs @@ -1,6 +1,6 @@ use crate::{ database::Database, - environment::{Environment, EnvironmentKind, NoWriteMap, TxnManagerMessage, TxnPtr}, + environment::{Environment, TxnManagerMessage, TxnPtr}, error::{mdbx_result, Result}, flags::{DatabaseFlags, WriteFlags}, Cursor, Error, Stat, TableObject, @@ -12,7 +12,6 @@ use parking_lot::Mutex; use std::{ fmt, fmt::Debug, - marker::PhantomData, mem::size_of, ptr, slice, sync::{atomic::AtomicBool, mpsc::sync_channel, Arc}, @@ -60,20 +59,18 @@ impl TransactionKind for RW { /// An MDBX transaction. /// /// All database operations require a transaction. -pub struct Transaction<'env, K, E> +pub struct Transaction<'env, K> where K: TransactionKind, - E: EnvironmentKind, { - inner: Arc>, + inner: Arc>, } -impl<'env, K, E> Transaction<'env, K, E> +impl<'env, K> Transaction<'env, K> where K: TransactionKind, - E: EnvironmentKind, { - pub(crate) fn new(env: &'env Environment) -> Result { + pub(crate) fn new(env: &'env Environment) -> Result { let mut txn: *mut ffi::MDBX_txn = ptr::null_mut(); unsafe { mdbx_result(ffi::mdbx_txn_begin_ex( @@ -87,13 +84,13 @@ where } } - pub(crate) fn new_from_ptr(env: &'env Environment, txn: *mut ffi::MDBX_txn) -> Self { + pub(crate) fn new_from_ptr(env: &'env Environment, txn: *mut ffi::MDBX_txn) -> Self { let inner = TransactionInner { txn: TransactionPtr::new(txn), primed_dbis: Mutex::new(IndexSet::new()), committed: AtomicBool::new(false), env, - _marker: PhantomData, + _marker: Default::default(), }; Self { inner: Arc::new(inner) } } @@ -118,7 +115,7 @@ where } /// Returns a raw pointer to the MDBX environment. - pub fn env(&self) -> &Environment { + pub fn env(&self) -> &Environment { self.inner.env } @@ -253,10 +250,9 @@ where } /// Internals of a transaction. -struct TransactionInner<'env, K, E> +struct TransactionInner<'env, K> where K: TransactionKind, - E: EnvironmentKind, { /// The transaction pointer itself. txn: TransactionPtr, @@ -264,14 +260,13 @@ where primed_dbis: Mutex>, /// Whether the transaction has committed. committed: AtomicBool, - env: &'env Environment, - _marker: PhantomData, + env: &'env Environment, + _marker: std::marker::PhantomData, } -impl<'env, K, E> TransactionInner<'env, K, E> +impl<'env, K> TransactionInner<'env, K> where K: TransactionKind, - E: EnvironmentKind, { /// Marks the transaction as committed. fn set_committed(&self) { @@ -288,10 +283,9 @@ where } } -impl<'env, K, E> Drop for TransactionInner<'env, K, E> +impl<'env, K> Drop for TransactionInner<'env, K> where K: TransactionKind, - E: EnvironmentKind, { fn drop(&mut self) { self.txn_execute(|txn| { @@ -314,10 +308,7 @@ where } } -impl<'env, E> Transaction<'env, RW, E> -where - E: EnvironmentKind, -{ +impl<'env> Transaction<'env, RW> { fn open_db_with_flags(&self, name: Option<&str>, flags: DatabaseFlags) -> Result> { Database::new(self, name, flags.bits()) } @@ -451,10 +442,7 @@ where } } -impl<'env, E> Transaction<'env, RO, E> -where - E: EnvironmentKind, -{ +impl<'env> Transaction<'env, RO> { /// Closes the database handle. /// /// # Safety @@ -467,9 +455,12 @@ where } } -impl<'env> Transaction<'env, RW, NoWriteMap> { +impl<'env> Transaction<'env, RW> { /// Begins a new nested transaction inside of this transaction. - pub fn begin_nested_txn(&mut self) -> Result> { + pub fn begin_nested_txn(&mut self) -> Result> { + if self.inner.env.is_write_map() { + return Err(Error::NestedTransactionsUnsupportedWithWriteMap) + } self.txn_execute(|txn| { let (tx, rx) = sync_channel(0); self.env() @@ -487,10 +478,9 @@ impl<'env> Transaction<'env, RW, NoWriteMap> { } } -impl<'env, K, E> fmt::Debug for Transaction<'env, K, E> +impl<'env, K> fmt::Debug for Transaction<'env, K> where K: TransactionKind, - E: EnvironmentKind, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RoTransaction").finish_non_exhaustive() @@ -526,15 +516,12 @@ unsafe impl Sync for TransactionPtr {} #[cfg(test)] mod tests { use super::*; - use crate::WriteMap; fn assert_send_sync() {} #[allow(dead_code)] fn test_txn_send_sync() { - assert_send_sync::>(); - assert_send_sync::>(); - assert_send_sync::>(); - assert_send_sync::>(); + assert_send_sync::>(); + assert_send_sync::>(); } } diff --git a/crates/storage/libmdbx-rs/tests/cursor.rs b/crates/storage/libmdbx-rs/tests/cursor.rs index efa1d85f0cb6..0e2ce403edf7 100644 --- a/crates/storage/libmdbx-rs/tests/cursor.rs +++ b/crates/storage/libmdbx-rs/tests/cursor.rs @@ -2,8 +2,6 @@ use reth_libmdbx::*; use std::borrow::Cow; use tempfile::tempdir; -type Environment = reth_libmdbx::Environment; - #[test] fn test_get() { let dir = tempdir().unwrap(); diff --git a/crates/storage/libmdbx-rs/tests/environment.rs b/crates/storage/libmdbx-rs/tests/environment.rs index 8dfb61ad4d7c..85cf9a62a584 100644 --- a/crates/storage/libmdbx-rs/tests/environment.rs +++ b/crates/storage/libmdbx-rs/tests/environment.rs @@ -2,8 +2,6 @@ use byteorder::{ByteOrder, LittleEndian}; use reth_libmdbx::*; use tempfile::tempdir; -type Environment = reth_libmdbx::Environment; - #[test] fn test_open() { let dir = tempdir().unwrap(); diff --git a/crates/storage/libmdbx-rs/tests/transaction.rs b/crates/storage/libmdbx-rs/tests/transaction.rs index c577017f4b5b..361bca0e2d49 100644 --- a/crates/storage/libmdbx-rs/tests/transaction.rs +++ b/crates/storage/libmdbx-rs/tests/transaction.rs @@ -7,8 +7,6 @@ use std::{ }; use tempfile::tempdir; -type Environment = reth_libmdbx::Environment; - #[test] fn test_put_get_del() { let dir = tempdir().unwrap();