fix: resize on failed MDB_FULL error

Resize everytime the database indicates it has run out of space rather
than checking for each and every write.

applications/tari_base_node/src/command_handler.rs

@@ -52,7 +52,6 @@ use tari_core::{
         LocalNodeCommsInterface,
     },
     blocks::BlockHeader,
-    chain_storage,
     chain_storage::{async_db::AsyncBlockchainDb, ChainHeader, LMDBDatabase},
     consensus::ConsensusManager,
     mempool::service::LocalMempoolService,
@@ -1057,21 +1056,12 @@ impl CommandHandler {
     }
 
     pub fn get_blockchain_db_stats(&self) {
-        let db = self.blockchain_db.clone();
+        const BYTES_PER_MB: usize = 1024 * 1024;
 
-        fn stat_to_row(stat: &chain_storage::DbStat) -> Vec<String> {
-            row![
-                stat.name,
-                stat.entries,
-                stat.depth,
-                stat.branch_pages,
-                stat.leaf_pages,
-                stat.overflow_pages,
-            ]
-        }
+        let db = self.blockchain_db.clone();
 
         self.executor.spawn(async move {
-            match db.get_stats().await {
+            let total_db_size = match db.get_stats().await {
                 Ok(stats) => {
                     let mut table = Table::new();
                     table.set_titles(vec![
@@ -1081,24 +1071,40 @@ impl CommandHandler {
                         "Branch Pages",
                         "Leaf Pages",
                         "Overflow Pages",
+                        "Est. Size (MiB)",
+                        "% of total",
                     ]);
+                    let total_db_size = stats.db_stats().iter().map(|s| s.total_page_size()).sum::<usize>();
                     stats.db_stats().iter().for_each(|stat| {
-                        table.add_row(stat_to_row(stat));
+                        table.add_row(row![
+                            stat.name,
+                            stat.entries,
+                            stat.depth,
+                            stat.branch_pages,
+                            stat.leaf_pages,
+                            stat.overflow_pages,
+                            format!("{:.2}", stat.total_page_size() as f32 / BYTES_PER_MB as f32),
+                            format!("{:.2}%", (stat.total_page_size() as f32 / total_db_size as f32) * 100.0)
+                        ]);
                     });
+
                     table.print_stdout();
                     println!();
                     println!(
-                        "{} databases, page size: {} bytes, env_info = ({})",
+                        "{} databases, {:.2} MiB used ({:.2}%), page size: {} bytes, env_info = ({})",
                         stats.root().entries,
+                        total_db_size as f32 / BYTES_PER_MB as f32,
+                        (total_db_size as f32 / stats.env_info().mapsize as f32) * 100.0,
                         stats.root().psize as usize,
                         stats.env_info()
                     );
+                    total_db_size
                 },
                 Err(err) => {
                     println!("{}", err);
                     return;
                 },
-            }
+            };
 
             println!();
             println!("Totalling DB entry sizes. This may take a few seconds...");
@@ -1107,21 +1113,31 @@ impl CommandHandler {
                 Ok(stats) => {
                     println!();
                     let mut table = Table::new();
-                    table.set_titles(vec!["Name", "Entries", "Total Size", "Avg. Size/Entry", "% of total"]);
-                    let total_db_size = stats.sizes().iter().map(|s| s.total()).sum::<u64>();
+                    table.set_titles(vec![
+                        "Name",
+                        "Entries",
+                        "Total Size (MiB)",
+                        "Avg. Size/Entry (bytes)",
+                        "% of total",
+                    ]);
+                    let total_data_size = stats.sizes().iter().map(|s| s.total()).sum::<u64>();
                     stats.sizes().iter().for_each(|size| {
-                        let total = size.total() as f32 / 1024.0 / 1024.0;
+                        let total = size.total() as f32 / BYTES_PER_MB as f32;
                         table.add_row(row![
                             size.name,
                             size.num_entries,
-                            format!("{:.2} MiB", total),
-                            format!("{} bytes", size.avg_bytes_per_entry()),
-                            format!("{:.2}%", (size.total() as f32 / total_db_size as f32) * 100.0)
+                            format!("{:.2}", total),
+                            format!("{}", size.avg_bytes_per_entry()),
+                            format!("{:.2}%", (size.total() as f32 / total_data_size as f32) * 100.0)
                         ])
                     });
                     table.print_stdout();
                     println!();
-                    println!("Total data size: {:.2} MiB", total_db_size as f32 / 1024.0 / 1024.0);
+                    println!(
+                        "Total blockchain data size: {:.2} MiB ({:.2} % of LMDB map size)",
+                        total_data_size as f32 / BYTES_PER_MB as f32,
+                        (total_data_size as f32 / total_db_size as f32) * 100.0
+                    );
                 },
                 Err(err) => {
                     println!("{}", err);

base_layer/core/src/chain_storage/db_transaction.rs

@@ -255,10 +255,6 @@ impl DbTransaction {
         &self.operations
     }
 
-    pub(crate) fn into_operations(self) -> Vec<WriteOperation> {
-        self.operations
-    }
-
     /// This will store the seed key with the height. This is called when a block is accepted into the main chain.
     /// This will only update the hieght of the seed, if its lower then currently stored.
     pub fn insert_monero_seed_height(&mut self, monero_seed: Vec<u8>, height: u64) {

base_layer/core/src/chain_storage/error.rs

@@ -21,6 +21,7 @@
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 use crate::{chain_storage::MmrTree, proof_of_work::PowError, validation::ValidationError};
+use lmdb_zero::error;
 use tari_mmr::{error::MerkleMountainRangeError, MerkleProofError};
 use tari_storage::lmdb_store::LMDBError;
 use thiserror::Error;
@@ -109,6 +110,8 @@ pub enum ChainStorageError {
     CannotCalculateNonTipMmr(String),
     #[error("Key {key} in {table_name} already exists")]
     KeyExists { table_name: &'static str, key: String },
+    #[error("Database resize required")]
+    DbResizeRequired,
 }
 
 impl ChainStorageError {
@@ -131,11 +134,12 @@ impl From<lmdb_zero::Error> for ChainStorageError {
     fn from(err: lmdb_zero::Error) -> Self {
         use lmdb_zero::Error::*;
         match err {
-            Code(c) if c == lmdb_zero::error::NOTFOUND => ChainStorageError::ValueNotFound {
+            Code(error::NOTFOUND) => ChainStorageError::ValueNotFound {
                 entity: "<unspecified entity>",
                 field: "<unknown>",
                 value: "<unknown>".to_string(),
             },
+            Code(error::MAP_FULL) => ChainStorageError::DbResizeRequired,
             _ => ChainStorageError::AccessError(err.to_string()),
         }
     }

base_layer/core/src/chain_storage/lmdb_db/lmdb.rs

@@ -87,16 +87,17 @@ where
             val,
             e,
         );
-
         if let lmdb_zero::Error::Code(code) = &e {
             if *code == lmdb_zero::error::KEYEXIST {
                 return ChainStorageError::KeyExists {
                     table_name,
                     key: to_hex(key.as_lmdb_bytes()),
                 };
             }
+            if *code == lmdb_zero::error::MAP_FULL {
+                return ChainStorageError::DbResizeRequired;
+            }
         }
-
         ChainStorageError::InsertError {
             table: table_name,
             error: e.to_string(),
@@ -121,6 +122,11 @@ where
             target: LOG_TARGET,
             "Could not insert value into lmdb transaction: {:?}", e
         );
+        if let lmdb_zero::Error::Code(code) = &e {
+            if *code == lmdb_zero::error::MAP_FULL {
+                return ChainStorageError::DbResizeRequired;
+            }
+        }
         ChainStorageError::AccessError(e.to_string())
     })
 }

base_layer/core/src/chain_storage/stats.rs

@@ -61,21 +61,14 @@ impl DbBasicStats {
 
 impl Display for DbBasicStats {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "Root: psize = {}, {}", self.root.psize, stat_to_string(&self.root))?;
+        writeln!(f, "Root: psize = {}, {}", self.root.psize, self.root)?;
         for stat in &self.db_stats {
-            writeln!(f, "{}", stat_to_string(&stat))?;
+            writeln!(f, "{}", stat)?;
         }
         Ok(())
     }
 }
 
-fn stat_to_string(stat: &DbStat) -> String {
-    format!(
-        "name: {}, entries = {}, depth = {}, branch_pages = {}, leaf_pages = {}, overflow_pages = {}",
-        stat.name, stat.entries, stat.depth, stat.branch_pages, stat.leaf_pages, stat.overflow_pages
-    )
-}
-
 /// Statistics information about an environment.
 #[derive(Debug, Clone, Copy)]
 pub struct DbStat {
@@ -95,6 +88,13 @@ pub struct DbStat {
     pub entries: usize,
 }
 
+impl DbStat {
+    /// Returns the total size in bytes of all pages
+    pub fn total_page_size(&self) -> usize {
+        self.psize as usize * (self.leaf_pages + self.branch_pages + self.overflow_pages)
+    }
+}
+
 impl From<(&'static str, lmdb::Stat)> for DbStat {
     fn from((name, stat): (&'static str, lmdb::Stat)) -> Self {
         Self {
@@ -109,6 +109,23 @@ impl From<(&'static str, lmdb::Stat)> for DbStat {
     }
 }
 
+impl Display for DbStat {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "name: {}, Total page size: {}, entries: {}, depth: {}, branch_pages: {}, leaf_pages: {}, overflow_pages: \
+             {}",
+            self.name,
+            self.total_page_size(),
+            self.entries,
+            self.depth,
+            self.branch_pages,
+            self.leaf_pages,
+            self.overflow_pages,
+        )
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct DbTotalSizeStats {
     sizes: Vec<DbSize>,
@@ -186,8 +203,12 @@ impl Display for EnvInfo {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "mapsize: {}, last_pgno: {}, last_txnid: {}, maxreaders: {}, numreaders: {}",
-            self.mapsize, self.last_pgno, self.last_txnid, self.maxreaders, self.numreaders,
+            "mapsize: {:.2} MiB, last_pgno: {}, last_txnid: {}, maxreaders: {}, numreaders: {}",
+            self.mapsize as f32 / 1024.0 / 1024.0,
+            self.last_pgno,
+            self.last_txnid,
+            self.maxreaders,
+            self.numreaders,
         )
     }
 }

common/src/configuration/global.rs

@@ -41,12 +41,12 @@ use std::{
 use tari_storage::lmdb_store::LMDBConfig;
 
 const DB_INIT_DEFAULT_MB: usize = 1000;
-const DB_GROW_DEFAULT_MB: usize = 500;
-const DB_RESIZE_DEFAULT_MB: usize = 100;
+const DB_GROW_SIZE_DEFAULT_MB: usize = 500;
+const DB_RESIZE_THRESHOLD_DEFAULT_MB: usize = 100;
 
 const DB_INIT_MIN_MB: i64 = 100;
-const DB_GROW_MIN_MB: i64 = 20;
-const DB_RESIZE_MIN_MB: i64 = 10;
+const DB_GROW_SIZE_MIN_MB: i64 = 20;
+const DB_RESIZE_THRESHOLD_MIN_MB: i64 = 10;
 
 //-------------------------------------        Main Configuration Struct      --------------------------------------//
 
@@ -203,25 +203,28 @@ fn convert_node_config(
 
     let key = config_string("base_node", &net_str, "db_grow_size_mb");
     let grow_size_mb = match cfg.get_int(&key) {
-        Ok(mb) if mb < DB_GROW_MIN_MB => {
+        Ok(mb) if mb < DB_GROW_SIZE_MIN_MB => {
             return Err(ConfigurationError::new(
                 &key,
-                &format!("DB grow size must be at least {} MB.", DB_GROW_MIN_MB),
+                &format!("DB grow size must be at least {} MB.", DB_GROW_SIZE_MIN_MB),
             ))
         },
         Ok(mb) => mb as usize,
         Err(e) => match e {
-            ConfigError::NotFound(_) => DB_GROW_DEFAULT_MB, // default
+            ConfigError::NotFound(_) => DB_GROW_SIZE_DEFAULT_MB, // default
             other => return Err(ConfigurationError::new(&key, &other.to_string())),
         },
     };
 
     let key = config_string("base_node", &net_str, "db_resize_threshold_mb");
     let resize_threshold_mb = match cfg.get_int(&key) {
-        Ok(mb) if mb < DB_RESIZE_MIN_MB => {
+        Ok(mb) if mb < DB_RESIZE_THRESHOLD_MIN_MB => {
             return Err(ConfigurationError::new(
                 &key,
-                &format!("DB resize threshold must be at least {} MB.", DB_RESIZE_MIN_MB),
+                &format!(
+                    "DB resize threshold must be at least {} MB.",
+                    DB_RESIZE_THRESHOLD_MIN_MB
+                ),
             ))
         },
         Ok(mb) if mb as usize >= grow_size_mb => {
@@ -238,7 +241,7 @@ fn convert_node_config(
         },
         Ok(mb) => mb as usize,
         Err(e) => match e {
-            ConfigError::NotFound(_) => DB_RESIZE_DEFAULT_MB, // default
+            ConfigError::NotFound(_) => DB_RESIZE_THRESHOLD_DEFAULT_MB, // default
             other => return Err(ConfigurationError::new(&key, &other.to_string())),
         },
     };

infrastructure/storage/src/lmdb_store/store.rs

@@ -6,7 +6,8 @@ use crate::{
 };
 use lmdb_zero::{
     db,
-    error::{self, LmdbResultExt},
+    error,
+    error::LmdbResultExt,
     open,
     put,
     traits::AsLmdbBytes,
@@ -42,25 +43,25 @@ type DatabaseRef = Arc<Database<'static>>;
 #[derive(Debug, Clone)]
 pub struct LMDBConfig {
     init_size_bytes: usize,
-    grow_size_bytes: usize,
+    max_resize_growth_bytes: usize,
     resize_threshold_bytes: usize,
 }
 
 impl LMDBConfig {
     /// Specify LMDB config in bytes.
-    pub fn new(init_size_bytes: usize, grow_size_bytes: usize, resize_threshold_bytes: usize) -> Self {
+    pub fn new(init_size_bytes: usize, max_resize_growth_bytes: usize, resize_threshold_bytes: usize) -> Self {
         Self {
             init_size_bytes,
-            grow_size_bytes,
+            max_resize_growth_bytes,
             resize_threshold_bytes,
         }
     }
 
     /// Specify LMDB config in megabytes.
-    pub fn new_from_mb(init_size_mb: usize, grow_size_mb: usize, resize_threshold_mb: usize) -> Self {
+    pub fn new_from_mb(init_size_mb: usize, max_resize_growth_mb: usize, resize_threshold_mb: usize) -> Self {
         Self {
             init_size_bytes: init_size_mb * BYTES_PER_MB,
-            grow_size_bytes: grow_size_mb * BYTES_PER_MB,
+            max_resize_growth_bytes: max_resize_growth_mb * BYTES_PER_MB,
             resize_threshold_bytes: resize_threshold_mb * BYTES_PER_MB,
         }
     }
@@ -73,7 +74,7 @@ impl LMDBConfig {
     /// Get the grow size of the LMDB environment in bytes. The LMDB environment will be resized by this amount when
     /// `resize_threshold_bytes` are left.
     pub fn grow_size_bytes(&self) -> usize {
-        self.grow_size_bytes
+        self.max_resize_growth_bytes
     }
 
     /// Get the resize threshold in bytes. The LMDB environment will be resized when this much free space is left.
@@ -175,12 +176,12 @@ impl LMDBBuilder {
             builder.set_maxdbs(max_dbs)?;
             // Using open::Flags::NOTLS does not compile!?! NOTLS=0x200000
             let flags = open::Flags::from_bits(0x0020_0000).expect("LMDB open::Flag is correct");
-            builder.open(&path, flags, 0o600)?
+            let env = builder.open(&path, flags, 0o600)?;
+            // SAFETY: no transactions can be open at this point
+            LMDBStore::resize_if_required(&env, &self.env_config)?;
+            Arc::new(env)
         };
-        let env = Arc::new(env);
 
-        // Increase map size if usage gets close to the db size
-        LMDBStore::resize_if_required(&env, &self.env_config)?;
         info!(
             target: LOG_TARGET,
             "({}) LMDB environment created with a capacity of {} MB, {} MB remaining.",
@@ -393,23 +394,32 @@ impl LMDBStore {
         self.env.clone()
     }
 
-    /// Resize the LMDB environment if the resize threshold is breached.
-    pub fn resize_if_required(env: &Environment, config: &LMDBConfig) -> Result<(), LMDBError> {
+    /// Resize the LMDB environment if remaining mapsize is less than the configured resize threshold.
+    ///
+    /// # Safety
+    /// This may only be called if no write transactions are active in the current process. Note that the library does
+    /// not check for this condition, the caller must ensure it explicitly.
+    ///
+    /// http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5
+    pub unsafe fn resize_if_required(env: &Environment, config: &LMDBConfig) -> Result<(), LMDBError> {
         let env_info = env.info()?;
-        let size_used_bytes = env.stat()?.psize as usize * env_info.last_pgno;
+        let stat = env.stat()?;
+        let size_used_bytes = stat.psize as usize * env_info.last_pgno;
         let size_left_bytes = env_info.mapsize - size_used_bytes;
+        debug!(
+            target: LOG_TARGET,
+            "Resize check: Used bytes: {}, Remaining bytes: {}", size_used_bytes, size_left_bytes
+        );
 
         if size_left_bytes <= config.resize_threshold_bytes {
-            unsafe {
-                env.set_mapsize(size_used_bytes + config.grow_size_bytes)?;
-            }
+            env.set_mapsize(env_info.mapsize + config.max_resize_growth_bytes)?;
             debug!(
                 target: LOG_TARGET,
-                "({}) LMDB size used {:?} MB, environment space left {:?} MB, increased by {:?} MB.",
+                "({}) LMDB size used {:?} MB, environment space left {:?} MB, increased by {:?} MB",
                 env.path()?.to_str()?,
                 size_used_bytes / BYTES_PER_MB,
                 size_left_bytes / BYTES_PER_MB,
-                config.grow_size_bytes / BYTES_PER_MB,
+                config.max_resize_growth_bytes / BYTES_PER_MB,
             );
         }
         Ok(())
@@ -432,17 +442,39 @@ impl LMDBDatabase {
         K: AsLmdbBytes + ?Sized,
         V: Serialize,
     {
-        let env = &(*self.db.env());
+        const MAX_RESIZES: usize = 3;
+
+        for _ in 0..MAX_RESIZES {
+            let tx = self.write_transaction()?;
+            {
+                let mut accessor = tx.access();
+                let buf = LMDBWriteTransaction::convert_value(value)?;
+                accessor.put(&*self.db, key, &buf, put::Flags::empty())?;
+            }
+            match tx.commit() {
+                Ok(txn) => return Ok(txn),
+                Err(error::Error::Code(code)) if code == error::MAP_RESIZED => {
+                    info!(
+                        target: LOG_TARGET,
+                        "Failed to obtain write transaction because the database needs to be resized"
+                    );
+                    // SAFETY: We know that there are no open transactions at this point because ...
+                    // TODO: we don't guarantee this here but it works because the caller does this.
+                    unsafe {
+                        LMDBStore::resize_if_required(&self.env, &self.env_config)?;
+                    }
+                },
+                Err(e) => return Err(e.into()),
+            }
+        }
 
-        LMDBStore::resize_if_required(env, &self.env_config)?;
+        // Failed to resize
+        Err(error::Error::Code(error::MAP_RESIZED).into())
+    }
 
-        let tx = WriteTransaction::new(env)?;
-        {
-            let mut accessor = tx.access();
-            let buf = LMDBWriteTransaction::convert_value(value)?;
-            accessor.put(&*self.db, key, &buf, put::Flags::empty())?;
-        }
-        tx.commit().map_err(LMDBError::from)
+    pub fn write_transaction(&self) -> Result<WriteTransaction<'_>, LMDBError> {
+        let txn = WriteTransaction::new(self.db.env())?;
+        Ok(txn)
     }
 
     /// Get a value from the database. This is an atomic operation. A read transaction is created, the value

infrastructure/storage/tests/lmdb.rs

@@ -360,10 +360,7 @@ fn test_lmdb_resize_before_full() {
             // our 1MB env size should be out of space
             // however the db should now be allocating additional space as it fills up
             for key in 0..32 {
-                if let Err(_e) = db.insert(&key, &value) {
-                    // println!("LMDBError {:#?}", _e);
-                    panic!("Failed to resize the LMDB store.");
-                }
+                db.insert(&key, &value).unwrap();
             }
             let env_info = store.env().info().unwrap();
             let psize = store.env().stat().unwrap().psize as usize;