From a001e6d0edb21922bb2f9365456cd1a2eaf214d1 Mon Sep 17 00:00:00 2001 From: Stanimal Date: Tue, 31 Aug 2021 11:05:36 +0400 Subject: [PATCH] feat: add get-blockchain-db-stats command Adds `get-db-stats` command. This returns the LMDB entry stats and the total entry sizes for each internal blockchain db. --- .../tari_base_node/src/command_handler.rs | 79 ++++++- applications/tari_base_node/src/parser.rs | 7 + applications/tari_base_node/src/table.rs | 24 ++- base_layer/core/src/chain_storage/async_db.rs | 6 + .../src/chain_storage/blockchain_backend.rs | 9 + .../src/chain_storage/blockchain_database.rs | 14 ++ .../core/src/chain_storage/lmdb_db/lmdb.rs | 17 ++ .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 59 ++++++ base_layer/core/src/chain_storage/mod.rs | 3 + base_layer/core/src/chain_storage/stats.rs | 193 ++++++++++++++++++ .../tests/blockchain_database.rs | 23 +++ .../core/src/test_helpers/blockchain.rs | 10 + 12 files changed, 440 insertions(+), 4 deletions(-) create mode 100644 base_layer/core/src/chain_storage/stats.rs diff --git a/applications/tari_base_node/src/command_handler.rs b/applications/tari_base_node/src/command_handler.rs index 562fa65e907..e4aaf0957f9 100644 --- a/applications/tari_base_node/src/command_handler.rs +++ b/applications/tari_base_node/src/command_handler.rs @@ -52,6 +52,7 @@ use tari_core::{ LocalNodeCommsInterface, }, blocks::BlockHeader, + chain_storage, chain_storage::{async_db::AsyncBlockchainDb, ChainHeader, LMDBDatabase}, consensus::ConsensusManager, mempool::service::LocalMempoolService, @@ -503,7 +504,7 @@ impl CommandHandler { info_str, ]); } - table.print_std(); + table.print_stdout(); println!("{} peer(s) known by this node", num_peers); }, @@ -677,7 +678,7 @@ impl CommandHandler { ]); } - table.print_std(); + table.print_stdout(); println!("{} active connection(s)", num_connections); }, @@ -1054,6 +1055,80 @@ impl CommandHandler { pub(crate) fn get_software_updater(&self) -> SoftwareUpdaterHandle { self.software_updater.clone() } + + pub fn get_blockchain_db_stats(&self) { + let db = self.blockchain_db.clone(); + + fn stat_to_row(stat: &chain_storage::DbStat) -> Vec { + row![ + stat.name, + stat.entries, + stat.depth, + stat.branch_pages, + stat.leaf_pages, + stat.overflow_pages, + ] + } + + self.executor.spawn(async move { + match db.get_stats().await { + Ok(stats) => { + let mut table = Table::new(); + table.set_titles(vec![ + "Name", + "Entries", + "Depth", + "Branch Pages", + "Leaf Pages", + "Overflow Pages", + ]); + stats.db_stats().iter().for_each(|stat| { + table.add_row(stat_to_row(stat)); + }); + table.print_stdout(); + println!(); + println!( + "{} databases, total page size: {} bytes, env_info = ({})", + stats.root().entries, + stats.root().psize as usize * stats.env_info().last_pgno, + stats.env_info() + ); + }, + Err(err) => { + println!("{}", err); + return; + }, + } + + println!(); + println!("Totalling DB entry sizes. This may take a few seconds..."); + println!(); + match db.fetch_total_size_stats().await { + Ok(stats) => { + println!(); + let mut table = Table::new(); + table.set_titles(vec!["Name", "Entries", "Total Size", "Avg. Size/Entry", "% of total"]); + let total_db_size = stats.sizes().iter().map(|s| s.total()).sum::(); + stats.sizes().iter().for_each(|size| { + let total = size.total() as f32 / 1024.0 / 1024.0; + table.add_row(row![ + size.name, + size.num_entries, + format!("{:.2} MiB", total), + format!("{} bytes", size.avg_bytes_per_entry()), + format!("{:.2}%", (size.total() as f32 / total_db_size as f32) * 100.0) + ]) + }); + table.print_stdout(); + println!(); + println!("Total data size: {:.2} MiB", total_db_size as f32 / 1024.0 / 1024.0); + }, + Err(err) => { + println!("{}", err); + }, + } + }); + } } async fn fetch_banned_peers(pm: &PeerManager) -> Result, PeerManagerError> { diff --git a/applications/tari_base_node/src/parser.rs b/applications/tari_base_node/src/parser.rs index 79fdf27efc4..9296496eea5 100644 --- a/applications/tari_base_node/src/parser.rs +++ b/applications/tari_base_node/src/parser.rs @@ -53,6 +53,7 @@ pub enum BaseNodeCommand { CheckForUpdates, Status, GetChainMetadata, + GetDbStats, GetPeer, ListPeers, DialPeer, @@ -184,6 +185,9 @@ impl Parser { GetChainMetadata => { self.command_handler.get_chain_meta(); }, + GetDbStats => { + self.command_handler.get_blockchain_db_stats(); + }, DialPeer => { self.process_dial_peer(args); }, @@ -285,6 +289,9 @@ impl Parser { GetChainMetadata => { println!("Gets your base node chain meta data"); }, + GetDbStats => { + println!("Gets your base node database stats"); + }, DialPeer => { println!("Attempt to connect to a known peer"); }, diff --git a/applications/tari_base_node/src/table.rs b/applications/tari_base_node/src/table.rs index 89b01f7d522..816909652b2 100644 --- a/applications/tari_base_node/src/table.rs +++ b/applications/tari_base_node/src/table.rs @@ -48,6 +48,8 @@ impl<'t, 's> Table<'t, 's> { pub fn render(&self, out: &mut T) -> io::Result<()> { self.render_titles(out)?; + out.write_all(b"\n")?; + self.render_separator(out)?; if !self.rows.is_empty() { out.write_all(b"\n")?; self.render_rows(out)?; @@ -56,7 +58,7 @@ impl<'t, 's> Table<'t, 's> { Ok(()) } - pub fn print_std(&self) { + pub fn print_stdout(&self) { self.render(&mut io::stdout()).unwrap(); } @@ -106,6 +108,23 @@ impl<'t, 's> Table<'t, 's> { } Ok(()) } + + fn render_separator(&self, out: &mut T) -> io::Result<()> { + if let Some(rows_len) = self.rows.first().map(|r| r.len()) { + for i in 0..rows_len { + let width = self.col_width(i); + let pad_left = if i == 0 { "" } else { " " }; + out.write_all(pad_left.as_bytes())?; + let sep = "-".repeat(width); + out.write_all(sep.as_bytes())?; + out.write_all(" ".as_bytes())?; + if i < rows_len - 1 { + out.write_all(self.delim_str.as_bytes())?; + } + } + } + Ok(()) + } } macro_rules! row { @@ -141,7 +160,8 @@ mod test { table.render(&mut buf).unwrap(); assert_eq!( String::from_utf8_lossy(&buf.into_inner()), - "Name | Age | Telephone Number | Favourite Headwear \nTrevor | 132 | +123 12323223 | Pith Helmet \n\nHatless | 2 \n" + "Name | Age | Telephone Number | Favourite Headwear \n------- | --- | ---------------- | \ + ------------------ \nTrevor | 132 | +123 12323223 | Pith Helmet \n\nHatless | 2 \n" ); } } diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs index ca1aea97c5c..579d319d3cf 100644 --- a/base_layer/core/src/chain_storage/async_db.rs +++ b/base_layer/core/src/chain_storage/async_db.rs @@ -32,6 +32,8 @@ use crate::{ ChainHeader, ChainStorageError, CompleteDeletedBitmap, + DbBasicStats, + DbTotalSizeStats, DbTransaction, HistoricalBlock, HorizonData, @@ -226,6 +228,10 @@ impl AsyncBlockchainDb { make_async_fn!(fetch_block_hashes_from_header_tip(n: usize, offset: usize) -> Vec, "fetch_block_hashes_from_header_tip"); make_async_fn!(fetch_complete_deleted_bitmap_at(hash: HashOutput) -> CompleteDeletedBitmap, "fetch_deleted_bitmap"); + + make_async_fn!(get_stats() -> DbBasicStats, "get_stats"); + + make_async_fn!(fetch_total_size_stats() -> DbTotalSizeStats, "fetch_total_size_stats"); } impl From> for AsyncBlockchainDb { diff --git a/base_layer/core/src/chain_storage/blockchain_backend.rs b/base_layer/core/src/chain_storage/blockchain_backend.rs index 505d25dda7c..c1f57c09d35 100644 --- a/base_layer/core/src/chain_storage/blockchain_backend.rs +++ b/base_layer/core/src/chain_storage/blockchain_backend.rs @@ -8,7 +8,9 @@ use crate::{ ChainBlock, ChainHeader, ChainStorageError, + DbBasicStats, DbKey, + DbTotalSizeStats, DbTransaction, DbValue, HorizonData, @@ -158,4 +160,11 @@ pub trait BlockchainBackend: Send + Sync { fn fetch_monero_seed_first_seen_height(&self, seed: &[u8]) -> Result; fn fetch_horizon_data(&self) -> Result, ChainStorageError>; + + /// Returns basic database stats for each internal database, such as number of entries and page sizes. This call may + /// not apply to every database implementation. + fn get_stats(&self) -> Result; + /// Returns total size information about each internal database. This call may be very slow and will obtain a read + /// lock for the duration. + fn fetch_total_size_stats(&self) -> Result; } diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs index f2bf132cf40..c4d03fe64e3 100644 --- a/base_layer/core/src/chain_storage/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/blockchain_database.rs @@ -35,6 +35,8 @@ use crate::{ BlockchainBackend, ChainBlock, ChainHeader, + DbBasicStats, + DbTotalSizeStats, HistoricalBlock, HorizonData, MmrTree, @@ -903,6 +905,18 @@ where B: BlockchainBackend chain_metadata.best_block().clone(), )) } + + pub fn get_stats(&self) -> Result { + let lock = self.db_read_access()?; + lock.get_stats() + } + + /// Returns total size information about each internal database. This call may be very slow and will obtain a read + /// lock for the duration. + pub fn fetch_total_size_stats(&self) -> Result { + let lock = self.db_read_access()?; + lock.fetch_total_size_stats() + } } fn unexpected_result(req: DbKey, res: DbValue) -> Result { diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs index dbaf0e6870f..b2c6bc7d9c2 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb.rs @@ -77,6 +77,7 @@ where V: Serialize + Debug, { let val_buf = serialize(val)?; + trace!(target: LOG_TARGET, "LMDB: {} bytes inserted", val_buf.len()); txn.access().put(&db, key, &val_buf, put::NOOVERWRITE).map_err(|e| { error!( target: LOG_TARGET, @@ -387,3 +388,19 @@ where } Ok(result) } + +/// Fetches all the size of all key/values in the given DB. Returns the number of entries, the total size of all the +/// keys and values in bytes. +pub fn fetch_db_entry_sizes(txn: &ConstTransaction<'_>, db: &Database) -> Result<(u64, u64, u64), ChainStorageError> { + let access = txn.access(); + let mut cursor = txn.cursor(db)?; + let mut num_entries = 0; + let mut total_key_size = 0; + let mut total_value_size = 0; + while let Some((key, value)) = cursor.next::<[u8], [u8]>(&access).to_opt()? { + num_entries += 1; + total_key_size += key.len() as u64; + total_value_size += value.len() as u64; + } + Ok((num_entries, total_key_size, total_value_size)) +} diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index e49aee1fedd..cd15d4d38c8 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -27,6 +27,7 @@ use crate::{ error::{ChainStorageError, OrNotFound}, lmdb_db::{ lmdb::{ + fetch_db_entry_sizes, lmdb_delete, lmdb_delete_key_value, lmdb_delete_keys_starting_with, @@ -65,9 +66,12 @@ use crate::{ LMDB_DB_UTXO_COMMITMENT_INDEX, LMDB_DB_UTXO_MMR_SIZE_INDEX, }, + stats::DbTotalSizeStats, BlockchainBackend, ChainBlock, ChainHeader, + DbBasicStats, + DbSize, HorizonData, MmrTree, PrunedOutput, @@ -360,6 +364,33 @@ impl LMDBDatabase { Ok(()) } + fn all_dbs(&self) -> [(&'static str, &DatabaseRef); 19] { + [ + ("metadata_db", &self.metadata_db), + ("headers_db", &self.headers_db), + ("header_accumulated_data_db", &self.header_accumulated_data_db), + ("block_accumulated_data_db", &self.block_accumulated_data_db), + ("block_hashes_db", &self.block_hashes_db), + ("utxos_db", &self.utxos_db), + ("inputs_db", &self.inputs_db), + ("txos_hash_to_index_db", &self.txos_hash_to_index_db), + ("kernels_db", &self.kernels_db), + ("kernel_excess_index", &self.kernel_excess_index), + ("kernel_excess_sig_index", &self.kernel_excess_sig_index), + ("kernel_mmr_size_index", &self.kernel_mmr_size_index), + ("output_mmr_size_index", &self.output_mmr_size_index), + ("utxo_commitment_index", &self.utxo_commitment_index), + ("orphans_db", &self.orphans_db), + ( + "orphan_header_accumulated_data_db", + &self.orphan_header_accumulated_data_db, + ), + ("monero_seed_height_db", &self.monero_seed_height_db), + ("orphan_chain_tips_db", &self.orphan_chain_tips_db), + ("orphan_parent_map_index", &self.orphan_parent_map_index), + ] + } + fn prune_output( &self, txn: &WriteTransaction<'_>, @@ -2043,6 +2074,34 @@ impl BlockchainBackend for LMDBDatabase { let txn = self.read_transaction()?; fetch_horizon_data(&txn, &self.metadata_db) } + + fn get_stats(&self) -> Result { + let global = self.env.stat()?; + let env_info = self.env.info()?; + + let txn = self.read_transaction()?; + let db_stats = self + .all_dbs() + .iter() + .map(|(name, db)| txn.db_stat(db).map(|s| (*name, s))) + .collect::, _>>()?; + Ok(DbBasicStats::new(global, env_info, db_stats)) + } + + fn fetch_total_size_stats(&self) -> Result { + let txn = self.read_transaction()?; + self.all_dbs() + .iter() + .map(|(name, db)| { + fetch_db_entry_sizes(&txn, db).map(|(num_entries, total_key_size, total_value_size)| DbSize { + name, + num_entries, + total_key_size, + total_value_size, + }) + }) + .collect() + } } // Fetch the chain metadata diff --git a/base_layer/core/src/chain_storage/mod.rs b/base_layer/core/src/chain_storage/mod.rs index 11755caccbd..c6fdc3f6390 100644 --- a/base_layer/core/src/chain_storage/mod.rs +++ b/base_layer/core/src/chain_storage/mod.rs @@ -92,5 +92,8 @@ pub use lmdb_db::{ LMDB_DB_UTXOS, }; +mod stats; +pub use stats::{DbBasicStats, DbSize, DbStat, DbTotalSizeStats}; + mod target_difficulties; pub use target_difficulties::TargetDifficulties; diff --git a/base_layer/core/src/chain_storage/stats.rs b/base_layer/core/src/chain_storage/stats.rs new file mode 100644 index 00000000000..e38092b7cb3 --- /dev/null +++ b/base_layer/core/src/chain_storage/stats.rs @@ -0,0 +1,193 @@ +// Copyright 2021, The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use lmdb_zero as lmdb; +use std::{ + fmt::{Display, Formatter}, + iter::FromIterator, +}; + +#[derive(Debug, Clone)] +pub struct DbBasicStats { + root: DbStat, + env_info: EnvInfo, + db_stats: Vec, +} + +impl DbBasicStats { + pub(super) fn new>( + global: lmdb::Stat, + env_info: lmdb::EnvInfo, + db_stats: I, + ) -> Self { + Self { + root: ("[root]", global).into(), + env_info: env_info.into(), + db_stats: db_stats.into_iter().map(Into::into).collect(), + } + } + + pub fn root(&self) -> &DbStat { + &self.root + } + + pub fn env_info(&self) -> &EnvInfo { + &self.env_info + } + + pub fn db_stats(&self) -> &[DbStat] { + &self.db_stats + } +} + +impl Display for DbBasicStats { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + writeln!(f, "Root: psize = {}, {}", self.root.psize, stat_to_string(&self.root))?; + for stat in &self.db_stats { + writeln!(f, "{}", stat_to_string(&stat))?; + } + Ok(()) + } +} + +fn stat_to_string(stat: &DbStat) -> String { + format!( + "name: {}, entries = {}, depth = {}, branch_pages = {}, leaf_pages = {}, overflow_pages = {}", + stat.name, stat.entries, stat.depth, stat.branch_pages, stat.leaf_pages, stat.overflow_pages + ) +} + +/// Statistics information about an environment. +#[derive(Debug, Clone, Copy)] +pub struct DbStat { + /// Name of the db + pub name: &'static str, + /// Size of a database page. This is currently the same for all databases. + pub psize: u32, + /// Depth (height) of the B-tree + pub depth: u32, + /// Number of internal (non-leaf) pages + pub branch_pages: usize, + /// Number of leaf pages + pub leaf_pages: usize, + /// Number of overflow pages + pub overflow_pages: usize, + /// Number of data items + pub entries: usize, +} + +impl From<(&'static str, lmdb::Stat)> for DbStat { + fn from((name, stat): (&'static str, lmdb::Stat)) -> Self { + Self { + name, + psize: stat.psize, + depth: stat.depth, + branch_pages: stat.branch_pages, + leaf_pages: stat.leaf_pages, + overflow_pages: stat.overflow_pages, + entries: stat.entries, + } + } +} + +#[derive(Debug, Clone)] +pub struct DbTotalSizeStats { + sizes: Vec, +} +impl DbTotalSizeStats { + pub fn sizes(&self) -> &[DbSize] { + &self.sizes + } +} + +#[derive(Debug, Clone, Copy)] +pub struct DbSize { + pub name: &'static str, + pub num_entries: u64, + pub total_key_size: u64, + pub total_value_size: u64, +} + +impl DbSize { + pub fn total(&self) -> u64 { + self.total_key_size.saturating_add(self.total_value_size) + } + + pub fn avg_bytes_per_entry(&self) -> u64 { + if self.num_entries == 0 { + return 0; + } + + self.total() / self.num_entries + } +} + +impl From> for DbTotalSizeStats { + fn from(sizes: Vec) -> Self { + Self { sizes } + } +} + +impl FromIterator for DbTotalSizeStats { + fn from_iter>(iter: T) -> Self { + Self { + sizes: iter.into_iter().collect(), + } + } +} + +/// Configuration information about an environment. +#[derive(Debug, Clone, Copy)] +pub struct EnvInfo { + /// Size of the data memory map + pub mapsize: usize, + /// ID of the last used page + pub last_pgno: usize, + /// ID of the last committed transaction + pub last_txnid: usize, + /// max reader slots in the environment + pub maxreaders: u32, + /// max reader slots used in the environment + pub numreaders: u32, +} + +impl From for EnvInfo { + fn from(info: lmdb::EnvInfo) -> Self { + Self { + mapsize: info.mapsize, + last_pgno: info.last_pgno, + last_txnid: info.last_txnid, + maxreaders: info.maxreaders, + numreaders: info.numreaders, + } + } +} + +impl Display for EnvInfo { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "mapsize: {}, last_pgno: {}, last_txnid: {}, maxreaders: {}, numreaders: {}", + self.mapsize, self.last_pgno, self.last_txnid, self.maxreaders, self.numreaders, + ) + } +} diff --git a/base_layer/core/src/chain_storage/tests/blockchain_database.rs b/base_layer/core/src/chain_storage/tests/blockchain_database.rs index 0a3807fad39..fed8b18104f 100644 --- a/base_layer/core/src/chain_storage/tests/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/tests/blockchain_database.rs @@ -406,3 +406,26 @@ mod add_block { db.add_block(block).unwrap().assert_added(); } } + +mod get_stats { + use super::*; + + #[test] + fn it_works_when_db_is_empty() { + let db = setup(); + let stats = db.get_stats().unwrap(); + assert_eq!(stats.root().depth, 1); + } +} + +mod fetch_total_size_stats { + use super::*; + + #[test] + fn it_works_when_db_is_empty() { + let db = setup(); + let stats = db.fetch_total_size_stats().unwrap(); + // Returns one per db + assert_eq!(stats.sizes().len(), 19); + } +} diff --git a/base_layer/core/src/test_helpers/blockchain.rs b/base_layer/core/src/test_helpers/blockchain.rs index d52df6deb7a..7f1af539e72 100644 --- a/base_layer/core/src/test_helpers/blockchain.rs +++ b/base_layer/core/src/test_helpers/blockchain.rs @@ -45,7 +45,9 @@ use crate::{ ChainBlock, ChainHeader, ChainStorageError, + DbBasicStats, DbKey, + DbTotalSizeStats, DbTransaction, DbValue, DeletedBitmap, @@ -330,4 +332,12 @@ impl BlockchainBackend for TempDatabase { fn fetch_horizon_data(&self) -> Result, ChainStorageError> { self.db.fetch_horizon_data() } + + fn get_stats(&self) -> Result { + self.db.get_stats() + } + + fn fetch_total_size_stats(&self) -> Result { + self.db.fetch_total_size_stats() + } }