diff --git a/tools/database/src/analyze_delayed_receipt.rs b/tools/database/src/analyze_delayed_receipt.rs new file mode 100644 index 00000000000..4fd87a5ba15 --- /dev/null +++ b/tools/database/src/analyze_delayed_receipt.rs @@ -0,0 +1,123 @@ +use clap::Parser; +use near_store::flat::FlatStorageManager; +use near_store::{get_delayed_receipt_indices, ShardTries, StateSnapshotConfig, TrieConfig}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::rc::Rc; + +use near_chain::ChainStore; +use near_chain_configs::GenesisValidationMode; +use nearcore::config::load_config; + +use near_primitives::hash::CryptoHash; +use near_primitives::shard_layout::{ShardLayout, ShardUId}; +use near_primitives::types::BlockHeight; + +use crate::block_iterators::{ + make_block_iterator_from_command_args, CommandArgs, LastNBlocksIterator, +}; +use nearcore::open_storage; + +#[derive(Parser)] +pub(crate) struct AnalyzeDelayedReceiptCommand { + /// Analyse the last N blocks in the blockchain + #[arg(long)] + last_blocks: Option, + + /// Analyse blocks from the given block height, inclusive + #[arg(long)] + from_block_height: Option, + + /// Analyse blocks up to the given block height, inclusive + #[arg(long)] + to_block_height: Option, +} + +impl AnalyzeDelayedReceiptCommand { + pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { + let mut near_config = load_config(home, GenesisValidationMode::Full).unwrap(); + let node_storage = open_storage(&home, &mut near_config).unwrap(); + let store = node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); + let chain_store = Rc::new(ChainStore::new( + store.clone(), + near_config.genesis.config.genesis_height, + false, + )); + let shard_layout = ShardLayout::get_simple_nightshade_layout(); + // hard code the shard ids + let shard_ids = vec![0, 1, 2, 3]; + let shard_uids = shard_ids + .iter().map(|i| ShardUId::from_shard_id_and_layout(*i, &shard_layout)) + .collect::>(); + let shard_tries = ShardTries::new( + store.clone(), + TrieConfig::default(), + &shard_uids, + FlatStorageManager::new(store.clone()), + StateSnapshotConfig::default(), + ); + // Create an iterator over the blocks that should be analysed + let blocks_iter_opt = make_block_iterator_from_command_args( + CommandArgs { + last_blocks: self.last_blocks, + from_block_height: self.from_block_height, + to_block_height: self.to_block_height, + }, + chain_store.clone(), + ); + + let blocks_iter = match blocks_iter_opt { + Some(iter) => iter, + None => { + println!("No arguments, defaulting to last 100 blocks"); + Box::new(LastNBlocksIterator::new(100, chain_store.clone())) + } + }; + + let mut blocks_count: usize = 0; + let mut first_analysed_block: Option<(BlockHeight, CryptoHash)> = None; + let mut last_analysed_block: Option<(BlockHeight, CryptoHash)> = None; + let mut shard_id_to_congested = HashMap::new(); + + for block in blocks_iter { + blocks_count += 1; + if first_analysed_block.is_none() { + first_analysed_block = Some((block.header().height(), *block.hash())); + } + last_analysed_block = Some((block.header().height(), *block.hash())); + + for chunk_header in block.chunks().iter() { + let state_root = chunk_header.prev_state_root(); + let trie_update = shard_tries.get_trie_for_shard( + ShardUId::from_shard_id_and_layout(chunk_header.shard_id(), &shard_layout), + state_root, + ); + let delayed_receipt_indices = get_delayed_receipt_indices(&trie_update)?; + if delayed_receipt_indices.len() > 0 { + *shard_id_to_congested.entry(chunk_header.shard_id()).or_insert(0) += 1; + } + println!( + "block height {} shard {} delayed receipts {}", + block.header().height(), + chunk_header.shard_id(), + delayed_receipt_indices.len() + ); + } + } + + println!("Analysed {} blocks between:", blocks_count); + if let Some((block_height, block_hash)) = first_analysed_block { + println!("Block: height = {block_height}, hash = {block_hash}"); + } + if let Some((block_height, block_hash)) = last_analysed_block { + println!("Block: height = {block_height}, hash = {block_hash}"); + } + + for shard_id in shard_ids { + let congested_chunks = *shard_id_to_congested.get(&shard_id).unwrap_or(&0); + println!("shard {} congested ratio {}", shard_id, congested_chunks as f64 / blocks_count as f64); + } + + Ok(()) + } +} diff --git a/tools/database/src/commands.rs b/tools/database/src/commands.rs index ef9060b7118..ae08a249bc7 100644 --- a/tools/database/src/commands.rs +++ b/tools/database/src/commands.rs @@ -1,6 +1,7 @@ use crate::adjust_database::ChangeDbKindCommand; use crate::analyse_data_size_distribution::AnalyseDataSizeDistributionCommand; use crate::analyse_gas_usage::AnalyseGasUsageCommand; +use crate::analyze_delayed_receipt::AnalyzeDelayedReceiptCommand; use crate::compact::RunCompactionCommand; use crate::corrupt::CorruptStateSnapshotCommand; use crate::make_snapshot::MakeSnapshotCommand; @@ -46,6 +47,8 @@ enum SubCommand { /// Loads an in-memory trie for research purposes. LoadMemTrie(LoadMemTrieCommand), + + AnalyzeDelayedReceipt(AnalyzeDelayedReceiptCommand), } impl DatabaseCommand { @@ -74,6 +77,7 @@ impl DatabaseCommand { .unwrap_or_else(|e| panic!("Error loading config: {:#}", e)); cmd.run(near_config, home) } + SubCommand::AnalyzeDelayedReceipt(cmd) => cmd.run(home), } } } diff --git a/tools/database/src/lib.rs b/tools/database/src/lib.rs index 827b5da4a63..3493dd84517 100644 --- a/tools/database/src/lib.rs +++ b/tools/database/src/lib.rs @@ -1,6 +1,7 @@ mod adjust_database; mod analyse_data_size_distribution; mod analyse_gas_usage; +mod analyze_delayed_receipt; mod block_iterators; pub mod commands; mod compact;