analyze delayed receipts

tools/database/src/analyze_delayed_receipt.rs

@@ -0,0 +1,123 @@
+use clap::Parser;
+use near_store::flat::FlatStorageManager;
+use near_store::{get_delayed_receipt_indices, ShardTries, StateSnapshotConfig, TrieConfig};
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::rc::Rc;
+
+use near_chain::ChainStore;
+use near_chain_configs::GenesisValidationMode;
+use nearcore::config::load_config;
+
+use near_primitives::hash::CryptoHash;
+use near_primitives::shard_layout::{ShardLayout, ShardUId};
+use near_primitives::types::BlockHeight;
+
+use crate::block_iterators::{
+    make_block_iterator_from_command_args, CommandArgs, LastNBlocksIterator,
+};
+use nearcore::open_storage;
+
+#[derive(Parser)]
+pub(crate) struct AnalyzeDelayedReceiptCommand {
+    /// Analyse the last N blocks in the blockchain
+    #[arg(long)]
+    last_blocks: Option<u64>,
+
+    /// Analyse blocks from the given block height, inclusive
+    #[arg(long)]
+    from_block_height: Option<BlockHeight>,
+
+    /// Analyse blocks up to the given block height, inclusive
+    #[arg(long)]
+    to_block_height: Option<BlockHeight>,
+}
+
+impl AnalyzeDelayedReceiptCommand {
+    pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> {
+        let mut near_config = load_config(home, GenesisValidationMode::Full).unwrap();
+        let node_storage = open_storage(&home, &mut near_config).unwrap();
+        let store = node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store());
+        let chain_store = Rc::new(ChainStore::new(
+            store.clone(),
+            near_config.genesis.config.genesis_height,
+            false,
+        ));
+        let shard_layout = ShardLayout::get_simple_nightshade_layout();
+        // hard code the shard ids
+        let shard_ids = vec![0, 1, 2, 3]; 
+        let shard_uids = shard_ids
+            .iter().map(|i| ShardUId::from_shard_id_and_layout(*i, &shard_layout))
+            .collect::<Vec<_>>();
+        let shard_tries = ShardTries::new(
+            store.clone(),
+            TrieConfig::default(),
+            &shard_uids,
+            FlatStorageManager::new(store.clone()),
+            StateSnapshotConfig::default(),
+        );
+        // Create an iterator over the blocks that should be analysed
+        let blocks_iter_opt = make_block_iterator_from_command_args(
+            CommandArgs {
+                last_blocks: self.last_blocks,
+                from_block_height: self.from_block_height,
+                to_block_height: self.to_block_height,
+            },
+            chain_store.clone(),
+        );
+
+        let blocks_iter = match blocks_iter_opt {
+            Some(iter) => iter,
+            None => {
+                println!("No arguments, defaulting to last 100 blocks");
+                Box::new(LastNBlocksIterator::new(100, chain_store.clone()))
+            }
+        };
+
+        let mut blocks_count: usize = 0;
+        let mut first_analysed_block: Option<(BlockHeight, CryptoHash)> = None;
+        let mut last_analysed_block: Option<(BlockHeight, CryptoHash)> = None;
+        let mut shard_id_to_congested = HashMap::new();
+
+        for block in blocks_iter {
+            blocks_count += 1;
+            if first_analysed_block.is_none() {
+                first_analysed_block = Some((block.header().height(), *block.hash()));
+            }
+            last_analysed_block = Some((block.header().height(), *block.hash()));
+
+            for chunk_header in block.chunks().iter() {
+                let state_root = chunk_header.prev_state_root();
+                let trie_update = shard_tries.get_trie_for_shard(
+                    ShardUId::from_shard_id_and_layout(chunk_header.shard_id(), &shard_layout),
+                    state_root,
+                );
+                let delayed_receipt_indices = get_delayed_receipt_indices(&trie_update)?;
+                if delayed_receipt_indices.len() > 0 {
+                    *shard_id_to_congested.entry(chunk_header.shard_id()).or_insert(0) += 1;
+                }
+                println!(
+                    "block height {} shard {} delayed receipts {}",
+                    block.header().height(),
+                    chunk_header.shard_id(),
+                    delayed_receipt_indices.len()
+                );
+            }
+        }
+
+        println!("Analysed {} blocks between:", blocks_count);
+        if let Some((block_height, block_hash)) = first_analysed_block {
+            println!("Block: height = {block_height}, hash = {block_hash}");
+        }
+        if let Some((block_height, block_hash)) = last_analysed_block {
+            println!("Block: height = {block_height}, hash = {block_hash}");
+        }
+
+        for shard_id in shard_ids {
+            let congested_chunks = *shard_id_to_congested.get(&shard_id).unwrap_or(&0);
+            println!("shard {} congested ratio {}", shard_id, congested_chunks as f64 / blocks_count as f64);
+        }
+
+        Ok(())
+    }
+}

tools/database/src/commands.rs

@@ -1,6 +1,7 @@
 use crate::adjust_database::ChangeDbKindCommand;
 use crate::analyse_data_size_distribution::AnalyseDataSizeDistributionCommand;
 use crate::analyse_gas_usage::AnalyseGasUsageCommand;
+use crate::analyze_delayed_receipt::AnalyzeDelayedReceiptCommand;
 use crate::compact::RunCompactionCommand;
 use crate::corrupt::CorruptStateSnapshotCommand;
 use crate::make_snapshot::MakeSnapshotCommand;
@@ -46,6 +47,8 @@ enum SubCommand {
 
     /// Loads an in-memory trie for research purposes.
     LoadMemTrie(LoadMemTrieCommand),
+
+    AnalyzeDelayedReceipt(AnalyzeDelayedReceiptCommand),
 }
 
 impl DatabaseCommand {
@@ -74,6 +77,7 @@ impl DatabaseCommand {
                 .unwrap_or_else(|e| panic!("Error loading config: {:#}", e));
                 cmd.run(near_config, home)
             }
+            SubCommand::AnalyzeDelayedReceipt(cmd) => cmd.run(home),
         }
     }
 }

tools/database/src/lib.rs

@@ -1,6 +1,7 @@
 mod adjust_database;
 mod analyse_data_size_distribution;
 mod analyse_gas_usage;
+mod analyze_delayed_receipt;
 mod block_iterators;
 pub mod commands;
 mod compact;