-
Notifications
You must be signed in to change notification settings - Fork 618
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(tool): add analysis for delayed receipts (#10729)
Add a script that analyzes congestion of shards by looking at delayed receipts for each shard.
- Loading branch information
1 parent
1114e89
commit a24ce9d
Showing
3 changed files
with
141 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,135 @@ | ||
use clap::Parser; | ||
use near_store::flat::FlatStorageManager; | ||
use near_store::{get_delayed_receipt_indices, ShardTries, StateSnapshotConfig, TrieConfig}; | ||
use std::collections::HashMap; | ||
use std::path::PathBuf; | ||
use std::rc::Rc; | ||
|
||
use near_chain::ChainStore; | ||
use near_chain::ChainStoreAccess; | ||
use near_chain_configs::GenesisValidationMode; | ||
use near_epoch_manager::EpochManager; | ||
use nearcore::config::load_config; | ||
|
||
use near_primitives::hash::CryptoHash; | ||
use near_primitives::shard_layout::ShardUId; | ||
use near_primitives::types::BlockHeight; | ||
|
||
use crate::block_iterators::{ | ||
make_block_iterator_from_command_args, CommandArgs, LastNBlocksIterator, | ||
}; | ||
use nearcore::open_storage; | ||
|
||
/// Analyze delayed receipts in a piece of history of the blockchain to understand congestion of each shard | ||
#[derive(Parser)] | ||
pub(crate) struct AnalyzeDelayedReceiptCommand { | ||
/// Analyse the last N blocks in the blockchain | ||
#[arg(long)] | ||
last_blocks: Option<u64>, | ||
|
||
/// Analyse blocks from the given block height, inclusive | ||
#[arg(long)] | ||
from_block_height: Option<BlockHeight>, | ||
|
||
/// Analyse blocks up to the given block height, inclusive | ||
#[arg(long)] | ||
to_block_height: Option<BlockHeight>, | ||
} | ||
|
||
impl AnalyzeDelayedReceiptCommand { | ||
pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { | ||
let mut near_config = load_config(home, GenesisValidationMode::Full).unwrap(); | ||
let node_storage = open_storage(&home, &mut near_config).unwrap(); | ||
let store = node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); | ||
let chain_store = Rc::new(ChainStore::new( | ||
store.clone(), | ||
near_config.genesis.config.genesis_height, | ||
false, | ||
)); | ||
let epoch_manager = | ||
EpochManager::new_from_genesis_config(store.clone(), &near_config.genesis.config) | ||
.unwrap(); | ||
|
||
let tip = chain_store.head().unwrap(); | ||
let shard_layout = epoch_manager.get_shard_layout(&tip.epoch_id).unwrap(); | ||
let shard_uids = shard_layout.shard_uids().collect::<Vec<_>>(); | ||
let shard_tries = ShardTries::new( | ||
store.clone(), | ||
TrieConfig::default(), | ||
&shard_uids, | ||
FlatStorageManager::new(store), | ||
StateSnapshotConfig::default(), | ||
); | ||
// Create an iterator over the blocks that should be analysed | ||
let blocks_iter_opt = make_block_iterator_from_command_args( | ||
CommandArgs { | ||
last_blocks: self.last_blocks, | ||
from_block_height: self.from_block_height, | ||
to_block_height: self.to_block_height, | ||
}, | ||
chain_store.clone(), | ||
); | ||
|
||
let blocks_iter = match blocks_iter_opt { | ||
Some(iter) => iter, | ||
None => { | ||
println!("No arguments, defaulting to last 100 blocks"); | ||
Box::new(LastNBlocksIterator::new(100, chain_store)) | ||
} | ||
}; | ||
|
||
let mut blocks_count: usize = 0; | ||
let mut first_analysed_block: Option<(BlockHeight, CryptoHash)> = None; | ||
let mut last_analysed_block: Option<(BlockHeight, CryptoHash)> = None; | ||
let mut shard_id_to_congested = HashMap::new(); | ||
|
||
for block in blocks_iter { | ||
blocks_count += 1; | ||
if first_analysed_block.is_none() { | ||
first_analysed_block = Some((block.header().height(), *block.hash())); | ||
} | ||
last_analysed_block = Some((block.header().height(), *block.hash())); | ||
let shard_layout = epoch_manager.get_shard_layout(block.header().epoch_id()).unwrap(); | ||
|
||
for chunk_header in block.chunks().iter() { | ||
let state_root = chunk_header.prev_state_root(); | ||
let trie_update = shard_tries.get_trie_for_shard( | ||
ShardUId::from_shard_id_and_layout(chunk_header.shard_id(), &shard_layout), | ||
state_root, | ||
); | ||
let delayed_receipt_indices = get_delayed_receipt_indices(&trie_update)?; | ||
if delayed_receipt_indices.len() > 0 { | ||
*shard_id_to_congested.entry(chunk_header.shard_id()).or_insert(0) += 1; | ||
} | ||
println!( | ||
"block height {} shard {} delayed receipts {}", | ||
block.header().height(), | ||
chunk_header.shard_id(), | ||
delayed_receipt_indices.len() | ||
); | ||
} | ||
} | ||
|
||
println!("Analysed {} blocks between:", blocks_count); | ||
if let Some((block_height, block_hash)) = first_analysed_block { | ||
println!("Block: height = {block_height}, hash = {block_hash}"); | ||
} | ||
if let Some((block_height, block_hash)) = last_analysed_block { | ||
println!("Block: height = {block_height}, hash = {block_hash}"); | ||
} | ||
|
||
if blocks_count == 0 { | ||
return Ok(()); | ||
} | ||
for shard_id in shard_layout.shard_ids() { | ||
let congested_chunks = *shard_id_to_congested.get(&shard_id).unwrap_or(&0); | ||
println!( | ||
"shard {} congested ratio {}", | ||
shard_id, | ||
congested_chunks as f64 / blocks_count as f64 | ||
); | ||
} | ||
|
||
Ok(()) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters