From 305e06744f7f11bca88ab63a2642838cada74e1a Mon Sep 17 00:00:00 2001 From: emailtovamos Date: Tue, 19 Nov 2024 11:47:36 +0000 Subject: [PATCH 1/2] util: python script to get stats of reorg --- cmd/pyutils/reorg_analyzer.py | 153 ++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 cmd/pyutils/reorg_analyzer.py diff --git a/cmd/pyutils/reorg_analyzer.py b/cmd/pyutils/reorg_analyzer.py new file mode 100644 index 0000000000..61dcd57970 --- /dev/null +++ b/cmd/pyutils/reorg_analyzer.py @@ -0,0 +1,153 @@ +# Example commands: (You can either given path to the directory which has all the .log files or just one .log file name) +# python reorg_analyzer.py /path/to/logs_directory +# or +# python reorg_analyzer.py bsc.log +import re +import os +import argparse +from collections import defaultdict + +def parse_logs(file_paths): + # Regular expressions to match log lines + re_import = re.compile( + r't=.* lvl=info msg="Imported new chain segment" number=(\d+) ' + r'hash=([0-9a-fx]+) miner=([0-9a-zA-Zx]+)' + ) + re_reorg = re.compile( + r't=.* lvl=info msg="Chain reorg detected" number=(\d+) hash=([0-9a-fx]+) ' + r'drop=(\d+) dropfrom=([0-9a-fx]+) add=(\d+) addfrom=([0-9a-fx]+)' + ) + + # Dictionaries to store block information + block_info = {} + reorgs = [] + + for log_file_path in file_paths: + with open(log_file_path, 'r') as f: + for line in f: + # Check for imported block lines + match_import = re_import.search(line) + if match_import: + block_number = int(match_import.group(1)) + block_hash = match_import.group(2) + miner = match_import.group(3) + block_info[block_hash] = { + 'number': block_number, + 'miner': miner + } + continue + + # Check for reorg lines + match_reorg = re_reorg.search(line) + if match_reorg: + reorg_number = int(match_reorg.group(1)) + reorg_hash = match_reorg.group(2) + drop_count = int(match_reorg.group(3)) + drop_from_hash = match_reorg.group(4) + add_count = int(match_reorg.group(5)) + add_from_hash = match_reorg.group(6) + reorgs.append({ + 'number': reorg_number, + 'hash': reorg_hash, + 'drop_count': drop_count, + 'drop_from_hash': drop_from_hash, + 'add_count': add_count, + 'add_from_hash': add_from_hash + }) + + return block_info, reorgs + +def analyze_reorgs(block_info, reorgs): + results = [] + validator_reorgs = defaultdict(lambda: {'count': 0, 'blocks': []}) + + for reorg in reorgs: + # Get the dropped and added block hashes + dropped_hash = reorg['drop_from_hash'] + added_hash = reorg['add_from_hash'] + + # Get miner information + dropped_miner = block_info.get(dropped_hash, {}).get('miner', 'Unknown') + added_miner = block_info.get(added_hash, {}).get('miner', 'Unknown') + + # Construct the result + result = { + 'reorg_at_block': reorg['number'], + 'dropped_block_hash': dropped_hash, + 'added_block_hash': added_hash, + 'dropped_miner': dropped_miner, + 'added_miner': added_miner, + 'responsible_validator': added_miner + } + results.append(result) + + # Update the validator reorgs data + validator = added_miner + validator_reorgs[validator]['count'] += 1 + validator_reorgs[validator]['blocks'].append(reorg['number']) + + return results, validator_reorgs + +def get_log_files(path): + if os.path.isfile(path): + return [path] + elif os.path.isdir(path): + # Get all .log files in the directory + log_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.log')] + if not log_files: + print(f"No .log files found in directory: {path}") + exit(1) + return log_files + else: + print(f"Invalid path: {path}") + exit(1) + +def main(): + parser = argparse.ArgumentParser(description='Analyze BSC node logs for reorgs.') + parser.add_argument('path', help='Path to the log file or directory containing log files.') + args = parser.parse_args() + + log_files = get_log_files(args.path) + block_info, reorgs = parse_logs(log_files) + results, validator_reorgs = analyze_reorgs(block_info, reorgs) + + # Print the detailed reorg results + for res in results: + print(f"Reorg detected at block number {res['reorg_at_block']}:") + print(f" Dropped block hash: {res['dropped_block_hash']}") + print(f" Dropped miner: {res['dropped_miner']}") + print(f" Added block hash: {res['added_block_hash']}") + print(f" Added miner: {res['added_miner']}") + print(f" Validator responsible for reorg: {res['responsible_validator']}") + print('-' * 60) + + # Print the aggregated summary + print("\nAggregated Validators Responsible for Reorgs:\n") + print(f"{'Validator Address':<46} {'Number of Reorgs':<16} {'Block Numbers'}") + print('-' * 90) + for validator, data in sorted(validator_reorgs.items(), key=lambda x: x[1]['count'], reverse=True): + block_numbers = ', '.join(map(str, data['blocks'])) + print(f"{validator:<46} {data['count']:<16} {block_numbers}") + +if __name__ == '__main__': + main() + +# Example Output: +# Reorg detected at block number 43989479: +# Dropped block hash: 0x8f97c466adc41449f98a51efd6c9b0ee480373a0d87d23fe0cbc78bcedb32f34 +# Dropped miner: 0xB4647b856CB9C3856d559C885Bed8B43e0846a48 +# Added block hash: 0x057f65b6852d269b61766387fecfbeed5b360fb3ffc8d80a73d674c3ad3237cc +# Added miner: 0x58567F7A51a58708C8B40ec592A38bA64C0697Df +# Validator responsible for reorg: 0x58567F7A51a58708C8B40ec592A38bA64C0697Df +# ------------------------------------------------------------ +# ... (additional reorg details) +# ------------------------------------------------------------ + +# Aggregated Validators Responsible for Reorgs: + +# Validator Address Number of Reorgs Block Numbers +# ------------------------------------------------------------------------------------------ +# 0x4e5acf9684652BEa56F2f01b7101a225Ee33d23g 13 43962513, 43966037, 43971672, ... +# 0x58567F7A51a58708C8B40ec592A38bA64C0697Df 9 43989479, 43996288, 43998896, ... +# 0x7E1FdF03Eb3aC35BF0256694D7fBe6B6d7b3E0c9 4 43990167, 43977391, 43912043, ... +# ... (additional validators) From c50fcf4ac48a93a7dee1b55802ff884695fb5c5e Mon Sep 17 00:00:00 2001 From: emailtovamos Date: Wed, 20 Nov 2024 11:26:39 +0000 Subject: [PATCH 2/2] utils: support regex and all files. Not just .log extension --- cmd/pyutils/reorg_analyzer.py | 113 ++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 45 deletions(-) diff --git a/cmd/pyutils/reorg_analyzer.py b/cmd/pyutils/reorg_analyzer.py index 61dcd57970..c71dc90764 100644 --- a/cmd/pyutils/reorg_analyzer.py +++ b/cmd/pyutils/reorg_analyzer.py @@ -2,10 +2,13 @@ # python reorg_analyzer.py /path/to/logs_directory # or # python reorg_analyzer.py bsc.log +# or +# python reorg_analyzer.py bsc.log.2024-10-3* import re import os import argparse from collections import defaultdict +import glob def parse_logs(file_paths): # Regular expressions to match log lines @@ -23,37 +26,40 @@ def parse_logs(file_paths): reorgs = [] for log_file_path in file_paths: - with open(log_file_path, 'r') as f: - for line in f: - # Check for imported block lines - match_import = re_import.search(line) - if match_import: - block_number = int(match_import.group(1)) - block_hash = match_import.group(2) - miner = match_import.group(3) - block_info[block_hash] = { - 'number': block_number, - 'miner': miner - } - continue - - # Check for reorg lines - match_reorg = re_reorg.search(line) - if match_reorg: - reorg_number = int(match_reorg.group(1)) - reorg_hash = match_reorg.group(2) - drop_count = int(match_reorg.group(3)) - drop_from_hash = match_reorg.group(4) - add_count = int(match_reorg.group(5)) - add_from_hash = match_reorg.group(6) - reorgs.append({ - 'number': reorg_number, - 'hash': reorg_hash, - 'drop_count': drop_count, - 'drop_from_hash': drop_from_hash, - 'add_count': add_count, - 'add_from_hash': add_from_hash - }) + try: + with open(log_file_path, 'r', encoding='utf-8') as f: + for line in f: + # Check for imported block lines + match_import = re_import.search(line) + if match_import: + block_number = int(match_import.group(1)) + block_hash = match_import.group(2) + miner = match_import.group(3) + block_info[block_hash] = { + 'number': block_number, + 'miner': miner + } + continue + + # Check for reorg lines + match_reorg = re_reorg.search(line) + if match_reorg: + reorg_number = int(match_reorg.group(1)) + reorg_hash = match_reorg.group(2) + drop_count = int(match_reorg.group(3)) + drop_from_hash = match_reorg.group(4) + add_count = int(match_reorg.group(5)) + add_from_hash = match_reorg.group(6) + reorgs.append({ + 'number': reorg_number, + 'hash': reorg_hash, + 'drop_count': drop_count, + 'drop_from_hash': drop_from_hash, + 'add_count': add_count, + 'add_from_hash': add_from_hash + }) + except Exception as e: + print(f"Error reading file {log_file_path}: {e}") return block_info, reorgs @@ -88,26 +94,43 @@ def analyze_reorgs(block_info, reorgs): return results, validator_reorgs -def get_log_files(path): - if os.path.isfile(path): - return [path] - elif os.path.isdir(path): - # Get all .log files in the directory - log_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.log')] - if not log_files: - print(f"No .log files found in directory: {path}") - exit(1) - return log_files - else: - print(f"Invalid path: {path}") +def get_log_files(paths): + log_files = [] + for path in paths: + # Expand patterns + expanded_paths = glob.glob(path) + if not expanded_paths: + print(f"No files matched the pattern: {path}") + continue + for expanded_path in expanded_paths: + if os.path.isfile(expanded_path): + log_files.append(expanded_path) + elif os.path.isdir(expanded_path): + # Get all files in the directory + files_in_dir = [ + os.path.join(expanded_path, f) + for f in os.listdir(expanded_path) + if os.path.isfile(os.path.join(expanded_path, f)) + ] + log_files.extend(files_in_dir) + else: + print(f"Invalid path: {expanded_path}") + if not log_files: + print("No log files to process.") exit(1) + return log_files def main(): parser = argparse.ArgumentParser(description='Analyze BSC node logs for reorgs.') - parser.add_argument('path', help='Path to the log file or directory containing log files.') + parser.add_argument('paths', nargs='+', help='Path(s) to log files, directories, or patterns.') args = parser.parse_args() - log_files = get_log_files(args.path) + log_files = get_log_files(args.paths) + + print("Processing the following files:") + for f in log_files: + print(f" - {f}") + block_info, reorgs = parse_logs(log_files) results, validator_reorgs = analyze_reorgs(block_info, reorgs)