diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 302effb714..460dd550d7 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4733,10 +4733,16 @@ struct controller_impl { return conf.block_validation_mode == validation_mode::LIGHT || conf.trusted_producers.count(producer); } - bool should_terminate(block_num_type head_block_num) const { - if (conf.terminate_at_block > 0 && conf.terminate_at_block <= head_block_num) { + bool should_terminate(block_num_type reversible_block_num) const { + assert(reversible_block_num > 0); + if (conf.terminate_at_block > 0 && conf.terminate_at_block <= reversible_block_num) { ilog("Block ${n} reached configured maximum block ${num}; terminating", - ("n", head_block_num)("num", conf.terminate_at_block) ); + ("n", reversible_block_num)("num", conf.terminate_at_block) ); + return true; + } + if (conf.max_reversible_blocks > 0 && fork_db.size() >= conf.max_reversible_blocks) { + elog("Exceeded max reversible blocks allowed, fork db size ${s} >= max-reversible-blocks ${m}", + ("s", fork_db.size())("m", conf.max_reversible_blocks)); return true; } return false; @@ -5539,10 +5545,6 @@ bool controller::should_terminate() const { return my->should_terminate(); } -bool controller::should_terminate(block_num_type head_block_num) const { - return my->should_terminate(head_block_num); -} - const apply_handler* controller::find_apply_handler( account_name receiver, account_name scope, action_name act ) const { auto native_handler_scope = my->apply_handlers.find( receiver ); diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 7434033e75..1376004b21 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -152,6 +152,12 @@ namespace eosio::chain { my->close_impl(out); } + template + size_t fork_database_t::size() const { + std::lock_guard g( my->mtx ); + return my->index.size(); + } + template void fork_database_impl::close_impl(std::ofstream& out) { assert(!!root); // if head or root are null, we don't save and shouldn't get here @@ -725,6 +731,12 @@ namespace eosio::chain { } } + size_t fork_database::size() const { + return apply([](const auto& forkdb) { + return forkdb.size(); + }); + } + // only called from the main thread void fork_database::switch_from_legacy(const block_state_ptr& root) { // no need to close fork_db because we don't want to write anything out, file is removed on open diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index f2822e9715..0016980a49 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -84,6 +84,7 @@ const static uint16_t default_controller_thread_pool_size = 2; const static uint16_t default_vote_thread_pool_size = 4; const static uint32_t default_max_variable_signature_length = 16384u; const static uint32_t default_max_action_return_value_size = 256; +const static uint32_t default_max_reversible_blocks = 3600u; const static uint32_t default_max_transaction_finality_status_success_duration_sec = 180; const static uint32_t default_max_transaction_finality_status_failure_duration_sec = 180; diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 176973ab81..4466c6ebe5 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -91,6 +91,7 @@ namespace eosio::chain { uint32_t sig_cpu_bill_pct = chain::config::default_sig_cpu_bill_pct; uint16_t chain_thread_pool_size = chain::config::default_controller_thread_pool_size; uint16_t vote_thread_pool_size = 0; + uint32_t max_reversible_blocks = chain::config::default_max_reversible_blocks; bool read_only = false; bool force_all_checks = false; bool disable_replay_opts = false; @@ -379,9 +380,7 @@ namespace eosio::chain { db_read_mode get_read_mode()const; validation_mode get_validation_mode()const; - /// @return true if terminate-at-block reaches terminate block number - /// thread-safe - bool should_terminate(block_num_type head_block_num) const; + /// @return true if terminate-at-block reached, or max-reversible-blocks reached /// not-thread-safe bool should_terminate() const; diff --git a/libraries/chain/include/eosio/chain/fork_database.hpp b/libraries/chain/include/eosio/chain/fork_database.hpp index 3c57bd3c81..6b4836192e 100644 --- a/libraries/chain/include/eosio/chain/fork_database.hpp +++ b/libraries/chain/include/eosio/chain/fork_database.hpp @@ -47,6 +47,7 @@ namespace eosio::chain { void open( const char* desc, const std::filesystem::path& fork_db_file, fc::cfile_datastream& ds, validator_t& validator ); void close( std::ofstream& out ); + size_t size() const; bsp_t get_block( const block_id_type& id, include_root_t include_root = include_root_t::no ) const; bool block_exists( const block_id_type& id ) const; @@ -169,6 +170,9 @@ namespace eosio::chain { void open( validator_t& validator ); void close(); + // return the size of the active fork_database + size_t size() const; + // switches to using both legacy and savanna during transition void switch_from_legacy(const block_state_ptr& root); void switch_to(in_use_t v) { in_use = v; } diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index f4a98e7aa8..34766c6847 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -361,6 +361,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "'none' - EOS VM OC tier-up is completely disabled.\n") #endif ("enable-account-queries", bpo::value()->default_value(false), "enable queries to find accounts by various metadata.") + ("max-reversible-blocks", bpo::value()->default_value(config::default_max_reversible_blocks), + "Approximate maximum allowed reversible blocks before shutdown. Will shut down if limit reached. Specify 0 to disable.") ("transaction-retry-max-storage-size-gb", bpo::value(), "Maximum size (in GiB) allowed to be allocated for the Transaction Retry feature. Setting above 0 enables this feature.") ("transaction-retry-interval-sec", bpo::value()->default_value(20), @@ -947,6 +949,8 @@ void chain_plugin_impl::plugin_initialize(const variables_map& options) { account_queries_enabled = options.at("enable-account-queries").as(); + chain_config->max_reversible_blocks = options.at("max-reversible-blocks").as(); + chain_config->integrity_hash_on_start = options.at("integrity-hash-on-start").as(); chain_config->integrity_hash_on_stop = options.at("integrity-hash-on-stop").as(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index da88d41ed9..8c9888febb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,6 +23,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/spring_util_bls_test.py ${CMAKE_CURRE configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sample-cluster-map.json ${CMAKE_CURRENT_BINARY_DIR}/sample-cluster-map.json COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/restart-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/restart-scenarios-test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/terminate-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/terminate-scenarios-test.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/terminate_scenarios_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/terminate_scenarios_test_shape.json COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/liveness_test.py ${CMAKE_CURRENT_BINARY_DIR}/liveness_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_startup_catchup.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_startup_catchup.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_diff_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_diff_test.py COPYONLY) @@ -239,6 +240,10 @@ add_test(NAME terminate-scenarios-if-test-replay-pass-transition COMMAND tests/t set_property(TEST terminate-scenarios-if-test-replay-pass-transition PROPERTY LABELS nonparallelizable_tests) add_test(NAME terminate-scenarios-if-test-hard_replay-pass-transition COMMAND tests/terminate-scenarios-test.py -c hardReplay --terminate-at-block 150 --kill-sig term --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST terminate-scenarios-if-test-hard_replay-pass-transition PROPERTY LABELS nonparallelizable_tests) +add_test(NAME terminate-scenarios-if-test-max-reversible-blocks-sync COMMAND tests/terminate-scenarios-test.py -c resync --max-reversible-blocks 25 --kill-sig term --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST terminate-scenarios-if-test-max-reversible-blocks-sync PROPERTY LABELS nonparallelizable_tests) +add_test(NAME terminate-scenarios-if-test-max-reversible-blocks-replay COMMAND tests/terminate-scenarios-test.py -c replay --max-reversible-blocks 25 --kill-sig term --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST terminate-scenarios-if-test-max-reversible-blocks-replay PROPERTY LABELS nonparallelizable_tests) add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST validate_dirty_db_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME keosd_auto_launch_test COMMAND tests/keosd_auto_launch_test.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/terminate-scenarios-test.py b/tests/terminate-scenarios-test.py index 319db32fc1..f41da485ab 100755 --- a/tests/terminate-scenarios-test.py +++ b/tests/terminate-scenarios-test.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import random +import signal from TestHarness import Cluster, TestHelper, Utils, WalletMgr +from TestHarness.TestHelper import AppArgs ############################################################### # terminate-scenarios-test @@ -11,25 +13,31 @@ # (--delete-all-blocks), "hardReplay"(--hard-replay-blockchain), and "none" to indicate what kind of restart flag should # be used. This is one of the only test that actually verify that nodeos terminates with a good exit status. # +# Also used to test max-reversible-blocks in savanna. +# ############################################################### Print=Utils.Print errorExit=Utils.errorExit -args=TestHelper.parse_args({"-d","-s","-c","--kill-sig","--keep-logs" +appArgs=AppArgs() +appArgs.add(flag="--max-reversible-blocks", type=int, help="pass max-reversible-blocks to nodeos", default=0) + +args=TestHelper.parse_args({"-d","-c","--kill-sig","--keep-logs" ,"--activate-if","--dump-error-details","-v","--leave-running" - ,"--terminate-at-block","--unshared"}) + ,"--terminate-at-block","--unshared"}, applicationSpecificArgs=appArgs) pnodes=1 -topo=args.s +topo="./tests/terminate_scenarios_test_shape.json" delay=args.d chainSyncStrategyStr=args.c debug=args.v -total_nodes = pnodes -killSignal=args.kill_sig +total_nodes = pnodes+1 +killSignalStr=args.kill_sig activateIF=args.activate_if dumpErrorDetails=args.dump_error_details terminate=args.terminate_at_block +maxReversibleBlocks=args.max_reversible_blocks seed=1 Utils.Debug=debug @@ -41,18 +49,24 @@ try: TestHelper.printSystemInfo("BEGIN") - cluster.setWalletMgr(walletMgr) + if maxReversibleBlocks > 0 and not activateIF: + errorExit("--max-reversible-blocks requires --activate-if") + if maxReversibleBlocks > 0 and terminate > 0: + errorExit("Test supports only one of --max-reversible-blocks requires --terminate-at-block") - cluster.setChainStrategy(chainSyncStrategyStr) cluster.setWalletMgr(walletMgr) + cluster.setChainStrategy(chainSyncStrategyStr) Print ("producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % ( - pnodes, topo, delay, chainSyncStrategyStr)) + pnodes, topo, delay, chainSyncStrategyStr)) Print("Stand up cluster") - if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, activateIF=activateIF) is False: + if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, totalProducers=pnodes, topo=topo, delay=delay, activateIF=activateIF) is False: errorExit("Failed to stand up eos cluster.") + Print ("Kill bios node") + cluster.biosNode.kill(signal.SIGTERM) + Print ("Wait for Cluster stabilization") # wait for cluster to start producing blocks if not cluster.waitOnClusterBlockNumSync(3): @@ -61,18 +75,24 @@ # make sure enough blocks produced to verify truncate works on restart cluster.getNode(0).waitForBlock(terminate+5) - Print("Kill cluster node instance.") - if cluster.killSomeEosInstances(1, killSignal) is False: + Print(f"Kill signal {killSignalStr} cluster node instance 0.") + killSignal = signal.SIGKILL + if killSignalStr == Utils.SigTermTag: + killSignal = signal.SIGTERM + if not cluster.getNode(0).kill(killSignal): errorExit("Failed to kill Eos instances") assert not cluster.getNode(0).verifyAlive() - Print("nodeos instances killed.") + Print("nodeos instances 0 killed.") Print ("Relaunch dead cluster node instance.") nodeArg = "--terminate-at-block %d" % terminate if terminate > 0 else "" + if nodeArg == "": + nodeArg = "--max-reversible-blocks %d" % maxReversibleBlocks if maxReversibleBlocks > 0 else "" if nodeArg != "": if chainSyncStrategyStr == "hardReplay": nodeArg += " --truncate-at-block %d" % terminate - if cluster.relaunchEosInstances(nodeArgs=nodeArg, waitForTerm=(terminate > 0)) is False: + nodeArg += " --enable-stale-production " + if cluster.relaunchEosInstances(nodeArgs=nodeArg, waitForTerm=(terminate > 0 or maxReversibleBlocks > 0)) is False: errorExit("Failed to relaunch Eos instance") Print("nodeos instance relaunched.") diff --git a/tests/terminate_scenarios_test_shape.json b/tests/terminate_scenarios_test_shape.json new file mode 100644 index 0000000000..05c3332f84 --- /dev/null +++ b/tests/terminate_scenarios_test_shape.json @@ -0,0 +1,56 @@ +{ + "name": "testnet_", + "ssh_helper": { + "ssh_cmd": "/usr/bin/ssh", + "scp_cmd": "/usr/bin/scp", + "ssh_identity": "", + "ssh_args": "" + }, + "nodes": { + "bios":{ + "name": "bios", + "keys": [ + { + "privkey":"5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3", + "pubkey":"EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV", + "blspubkey":"PUB_BLS_Uf3df_EqPpR31ZkenPtwgGUtd69cahyuY2lc9jPwEta7Q6t7REV-Hd35hUIDel4N7pQdCGZdnVZzs_UmJghEjGhVHN1QVVAQjOca8Fs10D_jqTiUzffzqyBAvTHyZtoEEPyXkg", + "blsprivkey":"PVT_BLS_t2sZsoDWTQFIKg75bhJn8pBA0iDYcWyn3HlEfKIzTzKozgKO", + "blspop":"SIG_BLS_TnwBY4dpG54mCue3ZXwjCio0AIdWYwFdz5ipLdnXlg64FkYkhMUtkOdQIs1IYbMWOXlD6OnCP6jcCWi5VziWKNbLfMX64SdIkNPKOHrfE_8fBfIk9Onj7GbWx3q0LbYP7NfJQk1mk-gOjz1G3elZDDHt367YUgzYDKhtl1FSkfZzDRzDsCSei7H1MjLi_e0RVdUfgqAznGaq2Yss6gY-HzwzgHU4y-SNQpzdCuDlLEEIjkHq8fXuMiPWT2Dlt8kOML0uqg" + } + ], + "peers": [], + "producers": [ + "eosio" + ], + "dont_start": false + }, + "testnet_00":{ + "name": "testnet_00", + "keys": [ + { + "privkey":"5Jf4sTk7vwX1MYpLJ2eQFanVvKYXFqGBrCyANPukuP2BJ5WAAKZ", + "pubkey":"EOS58B33q9S7oNkgeFfcoW3VJYu4obfDiqn5RHGE2ige6jVjUhymR" + } + ], + "peers": [ + "bios", + "testnet_01" + ], + "producers": [ + "defproducera" + ], + "dont_start": false + }, + "testnet_01":{ + "name": "testnet_01", + "keys": [ + ], + "peers": [ + "testnet_00" + ], + "producers": [ + ], + "dont_start": false + } + } +}