[1.0-beta4 -> main] Add max-reversible-blocks option

libraries/chain/controller.cpp

@@ -4733,10 +4733,16 @@ struct controller_impl {
       return conf.block_validation_mode == validation_mode::LIGHT || conf.trusted_producers.count(producer);
    }
 
-   bool should_terminate(block_num_type head_block_num) const {
-      if (conf.terminate_at_block > 0 && conf.terminate_at_block <= head_block_num) {
+   bool should_terminate(block_num_type reversible_block_num) const {
+      assert(reversible_block_num > 0);
+      if (conf.terminate_at_block > 0 && conf.terminate_at_block <= reversible_block_num) {
          ilog("Block ${n} reached configured maximum block ${num}; terminating",
-              ("n", head_block_num)("num", conf.terminate_at_block) );
+              ("n", reversible_block_num)("num", conf.terminate_at_block) );
+         return true;
+      }
+      if (conf.max_reversible_blocks > 0 && fork_db.size() >= conf.max_reversible_blocks) {
+         elog("Exceeded max reversible blocks allowed, fork db size ${s} >= max-reversible-blocks ${m}",
+              ("s", fork_db.size())("m", conf.max_reversible_blocks));
          return true;
       }
       return false;
@@ -5539,10 +5545,6 @@ bool controller::should_terminate() const {
    return my->should_terminate();
 }
 
-bool controller::should_terminate(block_num_type head_block_num) const {
-   return my->should_terminate(head_block_num);
-}
-
 const apply_handler* controller::find_apply_handler( account_name receiver, account_name scope, action_name act ) const
 {
    auto native_handler_scope = my->apply_handlers.find( receiver );

libraries/chain/fork_database.cpp

@@ -152,6 +152,12 @@ namespace eosio::chain {
       my->close_impl(out);
    }
 
+   template<class BSP>
+   size_t fork_database_t<BSP>::size() const {
+      std::lock_guard g( my->mtx );
+      return my->index.size();
+   }
+
    template<class BSP>
    void fork_database_impl<BSP>::close_impl(std::ofstream& out) {
       assert(!!root); // if head or root are null, we don't save and shouldn't get here
@@ -725,6 +731,12 @@ namespace eosio::chain {
       }
    }
 
+   size_t fork_database::size() const {
+      return apply<size_t>([](const auto& forkdb) {
+         return forkdb.size();
+      });
+   }
+
    // only called from the main thread
    void fork_database::switch_from_legacy(const block_state_ptr& root) {
       // no need to close fork_db because we don't want to write anything out, file is removed on open

libraries/chain/include/eosio/chain/config.hpp

@@ -84,6 +84,7 @@ const static uint16_t   default_controller_thread_pool_size          = 2;
 const static uint16_t   default_vote_thread_pool_size                = 4;
 const static uint32_t   default_max_variable_signature_length        = 16384u;
 const static uint32_t   default_max_action_return_value_size         = 256;
+const static uint32_t   default_max_reversible_blocks                = 3600u;
 
 const static uint32_t   default_max_transaction_finality_status_success_duration_sec = 180;
 const static uint32_t   default_max_transaction_finality_status_failure_duration_sec = 180;

libraries/chain/include/eosio/chain/controller.hpp

@@ -91,6 +91,7 @@ namespace eosio::chain {
             uint32_t                 sig_cpu_bill_pct       =  chain::config::default_sig_cpu_bill_pct;
             uint16_t                 chain_thread_pool_size =  chain::config::default_controller_thread_pool_size;
             uint16_t                 vote_thread_pool_size  =  0;
+            uint32_t                 max_reversible_blocks  =  chain::config::default_max_reversible_blocks;
             bool                     read_only              =  false;
             bool                     force_all_checks       =  false;
             bool                     disable_replay_opts    =  false;
@@ -379,9 +380,7 @@ namespace eosio::chain {
 
          db_read_mode get_read_mode()const;
          validation_mode get_validation_mode()const;
-         /// @return true if terminate-at-block reaches terminate block number
-         /// thread-safe
-         bool should_terminate(block_num_type head_block_num) const;
+         /// @return true if terminate-at-block reached, or max-reversible-blocks reached
          /// not-thread-safe
          bool should_terminate() const;
 

libraries/chain/include/eosio/chain/fork_database.hpp

@@ -47,6 +47,7 @@ namespace eosio::chain {
 
       void open( const char* desc, const std::filesystem::path& fork_db_file, fc::cfile_datastream& ds, validator_t& validator );
       void close( std::ofstream& out );
+      size_t size() const;
 
       bsp_t get_block( const block_id_type& id, include_root_t include_root = include_root_t::no ) const;
       bool block_exists( const block_id_type& id ) const;
@@ -169,6 +170,9 @@ namespace eosio::chain {
       void open( validator_t& validator );
       void close();
 
+      // return the size of the active fork_database
+      size_t size() const;
+
       // switches to using both legacy and savanna during transition
       void switch_from_legacy(const block_state_ptr& root);
       void switch_to(in_use_t v) { in_use = v; }

plugins/chain_plugin/chain_plugin.cpp

@@ -361,6 +361,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
           "'none' - EOS VM OC tier-up is completely disabled.\n")
 #endif
          ("enable-account-queries", bpo::value<bool>()->default_value(false), "enable queries to find accounts by various metadata.")
+         ("max-reversible-blocks", bpo::value<uint32_t>()->default_value(config::default_max_reversible_blocks),
+          "Approximate maximum allowed reversible blocks before shutdown. Will shut down if limit reached. Specify 0 to disable.")
          ("transaction-retry-max-storage-size-gb", bpo::value<uint64_t>(),
           "Maximum size (in GiB) allowed to be allocated for the Transaction Retry feature. Setting above 0 enables this feature.")
          ("transaction-retry-interval-sec", bpo::value<uint32_t>()->default_value(20),
@@ -947,6 +949,8 @@ void chain_plugin_impl::plugin_initialize(const variables_map& options) {
 
       account_queries_enabled = options.at("enable-account-queries").as<bool>();
 
+      chain_config->max_reversible_blocks = options.at("max-reversible-blocks").as<uint32_t>();
+
       chain_config->integrity_hash_on_start = options.at("integrity-hash-on-start").as<bool>();
       chain_config->integrity_hash_on_stop = options.at("integrity-hash-on-stop").as<bool>();
 

tests/CMakeLists.txt

@@ -23,6 +23,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/spring_util_bls_test.py ${CMAKE_CURRE
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sample-cluster-map.json ${CMAKE_CURRENT_BINARY_DIR}/sample-cluster-map.json COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/restart-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/restart-scenarios-test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/terminate-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/terminate-scenarios-test.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/terminate_scenarios_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/terminate_scenarios_test_shape.json COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/liveness_test.py ${CMAKE_CURRENT_BINARY_DIR}/liveness_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_startup_catchup.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_startup_catchup.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_diff_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_diff_test.py COPYONLY)
@@ -239,6 +240,10 @@ add_test(NAME terminate-scenarios-if-test-replay-pass-transition COMMAND tests/t
 set_property(TEST terminate-scenarios-if-test-replay-pass-transition PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME terminate-scenarios-if-test-hard_replay-pass-transition COMMAND tests/terminate-scenarios-test.py -c hardReplay --terminate-at-block 150 --kill-sig term --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-if-test-hard_replay-pass-transition PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME terminate-scenarios-if-test-max-reversible-blocks-sync COMMAND tests/terminate-scenarios-test.py -c resync --max-reversible-blocks 25 --kill-sig term --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST terminate-scenarios-if-test-max-reversible-blocks-sync PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME terminate-scenarios-if-test-max-reversible-blocks-replay COMMAND tests/terminate-scenarios-test.py -c replay --max-reversible-blocks 25 --kill-sig term --activate-if ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST terminate-scenarios-if-test-max-reversible-blocks-replay PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST validate_dirty_db_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME keosd_auto_launch_test COMMAND tests/keosd_auto_launch_test.py WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/terminate-scenarios-test.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
 import random
+import signal
 
 from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # terminate-scenarios-test
@@ -11,25 +13,31 @@
 # (--delete-all-blocks), "hardReplay"(--hard-replay-blockchain), and "none" to indicate what kind of restart flag should
 # be used. This is one of the only test that actually verify that nodeos terminates with a good exit status.
 #
+# Also used to test max-reversible-blocks in savanna.
+#
 ###############################################################
 
 
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-args=TestHelper.parse_args({"-d","-s","-c","--kill-sig","--keep-logs"
+appArgs=AppArgs()
+appArgs.add(flag="--max-reversible-blocks", type=int, help="pass max-reversible-blocks to nodeos", default=0)
+
+args=TestHelper.parse_args({"-d","-c","--kill-sig","--keep-logs"
                             ,"--activate-if","--dump-error-details","-v","--leave-running"
-                            ,"--terminate-at-block","--unshared"})
+                            ,"--terminate-at-block","--unshared"}, applicationSpecificArgs=appArgs)
 pnodes=1
-topo=args.s
+topo="./tests/terminate_scenarios_test_shape.json"
 delay=args.d
 chainSyncStrategyStr=args.c
 debug=args.v
-total_nodes = pnodes
-killSignal=args.kill_sig
+total_nodes = pnodes+1
+killSignalStr=args.kill_sig
 activateIF=args.activate_if
 dumpErrorDetails=args.dump_error_details
 terminate=args.terminate_at_block
+maxReversibleBlocks=args.max_reversible_blocks
 
 seed=1
 Utils.Debug=debug
@@ -41,18 +49,24 @@
 
 try:
     TestHelper.printSystemInfo("BEGIN")
-    cluster.setWalletMgr(walletMgr)
+    if maxReversibleBlocks > 0 and not activateIF:
+        errorExit("--max-reversible-blocks requires --activate-if")
+    if maxReversibleBlocks > 0 and terminate > 0:
+        errorExit("Test supports only one of --max-reversible-blocks requires --terminate-at-block")
 
-    cluster.setChainStrategy(chainSyncStrategyStr)
     cluster.setWalletMgr(walletMgr)
+    cluster.setChainStrategy(chainSyncStrategyStr)
 
     Print ("producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % (
-    pnodes, topo, delay, chainSyncStrategyStr))
+           pnodes, topo, delay, chainSyncStrategyStr))
 
     Print("Stand up cluster")
-    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, activateIF=activateIF) is False:
+    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, totalProducers=pnodes, topo=topo, delay=delay, activateIF=activateIF) is False:
         errorExit("Failed to stand up eos cluster.")
 
+    Print ("Kill bios node")
+    cluster.biosNode.kill(signal.SIGTERM)
+
     Print ("Wait for Cluster stabilization")
     # wait for cluster to start producing blocks
     if not cluster.waitOnClusterBlockNumSync(3):
@@ -61,18 +75,24 @@
     # make sure enough blocks produced to verify truncate works on restart
     cluster.getNode(0).waitForBlock(terminate+5)
 
-    Print("Kill cluster node instance.")
-    if cluster.killSomeEosInstances(1, killSignal) is False:
+    Print(f"Kill signal {killSignalStr} cluster node instance 0.")
+    killSignal = signal.SIGKILL
+    if killSignalStr == Utils.SigTermTag:
+        killSignal = signal.SIGTERM
+    if not cluster.getNode(0).kill(killSignal):
         errorExit("Failed to kill Eos instances")
     assert not cluster.getNode(0).verifyAlive()
-    Print("nodeos instances killed.")
+    Print("nodeos instances 0 killed.")
 
     Print ("Relaunch dead cluster node instance.")
     nodeArg = "--terminate-at-block %d" % terminate if terminate > 0 else ""
+    if nodeArg == "":
+        nodeArg = "--max-reversible-blocks %d" % maxReversibleBlocks if maxReversibleBlocks > 0 else ""
     if nodeArg != "":
         if chainSyncStrategyStr == "hardReplay":
             nodeArg += " --truncate-at-block %d" % terminate
-    if cluster.relaunchEosInstances(nodeArgs=nodeArg, waitForTerm=(terminate > 0)) is False:
+    nodeArg += " --enable-stale-production "
+    if cluster.relaunchEosInstances(nodeArgs=nodeArg, waitForTerm=(terminate > 0 or maxReversibleBlocks > 0)) is False:
         errorExit("Failed to relaunch Eos instance")
     Print("nodeos instance relaunched.")
 

tests/terminate_scenarios_test_shape.json

@@ -0,0 +1,56 @@
+{
+  "name": "testnet_",
+  "ssh_helper": {
+    "ssh_cmd": "/usr/bin/ssh",
+    "scp_cmd": "/usr/bin/scp",
+    "ssh_identity": "",
+    "ssh_args": ""
+  },
+  "nodes": {
+      "bios":{
+        "name": "bios",
+        "keys": [
+          {
+            "privkey":"5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3",
+            "pubkey":"EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV",
+            "blspubkey":"PUB_BLS_Uf3df_EqPpR31ZkenPtwgGUtd69cahyuY2lc9jPwEta7Q6t7REV-Hd35hUIDel4N7pQdCGZdnVZzs_UmJghEjGhVHN1QVVAQjOca8Fs10D_jqTiUzffzqyBAvTHyZtoEEPyXkg",
+            "blsprivkey":"PVT_BLS_t2sZsoDWTQFIKg75bhJn8pBA0iDYcWyn3HlEfKIzTzKozgKO",
+            "blspop":"SIG_BLS_TnwBY4dpG54mCue3ZXwjCio0AIdWYwFdz5ipLdnXlg64FkYkhMUtkOdQIs1IYbMWOXlD6OnCP6jcCWi5VziWKNbLfMX64SdIkNPKOHrfE_8fBfIk9Onj7GbWx3q0LbYP7NfJQk1mk-gOjz1G3elZDDHt367YUgzYDKhtl1FSkfZzDRzDsCSei7H1MjLi_e0RVdUfgqAznGaq2Yss6gY-HzwzgHU4y-SNQpzdCuDlLEEIjkHq8fXuMiPWT2Dlt8kOML0uqg"
+          }
+        ],
+        "peers": [],
+        "producers": [
+          "eosio"
+        ],
+        "dont_start": false
+      },
+      "testnet_00":{
+        "name": "testnet_00",
+        "keys": [
+          {
+            "privkey":"5Jf4sTk7vwX1MYpLJ2eQFanVvKYXFqGBrCyANPukuP2BJ5WAAKZ",
+            "pubkey":"EOS58B33q9S7oNkgeFfcoW3VJYu4obfDiqn5RHGE2ige6jVjUhymR"
+          }
+        ],
+        "peers": [
+          "bios",
+          "testnet_01"
+        ],
+        "producers": [
+          "defproducera"
+        ],
+        "dont_start": false
+      },
+      "testnet_01":{
+        "name": "testnet_01",
+        "keys": [
+        ],
+        "peers": [
+          "testnet_00"
+        ],
+        "producers": [
+        ],
+        "dont_start": false
+      }
+  }
+}