Implement last three Savanna unittests modeled after the fast testnet wave tests.

unittests/savanna_cluster.cpp

@@ -27,10 +27,11 @@ node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = set
       }
    };
 
-   auto node_initialization_fn = [&]() {
+   auto node_initialization_fn = [&, node_idx]() {
       [[maybe_unused]] auto _a = control->voted_block().connect(voted_block_cb);
       [[maybe_unused]] auto _b = control->accepted_block().connect(accepted_block_cb);
       tester::set_node_finalizers(node_finalizers);
+      cluster.get_new_blocks_from_peers(node_idx);
    };
 
    node_initialization_fn();                  // initialize the node when it is first created

unittests/savanna_cluster.hpp

@@ -59,6 +59,7 @@ namespace savanna_cluster {
 
       std::function<void(const block_signal_params&)> accepted_block_cb;
       std::function<void(const vote_signal_params&)>  voted_block_cb;
+
    public:
       std::vector<account_name> node_finalizers;
 
@@ -141,10 +142,12 @@ namespace savanna_cluster {
       }
 
       void push_block(const signed_block_ptr& b) {
-         assert(!pushing_a_block);
-         pushing_a_block = true;
-         auto reset_pending_on_exit = fc::make_scoped_exit([this]{ pushing_a_block = false; });
-         tester::push_block(b);
+         if (is_open() && !fetch_block_by_id(b->calculate_id())) {
+            assert(!pushing_a_block);
+            pushing_a_block = true;
+            auto reset_pending_on_exit = fc::make_scoped_exit([this]{ pushing_a_block = false; });
+            tester::push_block(b);
+         }
       }
 
       template <class Node>
@@ -419,7 +422,7 @@ namespace savanna_cluster {
       // -----------------------------------------------------------------------------
       void push_blocks(size_t src_idx, size_t dst_idx, uint32_t start_block_num) {
          auto& src = _nodes[src_idx];
-         assert(src.is_open() &&  _nodes[dst_idx].is_open());
+         assert(src.is_open() && _nodes[dst_idx].is_open());
 
          auto end_block_num   = src.fork_db_head().block_num();
 
@@ -464,26 +467,35 @@ namespace savanna_cluster {
       void dispatch_vote_to_peers(size_t node_idx, skip_self_t skip_self, const vote_message_ptr& msg) {
          static uint32_t connection_id = 0;
          for_each_peer(node_idx, skip_self, [&](node_t& n) {
-            n.control->process_vote_message(++connection_id, msg);
+            if (n.is_open())
+               n.control->process_vote_message(++connection_id, msg);
          });
       }
 
       void push_block_to_peers(size_t node_idx, skip_self_t skip_self, const signed_block_ptr& b) {
          for_each_peer(node_idx, skip_self, [&](node_t& n) {
-            if (!n.fetch_block_by_id(b->calculate_id()))
-               n.push_block(b);
+            n.push_block(b);
+         });
+      }
+
+      // when a node restarts, simulate receiving newly produced blocks from peers
+      void get_new_blocks_from_peers(size_t node_idx) {
+         assert(_nodes[node_idx].is_open());
+         for_each_peer(node_idx, skip_self_t::yes, [&](node_t& n) {
+            if (n.is_open())
+               n.push_blocks_to(_nodes[node_idx]);
          });
       }
 
       template<class CB>
       void for_each_peer(size_t node_idx, skip_self_t skip_self, const CB& cb) {
-         if (_shutting_down)
+         if (_shutting_down || _peers.empty())
             return;
          assert(_peers.find(node_idx) != _peers.end());
          const auto& peers = _peers[node_idx];
          for (auto i : peers) {
             bool dont_skip = skip_self == skip_self_t::no || i != node_idx;
-            if (dont_skip && _nodes[i].is_open())
+            if (dont_skip)
                cb(_nodes[i]);
          }
       }

unittests/savanna_cluster_tests.cpp

@@ -54,7 +54,7 @@ BOOST_FIXTURE_TEST_CASE(simple_test, savanna_cluster::cluster_t) { try {
       // all 4 blocks produced by _nodes[0] will have the same `latest_qc_claim_block_num`, which is node0_lib+2
 
       set_partition({});                                    // Reunite the two partitions
-      push_blocks(0, partition);                            // Push the blocks that _nodes[0] produced to the other
+      propagate_heads();                                    // Push the blocks that _nodes[0] produced to the other
                                                             // nodes which will vote
       _nodes[0].produce_block();                            // Produce one block so the newly produced QC propagates.
                                                             // this is needed because we don't advance lib when

unittests/savanna_disaster_recovery_tests.cpp

@@ -15,7 +15,6 @@ BOOST_FIXTURE_TEST_CASE(node_goes_down, savanna_cluster::cluster_t) try {
    C.close();                                                                  // shutdown node C
    BOOST_REQUIRE_EQUAL(4u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // lib still advances with 3 finalizers
    C.open();                                                                   // restart node C
-   A.push_blocks_to(C);                                                        // propagate blocks A -> C
    BOOST_REQUIRE_EQUAL(4u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -37,7 +36,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_with_old_fsi, savanna_cluster::clust
    C.remove_state();
    C.overwrite_fsi(fsi);
    C.open_from_snapshot(snapshot);
-   A.push_blocks_to(C);
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -57,7 +55,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_with_deleted_fsi, savanna_cluster::c
    C.remove_state();
    C.remove_fsi();
    C.open_from_snapshot(snapshot);
-   A.push_blocks_to(C);
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -76,7 +73,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_while_retaining_fsi, savanna_cluster
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // lib still advances with 3 finalizers
    C.remove_state();
    C.open_from_snapshot(snapshot);
-   A.push_blocks_to(C);
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -97,7 +93,6 @@ BOOST_FIXTURE_TEST_CASE(nodes_go_down, savanna_cluster::cluster_t) try {
    for (auto& N : failing_nodes) N->close();
    BOOST_REQUIRE_EQUAL(1u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // lib stalls with 3 finalizers down, 1 QC in flight
    for (auto& N : failing_nodes) N->open();
-   for (auto& N : failing_nodes) A.push_blocks_to(*N);
    BOOST_REQUIRE_EQUAL(7u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // all 4 finalizers should be back voting
    for (auto& N : failing_nodes) BOOST_REQUIRE(!N->is_head_missing_finalizer_votes());
 } FC_LOG_AND_RETHROW()
@@ -125,7 +120,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_with_old_fsi, savanna_cluster::clus
       N->remove_state();
       N->overwrite_fsi(fsis[i]);
       N->open_from_snapshot(snapshots[i]);
-      A.push_blocks_to(*N);
       ++i;
    }
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
@@ -152,7 +146,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_with_deleted_fsi, savanna_cluster::
       N->remove_state();
       N->remove_fsi();
       N->open_from_snapshot(snapshots[i]);
-      A.push_blocks_to(*N);
       ++i;
    }
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
@@ -178,7 +171,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_while_retaining_fsi, savanna_cluste
    for (auto& N : failing_nodes) {
       N->remove_state();
       N->open_from_snapshot(snapshots[i]);
-      A.push_blocks_to(*N);
       ++i;
    }
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
@@ -238,9 +230,13 @@ BOOST_FIXTURE_TEST_CASE(all_nodes_shutdown_with_reversible_blocks_lost, savanna_
       N->close();
       N->remove_state();
       remove_blocks_log ? N->remove_reversible_data_and_blocks_log() : N->remove_reversible_data();
-      N->open_from_snapshot(snapshot);
    }
 
+   // reopen after all nodes closed
+   // -----------------------------
+   for (auto& N : failing_nodes)
+      N->open_from_snapshot(snapshot);
+
    propagate_heads(); // needed only if we don't remove the blocks log, otherwise lib advanced by 1 block
                       // which was stored in the blocks log, and when replayed after loading A and B's
                       // snapshots advanced head() by one

unittests/savanna_finalizer_policy_tests.cpp

@@ -6,7 +6,7 @@ using namespace eosio::testing;
 BOOST_AUTO_TEST_SUITE(savanna_finalizer_policy)
 
 // ---------------------------------------------------------------------------------------------------
-//
+//     Policy change - new key on one node - node shutdown and restarted while policy pending
 // ---------------------------------------------------------------------------------------------------
 BOOST_FIXTURE_TEST_CASE(policy_change, savanna_cluster::cluster_t) try {
    auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2];
@@ -67,5 +67,203 @@ BOOST_FIXTURE_TEST_CASE(policy_change, savanna_cluster::cluster_t) try {
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(3); }));
 } FC_LOG_AND_RETHROW()
 
+// ---------------------------------------------------------------------------------------------------
+//                           Policy change including weight and threshold
+// ---------------------------------------------------------------------------------------------------
+BOOST_FIXTURE_TEST_CASE(policy_change_including_weight_and_threshold, savanna_cluster::cluster_t) try {
+   auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3];
+   auto initial_gen = A.head_active_finalizer_policy()->generation;
+
+   // shutdown C, verify that lib still advances (since threshold is 3)
+   // -----------------------------------------------------------------
+   C.close();
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // update finalizer_policy, so that C's weight is 3, B and D are removed, and the threshold is 4
+   // ---------------------------------------------------------------------------------------------
+   base_tester::finalizer_policy_input input;
+   input.finalizers.emplace_back(_fin_keys[0], 1);
+   input.finalizers.emplace_back(_fin_keys[2], 3);
+   input.threshold = 4;
+   A.set_finalizers(input);
+   A.produce_block();                                   // so the block with `set_finalizers` is `head`
+
+   // produce blocks on A, waiting for the new policy to become pending
+   // -----------------------------------------------------------------
+   BOOST_REQUIRE(!A.head_pending_finalizer_policy());     // we shouldn't have a pending policy
+   size_t num_to_pending = 0;
+   do {
+      A.produce_block();
+      ++num_to_pending;
+   } while (!A.head_pending_finalizer_policy());
+   BOOST_REQUIRE_EQUAL(num_to_pending, num_chains_to_final); // becames pending when proposed block is final
+
+   // verify that lib stops advancing (because C is down so we can't get a QC on the pending policy
+   // which needs three C votes)
+   // ---------------------------------------------------------------------------------------------
+   BOOST_REQUIRE_EQUAL(0u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // restart C.
+   // ---------
+   C.open();
+
+   // produce blocks on A, waiting for transition to complete (until the updated policy is active on A's head)
+   // --------------------------------------------------------------------------------------------------------
+   size_t num_to_active = 0;
+   do {
+      A.produce_block();
+      ++num_to_active;
+   } while (A.head_active_finalizer_policy()->generation != initial_gen+1);
+   BOOST_REQUIRE_EQUAL(num_to_active, num_chains_to_final); // becomes active when "pending" block is final
+
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // shutdown B and D which are not used in new policy.
+   // A produces 2 blocks, verify that *lib* advances by 2
+   // ----------------------------------------------------
+   B.close();
+   D.close();
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+} FC_LOG_AND_RETHROW()
+
+
+// ---------------------------------------------------------------------------------------------------
+//                    Policy change, reducing threshold and replacing all keys
+// ---------------------------------------------------------------------------------------------------
+BOOST_FIXTURE_TEST_CASE(policy_change_reduce_threshold_replace_all_keys, savanna_cluster::cluster_t) try {
+   auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3];
+   auto initial_gen = A.head_active_finalizer_policy()->generation;
+
+   // shutdown D, verify that lib still advances (since threshold is 3)
+   // -----------------------------------------------------------------
+   D.close();
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // update signing keys on each of { A, B }, so every node has 2 keys, the previous one + a new one
+   // -----------------------------------------------------------------------------------------------
+   A.close();
+   B.close();
+   A.node_finalizers = std::vector<account_name>{ _fin_keys[0], _fin_keys[num_nodes()] };
+   B.node_finalizers = std::vector<account_name>{ _fin_keys[1], _fin_keys[num_nodes() + 1] };
+   A.open();
+   B.open();
+
+   // verify that lib still advances even though D is down (since threshold is 3)
+   // ---------------------------------------------------------------------------
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // update finalizer_policy to include only { A, B }'s new keys, and threshold is 2
+   // -------------------------------------------------------------------------------
+   base_tester::finalizer_policy_input input;
+   input.finalizers.emplace_back(_fin_keys[num_nodes()], 1);
+   input.finalizers.emplace_back(_fin_keys[num_nodes() + 1], 1);
+   input.threshold = 2;
+   A.set_finalizers(input);
+   A.produce_block();                                   // so the block with `set_finalizers` is `head`
+
+   // produce blocks on A, waiting for the new policy to become pending
+   // -----------------------------------------------------------------
+   BOOST_REQUIRE(!A.head_pending_finalizer_policy());     // we shouldn't have a pending policy
+   size_t num_to_pending = 0;
+   do {
+      A.produce_block();
+      ++num_to_pending;
+   } while (!A.head_pending_finalizer_policy());
+   BOOST_REQUIRE_EQUAL(num_to_pending, num_chains_to_final); // becames pending when proposed block is final
+
+   // produce blocks on A, waiting for the new policy to become final
+   // ---------------------------------------------------------------
+   size_t num_to_active = 0;
+   do {
+      A.produce_block();
+      ++num_to_active;
+   } while (A.head_active_finalizer_policy()->generation != initial_gen+1);
+   BOOST_REQUIRE_EQUAL(num_to_active, num_chains_to_final); // becomes active when "pending" block is final
+
+   // A produces 2 blocks, verify that *lib* advances by 2
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // shutdown C and D which are not used in new policy.
+   // A produces 2 blocks, verify that *lib* advances by 2
+   // ----------------------------------------------------
+   C.close();
+   D.close();
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+} FC_LOG_AND_RETHROW()
+
+
+// ---------------------------------------------------------------------------------------------------
+//                    Policy change: restart from snapshot
+// ---------------------------------------------------------------------------------------------------
+BOOST_FIXTURE_TEST_CASE(policy_change_restart_from_snapshot, savanna_cluster::cluster_t) try {
+   auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3];
+   auto initial_gen = A.head_active_finalizer_policy()->generation;
+
+   // update signing keys on each of { A, B, C }, so every node has 2 keys, the previous one + a new one
+   // --------------------------------------------------------------------------------------------------
+   A.close();
+   B.close();
+   C.close();
+   A.node_finalizers = std::vector<account_name>{ _fin_keys[0], _fin_keys[num_nodes()] };
+   B.node_finalizers = std::vector<account_name>{ _fin_keys[1], _fin_keys[num_nodes() + 1] };
+   C.node_finalizers = std::vector<account_name>{ _fin_keys[2], _fin_keys[num_nodes() + 2] };
+   A.open();
+   B.open();
+   C.open();
+
+   // update the  *finalizer_policy* to include only { A, B, C }'s new keys, C's weight is 2, and threshold is 3
+   // ----------------------------------------------------------------------------------------------------------
+   base_tester::finalizer_policy_input input;
+   input.finalizers.emplace_back(_fin_keys[num_nodes()], 1);
+   input.finalizers.emplace_back(_fin_keys[num_nodes() + 1], 1);
+   input.finalizers.emplace_back(_fin_keys[num_nodes() + 2], 2);
+   input.threshold = 3;
+   A.set_finalizers(input);
+   A.produce_block();                                   // so the block with `set_finalizers` is `head`
+
+   // Take a snapshot of C. Produce 2 blocks on A so the snapshot block is stored in block log
+   // ----------------------------------------------------------------------------------------
+   auto snapshot_C = C.snapshot();
+   BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); }));
+
+   // for each of { A, B, C, D }, shutdown, delete *state*, but not *blocks log*, *reversible data* or *fsi*
+   // ------------------------------------------------------------------------------------------------------
+   for (size_t i=0; i<4; ++i) {
+      _nodes[i].close();
+      _nodes[i].remove_state();
+   }
+
+   // for each of { A, B, D }, restart from the snapshot
+   // --------------------------------------------------
+   A.open_from_snapshot(snapshot_C);
+   B.open_from_snapshot(snapshot_C);
+   D.open_from_snapshot(snapshot_C);
+
+   // A produces 4 blocks, verify that *lib* advances only by one and that the new policy is only pending
+   // (because  C is down so no quorum on new policy)
+   // ---------------------------------------------------------------------------------------------------
+   BOOST_REQUIRE_EQUAL(1u, A.lib_advances_by([&]() { A.produce_blocks(4); }));
+   BOOST_REQUIRE(!!A.head_pending_finalizer_policy());
+
+   // restart C from the snapshot
+   // ---------------------------
+   C.open_from_snapshot(snapshot_C);
+
+   // A produces 4 blocks, verify that the new policy is active and *lib* starts advancing again
+   // ------------------------------------------------------------------------------------------
+   BOOST_REQUIRE_LT(4u, A.lib_advances_by([&]() { A.produce_blocks(4); }));
+   BOOST_REQUIRE_EQUAL(A.head_active_finalizer_policy()->generation, initial_gen+1);
+
+   // shutdown B and D. A produces 3 blocks, verify that *lib* advances by 3
+   // (because together A and C meet the 3 votes quorum for the new policy)
+   // ----------------------------------------------------------------------
+   B.close();
+   D.close();
+   BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(3); }));
+
+} FC_LOG_AND_RETHROW()
+
 
 BOOST_AUTO_TEST_SUITE_END()