diff --git a/unittests/savanna_cluster.cpp b/unittests/savanna_cluster.cpp index 8f9bf4bc7f..428f9a063a 100644 --- a/unittests/savanna_cluster.cpp +++ b/unittests/savanna_cluster.cpp @@ -27,10 +27,11 @@ node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = set } }; - auto node_initialization_fn = [&]() { + auto node_initialization_fn = [&, node_idx]() { [[maybe_unused]] auto _a = control->voted_block().connect(voted_block_cb); [[maybe_unused]] auto _b = control->accepted_block().connect(accepted_block_cb); tester::set_node_finalizers(node_finalizers); + cluster.get_new_blocks_from_peers(node_idx); }; node_initialization_fn(); // initialize the node when it is first created diff --git a/unittests/savanna_cluster.hpp b/unittests/savanna_cluster.hpp index 92380d3bac..64ac1e8323 100644 --- a/unittests/savanna_cluster.hpp +++ b/unittests/savanna_cluster.hpp @@ -59,6 +59,7 @@ namespace savanna_cluster { std::function accepted_block_cb; std::function voted_block_cb; + public: std::vector node_finalizers; @@ -141,10 +142,12 @@ namespace savanna_cluster { } void push_block(const signed_block_ptr& b) { - assert(!pushing_a_block); - pushing_a_block = true; - auto reset_pending_on_exit = fc::make_scoped_exit([this]{ pushing_a_block = false; }); - tester::push_block(b); + if (is_open() && !fetch_block_by_id(b->calculate_id())) { + assert(!pushing_a_block); + pushing_a_block = true; + auto reset_pending_on_exit = fc::make_scoped_exit([this]{ pushing_a_block = false; }); + tester::push_block(b); + } } template @@ -419,7 +422,7 @@ namespace savanna_cluster { // ----------------------------------------------------------------------------- void push_blocks(size_t src_idx, size_t dst_idx, uint32_t start_block_num) { auto& src = _nodes[src_idx]; - assert(src.is_open() && _nodes[dst_idx].is_open()); + assert(src.is_open() && _nodes[dst_idx].is_open()); auto end_block_num = src.fork_db_head().block_num(); @@ -464,26 +467,35 @@ namespace savanna_cluster { void dispatch_vote_to_peers(size_t node_idx, skip_self_t skip_self, const vote_message_ptr& msg) { static uint32_t connection_id = 0; for_each_peer(node_idx, skip_self, [&](node_t& n) { - n.control->process_vote_message(++connection_id, msg); + if (n.is_open()) + n.control->process_vote_message(++connection_id, msg); }); } void push_block_to_peers(size_t node_idx, skip_self_t skip_self, const signed_block_ptr& b) { for_each_peer(node_idx, skip_self, [&](node_t& n) { - if (!n.fetch_block_by_id(b->calculate_id())) - n.push_block(b); + n.push_block(b); + }); + } + + // when a node restarts, simulate receiving newly produced blocks from peers + void get_new_blocks_from_peers(size_t node_idx) { + assert(_nodes[node_idx].is_open()); + for_each_peer(node_idx, skip_self_t::yes, [&](node_t& n) { + if (n.is_open()) + n.push_blocks_to(_nodes[node_idx]); }); } template void for_each_peer(size_t node_idx, skip_self_t skip_self, const CB& cb) { - if (_shutting_down) + if (_shutting_down || _peers.empty()) return; assert(_peers.find(node_idx) != _peers.end()); const auto& peers = _peers[node_idx]; for (auto i : peers) { bool dont_skip = skip_self == skip_self_t::no || i != node_idx; - if (dont_skip && _nodes[i].is_open()) + if (dont_skip) cb(_nodes[i]); } } diff --git a/unittests/savanna_cluster_tests.cpp b/unittests/savanna_cluster_tests.cpp index cc51fd1f4b..698e276ec0 100644 --- a/unittests/savanna_cluster_tests.cpp +++ b/unittests/savanna_cluster_tests.cpp @@ -54,7 +54,7 @@ BOOST_FIXTURE_TEST_CASE(simple_test, savanna_cluster::cluster_t) { try { // all 4 blocks produced by _nodes[0] will have the same `latest_qc_claim_block_num`, which is node0_lib+2 set_partition({}); // Reunite the two partitions - push_blocks(0, partition); // Push the blocks that _nodes[0] produced to the other + propagate_heads(); // Push the blocks that _nodes[0] produced to the other // nodes which will vote _nodes[0].produce_block(); // Produce one block so the newly produced QC propagates. // this is needed because we don't advance lib when diff --git a/unittests/savanna_disaster_recovery_tests.cpp b/unittests/savanna_disaster_recovery_tests.cpp index 512ec05c3d..6e3fc93e17 100644 --- a/unittests/savanna_disaster_recovery_tests.cpp +++ b/unittests/savanna_disaster_recovery_tests.cpp @@ -15,7 +15,6 @@ BOOST_FIXTURE_TEST_CASE(node_goes_down, savanna_cluster::cluster_t) try { C.close(); // shutdown node C BOOST_REQUIRE_EQUAL(4u, A.lib_advances_by([&]() { A.produce_blocks(4); })); // lib still advances with 3 finalizers C.open(); // restart node C - A.push_blocks_to(C); // propagate blocks A -> C BOOST_REQUIRE_EQUAL(4u, A.lib_advances_by([&]() { A.produce_blocks(4); })); // all 4 finalizers should be back voting BOOST_REQUIRE(!C.is_head_missing_finalizer_votes()); // let's make sure of that } FC_LOG_AND_RETHROW() @@ -37,7 +36,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_with_old_fsi, savanna_cluster::clust C.remove_state(); C.overwrite_fsi(fsi); C.open_from_snapshot(snapshot); - A.push_blocks_to(C); BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // all 4 finalizers should be back voting BOOST_REQUIRE(!C.is_head_missing_finalizer_votes()); // let's make sure of that } FC_LOG_AND_RETHROW() @@ -57,7 +55,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_with_deleted_fsi, savanna_cluster::c C.remove_state(); C.remove_fsi(); C.open_from_snapshot(snapshot); - A.push_blocks_to(C); BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // all 4 finalizers should be back voting BOOST_REQUIRE(!C.is_head_missing_finalizer_votes()); // let's make sure of that } FC_LOG_AND_RETHROW() @@ -76,7 +73,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_while_retaining_fsi, savanna_cluster BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // lib still advances with 3 finalizers C.remove_state(); C.open_from_snapshot(snapshot); - A.push_blocks_to(C); BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // all 4 finalizers should be back voting BOOST_REQUIRE(!C.is_head_missing_finalizer_votes()); // let's make sure of that } FC_LOG_AND_RETHROW() @@ -97,7 +93,6 @@ BOOST_FIXTURE_TEST_CASE(nodes_go_down, savanna_cluster::cluster_t) try { for (auto& N : failing_nodes) N->close(); BOOST_REQUIRE_EQUAL(1u, A.lib_advances_by([&]() { A.produce_blocks(4); })); // lib stalls with 3 finalizers down, 1 QC in flight for (auto& N : failing_nodes) N->open(); - for (auto& N : failing_nodes) A.push_blocks_to(*N); BOOST_REQUIRE_EQUAL(7u, A.lib_advances_by([&]() { A.produce_blocks(4); })); // all 4 finalizers should be back voting for (auto& N : failing_nodes) BOOST_REQUIRE(!N->is_head_missing_finalizer_votes()); } FC_LOG_AND_RETHROW() @@ -125,7 +120,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_with_old_fsi, savanna_cluster::clus N->remove_state(); N->overwrite_fsi(fsis[i]); N->open_from_snapshot(snapshots[i]); - A.push_blocks_to(*N); ++i; } BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // all 4 finalizers should be back voting @@ -152,7 +146,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_with_deleted_fsi, savanna_cluster:: N->remove_state(); N->remove_fsi(); N->open_from_snapshot(snapshots[i]); - A.push_blocks_to(*N); ++i; } BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // all 4 finalizers should be back voting @@ -178,7 +171,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_while_retaining_fsi, savanna_cluste for (auto& N : failing_nodes) { N->remove_state(); N->open_from_snapshot(snapshots[i]); - A.push_blocks_to(*N); ++i; } BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2); })); // all 4 finalizers should be back voting @@ -238,9 +230,13 @@ BOOST_FIXTURE_TEST_CASE(all_nodes_shutdown_with_reversible_blocks_lost, savanna_ N->close(); N->remove_state(); remove_blocks_log ? N->remove_reversible_data_and_blocks_log() : N->remove_reversible_data(); - N->open_from_snapshot(snapshot); } + // reopen after all nodes closed + // ----------------------------- + for (auto& N : failing_nodes) + N->open_from_snapshot(snapshot); + propagate_heads(); // needed only if we don't remove the blocks log, otherwise lib advanced by 1 block // which was stored in the blocks log, and when replayed after loading A and B's // snapshots advanced head() by one diff --git a/unittests/savanna_finalizer_policy_tests.cpp b/unittests/savanna_finalizer_policy_tests.cpp index a9b79b4df0..4af240c9c2 100644 --- a/unittests/savanna_finalizer_policy_tests.cpp +++ b/unittests/savanna_finalizer_policy_tests.cpp @@ -6,7 +6,7 @@ using namespace eosio::testing; BOOST_AUTO_TEST_SUITE(savanna_finalizer_policy) // --------------------------------------------------------------------------------------------------- -// +// Policy change - new key on one node - node shutdown and restarted while policy pending // --------------------------------------------------------------------------------------------------- BOOST_FIXTURE_TEST_CASE(policy_change, savanna_cluster::cluster_t) try { auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; @@ -32,7 +32,7 @@ BOOST_FIXTURE_TEST_CASE(policy_change, savanna_cluster::cluster_t) try { A.produce_block(); ++num_to_pending; } while (!A.head_pending_finalizer_policy()); - BOOST_REQUIRE_EQUAL(num_to_pending, num_chains_to_final); // becames pending when proposed block is final + BOOST_REQUIRE_EQUAL(num_to_pending, num_chains_to_final); // becomes pending when proposed block is final // now that the new policy is pending, we need B to vote on it for finality to advance, as C is down. // -------------------------------------------------------------------------------------------------- @@ -67,5 +67,203 @@ BOOST_FIXTURE_TEST_CASE(policy_change, savanna_cluster::cluster_t) try { BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(3); })); } FC_LOG_AND_RETHROW() +// --------------------------------------------------------------------------------------------------- +// Policy change including weight and threshold +// --------------------------------------------------------------------------------------------------- +BOOST_FIXTURE_TEST_CASE(policy_change_including_weight_and_threshold, savanna_cluster::cluster_t) try { + auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3]; + auto initial_gen = A.head_active_finalizer_policy()->generation; + + // shutdown C, verify that lib still advances (since threshold is 3) + // ----------------------------------------------------------------- + C.close(); + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // update finalizer_policy, so that C's weight is 3, B and D are removed, and the threshold is 4 + // --------------------------------------------------------------------------------------------- + base_tester::finalizer_policy_input input; + input.finalizers.emplace_back(_fin_keys[0], 1); + input.finalizers.emplace_back(_fin_keys[2], 3); + input.threshold = 4; + A.set_finalizers(input); + A.produce_block(); // so the block with `set_finalizers` is `head` + + // produce blocks on A, waiting for the new policy to become pending + // ----------------------------------------------------------------- + BOOST_REQUIRE(!A.head_pending_finalizer_policy()); // we shouldn't have a pending policy + size_t num_to_pending = 0; + do { + A.produce_block(); + ++num_to_pending; + } while (!A.head_pending_finalizer_policy()); + BOOST_REQUIRE_EQUAL(num_to_pending, num_chains_to_final); // becomes pending when proposed block is final + + // verify that lib stops advancing (because C is down so we can't get a QC on the pending policy + // which needs three C votes) + // --------------------------------------------------------------------------------------------- + BOOST_REQUIRE_EQUAL(0u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // restart C. + // --------- + C.open(); + + // produce blocks on A, waiting for transition to complete (until the updated policy is active on A's head) + // -------------------------------------------------------------------------------------------------------- + size_t num_to_active = 0; + do { + A.produce_block(); + ++num_to_active; + } while (A.head_active_finalizer_policy()->generation != initial_gen+1); + BOOST_REQUIRE_EQUAL(num_to_active, num_chains_to_final); // becomes active when "pending" block is final + + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // shutdown B and D which are not used in new policy. + // A produces 2 blocks, verify that *lib* advances by 2 + // ---------------------------------------------------- + B.close(); + D.close(); + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + +} FC_LOG_AND_RETHROW() + + +// --------------------------------------------------------------------------------------------------- +// Policy change, reducing threshold and replacing all keys +// --------------------------------------------------------------------------------------------------- +BOOST_FIXTURE_TEST_CASE(policy_change_reduce_threshold_replace_all_keys, savanna_cluster::cluster_t) try { + auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3]; + auto initial_gen = A.head_active_finalizer_policy()->generation; + + // shutdown D, verify that lib still advances (since threshold is 3) + // ----------------------------------------------------------------- + D.close(); + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // update signing keys on each of { A, B }, so every node has 2 keys, the previous one + a new one + // ----------------------------------------------------------------------------------------------- + A.close(); + B.close(); + A.node_finalizers = std::vector{ _fin_keys[0], _fin_keys[num_nodes()] }; + B.node_finalizers = std::vector{ _fin_keys[1], _fin_keys[num_nodes() + 1] }; + A.open(); + B.open(); + + // verify that lib still advances even though D is down (since threshold is 3) + // --------------------------------------------------------------------------- + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // update finalizer_policy to include only { A, B }'s new keys, and threshold is 2 + // ------------------------------------------------------------------------------- + base_tester::finalizer_policy_input input; + input.finalizers.emplace_back(_fin_keys[num_nodes()], 1); + input.finalizers.emplace_back(_fin_keys[num_nodes() + 1], 1); + input.threshold = 2; + A.set_finalizers(input); + A.produce_block(); // so the block with `set_finalizers` is `head` + + // produce blocks on A, waiting for the new policy to become pending + // ----------------------------------------------------------------- + BOOST_REQUIRE(!A.head_pending_finalizer_policy()); // we shouldn't have a pending policy + size_t num_to_pending = 0; + do { + A.produce_block(); + ++num_to_pending; + } while (!A.head_pending_finalizer_policy()); + BOOST_REQUIRE_EQUAL(num_to_pending, num_chains_to_final); // becomes pending when proposed block is final + + // produce blocks on A, waiting for the new policy to become final + // --------------------------------------------------------------- + size_t num_to_active = 0; + do { + A.produce_block(); + ++num_to_active; + } while (A.head_active_finalizer_policy()->generation != initial_gen+1); + BOOST_REQUIRE_EQUAL(num_to_active, num_chains_to_final); // becomes active when "pending" block is final + + // A produces 2 blocks, verify that *lib* advances by 2 + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // shutdown C and D which are not used in new policy. + // A produces 2 blocks, verify that *lib* advances by 2 + // ---------------------------------------------------- + C.close(); + D.close(); + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + +} FC_LOG_AND_RETHROW() + + +// --------------------------------------------------------------------------------------------------- +// Policy change: restart from snapshot +// --------------------------------------------------------------------------------------------------- +BOOST_FIXTURE_TEST_CASE(policy_change_restart_from_snapshot, savanna_cluster::cluster_t) try { + auto& A=_nodes[0]; auto& B=_nodes[1]; auto& C=_nodes[2]; auto& D=_nodes[3]; + auto initial_gen = A.head_active_finalizer_policy()->generation; + + // update signing keys on each of { A, B, C }, so every node has 2 keys, the previous one + a new one + // -------------------------------------------------------------------------------------------------- + A.close(); + B.close(); + C.close(); + A.node_finalizers = std::vector{ _fin_keys[0], _fin_keys[num_nodes()] }; + B.node_finalizers = std::vector{ _fin_keys[1], _fin_keys[num_nodes() + 1] }; + C.node_finalizers = std::vector{ _fin_keys[2], _fin_keys[num_nodes() + 2] }; + A.open(); + B.open(); + C.open(); + + // update the *finalizer_policy* to include only { A, B, C }'s new keys, C's weight is 2, and threshold is 3 + // ---------------------------------------------------------------------------------------------------------- + base_tester::finalizer_policy_input input; + input.finalizers.emplace_back(_fin_keys[num_nodes()], 1); + input.finalizers.emplace_back(_fin_keys[num_nodes() + 1], 1); + input.finalizers.emplace_back(_fin_keys[num_nodes() + 2], 2); + input.threshold = 3; + A.set_finalizers(input); + A.produce_block(); // so the block with `set_finalizers` is `head` + + // Take a snapshot of C. Produce 2 blocks on A so the snapshot block is stored in block log + // ---------------------------------------------------------------------------------------- + auto snapshot_C = C.snapshot(); + BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2); })); + + // for each of { A, B, C, D }, shutdown, delete *state*, but not *blocks log*, *reversible data* or *fsi* + // ------------------------------------------------------------------------------------------------------ + for (size_t i=0; i<4; ++i) { + _nodes[i].close(); + _nodes[i].remove_state(); + } + + // for each of { A, B, D }, restart from the snapshot + // -------------------------------------------------- + A.open_from_snapshot(snapshot_C); + B.open_from_snapshot(snapshot_C); + D.open_from_snapshot(snapshot_C); + + // A produces 4 blocks, verify that *lib* advances only by one and that the new policy is only pending + // (because C is down so no quorum on new policy) + // --------------------------------------------------------------------------------------------------- + BOOST_REQUIRE_EQUAL(1u, A.lib_advances_by([&]() { A.produce_blocks(4); })); + BOOST_REQUIRE(!!A.head_pending_finalizer_policy()); + + // restart C from the snapshot + // --------------------------- + C.open_from_snapshot(snapshot_C); + + // A produces 4 blocks, verify that the new policy is active and *lib* starts advancing again + // ------------------------------------------------------------------------------------------ + BOOST_REQUIRE_LT(4u, A.lib_advances_by([&]() { A.produce_blocks(4); })); + BOOST_REQUIRE_EQUAL(A.head_active_finalizer_policy()->generation, initial_gen+1); + + // shutdown B and D. A produces 3 blocks, verify that *lib* advances by 3 + // (because together A and C meet the 3 votes quorum for the new policy) + // ---------------------------------------------------------------------- + B.close(); + D.close(); + BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(3); })); + +} FC_LOG_AND_RETHROW() + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/unittests/savanna_transition_tests.cpp b/unittests/savanna_transition_tests.cpp index 3ec5879598..f615a48089 100644 --- a/unittests/savanna_transition_tests.cpp +++ b/unittests/savanna_transition_tests.cpp @@ -151,7 +151,6 @@ BOOST_FIXTURE_TEST_CASE(restart_from_snapshot_at_beginning_of_transition_while_p for (auto& N : failing_nodes) { N->open_from_snapshot(snapshot_C); - A.push_blocks_to(*N); } // A produces blocks until we reach the critical block (i.e. until lib advances past the genesis block) @@ -253,7 +252,6 @@ BOOST_FIXTURE_TEST_CASE(restart_from_snapshot_at_end_of_transition_while_preserv for (auto& N : failing_nodes) { N->open_from_snapshot(snapshot_C); - A.push_blocks_to(*N); } // with partition gone, transition to Savanna will complete and lib will start advancing again @@ -331,7 +329,6 @@ BOOST_FIXTURE_TEST_CASE(restart_from_snapshot_at_beginning_of_transition_with_lo for (auto& N : failing_nodes) { N->open_from_snapshot(snapshot_C); - A.push_blocks_to(*N); } // A produces blocks until we reach the critical block (i.e. until lib advances past the genesis block)