From 350e15ff45d822f3eb875da9511accef6ae738e4 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Mon, 22 Jul 2019 10:16:52 -0400 Subject: [PATCH 1/7] publishes from rpc send, publish and wallet send, publish will no longer be eligible to drop due to bandwidth limiting --- nano/node/blockprocessor.cpp | 2 +- nano/node/election.cpp | 2 +- nano/node/network.cpp | 10 ++++++++-- nano/node/network.hpp | 8 ++------ nano/node/wallet.cpp | 4 ++-- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index 161f630123..e6c3ea0262 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -352,7 +352,7 @@ void nano::block_processor::process_live (nano::block_hash const & hash_a, std:: // Start collecting quorum on block node.active.start (block_a); // Announce block contents to the network - node.network.flood_block (block_a); + node.network.flood_block (block_a, false); if (node.config.enable_voting) { // Announce our weighted vote to the network diff --git a/nano/node/election.cpp b/nano/node/election.cpp index c1e69844b2..a73a753cf6 100644 --- a/nano/node/election.cpp +++ b/nano/node/election.cpp @@ -223,7 +223,7 @@ bool nano::election::publish (std::shared_ptr block_a) { blocks.insert (std::make_pair (block_a->hash (), block_a)); confirm_if_quorum (transaction); - node.network.flood_block (block_a); + node.network.flood_block (block_a, false); } else { diff --git a/nano/node/network.cpp b/nano/node/network.cpp index ff6ed3aaa8..a01fb3a836 100644 --- a/nano/node/network.cpp +++ b/nano/node/network.cpp @@ -215,15 +215,21 @@ bool nano::network::send_votes_cache (std::shared_ptr return result; } -void nano::network::flood_message (nano::message const & message_a) +void nano::network::flood_message (nano::message const & message_a, bool const & is_dropable) { auto list (list_fanout ()); for (auto i (list.begin ()), n (list.end ()); i != n; ++i) { - (*i)->send (message_a); + (*i)->send (message_a, nullptr, is_dropable); } } +void nano::network::flood_block (std::shared_ptr block_a, bool const & is_dropable) +{ + nano::publish publish (block_a); + flood_message (publish, is_dropable); +} + void nano::network::flood_block_batch (std::deque> blocks_a, unsigned delay_a) { auto block (blocks_a.front ()); diff --git a/nano/node/network.hpp b/nano/node/network.hpp index fcfcb01940..e5e946d7b9 100644 --- a/nano/node/network.hpp +++ b/nano/node/network.hpp @@ -114,7 +114,7 @@ class network final ~network (); void start (); void stop (); - void flood_message (nano::message const &); + void flood_message (nano::message const &, bool const & = true); void flood_keepalive () { nano::keepalive message; @@ -126,11 +126,7 @@ class network final nano::confirm_ack message (vote_a); flood_message (message); } - void flood_block (std::shared_ptr block_a) - { - nano::publish publish (block_a); - flood_message (publish); - } + void flood_block (std::shared_ptr, bool const & = true); void flood_block_batch (std::deque>, unsigned = broadcast_interval_ms); void merge_peers (std::array const &); void merge_peer (nano::endpoint const &); diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 14544cde1b..99d993fdbf 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -1060,7 +1060,7 @@ std::shared_ptr nano::wallet::send_action (nano::account const & so if (block != nullptr) { cached_block = true; - wallets.node.network.flood_block (block); + wallets.node.network.flood_block (block, false); } } else if (status != MDB_NOTFOUND) @@ -1487,7 +1487,7 @@ void nano::work_watcher::run () current->second = block; } } - node.network.flood_block (block); + node.network.flood_block (block, false); node.active.update_difficulty (*block.get ()); lock.lock (); if (stopped) From 7a9300c1e28421a04362338cd8f12c98e0c0662a Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Tue, 23 Jul 2019 09:43:18 -0400 Subject: [PATCH 2/7] PR feedback edits --- nano/node/bootstrap.cpp | 8 ++++---- nano/node/network.cpp | 10 ++-------- nano/node/network.hpp | 9 +++++++-- nano/node/transport/transport.cpp | 4 ++-- nano/node/transport/transport.hpp | 2 +- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/nano/node/bootstrap.cpp b/nano/node/bootstrap.cpp index a92ed1aed1..8b167009c7 100644 --- a/nano/node/bootstrap.cpp +++ b/nano/node/bootstrap.cpp @@ -81,7 +81,7 @@ void nano::frontier_req_client::run () } } }, - false); // is bootstrap traffic is_dropable false + false); // is bootstrap traffic is_droppable false } std::shared_ptr nano::bootstrap_client::shared () @@ -352,7 +352,7 @@ void nano::bulk_pull_client::request () this_l->connection->node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_request_failure, nano::stat::dir::in); } }, - false); // is bootstrap traffic is_dropable false + false); // is bootstrap traffic is_droppable false } void nano::bulk_pull_client::receive_block () @@ -539,7 +539,7 @@ void nano::bulk_push_client::start () } } }, - false); // is bootstrap traffic is_dropable false + false); // is bootstrap traffic is_droppable false } void nano::bulk_push_client::push (nano::transaction const & transaction_a) @@ -678,7 +678,7 @@ void nano::bulk_pull_account_client::request () this_l->connection->node->stats.inc (nano::stat::type::bootstrap, nano::stat::detail::bulk_pull_error_starting_request, nano::stat::dir::in); } }, - false); // is bootstrap traffic is_dropable false + false); // is bootstrap traffic is_droppable false } void nano::bulk_pull_account_client::receive_pending () diff --git a/nano/node/network.cpp b/nano/node/network.cpp index a01fb3a836..6ed4bbdc16 100644 --- a/nano/node/network.cpp +++ b/nano/node/network.cpp @@ -215,21 +215,15 @@ bool nano::network::send_votes_cache (std::shared_ptr return result; } -void nano::network::flood_message (nano::message const & message_a, bool const & is_dropable) +void nano::network::flood_message (nano::message const & message_a, bool const is_droppable_a) { auto list (list_fanout ()); for (auto i (list.begin ()), n (list.end ()); i != n; ++i) { - (*i)->send (message_a, nullptr, is_dropable); + (*i)->send (message_a, nullptr, is_droppable_a); } } -void nano::network::flood_block (std::shared_ptr block_a, bool const & is_dropable) -{ - nano::publish publish (block_a); - flood_message (publish, is_dropable); -} - void nano::network::flood_block_batch (std::deque> blocks_a, unsigned delay_a) { auto block (blocks_a.front ()); diff --git a/nano/node/network.hpp b/nano/node/network.hpp index e5e946d7b9..3df232e415 100644 --- a/nano/node/network.hpp +++ b/nano/node/network.hpp @@ -114,7 +114,7 @@ class network final ~network (); void start (); void stop (); - void flood_message (nano::message const &, bool const & = true); + void flood_message (nano::message const &, bool const = true); void flood_keepalive () { nano::keepalive message; @@ -126,7 +126,12 @@ class network final nano::confirm_ack message (vote_a); flood_message (message); } - void flood_block (std::shared_ptr, bool const & = true); + void flood_block (std::shared_ptr block_a, bool const is_droppable_a = true) + { + nano::publish publish (block_a); + flood_message (publish, is_droppable_a); + } + void flood_block_batch (std::deque>, unsigned = broadcast_interval_ms); void merge_peers (std::array const &); void merge_peer (nano::endpoint const &); diff --git a/nano/node/transport/transport.cpp b/nano/node/transport/transport.cpp index 16466a37c3..548e421767 100644 --- a/nano/node/transport/transport.cpp +++ b/nano/node/transport/transport.cpp @@ -75,13 +75,13 @@ node (node_a) { } -void nano::transport::channel::send (nano::message const & message_a, std::function const & callback_a, bool const & is_dropable) +void nano::transport::channel::send (nano::message const & message_a, std::function const & callback_a, bool const is_droppable_a) { callback_visitor visitor; message_a.visit (visitor); auto buffer (message_a.to_bytes ()); auto detail (visitor.result); - if (!is_dropable || !limiter.should_drop (buffer->size ())) + if (!is_droppable_a || !limiter.should_drop (buffer->size ())) { send_buffer (buffer, detail, callback_a); node.stats.inc (nano::stat::type::message, detail, nano::stat::dir::out); diff --git a/nano/node/transport/transport.hpp b/nano/node/transport/transport.hpp index 2fda8ea2e0..af8f2c29f4 100644 --- a/nano/node/transport/transport.hpp +++ b/nano/node/transport/transport.hpp @@ -53,7 +53,7 @@ namespace transport virtual ~channel () = default; virtual size_t hash_code () const = 0; virtual bool operator== (nano::transport::channel const &) const = 0; - void send (nano::message const &, std::function const & = nullptr, bool const & = true); + void send (nano::message const &, std::function const & = nullptr, bool const = true); virtual void send_buffer (std::shared_ptr>, nano::stat::detail, std::function const & = nullptr) = 0; virtual std::function callback (std::shared_ptr>, nano::stat::detail, std::function const & = nullptr) const = 0; virtual std::string to_string () const = 0; From fda46e5436d32d868ccb64e5f8e0778b7329dd1c Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Tue, 23 Jul 2019 13:01:15 -0400 Subject: [PATCH 3/7] formatting --- nano/node/network.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nano/node/network.hpp b/nano/node/network.hpp index 3df232e415..a720c36796 100644 --- a/nano/node/network.hpp +++ b/nano/node/network.hpp @@ -128,8 +128,8 @@ class network final } void flood_block (std::shared_ptr block_a, bool const is_droppable_a = true) { - nano::publish publish (block_a); - flood_message (publish, is_droppable_a); + nano::publish publish (block_a); + flood_message (publish, is_droppable_a); } void flood_block_batch (std::deque>, unsigned = broadcast_interval_ms); From 7fce1e676d55042a94fd51507b263a95cf5a5202 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Tue, 30 Jul 2019 13:40:31 -0400 Subject: [PATCH 4/7] add flag watch_work, defaulting to true to process RPC call --- nano/node/blockprocessor.cpp | 15 ++++++--- nano/node/blockprocessor.hpp | 6 ++-- nano/node/json_handler.cpp | 3 +- nano/rpc_test/rpc.cpp | 60 ++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index e6c3ea0262..e2c8d2c383 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -347,10 +347,15 @@ void nano::block_processor::process_batch (std::unique_lock & lock_a } } -void nano::block_processor::process_live (nano::block_hash const & hash_a, std::shared_ptr block_a) +void nano::block_processor::process_live (nano::block_hash const & hash_a, std::shared_ptr block_a, const bool watch_work_a) { // Start collecting quorum on block node.active.start (block_a); + //add block to watcher if desired after block has been added to active + if (watch_work_a) + { + node.wallets.watcher.add (block_a); + } // Announce block contents to the network node.network.flood_block (block_a, false); if (node.config.enable_voting) @@ -382,7 +387,7 @@ void nano::block_processor::process_live (nano::block_hash const & hash_a, std:: }); } -nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, nano::unchecked_info info_a) +nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, nano::unchecked_info info_a, const bool work_watcher_a) { nano::process_return result; auto hash (info_a.block->hash ()); @@ -400,7 +405,7 @@ nano::process_return nano::block_processor::process_one (nano::transaction const } if (info_a.modified > nano::seconds_since_epoch () - 300 && node.block_arrival.recent (hash)) { - process_live (hash, info_a.block); + process_live (hash, info_a.block, work_watcher_a); } queue_unchecked (transaction_a, hash); break; @@ -515,10 +520,10 @@ nano::process_return nano::block_processor::process_one (nano::transaction const return result; } -nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, std::shared_ptr block_a) +nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, std::shared_ptr block_a, const bool watch_work_a) { nano::unchecked_info info (block_a, block_a->account (), 0, nano::signature_verification::unknown); - auto result (process_one (transaction_a, info)); + auto result (process_one (transaction_a, info, watch_work_a)); return result; } diff --git a/nano/node/blockprocessor.hpp b/nano/node/blockprocessor.hpp index ad3090edc1..d865ab951c 100644 --- a/nano/node/blockprocessor.hpp +++ b/nano/node/blockprocessor.hpp @@ -44,8 +44,8 @@ class block_processor final bool should_log (bool); bool have_blocks (); void process_blocks (); - nano::process_return process_one (nano::transaction const &, nano::unchecked_info); - nano::process_return process_one (nano::transaction const &, std::shared_ptr); + nano::process_return process_one (nano::transaction const &, nano::unchecked_info, const bool = false); + nano::process_return process_one (nano::transaction const &, std::shared_ptr, const bool = false); nano::vote_generator generator; // Delay required for average network propagartion before requesting confirmation static std::chrono::milliseconds constexpr confirmation_request_delay{ 1500 }; @@ -54,7 +54,7 @@ class block_processor final void queue_unchecked (nano::transaction const &, nano::block_hash const &); void verify_state_blocks (nano::transaction const & transaction_a, std::unique_lock &, size_t = std::numeric_limits::max ()); void process_batch (std::unique_lock &); - void process_live (nano::block_hash const &, std::shared_ptr); + void process_live (nano::block_hash const &, std::shared_ptr, const bool = false); bool stopped; bool active; std::chrono::steady_clock::time_point next_log; diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 0c8c57d071..8b14a988a2 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -2818,6 +2818,7 @@ void nano::json_handler::payment_wait () void nano::json_handler::process () { const bool json_block_l = request.get ("json_block", false); + const bool watch_work_l = request.get ("watch_work", true); std::shared_ptr block; if (json_block_l) { @@ -2906,7 +2907,7 @@ void nano::json_handler::process () auto transaction (node.store.tx_begin_write ()); // Set current time to trigger automatic rebroadcast and election nano::unchecked_info info (block, block->account (), nano::seconds_since_epoch (), nano::signature_verification::unknown); - result = node.block_processor.process_one (transaction, info); + result = node.block_processor.process_one (transaction, info, watch_work_l); } switch (result.code) { diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 9157c32e85..ae0f77837c 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -1579,6 +1579,66 @@ TEST (rpc, process_block) ASSERT_EQ (send.hash ().to_string (), send_hash); } +TEST (rpc, process_block_with_work_watcher) +{ + nano::system system; + nano::node_config node_config (24000, system.logging); + node_config.enable_voting = false; + auto & node1 = *system.add_node (node_config); + nano::keypair key; + auto latest (system.nodes[0]->latest (nano::test_genesis_key.pub)); + auto send (std::make_shared (nano::test_genesis_key.pub, latest, nano::test_genesis_key.pub, nano::genesis_amount - 100, nano::test_genesis_key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, system.work.generate (latest))); + uint64_t difficulty1 (0); + nano::work_validate (*send, &difficulty1); + auto multiplier1 = nano::difficulty::to_multiplier (difficulty1, node1.network_params.network.publish_threshold); + enable_ipc_transport_tcp (node1.config.ipc_config.transport_tcp); + nano::node_rpc_config node_rpc_config; + nano::ipc::ipc_server ipc_server (node1, node_rpc_config); + nano::rpc_config rpc_config (true); + nano::ipc_rpc_processor ipc_rpc_processor (system.io_ctx, rpc_config); + nano::rpc rpc (system.io_ctx, rpc_config, ipc_rpc_processor); + rpc.start (); + boost::property_tree::ptree request; + request.put ("action", "process"); + request.put ("work_watcher", true); + std::string json; + send->serialize_json (json); + request.put ("block", json); + test_response response (request, rpc.config.port, system.io_ctx); + system.deadline_set (5s); + while (response.status == 0) + { + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_EQ (200, response.status); + system.deadline_set (10s); + while (system.nodes[0]->latest (nano::test_genesis_key.pub) != send->hash ()) + { + ASSERT_NO_ERROR (system.poll ()); + } + system.deadline_set (10s); + auto updated (false); + uint64_t updated_difficulty; + while (!updated) + { + std::unique_lock lock (node1.active.mutex); + //fill multipliers_cb and update active difficulty; + for (auto i (0); i < node1.active.multipliers_cb.size (); i++) + { + node1.active.multipliers_cb.push_back (multiplier1 * (1 + i / 100.)); + } + node1.active.update_active_difficulty (lock); + auto const existing (node1.active.roots.find (send->qualified_root ())); + //if existing is junk the block has been confirmed already + ASSERT_NE (existing, node1.active.roots.end ()); + updated = existing->difficulty != difficulty1; + updated_difficulty = existing->difficulty; + lock.unlock (); + ASSERT_NO_ERROR (system.poll ()); + } + ASSERT_GT (updated_difficulty, difficulty1); +} + TEST (rpc, process_block_no_work) { nano::system system (24000, 1); From b9d70a0e0510c9a08dbe23f9d018efef0ce2d6e4 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Fri, 2 Aug 2019 09:11:04 -0400 Subject: [PATCH 5/7] normalize bool name to fit other usages --- nano/node/blockprocessor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index e2c8d2c383..2ac7d39351 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -387,7 +387,7 @@ void nano::block_processor::process_live (nano::block_hash const & hash_a, std:: }); } -nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, nano::unchecked_info info_a, const bool work_watcher_a) +nano::process_return nano::block_processor::process_one (nano::transaction const & transaction_a, nano::unchecked_info info_a, const bool watch_work_a) { nano::process_return result; auto hash (info_a.block->hash ()); @@ -405,7 +405,7 @@ nano::process_return nano::block_processor::process_one (nano::transaction const } if (info_a.modified > nano::seconds_since_epoch () - 300 && node.block_arrival.recent (hash)) { - process_live (hash, info_a.block, work_watcher_a); + process_live (hash, info_a.block, watch_work_a); } queue_unchecked (transaction_a, hash); break; From b92291633bee3c24dc70a8cff26da60d8570cbe1 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Fri, 9 Aug 2019 09:42:41 -0400 Subject: [PATCH 6/7] add watch_work setting to false to require enable_control --- nano/rpc/rpc_handler.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nano/rpc/rpc_handler.cpp b/nano/rpc/rpc_handler.cpp index 298ba1b216..f7abfd4776 100644 --- a/nano/rpc/rpc_handler.cpp +++ b/nano/rpc/rpc_handler.cpp @@ -88,7 +88,8 @@ void nano::rpc_handler::process_request () else if (action == "process") { auto force = request.get_optional ("force"); - if (force.is_initialized () && *force && !rpc_config.enable_control) + auto watch_work = request.get_optional ("watch_work"); + if (((force.is_initialized () && *force) || (watch_work.is_initialized () && *watch_work)) && !rpc_config.enable_control) { json_error_response (response, rpc_control_disabled_ec.message ()); error = true; From 016616e00e6755824103963bafaa6f2316765761 Mon Sep 17 00:00:00 2001 From: Russel Waters Date: Fri, 9 Aug 2019 09:56:37 -0400 Subject: [PATCH 7/7] default watch_work is true, should check for false --- nano/rpc/rpc_handler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/rpc/rpc_handler.cpp b/nano/rpc/rpc_handler.cpp index f7abfd4776..f990c9b81a 100644 --- a/nano/rpc/rpc_handler.cpp +++ b/nano/rpc/rpc_handler.cpp @@ -89,7 +89,7 @@ void nano::rpc_handler::process_request () { auto force = request.get_optional ("force"); auto watch_work = request.get_optional ("watch_work"); - if (((force.is_initialized () && *force) || (watch_work.is_initialized () && *watch_work)) && !rpc_config.enable_control) + if (((force.is_initialized () && *force) || (watch_work.is_initialized () && !*watch_work)) && !rpc_config.enable_control) { json_error_response (response, rpc_control_disabled_ec.message ()); error = true;