Merge with develop

.github/workflows/tests.yml

@@ -20,6 +20,22 @@ jobs:
       - name: Run Tests
         run: ci/build-travis.sh "/tmp/qt/lib/cmake/Qt5";
 
+  clang_format:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@50fbc62
+      - name: Get Clang 9
+        env: 
+          DEBIAN_FRONTEND: noninteractive
+        run: |
+          APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - 
+          sudo add-apt-repository 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' -y
+          sudo apt-get update -y
+          sudo apt-get install clang-tools-9 clang-format-9
+          sudo ln -s /usr/bin/clang-format-9 /usr/bin/clang-format
+      - name: Clang Format
+        run: ci/check-commit-format.sh 
+
   gcc_test:
     runs-on: ubuntu-18.04
     steps:

ci/check-commit-format.sh

@@ -10,4 +10,7 @@ if [ "$RESULT" != "no modified files to format" ] && [ "$RESULT" != "clang-forma
     echo
     echo "Code formatting differs from expected - please run ci/clang-format-all.sh"
     exit 1
+else
+    echo "clang-format passed"
+    exit 0
 fi

nano/core_test/CMakeLists.txt

@@ -6,6 +6,7 @@ add_executable (core_test
 	block_store.cpp
 	bootstrap.cpp
 	confirmation_height.cpp
+	confirmation_solicitor.cpp
 	conflicts.cpp
 	difficulty.cpp
 	distributed_work.cpp

nano/core_test/active_transactions.cpp

@@ -637,11 +637,13 @@ TEST (active_transactions, restart_dropped)
 		}
 		ASSERT_NO_ERROR (system.poll ());
 	}
-	// Verify the block was updated in the ledger
+	std::shared_ptr<nano::block> block;
+	while (block == nullptr)
 	{
-		auto block (node.store.block_get (node.store.tx_begin_read (), send1->hash ()));
-		ASSERT_EQ (work2, block->block_work ());
+		ASSERT_NO_ERROR (system.poll ());
+		block = node.store.block_get (node.store.tx_begin_read (), send1->hash ());
 	}
+	ASSERT_EQ (work2, block->block_work ());
 	// Drop election
 	node.active.erase (*send2);
 	// Try to restart election with the lower difficulty block, should not work since the block as lower work

nano/core_test/confirmation_solicitor.cpp

@@ -0,0 +1,60 @@
+#include <nano/core_test/testutil.hpp>
+#include <nano/lib/jsonconfig.hpp>
+#include <nano/node/testing.hpp>
+
+#include <gtest/gtest.h>
+
+using namespace std::chrono_literals;
+
+TEST (confirmation_solicitor, batches)
+{
+	nano::system system;
+	nano::node_config node_config (nano::get_available_port (), system.logging);
+	node_config.enable_voting = false;
+	node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	auto & node1 = *system.add_node (node_config);
+	auto channel1 (node1.network.udp_channels.create (node1.network.endpoint ()));
+	// Solicitor will only solicit from this representative
+	nano::representative representative (nano::test_genesis_key.pub, nano::genesis_amount, channel1);
+	node_config.peering_port = nano::get_available_port ();
+	nano::node_flags node_flags;
+	// To prevent races on the solicitor
+	node_flags.disable_request_loop = true;
+	auto & node2 = *system.add_node (node_config, node_flags);
+	// Lock active_transactions which uses the solicitor
+	{
+		nano::lock_guard<std::mutex> active_guard (node2.active.mutex);
+		std::vector<nano::representative> representatives{ representative };
+		node2.active.solicitor.prepare (representatives);
+		// Ensure the representatives are correct
+		ASSERT_EQ (1, representatives.size ());
+		ASSERT_EQ (channel1, representatives.front ().channel);
+		ASSERT_EQ (nano::test_genesis_key.pub, representatives.front ().account);
+		nano::genesis genesis;
+		auto send (std::make_shared<nano::send_block> (genesis.open->hash (), nano::keypair ().pub, nano::genesis_amount - 100, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.open->hash ())));
+		for (size_t i (0); i < nano::network::confirm_req_hashes_max; ++i)
+		{
+			auto election (std::make_shared<nano::election> (node2, send, false, nullptr));
+			ASSERT_FALSE (node2.active.solicitor.add (*election));
+		}
+		ASSERT_EQ (1, node2.active.solicitor.max_confirm_req_batches);
+		// Reached the maximum amount of requests for the channel
+		auto election (std::make_shared<nano::election> (node2, send, false, nullptr));
+		ASSERT_TRUE (node2.active.solicitor.add (*election));
+		// Broadcasting should be immediate
+		ASSERT_EQ (0, node2.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out));
+		ASSERT_FALSE (node2.active.solicitor.broadcast (*election));
+		while (node2.stats.count (nano::stat::type::message, nano::stat::detail::publish, nano::stat::dir::out) < 1)
+		{
+			ASSERT_NO_ERROR (system.poll ());
+		}
+	}
+	// From rep crawler
+	ASSERT_EQ (1, node2.stats.count (nano::stat::type::message, nano::stat::detail::confirm_req, nano::stat::dir::out));
+	system.deadline_set (5s);
+	node2.active.solicitor.flush ();
+	while (node2.stats.count (nano::stat::type::message, nano::stat::detail::confirm_req, nano::stat::dir::out) < 2)
+	{
+		ASSERT_NO_ERROR (system.poll ());
+	}
+}

nano/core_test/ledger.cpp

@@ -735,9 +735,11 @@ TEST (votes, check_signature)
 		ASSERT_EQ (nano::process_result::progress, node1.ledger.process (transaction, *send1).code);
 	}
 	node1.active.start (send1);
-	nano::lock_guard<std::mutex> lock (node1.active.mutex);
-	auto votes1 (node1.active.roots.find (send1->qualified_root ())->election);
-	ASSERT_EQ (1, votes1->last_votes.size ());
+	{
+		nano::lock_guard<std::mutex> lock (node1.active.mutex);
+		auto votes1 (node1.active.roots.find (send1->qualified_root ())->election);
+		ASSERT_EQ (1, votes1->last_votes.size ());
+	}
 	auto vote1 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 1, send1));
 	vote1->signature.bytes[0] ^= 1;
 	auto transaction (node1.store.tx_begin_read ());
@@ -870,8 +872,11 @@ TEST (votes, add_old)
 	ASSERT_EQ (nano::process_result::progress, node1.ledger.process (transaction, *send1).code);
 	node1.active.start (send1);
 	auto vote1 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 2, send1));
-	nano::lock_guard<std::mutex> lock (node1.active.mutex);
-	auto votes1 (node1.active.roots.find (send1->qualified_root ())->election);
+	std::shared_ptr<nano::election> votes1;
+	{
+		nano::lock_guard<std::mutex> lock (node1.active.mutex);
+		votes1 = node1.active.roots.find (send1->qualified_root ())->election;
+	}
 	auto channel (std::make_shared<nano::transport::channel_udp> (node1.network.udp_channels, node1.network.endpoint (), node1.network_params.protocol.protocol_version));
 	node1.vote_processor.vote_blocking (transaction, vote1, channel);
 	nano::keypair key2;
@@ -880,7 +885,7 @@ TEST (votes, add_old)
 	auto vote2 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 1, send2));
 	votes1->last_votes[nano::test_genesis_key.pub].time = std::chrono::steady_clock::now () - std::chrono::seconds (20);
 	node1.vote_processor.vote_blocking (transaction, vote2, channel);
-	ASSERT_EQ (2, votes1->last_votes.size ());
+	ASSERT_EQ (2, votes1->last_votes_size ());
 	ASSERT_NE (votes1->last_votes.end (), votes1->last_votes.find (nano::test_genesis_key.pub));
 	ASSERT_EQ (send1->hash (), votes1->last_votes[nano::test_genesis_key.pub].hash);
 	auto winner (*votes1->tally ().begin ());
@@ -903,11 +908,15 @@ TEST (votes, add_old_different_account)
 	ASSERT_EQ (nano::process_result::progress, node1.ledger.process (transaction, *send2).code);
 	node1.active.start (send1);
 	node1.active.start (send2);
-	nano::unique_lock<std::mutex> lock (node1.active.mutex);
-	auto votes1 (node1.active.roots.find (send1->qualified_root ())->election);
-	auto votes2 (node1.active.roots.find (send2->qualified_root ())->election);
-	ASSERT_EQ (1, votes1->last_votes.size ());
-	ASSERT_EQ (1, votes2->last_votes.size ());
+	std::shared_ptr<nano::election> votes1;
+	std::shared_ptr<nano::election> votes2;
+	{
+		nano::unique_lock<std::mutex> lock (node1.active.mutex);
+		votes1 = node1.active.roots.find (send1->qualified_root ())->election;
+		votes2 = node1.active.roots.find (send2->qualified_root ())->election;
+	}
+	ASSERT_EQ (1, votes1->last_votes_size ());
+	ASSERT_EQ (1, votes2->last_votes_size ());
 	auto vote1 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 2, send1));
 	auto channel (std::make_shared<nano::transport::channel_udp> (node1.network.udp_channels, node1.network.endpoint (), node1.network_params.protocol.protocol_version));
 	auto vote_result1 (node1.vote_processor.vote_blocking (transaction, vote1, channel));
@@ -941,8 +950,11 @@ TEST (votes, add_cooldown)
 	auto transaction (node1.store.tx_begin_write ());
 	ASSERT_EQ (nano::process_result::progress, node1.ledger.process (transaction, *send1).code);
 	node1.active.start (send1);
-	nano::unique_lock<std::mutex> lock (node1.active.mutex);
-	auto votes1 (node1.active.roots.find (send1->qualified_root ())->election);
+	std::shared_ptr<nano::election> votes1;
+	{
+		nano::unique_lock<std::mutex> lock (node1.active.mutex);
+		votes1 = node1.active.roots.find (send1->qualified_root ())->election;
+	}
 	auto vote1 (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 1, send1));
 	auto channel (std::make_shared<nano::transport::channel_udp> (node1.network.udp_channels, node1.network.endpoint (), node1.network_params.protocol.protocol_version));
 	node1.vote_processor.vote_blocking (transaction, vote1, channel);

nano/core_test/node.cpp

@@ -2499,6 +2499,33 @@ TEST (node, local_votes_cache_generate_new_vote)
 	ASSERT_FALSE (node.votes_cache.find (send2->hash ()).empty ());
 }
 
+// Tests that the max cache size is inversely proportional to the number of voting accounts
+TEST (node, local_votes_cache_size)
+{
+	nano::system system;
+	nano::node_config node_config (nano::get_available_port (), system.logging);
+	node_config.frontiers_confirmation = nano::frontiers_confirmation_mode::disabled;
+	node_config.vote_minimum = 0; // wallet will pick up the second account as voting even if unopened
+	auto & node (*system.add_node (node_config));
+	ASSERT_EQ (node.network_params.voting.max_cache, 2); // effective cache size is 1 with 2 voting accounts
+	nano::genesis genesis;
+	nano::keypair key;
+	auto & wallet (*system.wallet (0));
+	wallet.insert_adhoc (nano::test_genesis_key.prv);
+	wallet.insert_adhoc (nano::keypair ().prv);
+	ASSERT_EQ (2, node.wallets.rep_counts ().voting);
+	auto transaction (node.store.tx_begin_read ());
+	auto vote1 (node.store.vote_generate (transaction, nano::test_genesis_key.pub, nano::test_genesis_key.prv, { genesis.open->hash () }));
+	nano::block_hash hash (1);
+	auto vote2 (node.store.vote_generate (transaction, nano::test_genesis_key.pub, nano::test_genesis_key.prv, { hash }));
+	node.votes_cache.add (vote1);
+	node.votes_cache.add (vote2);
+	auto existing2 (node.votes_cache.find (hash));
+	ASSERT_EQ (1, existing2.size ());
+	ASSERT_EQ (vote2, existing2.front ());
+	ASSERT_EQ (0, node.votes_cache.find (genesis.open->hash ()).size ());
+}
+
 TEST (node, vote_republish)
 {
 	nano::system system (2);
@@ -2769,7 +2796,6 @@ TEST (node, fork_invalid_block_signature_vote_by_hash)
 	auto vote (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 0, vote_blocks));
 	{
 		auto transaction (node1.store.tx_begin_read ());
-		nano::unique_lock<std::mutex> lock (node1.active.mutex);
 		node1.vote_processor.vote_blocking (transaction, vote, std::make_shared<nano::transport::channel_udp> (node1.network.udp_channels, node1.network.endpoint (), node1.network_params.protocol.protocol_version));
 	}
 	while (node1.block (send1->hash ()))
@@ -2938,7 +2964,6 @@ TEST (node, confirm_back)
 	auto vote (std::make_shared<nano::vote> (nano::test_genesis_key.pub, nano::test_genesis_key.prv, 0, vote_blocks));
 	{
 		auto transaction (node.store.tx_begin_read ());
-		nano::unique_lock<std::mutex> lock (node.active.mutex);
 		node.vote_processor.vote_blocking (transaction, vote, std::make_shared<nano::transport::channel_udp> (node.network.udp_channels, node.network.endpoint (), node.network_params.protocol.protocol_version));
 	}
 	system.deadline_set (10s);

nano/lib/config.hpp

@@ -73,7 +73,7 @@ class network_constants
 		default_rpc_port = is_live_network () ? 7076 : is_beta_network () ? 55000 : 45000;
 		default_ipc_port = is_live_network () ? 7077 : is_beta_network () ? 56000 : 46000;
 		default_websocket_port = is_live_network () ? 7078 : is_beta_network () ? 57000 : 47000;
-		request_interval_ms = is_test_network () ? (is_sanitizer_build ? 100 : 20) : 500;
+		request_interval_ms = is_test_network () ? 20 : 500;
 	}
 
 	/** Network work thresholds. ~5 seconds of work for the live network */