Add lazy bootstrap possible links and accounts (#2315)

* request bulk_pull for state blocks where previous block is legacy receive, open or change (doesn't contain balance in block)
* request bulk_pull for most used destinations accounts
* allow to drop lazy bulk_pulls after failed attempt for destiations or unknown state blocks
* split bootstrap core_test
* allow hash_or_account to be concrete
* improve requeue invalid signature function
* add bootstrap attempt duration

nano/core_test/CMakeLists.txt

@@ -4,6 +4,7 @@ add_executable (core_test
 	active_transactions.cpp
 	block.cpp
 	block_store.cpp
+	bootstrap.cpp
 	confirmation_height.cpp
 	conflicts.cpp
 	difficulty.cpp

nano/lib/numbers.cpp

@@ -827,39 +827,19 @@ nano::hash_or_account::operator nano::uint256_union const & () const
 	return raw;
 }
 
-nano::link::link (uint64_t value_a) :
-hash_or_account (value_a)
-{
-}
-
-nano::root::root (uint64_t value_a) :
-hash_or_account (value_a)
-{
-}
-
 nano::block_hash const & nano::root::previous () const
 {
 	return hash;
 }
 
-bool nano::root::operator== (nano::root const & root_a) const
-{
-	return bytes == root_a.bytes;
-}
-
-bool nano::root::operator!= (nano::root const & root_a) const
+bool nano::hash_or_account::operator== (nano::hash_or_account const & hash_or_account_a) const
 {
-	return !(*this == root_a);
+	return bytes == hash_or_account_a.bytes;
 }
 
-bool nano::link::operator== (nano::link const & link_a) const
+bool nano::hash_or_account::operator!= (nano::hash_or_account const & hash_or_account_a) const
 {
-	return bytes == link_a.bytes;
-}
-
-bool nano::link::operator!= (nano::link const & link_a) const
-{
-	return !(*this == link_a);
+	return !(*this == hash_or_account_a);
 }
 
 std::string nano::to_string_hex (uint64_t const value_a)
@@ -942,20 +922,30 @@ double nano::difficulty::to_multiplier (uint64_t const difficulty_a, uint64_t co
 
 nano::public_key::operator nano::link const & () const
 {
-	return nano::to_link (*this);
+	return reinterpret_cast<nano::link const &> (*this);
 }
 
 nano::public_key::operator nano::root const & () const
 {
-	return nano::to_root (*this);
+	return reinterpret_cast<nano::root const &> (*this);
+}
+
+nano::public_key::operator nano::hash_or_account const & () const
+{
+	return reinterpret_cast<nano::hash_or_account const &> (*this);
 }
 
 nano::block_hash::operator nano::link const & () const
 {
-	return nano::to_link (*this);
+	return reinterpret_cast<nano::link const &> (*this);
 }
 
 nano::block_hash::operator nano::root const & () const
 {
-	return nano::to_root (*this);
+	return reinterpret_cast<nano::root const &> (*this);
+}
+
+nano::block_hash::operator nano::hash_or_account const & () const
+{
+	return reinterpret_cast<nano::hash_or_account const &> (*this);
 }

nano/lib/numbers.hpp

@@ -100,6 +100,7 @@ static_assert (std::is_nothrow_move_constructible<uint256_union>::value, "uint25
 
 class link;
 class root;
+class hash_or_account;
 
 // All keys and hashes are 256 bit.
 class block_hash final : public uint256_union
@@ -108,6 +109,7 @@ class block_hash final : public uint256_union
 	using uint256_union::uint256_union;
 	operator nano::link const & () const;
 	operator nano::root const & () const;
+	operator nano::hash_or_account const & () const;
 };
 
 class public_key final : public uint256_union
@@ -122,6 +124,7 @@ class public_key final : public uint256_union
 
 	operator nano::link const & () const;
 	operator nano::root const & () const;
+	operator nano::hash_or_account const & () const;
 };
 
 class wallet_id : public uint256_union
@@ -135,6 +138,9 @@ using account = public_key;
 class hash_or_account
 {
 public:
+	hash_or_account () = default;
+	hash_or_account (uint64_t value_a);
+
 	bool is_zero () const;
 	void clear ();
 	std::string to_string () const;
@@ -146,40 +152,30 @@ class hash_or_account
 	operator nano::account const & () const;
 	operator nano::uint256_union const & () const;
 
+	bool operator== (nano::hash_or_account const &) const;
+	bool operator!= (nano::hash_or_account const &) const;
+
 	union
 	{
 		std::array<uint8_t, 32> bytes;
 		nano::uint256_union raw; // This can be used when you don't want to explicitly mention either of the types
 		nano::account account;
 		nano::block_hash hash;
 	};
-
-protected:
-	// Cannot instantiate a concrete version of this class
-	hash_or_account () = default;
-	hash_or_account (uint64_t value_a);
 };
 
 // A link can either be a destination account or source hash
 class link final : public hash_or_account
 {
 public:
-	link () = default;
-	link (uint64_t value_a);
-
-	bool operator== (nano::link const &) const;
-	bool operator!= (nano::link const &) const;
+	using hash_or_account::hash_or_account;
 };
 
 // A root can either be an open block hash or a previous hash
 class root final : public hash_or_account
 {
 public:
-	root () = default;
-	root (uint64_t value_a);
-
-	bool operator== (nano::root const &) const;
-	bool operator!= (nano::root const &) const;
+	using hash_or_account::hash_or_account;
 
 	nano::block_hash const & previous () const;
 };
@@ -227,33 +223,6 @@ class uint512_union
 };
 static_assert (std::is_nothrow_move_constructible<uint512_union>::value, "uint512_union should be noexcept MoveConstructible");
 
-inline nano::link const & to_link (nano::block_hash const & hash_a)
-{
-	return reinterpret_cast<nano::link const &> (hash_a);
-}
-
-inline nano::link const & to_link (nano::account const & account_a)
-{
-	return reinterpret_cast<nano::link const &> (account_a);
-}
-
-inline nano::root const & to_root (nano::block_hash const & hash_a)
-{
-	return reinterpret_cast<nano::root const &> (hash_a);
-}
-
-inline nano::root const & to_root (nano::account const & account_a)
-{
-	return reinterpret_cast<nano::root const &> (account_a);
-}
-
-inline nano::account const & root_as_account (nano::root const & root_a)
-{
-	static_assert (sizeof (nano::root) == sizeof (nano::account), "Sizes do not match");
-	static_assert (std::is_standard_layout<nano::root>::value && std::is_standard_layout<nano::account>::value, "Both types must have standard layout");
-	return reinterpret_cast<nano::account const &> (root_a);
-}
-
 class signature : public uint512_union
 {
 public:

nano/node/blockprocessor.cpp

@@ -226,7 +226,7 @@ void nano::block_processor::verify_state_blocks (nano::unique_lock<std::mutex> &
 			else
 			{
 				blocks_filter.erase (filter_item (hashes[i], blocks_signatures[i]));
-				requeue_invalid (hashes[i]);
+				requeue_invalid (hashes[i], item);
 			}
 			items.pop_front ();
 		}
@@ -456,7 +456,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction
 			{
 				node.logger.try_log (boost::str (boost::format ("Bad signature for: %1%") % hash.to_string ()));
 			}
-			requeue_invalid (hash);
+			requeue_invalid (hash, info_a);
 			break;
 		}
 		case nano::process_result::negative_spend:
@@ -552,11 +552,12 @@ nano::block_hash nano::block_processor::filter_item (nano::block_hash const & ha
 	return result;
 }
 
-void nano::block_processor::requeue_invalid (nano::block_hash const & hash_a)
+void nano::block_processor::requeue_invalid (nano::block_hash const & hash_a, nano::unchecked_info const & info_a)
 {
+	assert (hash_a == info_a.block->hash ());
 	auto attempt (node.bootstrap_initiator.current_attempt ());
 	if (attempt != nullptr && attempt->mode == nano::bootstrap_mode::lazy)
 	{
-		attempt->lazy_requeue (hash_a);
+		attempt->lazy_requeue (hash_a, info_a.block->previous (), info_a.confirmed);
 	}
 }

nano/node/blockprocessor.hpp

@@ -59,7 +59,7 @@ class block_processor final
 	void verify_state_blocks (nano::unique_lock<std::mutex> &, size_t = std::numeric_limits<size_t>::max ());
 	void process_batch (nano::unique_lock<std::mutex> &);
 	void process_live (nano::block_hash const &, std::shared_ptr<nano::block>, const bool = false);
-	void requeue_invalid (nano::block_hash const &);
+	void requeue_invalid (nano::block_hash const &, nano::unchecked_info const &);
 	bool stopped;
 	bool active;
 	bool awaiting_write{ false };