Merge pull request #4482 from simpago/rep-weight-table

rep_weights part 1 - Add data store for representative weights

nano/core_test/CMakeLists.txt

@@ -38,6 +38,7 @@ add_executable(
   processor_service.cpp
   rep_crawler.cpp
   peer_container.cpp
+  rep_weight_store.cpp
   scheduler_buckets.cpp
   request_aggregator.cpp
   signal_manager.cpp

nano/core_test/rep_weight_store.cpp

@@ -0,0 +1,75 @@
+#include <nano/lib/numbers.hpp>
+#include <nano/store/component.hpp>
+#include <nano/store/rep_weight.hpp>
+#include <nano/test_common/make_store.hpp>
+
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <iostream>
+
+TEST (rep_weight_store, empty)
+{
+	auto store = nano::test::make_store ();
+	ASSERT_TRUE (!store->init_error ());
+	auto txn{ store->tx_begin_read () };
+	ASSERT_EQ (0, store->rep_weight.count (txn));
+}
+
+TEST (rep_weight_store, add_item)
+{
+	auto store = nano::test::make_store ();
+	ASSERT_TRUE (!store->init_error ());
+	auto txn{ store->tx_begin_write () };
+
+	nano::account representative{ 123 };
+	nano::uint128_t weight{ 456 };
+	store->rep_weight.put (txn, representative, weight);
+
+	ASSERT_EQ (1, store->rep_weight.count (txn));
+	ASSERT_EQ (weight, store->rep_weight.get (txn, representative));
+}
+
+TEST (rep_weight_store, del)
+{
+	auto store = nano::test::make_store ();
+	ASSERT_TRUE (!store->init_error ());
+	auto txn{ store->tx_begin_write () };
+
+	store->rep_weight.put (txn, 1, 100);
+	store->rep_weight.put (txn, 2, 200);
+	store->rep_weight.put (txn, 3, 300);
+
+	store->rep_weight.del (txn, 2);
+
+	ASSERT_EQ (2, store->rep_weight.count (txn));
+	ASSERT_EQ (0, store->rep_weight.get (txn, 200));
+}
+
+TEST (rep_weight_store, for_each_par)
+{
+	auto store = nano::test::make_store ();
+	ASSERT_TRUE (!store->init_error ());
+	{
+		auto txn{ store->tx_begin_write () };
+		for (auto i = 0; i < 50; ++i)
+		{
+			store->rep_weight.put (txn, i, 100);
+		}
+	}
+
+	std::atomic_size_t rep_total{ 0 };
+	std::atomic_size_t weight_total{ 0 };
+
+	store->rep_weight.for_each_par (
+	[&rep_total, &weight_total] (auto const &, auto i, auto n) {
+		for (; i != n; ++i)
+		{
+			rep_total.fetch_add (static_cast<std::size_t> (i->first.number ()));
+			weight_total.fetch_add (static_cast<std::size_t> (i->second.number ()));
+		}
+	});
+
+	ASSERT_EQ (1225, rep_total.load ());
+	ASSERT_EQ (50 * 100, weight_total.load ());
+}

nano/node/make_store.cpp

@@ -1,3 +1,4 @@
+#include <nano/lib/logging.hpp>
 #include <nano/node/make_store.hpp>
 #include <nano/store/lmdb/lmdb.hpp>
 #include <nano/store/rocksdb/rocksdb.hpp>

nano/store/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(
   lmdb/peer.hpp
   lmdb/pending.hpp
   lmdb/pruned.hpp
+  lmdb/rep_weight.hpp
   lmdb/transaction_impl.hpp
   lmdb/version.hpp
   lmdb/wallet_value.hpp
@@ -43,6 +44,7 @@ add_library(
   rocksdb/peer.hpp
   rocksdb/pending.hpp
   rocksdb/pruned.hpp
+  rocksdb/rep_weight.hpp
   rocksdb/rocksdb.hpp
   rocksdb/iterator.hpp
   rocksdb/transaction_impl.hpp
@@ -73,6 +75,7 @@ add_library(
   lmdb/peer.cpp
   lmdb/pending.cpp
   lmdb/pruned.cpp
+  lmdb/rep_weight.cpp
   lmdb/version.cpp
   lmdb/wallet_value.cpp
   online_weight.cpp
@@ -89,6 +92,7 @@ add_library(
   rocksdb/peer.cpp
   rocksdb/pending.cpp
   rocksdb/pruned.cpp
+  rocksdb/rep_weight.cpp
   rocksdb/rocksdb.cpp
   rocksdb/transaction.cpp
   rocksdb/version.cpp

nano/store/component.cpp

@@ -6,8 +6,9 @@
 #include <nano/store/component.hpp>
 #include <nano/store/confirmation_height.hpp>
 #include <nano/store/frontier.hpp>
+#include <nano/store/rep_weight.hpp>
 
-nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a) :
+nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a, nano::store::rep_weight & rep_weight_a) :
 	block (block_store_a),
 	frontier (frontier_store_a),
 	account (account_store_a),
@@ -17,7 +18,8 @@ nano::store::component::component (nano::store::block & block_store_a, nano::sto
 	peer (peer_store_a),
 	confirmation_height (confirmation_height_store_a),
 	final_vote (final_vote_store_a),
-	version (version_store_a)
+	version (version_store_a),
+	rep_weight (rep_weight_a)
 {
 }
 

nano/store/component.hpp

@@ -27,6 +27,7 @@ namespace store
 	class pending;
 	class pruned;
 	class version;
+	class rep_weight;
 }
 class ledger_cache;
 
@@ -52,7 +53,8 @@ namespace store
 		nano::store::peer &,
 		nano::store::confirmation_height &,
 		nano::store::final_vote &,
-		nano::store::version &
+		nano::store::version &,
+		nano::store::rep_weight &
 	);
 		// clang-format on
 		virtual ~component () = default;
@@ -68,6 +70,7 @@ namespace store
 		store::frontier & frontier;
 		store::account & account;
 		store::pending & pending;
+		store::rep_weight & rep_weight;
 		static int constexpr version_minimum{ 21 };
 		static int constexpr version_current{ 22 };
 

nano/store/lmdb/lmdb.cpp

@@ -24,7 +24,8 @@ nano::store::lmdb::component::component (nano::logger & logger_a, std::filesyste
 		peer_store,
 		confirmation_height_store,
 		final_vote_store,
-		version_store
+		version_store,
+		rep_weight_store
 	},
 	// clang-format on
 	block_store{ *this },
@@ -37,6 +38,7 @@ nano::store::lmdb::component::component (nano::logger & logger_a, std::filesyste
 	confirmation_height_store{ *this },
 	final_vote_store{ *this },
 	version_store{ *this },
+	rep_weight_store{ *this },
 	logger{ logger_a },
 	env (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)),
 	mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a),
@@ -204,6 +206,7 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa
 	pending_store.pending_handle = pending_store.pending_v0_handle;
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0;
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0;
+	mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle);
 }
 
 bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming)
@@ -339,6 +342,8 @@ MDB_dbi nano::store::lmdb::component::table_to_dbi (tables table_a) const
 			return confirmation_height_store.confirmation_height_handle;
 		case tables::final_votes:
 			return final_vote_store.final_votes_handle;
+		case tables::rep_weights:
+			return rep_weight_store.rep_weights_handle;
 		default:
 			release_assert (false);
 			return peer_store.peers_handle;

nano/store/lmdb/lmdb.hpp

@@ -18,6 +18,7 @@
 #include <nano/store/lmdb/peer.hpp>
 #include <nano/store/lmdb/pending.hpp>
 #include <nano/store/lmdb/pruned.hpp>
+#include <nano/store/lmdb/rep_weight.hpp>
 #include <nano/store/lmdb/transaction_impl.hpp>
 #include <nano/store/lmdb/version.hpp>
 #include <nano/store/versioning.hpp>
@@ -50,6 +51,7 @@ class component : public nano::store::component
 	nano::store::lmdb::pending pending_store;
 	nano::store::lmdb::pruned pruned_store;
 	nano::store::lmdb::version version_store;
+	nano::store::lmdb::rep_weight rep_weight_store;
 
 	friend class nano::store::lmdb::account;
 	friend class nano::store::lmdb::block;
@@ -61,6 +63,7 @@ class component : public nano::store::component
 	friend class nano::store::lmdb::pending;
 	friend class nano::store::lmdb::pruned;
 	friend class nano::store::lmdb::version;
+	friend class nano::store::lmdb::rep_weight;
 
 public:
 	component (nano::logger &, std::filesystem::path const &, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);

nano/store/lmdb/rep_weight.cpp

@@ -0,0 +1,68 @@
+#include <nano/lib/numbers.hpp>
+#include <nano/secure/parallel_traversal.hpp>
+#include <nano/store/lmdb/lmdb.hpp>
+#include <nano/store/lmdb/rep_weight.hpp>
+
+#include <iostream>
+#include <stdexcept>
+
+nano::store::lmdb::rep_weight::rep_weight (nano::store::lmdb::component & store_a) :
+	store{ store_a }
+{
+}
+
+uint64_t nano::store::lmdb::rep_weight::count (store::transaction const & txn_a)
+{
+	return store.count (txn_a, tables::rep_weights);
+}
+
+nano::uint128_t nano::store::lmdb::rep_weight::get (store::transaction const & txn_a, nano::account const & representative_a)
+{
+	nano::store::lmdb::db_val value;
+	auto status = store.get (txn_a, tables::rep_weights, representative_a, value);
+	release_assert (store.success (status) || store.not_found (status));
+	nano::uint128_t weight{ 0 };
+	if (store.success (status))
+	{
+		nano::uint128_union weight_union{ value };
+		weight = weight_union.number ();
+	}
+	return weight;
+}
+
+void nano::store::lmdb::rep_weight::put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a)
+{
+	nano::uint128_union weight{ weight_a };
+	auto status = store.put (txn_a, tables::rep_weights, representative_a, weight);
+	store.release_assert_success (status);
+}
+
+void nano::store::lmdb::rep_weight::del (store::write_transaction const & txn_a, nano::account const & representative_a)
+{
+	auto status = store.del (txn_a, tables::rep_weights, representative_a);
+	store.release_assert_success (status);
+}
+
+nano::store::iterator<nano::account, nano::uint128_union> nano::store::lmdb::rep_weight::begin (store::transaction const & transaction_a, nano::account const & representative_a) const
+{
+	return store.make_iterator<nano::account, nano::uint128_union> (transaction_a, tables::rep_weights, representative_a);
+}
+
+nano::store::iterator<nano::account, nano::uint128_union> nano::store::lmdb::rep_weight::begin (store::transaction const & transaction_a) const
+{
+	return store.make_iterator<nano::account, nano::uint128_union> (transaction_a, tables::rep_weights);
+}
+
+nano::store::iterator<nano::account, nano::uint128_union> nano::store::lmdb::rep_weight::end () const
+{
+	return nano::store::iterator<nano::account, nano::uint128_union> (nullptr);
+}
+
+void nano::store::lmdb::rep_weight::for_each_par (std::function<void (store::read_transaction const &, store::iterator<nano::account, nano::uint128_union>, store::iterator<nano::account, nano::uint128_union>)> const & action_a) const
+{
+	parallel_traversal<nano::uint256_t> (
+	[&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) {
+		auto transaction (this->store.tx_begin_read ());
+		action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ());
+	});
+}

nano/store/lmdb/rep_weight.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <nano/store/rep_weight.hpp>
+
+#include <lmdb/libraries/liblmdb/lmdb.h>
+
+namespace nano::store::lmdb
+{
+class component;
+
+class rep_weight : public nano::store::rep_weight
+{
+private:
+	nano::store::lmdb::component & store;
+
+public:
+	explicit rep_weight (nano::store::lmdb::component & store_a);
+
+	uint64_t count (store::transaction const & txn) override;
+	nano::uint128_t get (store::transaction const & txn_a, nano::account const & representative_a) override;
+	void put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) override;
+	void del (store::write_transaction const &, nano::account const & representative_a) override;
+	store::iterator<nano::account, nano::uint128_union> begin (store::transaction const & transaction_a, nano::account const & representative_a) const override;
+	store::iterator<nano::account, nano::uint128_union> begin (store::transaction const & transaction_a) const override;
+	store::iterator<nano::account, nano::uint128_union> end () const override;
+	void for_each_par (std::function<void (store::read_transaction const &, store::iterator<nano::account, nano::uint128_union>, store::iterator<nano::account, nano::uint128_union>)> const & action_a) const override;
+
+	/**
+	 * Representative weights
+	 * nano::account -> uint128_t
+	 */
+	MDB_dbi rep_weights_handle{ 0 };
+};
+}

nano/store/rep_weight.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <nano/lib/numbers.hpp>
+#include <nano/store/component.hpp>
+#include <nano/store/iterator.hpp>
+
+#include <cstdint>
+#include <functional>
+
+namespace nano
+{
+// class account;
+}
+namespace nano::store
+{
+/**
+ * A lookup table of all representatives and their vote weight
+ */
+class rep_weight
+{
+public:
+	virtual ~rep_weight (){};
+	virtual uint64_t count (store::transaction const & txn_a) = 0;
+	virtual nano::uint128_t get (store::transaction const & txn_a, nano::account const & representative_a) = 0;
+	virtual void put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) = 0;
+	virtual void del (store::write_transaction const &, nano::account const & representative_a) = 0;
+	virtual store::iterator<nano::account, nano::uint128_union> begin (store::transaction const & transaction_a, nano::account const & representative_a) const = 0;
+	virtual store::iterator<nano::account, nano::uint128_union> begin (store::transaction const & transaction_a) const = 0;
+	virtual store::iterator<nano::account, nano::uint128_union> end () const = 0;
+	virtual void for_each_par (std::function<void (store::read_transaction const &, store::iterator<nano::account, nano::uint128_union>, store::iterator<nano::account, nano::uint128_union>)> const & action_a) const = 0;
+};
+}