Add CLI command for better LMDB compaction

nano/node/cli.cpp

@@ -51,6 +51,7 @@ void nano::add_node_options (boost::program_options::options_description & descr
 	("peer_clear", "Clear online peers database dump")
 	("unchecked_clear", "Clear unchecked blocks")
 	("confirmation_height_clear", "Clear confirmation height")
+	("rebuild_database", "Rebuild LMDB database with vacuum for best compaction")
 	("diagnostics", "Run internal diagnostics")
 	("generate_config", boost::program_options::value<std::string> (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node or rpc. See also use_defaults.")
 	("key_create", "Generates a adhoc random keypair and prints it to stdout")
@@ -155,7 +156,7 @@ void database_write_lock_error (std::error_code & ec)
 bool copy_database (boost::filesystem::path const & data_path, boost::program_options::variables_map & vm, boost::filesystem::path const & output_path, std::error_code & ec)
 {
 	bool success = false;
-	bool needs_to_write = vm.count ("unchecked_clear") || vm.count ("clear_send_ids") || vm.count ("online_weight_clear") || vm.count ("peer_clear") || vm.count ("confirmation_height_clear");
+	bool needs_to_write = vm.count ("unchecked_clear") || vm.count ("clear_send_ids") || vm.count ("online_weight_clear") || vm.count ("peer_clear") || vm.count ("confirmation_height_clear") || vm.count ("rebuild_database");
 
 	auto node_flags = nano::inactive_node_flag_defaults ();
 	node_flags.read_only = !needs_to_write;
@@ -186,6 +187,11 @@ bool copy_database (boost::filesystem::path const & data_path, boost::program_op
 		{
 			reset_confirmation_heights (node.node->store);
 		}
+		if (vm.count ("rebuild_database"))
+		{
+			auto transaction (node.node->store.tx_begin_write ());
+			node.node->store.rebuild_db (transaction);
+		}
 
 		success = node.node->copy_with_compaction (output_path);
 	}

nano/node/lmdb/lmdb.cpp

@@ -868,6 +868,56 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file)
 	return !mdb_env_copy2 (env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT);
 }
 
+void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
+{
+	// Tables with uint256_union key
+	std::vector<MDB_dbi> tables = { accounts, send_blocks, receive_blocks, open_blocks, change_blocks, state_blocks, vote, confirmation_height };
+	for (auto const & table : tables)
+	{
+		MDB_dbi temp;
+		mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp);
+		// Copy all values to temporary table
+		for (auto i (nano::store_iterator<nano::uint256_union, nano::mdb_val> (std::make_unique<nano::mdb_iterator<nano::uint256_union, nano::mdb_val>> (transaction_a, table))), n (nano::store_iterator<nano::uint256_union, nano::mdb_val> (nullptr)); i != n; ++i)
+		{
+			auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), i->second, MDB_APPEND);
+			release_assert (success (s));
+		}
+		release_assert (count (transaction_a, table) == count (transaction_a, temp));
+		// Clear existing table
+		mdb_drop (env.tx (transaction_a), table, 0);
+		// Put values from copy
+		for (auto i (nano::store_iterator<nano::uint256_union, nano::mdb_val> (std::make_unique<nano::mdb_iterator<nano::uint256_union, nano::mdb_val>> (transaction_a, temp))), n (nano::store_iterator<nano::uint256_union, nano::mdb_val> (nullptr)); i != n; ++i)
+		{
+			auto s = mdb_put (env.tx (transaction_a), table, nano::mdb_val (i->first), i->second, MDB_APPEND);
+			release_assert (success (s));
+		}
+		release_assert (count (transaction_a, table) == count (transaction_a, temp));
+		// Remove temporary table
+		mdb_drop (env.tx (transaction_a), temp, 1);
+	}
+	// Pending table
+	{
+		MDB_dbi temp;
+		mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp);
+		// Copy all values to temporary table
+		for (auto i (nano::store_iterator<nano::pending_key, nano::pending_info> (std::make_unique<nano::mdb_iterator<nano::pending_key, nano::pending_info>> (transaction_a, pending))), n (nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr)); i != n; ++i)
+		{
+			auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
+			release_assert (success (s));
+		}
+		release_assert (count (transaction_a, pending) == count (transaction_a, temp));
+		mdb_drop (env.tx (transaction_a), pending, 0);
+		// Put values from copy
+		for (auto i (nano::store_iterator<nano::pending_key, nano::pending_info> (std::make_unique<nano::mdb_iterator<nano::pending_key, nano::pending_info>> (transaction_a, temp))), n (nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr)); i != n; ++i)
+		{
+			auto s = mdb_put (env.tx (transaction_a), pending, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
+			release_assert (success (s));
+		}
+		release_assert (count (transaction_a, pending) == count (transaction_a, temp));
+		mdb_drop (env.tx (transaction_a), temp, 1);
+	}
+}
+
 bool nano::mdb_store::init_error () const
 {
 	return error;

nano/node/lmdb/lmdb.hpp

@@ -196,6 +196,7 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 	int del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const;
 
 	bool copy_db (boost::filesystem::path const & destination_file) override;
+	void rebuild_db (nano::write_transaction const & transaction_a) override;
 
 	template <typename Key, typename Value>
 	nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const

nano/node/rocksdb/rocksdb.cpp

@@ -603,6 +603,11 @@ bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_p
 	return false;
 }
 
+void nano::rocksdb_store::rebuild_db (nano::write_transaction const & transaction_a)
+{
+	release_assert (false && "Not available for RocksDB");
+}
+
 bool nano::rocksdb_store::init_error () const
 {
 	return error;

nano/node/rocksdb/rocksdb.hpp

@@ -55,6 +55,7 @@ class rocksdb_store : public block_store_partial<rocksdb::Slice, rocksdb_store>
 	}
 
 	bool copy_db (boost::filesystem::path const & destination) override;
+	void rebuild_db (nano::write_transaction const & transaction_a) override;
 
 	template <typename Key, typename Value>
 	nano::store_iterator<Key, Value> make_iterator (nano::transaction const & transaction_a, tables table_a) const

nano/secure/blockstore.hpp

@@ -732,6 +732,7 @@ class block_store
 	virtual std::mutex & get_cache_mutex () = 0;
 
 	virtual bool copy_db (boost::filesystem::path const & destination) = 0;
+	virtual void rebuild_db (nano::write_transaction const & transaction_a) = 0;
 
 	/** Not applicable to all sub-classes */
 	virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) = 0;