Provide optional automatic ledger/wallet backups before an upgrade

nano/core_test/block_store.cpp

@@ -1652,6 +1652,46 @@ TEST (mdb_block_store, upgrade_v13_v14)
 	ASSERT_EQ (error_node_id, MDB_NOTFOUND);
 }
 
+TEST (mdb_block_store, upgrade_backup)
+{
+	auto dir (nano::unique_path ());
+	namespace fs = boost::filesystem;
+	fs::create_directory (dir);
+	auto path = dir / "data.ldb";
+	/** Returns 'dir' if backup file cannot be found */
+	// clang-format off
+	auto get_backup_path = [&dir]() {
+		for (fs::directory_iterator itr (dir); itr != fs::directory_iterator (); ++itr)
+		{
+			if (itr->path ().filename ().string ().find ("data_backup_") != std::string::npos)
+			{
+				return itr->path ();
+			}
+		}
+		return dir;
+	};
+	// clang-format on
+
+	{
+		nano::logger_mt logger;
+		nano::genesis genesis;
+		auto error (false);
+		nano::mdb_store store (error, logger, path);
+		auto transaction (store.tx_begin_write ());
+		store.version_put (transaction, 14);
+	}
+	ASSERT_EQ (get_backup_path ().string (), dir.string ());
+
+	// Now do the upgrade and confirm that backup is saved
+	nano::logger_mt logger;
+	auto error (false);
+	nano::mdb_store store (error, logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), 128, false, 512, true);
+	ASSERT_FALSE (error);
+	auto transaction (store.tx_begin_read ());
+	ASSERT_LT (14, store.version_get (transaction));
+	ASSERT_NE (get_backup_path ().string (), dir.string ());
+}
+
 // Test various confirmation height values as well as clearing them
 TEST (block_store, confirmation_height)
 {

nano/core_test/node.cpp

@@ -842,6 +842,58 @@ TEST (node_config, v17_values)
 	ASSERT_EQ (config.conf_height_processor_batch_min_time.count (), 500);
 }
 
+TEST (node_config, v17_v18_upgrade)
+{
+	auto path (nano::unique_path ());
+	nano::jsonconfig tree;
+	add_required_children_node_config_tree (tree);
+	tree.put ("version", "17");
+
+	auto upgraded (false);
+	nano::node_config config;
+	config.logging.init (path);
+	// These config options should not be present
+	ASSERT_FALSE (tree.get_optional_child ("backup_before_upgrade"));
+
+	config.deserialize_json (upgraded, tree);
+	// The config options should be added after the upgrade
+	ASSERT_TRUE (!!tree.get_optional_child ("backup_before_upgrade"));
+
+	ASSERT_TRUE (upgraded);
+	auto version (tree.get<std::string> ("version"));
+
+	// Check version is updated
+	ASSERT_GT (std::stoull (version), 17);
+}
+
+TEST (node_config, v18_values)
+{
+	nano::jsonconfig tree;
+	add_required_children_node_config_tree (tree);
+
+	auto path (nano::unique_path ());
+	auto upgraded (false);
+	nano::node_config config;
+	config.logging.init (path);
+
+	// Check config is correct
+	{
+		tree.put ("backup_before_upgrade", true);
+	}
+
+	config.deserialize_json (upgraded, tree);
+	ASSERT_FALSE (upgraded);
+	ASSERT_EQ (config.backup_before_upgrade, true);
+
+	// Check config is correct with other values
+	tree.put ("backup_before_upgrade", false);
+
+	upgraded = false;
+	config.deserialize_json (upgraded, tree);
+	ASSERT_FALSE (upgraded);
+	ASSERT_EQ (config.backup_before_upgrade, false);
+}
+
 // Regression test to ensure that deserializing includes changes node via get_required_child
 TEST (node_config, required_child)
 {

nano/core_test/wallet.cpp

@@ -8,6 +8,7 @@
 #include <fstream>
 
 using namespace std::chrono_literals;
+unsigned constexpr nano::wallet_store::version_current;
 
 TEST (wallet, no_key)
 {
@@ -698,7 +699,7 @@ TEST (wallet, version_1_upgrade)
 	wallet->store.version_put (transaction, 1);
 	wallet->enter_password (transaction, "1");
 	ASSERT_TRUE (wallet->store.valid_password (transaction));
-	ASSERT_EQ (wallet->store.version_current, wallet->store.version (transaction));
+	ASSERT_EQ (nano::wallet_store::version_current, wallet->store.version (transaction));
 	nano::raw_key prv;
 	ASSERT_FALSE (wallet->store.fetch (transaction, key.pub, prv));
 	ASSERT_EQ (key.prv, prv);
@@ -710,7 +711,7 @@ TEST (wallet, version_1_upgrade)
 	wallet->store.version_put (transaction, 1);
 	wallet->enter_password (transaction, "1");
 	ASSERT_TRUE (wallet->store.valid_password (transaction));
-	ASSERT_EQ (wallet->store.version_current, wallet->store.version (transaction));
+	ASSERT_EQ (nano::wallet_store::version_current, wallet->store.version (transaction));
 	nano::raw_key prv2;
 	ASSERT_FALSE (wallet->store.fetch (transaction, key.pub, prv2));
 	ASSERT_EQ (key.prv, prv2);
@@ -821,7 +822,7 @@ TEST (wallet, version_2_upgrade)
 	ASSERT_FALSE (wallet->store.exists (transaction, nano::wallet_store::deterministic_index_special));
 	ASSERT_FALSE (wallet->store.exists (transaction, nano::wallet_store::seed_special));
 	wallet->store.attempt_password (transaction, "1");
-	ASSERT_EQ (wallet->store.version_current, wallet->store.version (transaction));
+	ASSERT_EQ (nano::wallet_store::version_current, wallet->store.version (transaction));
 	ASSERT_TRUE (wallet->store.exists (transaction, nano::wallet_store::deterministic_index_special));
 	ASSERT_TRUE (wallet->store.exists (transaction, nano::wallet_store::seed_special));
 	ASSERT_FALSE (wallet->deterministic_insert (transaction).is_zero ());
@@ -835,7 +836,7 @@ TEST (wallet, version_3_upgrade)
 	wallet->store.rekey (transaction, "1");
 	wallet->enter_password (transaction, "1");
 	ASSERT_TRUE (wallet->store.valid_password (transaction));
-	ASSERT_EQ (wallet->store.version_current, wallet->store.version (transaction));
+	ASSERT_EQ (nano::wallet_store::version_current, wallet->store.version (transaction));
 	nano::keypair key;
 	nano::raw_key seed;
 	nano::uint256_union seed_ciphertext;
@@ -853,7 +854,7 @@ TEST (wallet, version_3_upgrade)
 	wallet->store.version_put (transaction, 3);
 	wallet->enter_password (transaction, "1");
 	ASSERT_TRUE (wallet->store.valid_password (transaction));
-	ASSERT_EQ (wallet->store.version_current, wallet->store.version (transaction));
+	ASSERT_EQ (nano::wallet_store::version_current, wallet->store.version (transaction));
 	nano::raw_key prv;
 	ASSERT_FALSE (wallet->store.fetch (transaction, key.pub, prv));
 	ASSERT_EQ (key.prv, prv);
@@ -863,6 +864,66 @@ TEST (wallet, version_3_upgrade)
 	ASSERT_NE (seed_ciphertext, wallet->store.entry_get_raw (transaction, nano::wallet_store::seed_special).key);
 }
 
+TEST (wallet, upgrade_backup)
+{
+	nano::system system (24000, 1);
+	auto dir (nano::unique_path ());
+	namespace fs = boost::filesystem;
+	fs::create_directory (dir);
+	/** Returns 'dir' if backup file cannot be found */
+	// clang-format off
+	auto get_backup_path = [&dir]() {
+		for (fs::directory_iterator itr (dir); itr != fs::directory_iterator (); ++itr)
+		{
+			if (itr->path ().filename ().string ().find ("wallets_backup_") != std::string::npos)
+			{
+				return itr->path ();
+			}
+		}
+		return dir;
+	};
+	// clang-format on
+
+	nano::keypair id;
+	{
+		nano::node_init init1;
+		auto node1 (std::make_shared<nano::node> (init1, system.io_ctx, 24001, dir, system.alarm, system.logging, system.work));
+		ASSERT_FALSE (init1.error ());
+		auto wallet (node1->wallets.create (id.pub));
+		ASSERT_NE (nullptr, wallet);
+		auto transaction (node1->wallets.tx_begin_write ());
+		wallet->store.version_put (transaction, 3);
+	}
+	ASSERT_EQ (get_backup_path ().string (), dir.string ());
+
+	// Check with config backup_before_upgrade = false
+	{
+		nano::node_init init1;
+		auto node1 (std::make_shared<nano::node> (init1, system.io_ctx, 24001, dir, system.alarm, system.logging, system.work));
+		ASSERT_FALSE (init1.error ());
+		auto wallet (node1->wallets.open (id.pub));
+		ASSERT_NE (nullptr, wallet);
+		auto transaction (node1->wallets.tx_begin_write ());
+		ASSERT_LT (3, wallet->store.version (transaction));
+		wallet->store.version_put (transaction, 3);
+	}
+	ASSERT_EQ (get_backup_path ().string (), dir.string ());
+
+	// Now do the upgrade and confirm that backup is saved
+	{
+		nano::node_config node_config (24001, system.logging);
+		node_config.backup_before_upgrade = true;
+		nano::node_init init1;
+		auto node1 (std::make_shared<nano::node> (init1, system.io_ctx, dir, system.alarm, node_config, system.work));
+		ASSERT_FALSE (init1.error ());
+		auto wallet (node1->wallets.open (id.pub));
+		ASSERT_NE (nullptr, wallet);
+		auto transaction (node1->wallets.tx_begin_read ());
+		ASSERT_LT (3, wallet->store.version (transaction));
+	}
+	ASSERT_NE (get_backup_path ().string (), dir.string ());
+}
+
 TEST (wallet, no_work)
 {
 	nano::system system (24000, 1);

nano/node/lmdb/lmdb.cpp

@@ -39,7 +39,7 @@ void mdb_val::convert_buffer_to_value ()
 }
 }
 
-nano::mdb_store::mdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size) :
+nano::mdb_store::mdb_store (bool & error_a, nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, bool drop_unchecked, size_t const batch_size, bool backup_before_upgrade) :
 logger (logger_a),
 env (error_a, path_a, lmdb_max_dbs, true),
 mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a),
@@ -60,9 +60,13 @@ txn_tracking_enabled (txn_tracking_config_a.enable)
 
 		// Only open a write lock when upgrades are needed. This is because CLI commands
 		// open inactive nodes which can otherwise be locked here if there is a long write
-		// (can be a few minutes with the --fastbootstrap flag for instance)
+		// (can be a few minutes with the --fast_bootstrap flag for instance)
 		if (!is_fully_upgraded)
 		{
+			if (backup_before_upgrade)
+			{
+				create_backup_file (env, path_a, logger_a);
+			}
 			auto transaction (tx_begin_write ());
 			open_databases (error_a, transaction, MDB_CREATE);
 			if (!error_a)
@@ -496,6 +500,37 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::transaction const & transaction_
 	}
 }
 
+/** Takes a filepath, appends '_backup_<timestamp>' to the end (but before any extension) and saves that file in the same directory */
+void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a)
+{
+	auto extension = filepath_a.extension ();
+	auto filename_without_extension = filepath_a.filename ().replace_extension ("");
+	auto orig_filepath = filepath_a;
+	auto & backup_path = orig_filepath.remove_filename ();
+	auto backup_filename = filename_without_extension;
+	backup_filename += "_backup_";
+	backup_filename += std::to_string (std::chrono::system_clock::now ().time_since_epoch ().count ());
+	backup_filename += extension;
+	auto backup_filepath = backup_path / backup_filename;
+	auto start_message (boost::str (boost::format ("Performing %1% backup before database upgrade...") % filepath_a.filename ()));
+	logger_a.always_log (start_message);
+	std::cout << start_message << std::endl;
+	auto error (mdb_env_copy (env_a, backup_filepath.string ().c_str ()));
+	if (error)
+	{
+		auto error_message (boost::str (boost::format ("%1% backup failed") % filepath_a.filename ()));
+		logger_a.always_log (error_message);
+		std::cerr << error_message << std::endl;
+		std::exit (1);
+	}
+	else
+	{
+		auto success_message (boost::str (boost::format ("Backup created: %1%") % backup_filename));
+		logger_a.always_log (success_message);
+		std::cout << success_message << std::endl;
+	}
+}
+
 void nano::mdb_store::version_put (nano::transaction const & transaction_a, int version_a)
 {
 	nano::uint256_union version_key (1);

nano/node/lmdb/lmdb.hpp

@@ -32,7 +32,7 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 	using block_store_partial::block_exists;
 	using block_store_partial::unchecked_put;
 
-	mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512);
+	mdb_store (bool &, nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, bool drop_unchecked = false, size_t batch_size = 512, bool backup_before_upgrade = false);
 	nano::write_transaction tx_begin_write () override;
 	nano::read_transaction tx_begin_read () override;
 
@@ -42,6 +42,8 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 
 	void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override;
 
+	static void create_backup_file (nano::mdb_env &, boost::filesystem::path const &, nano::logger_mt &);
+
 	nano::logger_mt & logger;
 
 	nano::mdb_env env;

nano/node/node.cpp

@@ -123,7 +123,7 @@ flags (flags_a),
 alarm (alarm_a),
 work (work_a),
 logger (config_a.logging.min_time_between_log_output),
-store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, logger, application_path_a / "data.ldb", config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size)),
+store_impl (std::make_unique<nano::mdb_store> (init_a.block_store_init, logger, application_path_a / "data.ldb", config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, !flags.disable_unchecked_drop, flags.sideband_batch_size, config_a.backup_before_upgrade)),
 store (*store_impl),
 wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (init_a.wallets_store_init, application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)),
 wallets_store (*wallets_store_impl),

nano/node/nodeconfig.cpp

@@ -134,6 +134,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const
 	json.put ("confirmation_history_size", confirmation_history_size);
 	json.put ("active_elections_size", active_elections_size);
 	json.put ("bandwidth_limit", bandwidth_limit);
+	json.put ("backup_before_upgrade", backup_before_upgrade);
 
 	return json.get_error ();
 }
@@ -258,6 +259,10 @@ bool nano::node_config::upgrade_json (unsigned version_a, nano::jsonconfig & jso
 			json.put ("conf_height_processor_batch_min_time", conf_height_processor_batch_min_time.count ());
 		}
 		case 17:
+		{
+			json.put ("backup_before_upgrade", backup_before_upgrade);
+		}
+		case 18:
 			break;
 		default:
 			throw std::runtime_error ("Unknown node_config version");
@@ -409,6 +414,7 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco
 		json.get<size_t> ("confirmation_history_size", confirmation_history_size);
 		json.get<size_t> ("active_elections_size", active_elections_size);
 		json.get<size_t> ("bandwidth_limit", bandwidth_limit);
+		json.get<bool> ("backup_before_upgrade", backup_before_upgrade);
 
 		auto conf_height_processor_batch_min_time_l (conf_height_processor_batch_min_time.count ());
 		json.get ("conf_height_processor_batch_min_time", conf_height_processor_batch_min_time_l);

nano/node/nodeconfig.hpp

@@ -77,9 +77,10 @@ class node_config
 	static std::chrono::minutes constexpr wallet_backup_interval = std::chrono::minutes (5);
 	size_t bandwidth_limit{ 5 * 1024 * 1024 }; // 5Mb/s
 	std::chrono::milliseconds conf_height_processor_batch_min_time{ 50 };
+	bool backup_before_upgrade{ false };
 	static unsigned json_version ()
 	{
-		return 17;
+		return 18;
 	}
 };
 

nano/node/wallet.cpp

@@ -1607,6 +1607,27 @@ thread ([this]() {
 			}
 		}
 	}
+	// Backup before upgrade wallets
+	bool backup_required (false);
+	if (node.config.backup_before_upgrade)
+	{
+		auto transaction (tx_begin_read ());
+		for (auto & item : items)
+		{
+			if (item.second->store.version (transaction) != nano::wallet_store::version_current)
+			{
+				backup_required = true;
+				break;
+			}
+		}
+	}
+	if (backup_required)
+	{
+		const char * store_path;
+		mdb_env_get_path (env, &store_path);
+		const boost::filesystem::path path (store_path);
+		nano::mdb_store::create_backup_file (env, path, node_a.logger);
+	}
 	for (auto & item : items)
 	{
 		item.second->enter_initial_password ();

nano/node/wallet.hpp

@@ -98,7 +98,7 @@ class wallet_store final
 	static unsigned const version_2 = 2;
 	static unsigned const version_3 = 3;
 	static unsigned const version_4 = 4;
-	unsigned const version_current = version_4;
+	static unsigned constexpr version_current = version_4;
 	static nano::uint256_union const version_special;
 	static nano::uint256_union const wallet_key_special;
 	static nano::uint256_union const salt_special;