From 1585ede5c7e22d7bcbeab2a85dc2ac50cb953230 Mon Sep 17 00:00:00 2001 From: cryptocode Date: Sun, 23 Feb 2020 17:55:33 +0100 Subject: [PATCH] LMDB sync options and new config settings --- nano/core_test/block_store.cpp | 3 +- nano/core_test/node.cpp | 6 +-- nano/core_test/toml.cpp | 18 ++++++++- nano/core_test/wallets.cpp | 2 +- nano/lib/CMakeLists.txt | 2 + nano/lib/lmdbconfig.cpp | 72 ++++++++++++++++++++++++++++++++++ nano/lib/lmdbconfig.hpp | 50 +++++++++++++++++++++++ nano/node/lmdb/lmdb.cpp | 16 +++++--- nano/node/lmdb/lmdb.hpp | 5 ++- nano/node/lmdb/lmdb_env.cpp | 29 ++++++++++---- nano/node/lmdb/lmdb_env.hpp | 40 ++++++++++++++++++- nano/node/node.cpp | 8 ++-- nano/node/nodeconfig.cpp | 35 ++++++++++++++--- nano/node/nodeconfig.hpp | 4 +- nano/node/wallet.cpp | 4 +- nano/node/wallet.hpp | 3 +- nano/secure/blockstore.hpp | 3 +- 17 files changed, 262 insertions(+), 38 deletions(-) create mode 100644 nano/lib/lmdbconfig.cpp create mode 100644 nano/lib/lmdbconfig.hpp diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 7cf0596a69..f8d6fdccf6 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -2090,7 +2091,7 @@ TEST (mdb_block_store, upgrade_backup) // Now do the upgrade and confirm that backup is saved nano::logger_mt logger; - nano::mdb_store store (logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), 128, 512, true); + nano::mdb_store store (logger, path, nano::txn_tracking_config{}, std::chrono::seconds (5), nano::lmdb_config{}, 512, true); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_read ()); ASSERT_LT (14, store.version_get (transaction)); diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 9f5a53b442..fcf03034e9 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -555,7 +555,7 @@ TEST (node_config, serialization) config1.callback_address = "test"; config1.callback_port = 10; config1.callback_target = "test"; - config1.lmdb_max_dbs = 256; + config1.deprecated_lmdb_max_dbs = 256; nano::jsonconfig tree; config1.serialize_json (tree); nano::logging logging2; @@ -572,7 +572,7 @@ TEST (node_config, serialization) ASSERT_NE (config2.callback_address, config1.callback_address); ASSERT_NE (config2.callback_port, config1.callback_port); ASSERT_NE (config2.callback_target, config1.callback_target); - ASSERT_NE (config2.lmdb_max_dbs, config1.lmdb_max_dbs); + ASSERT_NE (config2.deprecated_lmdb_max_dbs, config1.deprecated_lmdb_max_dbs); ASSERT_FALSE (tree.get_optional ("epoch_block_link")); ASSERT_FALSE (tree.get_optional ("epoch_block_signer")); @@ -590,7 +590,7 @@ TEST (node_config, serialization) ASSERT_EQ (config2.callback_address, config1.callback_address); ASSERT_EQ (config2.callback_port, config1.callback_port); ASSERT_EQ (config2.callback_target, config1.callback_target); - ASSERT_EQ (config2.lmdb_max_dbs, config1.lmdb_max_dbs); + ASSERT_EQ (config2.deprecated_lmdb_max_dbs, config1.deprecated_lmdb_max_dbs); } TEST (node_config, v1_v2_upgrade) diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index c615379467..150efbb481 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -120,6 +120,7 @@ TEST (toml, daemon_config_deserialize_defaults) [node.statistics.log] [node.statistics.sampling] [node.websocket] + [node.lmdb] [node.rocksdb] [opencl] [rpc] @@ -157,7 +158,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.external_address, defaults.node.external_address); ASSERT_EQ (conf.node.external_port, defaults.node.external_port); ASSERT_EQ (conf.node.io_threads, defaults.node.io_threads); - ASSERT_EQ (conf.node.lmdb_max_dbs, defaults.node.lmdb_max_dbs); + ASSERT_EQ (conf.node.deprecated_lmdb_max_dbs, defaults.node.deprecated_lmdb_max_dbs); ASSERT_EQ (conf.node.max_work_generate_multiplier, defaults.node.max_work_generate_multiplier); ASSERT_EQ (conf.node.network_threads, defaults.node.network_threads); ASSERT_EQ (conf.node.secondary_work_peers, defaults.node.secondary_work_peers); @@ -244,6 +245,10 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.stat_config.log_counters_filename, defaults.node.stat_config.log_counters_filename); ASSERT_EQ (conf.node.stat_config.log_samples_filename, defaults.node.stat_config.log_samples_filename); + ASSERT_EQ (conf.node.lmdb_config.sync, defaults.node.lmdb_config.sync); + ASSERT_EQ (conf.node.lmdb_config.max_databases, defaults.node.lmdb_config.max_databases); + ASSERT_EQ (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size); + ASSERT_EQ (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable); ASSERT_EQ (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits); ASSERT_EQ (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache); @@ -493,6 +498,11 @@ TEST (toml, daemon_config_deserialize_no_defaults) enable = true port = 999 + [node.lmdb] + sync = "nosync_safe" + max_databases = 999 + map_size = 999 + [node.rocksdb] enable = true bloom_filter_bits = 10 @@ -554,7 +564,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.external_address, defaults.node.external_address); ASSERT_NE (conf.node.external_port, defaults.node.external_port); ASSERT_NE (conf.node.io_threads, defaults.node.io_threads); - ASSERT_NE (conf.node.lmdb_max_dbs, defaults.node.lmdb_max_dbs); + ASSERT_NE (conf.node.deprecated_lmdb_max_dbs, defaults.node.deprecated_lmdb_max_dbs); ASSERT_NE (conf.node.max_work_generate_multiplier, defaults.node.max_work_generate_multiplier); ASSERT_NE (conf.node.frontiers_confirmation, defaults.node.frontiers_confirmation); ASSERT_NE (conf.node.network_threads, defaults.node.network_threads); @@ -642,6 +652,10 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.stat_config.log_counters_filename, defaults.node.stat_config.log_counters_filename); ASSERT_NE (conf.node.stat_config.log_samples_filename, defaults.node.stat_config.log_samples_filename); + ASSERT_NE (conf.node.lmdb_config.sync, defaults.node.lmdb_config.sync); + ASSERT_NE (conf.node.lmdb_config.max_databases, defaults.node.lmdb_config.max_databases); + ASSERT_NE (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size); + ASSERT_NE (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable); ASSERT_NE (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits); ASSERT_NE (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache); diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index a3a851c1e1..7e4b10f858 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -134,7 +134,7 @@ TEST (wallets, DISABLED_wallet_create_max) bool error (false); nano::wallets wallets (error, *system.nodes[0]); const int nonWalletDbs = 19; - for (int i = 0; i < system.nodes[0]->config.lmdb_max_dbs - nonWalletDbs; i++) + for (int i = 0; i < system.nodes[0]->config.deprecated_lmdb_max_dbs - nonWalletDbs; i++) { auto wallet_id = nano::random_wallet_id (); auto wallet = wallets.create (wallet_id); diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index ec94f81028..61979f60e8 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -34,6 +34,8 @@ add_library (nano_lib json_error_response.hpp jsonconfig.hpp jsonconfig.cpp + lmdbconfig.hpp + lmdbconfig.cpp locks.hpp locks.cpp logger_mt.hpp diff --git a/nano/lib/lmdbconfig.cpp b/nano/lib/lmdbconfig.cpp new file mode 100644 index 0000000000..664ddabfe7 --- /dev/null +++ b/nano/lib/lmdbconfig.cpp @@ -0,0 +1,72 @@ +#include +#include +#include + +#include + +nano::error nano::lmdb_config::serialize_toml (nano::tomlconfig & toml) const +{ + std::string sync_string; + switch (sync) + { + case nano::lmdb_config::sync_strategy::always: + sync_string = "always"; + break; + case nano::lmdb_config::sync_strategy::nosync_safe: + sync_string = "nosync_safe"; + break; + case nano::lmdb_config::sync_strategy::nosync_unsafe: + sync_string = "nosync_unsafe"; + break; + case nano::lmdb_config::sync_strategy::nosync_unsafe_large_memory: + sync_string = "nosync_unsafe_large_memory"; + break; + } + + toml.put ("sync", sync_string, "Sync strategy for flushing commits to the ledger database. This does not affect the wallet database.\ntype:string,{always, nosync_safe, nosync_unsafe, nosync_unsafe_large_memory}"); + toml.put ("max_databases", max_databases, "Maximum open lmdb databases. Increase default if more than 100 wallets is required.\nNote: external management is recommended when a large amounts of wallets are required (see https://docs.nano.org/integration-guides/key-management/).\ntype:uin32"); + toml.put ("map_size", map_size, "Maximum ledger database map size in bytes.\ntype:uint64"); + return toml.get_error (); +} + +nano::error nano::lmdb_config::deserialize_toml (nano::tomlconfig & toml, bool is_deprecated_lmdb_dbs_used) +{ + static nano::network_params params; + auto default_max_databases = max_databases; + toml.get_optional ("max_databases", max_databases); + toml.get_optional ("map_size", map_size); + + // For now we accept either setting, but not both + if (!params.network.is_test_network () && is_deprecated_lmdb_dbs_used && default_max_databases != max_databases) + { + toml.get_error ().set ("Both the deprecated node.lmdb_max_dbs and the new node.lmdb.max_databases setting are used. Please use max_databases only."); + } + + if (!toml.get_error ()) + { + std::string sync_string = "always"; + toml.get_optional ("sync", sync_string); + if (sync_string == "always") + { + sync = nano::lmdb_config::sync_strategy::always; + } + else if (sync_string == "nosync_safe") + { + sync = nano::lmdb_config::sync_strategy::nosync_safe; + } + else if (sync_string == "nosync_unsafe") + { + sync = nano::lmdb_config::sync_strategy::nosync_unsafe; + } + else if (sync_string == "nosync_unsafe_large_memory") + { + sync = nano::lmdb_config::sync_strategy::nosync_unsafe_large_memory; + } + else + { + toml.get_error ().set (sync_string + " is not a valid sync option"); + } + } + + return toml.get_error (); +} diff --git a/nano/lib/lmdbconfig.hpp b/nano/lib/lmdbconfig.hpp new file mode 100644 index 0000000000..342c03d126 --- /dev/null +++ b/nano/lib/lmdbconfig.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include + +namespace nano +{ +class tomlconfig; + +/** Configuration options for LMDB */ +class lmdb_config final +{ +public: + /** + * Dictates how lmdb flushes to disk on commit. + * These options only apply to the ledger database; the wallet database + * always flush. + */ + enum sync_strategy + { + /** Always flush to disk on commit. This is default. */ + always, + + /** Do not flush meta data eagerly. This may cause loss of transactions, but maintains integrity. */ + nosync_safe, + + /** + * Let the OS decide when to flush to disk. On filesystems with write ordering, this has the same + * guarantees as nosync_safe, otherwise corruption may occur on system crash. + */ + nosync_unsafe, + /** + * Use a writeable memory map. Let the OS decide when to flush to disk, and make the request asynchronous. + * This may give better performance on systems where the database fits entirely in memory, otherwise is + * may be slower. + * @warning Do not use this option if external processes uses the database concurrently. + */ + nosync_unsafe_large_memory + }; + + nano::error serialize_toml (nano::tomlconfig & toml_a) const; + nano::error deserialize_toml (nano::tomlconfig & toml_a, bool is_deprecated_lmdb_dbs_used); + + /** Sync strategy for the ledger database */ + sync_strategy sync{ always }; + uint32_t max_databases{ 128 }; + size_t map_size{ 128ULL * 1024 * 1024 * 1024 }; +}; +} diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index 2eb700b249..b21da057d4 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -40,9 +40,9 @@ void mdb_val::convert_buffer_to_value () } } -nano::mdb_store::mdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, size_t const batch_size, bool backup_before_upgrade) : +nano::mdb_store::mdb_store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, size_t const batch_size, bool backup_before_upgrade) : logger (logger_a), -env (error, path_a, lmdb_max_dbs, true), +env (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), txn_tracking_enabled (txn_tracking_config_a.enable) { @@ -91,7 +91,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable) if (needs_vacuuming && !network_constants.is_test_network ()) { logger.always_log ("Preparing vacuum..."); - auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_max_dbs); + auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_config_a); logger.always_log (vacuum_success ? "Vacuum succeeded." : "Failed to vacuum. (Optional) Ensure enough disk space is available for a copy of the database and try to vacuum after shutting down the node"); } } @@ -103,7 +103,7 @@ txn_tracking_enabled (txn_tracking_config_a.enable) } } -bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path_a, int lmdb_max_dbs) +bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a) { // Vacuum the database. This is not a required step and may actually fail if there isn't enough storage space. auto vacuum_path = path_a.parent_path () / "vacuumed.ldb"; @@ -112,6 +112,7 @@ bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path if (vacuum_success) { // Need to close the database to release the file handle + mdb_env_sync (env.environment, true); mdb_env_close (env.environment); env.environment = nullptr; @@ -119,7 +120,10 @@ bool nano::mdb_store::vacuum_after_upgrade (boost::filesystem::path const & path boost::filesystem::rename (vacuum_path, path_a); // Set up the environment again - env.init (error, path_a, lmdb_max_dbs, true); + auto options = nano::mdb_env::options::make () + .set_config (lmdb_config_a) + .set_use_no_mem_init (true); + env.init (error, path_a, options); if (!error) { auto transaction (tx_begin_read ()); @@ -1305,4 +1309,4 @@ bool nano::mdb_store::upgrade_counters::are_equal () const } // Explicitly instantiate -template class nano::block_store_partial; \ No newline at end of file +template class nano::block_store_partial; diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index a0cc27f845..e46ebdf203 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -36,7 +37,7 @@ class mdb_store : public block_store_partial using block_store_partial::block_exists; using block_store_partial::unchecked_put; - mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, size_t batch_size = 512, bool backup_before_upgrade = false); + mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, size_t batch_size = 512, bool backup_before_upgrade = false); nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; nano::read_transaction tx_begin_read () override; @@ -261,7 +262,7 @@ class mdb_store : public block_store_partial size_t count (nano::transaction const & transaction_a, tables table_a) const override; - bool vacuum_after_upgrade (boost::filesystem::path const & path_a, int lmdb_max_dbs); + bool vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a); class upgrade_counters { diff --git a/nano/node/lmdb/lmdb_env.cpp b/nano/node/lmdb/lmdb_env.cpp index 61bc9f37f8..2c474ac200 100644 --- a/nano/node/lmdb/lmdb_env.cpp +++ b/nano/node/lmdb/lmdb_env.cpp @@ -2,12 +2,12 @@ #include -nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a) +nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, nano::mdb_env::options options_a) { - init (error_a, path_a, max_dbs_a, use_no_mem_init_a, map_size_a); + init (error_a, path_a, options_a); } -void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a, int max_dbs_a, bool use_no_mem_init_a, size_t map_size_a) +void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a, nano::mdb_env::options options_a) { boost::system::error_code error_mkdir, error_chmod; if (path_a.has_parent_path ()) @@ -18,11 +18,11 @@ void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a { auto status1 (mdb_env_create (&environment)); release_assert (status1 == 0); - auto status2 (mdb_env_set_maxdbs (environment, max_dbs_a)); + auto status2 (mdb_env_set_maxdbs (environment, options_a.config.max_databases)); release_assert (status2 == 0); - auto map_size = map_size_a; + auto map_size = options_a.config.map_size; auto max_valgrind_map_size = 16 * 1024 * 1024; - if (running_within_valgrind () && map_size_a > max_valgrind_map_size) + if (running_within_valgrind () && map_size > max_valgrind_map_size) { // In order to run LMDB under Valgrind, the maximum map size must be smaller than half your available RAM map_size = max_valgrind_map_size; @@ -34,7 +34,20 @@ void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a // MDB_NORDAHEAD will allow platforms that support it to load the DB in memory as needed. // MDB_NOMEMINIT prevents zeroing malloc'ed pages. Can provide improvement for non-sensitive data but may make memory checkers noisy (e.g valgrind). auto environment_flags = MDB_NOSUBDIR | MDB_NOTLS | MDB_NORDAHEAD; - if (!running_within_valgrind () && use_no_mem_init_a) + if (options_a.config.sync == nano::lmdb_config::sync_strategy::nosync_safe) + { + environment_flags |= MDB_NOMETASYNC; + } + else if (options_a.config.sync == nano::lmdb_config::sync_strategy::nosync_unsafe) + { + environment_flags |= MDB_NOSYNC; + } + else if (options_a.config.sync == nano::lmdb_config::sync_strategy::nosync_unsafe_large_memory) + { + environment_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + } + + if (!running_within_valgrind () && options_a.use_no_mem_init) { environment_flags |= MDB_NOMEMINIT; } @@ -69,6 +82,8 @@ nano::mdb_env::~mdb_env () { if (environment != nullptr) { + // Make sure the commits are flushed. This is a no-op unless MDB_NOSYNC is used. + mdb_env_sync (environment, true); mdb_env_close (environment); } } diff --git a/nano/node/lmdb/lmdb_env.hpp b/nano/node/lmdb/lmdb_env.hpp index 6bd3560c6d..05ff558785 100644 --- a/nano/node/lmdb/lmdb_env.hpp +++ b/nano/node/lmdb/lmdb_env.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -11,8 +12,43 @@ namespace nano class mdb_env final { public: - mdb_env (bool &, boost::filesystem::path const &, int max_dbs = 128, bool use_no_mem_init = false, size_t map_size = 128ULL * 1024 * 1024 * 1024); - void init (bool &, boost::filesystem::path const &, int max_dbs, bool use_no_mem_init, size_t map_size = 128ULL * 1024 * 1024 * 1024); + /** Environment options, most of which originates from the config file. */ + class options final + { + friend class mdb_env; + + public: + static options make () + { + return options (); + } + + options & set_config (nano::lmdb_config config_a) + { + config = config_a; + return *this; + } + + options & set_use_no_mem_init (int use_no_mem_init_a) + { + use_no_mem_init = use_no_mem_init_a; + return *this; + } + + /** Used by the wallet to override the config map size */ + options & override_config_map_size (size_t map_size_a) + { + config.map_size = map_size_a; + return *this; + } + + private: + bool use_no_mem_init{ false }; + nano::lmdb_config config; + }; + + mdb_env (bool &, boost::filesystem::path const &, nano::mdb_env::options options_a = nano::mdb_env::options::make ()); + void init (bool &, boost::filesystem::path const &, nano::mdb_env::options options_a = nano::mdb_env::options::make ()); ~mdb_env (); operator MDB_env * () const; nano::read_transaction tx_begin_read (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 636413bb3b..025a1e4309 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -118,9 +118,9 @@ alarm (alarm_a), work (work_a), distributed_work (*this), logger (config_a.logging.min_time_between_log_output), -store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_max_dbs, flags.sideband_batch_size, config_a.backup_before_upgrade, config_a.rocksdb_config.enable)), +store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, flags.sideband_batch_size, config_a.backup_before_upgrade, config_a.rocksdb_config.enable)), store (*store_impl), -wallets_store_impl (std::make_unique (application_path_a / "wallets.ldb", config_a.lmdb_max_dbs)), +wallets_store_impl (std::make_unique (application_path_a / "wallets.ldb", config_a.lmdb_config)), wallets_store (*wallets_store_impl), gap_cache (*this), ledger (store, stats, flags_a.generate_cache), @@ -1369,7 +1369,7 @@ nano::node_flags const & nano::inactive_node_flag_defaults () return node_flags; } -std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, int lmdb_max_dbs, size_t batch_size, bool backup_before_upgrade, bool use_rocksdb_backend) +std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, size_t batch_size, bool backup_before_upgrade, bool use_rocksdb_backend) { #if NANO_ROCKSDB auto make_rocksdb = [&logger, add_db_postfix, &path, &rocksdb_config, read_only]() { @@ -1400,5 +1400,5 @@ std::unique_ptr nano::make_store (nano::logger_mt & logger, b #endif } - return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_max_dbs, batch_size, backup_before_upgrade); + return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, batch_size, backup_before_upgrade); } diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index 45fb429557..eeb894d3a4 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -80,7 +79,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const toml.put ("enable_voting", enable_voting, "Enable or disable voting. Enabling this option requires additional system resources, namely increased CPU, bandwidth and disk usage.\ntype:bool"); toml.put ("bootstrap_connections", bootstrap_connections, "Number of outbound bootstrap connections. Must be a power of 2. Defaults to 4.\nWarning: a larger amount of connections may use substantially more system memory.\ntype:uint64"); toml.put ("bootstrap_connections_max", bootstrap_connections_max, "Maximum number of inbound bootstrap connections. Defaults to 64.\nWarning: a larger amount of connections may use additional system memory.\ntype:uint64"); - toml.put ("lmdb_max_dbs", lmdb_max_dbs, "Maximum open lmdb databases. Increase default if more than 100 wallets is required.\nNote: external management is recommended when a large amounts of wallets are required (see https://docs.nano.org/integration-guides/key-management/).\ntype:uint64"); + toml.put ("lmdb_max_dbs", deprecated_lmdb_max_dbs, "DEPRECATED: use node.lmdb.max_databases instead.\nMaximum open lmdb databases. Increase default if more than 100 wallets is required.\nNote: external management is recommended when a large amounts of wallets are required (see https://docs.nano.org/integration-guides/key-management/).\ntype:uint64"); toml.put ("block_processor_batch_max_time", block_processor_batch_max_time.count (), "The maximum time the block processor can continously process blocks for.\ntype:milliseconds"); toml.put ("allow_local_peers", allow_local_peers, "Enable or disable local host peering.\ntype:bool"); toml.put ("vote_minimum", vote_minimum.to_string_dec (), "Local representatives do not vote if the delegated weight is under this threshold. Saves on system resources.\ntype:string,amount,raw"); @@ -160,6 +159,10 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const rocksdb_config.serialize_toml (rocksdb_l); toml.put_child ("rocksdb", rocksdb_l); + nano::tomlconfig lmdb_l; + lmdb_config.serialize_toml (lmdb_l); + toml.put_child ("lmdb", lmdb_l); + return toml.get_error (); } @@ -302,11 +305,33 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) toml.get ("network_threads", network_threads); toml.get ("bootstrap_connections", bootstrap_connections); toml.get ("bootstrap_connections_max", bootstrap_connections_max); - toml.get ("lmdb_max_dbs", lmdb_max_dbs); toml.get ("enable_voting", enable_voting); toml.get ("allow_local_peers", allow_local_peers); toml.get (signature_checker_threads_key, signature_checker_threads); + auto lmdb_max_dbs_default = deprecated_lmdb_max_dbs; + toml.get ("lmdb_max_dbs", deprecated_lmdb_max_dbs); + bool is_deprecated_lmdb_dbs_used = lmdb_max_dbs_default != deprecated_lmdb_max_dbs; + + // Note: using the deprecated setting will result in a fail-fast config error in the future + if (!network_params.network.is_test_network () && is_deprecated_lmdb_dbs_used) + { + std::cerr << "WARNING: The node.lmdb_max_dbs setting is deprecated and will be removed in a future version." << std::endl; + std::cerr << "Please use the node.lmdb.max_databases setting instead." << std::endl; + } + + if (toml.has_key ("lmdb")) + { + auto lmdb_config_l (toml.get_required_child ("lmdb")); + lmdb_config.deserialize_toml (lmdb_config_l, is_deprecated_lmdb_dbs_used); + + // Note that the lmdb config fails is both the deprecated and new setting are changed. + if (is_deprecated_lmdb_dbs_used) + { + lmdb_config.max_databases = deprecated_lmdb_max_dbs; + } + } + boost::asio::ip::address_v6 external_address_l; toml.get ("external_address", external_address_l); external_address = external_address_l.to_string (); @@ -444,7 +469,7 @@ nano::error nano::node_config::serialize_json (nano::jsonconfig & json) const json.put ("callback_address", callback_address); json.put ("callback_port", callback_port); json.put ("callback_target", callback_target); - json.put ("lmdb_max_dbs", lmdb_max_dbs); + json.put ("lmdb_max_dbs", deprecated_lmdb_max_dbs); json.put ("block_processor_batch_max_time", block_processor_batch_max_time.count ()); json.put ("allow_local_peers", allow_local_peers); json.put ("vote_minimum", vote_minimum.to_string_dec ()); @@ -738,7 +763,7 @@ nano::error nano::node_config::deserialize_json (bool & upgraded_a, nano::jsonco json.get ("callback_address", callback_address); json.get ("callback_port", callback_port); json.get ("callback_target", callback_target); - json.get ("lmdb_max_dbs", lmdb_max_dbs); + json.get ("lmdb_max_dbs", deprecated_lmdb_max_dbs); json.get ("enable_voting", enable_voting); json.get ("allow_local_peers", allow_local_peers); json.get (signature_checker_threads_key, signature_checker_threads); diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index fdd497725e..f085b32f6a 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -69,7 +70,7 @@ class node_config std::string callback_address; uint16_t callback_port{ 0 }; std::string callback_target; - int lmdb_max_dbs{ 128 }; + int deprecated_lmdb_max_dbs{ 128 }; bool allow_local_peers{ !network_params.network.is_live_network () }; // disable by default for live network nano::stat_config stat_config; nano::ipc::ipc_config ipc_config; @@ -95,6 +96,7 @@ class node_config uint64_t max_work_generate_difficulty{ nano::network_constants::publish_full_threshold }; uint32_t max_queued_requests{ 512 }; nano::rocksdb_config rocksdb_config; + nano::lmdb_config lmdb_config; nano::frontiers_confirmation_mode frontiers_confirmation{ nano::frontiers_confirmation_mode::automatic }; std::string serialize_frontiers_confirmation (nano::frontiers_confirmation_mode) const; nano::frontiers_confirmation_mode deserialize_frontiers_confirmation (std::string const &); diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 3fb2842aa5..2408579204 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -1989,8 +1989,8 @@ nano::store_iterator nano::wallet_store::end { return nano::store_iterator (nullptr); } -nano::mdb_wallets_store::mdb_wallets_store (boost::filesystem::path const & path_a, int lmdb_max_dbs) : -environment (error, path_a, lmdb_max_dbs, false, 1ULL * 1024 * 1024 * 1024) +nano::mdb_wallets_store::mdb_wallets_store (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a) : +environment (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).override_config_map_size (1ULL * 1024 * 1024 * 1024)) { } diff --git a/nano/node/wallet.hpp b/nano/node/wallet.hpp index c447c86056..e9d828450c 100644 --- a/nano/node/wallet.hpp +++ b/nano/node/wallet.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -257,7 +258,7 @@ class wallets_store class mdb_wallets_store final : public wallets_store { public: - mdb_wallets_store (boost::filesystem::path const &, int lmdb_max_dbs = 128); + mdb_wallets_store (boost::filesystem::path const &, nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}); nano::mdb_env environment; bool init_error () const override; bool error{ false }; diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 2c5f8a3573..0e42dd217c 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -828,7 +829,7 @@ class block_store virtual std::string vendor_get () const = 0; }; -std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), int lmdb_max_dbs = 128, size_t batch_size = 512, bool backup_before_upgrade = false, bool rocksdb_backend = false); +std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool open_read_only = false, bool add_db_postfix = false, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, size_t batch_size = 512, bool backup_before_upgrade = false, bool rocksdb_backend = false); } namespace std