DatabaseSource::Auto (#9500)

* implement "auto" database backend in client/db, in progress, #9201 * move fn supports_ref_counting from DatabaseSource enum to Database trait to make it work correctly for all types of dbs * update kvdb_rocksdb to 0.13 and use it's new config feature to properly auto start existing database * tests for auto database reopening * introduce OpenDbError to cleanup opening database error handling and handle case when database is not enabled at the compile time * cargo fmt strings again * cargo fmt strings again * rename DataSettingsSrc to fix test compilation * fix the call to the new kvdb-rocksdb interdace in tests to fix compilation * simplify OpenDbError and make it compile even when paritydb and rocksdb are disabled * cargo fmt * fix compilation without flag with-parity-db * fix unused var compilation warning * support different paths for rocksdb and paritydb in DatabaseSouce::Auto * support "auto" database option in substrate cli * enable Lz4 compression for some of the parity-db colums as per review suggestion * applied review suggestions
paritytech · Aug 9, 2021 · b9d86e1 · b9d86e1
1 parent 91061a7
commit b9d86e1
Show file tree

Hide file tree

Showing 21 changed files with 548 additions and 218 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml
@@ -22,7 +22,7 @@ serde_json = "1.0.41"
 structopt = "0.3"
 derive_more = "0.99.2"
 kvdb = "0.10.0"
-kvdb-rocksdb = "0.12.0"
+kvdb-rocksdb = "0.14.0"
 sp-trie = { version = "4.0.0-dev", path = "../../../primitives/trie" }
 sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" }
 sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" }
@@ -37,7 +37,7 @@ hex = "0.4.0"
 rand = { version = "0.7.2", features = ["small_rng"] }
 lazy_static = "1.4.0"
 parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] }
-parity-db = { version = "0.2.4" }
+parity-db = { version = "0.3" }
 sc-transaction-pool = { version = "4.0.0-dev", path = "../../../client/transaction-pool" }
 sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" }
 futures = { version = "0.3.4", features = ["thread-pool"] }
diff --git a/bin/node/bench/src/tempdb.rs b/bin/node/bench/src/tempdb.rs
@@ -91,8 +91,7 @@ impl TempDatabase {
 		match db_type {
 			DatabaseType::RocksDb => {
 				let db_cfg = DatabaseConfig::with_columns(1);
-				let db = Database::open(&db_cfg, &self.0.path().to_string_lossy())
-					.expect("Database backend error");
+				let db = Database::open(&db_cfg, &self.0.path()).expect("Database backend error");
 				Arc::new(db)
 			},
 			DatabaseType::ParityDb => Arc::new(ParityDbWrapper({
@@ -101,7 +100,7 @@ impl TempDatabase {
 				column_options.ref_counted = true;
 				column_options.preimage = true;
 				column_options.uniform = true;
-				parity_db::Db::open(&options).expect("db open error")
+				parity_db::Db::open_or_create(&options).expect("db open error")
 			})),
 		}
 	}

diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs
@@ -220,10 +220,10 @@ pub enum DatabaseType {
 }
 
 impl DatabaseType {
-	fn into_settings(self, path: PathBuf) -> sc_client_db::DatabaseSettingsSrc {
+	fn into_settings(self, path: PathBuf) -> sc_client_db::DatabaseSource {
 		match self {
-			Self::RocksDb => sc_client_db::DatabaseSettingsSrc::RocksDb { path, cache_size: 512 },
-			Self::ParityDb => sc_client_db::DatabaseSettingsSrc::ParityDb { path },
+			Self::RocksDb => sc_client_db::DatabaseSource::RocksDb { path, cache_size: 512 },
+			Self::ParityDb => sc_client_db::DatabaseSource::ParityDb { path },
 		}
 	}
 }

diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs
@@ -197,6 +197,9 @@ pub enum Database {
 	RocksDb,
 	/// ParityDb. <https://github.com/paritytech/parity-db/>
 	ParityDb,
+	/// Detect whether there is an existing database. Use it, if there is, if not, create new
+	/// instance of paritydb
+	Auto,
 }
 
 impl std::str::FromStr for Database {
@@ -207,6 +210,8 @@ impl std::str::FromStr for Database {
 			Ok(Self::RocksDb)
 		} else if s.eq_ignore_ascii_case("paritydb-experimental") {
 			Ok(Self::ParityDb)
+		} else if s.eq_ignore_ascii_case("auto") {
+			Ok(Self::Auto)
 		} else {
 			Err(format!("Unknown variant `{}`, known variants: {:?}", s, Self::variants()))
 		}
@@ -216,7 +221,7 @@ impl std::str::FromStr for Database {
 impl Database {
 	/// Returns all the variants of this enum to be shown in the cli.
 	pub fn variants() -> &'static [&'static str] {
-		&["rocksdb", "paritydb-experimental"]
+		&["rocksdb", "paritydb-experimental", "auto"]
 	}
 }
 

diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs
@@ -23,7 +23,7 @@ use crate::{
 };
 use log::info;
 use sc_client_api::{BlockBackend, UsageProvider};
-use sc_service::{chain_ops::export_blocks, config::DatabaseConfig};
+use sc_service::{chain_ops::export_blocks, config::DatabaseSource};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 use std::{fmt::Debug, fs, io, path::PathBuf, str::FromStr, sync::Arc};
 use structopt::StructOpt;
@@ -69,14 +69,14 @@ impl ExportBlocksCmd {
 	pub async fn run<B, C>(
 		&self,
 		client: Arc<C>,
-		database_config: DatabaseConfig,
+		database_config: DatabaseSource,
 	) -> error::Result<()>
 	where
 		B: BlockT,
 		C: BlockBackend<B> + UsageProvider<B> + 'static,
 		<<B::Header as HeaderT>::Number as FromStr>::Err: Debug,
 	{
-		if let DatabaseConfig::RocksDb { ref path, .. } = database_config {
+		if let DatabaseSource::RocksDb { ref path, .. } = database_config {
 			info!("DB path: {}", path.display());
 		}
 

diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs
@@ -21,7 +21,7 @@ use crate::{
 	params::{DatabaseParams, SharedParams},
 	CliConfiguration,
 };
-use sc_service::DatabaseConfig;
+use sc_service::DatabaseSource;
 use std::{
 	fmt::Debug,
 	fs,
@@ -47,7 +47,7 @@ pub struct PurgeChainCmd {
 
 impl PurgeChainCmd {
 	/// Run the purge command
-	pub fn run(&self, database_config: DatabaseConfig) -> error::Result<()> {
+	pub fn run(&self, database_config: DatabaseSource) -> error::Result<()> {
 		let db_path = database_config.path().ok_or_else(|| {
 			error::Error::Input("Cannot purge custom database implementation".into())
 		})?;

diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs
@@ -27,7 +27,7 @@ use names::{Generator, Name};
 use sc_client_api::execution_extensions::ExecutionStrategies;
 use sc_service::{
 	config::{
-		BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig,
+		BasePath, Configuration, DatabaseSource, ExtTransport, KeystoreConfig,
 		NetworkConfiguration, NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode,
 		Role, RpcMethods, TaskExecutor, TelemetryEndpoints, TransactionPoolOptions,
 		WasmExecutionMethod,
@@ -220,10 +220,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		base_path: &PathBuf,
 		cache_size: usize,
 		database: Database,
-	) -> Result<DatabaseConfig> {
+	) -> Result<DatabaseSource> {
+		let rocksdb_path = base_path.join("db");
+		let paritydb_path = base_path.join("paritydb");
 		Ok(match database {
-			Database::RocksDb => DatabaseConfig::RocksDb { path: base_path.join("db"), cache_size },
-			Database::ParityDb => DatabaseConfig::ParityDb { path: base_path.join("paritydb") },
+			Database::RocksDb => DatabaseSource::RocksDb { path: rocksdb_path, cache_size },
+			Database::ParityDb => DatabaseSource::ParityDb { path: rocksdb_path },
+			Database::Auto => DatabaseSource::Auto { paritydb_path, rocksdb_path, cache_size },
 		})
 	}
 

diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml
@@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 parking_lot = "0.11.1"
 log = "0.4.8"
 kvdb = "0.10.0"
-kvdb-rocksdb = { version = "0.12.0", optional = true }
+kvdb-rocksdb = { version = "0.14.0", optional = true }
 kvdb-memorydb = "0.10.0"
 linked-hash-map = "0.5.4"
 hash-db = "0.15.2"
@@ -34,15 +34,15 @@ sc-state-db = { version = "0.10.0-dev", path = "../state-db" }
 sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" }
 sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" }
 sp-database = { version = "4.0.0-dev", path = "../../primitives/database" }
-parity-db = { version = "0.2.4", optional = true }
+parity-db = { version = "0.3.1", optional = true }
 prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.9.0", path = "../../utils/prometheus" }
 
 [dev-dependencies]
 sp-keyring = { version = "4.0.0-dev", path = "../../primitives/keyring" }
 sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" }
 substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }
 quickcheck = "1.0.3"
-kvdb-rocksdb = "0.12.0"
+kvdb-rocksdb = "0.14.0"
 tempfile = "3"
 
 [features]