Bump to polkadot-v0.9.40 as a base for substrate related deps (#848)

* Use locked/polkadot-v0.9.40 substrate and frontier deps

* [substrate-apply] Move grandpa crates to consensus folder #13458

* Use suitable evm version

* Fix naming

* [frontier-apply] EVM + Weight v2 support #1039

* [substrate-apply] Deprecate Weight::from_{ref_time, proof_size} #13475

* [substrate-apply] Offences report system rework #13425

* Add missing GasLimitPovSizeRatio for pallet_evm::Config

* [frontier-apply] Add pending support for eth_getBlockByNumber #1048

* Fix is_precompile return value

* Add missing args for weight limit and transaction len at runner call and created methods

* Add missing trait OnFinalize

* Use SyncingService for Eth

* [frontier-apply] sql backend w/ receipt log indexing #883 (part 1)

* [frontier-apply] Add txpool RPC and impl new_pending_transaction_filter #1073

* [substrate-apply] Extract syncing protocol from sc-network #12828

* sql backend w/ receipt log indexing #883 (part 2)

* [frontier-apply] sql backend w/ receipt log indexing #883 (part 3)

* Fix tests

* Update features snapshot

* Fux typo

* Fix clippy default usage

* Fix clippy ref usage

* Add missing docs

* Introduce frontier backend configuration

* [frontier-apply] make rocksdb and paritydb optional. (#1088)

* Make new_partial async

* Return result for frontier backend creation

* Implement a helper function to remove frontier offchain db

* Update features snapshot

* Fix clippy

* Update help stdout fixture

* [substrate-apply] Pub enum runtime to pub struct runtime #13250

* Update frontier

* Properly use evm deps

* Fix fmt

* Use fully qualified TxPool usage

* Link TODO to humanode issue

* Add test to verify pending block extraction

* Return new_partial to be synchronously

* Remove missed added test file

* Fix help sdtout fixture

* Remove pending block test

Author: dmitrylavrenov

Cargo.lock

@@ -8,5 +8,5 @@ publish = false
 frame-support = { workspace = true }
 pallet-im-online = { workspace = true }
 sp-consensus-babe = { workspace = true }
-sp-finality-grandpa = { workspace = true }
+sp-consensus-grandpa = { workspace = true }
 sp-runtime = { workspace = true }

Cargo.toml

@@ -2,7 +2,7 @@
 
 use pallet_im_online::sr25519::AuthorityId as ImOnlineId;
 use sp_consensus_babe::AuthorityId as BabeId;
-use sp_finality_grandpa::AuthorityId as GrandpaId;
+use sp_consensus_grandpa::AuthorityId as GrandpaId;
 use sp_runtime::{
     app_crypto::{Pair, Public},
     traits::IdentifyAccount,

crates/crypto-utils/Cargo.toml

@@ -27,8 +27,8 @@ clap = { workspace = true, features = ["derive"] }
 codec = { workspace = true, package = "parity-scale-codec" }
 fc-cli = { workspace = true }
 fc-consensus = { workspace = true }
-fc-db = { workspace = true }
-fc-mapping-sync = { workspace = true }
+fc-db = { workspace = true, features = ["sql"] }
+fc-mapping-sync = { workspace = true, features = ["sql"] }
 fc-rpc = { workspace = true }
 fc-rpc-core = { workspace = true }
 fc-storage = { workspace = true }
@@ -53,8 +53,8 @@ sc-cli = { workspace = true }
 sc-client-api = { workspace = true }
 sc-consensus = { workspace = true }
 sc-consensus-babe = { workspace = true }
+sc-consensus-grandpa = { workspace = true }
 sc-executor = { workspace = true }
-sc-finality-grandpa = { workspace = true }
 sc-service = { workspace = true }
 sc-telemetry = { workspace = true }
 sc-transaction-pool = { workspace = true }
@@ -63,8 +63,8 @@ serde_json = { workspace = true }
 sp-api = { workspace = true }
 sp-application-crypto = { workspace = true }
 sp-consensus-babe = { workspace = true }
+sp-consensus-grandpa = { workspace = true }
 sp-core = { workspace = true }
-sp-finality-grandpa = { workspace = true }
 sp-inherents = { workspace = true }
 sp-io = { workspace = true }
 sp-keyring = { workspace = true }
@@ -87,7 +87,8 @@ sp-io = { workspace = true }
 vergen = { workspace = true, features = ["cargo", "git", "gitcl"] }
 
 [features]
-default = []
+default = ["rocksdb"]
+rocksdb = ["fc-db/rocksdb"]
 runtime-benchmarks = [
   "frame-benchmarking-cli/runtime-benchmarks",
   "frame-benchmarking/runtime-benchmarks",

crates/crypto-utils/src/lib.rs

@@ -15,7 +15,7 @@ use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup};
 use sc_service::ChainType;
 use serde::{Deserialize, Serialize};
 use sp_consensus_babe::AuthorityId as BabeId;
-use sp_finality_grandpa::AuthorityId as GrandpaId;
+use sp_consensus_grandpa::AuthorityId as GrandpaId;
 use sp_runtime::{app_crypto::sr25519, traits::Verify};
 
 /// The concrete chain spec type we're using for the humanode network.

crates/humanode-peer/Cargo.toml

@@ -54,6 +54,15 @@ pub trait CliConfigurationExt: SubstrateCliConfigurationProvider {
                 execute_gas_limit_multiplier: params.execute_gas_limit_multiplier,
             });
 
+        let fb_params = self.frontier_backend();
+        let frontier_backend = configuration::FrontierBackend {
+            frontier_backend_type: fb_params.frontier_backend_type,
+            frontier_sql_backend_pool_size: fb_params.frontier_sql_backend_pool_size,
+            frontier_sql_backend_num_ops_timeout: fb_params.frontier_sql_backend_num_ops_timeout,
+            frontier_sql_backend_thread_count: fb_params.frontier_sql_backend_thread_count,
+            frontier_sql_backend_cache_size: fb_params.frontier_sql_backend_cache_size,
+        };
+
         let time_warp = self.time_warp_params().and_then(|params| {
             params
                 .time_warp_fork_timestamp
@@ -71,6 +80,7 @@ pub trait CliConfigurationExt: SubstrateCliConfigurationProvider {
             substrate,
             bioauth_flow,
             ethereum_rpc,
+            frontier_backend,
             time_warp,
         })
     }
@@ -85,6 +95,11 @@ pub trait CliConfigurationExt: SubstrateCliConfigurationProvider {
         None
     }
 
+    /// Provide the Frontier backend params.
+    fn frontier_backend(&self) -> params::FrontierBackendParams {
+        Default::default()
+    }
+
     /// Provide the time warp related params, if available.
     fn time_warp_params(&self) -> Option<&params::TimeWarpParams> {
         None

crates/humanode-peer/src/chain_spec.rs

@@ -1,5 +1,7 @@
 //! Shared CLI parameters.
 
+use crate::configuration::FrontierBackendType;
+
 /// Possible RPC URL scheme preference options.
 #[derive(Debug, clap::ValueEnum, Clone)]
 pub enum RpcUrlSchemePreference {
@@ -75,6 +77,31 @@ pub struct EthereumRpcParams {
     pub execute_gas_limit_multiplier: u64,
 }
 
+/// Shared CLI parameters used to configure Frontier backend.
+#[derive(Debug, Default, clap::Parser, Clone)]
+pub struct FrontierBackendParams {
+    /// Sets the frontier backend type (KeyValue or Sql).
+    #[arg(long, value_enum, ignore_case = true, default_value_t = FrontierBackendType::default())]
+    pub frontier_backend_type: FrontierBackendType,
+
+    /// Sets the SQL backend's pool size.
+    #[arg(long, default_value = "100")]
+    pub frontier_sql_backend_pool_size: u32,
+
+    /// Sets the SQL backend's query timeout in number of VM ops.
+    #[arg(long, default_value = "10000000")]
+    pub frontier_sql_backend_num_ops_timeout: u32,
+
+    /// Sets the SQL backend's auxiliary thread limit.
+    #[arg(long, default_value = "4")]
+    pub frontier_sql_backend_thread_count: u32,
+
+    /// Sets the SQL backend's query timeout in number of VM ops.
+    /// Default value is 200MB.
+    #[arg(long, default_value = "209715200")]
+    pub frontier_sql_backend_cache_size: u64,
+}
+
 /// Shared CLI parameters used to configure time warp mode.
 #[derive(Debug, clap::Parser, Clone)]
 pub struct TimeWarpParams {

crates/humanode-peer/src/cli/config.rs

@@ -6,7 +6,7 @@ use frame_benchmarking_cli::*;
 use humanode_runtime::Block;
 #[cfg(feature = "runtime-benchmarks")]
 use humanode_runtime::Runtime;
-use sc_service::PartialComponents;
+use sc_service::{DatabaseSource, PartialComponents};
 #[cfg(feature = "runtime-benchmarks")]
 use sp_core::Get;
 #[cfg(feature = "try-runtime")]
@@ -18,7 +18,7 @@ use {
 use super::{bioauth, Root, Subcommand};
 #[cfg(feature = "runtime-benchmarks")]
 use crate::benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder};
-use crate::service;
+use crate::{configuration, service};
 
 /// Parse command line arguments and run the requested operation.
 pub async fn run() -> sc_cli::Result<()> {
@@ -86,7 +86,10 @@ pub async fn run() -> sc_cli::Result<()> {
         }
         Some(Subcommand::PurgeChain(cmd)) => {
             let runner = root.create_humanode_runner(cmd)?;
-            runner.sync_run(|config| cmd.run(config.substrate.database))
+            runner.sync_run(|config| {
+                remove_frontier_offchain_db(cmd, &config)?;
+                cmd.run(config.substrate.database)
+            })
         }
         Some(Subcommand::Revert(cmd)) => {
             let runner = root.create_humanode_runner(cmd)?;
@@ -100,7 +103,7 @@ pub async fn run() -> sc_cli::Result<()> {
                     } = service::new_partial(&config)?;
                     let aux_revert = Box::new(|client, backend, blocks| {
                         sc_consensus_babe::revert(Arc::clone(&client), backend, blocks)?;
-                        sc_finality_grandpa::revert(client, blocks)?;
+                        sc_consensus_grandpa::revert(client, blocks)?;
                         Ok(())
                     });
                     Ok((cmd.run(client, backend, Some(aux_revert)), task_manager))
@@ -230,7 +233,12 @@ pub async fn run() -> sc_cli::Result<()> {
             let runner = root.create_humanode_runner(cmd)?;
             runner.sync_run(|config| {
                 let partial = service::new_partial(&config)?;
-                let frontier_backend = partial.other.4;
+                let frontier_backend = match partial.other.4 {
+                    fc_db::Backend::KeyValue(kv_fb) => Arc::new(kv_fb),
+                    _ => {
+                        panic!("Only fc_db::Backend::KeyValue supported for FrontierDb command")
+                    }
+                };
                 cmd.run(partial.client, frontier_backend)
             })
         }
@@ -297,3 +305,44 @@ fn print_build_info() {
         crate::build_info::CARGO_FEATURES
     );
 }
+
+/// Remove Frontier offchain db.
+fn remove_frontier_offchain_db(
+    cmd: &sc_cli::PurgeChainCmd,
+    config: &configuration::Configuration,
+) -> sc_cli::Result<()> {
+    let fdb_config_dir = service::frontier::db_config_dir(&config.substrate);
+
+    match config.frontier_backend.frontier_backend_type {
+        crate::configuration::FrontierBackendType::KeyValue => {
+            let frontier_database_config = match config.substrate.database {
+                DatabaseSource::RocksDb { .. } => DatabaseSource::RocksDb {
+                    path: fc_db::kv::frontier_database_dir(&fdb_config_dir, "db"),
+                    cache_size: 0,
+                },
+                DatabaseSource::ParityDb { .. } => DatabaseSource::ParityDb {
+                    path: fc_db::kv::frontier_database_dir(&fdb_config_dir, "paritydb"),
+                },
+                _ => panic!("frontier supports either rocksdb or paritydb"),
+            };
+            cmd.run(frontier_database_config)?;
+        }
+        crate::configuration::FrontierBackendType::Sql => {
+            let db_path = fdb_config_dir.join("sql");
+
+            match std::fs::remove_dir_all(&db_path) {
+                Ok(_) => {
+                    tracing::info!("{:?} removed.", &db_path);
+                }
+                Err(ref err) if err.kind() == std::io::ErrorKind::NotFound => {
+                    tracing::error!("{:?} did not exist.", &db_path);
+                }
+                Err(err) => {
+                    return Err(format!("Cannot purge `{:?}` database: {:?}", db_path, err).into())
+                }
+            };
+        }
+    };
+
+    Ok(())
+}

crates/humanode-peer/src/cli/params.rs

@@ -21,6 +21,9 @@ pub struct Configuration {
     /// Ethereum RPC configuration.
     pub ethereum_rpc: Option<EthereumRpc>,
 
+    /// Frontier backend configuration.
+    pub frontier_backend: FrontierBackend,
+
     /// Time warp mode configuration.
     /// If not defined, time warp mode isn't enabled.
     pub time_warp: Option<TimeWarp>,
@@ -73,3 +76,32 @@ pub struct EthereumRpc {
     /// block.gas_limit * execute_gas_limit_multiplier.
     pub execute_gas_limit_multiplier: u64,
 }
+
+/// Frontier backend configuration parameters.
+pub struct FrontierBackend {
+    /// Sets the frontier backend type (KeyValue or Sql).
+    pub frontier_backend_type: FrontierBackendType,
+
+    /// Sets the SQL backend's pool size.
+    pub frontier_sql_backend_pool_size: u32,
+
+    /// Sets the SQL backend's query timeout in number of VM ops.
+    pub frontier_sql_backend_num_ops_timeout: u32,
+
+    /// Sets the SQL backend's auxiliary thread limit.
+    pub frontier_sql_backend_thread_count: u32,
+
+    /// Sets the SQL backend's query timeout in number of VM ops.
+    /// Default value is 200MB.
+    pub frontier_sql_backend_cache_size: u64,
+}
+
+/// Avalailable frontier backend types.
+#[derive(Default, Debug, Copy, Clone, clap::ValueEnum)]
+pub enum FrontierBackendType {
+    /// Either RocksDb or ParityDb as per inherited from the global backend settings.
+    #[default]
+    KeyValue,
+    /// Sql database with custom log indexing.
+    Sql,
+}

crates/humanode-peer/src/cli/run.rs

@@ -1,9 +1,15 @@
 //! The Frontier related stuff.
 
+use std::{path::Path, sync::Arc};
+
+use fc_storage::OverrideHandle;
 use humanode_runtime::opaque::Block;
 use sc_cli::SubstrateCli;
 use sc_client_api::backend::Backend;
-use sc_service::BasePath;
+use sc_service::{BasePath, Configuration};
+
+use super::{FrontierBackend, FullClient, ServiceError};
+use crate::configuration::{self, FrontierBackendType};
 
 /// Create frontier dir.
 pub fn db_config_dir(config: &sc_service::Configuration) -> std::path::PathBuf {
@@ -17,6 +23,45 @@ pub fn db_config_dir(config: &sc_service::Configuration) -> std::path::PathBuf {
         })
 }
 
+/// Create frontier backend.
+pub fn backend(
+    config: &Configuration,
+    client: Arc<FullClient>,
+    fb_config: &configuration::FrontierBackend,
+    eth_overrides: Arc<OverrideHandle<Block>>,
+) -> Result<FrontierBackend, ServiceError> {
+    match fb_config.frontier_backend_type {
+        FrontierBackendType::KeyValue => Ok(FrontierBackend::KeyValue(fc_db::kv::Backend::open(
+            Arc::clone(&client),
+            &config.database,
+            &db_config_dir(config),
+        )?)),
+        FrontierBackendType::Sql => {
+            let db_path = db_config_dir(config).join("sql");
+            std::fs::create_dir_all(&db_path)?;
+
+            let backend = futures::executor::block_on(fc_db::sql::Backend::new(
+                fc_db::sql::BackendConfig::Sqlite(fc_db::sql::SqliteBackendConfig {
+                    path: Path::new("sqlite:///")
+                        .join(db_path)
+                        .join("frontier.db3")
+                        .to_str()
+                        .ok_or(ServiceError::Other("invalid sqlite path".to_owned()))?,
+                    create_if_missing: true,
+                    thread_count: fb_config.frontier_sql_backend_thread_count,
+                    cache_size: fb_config.frontier_sql_backend_cache_size,
+                }),
+                fb_config.frontier_sql_backend_pool_size,
+                std::num::NonZeroU32::new(fb_config.frontier_sql_backend_num_ops_timeout),
+                Arc::clone(&eth_overrides),
+            ))
+            .map_err(|err| ServiceError::Application(err.into()))?;
+
+            Ok(FrontierBackend::Sql(backend))
+        }
+    }
+}
+
 /// Default ethereum config.
 pub struct DefaultEthConfig<C, BE>(std::marker::PhantomData<(C, BE)>);