From 0f894f069ace6a7c6620e5b77ee18b3ebe40924f Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sat, 14 Sep 2019 17:45:55 +0200 Subject: [PATCH 01/27] consensus-pow: add difficulty data to auxiliary --- core/consensus/pow/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 343c7b14c6af1..db7201b8faa73 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -63,6 +63,8 @@ fn aux_key(hash: &H256) -> Vec { /// Auxiliary storage data for PoW. #[derive(Encode, Decode, Clone, Debug, Default)] pub struct PowAux { + /// Difficulty. + pub difficulty: Difficulty, /// Total difficulty. pub total_difficulty: Difficulty, } @@ -208,6 +210,7 @@ impl, C, Algorithm> Verifier for PowVerifier, C, Algorithm, E>( } }; + aux.difficulty = difficulty; aux.total_difficulty = aux.total_difficulty.saturating_add(difficulty); let hash = header.hash(); From debbb5b728ae182c34a569caf18b79380e519c11 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sat, 14 Sep 2019 20:22:10 +0200 Subject: [PATCH 02/27] Timestamp api --- core/consensus/pow/primitives/Cargo.toml | 2 ++ core/consensus/pow/primitives/src/lib.rs | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/core/consensus/pow/primitives/Cargo.toml b/core/consensus/pow/primitives/Cargo.toml index a7e0d284b09fa..021e4c80a754c 100644 --- a/core/consensus/pow/primitives/Cargo.toml +++ b/core/consensus/pow/primitives/Cargo.toml @@ -10,6 +10,7 @@ substrate-client = { path = "../../../client", default-features = false } rstd = { package = "sr-std", path = "../../../sr-std", default-features = false } sr-primitives = { path = "../../../sr-primitives", default-features = false } primitives = { package = "substrate-primitives", path = "../../../primitives", default-features = false } +codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } [features] default = ["std"] @@ -18,4 +19,5 @@ std = [ "substrate-client/std", "sr-primitives/std", "primitives/std", + "codec/std", ] diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs index 807a7b2df2c90..361c5cb13e261 100644 --- a/core/consensus/pow/primitives/src/lib.rs +++ b/core/consensus/pow/primitives/src/lib.rs @@ -20,6 +20,8 @@ use rstd::vec::Vec; use sr_primitives::ConsensusEngineId; +use codec::Decode; +use substrate_client::decl_runtime_apis; /// The `ConsensusEngineId` of PoW. pub const POW_ENGINE_ID: ConsensusEngineId = [b'p', b'o', b'w', b'_']; @@ -34,3 +36,11 @@ pub type Difficulty = u128; /// Type of seal. pub type Seal = Vec; + +decl_runtime_apis! { + /// API necessary for timestamp-based difficulty adjustment algorithms. + pub trait TimestampApi { + /// Return the timestamp in the current block. + fn timestamp() -> Moment; + } +} From 62fb99a8419a761d0e992843ac4e9e81f678b5b0 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sat, 14 Sep 2019 20:42:40 +0200 Subject: [PATCH 03/27] Implement FinalityProofProvider for () --- core/network/src/chain.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/network/src/chain.rs b/core/network/src/chain.rs index a4767caf13d28..f9a4e04ad7f4b 100644 --- a/core/network/src/chain.rs +++ b/core/network/src/chain.rs @@ -78,6 +78,12 @@ pub trait FinalityProofProvider: Send + Sync { fn prove_finality(&self, for_block: Block::Hash, request: &[u8]) -> Result>, Error>; } +impl FinalityProofProvider for () { + fn prove_finality(&self, _for_block: Block::Hash, _request: &[u8]) -> Result>, Error> { + Ok(None) + } +} + impl Client for SubstrateClient where B: client::backend::Backend + Send + Sync + 'static, E: CallExecutor + Send + Sync + 'static, From fcb81d7c6554a6f815a662c11c458b4e92e6d270 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 17 Sep 2019 10:04:57 +0200 Subject: [PATCH 04/27] Add DifficultyApi --- core/consensus/pow/primitives/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs index 361c5cb13e261..93489343b091c 100644 --- a/core/consensus/pow/primitives/src/lib.rs +++ b/core/consensus/pow/primitives/src/lib.rs @@ -43,4 +43,12 @@ decl_runtime_apis! { /// Return the timestamp in the current block. fn timestamp() -> Moment; } + + /// API for those chains that put their difficulty adjustment algorithm directly + /// onto runtime. Note that while putting difficulty adjustment algorithm to + /// runtime is safe, putting the PoW algorithm on runtime is not. + pub trait DifficultyApi { + /// Return the target difficulty of the next block. + fn difficulty() -> Difficulty; + } } From 74b363441ac1d34a2c799b78592674e704f273ee Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 19 Sep 2019 12:49:28 +0200 Subject: [PATCH 05/27] Remove assumption that Difficulty is u128 --- core/consensus/pow/primitives/src/lib.rs | 12 ++--------- core/consensus/pow/src/lib.rs | 26 +++++++++++++++--------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs index 93489343b091c..157cbe281bc2e 100644 --- a/core/consensus/pow/primitives/src/lib.rs +++ b/core/consensus/pow/primitives/src/lib.rs @@ -20,20 +20,12 @@ use rstd::vec::Vec; use sr_primitives::ConsensusEngineId; -use codec::Decode; +use codec::{Encode, Decode}; use substrate_client::decl_runtime_apis; /// The `ConsensusEngineId` of PoW. pub const POW_ENGINE_ID: ConsensusEngineId = [b'p', b'o', b'w', b'_']; -/// Type of difficulty. -/// -/// For runtime designed for Substrate, it's always possible to fit its total -/// difficulty range under `u128::max_value()` because it can be freely scaled -/// up or scaled down. Very few PoW chains use difficulty values -/// larger than `u128::max_value()`. -pub type Difficulty = u128; - /// Type of seal. pub type Seal = Vec; @@ -47,7 +39,7 @@ decl_runtime_apis! { /// API for those chains that put their difficulty adjustment algorithm directly /// onto runtime. Note that while putting difficulty adjustment algorithm to /// runtime is safe, putting the PoW algorithm on runtime is not. - pub trait DifficultyApi { + pub trait DifficultyApi { /// Return the target difficulty of the next block. fn difficulty() -> Difficulty; } diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index db7201b8faa73..e52872cdd4ff2 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -38,9 +38,9 @@ use client::{ }; use sr_primitives::Justification; use sr_primitives::generic::{BlockId, Digest, DigestItem}; -use sr_primitives::traits::{Block as BlockT, Header as HeaderT, ProvideRuntimeApi}; +use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Saturating, ProvideRuntimeApi}; use srml_timestamp::{TimestampInherentData, InherentError as TIError}; -use pow_primitives::{Difficulty, Seal, POW_ENGINE_ID}; +use pow_primitives::{Seal, POW_ENGINE_ID}; use primitives::H256; use inherents::{InherentDataProviders, InherentData}; use consensus_common::{ @@ -62,36 +62,42 @@ fn aux_key(hash: &H256) -> Vec { /// Auxiliary storage data for PoW. #[derive(Encode, Decode, Clone, Debug, Default)] -pub struct PowAux { +pub struct PowAux { /// Difficulty. pub difficulty: Difficulty, /// Total difficulty. pub total_difficulty: Difficulty, } -impl PowAux { +impl PowAux where + Difficulty: Decode + Default, +{ /// Read the auxiliary from client. pub fn read(client: &C, hash: &H256) -> Result { let key = aux_key(hash); match client.get_aux(&key).map_err(|e| format!("{:?}", e))? { - Some(bytes) => PowAux::decode(&mut &bytes[..]).map_err(|e| format!("{:?}", e)), - None => Ok(PowAux::default()), + Some(bytes) => Self::decode(&mut &bytes[..]) + .map_err(|e| format!("{:?}", e)), + None => Ok(Self::default()), } } } /// Algorithm used for proof of work. pub trait PowAlgorithm { + /// Difficulty for the algorithm. + type Difficulty: Saturating + Default + Encode + Decode + Ord + Clone + Copy; + /// Get the next block's difficulty. - fn difficulty(&self, parent: &BlockId) -> Result; + fn difficulty(&self, parent: &BlockId) -> Result; /// Verify proof of work against the given difficulty. fn verify( &self, parent: &BlockId, pre_hash: &H256, seal: &Seal, - difficulty: Difficulty, + difficulty: Self::Difficulty, ) -> Result; /// Mine a seal that satisfy the given difficulty. fn mine( @@ -99,7 +105,7 @@ pub trait PowAlgorithm { parent: &BlockId, pre_hash: &H256, seed: &H256, - difficulty: Difficulty, + difficulty: Self::Difficulty, round: u32, ) -> Result, String>; } @@ -116,7 +122,7 @@ impl PowVerifier { &self, mut header: B::Header, parent_block_id: BlockId, - ) -> Result<(B::Header, Difficulty, DigestItem), String> where + ) -> Result<(B::Header, Algorithm::Difficulty, DigestItem), String> where Algorithm: PowAlgorithm, { let hash = header.hash(); From a1b49b1dc7cf2e83069ad154f1348143a0a7ed3f Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 00:36:35 +0200 Subject: [PATCH 06/27] Use a separate trait for add instead of hard-code it as Saturating --- core/consensus/pow/primitives/src/lib.rs | 5 +++++ core/consensus/pow/src/lib.rs | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs index 157cbe281bc2e..e551d060129dc 100644 --- a/core/consensus/pow/primitives/src/lib.rs +++ b/core/consensus/pow/primitives/src/lib.rs @@ -29,6 +29,11 @@ pub const POW_ENGINE_ID: ConsensusEngineId = [b'p', b'o', b'w', b'_']; /// Type of seal. pub type Seal = Vec; +/// Define methods that total difficulty should implement. +pub trait TotalDifficulty { + fn add(&mut self, other: Self); +} + decl_runtime_apis! { /// API necessary for timestamp-based difficulty adjustment algorithms. pub trait TimestampApi { diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index e52872cdd4ff2..4b2b3ebd9c3d4 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -38,9 +38,9 @@ use client::{ }; use sr_primitives::Justification; use sr_primitives::generic::{BlockId, Digest, DigestItem}; -use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Saturating, ProvideRuntimeApi}; +use sr_primitives::traits::{Block as BlockT, Header as HeaderT, ProvideRuntimeApi}; use srml_timestamp::{TimestampInherentData, InherentError as TIError}; -use pow_primitives::{Seal, POW_ENGINE_ID}; +use pow_primitives::{Seal, TotalDifficulty, POW_ENGINE_ID}; use primitives::H256; use inherents::{InherentDataProviders, InherentData}; use consensus_common::{ @@ -87,7 +87,7 @@ impl PowAux where /// Algorithm used for proof of work. pub trait PowAlgorithm { /// Difficulty for the algorithm. - type Difficulty: Saturating + Default + Encode + Decode + Ord + Clone + Copy; + type Difficulty: TotalDifficulty + Default + Encode + Decode + Ord + Clone + Copy; /// Get the next block's difficulty. fn difficulty(&self, parent: &BlockId) -> Result; @@ -217,7 +217,7 @@ impl, C, Algorithm> Verifier for PowVerifier, C, Algorithm, E>( }; aux.difficulty = difficulty; - aux.total_difficulty = aux.total_difficulty.saturating_add(difficulty); + aux.total_difficulty.add(difficulty); let hash = header.hash(); let key = aux_key(&hash); From 9775ece1877ded9c2fc1cc4980d8718cd408e895 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 03:49:52 +0200 Subject: [PATCH 07/27] Some convenience functions to work with PowVerifier --- core/consensus/pow/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 4b2b3ebd9c3d4..834b884d8f269 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -118,6 +118,14 @@ pub struct PowVerifier { } impl PowVerifier { + pub fn new( + client: Arc, + algorithm: Algorithm, + inherent_data_providers: inherents::InherentDataProviders, + ) -> Self { + Self { client, algorithm, inherent_data_providers } + } + fn check_header>( &self, mut header: B::Header, @@ -249,7 +257,7 @@ impl, C, Algorithm> Verifier for PowVerifier Result<(), consensus_common::Error> { if !inherent_data_providers.has_provider(&srml_timestamp::INHERENT_IDENTIFIER) { From 526b8605d102cefc8b08615f107d3beb2c7d5785 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 16:15:48 +0200 Subject: [PATCH 08/27] Try to fix mining unstability --- core/consensus/pow/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 834b884d8f269..3fc5d98e24fd0 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -418,6 +418,16 @@ fn mine_loop, C, Algorithm, E>( let hash = header.hash(); let key = aux_key(&hash); + let best_hash = client.info().best_hash; + let best_header = client.header(BlockId::Hash(best_hash)) + .map_err(|e| format!("Fetching best header failed: {:?}", e))? + .ok_or("Best header does not exist")?; + let best_aux = PowAux::read(client, &best_hash)?; + + if best_aux.total_difficulty > aux.total_difficulty { + continue 'outer + } + let import_block = BlockImportParams { origin: BlockOrigin::Own, header, From 912c03200f265adf9e24b640ce499b38f78afabe Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 16:17:55 +0200 Subject: [PATCH 09/27] Fix generic resolution --- core/consensus/pow/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 3fc5d98e24fd0..4bfe958f4a060 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -422,7 +422,7 @@ fn mine_loop, C, Algorithm, E>( let best_header = client.header(BlockId::Hash(best_hash)) .map_err(|e| format!("Fetching best header failed: {:?}", e))? .ok_or("Best header does not exist")?; - let best_aux = PowAux::read(client, &best_hash)?; + let best_aux = PowAux::::read(client, &best_hash)?; if best_aux.total_difficulty > aux.total_difficulty { continue 'outer From 4beec1a3c4470fd34bd8eef0b050ba687ddf0578 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 16:19:11 +0200 Subject: [PATCH 10/27] Unused best_header variable --- core/consensus/pow/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 4bfe958f4a060..d5048bb194791 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -419,9 +419,6 @@ fn mine_loop, C, Algorithm, E>( let key = aux_key(&hash); let best_hash = client.info().best_hash; - let best_header = client.header(BlockId::Hash(best_hash)) - .map_err(|e| format!("Fetching best header failed: {:?}", e))? - .ok_or("Best header does not exist")?; let best_aux = PowAux::::read(client, &best_hash)?; if best_aux.total_difficulty > aux.total_difficulty { From ff7a3f89fcf91f43992b7006b778a08426c7e008 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 16:55:55 +0200 Subject: [PATCH 11/27] Fix hash calculation --- core/consensus/pow/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index d5048bb194791..906d3e6215abe 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -415,7 +415,11 @@ fn mine_loop, C, Algorithm, E>( aux.difficulty = difficulty; aux.total_difficulty.add(difficulty); - let hash = header.hash(); + let hash = { + let mut header = header.clone(); + header.digest_mut().push(DigestItem::Seal(POW_ENGINE_ID, seal.clone())); + header.hash() + }; let key = aux_key(&hash); let best_hash = client.info().best_hash; From a484830eee97141749210ae0d5c6e72019e0839b Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 17:22:46 +0200 Subject: [PATCH 12/27] Remove artificial sleep --- core/consensus/pow/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 906d3e6215abe..3d6664cf56f5e 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -347,8 +347,6 @@ pub fn start_mine, C, Algorithm, E>( e ), } - - std::thread::sleep(std::time::Duration::new(1, 0)); } }); } From 58395dc4409cb15dba03dae669d85acb7c851ddf Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 17:23:46 +0200 Subject: [PATCH 13/27] Tweak proposer waiting time --- core/consensus/pow/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 3d6664cf56f5e..0e5dab5f4f597 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -382,7 +382,7 @@ fn mine_loop, C, Algorithm, E>( let block = futures::executor::block_on(proposer.propose( inherent_data, inherent_digest, - std::time::Duration::new(0, 0) + std::time::Duration::new(0, 100) )).map_err(|e| format!("Block proposing error: {:?}", e))?; let (header, body) = block.deconstruct(); From f431d63b1b881794bbf54c5cd7c36a09cfb54f82 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Sep 2019 17:28:30 +0200 Subject: [PATCH 14/27] Revert sleep removal The reason why it was there is because when mine_loop returns, it means an error happened. In that case, we'd better sleep for a moment before trying again, because immediately trying would most likely just fail. --- core/consensus/pow/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 0e5dab5f4f597..e7cc18d4bdd05 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -347,6 +347,7 @@ pub fn start_mine, C, Algorithm, E>( e ), } + std::thread::sleep(std::time::Duration::new(1, 0)); } }); } From 53e8beabc085dcce9c01ea9caf128a947f7569cd Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sat, 21 Sep 2019 00:24:25 +0200 Subject: [PATCH 15/27] Pass sync oracle to mining So that it does not mine when major syncing --- core/consensus/pow/src/lib.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index e7cc18d4bdd05..c6ffcaeea92cb 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -44,7 +44,7 @@ use pow_primitives::{Seal, TotalDifficulty, POW_ENGINE_ID}; use primitives::H256; use inherents::{InherentDataProviders, InherentData}; use consensus_common::{ - BlockImportParams, BlockOrigin, ForkChoiceStrategy, + BlockImportParams, BlockOrigin, ForkChoiceStrategy, SyncOracle, well_known_cache_keys::Id as CacheKeyId, Environment, Proposer, }; use consensus_common::import_queue::{BoxBlockImport, BasicQueue, Verifier}; @@ -312,19 +312,21 @@ pub fn import_queue( /// information, or just be a graffiti. `round` is for number of rounds the /// CPU miner runs each time. This parameter should be tweaked so that each /// mining round is within sub-second time. -pub fn start_mine, C, Algorithm, E>( +pub fn start_mine, C, Algorithm, E, SO>( mut block_import: BoxBlockImport, client: Arc, algorithm: Algorithm, mut env: E, preruntime: Option>, round: u32, + mut sync_oracle: SO, inherent_data_providers: inherents::InherentDataProviders, ) where C: HeaderBackend + AuxStore + 'static, Algorithm: PowAlgorithm + Send + Sync + 'static, E: Environment + Send + Sync + 'static, E::Error: std::fmt::Debug, + SO: SyncOracle + Send + Sync + 'static, { if let Err(_) = register_pow_inherent_data_provider(&inherent_data_providers) { warn!("Registering inherent data provider for timestamp failed"); @@ -339,6 +341,7 @@ pub fn start_mine, C, Algorithm, E>( &mut env, preruntime.as_ref(), round, + &mut sync_oracle, &inherent_data_providers ) { Ok(()) => (), @@ -352,21 +355,29 @@ pub fn start_mine, C, Algorithm, E>( }); } -fn mine_loop, C, Algorithm, E>( +fn mine_loop, C, Algorithm, E, SO>( block_import: &mut BoxBlockImport, client: &C, algorithm: &Algorithm, env: &mut E, preruntime: Option<&Vec>, round: u32, + sync_oracle: &mut SO, inherent_data_providers: &inherents::InherentDataProviders, ) -> Result<(), String> where C: HeaderBackend + AuxStore, Algorithm: PowAlgorithm, E: Environment, E::Error: std::fmt::Debug, + SO: SyncOracle, { 'outer: loop { + if sync_oracle.is_major_syncing() { + debug!(target: "pow", "Skipping proposal due to sync."); + std::thread::sleep(std::time::Duration::new(1, 0)); + continue 'outer + } + let best_hash = client.info().best_hash; let best_header = client.header(BlockId::Hash(best_hash)) .map_err(|e| format!("Fetching best header failed: {:?}", e))? From e417b3fe8b76d762ece028f39bc918a62b400637 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sat, 21 Sep 2019 00:41:06 +0200 Subject: [PATCH 16/27] Expose build time as a parameter Instead of hardcode it as previously 100ms. --- core/consensus/pow/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index c6ffcaeea92cb..abae75eeeb2f9 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -320,6 +320,7 @@ pub fn start_mine, C, Algorithm, E, SO>( preruntime: Option>, round: u32, mut sync_oracle: SO, + build_time: std::time::Duration, inherent_data_providers: inherents::InherentDataProviders, ) where C: HeaderBackend + AuxStore + 'static, @@ -342,6 +343,7 @@ pub fn start_mine, C, Algorithm, E, SO>( preruntime.as_ref(), round, &mut sync_oracle, + build_time.clone(), &inherent_data_providers ) { Ok(()) => (), @@ -363,6 +365,7 @@ fn mine_loop, C, Algorithm, E, SO>( preruntime: Option<&Vec>, round: u32, sync_oracle: &mut SO, + build_time: std::time::Duration, inherent_data_providers: &inherents::InherentDataProviders, ) -> Result<(), String> where C: HeaderBackend + AuxStore, @@ -394,7 +397,7 @@ fn mine_loop, C, Algorithm, E, SO>( let block = futures::executor::block_on(proposer.propose( inherent_data, inherent_digest, - std::time::Duration::new(0, 100) + build_time.clone(), )).map_err(|e| format!("Block proposing error: {:?}", e))?; let (header, body) = block.deconstruct(); From f9b76ec2703b63de1120116ec29e4b3cc3e94f1c Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sun, 22 Sep 2019 06:51:39 +0800 Subject: [PATCH 17/27] Update lock file --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index c852d480edd5d..2e1fe9c5618c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4856,6 +4856,7 @@ dependencies = [ name = "substrate-consensus-pow-primitives" version = "2.0.0" dependencies = [ + "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 2.0.0", "sr-std 2.0.0", "substrate-client 2.0.0", From fb71f7776fc156d44780059da00f7f7c4d657fa9 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sun, 22 Sep 2019 06:59:24 +0800 Subject: [PATCH 18/27] Fix compile --- core/consensus/pow/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 9040bb6ae28a8..d27b9764ebea9 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -45,8 +45,7 @@ use pow_primitives::{Seal, TotalDifficulty, POW_ENGINE_ID}; use primitives::H256; use inherents::{InherentDataProviders, InherentData}; use consensus_common::{ - BlockImportParams, BlockOrigin, ForkChoiceStrategy, SyncOracle, - well_known_cache_keys::Id as CacheKeyId, Environment, Proposer, + BlockImportParams, BlockOrigin, ForkChoiceStrategy, SyncOracle, Environment, Proposer, }; use consensus_common::import_queue::{BoxBlockImport, BasicQueue, Verifier}; use codec::{Encode, Decode}; From 8888f161ad2cdcae9c4113b81c50bbff246a8b17 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Sun, 22 Sep 2019 16:22:35 +0800 Subject: [PATCH 19/27] Support skipping check_inherents for ancient blocks For PoW, older blocks are secured by the work, and can mostly be considered to be finalized. Thus we can save both code complexity and validation time by skipping checking inherents for them. --- core/consensus/pow/src/lib.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index d27b9764ebea9..42bb02a918c1c 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -111,22 +111,24 @@ pub trait PowAlgorithm { } /// A verifier for PoW blocks. -pub struct PowVerifier { +pub struct PowVerifier, C, Algorithm> { client: Arc, algorithm: Algorithm, inherent_data_providers: inherents::InherentDataProviders, + check_inherents_after: <::Header as HeaderT>::Number, } -impl PowVerifier { +impl, C, Algorithm> PowVerifier { pub fn new( client: Arc, algorithm: Algorithm, + check_inherents_after: <::Header as HeaderT>::Number, inherent_data_providers: inherents::InherentDataProviders, ) -> Self { - Self { client, algorithm, inherent_data_providers } + Self { client, algorithm, inherent_data_providers, check_inherents_after } } - fn check_header>( + fn check_header( &self, mut header: B::Header, parent_block_id: BlockId, @@ -161,7 +163,7 @@ impl PowVerifier { Ok((header, difficulty, seal)) } - fn check_inherents>( + fn check_inherents( &self, block: B, block_id: BlockId, @@ -172,6 +174,10 @@ impl PowVerifier { { const MAX_TIMESTAMP_DRIFT_SECS: u64 = 60; + if *block.header().number() < self.check_inherents_after { + return Ok(()) + } + let inherent_res = self.client.runtime_api().check_inherents( &block_id, block, @@ -198,7 +204,7 @@ impl PowVerifier { } } -impl, C, Algorithm> Verifier for PowVerifier where +impl, C, Algorithm> Verifier for PowVerifier where C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + ProvideCache + BlockOf, C::Api: BlockBuilderApi, Algorithm: PowAlgorithm + Send + Sync, @@ -220,7 +226,7 @@ impl, C, Algorithm> Verifier for PowVerifier( + let (checked_header, difficulty, seal) = self.check_header( header, BlockId::Hash(parent_hash), )?; @@ -278,6 +284,7 @@ pub fn import_queue( block_import: BoxBlockImport, client: Arc, algorithm: Algorithm, + check_inherents_after: <::Header as HeaderT>::Number, inherent_data_providers: InherentDataProviders, ) -> Result, consensus_common::Error> where B: BlockT, @@ -288,11 +295,12 @@ pub fn import_queue( { register_pow_inherent_data_provider(&inherent_data_providers)?; - let verifier = PowVerifier { - client: client.clone(), + let verifier = PowVerifier::new( + client.clone(), algorithm, + check_inherents_after, inherent_data_providers, - }; + ); Ok(BasicQueue::new( verifier, From 23bbb5348e66f65f592c92c151e419f3c6fc788c Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 23 Sep 2019 15:13:40 +0800 Subject: [PATCH 20/27] Move difficulty fetch function out of loop To make things faster --- core/consensus/pow/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 42bb02a918c1c..6d045240cd367 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -411,11 +411,11 @@ fn mine_loop, C, Algorithm, E, SO>( let (header, body) = block.deconstruct(); let seed = H256::random(); let (difficulty, seal) = { - loop { - let difficulty = algorithm.difficulty( - &BlockId::Hash(best_hash), - )?; + let difficulty = algorithm.difficulty( + &BlockId::Hash(best_hash), + )?; + loop { let seal = algorithm.mine( &BlockId::Hash(best_hash), &header.hash(), From b8745ba42f70d88d8be264bc1a46e9297fb68bbf Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 24 Sep 2019 08:13:08 +0800 Subject: [PATCH 21/27] Remove seed from mining Each engine can use its own Rng source. --- core/consensus/pow/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 6d045240cd367..93ea670bae41e 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -104,7 +104,6 @@ pub trait PowAlgorithm { &self, parent: &BlockId, pre_hash: &H256, - seed: &H256, difficulty: Self::Difficulty, round: u32, ) -> Result, String>; @@ -409,7 +408,6 @@ fn mine_loop, C, Algorithm, E, SO>( )).map_err(|e| format!("Block proposing error: {:?}", e))?; let (header, body) = block.deconstruct(); - let seed = H256::random(); let (difficulty, seal) = { let difficulty = algorithm.difficulty( &BlockId::Hash(best_hash), @@ -419,7 +417,6 @@ fn mine_loop, C, Algorithm, E, SO>( let seal = algorithm.mine( &BlockId::Hash(best_hash), &header.hash(), - &seed, difficulty, round, )?; From bf26edf0022242a77e044da23e5636d840dd9414 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 24 Sep 2019 08:29:24 +0800 Subject: [PATCH 22/27] Better comments --- core/consensus/pow/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index 93ea670bae41e..bf6ce568c209f 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -63,9 +63,9 @@ fn aux_key(hash: &H256) -> Vec { /// Auxiliary storage data for PoW. #[derive(Encode, Decode, Clone, Debug, Default)] pub struct PowAux { - /// Difficulty. + /// Difficulty of the current block. pub difficulty: Difficulty, - /// Total difficulty. + /// Total difficulty up to current block. pub total_difficulty: Difficulty, } From d94fe48ae53dee57a53feecd4f14941c4f7010d3 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 24 Sep 2019 08:36:51 +0800 Subject: [PATCH 23/27] Add TotalDifficulty definition for U256 and u128 --- core/consensus/pow/primitives/src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs index e551d060129dc..bb0b9cc964a9d 100644 --- a/core/consensus/pow/primitives/src/lib.rs +++ b/core/consensus/pow/primitives/src/lib.rs @@ -20,7 +20,7 @@ use rstd::vec::Vec; use sr_primitives::ConsensusEngineId; -use codec::{Encode, Decode}; +use codec::Decode; use substrate_client::decl_runtime_apis; /// The `ConsensusEngineId` of PoW. @@ -34,6 +34,20 @@ pub trait TotalDifficulty { fn add(&mut self, other: Self); } +impl TotalDifficulty for primitives::U256 { + fn add(&mut self, other: Self) { + let ret = self.saturating_add(other); + *self = ret; + } +} + +impl TotalDifficulty for u128 { + fn add(&mut self, other: Self) { + let ret = self.saturating_add(other); + *self = ret; + } +} + decl_runtime_apis! { /// API necessary for timestamp-based difficulty adjustment algorithms. pub trait TimestampApi { From 17199059b005fa686c0e2f50f275dedfbde94439 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 3 Oct 2019 09:42:28 +0800 Subject: [PATCH 24/27] Update core/consensus/pow/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: André Silva --- core/consensus/pow/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index bf6ce568c209f..e7db0141ba6fb 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -443,6 +443,7 @@ fn mine_loop, C, Algorithm, E, SO>( let best_hash = client.info().best_hash; let best_aux = PowAux::::read(client, &best_hash)?; + // if the best block has changed in the meantime drop our proposal if best_aux.total_difficulty > aux.total_difficulty { continue 'outer } From 96d6a7e8852f99b3aa49cdd6c0bd808fb7f2abde Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Wed, 2 Oct 2019 18:48:47 -0700 Subject: [PATCH 25/27] Rename TotalDifficulty::add -> increment --- core/consensus/pow/primitives/src/lib.rs | 6 +++--- core/consensus/pow/src/lib.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs index bb0b9cc964a9d..2079b1cbe7e88 100644 --- a/core/consensus/pow/primitives/src/lib.rs +++ b/core/consensus/pow/primitives/src/lib.rs @@ -31,18 +31,18 @@ pub type Seal = Vec; /// Define methods that total difficulty should implement. pub trait TotalDifficulty { - fn add(&mut self, other: Self); + fn increment(&mut self, other: Self); } impl TotalDifficulty for primitives::U256 { - fn add(&mut self, other: Self) { + fn increment(&mut self, other: Self) { let ret = self.saturating_add(other); *self = ret; } } impl TotalDifficulty for u128 { - fn add(&mut self, other: Self) { + fn increment(&mut self, other: Self) { let ret = self.saturating_add(other); *self = ret; } diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index e7db0141ba6fb..d2b1aad772833 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -230,7 +230,7 @@ impl, C, Algorithm> Verifier for PowVerifier, C, Algorithm, E, SO>( }; aux.difficulty = difficulty; - aux.total_difficulty.add(difficulty); + aux.total_difficulty.increment(difficulty); let hash = { let mut header = header.clone(); header.digest_mut().push(DigestItem::Seal(POW_ENGINE_ID, seal.clone())); From bf7610253098b6f583d8756ee8407553d47dd2c1 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Wed, 2 Oct 2019 19:19:34 -0700 Subject: [PATCH 26/27] Use SelectChain to fetch the best header/hash --- core/consensus/pow/src/lib.rs | 59 +++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs index d2b1aad772833..7f282e20523bf 100644 --- a/core/consensus/pow/src/lib.rs +++ b/core/consensus/pow/src/lib.rs @@ -46,6 +46,7 @@ use primitives::H256; use inherents::{InherentDataProviders, InherentData}; use consensus_common::{ BlockImportParams, BlockOrigin, ForkChoiceStrategy, SyncOracle, Environment, Proposer, + SelectChain, }; use consensus_common::import_queue::{BoxBlockImport, BasicQueue, Verifier}; use codec::{Encode, Decode}; @@ -110,21 +111,23 @@ pub trait PowAlgorithm { } /// A verifier for PoW blocks. -pub struct PowVerifier, C, Algorithm> { +pub struct PowVerifier, C, S, Algorithm> { client: Arc, algorithm: Algorithm, inherent_data_providers: inherents::InherentDataProviders, + select_chain: Option, check_inherents_after: <::Header as HeaderT>::Number, } -impl, C, Algorithm> PowVerifier { +impl, C, S, Algorithm> PowVerifier { pub fn new( client: Arc, algorithm: Algorithm, check_inherents_after: <::Header as HeaderT>::Number, + select_chain: Option, inherent_data_providers: inherents::InherentDataProviders, ) -> Self { - Self { client, algorithm, inherent_data_providers, check_inherents_after } + Self { client, algorithm, inherent_data_providers, select_chain, check_inherents_after } } fn check_header( @@ -203,9 +206,10 @@ impl, C, Algorithm> PowVerifier { } } -impl, C, Algorithm> Verifier for PowVerifier where +impl, C, S, Algorithm> Verifier for PowVerifier where C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + ProvideCache + BlockOf, C::Api: BlockBuilderApi, + S: SelectChain, Algorithm: PowAlgorithm + Send + Sync, { fn verify( @@ -219,7 +223,12 @@ impl, C, Algorithm> Verifier for PowVerifier select_chain.best_chain() + .map_err(|e| format!("Fetch best chain failed via select chain: {:?}", e))? + .hash(), + None => self.client.info().best_hash, + }; let hash = header.hash(); let parent_hash = *header.parent_hash(); let best_aux = PowAux::read(self.client.as_ref(), &best_hash)?; @@ -279,11 +288,12 @@ pub fn register_pow_inherent_data_provider( pub type PowImportQueue = BasicQueue; /// Import queue for PoW engine. -pub fn import_queue( +pub fn import_queue( block_import: BoxBlockImport, client: Arc, algorithm: Algorithm, check_inherents_after: <::Header as HeaderT>::Number, + select_chain: Option, inherent_data_providers: InherentDataProviders, ) -> Result, consensus_common::Error> where B: BlockT, @@ -291,6 +301,7 @@ pub fn import_queue( C: Send + Sync + AuxStore + 'static, C::Api: BlockBuilderApi, Algorithm: PowAlgorithm + Send + Sync + 'static, + S: SelectChain + 'static, { register_pow_inherent_data_provider(&inherent_data_providers)?; @@ -298,6 +309,7 @@ pub fn import_queue( client.clone(), algorithm, check_inherents_after, + select_chain, inherent_data_providers, ); @@ -319,7 +331,7 @@ pub fn import_queue( /// information, or just be a graffiti. `round` is for number of rounds the /// CPU miner runs each time. This parameter should be tweaked so that each /// mining round is within sub-second time. -pub fn start_mine, C, Algorithm, E, SO>( +pub fn start_mine, C, Algorithm, E, SO, S>( mut block_import: BoxBlockImport, client: Arc, algorithm: Algorithm, @@ -328,6 +340,7 @@ pub fn start_mine, C, Algorithm, E, SO>( round: u32, mut sync_oracle: SO, build_time: std::time::Duration, + select_chain: Option, inherent_data_providers: inherents::InherentDataProviders, ) where C: HeaderBackend + AuxStore + 'static, @@ -335,6 +348,7 @@ pub fn start_mine, C, Algorithm, E, SO>( E: Environment + Send + Sync + 'static, E::Error: std::fmt::Debug, SO: SyncOracle + Send + Sync + 'static, + S: SelectChain + 'static, { if let Err(_) = register_pow_inherent_data_provider(&inherent_data_providers) { warn!("Registering inherent data provider for timestamp failed"); @@ -351,6 +365,7 @@ pub fn start_mine, C, Algorithm, E, SO>( round, &mut sync_oracle, build_time.clone(), + select_chain.as_ref(), &inherent_data_providers ) { Ok(()) => (), @@ -364,7 +379,7 @@ pub fn start_mine, C, Algorithm, E, SO>( }); } -fn mine_loop, C, Algorithm, E, SO>( +fn mine_loop, C, Algorithm, E, SO, S>( block_import: &mut BoxBlockImport, client: &C, algorithm: &Algorithm, @@ -373,6 +388,7 @@ fn mine_loop, C, Algorithm, E, SO>( round: u32, sync_oracle: &mut SO, build_time: std::time::Duration, + select_chain: Option<&S>, inherent_data_providers: &inherents::InherentDataProviders, ) -> Result<(), String> where C: HeaderBackend + AuxStore, @@ -380,6 +396,7 @@ fn mine_loop, C, Algorithm, E, SO>( E: Environment, E::Error: std::fmt::Debug, SO: SyncOracle, + S: SelectChain, { 'outer: loop { if sync_oracle.is_major_syncing() { @@ -388,10 +405,21 @@ fn mine_loop, C, Algorithm, E, SO>( continue 'outer } - let best_hash = client.info().best_hash; - let best_header = client.header(BlockId::Hash(best_hash)) - .map_err(|e| format!("Fetching best header failed: {:?}", e))? - .ok_or("Best header does not exist")?; + let (best_hash, best_header) = match select_chain { + Some(select_chain) => { + let header = select_chain.best_chain() + .map_err(|e| format!("Fetching best header failed using select chain: {:?}", e))?; + let hash = header.hash(); + (hash, header) + }, + None => { + let hash = client.info().best_hash; + let header = client.header(BlockId::Hash(hash)) + .map_err(|e| format!("Fetching best header failed: {:?}", e))? + .ok_or("Best header does not exist")?; + (hash, header) + }, + }; let mut aux = PowAux::read(client, &best_hash)?; let mut proposer = env.init(&best_header).map_err(|e| format!("{:?}", e))?; @@ -440,7 +468,12 @@ fn mine_loop, C, Algorithm, E, SO>( }; let key = aux_key(&hash); - let best_hash = client.info().best_hash; + let best_hash = match select_chain { + Some(select_chain) => select_chain.best_chain() + .map_err(|e| format!("Fetch best hash failed via select chain: {:?}", e))? + .hash(), + None => client.info().best_hash, + }; let best_aux = PowAux::::read(client, &best_hash)?; // if the best block has changed in the meantime drop our proposal From 7229cd04332b4a0524a220d9403d126cd7bbf89b Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Wed, 2 Oct 2019 19:51:15 -0700 Subject: [PATCH 27/27] Update lock file --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 53e724b73346c..70433417519d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4861,7 +4861,7 @@ dependencies = [ name = "substrate-consensus-pow-primitives" version = "2.0.0" dependencies = [ - "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 2.0.0", "sr-std 2.0.0", "substrate-client 2.0.0",