From 1ec0ba8633597c49d617760045293d2906724435 Mon Sep 17 00:00:00 2001 From: Guillaume Thiolliere Date: Fri, 31 Jul 2020 10:29:39 +0200 Subject: [PATCH 01/32] Fix link (#6775) --- frame/staking/reward-curve/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/reward-curve/src/lib.rs b/frame/staking/reward-curve/src/lib.rs index 9b55b346d5fd1..275669fe26b3b 100644 --- a/frame/staking/reward-curve/src/lib.rs +++ b/frame/staking/reward-curve/src/lib.rs @@ -29,7 +29,7 @@ use syn::parse::{Parse, ParseStream}; /// Accepts a number of expressions to create a instance of PiecewiseLinear which represents the /// NPoS curve (as detailed -/// [here](http://research.web3.foundation/en/latest/polkadot/Token%20Economics/#inflation-model)) +/// [here](https://research.web3.foundation/en/latest/polkadot/Token%20Economics.html#inflation-model)) /// for those parameters. Parameters are: /// - `min_inflation`: the minimal amount to be rewarded between validators, expressed as a fraction /// of total issuance. Known as `I_0` in the literature. From 9ae3a1ce15dfaaf4187152d90f39d7be66b51bbf Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 31 Jul 2020 14:32:13 +0200 Subject: [PATCH 02/32] Allow blacklisting blocks from being finalized again after block revert (#6301) * Allow blacklisting blocks from being finalized again after block revert * Use BlockRules for storing unfinalized and add have_state_at in revert * Move finalization_check in finalize_block upward * Directly mark finalization blacklist as badblocks * Remove obselete comment --- Cargo.lock | 1 + client/api/src/backend.rs | 18 +++-- client/api/src/in_mem.rs | 11 ++-- client/db/Cargo.toml | 1 + client/db/src/lib.rs | 69 ++++++++++++++------ client/light/src/backend.rs | 9 ++- client/service/src/chain_ops/revert_chain.rs | 4 +- client/service/src/client/block_rules.rs | 9 ++- client/service/src/client/client.rs | 33 ++++++---- 9 files changed, 105 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 525ea78d9d4e2..b86507cf072f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6309,6 +6309,7 @@ dependencies = [ "sc-client-api", "sc-executor", "sc-state-db", + "sp-arithmetic", "sp-blockchain", "sp-consensus", "sp-core", diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 9482a6118d71a..efc5ca4ee8ca0 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -19,7 +19,7 @@ //! Substrate Client data backend use std::sync::Arc; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use sp_core::ChangesTrieConfigurationRange; use sp_core::offchain::{OffchainStorage,storage::OffchainOverlayedChanges}; use sp_runtime::{generic::BlockId, Justification, Storage}; @@ -418,7 +418,10 @@ pub trait Backend: AuxStore + Send + Sync { ) -> sp_blockchain::Result<()>; /// Commit block insertion. - fn commit_operation(&self, transaction: Self::BlockImportOperation) -> sp_blockchain::Result<()>; + fn commit_operation( + &self, + transaction: Self::BlockImportOperation, + ) -> sp_blockchain::Result<()>; /// Finalize block with given Id. /// @@ -449,16 +452,17 @@ pub trait Backend: AuxStore + Send + Sync { /// Returns state backend with post-state of given block. fn state_at(&self, block: BlockId) -> sp_blockchain::Result; - /// Attempts to revert the chain by `n` blocks. If `revert_finalized` is set - /// it will attempt to revert past any finalized block, this is unsafe and - /// can potentially leave the node in an inconsistent state. + /// Attempts to revert the chain by `n` blocks. If `revert_finalized` is set it will attempt to + /// revert past any finalized block, this is unsafe and can potentially leave the node in an + /// inconsistent state. /// - /// Returns the number of blocks that were successfully reverted. + /// Returns the number of blocks that were successfully reverted and the list of finalized + /// blocks that has been reverted. fn revert( &self, n: NumberFor, revert_finalized: bool, - ) -> sp_blockchain::Result>; + ) -> sp_blockchain::Result<(NumberFor, HashSet)>; /// Insert auxiliary data into key-value store. fn insert_aux< diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 7d27326678f58..306c3c2b2f10c 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -18,7 +18,7 @@ //! In memory client backend -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ptr; use std::sync::Arc; use parking_lot::RwLock; @@ -646,7 +646,10 @@ impl backend::Backend for Backend where Block::Hash Ok(()) } - fn commit_operation(&self, operation: Self::BlockImportOperation) -> sp_blockchain::Result<()> { + fn commit_operation( + &self, + operation: Self::BlockImportOperation, + ) -> sp_blockchain::Result<()> { if !operation.finalized_blocks.is_empty() { for (block, justification) in operation.finalized_blocks { self.blockchain.finalize_header(block, justification)?; @@ -722,8 +725,8 @@ impl backend::Backend for Backend where Block::Hash &self, _n: NumberFor, _revert_finalized: bool, - ) -> sp_blockchain::Result> { - Ok(Zero::zero()) + ) -> sp_blockchain::Result<(NumberFor, HashSet)> { + Ok((Zero::zero(), HashSet::new())) } fn get_import_lock(&self) -> &RwLock<()> { diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index c26f7121493cf..50e14fcaae602 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -24,6 +24,7 @@ codec = { package = "parity-scale-codec", version = "1.3.4", features = ["derive blake2-rfc = "0.2.18" sc-client-api = { version = "2.0.0-rc5", path = "../api" } +sp-arithmetic = { version = "2.0.0-rc5", path = "../../primitives/arithmetic" } sp-core = { version = "2.0.0-rc5", path = "../../primitives/core" } sp-runtime = { version = "2.0.0-rc5", path = "../../primitives/runtime" } sp-state-machine = { version = "0.8.0-rc5", path = "../../primitives/state-machine" } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 086db73728f1b..d854c80bf3535 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -50,8 +50,7 @@ mod subdb; use std::sync::Arc; use std::path::{Path, PathBuf}; use std::io; -use std::collections::HashMap; - +use std::collections::{HashMap, HashSet}; use sc_client_api::{ UsageInfo, MemoryInfo, IoInfo, MemorySize, @@ -70,6 +69,7 @@ use parking_lot::RwLock; use sp_core::ChangesTrieConfiguration; use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; +use sp_arithmetic::traits::Saturating; use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, @@ -962,6 +962,7 @@ impl Backend { // TODO: ensure best chain contains this block. let number = *header.number(); self.ensure_sequential_finalization(header, last_finalized)?; + self.note_finalized( transaction, false, @@ -1015,9 +1016,10 @@ impl Backend { Ok(()) } - fn try_commit_operation(&self, mut operation: BlockImportOperation) - -> ClientResult<()> - { + fn try_commit_operation( + &self, + mut operation: BlockImportOperation, + ) -> ClientResult<()> { let mut transaction = Transaction::new(); let mut finalization_displaced_leaves = None; @@ -1404,7 +1406,10 @@ impl sc_client_api::backend::Backend for Backend { Ok(()) } - fn commit_operation(&self, operation: Self::BlockImportOperation) -> ClientResult<()> { + fn commit_operation( + &self, + operation: Self::BlockImportOperation, + ) -> ClientResult<()> { let usage = operation.old_state.usage_info(); self.state_usage.merge_sm(usage); @@ -1420,9 +1425,11 @@ impl sc_client_api::backend::Backend for Backend { } } - fn finalize_block(&self, block: BlockId, justification: Option) - -> ClientResult<()> - { + fn finalize_block( + &self, + block: BlockId, + justification: Option, + ) -> ClientResult<()> { let mut transaction = Transaction::new(); let hash = self.blockchain.expect_block_hash_from_id(&block)?; let header = self.blockchain.expect_header(block)?; @@ -1488,7 +1495,13 @@ impl sc_client_api::backend::Backend for Backend { }) } - fn revert(&self, n: NumberFor, revert_finalized: bool) -> ClientResult> { + fn revert( + &self, + n: NumberFor, + revert_finalized: bool, + ) -> ClientResult<(NumberFor, HashSet)> { + let mut reverted_finalized = HashSet::new(); + let mut best_number = self.blockchain.info().best_number; let mut best_hash = self.blockchain.info().best_hash; @@ -1507,18 +1520,28 @@ impl sc_client_api::backend::Backend for Backend { return Ok(c.saturated_into::>()) } let mut transaction = Transaction::new(); + let removed_number = best_number; + let removed = self.blockchain.header(BlockId::Number(best_number))?.ok_or_else( + || sp_blockchain::Error::UnknownBlock( + format!("Error reverting to {}. Block hash not found.", best_number)))?; + let removed_hash = removed.hash(); + + let prev_number = best_number.saturating_sub(One::one()); + let prev_hash = self.blockchain.hash(prev_number)?.ok_or_else( + || sp_blockchain::Error::UnknownBlock( + format!("Error reverting to {}. Block hash not found.", best_number)) + )?; + + if !self.have_state_at(&prev_hash, prev_number) { + return Ok(c.saturated_into::>()) + } + match self.storage.state_db.revert_one() { Some(commit) => { apply_state_commit(&mut transaction, commit); - let removed_number = best_number; - let removed = self.blockchain.header(BlockId::Number(best_number))?.ok_or_else( - || sp_blockchain::Error::UnknownBlock( - format!("Error reverting to {}. Block hash not found.", best_number)))?; - best_number -= One::one(); // prev block - best_hash = self.blockchain.hash(best_number)?.ok_or_else( - || sp_blockchain::Error::UnknownBlock( - format!("Error reverting to {}. Block hash not found.", best_number)))?; + best_number = prev_number; + best_hash = prev_hash; let update_finalized = best_number < finalized; @@ -1531,7 +1554,12 @@ impl sc_client_api::backend::Backend for Backend { ), )?; if update_finalized { - transaction.set_from_vec(columns::META, meta_keys::FINALIZED_BLOCK, key.clone()); + transaction.set_from_vec( + columns::META, + meta_keys::FINALIZED_BLOCK, + key.clone() + ); + reverted_finalized.insert(removed_hash); } transaction.set_from_vec(columns::META, meta_keys::BEST_BLOCK, key); transaction.remove(columns::KEY_LOOKUP, removed.hash().as_ref()); @@ -1562,7 +1590,7 @@ impl sc_client_api::backend::Backend for Backend { revert_leaves()?; - Ok(reverted) + Ok((reverted, reverted_finalized)) } fn blockchain(&self) -> &BlockchainDb { @@ -1986,7 +2014,6 @@ pub(crate) mod tests { backend.commit_operation(op).unwrap(); - assert!(backend.storage.db.get( columns::STATE, &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 2cf994d3f5993..be7953e528bd8 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -19,7 +19,7 @@ //! Light client backend. Only stores headers and justifications of blocks. //! Everything else is requested from full nodes on demand. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use parking_lot::RwLock; @@ -146,7 +146,10 @@ impl ClientBackend for Backend> Ok(()) } - fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> ClientResult<()> { + fn commit_operation( + &self, + mut operation: Self::BlockImportOperation, + ) -> ClientResult<()> { if !operation.finalized_blocks.is_empty() { for block in operation.finalized_blocks { self.blockchain.storage().finalize_header(block)?; @@ -231,7 +234,7 @@ impl ClientBackend for Backend> &self, _n: NumberFor, _revert_finalized: bool, - ) -> ClientResult> { + ) -> ClientResult<(NumberFor, HashSet)> { Err(ClientError::NotAvailableOnLightClient) } diff --git a/client/service/src/chain_ops/revert_chain.rs b/client/service/src/chain_ops/revert_chain.rs index 129aea0408685..eaee2c03f9b31 100644 --- a/client/service/src/chain_ops/revert_chain.rs +++ b/client/service/src/chain_ops/revert_chain.rs @@ -34,10 +34,10 @@ where let reverted = backend.revert(blocks, false)?; let info = client.usage_info().chain; - if reverted.is_zero() { + if reverted.0.is_zero() { info!("There aren't any non-finalized blocks to revert."); } else { - info!("Reverted {} blocks. Best: #{} ({})", reverted, info.best_number, info.best_hash); + info!("Reverted {} blocks. Best: #{} ({})", reverted.0, info.best_number, info.best_hash); } Ok(()) } diff --git a/client/service/src/client/block_rules.rs b/client/service/src/client/block_rules.rs index e862379a56414..be84614c2a590 100644 --- a/client/service/src/client/block_rules.rs +++ b/client/service/src/client/block_rules.rs @@ -30,7 +30,7 @@ use sc_client_api::{ForkBlocks, BadBlocks}; pub enum LookupResult { /// Specification rules do not contain any special rules about this block NotSpecial, - /// The bock is known to be bad and should not be imported + /// The block is known to be bad and should not be imported KnownBad, /// There is a specified canonical block hash for the given height Expected(B::Hash) @@ -57,6 +57,11 @@ impl BlockRules { } } + /// Mark a new block as bad. + pub fn mark_bad(&mut self, hash: B::Hash) { + self.bad.insert(hash); + } + /// Check if there's any rule affecting the given block. pub fn lookup(&self, number: NumberFor, hash: &B::Hash) -> LookupResult { if let Some(hash_for_height) = self.forks.get(&number) { @@ -66,7 +71,7 @@ impl BlockRules { } if self.bad.contains(hash) { - return LookupResult::KnownBad; + return LookupResult::KnownBad } LookupResult::NotSpecial diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index b152415a4a89d..d0859f4ee0392 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1054,20 +1054,31 @@ impl Client where /// reverted past the last finalized block. Returns the number of blocks /// that were successfully reverted. pub fn revert(&self, n: NumberFor) -> sp_blockchain::Result> { - Ok(self.backend.revert(n, false)?) + let (number, _) = self.backend.revert(n, false)?; + Ok(number) } - /// Attempts to revert the chain by `n` blocks disregarding finality. This - /// method will revert any finalized blocks as requested and can potentially - /// leave the node in an inconsistent state. Other modules in the system that - /// persist data and that rely on finality (e.g. consensus parts) will be - /// unaffected by the revert. Use this method with caution and making sure - /// that no other data needs to be reverted for consistency aside from the - /// block data. + /// Attempts to revert the chain by `n` blocks disregarding finality. This method will revert + /// any finalized blocks as requested and can potentially leave the node in an inconsistent + /// state. Other modules in the system that persist data and that rely on finality + /// (e.g. consensus parts) will be unaffected by the revert. Use this method with caution and + /// making sure that no other data needs to be reverted for consistency aside from the block + /// data. If `blacklist` is set to true, will also blacklist reverted blocks from finalizing + /// again. The blacklist is reset upon client restart. /// /// Returns the number of blocks that were successfully reverted. - pub fn unsafe_revert(&self, n: NumberFor) -> sp_blockchain::Result> { - Ok(self.backend.revert(n, true)?) + pub fn unsafe_revert( + &mut self, + n: NumberFor, + blacklist: bool, + ) -> sp_blockchain::Result> { + let (number, reverted) = self.backend.revert(n, true)?; + if blacklist { + for b in reverted { + self.block_rules.mark_bad(b); + } + } + Ok(number) } /// Get blockchain info. @@ -1921,7 +1932,7 @@ impl BlockBackend for Client fn block_hash(&self, number: NumberFor) -> sp_blockchain::Result> { self.backend.blockchain().hash(number) - } + } } impl backend::AuxStore for Client From 64543a310e65164b6e0185612d61fe0c9d085b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 31 Jul 2020 14:58:38 +0200 Subject: [PATCH 03/32] Order delta before calculating the storage root (#6780) We need to order the delta before calculating the storage root, because the order is important if the storage root is calculated using a storage proof. The problem is arises when the delta is different than at the time the storage root was recorded, because we may require a different node that is not part of the proof and so, the storage root can not be calculated. The problem is solved by always order the delta to use the same order when calculating the storage root while recording the stroage proof and when calculating the storage root using the storage proof. To prevent this bug in future again, a regression test is added. Fixes: https://github.com/paritytech/cumulus/issues/146 --- primitives/trie/src/lib.rs | 54 ++++++++++++++----- primitives/trie/test-res/invalid-delta-order | Bin 0 -> 3476 bytes primitives/trie/test-res/proof | Bin 0 -> 4680 bytes primitives/trie/test-res/storage_root | 1 + primitives/trie/test-res/valid-delta-order | Bin 0 -> 3476 bytes 5 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 primitives/trie/test-res/invalid-delta-order create mode 100644 primitives/trie/test-res/proof create mode 100644 primitives/trie/test-res/storage_root create mode 100644 primitives/trie/test-res/valid-delta-order diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 5ab06cecca657..73a4a8029b2d7 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -186,6 +186,9 @@ pub fn delta_trie_root( { let mut trie = TrieDBMut::::from_existing(&mut *db, &mut root)?; + let mut delta = delta.into_iter().collect::>(); + delta.sort_by(|l, r| l.0.borrow().cmp(r.0.borrow())); + for (key, change) in delta { match change.borrow() { Some(val) => trie.insert(key.borrow(), val.borrow())?, @@ -259,19 +262,12 @@ pub fn child_delta_trie_root( // root is fetched from DB, not writable by runtime, so it's always valid. root.as_mut().copy_from_slice(root_data.as_ref()); - { - let mut db = KeySpacedDBMut::new(&mut *db, keyspace); - let mut trie = TrieDBMut::::from_existing(&mut db, &mut root)?; - - for (key, change) in delta { - match change.borrow() { - Some(val) => trie.insert(key.borrow(), val.borrow())?, - None => trie.remove(key.borrow())?, - }; - } - } - - Ok(root) + let mut db = KeySpacedDBMut::new(&mut *db, keyspace); + delta_trie_root::( + &mut db, + root, + delta, + ) } /// Call `f` for all keys in a child trie. @@ -468,7 +464,7 @@ mod trie_constants { #[cfg(test)] mod tests { use super::*; - use codec::{Encode, Compact}; + use codec::{Encode, Decode, Compact}; use sp_core::Blake2Hasher; use hash_db::{HashDB, Hasher}; use trie_db::{DBValue, TrieMut, Trie, NodeCodec as NodeCodecT}; @@ -856,4 +852,34 @@ mod tests { ).is_err() ); } + + #[test] + fn generate_storage_root_with_proof_works_independently_from_the_delta_order() { + let proof = StorageProof::decode(&mut &include_bytes!("../test-res/proof")[..]).unwrap(); + let storage_root = sp_core::H256::decode( + &mut &include_bytes!("../test-res/storage_root")[..], + ).unwrap(); + // Delta order that is "invalid" so that it would require a different proof. + let invalid_delta = Vec::<(Vec, Option>)>::decode( + &mut &include_bytes!("../test-res/invalid-delta-order")[..], + ).unwrap(); + // Delta order that is "valid" + let valid_delta = Vec::<(Vec, Option>)>::decode( + &mut &include_bytes!("../test-res/valid-delta-order")[..], + ).unwrap(); + + let proof_db = proof.into_memory_db::(); + let first_storage_root = delta_trie_root::( + &mut proof_db.clone(), + storage_root, + valid_delta, + ).unwrap(); + let second_storage_root = delta_trie_root::( + &mut proof_db.clone(), + storage_root, + invalid_delta, + ).unwrap(); + + assert_eq!(first_storage_root, second_storage_root); + } } diff --git a/primitives/trie/test-res/invalid-delta-order b/primitives/trie/test-res/invalid-delta-order new file mode 100644 index 0000000000000000000000000000000000000000..e46f280dc29bb2cbc3a28faeabe51efc747ba4af GIT binary patch literal 3476 zcmai1XHXN^7AAm-)PR6AS(GBurAP^#rT0Z6NDaLOLP9S~K&n9yk>2ZuDlHhQU|1kj zl_o_>XoB<-K)^h=%+7m{AAA1XbIyGCd^2~>o%z0pwE`9LMjK#>g;1Ru+Enx5;}UI3 zjO#~M7a8{_S7;eywQ{zqrw(Q)Mefj$k&z`RIQT*RJpB;1{t!<$M~JVRx0kK6pO?d* z2@C>*z;ALCWXS&~kghx$ke=Yv2ppQ(ON`1*1E}`%ydeKE{GUJ~ zffI526*R_;p{Ks7>EY?P=)D*Q9&;^8HAvC^0DhPZxr=kcjaW?he1%Wb2dmxaZv8;j z9iQoVeC^`(S)_xV(`8!fjEjz()!)r!AA8#MzLqlq04aVDswy}I`oi735N-~(ZeET~ zftN?;|8#}r(0*tnYGH4J()zp=z|;KN4V)`hgVhfYlc0=g*P0)A@sBj{m&5A1nX@ceI5%ArKH3CxpV~ z%V=n6xumNxd#e0X?}3Ck@5_iMYyY+SHexI$*?!Gr+uf|`IpJ(0v+R}a#ycW4d0v&nt5J;uJV2$s4{?^#5a z6GdQeKNUrF6PL?a-^MK-v^2{0Zr+aLPZJdr{*G!bw4Ho8WIs#UaJQd12$$T%#l(wm zrs@^FkPhuImXmGx7)^`004Gf_4n5gHhI|t*RQ5kWhyC2%2Z@I-uKi4z&g_R@VBgrppi_B%SJmi zKen-x2~W(^qzl(3J2vZ&vKx_l(OTuQmJR?F+qj<|?su=tsXRy(2(Q0`7Z+dkXbhcF zO550ENn5+h!y%)EEg3f^a%oI@3acuIMzwY+KZx&}IL}G9_~fRiH^O4nl_f}}k!ux)si}7XtvNF_A6q)UWa{zq_DEMnowpRQv)etcwvJO1{q~?;B0F!< z@Y;<~HUMz#iU6&V&xL9-@eN+g7K}Sy(za1{$lRF-2$!4IwjItuUi4?&M;eDIAO8|3m(bXrE+&`||Qz(Q! zW{@W;hVv?fvSc%_+Z%SE<4wbQvU{6$gGM%Gz#Y9E&wcws-`)Mn-w5(H6mNd)Tx_!z3S6 z&38$Jt|re#=jwY6w?ADq_qxcJMzJxNoEQVfjnS46e(qokgN z&Ctayv^{X0jgH!!qt=GTajCa!Dk-67fr~Ve7R9U~e#`aN2FLieX2irKy&{&oxX{{z zw~nC)wKep~@!-Uj*Lm_rZDyq0OoPpy8EKsMQ+|iU>c`zVt+y#{TTwZt4mZ%&ceiG1 zV7l5XOlb-oKF0RqE2$+@+k_&N)Ig#Yv)h|Cv$ppZjykA8?TdZ$>ZYNX*Pz<1@Xsr^ zXcp5Y)py=c6X?6B()HviSUnH*qB8>*5cKmk39A6}uJX9E4Az8g!*=sPB+anNipf21 zm!N@g##Q;zObST7d*64(e1Z@~GOCyOu3$)o(fNc1&h?D+r3F+-KZg0RJzkIzw~C|{ z0&N^{Hfr4vws_95CY<_K7yCZ+%@KLKT+(M2TQgH(olI<6yoI3V$WDXiwq4sj*{mek z$HSFrLtpAQ10pAwZ)O|@fkpV-MJIsO*ke}iUY|mv%9e|%Vm{WdqwT4)HvH)&PQ@Ym z^C4dxev1%=vd2!YD^A}w^^!2xTV7r80yHiz6VJ->ToJ`t>NaSQ!<5ihrS5guxq>G= z2**u`b=KxNgRr4|bSniS!iLd|^3yst14VT)3A%!Vzvu3K%i!ImSk$#ZvbXZ7W9+-I1TJ}a$uK^0 zk`nHkNU9OeRd+LBKljU*is!#)=g45Y&4QLQdnr=B2D6&3GKJjp=ssC7^$mTIrYg95 zN}OrjVRLh9uH>l_F;|Ba1PV*PxKaxacR|}2G~4bGi8qc;n;Jv}sz((8?%+)|^0LoG z^%g4M)<3u}N~U5KZD%I2uPgOg;>JP<9jl&$8ip%*zK3}N&Sh2E{CL$(O#gE~{fP!5 zeM>!K?R{HQI*RJNbdi1+`;aaE@#Mbe>~7z6tP)4EY32n^XP@?F_9%HRxU{4*@~@Ga zXin-(v{`Tm*D`z*2FaxdM^CBYyBZTy`>4bqB{QK+Y@61c*r{6(@H6vxY4LoAv0Vir zr}QU}&gbFD7@?gum^yBYGRIjf0QqDwiVlMT(~C$|G}SuHfny_KiFATZ|o+5wG7(LUkaJoZT)LSxK)t>R=-l zx#S`LSLx3J8L46lnTCw8kL$1itR_&U-&2w~6hKoqGoP2ogB#D^cszY`T5EY+Jan3)&PLh~hn0Fr>0ycbp2L8K~X1*b$BOorX|zr(=2GRm2}P_xQZ z*R18*LaDB*s(-_nBQ1*olD?EX<}lu$2-!LV6&CMxzWPTXqphQNfY<+u zY5?#L(Rs|l-Fa`g4 X{$X-{*oOa#KRSO|AL_rmKdS!#%te!7 literal 0 HcmV?d00001 diff --git a/primitives/trie/test-res/proof b/primitives/trie/test-res/proof new file mode 100644 index 0000000000000000000000000000000000000000..278bea0bfbbc7fb16364eddbded70c884dd7ab71 GIT binary patch literal 4680 zcma)9WmMFSvt|JSSxRCB1eAuQq@=r3ctJvP>0FR*el)m*tJ1lYwAA8~N_QyTAt5c& zBJSV!z4xB;-uvm!r)TCoGiT16IdkUGP9n<3SHnj;eStWNx;(!_axYF5iv^U+3yET? zXX}o8(}ljar*9z^a61u|H@RLPNT5v5-Le3dvcC?h7U^bIj6U;!NfS*8Rw{0yu{&?k z(XqQKxYC^kES{;j&A*hQt{tVgAsa84l_F6!PGNZc^9Gz+Lj0ti5O^lKv9MSSk)K-EQc)1#3%WCT>~{L#1$KF(A8O7j~Pe8>OEZFO~%NF;Z3&X>j$a_ z^uYj{Z;pmqk-yjYTbovLHNY$D>8b)~Qi`zJewC=8k~|h*i(8f>zgJ;EW{w+ynW?aN zC;L!>@ZiVz7wKZjR#bZwU}KmA=VTXW>CB^f7B69|$jy~21DWLrgr-XDWEx*b0zRFe zkzi@lC#kr(eh8b=;6x{jSTT5Vm8#8=MCg{_Vnk=1$;yw%TbzfHN zUYe3N1@pFg-vUun#&;PjYzr6of*9$T1u!-huTo5Re6M-Od_@`f)yn|v@g%lYDsM%f zw#!`ldF@9Msqx3g5)~b(t;kU2kW&$$tezlizvcr2>B29|IfT4G_n`IfN-2&xmE6E* zgzFwxfK?TZj6}kcL3DRZno?juH~Um%HFrr3Kkcobu!dMnIuIYlO3RkO#lUq+bJZ_V z>eme!t?QQ*bP+@0e8X}*QvebLn!>{kjt1IB6wCRPS?vzlLj0PXx@mUk3|MwfqSbP& z?vCsZ#H(i46)f$cxE=Yg3@NnzVczb1KK9-|e4a2bn5`4c8BPNNaH4@a=D5xbt%ni` zc2fjDh|E-vPoYFqC2;%+hwMb!IomPd4ldswOy{iHVv7ou!BOPIhU56MiZX;azVlJf zyF*409VpWF6Aob7JDqt$!-%M=N^yHWDJN2sF-@Xurno*`2^v>*2z%+(pMotP_ogiQNOY1r2-CcHsCI9J7w}SFyM1BoYX|- z>tXU41RTsBX`0G<6CcwBP(f=jY7Mn3vXr$oU(!8LX6DUZwXz((g?o>0TYz^wJz><2 z1!--w^kyz35zjV=DCGE8rB=lm_rD5O`lwc~F3IEkAB*<$^02k{_O`dPvh(nV`@_8K ztlafruJXv2vU=GPCS$wP*W4TZu#5n?&3ha8Exf`7+HaV0=gflFjZrNgH zp4e8|7)m*OJ>L(S%Op^$IKsuB*P;RayUBl=_;=v{i`nr2-V9)Ji}D4T_4ABgaWlrd z%jH9H!EtQUHC&y@H>S#ca~luO{{(5JsVufGaOb|Nhojr^JjesqYeV;wzq5 zHh5WwG9SHMP<=%y5C};A#lc|r_Iu>81}?l;g**=;cBG8Qh`~OCQpX3?BbEU)bRWq- z*qA0CHEq>cM%zR1DtQjf<`}z-5xh?|H!Jqa3+Co*2lH|EfLl5E!fpQnf3LK=HyaL4 z0Zv^0pP|rkhvSe!|9TUsrWZfCk%dg03h<(rIPeRe=0M)t)__8MJ)sF)Dxd&O-sJC2 zu2{ydG9}HM*&k&#Xq(1D&(tF_w?dqp=ncB|Yd%EJ#bznwWBmGQOQTr-0dqWi8XrH; zZ$vA~YWDT0%@TeC=MZ&ZH?oO=l6A0|sGsjvB&6f1xLCuN7!vfYQOeA{(dWlN@J|7h ztlw!_sB-%NECFg?lAj#@**Ejz;!Ete3OD7xO*<*MQ;OS03%~mpZR67k^&jpJXH3lQ3vZCk-ja_6PIO1L5oxQ~wNKsz@3t6~%N_a|M=ONCb_$iKqj0`U zuh3wGfA#>p)eI9yz00Zy9M42M?zOw;$2ESU9K6U=Aq`iqigCNrFI-Bn=j4mUQ_nL zJzUe}5x-gr=;*ACt1aRc_WH#h>vY`iVsjF3pD^< zAnO8Y!YC_GRkQ(bfxrvSPELXaH9&szqiz&sotE##(6a`73z$hwJB+RTzNG&JkylQ7 zfw=Dxw)kO5u4{CW7pWYg;XaV8lcS02e$m9*Y)M*eO0Qhpm149bsM<{*%}UO--; z6UK_mV^}|DZN5>?g*6=)t7Ryrd4<9Kd2A*>ufO0)@m8h`UkAHLL!i0lp6YO2N;Jro z*~N4vdeFxZu99eZs$b3-J6J#(b_|v9x$Ij?&0bGQwcuSR&LJfGt5X=7 z^(d8=j*zP$f`-@ImW!gIPsrHR>4p>tyK+aKY#vJF#21x?C1S;abh30GNAmPK7XkZv z-9-yRJB5P}EQ{KDO1R}mPaMOan1c@674no?+!x6%h@hx%XpU22p@dVuV;+vd26*}1 z-3J202CuooMe`fQPwULP8)uJQ6_5r}0~w`-gL=Xn2Hqab)SVpY+uP0oC~|08whsj@ zyCj#mkPc^+457)U-nG%`C`-@}?aS_O`W`rbAZpan-{Jmb$|rHNDzNHlo}4z_W!wgZ ze2}(_xw#SjjIO_dI~-I}^2_K=x3J8&^C|m0z87)~Vmqq(4BVF)fr9*;ljwxLUzr2V zZwq-*kT=ma%~?#~6d|vLUZ$omk7qjPqrS5B-BvsAU2k>ZODmL~?-X_C48j>p?yzOSf$eA9Lqouh9{Rc66? zxK!h*p|(YtCfV`gsg1x^YVrKhMj=9YIN6-q`9qsQ+k~l|I$}ia_J>Jz(`f8_soKNv z>w1VD7p-$b8Be!qoo)rCZr@J72^G*`I7rU9x0PFSaiV z)>;kzX8SIJ&z&xAc2|0VSs(t`M00a{4GwBt-@Ldf&2#cD%2Ko}ld_$M?#uM-dj67h z^9t-{+K6*ddvpV%MZD0c2BC#WsjF;j%4Z#GehNbZ46}C##bD&#i$MV7M5bN3a>f8>E>mifWxrhY;{rP) zJ=t)vGTnr{sk+@x8VCMAa=(46Nb1eb5ykWv1a4;b+IZI`KXH5L=jdHz$5bNPOVY6W0|FPv(?GP1=aa=Qv)wY^`tb&#ppl* zw{*F%-c;`A?q`>Ep;Yv`)#5DXeWeyn{8aEp$F|#ulFnA1&siRaLupku`=I;*dML?P zYo_6Xtfl^5t)P`YDPDD6I`5F9P4FT6G;$z`&iS{xdGdHRc$MRNDXKrSPX@l`*phya zeFxhFzN*txqv<{+%y=O=f-5pIc}?u~U1?_i2O>F8#sHd$X;WR{zjh9!Sf-vXDO&A# zYF)9BQ}WRDQ&Kn*4Lxr2RO~*)&v8)o2cEA-lcLc^WW2%^O|`ZwMsX2eFO$o8;;Y!9 z(_`fQolh%qALW%hm!O`r&=n{8X!^!=my)nG4Dh~x)w^OpoXwm#Qr@{PAR17N;X1Ns z{}Jt2mQAEozo>|3-?-aWnCZz@bkL+**62Y?gf#I9lBrPTp#k^?D%uXmtofF*>Fp$W zM}PC95tUC`5&TlIl5tmY?<0UgXp7%1#f9p5uIu&FOoy*A?_e*Yu8BMaM)~#kJGOLT zC;J4K#$PnvL=F;vCTLY?OI4{O`5L+F8rpT1chNZotY1= z@f^VzWlt`A+G~<`r>l+;IA>_}X-*QC4oZl5kQeHHVlp^zp%lQ zhaXRIMKded9$LbCfRWbdFoPZMq39jcX-N1#T`@jQS+VM%=N9|RPY6YOS5a(~WJw5K zM;xR}BaR&Pdz=0;;2u#digHvJV_b&;VN@vOhTMdxLqOoV#nrsLN8M9wmA}3ls|(pQ z(}SESzG_l!3a~`S1PS+NjAZ5|(15@BZ%94%DxTnzJoZkN&I)=HsXZIY?8idRv^Dtv zxmL4HS=X%U)Pk=lFRw-Qls+ws9G3nycgl8pC>eHmBZVqD`ScA?z%jH;7=NPjgXyf~ zzVeAITvvs{?$3t_Y`0HgQT-hx&w59yohw(A=b{8&9gYvs*gO=u4)NKWDaQl8F#tNh zLIhew<}_9yi-|L?8cXt(A<3OTN@De_)r08GAOP?wb2Z@_{sx#ZgCvKp7`iX^9{ntA zPTM-esP?h)5M=_=DoJ5bZc1#Jb#c1&#V*NjSJxWOo~#3LWHz_;2ag8e4*QTHVr45} z17g);mNdk(YU~=O^O2@^l1OhC%rLPB%rVn6MR8hIh0|nz`3u|g13E|adh8s7|Hf$l z+Ns|g3E(IF6x~Im@K!}!cCF+dw3>B--cxK;kl;QQGQK8hmg8*HS5z$L zC|e`6CXr#kL^Iy&pR5$Stw;SWBo%=ntZ=57EFpIs?c1VZF%({dvr8oZL!b=$2mPJ!za$pU-;w_+L;T;|Kb{0$UWJ72 zp3&pA5cL|8RFkpGyGbTi#2L38IorJ^V#aRP{qM!~DDeJd0HyOU$ev1Ty!}uw6^l8N z-@{k9uRT-#D?m+>T<(GGwGTk)Z6=L_r~!s+OE~vfNIr{8bid5rdaP6(ub3Ky7r+VN zzkX<8l!|Y=&;?St&1ZbiZuf95*T8r^Mq_ykjE3m|E1MsOBz-O>qkfMRofzU)KA^vb yYgqA(s(;A~q$hH=15Q*ugw?8oPG<89c_3;8`+0P>DR-FH&-xKw5*Z|1p#K0WpTDL6 literal 0 HcmV?d00001 diff --git a/primitives/trie/test-res/storage_root b/primitives/trie/test-res/storage_root new file mode 100644 index 0000000000000..758626802e15f --- /dev/null +++ b/primitives/trie/test-res/storage_root @@ -0,0 +1 @@ +‡=Ô[Á42%áJP ¢Áh¦Kwé)Rª 0¤Ô­u¿Ã \ No newline at end of file diff --git a/primitives/trie/test-res/valid-delta-order b/primitives/trie/test-res/valid-delta-order new file mode 100644 index 0000000000000000000000000000000000000000..4df6e62ec133b05cab2b53f00c8a4286eef70865 GIT binary patch literal 3476 zcma)2vS3DfsoM45s+#SM5On+p-KyeDi{_B zRi#Oh5}F{r1Q0Oi?%nQh@Az;xpJwL$XWqg;1Ru+Enx5WAf6j zx~w+^#%7+TS^c_CtH>@9(v?R8(i40dfkQKUiBY*}0M&k;7vyArlS?7UF>V{yD=toL zQIKdL96t$|rTr6U3jRa;Js$9<4Uq&L%V?@@@8*PcVuq%8d-8V@6j zIKggnzZ_&Nsh!?TP;l^r`g!^xZ2cjgZjKONH*YUnXFo58KNJ`Q4uK*7(qv=>WNZ8s zWPo4gDE^NMwyLKNW++AO(EMoZ)n6CD;NA{S2!xZPt)q87_3oI`%oKV)1WnQ>KDh}91!1c(>Z1O)1d zB<{L;kZ!$0Yv`R#?xXE7_Kiib)NOjtBC?z)0(<+ZD5{&dT*mq~Zt7v}|C_|Nk@5KYH#ht8 zBr&9lxt9zZae}K8=9V|52=tQ|48uY1aZlT*3^X0vN6$jmTfpTiTmFxu)WTo7hRN3f z-L7+}HGnA(IcZj!A##PkW>o}@WMW)4+M)Tejh#$*VxA^lxHj3bS$~w>h}4VLDwnl% z0I1l;{q%6ZdtFZDL8?G_{T;lx_^L-^=#*00#wJVJ+EpG787*wdxG|ASW71PtRXH@O zwM+RyeBZ=*PP)Y>H$A;M<~BgRRX`=}GZU+iBiE@cK`M=0t2j(ey$fi~nW_2M((xrz zkC(Scx+?0trGTB??s2tsoSNvj2kjEsd5ebEZiKP{fNNJg@Q0|_N>0hA^RW`^6TIZx zdO9onDiqQRGl)QFaL>bLuZeC!zXS13Z>nYdq^w0F`1W@ubeQ!JMa0{-I(5#%(&-u` zMG)^aUHlNQa8;CL z^a44*ZBofkrMVta#c(O(=sZ& zVCFmS6Pe8(u5ir+n*-DGyD|6zx_|5l$@?Dn%%)~9CZ$@7EM7GXavoA9J_L6&WLnm# z^-p@2DJn*&R8R?p*E^KU6h`gQ^XahntNpJx;hLk`=9@b+1|}Nh<=YQj!XH=xw%XP5 zG+LlD^h7G5!Y>$s18T|m1JNB}0epj3`So>9@d2}!g5fgxjdBNdR=CE=9S^l+Gf-bf zSrOhuieQHDzRlbvfVsHn>Jv)tA6M=v6ha>}$P*RAc@;ugvYFTI4Li{BreQtVy-mA8 zBbzedj@}OFgE8NP<*J}6<2)4u_TyLrP&L@V-O38gF<}&72K54zmY#!Ca8gQNj>eqw zM154aWY@Gzx$Yfj1lbSJQ~4*6ES zicw$WofWmlcTP4yvc;}Ux*yz1Gw`7bO)D4+9rYuBll|=BDT2G`zIzeD6zw!+$KII3 zIOfS}5E}OGoSQoa4kFBGi{Q3BY+9pXk`JopyCgzaljovy^}UAMpRSsFUF1un*cePs zjEGZrZLp;H`}q~M*Vm)%^h>;Ic)Cn1K<(mBhU*Xy8M54cm$9F@Epdq<3ox3ca49V{ zMSrQz!WH4U5IN~H6t6P%K>6q^qT2XTQqRI>=;9XI9=Og%M{UkgYeVC>)Y~-8Akm81 z?M<6m+j|Q~9n_%q#lCrU(@@N7Q0-Ru=apMDi|LZ;JMX6n^j%cxdh!&ko`-tTnSl!k z`uUoKRe*U{dE8kBYr?i+yLljzX4quKohd_n`~dPe%v0xF~*!+h8tFUW{nMbZj^HV!x&wQdMoJm**wPJOG3eINSfh`e1c z=`)M1nW?Z&CN?eJLQr#Lr$KYuuI-*|Rub&v;mWk3FZG)NkrT`}GY*5mB7E+m6ToWh zF)MekPoYs|%f(bNAM4lA_S9J${`3;3;t>7$kS`9uMuY<0iy9YxA5z*ib&Yl>!lA!)QkNX&sw^ zqPmy_UBSWMa`(Pv@a|G9>RKRL1|Ip2T!{>T#4&p`Ut|Zo-z5}p;>w<<(1Uj%xwj@H zGeKt3wI`g{5CysRf6- zpluAAZTE=88%L*24I%>7qly4`@TMAh+2^8q3zcu{AKVuuQ!$ISGn3fYmHI4kVVui6hxG^8%-{PkS?al)M&PTGAQ$*GNq?Cv_&;EVzSf89oYw0i<)GY}3nR&dlc)r8fu7Z$L`jbcJ^YCPh&`ujn9k)f9*pt}T5lFmHi~j|XsCAU< zacZ3DycGQ_#OLWL6-<0c%=Bx=iV(%4OA9YhA@zfu#wyL$IYSz&%MXq~MM1R7eeM5DU&8o$lOdQiZZtM~feEwfZ(czi)8m>JbzW zAI+H;2HgWsk-uk*p6ORO-JvXMun`ucP@mJw3ULxD71_ip>ULN{b~Xw*T+&%+#X3)J z87Ixzc5vL2{l#ChknOs<>=5m&q*onvun~(~@{s?l^k;#LR5682Lq^!gbyxsa6R6Vf zDajlPpsAag&&%V%jqz2U?>^R%v~T7BxB{2-ZY+!6iHZ)E>dhF;%!_BC`4T_?Nx)0q ziz@6OQWdj;Qz8u}!*2WEVc=dFF4nmhQ9>j|fyi6|s5Hyxgs?OZO{%wW{Z50jj_McKj$dt_)N#+YZtH2A|33U$dE|jMBIJ_jd5e>sc&j}csee6FNT4~ zTuV|7QnWvSA0|^(a0>K=yLlnp9BkdZ9GwEmE@jRC=?crC{m@9%!rla>^?56Rr~4Tw z8}VZ?xwLq+ceYC?XFQ5fKo`>)otrXUCP See7x1`&!Nf0HpZQ^gjVW Date: Fri, 31 Jul 2020 16:42:53 +0200 Subject: [PATCH 04/32] Don't close inbound notifications substreams immediately (#6781) * Don't close inbound notifications substreams immediately * Fix not closing in return to node A closing --- client/network/src/lib.rs | 5 +- .../generic_proto/handler/notif_in.rs | 6 +-- .../generic_proto/upgrade/notifications.rs | 51 +++++++++++++++---- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index b8e5d7582b96b..fc5cab321d127 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -197,13 +197,14 @@ //! handshake message can be of length 0, in which case the sender has to send a single `0`. //! - The receiver then either immediately closes the substream, or answers with its own //! LEB128-prefixed protocol-specific handshake response. The message can be of length 0, in which -//! case a single `0` has to be sent back. The receiver is then encouraged to close its sending -//! side. +//! case a single `0` has to be sent back. //! - Once the handshake has completed, the notifications protocol is unidirectional. Only the //! node which initiated the substream can push notifications. If the remote wants to send //! notifications as well, it has to open its own undirectional substream. //! - Each notification must be prefixed with an LEB128-encoded length. The encoding of the //! messages is specific to each protocol. +//! - Either party can signal that it doesn't want a notifications substream anymore by closing +//! its writing side. The other party should respond by closing its own writing side soon after. //! //! The API of `sc-network` allows one to register user-defined notification protocols. //! `sc-network` automatically tries to open a substream towards each node for which the legacy diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs index be78fb970e90b..ddd78566fcd2a 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_in.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs @@ -163,11 +163,9 @@ impl ProtocolsHandler for NotifsInHandler { } // Note that we drop the existing substream, which will send an equivalent to a TCP "RST" - // to the remote and force-close the substream. It might seem like an unclean way to get + // to the remote and force-close the substream. It might seem like an unclean way to get // rid of a substream. However, keep in mind that it is invalid for the remote to open - // multiple such substreams, and therefore sending a "RST" is the correct thing to do. - // Also note that we have already closed our writing side during the initial handshake, - // and we can't close "more" than that anyway. + // multiple such substreams, and therefore sending a "RST" is not an incorrect thing to do. self.substream = Some(proto); self.events_queue.push_back(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg))); diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs index f1f41d5bccf8e..80fd7761f8088 100644 --- a/client/network/src/protocol/generic_proto/upgrade/notifications.rs +++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs @@ -22,12 +22,13 @@ /// higher-level logic. This message is prefixed with a variable-length integer message length. /// This message can be empty, in which case `0` is sent. /// - If node B accepts the substream, it sends back a message with the same properties. -/// Afterwards, the sending side of B is closed. /// - If instead B refuses the connection (which typically happens because no empty slot is /// available), then it immediately closes the substream without sending back anything. /// - Node A can then send notifications to B, prefixed with a variable-length integer indicating /// the length of the message. -/// - Node A closes its writing side if it doesn't want the notifications substream anymore. +/// - Either node A or node B can signal that it doesn't want this notifications substream anymore +/// by closing its writing side. The other party should respond by also closing their own +/// writing side soon after. /// /// Notification substreams are unidirectional. If A opens a substream with B, then B is /// encouraged but not required to open a substream to A as well. @@ -80,9 +81,13 @@ enum NotificationsInSubstreamHandshake { /// User gave us the handshake message. Trying to push it in the socket. PendingSend(Vec), /// Handshake message was pushed in the socket. Still need to flush. - Close, - /// Handshake message successfully sent. + Flush, + /// Handshake message successfully sent and flushed. Sent, + /// Remote has closed their writing side. We close our own writing side in return. + ClosingInResponseToRemote, + /// Both our side and the remote have closed their writing side. + BothSidesClosed, } /// A substream for outgoing notification messages. @@ -177,8 +182,6 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin, // This `Stream` implementation first tries to send back the handshake if necessary. loop { match mem::replace(this.handshake, NotificationsInSubstreamHandshake::Sent) { - NotificationsInSubstreamHandshake::Sent => - return Stream::poll_next(this.socket.as_mut(), cx), NotificationsInSubstreamHandshake::NotSent => { *this.handshake = NotificationsInSubstreamHandshake::NotSent; return Poll::Pending @@ -186,7 +189,7 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin, NotificationsInSubstreamHandshake::PendingSend(msg) => match Sink::poll_ready(this.socket.as_mut(), cx) { Poll::Ready(_) => { - *this.handshake = NotificationsInSubstreamHandshake::Close; + *this.handshake = NotificationsInSubstreamHandshake::Flush; match Sink::start_send(this.socket.as_mut(), io::Cursor::new(msg)) { Ok(()) => {}, Err(err) => return Poll::Ready(Some(Err(err))), @@ -197,15 +200,43 @@ where TSubstream: AsyncRead + AsyncWrite + Unpin, return Poll::Pending } }, - NotificationsInSubstreamHandshake::Close => - match Sink::poll_close(this.socket.as_mut(), cx)? { + NotificationsInSubstreamHandshake::Flush => + match Sink::poll_flush(this.socket.as_mut(), cx)? { Poll::Ready(()) => *this.handshake = NotificationsInSubstreamHandshake::Sent, Poll::Pending => { - *this.handshake = NotificationsInSubstreamHandshake::Close; + *this.handshake = NotificationsInSubstreamHandshake::Flush; return Poll::Pending } }, + + NotificationsInSubstreamHandshake::Sent => { + match Stream::poll_next(this.socket.as_mut(), cx) { + Poll::Ready(None) => *this.handshake = + NotificationsInSubstreamHandshake::ClosingInResponseToRemote, + Poll::Ready(Some(msg)) => { + *this.handshake = NotificationsInSubstreamHandshake::Sent; + return Poll::Ready(Some(msg)) + }, + Poll::Pending => { + *this.handshake = NotificationsInSubstreamHandshake::Sent; + return Poll::Pending + }, + } + }, + + NotificationsInSubstreamHandshake::ClosingInResponseToRemote => + match Sink::poll_close(this.socket.as_mut(), cx)? { + Poll::Ready(()) => + *this.handshake = NotificationsInSubstreamHandshake::BothSidesClosed, + Poll::Pending => { + *this.handshake = NotificationsInSubstreamHandshake::ClosingInResponseToRemote; + return Poll::Pending + } + }, + + NotificationsInSubstreamHandshake::BothSidesClosed => + return Poll::Ready(None), } } } From 8281317727440ccdd8c95378ccfa3c365ef7bd1c Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 3 Aug 2020 10:30:06 +0200 Subject: [PATCH 05/32] client/network: Expose DHT query duration to Prometheus (#6784) Expose duration of DHT put and get request as a Prometheus histogram. --- client/network/src/behaviour.rs | 21 ++++++++++---------- client/network/src/discovery.rs | 34 ++++++++++++++++++++------------- client/network/src/service.rs | 28 +++++++++++++++++++++++++-- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 2afcd2741381f..a43c61054d974 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -145,8 +145,9 @@ pub enum BehaviourOut { messages: Vec<(ConsensusEngineId, Bytes)>, }, - /// Event generated by a DHT. - Dht(DhtEvent), + /// Events generated by a DHT as a response to get_value or put_value requests as well as the + /// request duration. + Dht(DhtEvent, Duration), } impl Behaviour { @@ -454,17 +455,17 @@ impl NetworkBehaviourEventProcess DiscoveryOut::Discovered(peer_id) => { self.substrate.add_discovered_nodes(iter::once(peer_id)); } - DiscoveryOut::ValueFound(results) => { - self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueFound(results))); + DiscoveryOut::ValueFound(results, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueFound(results), duration)); } - DiscoveryOut::ValueNotFound(key) => { - self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueNotFound(key))); + DiscoveryOut::ValueNotFound(key, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueNotFound(key), duration)); } - DiscoveryOut::ValuePut(key) => { - self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePut(key))); + DiscoveryOut::ValuePut(key, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePut(key), duration)); } - DiscoveryOut::ValuePutFailed(key) => { - self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePutFailed(key))); + DiscoveryOut::ValuePutFailed(key, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePutFailed(key), duration)); } DiscoveryOut::RandomKademliaStarted(protocols) => { for protocol in protocols { diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index 8216d6b2cbe73..7cb977e8e1ab3 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -312,7 +312,7 @@ impl DiscoveryBehaviour { for k in self.kademlias.values_mut() { if let Err(e) = k.put_record(Record::new(key.clone(), value.clone()), Quorum::All) { warn!(target: "sub-libp2p", "Libp2p => Failed to put record: {:?}", e); - self.pending_events.push_back(DiscoveryOut::ValuePutFailed(key.clone())); + self.pending_events.push_back(DiscoveryOut::ValuePutFailed(key.clone(), Duration::from_secs(0))); } } } @@ -379,17 +379,25 @@ pub enum DiscoveryOut { /// the `identify` protocol. UnroutablePeer(PeerId), - /// The DHT yielded results for the record request, grouped in (key, value) pairs. - ValueFound(Vec<(record::Key, Vec)>), + /// The DHT yielded results for the record request. + /// + /// Returning the result grouped in (key, value) pairs as well as the request duration.. + ValueFound(Vec<(record::Key, Vec)>, Duration), /// The record requested was not found in the DHT. - ValueNotFound(record::Key), + /// + /// Returning the corresponding key as well as the request duration. + ValueNotFound(record::Key, Duration), /// The record with a given key was successfully inserted into the DHT. - ValuePut(record::Key), + /// + /// Returning the corresponding key as well as the request duration. + ValuePut(record::Key, Duration), /// Inserting a value into the DHT failed. - ValuePutFailed(record::Key), + /// + /// Returning the corresponding key as well as the request duration. + ValuePutFailed(record::Key, Duration), /// Started a random Kademlia query for each DHT identified by the given `ProtocolId`s. RandomKademliaStarted(Vec), @@ -620,7 +628,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { } } } - KademliaEvent::QueryResult { result: QueryResult::GetRecord(res), .. } => { + KademliaEvent::QueryResult { result: QueryResult::GetRecord(res), stats, .. } => { let ev = match res { Ok(ok) => { let results = ok.records @@ -628,28 +636,28 @@ impl NetworkBehaviour for DiscoveryBehaviour { .map(|r| (r.record.key, r.record.value)) .collect(); - DiscoveryOut::ValueFound(results) + DiscoveryOut::ValueFound(results, stats.duration().unwrap_or_else(Default::default)) } Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => { trace!(target: "sub-libp2p", "Libp2p => Failed to get record: {:?}", e); - DiscoveryOut::ValueNotFound(e.into_key()) + DiscoveryOut::ValueNotFound(e.into_key(), stats.duration().unwrap_or_else(Default::default)) } Err(e) => { warn!(target: "sub-libp2p", "Libp2p => Failed to get record: {:?}", e); - DiscoveryOut::ValueNotFound(e.into_key()) + DiscoveryOut::ValueNotFound(e.into_key(), stats.duration().unwrap_or_else(Default::default)) } }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } - KademliaEvent::QueryResult { result: QueryResult::PutRecord(res), .. } => { + KademliaEvent::QueryResult { result: QueryResult::PutRecord(res), stats, .. } => { let ev = match res { - Ok(ok) => DiscoveryOut::ValuePut(ok.key), + Ok(ok) => DiscoveryOut::ValuePut(ok.key, stats.duration().unwrap_or_else(Default::default)), Err(e) => { warn!(target: "sub-libp2p", "Libp2p => Failed to put record: {:?}", e); - DiscoveryOut::ValuePutFailed(e.into_key()) + DiscoveryOut::ValuePutFailed(e.into_key(), stats.duration().unwrap_or_else(Default::default)) } }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index c11a620c56720..65f93f58a5553 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -31,6 +31,7 @@ use crate::{ ExHashT, NetworkStateInfo, behaviour::{Behaviour, BehaviourOut}, config::{parse_addr, parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig}, + DhtEvent, discovery::DiscoveryConfig, error::Error, network_state::{ @@ -1119,6 +1120,7 @@ struct Metrics { incoming_connections_total: Counter, is_major_syncing: Gauge, issued_light_requests: Counter, + kademlia_query_duration: HistogramVec, kademlia_random_queries_total: CounterVec, kademlia_records_count: GaugeVec, kademlia_records_sizes_total: GaugeVec, @@ -1196,6 +1198,17 @@ impl Metrics { "issued_light_requests", "Number of light client requests that our node has issued.", )?, registry)?, + kademlia_query_duration: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "sub_libp2p_kademlia_query_duration", + "Duration of Kademlia queries per protocol and query type" + ), + buckets: prometheus_endpoint::exponential_buckets(0.5, 2.0, 10) + .expect("parameters are always valid values; qed"), + }, + &["type"] + )?, registry)?, kademlia_random_queries_total: register(CounterVec::new( Opts::new( "sub_libp2p_kademlia_random_queries_total", @@ -1508,8 +1521,19 @@ impl Future for NetworkWorker { messages, }); }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Dht(ev))) => { - this.event_streams.send(Event::Dht(ev)); + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Dht(event, duration))) => { + if let Some(metrics) = this.metrics.as_ref() { + let query_type = match event { + DhtEvent::ValueFound(_) => "value-found", + DhtEvent::ValueNotFound(_) => "value-not-found", + DhtEvent::ValuePut(_) => "value-put", + DhtEvent::ValuePutFailed(_) => "value-put-failed", + }; + metrics.kademlia_query_duration.with_label_values(&[query_type]) + .observe(duration.as_secs_f64()); + } + + this.event_streams.send(Event::Dht(event)); }, Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, num_established }) => { trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); From 54e6298523ea896f2129546c26dd7a47dbbeedc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 3 Aug 2020 10:46:53 +0200 Subject: [PATCH 06/32] Fix transaction payment runtime api (#6792) The transaction payment runtime api used its own extrinsic generic parameter. This is wrong, because this resulted in using always the native extrinsic. If there was a runtime upgrade that changed the extrinsic in some way, it would result in the api breaking. The correct way is to use the `Extrinsic` from the `Block` parameter. This is on the node side the opaque extrinsic and on the runtime side the real extrinsic. --- bin/node/rpc/src/lib.rs | 3 +-- bin/node/runtime/src/lib.rs | 3 +-- frame/transaction-payment/rpc/runtime-api/src/lib.rs | 5 ++--- frame/transaction-payment/rpc/src/lib.rs | 9 ++++----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 7f1457356d908..106353983febe 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -33,7 +33,6 @@ use std::sync::Arc; use node_primitives::{Block, BlockNumber, AccountId, Index, Balance, Hash}; -use node_runtime::UncheckedExtrinsic; use sp_api::ProvideRuntimeApi; use sp_transaction_pool::TransactionPool; use sp_blockchain::{Error as BlockChainError, HeaderMetadata, HeaderBackend}; @@ -106,7 +105,7 @@ pub fn create_full( C: Send + Sync + 'static, C::Api: substrate_frame_rpc_system::AccountNonceApi, C::Api: pallet_contracts_rpc::ContractsRuntimeApi, - C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, + C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, C::Api: BabeApi, C::Api: BlockBuilder, P: TransactionPool + 'static, diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 373a01b8ea279..fff0dd3427fee 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1101,9 +1101,8 @@ impl_runtime_apis! { impl pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi< Block, Balance, - UncheckedExtrinsic, > for Runtime { - fn query_info(uxt: UncheckedExtrinsic, len: u32) -> RuntimeDispatchInfo { + fn query_info(uxt: ::Extrinsic, len: u32) -> RuntimeDispatchInfo { TransactionPayment::query_info(uxt, len) } } diff --git a/frame/transaction-payment/rpc/runtime-api/src/lib.rs b/frame/transaction-payment/rpc/runtime-api/src/lib.rs index 17a8bcdf44e87..5575f8f7d0950 100644 --- a/frame/transaction-payment/rpc/runtime-api/src/lib.rs +++ b/frame/transaction-payment/rpc/runtime-api/src/lib.rs @@ -56,11 +56,10 @@ fn deserialize_from_string<'de, D: Deserializer<'de>, T: std::str::FromStr>(dese } sp_api::decl_runtime_apis! { - pub trait TransactionPaymentApi where + pub trait TransactionPaymentApi where Balance: Codec + MaybeDisplay + MaybeFromStr, - Extrinsic: Codec, { - fn query_info(uxt: Extrinsic, len: u32) -> RuntimeDispatchInfo; + fn query_info(uxt: Block::Extrinsic, len: u32) -> RuntimeDispatchInfo; } } diff --git a/frame/transaction-payment/rpc/src/lib.rs b/frame/transaction-payment/rpc/src/lib.rs index d99907a6ac3f4..5043f0257fc36 100644 --- a/frame/transaction-payment/rpc/src/lib.rs +++ b/frame/transaction-payment/rpc/src/lib.rs @@ -69,14 +69,13 @@ impl From for i64 { } } -impl TransactionPaymentApi<::Hash, RuntimeDispatchInfo> - for TransactionPayment +impl TransactionPaymentApi<::Hash, RuntimeDispatchInfo> + for TransactionPayment where Block: BlockT, C: Send + Sync + 'static + ProvideRuntimeApi + HeaderBackend, - C::Api: TransactionPaymentRuntimeApi, + C::Api: TransactionPaymentRuntimeApi, Balance: Codec + MaybeDisplay + MaybeFromStr, - Extrinsic: Codec + Send + Sync + 'static, { fn query_info( &self, @@ -91,7 +90,7 @@ where let encoded_len = encoded_xt.len() as u32; - let uxt: Extrinsic = Decode::decode(&mut &*encoded_xt).map_err(|e| RpcError { + let uxt: Block::Extrinsic = Decode::decode(&mut &*encoded_xt).map_err(|e| RpcError { code: ErrorCode::ServerError(Error::DecodeError.into()), message: "Unable to query dispatch info.".into(), data: Some(format!("{:?}", e).into()), From 125f77ddb7d8126318fe08f2c0ea967fb934d272 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 10:52:43 +0200 Subject: [PATCH 07/32] Bump elliptic from 6.5.2 to 6.5.3 in /.maintain/chaostest (#6791) Bumps [elliptic](https://github.com/indutny/elliptic) from 6.5.2 to 6.5.3. - [Release notes](https://github.com/indutny/elliptic/releases) - [Commits](https://github.com/indutny/elliptic/compare/v6.5.2...v6.5.3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .maintain/chaostest/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.maintain/chaostest/package-lock.json b/.maintain/chaostest/package-lock.json index d975c9faf9e72..8855f221a133d 100644 --- a/.maintain/chaostest/package-lock.json +++ b/.maintain/chaostest/package-lock.json @@ -1720,9 +1720,9 @@ } }, "elliptic": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz", - "integrity": "sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw==", + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", "requires": { "bn.js": "^4.4.0", "brorand": "^1.0.1", From eb57b07a369441eee0dcd5065c508b5606f86d1d Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 3 Aug 2020 11:08:06 +0200 Subject: [PATCH 08/32] client/network: Fix wrong metric help text (#6794) The `sub_libp2p_kademlia_query_duration` metric only has the dimension `type` not `protocol`. --- client/network/src/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 65f93f58a5553..e4ba36be587fe 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1202,7 +1202,7 @@ impl Metrics { HistogramOpts { common_opts: Opts::new( "sub_libp2p_kademlia_query_duration", - "Duration of Kademlia queries per protocol and query type" + "Duration of Kademlia queries per query type" ), buckets: prometheus_endpoint::exponential_buckets(0.5, 2.0, 10) .expect("parameters are always valid values; qed"), From e81a3160d963456c88ce05b9391103ed405e0851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Thei=C3=9Fen?= Date: Mon, 3 Aug 2020 12:03:22 +0200 Subject: [PATCH 09/32] seal: Fix and improve error reporting (#6773) * seal: Rework ext_transfer, ext_instantiate, ext_call error handling * Deny calling plain accounts (must use transfer now) * Return proper module error rather than ad-hoc strings * Return the correct error codes from call,instantiate (documentation was wrong) * Make ext_transfer fallible again to make it consistent with ext_call * seal: Improve error messages on memory access failures * seal: Convert contract trapped to module error * seal: Add additional tests for transfer, call, instantiate These tests verify that those functions return the error types which are declared in its docs. * Make it more pronounced that to_execution_result handles trap_reason * Improve ReturnCode docs * Fix whitespace issues in wat files * Improve ReturnCode doc * Improve ErrorOrigin doc and variant naming * Improve docs on ExecResult and ExecError * Encode u32 sentinel value as hex * with_nested_context no longer accepts an Option for trie * Fix successful typo * Rename InvalidContractCalled to NotCallable --- frame/contracts/fixtures/call_return_code.wat | 44 +++ frame/contracts/fixtures/caller_contract.wat | 4 +- .../fixtures/destroy_and_transfer.wat | 8 +- frame/contracts/fixtures/drain.wat | 11 +- .../fixtures/instantiate_return_code.wat | 47 +++ frame/contracts/fixtures/ok_trap_revert.wat | 35 +++ .../fixtures/self_destructing_constructor.wat | 39 +-- frame/contracts/fixtures/set_rent.wat | 9 +- .../fixtures/transfer_return_code.wat | 31 ++ frame/contracts/src/exec.rs | 201 ++++++++----- frame/contracts/src/gas.rs | 11 +- frame/contracts/src/lib.rs | 24 +- frame/contracts/src/storage.rs | 1 + frame/contracts/src/tests.rs | 267 +++++++++++++++-- frame/contracts/src/wasm/mod.rs | 111 ++++++- frame/contracts/src/wasm/runtime.rs | 277 +++++++++++------- 16 files changed, 855 insertions(+), 265 deletions(-) create mode 100644 frame/contracts/fixtures/call_return_code.wat create mode 100644 frame/contracts/fixtures/instantiate_return_code.wat create mode 100644 frame/contracts/fixtures/ok_trap_revert.wat create mode 100644 frame/contracts/fixtures/transfer_return_code.wat diff --git a/frame/contracts/fixtures/call_return_code.wat b/frame/contracts/fixtures/call_return_code.wat new file mode 100644 index 0000000000000..d724f90446245 --- /dev/null +++ b/frame/contracts/fixtures/call_return_code.wat @@ -0,0 +1,44 @@ +;; This calls Django (4) and transfers 100 balance during this call and copies the return code +;; of this call to the output buffer. +;; It also forwards its input to the callee. +(module + (import "env" "ext_input" (func $ext_input (param i32 i32))) + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "ext_return" (func $ext_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 8) address of django + (data (i32.const 0) "\04\00\00\00\00\00\00\00") + + ;; [8, 16) 100 balance + (data (i32.const 8) "\64\00\00\00\00\00\00\00") + + ;; [16, 20) here we store the return code of the transfer + + ;; [20, 24) here we store the input data + + ;; [24, 28) size of the input data + (data (i32.const 24) "\04") + + (func (export "deploy")) + + (func (export "call") + (call $ext_input (i32.const 20) (i32.const 24)) + (i32.store + (i32.const 16) + (call $ext_call + (i32.const 0) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 8) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 20) ;; Pointer to input data buffer address + (i32.load (i32.const 24)) ;; Length of input data buffer + (i32.const 0xffffffff) ;; u32 max sentinel value: do not copy output + (i32.const 0) ;; Ptr to output buffer len + ) + ) + ;; exit with success and take transfer return code to the output buffer + (call $ext_return (i32.const 0) (i32.const 16) (i32.const 4)) + ) +) diff --git a/frame/contracts/fixtures/caller_contract.wat b/frame/contracts/fixtures/caller_contract.wat index 369007834dc7b..ee2e16098d595 100644 --- a/frame/contracts/fixtures/caller_contract.wat +++ b/frame/contracts/fixtures/caller_contract.wat @@ -89,7 +89,7 @@ (call $ext_instantiate (i32.const 24) ;; Pointer to the code hash. (i32.const 32) ;; Length of the code hash. - (i64.const 200) ;; How much gas to devote for the execution. + (i64.const 187500000) ;; Just enough to pay for the instantiate (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 8) ;; Length of the buffer with value to transfer. (i32.const 8) ;; Pointer to input data buffer address @@ -206,7 +206,7 @@ (call $ext_call (i32.const 16) ;; Pointer to "callee" address. (i32.const 8) ;; Length of "callee" address. - (i64.const 100) ;; How much gas to devote for the execution. + (i64.const 117500000) ;; Just enough to make the call (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 8) ;; Length of the buffer with value to transfer. (i32.const 8) ;; Pointer to input data buffer address diff --git a/frame/contracts/fixtures/destroy_and_transfer.wat b/frame/contracts/fixtures/destroy_and_transfer.wat index ee191aa019bfb..3f8a8c89b02cd 100644 --- a/frame/contracts/fixtures/destroy_and_transfer.wat +++ b/frame/contracts/fixtures/destroy_and_transfer.wat @@ -3,6 +3,7 @@ (import "env" "ext_get_storage" (func $ext_get_storage (param i32 i32 i32) (result i32))) (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32) (result i32))) (import "env" "ext_instantiate" (func $ext_instantiate (param i32 i32 i64 i32 i32 i32 i32 i32 i32 i32 i32) (result i32))) (import "env" "memory" (memory 1 1)) @@ -139,16 +140,11 @@ ;; does not keep the contract alive. (call $assert (i32.eq - (call $ext_call + (call $ext_transfer (i32.const 80) ;; Pointer to destination address (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 1) ;; Length of input data buffer - (i32.const 4294967295) ;; u32 max sentinel value: do not copy output - (i32.const 0) ;; Length is ignored in this case ) (i32.const 0) ) diff --git a/frame/contracts/fixtures/drain.wat b/frame/contracts/fixtures/drain.wat index 1b3172b2a01a4..22422bb859d0f 100644 --- a/frame/contracts/fixtures/drain.wat +++ b/frame/contracts/fixtures/drain.wat @@ -1,6 +1,6 @@ (module (import "env" "ext_balance" (func $ext_balance (param i32 i32))) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32) (result i32))) (import "env" "memory" (memory 1 1)) ;; [0, 8) reserved for $ext_balance output @@ -36,18 +36,13 @@ ;; Self-destruct by sending full balance to the 0 address. (call $assert (i32.eq - (call $ext_call + (call $ext_transfer (i32.const 16) ;; Pointer to destination address (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - (i32.const 4294967295) ;; u32 max sentinel value: do not copy output - (i32.const 0) ;; Length is ignored in this case ) - (i32.const 0) + (i32.const 4) ;; ReturnCode::BelowSubsistenceThreshold ) ) ) diff --git a/frame/contracts/fixtures/instantiate_return_code.wat b/frame/contracts/fixtures/instantiate_return_code.wat new file mode 100644 index 0000000000000..bce80ca01faac --- /dev/null +++ b/frame/contracts/fixtures/instantiate_return_code.wat @@ -0,0 +1,47 @@ +;; This instantiats Charlie (3) and transfers 100 balance during this call and copies the return code +;; of this call to the output buffer. +;; The first 32 byte of input is the code hash to instantiate +;; The rest of the input is forwarded to the constructor of the callee +(module + (import "env" "ext_input" (func $ext_input (param i32 i32))) + (import "env" "ext_instantiate" (func $ext_instantiate (param i32 i32 i64 i32 i32 i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "ext_return" (func $ext_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 8) address of django + (data (i32.const 0) "\04\00\00\00\00\00\00\00") + + ;; [8, 16) 100 balance + (data (i32.const 8) "\64\00\00\00\00\00\00\00") + + ;; [16, 20) here we store the return code of the transfer + + ;; [20, 24) size of the input buffer + (data (i32.const 20) "\FF") + + ;; [24, inf) input buffer + + (func (export "deploy")) + + (func (export "call") + (call $ext_input (i32.const 24) (i32.const 20)) + (i32.store + (i32.const 16) + (call $ext_instantiate + (i32.const 24) ;; Pointer to the code hash. + (i32.const 32) ;; Length of the code hash. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 8) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 56) ;; Pointer to input data buffer address + (i32.sub (i32.load (i32.const 20)) (i32.const 32)) ;; Length of input data buffer + (i32.const 0xffffffff) ;; u32 max sentinel value: do not copy address + (i32.const 0) ;; Length is ignored in this case + (i32.const 0xffffffff) ;; u32 max sentinel value: do not copy output + (i32.const 0) ;; Length is ignored in this case + ) + ) + ;; exit with success and take transfer return code to the output buffer + (call $ext_return (i32.const 0) (i32.const 16) (i32.const 4)) + ) +) diff --git a/frame/contracts/fixtures/ok_trap_revert.wat b/frame/contracts/fixtures/ok_trap_revert.wat new file mode 100644 index 0000000000000..5877e55d0e70e --- /dev/null +++ b/frame/contracts/fixtures/ok_trap_revert.wat @@ -0,0 +1,35 @@ +(module + (import "env" "ext_input" (func $ext_input (param i32 i32))) + (import "env" "ext_return" (func $ext_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "deploy") + (call $ok_trap_revert) + ) + + (func (export "call") + (call $ok_trap_revert) + ) + + (func $ok_trap_revert + (i32.store (i32.const 4) (i32.const 4)) + (call $ext_input (i32.const 0) (i32.const 4)) + (block $IF_2 + (block $IF_1 + (block $IF_0 + (br_table $IF_0 $IF_1 $IF_2 + (i32.load8_u (i32.const 0)) + ) + (unreachable) + ) + ;; 0 = return with success + return + ) + ;; 1 = revert + (call $ext_return (i32.const 1) (i32.const 0) (i32.const 0)) + (unreachable) + ) + ;; 2 = trap + (unreachable) + ) +) \ No newline at end of file diff --git a/frame/contracts/fixtures/self_destructing_constructor.wat b/frame/contracts/fixtures/self_destructing_constructor.wat index 3b99db001cd38..ece5679f4f691 100644 --- a/frame/contracts/fixtures/self_destructing_constructor.wat +++ b/frame/contracts/fixtures/self_destructing_constructor.wat @@ -1,15 +1,7 @@ (module - (import "env" "ext_balance" (func $ext_balance (param i32 i32))) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "ext_terminate" (func $ext_terminate (param i32 i32))) (import "env" "memory" (memory 1 1)) - ;; [0, 8) reserved for $ext_balance output - - ;; [8, 16) length of the buffer - (data (i32.const 8) "\08") - - ;; [16, inf) zero initialized - (func $assert (param i32) (block $ok (br_if $ok @@ -20,33 +12,10 @@ ) (func (export "deploy") - ;; Send entire remaining balance to the 0 address. - (call $ext_balance (i32.const 0) (i32.const 8)) - - ;; Balance should be encoded as a u64. - (call $assert - (i32.eq - (i32.load (i32.const 8)) - (i32.const 8) - ) - ) - ;; Self-destruct by sending full balance to the 0 address. - (call $assert - (i32.eq - (call $ext_call - (i32.const 16) ;; Pointer to destination address - (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - (i32.const 4294967295) ;; u32 max sentinel value: do not copy output - (i32.const 0) ;; Length is ignored in this case - ) - (i32.const 0) - ) + (call $ext_terminate + (i32.const 0) ;; Pointer to destination address + (i32.const 8) ;; Length of destination address ) ) diff --git a/frame/contracts/fixtures/set_rent.wat b/frame/contracts/fixtures/set_rent.wat index 4e6424e720104..ba52e9ed752ce 100644 --- a/frame/contracts/fixtures/set_rent.wat +++ b/frame/contracts/fixtures/set_rent.wat @@ -1,5 +1,5 @@ (module - (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32))) + (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32) (result i32))) (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) (import "env" "ext_clear_storage" (func $ext_clear_storage (param i32))) (import "env" "ext_set_rent_allowance" (func $ext_set_rent_allowance (param i32 i32))) @@ -24,7 +24,12 @@ ;; transfer 50 to CHARLIE (func $call_2 - (call $ext_transfer (i32.const 68) (i32.const 8) (i32.const 76) (i32.const 8)) + (call $assert + (i32.eq + (call $ext_transfer (i32.const 68) (i32.const 8) (i32.const 76) (i32.const 8)) + (i32.const 0) + ) + ) ) ;; do nothing diff --git a/frame/contracts/fixtures/transfer_return_code.wat b/frame/contracts/fixtures/transfer_return_code.wat new file mode 100644 index 0000000000000..87d186811e757 --- /dev/null +++ b/frame/contracts/fixtures/transfer_return_code.wat @@ -0,0 +1,31 @@ +;; This transfers 100 balance to the zero account and copies the return code +;; of this transfer to the output buffer. +(module + (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32) (result i32))) + (import "env" "ext_return" (func $ext_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 8) zero-adress + (data (i32.const 0) "\00\00\00\00\00\00\00\00") + + ;; [8, 16) 100 balance + (data (i32.const 8) "\64\00\00\00\00\00\00\00") + + ;; [16, 20) here we store the return code of the transfer + + (func (export "deploy")) + + (func (export "call") + (i32.store + (i32.const 16) + (call $ext_transfer + (i32.const 0) ;; ptr to destination address + (i32.const 8) ;; length of destination address + (i32.const 8) ;; ptr to value to transfer + (i32.const 8) ;; length of value to transfer + ) + ) + ;; exit with success and take transfer return code to the output buffer + (call $ext_return (i32.const 0) (i32.const 16) (i32.const 4)) + ) +) diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index e8965692aa243..a2fb50dd3f319 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -14,9 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use super::{CodeHash, Config, ContractAddressFor, Event, RawEvent, Trait, - TrieId, BalanceOf, ContractInfo, TrieIdGenerator}; -use crate::{gas::{Gas, GasMeter, Token}, rent, storage, Error, ContractInfoOf}; +use crate::{ + CodeHash, Config, ContractAddressFor, Event, RawEvent, Trait, + TrieId, BalanceOf, ContractInfo, TrieIdGenerator, + gas::{Gas, GasMeter, Token}, rent, storage, Error, ContractInfoOf +}; use bitflags::bitflags; use sp_std::prelude::*; use sp_runtime::traits::{Bounded, Zero, Convert, Saturating}; @@ -69,7 +71,39 @@ impl ExecReturnValue { } } -pub type ExecResult = Result; +/// Call or instantiate both call into other contracts and pass through errors happening +/// in those to the caller. This enum is for the caller to distinguish whether the error +/// happened during the execution of the callee or in the current execution context. +#[cfg_attr(test, derive(PartialEq, Eq, Debug))] +pub enum ErrorOrigin { + /// The error happened in the current exeuction context rather than in the one + /// of the contract that is called into. + Caller, + /// The error happened during execution of the called contract. + Callee, +} + +/// Error returned by contract exection. +#[cfg_attr(test, derive(PartialEq, Eq, Debug))] +pub struct ExecError { + /// The reason why the execution failed. + pub error: DispatchError, + /// Origin of the error. + pub origin: ErrorOrigin, +} + +impl> From for ExecError { + fn from(error: T) -> Self { + Self { + error: error.into(), + origin: ErrorOrigin::Caller, + } + } +} + +/// The result that is returned from contract execution. It either contains the output +/// buffer or an error describing the reason for failure. +pub type ExecResult = Result; /// An interface that provides access to the external environment in which the /// smart-contract is executed. @@ -99,7 +133,7 @@ pub trait Ext { value: BalanceOf, gas_meter: &mut GasMeter, input_data: Vec, - ) -> Result<(AccountIdOf, ExecReturnValue), DispatchError>; + ) -> Result<(AccountIdOf, ExecReturnValue), ExecError>; /// Transfer some amount of funds into the specified account. fn transfer( @@ -282,12 +316,12 @@ where } } - fn nested<'b, 'c: 'b>(&'c self, dest: T::AccountId, trie_id: Option) + fn nested<'b, 'c: 'b>(&'c self, dest: T::AccountId, trie_id: TrieId) -> ExecutionContext<'b, T, V, L> { ExecutionContext { caller: Some(self), - self_trie_id: trie_id, + self_trie_id: Some(trie_id), self_account: dest, depth: self.depth + 1, config: self.config, @@ -307,31 +341,31 @@ where input_data: Vec, ) -> ExecResult { if self.depth == self.config.max_depth as usize { - Err("reached maximum depth, cannot make a call")? + Err(Error::::MaxCallDepthReached)? } if gas_meter .charge(self.config, ExecFeeToken::Call) .is_out_of_gas() { - Err("not enough gas to pay base call fee")? + Err(Error::::OutOfGas)? } // Assumption: `collect_rent` doesn't collide with overlay because // `collect_rent` will be done on first call and destination contract and balance // cannot be changed before the first call - let contract_info = rent::collect_rent::(&dest); - - // Calls to dead contracts always fail. - if let Some(ContractInfo::Tombstone(_)) = contract_info { - Err("contract has been evicted")? + // We do not allow 'calling' plain accounts. For transfering value + // `ext_transfer` must be used. + let contract = if let Some(ContractInfo::Alive(info)) = rent::collect_rent::(&dest) { + info + } else { + Err(Error::::NotCallable)? }; let transactor_kind = self.transactor_kind(); let caller = self.self_account.clone(); - let dest_trie_id = contract_info.and_then(|i| i.as_alive().map(|i| i.trie_id.clone())); - self.with_nested_context(dest.clone(), dest_trie_id, |nested| { + self.with_nested_context(dest.clone(), contract.trie_id.clone(), |nested| { if value > BalanceOf::::zero() { transfer( gas_meter, @@ -344,22 +378,15 @@ where )? } - // If code_hash is not none, then the destination account is a live contract, otherwise - // it is a regular account since tombstone accounts have already been rejected. - match storage::code_hash::(&dest) { - Ok(dest_code_hash) => { - let executable = nested.loader.load_main(&dest_code_hash)?; - let output = nested.vm - .execute( - &executable, - nested.new_call_context(caller, value), - input_data, - gas_meter, - )?; - Ok(output) - } - Err(storage::ContractAbsentError) => Ok(ExecReturnValue { flags: ReturnFlags::empty(), data: Vec::new() }), - } + let executable = nested.loader.load_main(&contract.code_hash) + .map_err(|_| Error::::CodeNotFound)?; + let output = nested.vm.execute( + &executable, + nested.new_call_context(caller, value), + input_data, + gas_meter, + ).map_err(|e| ExecError { error: e.error, origin: ErrorOrigin::Callee })?; + Ok(output) }) } @@ -369,16 +396,16 @@ where gas_meter: &mut GasMeter, code_hash: &CodeHash, input_data: Vec, - ) -> Result<(T::AccountId, ExecReturnValue), DispatchError> { + ) -> Result<(T::AccountId, ExecReturnValue), ExecError> { if self.depth == self.config.max_depth as usize { - Err("reached maximum depth, cannot instantiate")? + Err(Error::::MaxCallDepthReached)? } if gas_meter .charge(self.config, ExecFeeToken::Instantiate) .is_out_of_gas() { - Err("not enough gas to pay base instantiate fee")? + Err(Error::::OutOfGas)? } let transactor_kind = self.transactor_kind(); @@ -394,7 +421,7 @@ where // Generate it now. let dest_trie_id = ::TrieIdGenerator::trie_id(&dest); - let output = self.with_nested_context(dest.clone(), Some(dest_trie_id), |nested| { + let output = self.with_nested_context(dest.clone(), dest_trie_id, |nested| { storage::place_contract::( &dest, nested @@ -416,21 +443,21 @@ where nested, )?; - let executable = nested.loader.load_init(&code_hash)?; + let executable = nested.loader.load_init(&code_hash) + .map_err(|_| Error::::CodeNotFound)?; let output = nested.vm .execute( &executable, nested.new_call_context(caller.clone(), endowment), input_data, gas_meter, - )?; + ).map_err(|e| ExecError { error: e.error, origin: ErrorOrigin::Callee })?; - // Error out if insufficient remaining balance. // We need each contract that exists to be above the subsistence threshold // in order to keep up the guarantuee that we always leave a tombstone behind // with the exception of a contract that called `ext_terminate`. - if T::Currency::free_balance(&dest) < nested.config.subsistence_threshold() { - Err("insufficient remaining balance")? + if T::Currency::total_balance(&dest) < nested.config.subsistence_threshold() { + Err(Error::::NewContractNotFunded)? } // Deposit an instantiation event. @@ -459,7 +486,7 @@ where } /// Execute the given closure within a nested execution context. - fn with_nested_context(&mut self, dest: T::AccountId, trie_id: Option, func: F) + fn with_nested_context(&mut self, dest: T::AccountId, trie_id: TrieId, func: F) -> ExecResult where F: FnOnce(&mut ExecutionContext) -> ExecResult { @@ -569,7 +596,7 @@ fn transfer<'a, T: Trait, V: Vm, L: Loader>( }; if gas_meter.charge(ctx.config, token).is_out_of_gas() { - Err("not enough gas to pay transfer fee")? + Err(Error::::OutOfGas)? } // Only ext_terminate is allowed to bring the sender below the subsistence @@ -580,13 +607,15 @@ fn transfer<'a, T: Trait, V: Vm, L: Loader>( ensure!( T::Currency::total_balance(transactor).saturating_sub(value) >= ctx.config.subsistence_threshold(), - Error::::InsufficientBalance, + Error::::BelowSubsistenceThreshold, ); ExistenceRequirement::KeepAlive }, (_, PlainAccount) => ExistenceRequirement::KeepAlive, }; - T::Currency::transfer(transactor, dest, value, existence_requirement)?; + + T::Currency::transfer(transactor, dest, value, existence_requirement) + .map_err(|_| Error::::TransferFailed)?; Ok(()) } @@ -653,7 +682,7 @@ where endowment: BalanceOf, gas_meter: &mut GasMeter, input_data: Vec, - ) -> Result<(AccountIdOf, ExecReturnValue), DispatchError> { + ) -> Result<(AccountIdOf, ExecReturnValue), ExecError> { self.ctx.instantiate(endowment, gas_meter, code_hash, input_data) } @@ -837,13 +866,13 @@ fn deposit_event( mod tests { use super::{ BalanceOf, Event, ExecFeeToken, ExecResult, ExecutionContext, Ext, Loader, - RawEvent, TransferFeeKind, TransferFeeToken, Vm, ReturnFlags, + RawEvent, TransferFeeKind, TransferFeeToken, Vm, ReturnFlags, ExecError, ErrorOrigin }; use crate::{ gas::GasMeter, tests::{ExtBuilder, Test, MetaEvent}, exec::ExecReturnValue, CodeHash, Config, gas::Gas, - storage, + storage, Error }; use crate::tests::test_utils::{place_contract, set_balance, get_balance}; use sp_runtime::DispatchError; @@ -999,11 +1028,19 @@ mod tests { let mut gas_meter = GasMeter::::new(GAS_LIMIT); - let result = ctx.call(dest, 0, &mut gas_meter, vec![]); + let result = super::transfer( + &mut gas_meter, + super::TransferCause::Call, + super::TransactorKind::PlainAccount, + &origin, + &dest, + 0, + &mut ctx, + ); assert_matches!(result, Ok(_)); let mut toks = gas_meter.tokens().iter(); - match_tokens!(toks, ExecFeeToken::Call,); + match_tokens!(toks, TransferFeeToken { kind: TransferFeeKind::Transfer },); }); // This test verifies that base fee for instantiation is taken. @@ -1043,14 +1080,18 @@ mod tests { set_balance(&origin, 100); set_balance(&dest, 0); - let output = ctx.call( - dest, + let mut gas_meter = GasMeter::::new(GAS_LIMIT); + + super::transfer( + &mut gas_meter, + super::TransferCause::Call, + super::TransactorKind::PlainAccount, + &origin, + &dest, 55, - &mut GasMeter::::new(GAS_LIMIT), - vec![], + &mut ctx, ).unwrap(); - assert!(output.is_success()); assert_eq!(get_balance(&origin), 45); assert_eq!(get_balance(&dest), 55); }); @@ -1107,13 +1148,20 @@ mod tests { let mut gas_meter = GasMeter::::new(GAS_LIMIT); - let result = ctx.call(dest, 50, &mut gas_meter, vec![]); + let result = super::transfer( + &mut gas_meter, + super::TransferCause::Call, + super::TransactorKind::PlainAccount, + &origin, + &dest, + 50, + &mut ctx, + ); assert_matches!(result, Ok(_)); let mut toks = gas_meter.tokens().iter(); match_tokens!( toks, - ExecFeeToken::Call, TransferFeeToken { kind: TransferFeeKind::Transfer, }, @@ -1132,13 +1180,20 @@ mod tests { let mut gas_meter = GasMeter::::new(GAS_LIMIT); - let result = ctx.call(dest, 50, &mut gas_meter, vec![]); + let result = super::transfer( + &mut gas_meter, + super::TransferCause::Call, + super::TransactorKind::PlainAccount, + &origin, + &dest, + 50, + &mut ctx, + ); assert_matches!(result, Ok(_)); let mut toks = gas_meter.tokens().iter(); match_tokens!( toks, - ExecFeeToken::Call, TransferFeeToken { kind: TransferFeeKind::Transfer, }, @@ -1189,16 +1244,19 @@ mod tests { let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); set_balance(&origin, 0); - let result = ctx.call( - dest, - 100, + let result = super::transfer( &mut GasMeter::::new(GAS_LIMIT), - vec![], + super::TransferCause::Call, + super::TransactorKind::PlainAccount, + &origin, + &dest, + 100, + &mut ctx, ); - assert_matches!( + assert_eq!( result, - Err(DispatchError::Module { message: Some("InsufficientBalance"), .. }) + Err(Error::::TransferFailed.into()) ); assert_eq!(get_balance(&origin), 0); assert_eq!(get_balance(&dest), 0); @@ -1335,9 +1393,9 @@ mod tests { if !*reached_bottom { // We are first time here, it means we just reached bottom. // Verify that we've got proper error and set `reached_bottom`. - assert_matches!( + assert_eq!( r, - Err(DispatchError::Other("reached maximum depth, cannot make a call")) + Err(Error::::MaxCallDepthReached.into()) ); *reached_bottom = true; } else { @@ -1604,7 +1662,10 @@ mod tests { ctx.gas_meter, vec![] ), - Err(DispatchError::Other("It's a trap!")) + Err(ExecError { + error: DispatchError::Other("It's a trap!"), + origin: ErrorOrigin::Callee, + }) ); exec_success() @@ -1648,14 +1709,14 @@ mod tests { let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); set_balance(&ALICE, 1000); - assert_matches!( + assert_eq!( ctx.instantiate( 100, &mut GasMeter::::new(GAS_LIMIT), &terminate_ch, vec![], ), - Err(DispatchError::Other("insufficient remaining balance")) + Err(Error::::NewContractNotFunded.into()) ); assert_eq!( diff --git a/frame/contracts/src/gas.rs b/frame/contracts/src/gas.rs index d6c7f82753ebd..decaf11b796f7 100644 --- a/frame/contracts/src/gas.rs +++ b/frame/contracts/src/gas.rs @@ -14,11 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::Trait; +use crate::{Trait, exec::ExecError}; use sp_std::marker::PhantomData; use sp_runtime::traits::Zero; use frame_support::dispatch::{ - DispatchError, DispatchResultWithPostInfo, PostDispatchInfo, DispatchErrorWithPostInfo, + DispatchResultWithPostInfo, PostDispatchInfo, DispatchErrorWithPostInfo, }; #[cfg(test)] @@ -189,8 +189,9 @@ impl GasMeter { } /// Turn this GasMeter into a DispatchResult that contains the actually used gas. - pub fn into_dispatch_result(self, result: Result) -> DispatchResultWithPostInfo where - E: Into, + pub fn into_dispatch_result(self, result: Result) -> DispatchResultWithPostInfo + where + E: Into, { let post_info = PostDispatchInfo { actual_weight: Some(self.gas_spent()), @@ -199,7 +200,7 @@ impl GasMeter { result .map(|_| post_info) - .map_err(|e| DispatchErrorWithPostInfo { post_info, error: e.into() }) + .map_err(|e| DispatchErrorWithPostInfo { post_info, error: e.into().error }) } #[cfg(test)] diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 6d0b481dd0d47..24e5ece5bb8d8 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -95,6 +95,7 @@ use crate::wasm::{WasmLoader, WasmVm}; pub use crate::gas::{Gas, GasMeter}; pub use crate::exec::{ExecResult, ExecReturnValue}; +pub use crate::wasm::ReturnCode as RuntimeReturnCode; #[cfg(feature = "std")] use serde::{Serialize, Deserialize}; @@ -420,9 +421,30 @@ decl_error! { /// the subsistence threshold. No transfer is allowed to do this in order to allow /// for a tombstone to be created. Use `ext_terminate` to remove a contract without /// leaving a tombstone behind. - InsufficientBalance, + BelowSubsistenceThreshold, + /// The newly created contract is below the subsistence threshold after executing + /// its contructor. No contracts are allowed to exist below that threshold. + NewContractNotFunded, + /// Performing the requested transfer failed for a reason originating in the + /// chosen currency implementation of the runtime. Most probably the balance is + /// too low or locks are placed on it. + TransferFailed, + /// Performing a call was denied because the calling depth reached the limit + /// of what is specified in the schedule. + MaxCallDepthReached, + /// The contract that was called is either no contract at all (a plain account) + /// or is a tombstone. + NotCallable, /// The code supplied to `put_code` exceeds the limit specified in the current schedule. CodeTooLarge, + /// No code could be found at the supplied code hash. + CodeNotFound, + /// A buffer outside of sandbox memory was passed to a contract API function. + OutOfBounds, + /// Input passed to a contract API function failed to decode as expected type. + DecodingFailed, + /// Contract trapped during execution. + ContractTrapped, } } diff --git a/frame/contracts/src/storage.rs b/frame/contracts/src/storage.rs index 4c5ad892a967b..3740952778fd3 100644 --- a/frame/contracts/src/storage.rs +++ b/frame/contracts/src/storage.rs @@ -149,6 +149,7 @@ pub fn set_rent_allowance( } /// Returns the code hash of the contract specified by `account` ID. +#[cfg(test)] pub fn code_hash(account: &AccountIdOf) -> Result, ContractAbsentError> { >::get(account) .and_then(|i| i.as_alive().map(|i| i.code_hash)) diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 5b2d7feb3d128..37ded30a6934b 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -17,7 +17,7 @@ use crate::{ BalanceOf, ContractAddressFor, ContractInfo, ContractInfoOf, GenesisConfig, Module, RawAliveContractInfo, RawEvent, Trait, TrieId, Schedule, TrieIdGenerator, gas::Gas, - Error, + Error, Config, RuntimeReturnCode, }; use assert_matches::assert_matches; use hex_literal::*; @@ -30,8 +30,9 @@ use sp_runtime::{ use frame_support::{ assert_ok, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event, impl_outer_origin, parameter_types, StorageMap, StorageValue, - traits::{Currency, Get}, + traits::{Currency, Get, ReservableCurrency}, weights::{Weight, PostDispatchInfo}, + dispatch::DispatchErrorWithPostInfo, }; use std::cell::RefCell; use frame_system::{self as system, EventRecord, Phase}; @@ -63,6 +64,7 @@ impl_outer_dispatch! { } } +#[macro_use] pub mod test_utils { use super::{Test, Balances}; use crate::{ContractInfoOf, TrieIdGenerator, CodeHash}; @@ -89,6 +91,12 @@ pub mod test_utils { pub fn get_balance(who: &u64) -> u64 { Balances::free_balance(who) } + macro_rules! assert_return_code { + ( $x:expr , $y:expr $(,)? ) => {{ + use sp_std::convert::TryInto; + assert_eq!(u32::from_le_bytes($x.data[..].try_into().unwrap()), $y as u32); + }} + } } thread_local! { @@ -279,19 +287,23 @@ where Ok((wasm_binary, code_hash)) } -// Perform a simple transfer to a non-existent account. +// Perform a call to a plain account. +// The actual transfer fails because we can only call contracts. // Then we check that only the base costs are returned as actual costs. #[test] -fn returns_base_call_cost() { +fn calling_plain_account_fails() { ExtBuilder::default().build().execute_with(|| { let _ = Balances::deposit_creating(&ALICE, 100_000_000); assert_eq!( Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, Vec::new()), - Ok( - PostDispatchInfo { - actual_weight: Some(67500000), - pays_fee: Default::default(), + Err( + DispatchErrorWithPostInfo { + error: Error::::NotCallable.into(), + post_info: PostDispatchInfo { + actual_weight: Some(67500000), + pays_fee: Default::default(), + }, } ) ); @@ -987,7 +999,7 @@ fn call_removed_contract() { // Calling contract should remove contract and fail. assert_err_ignore_postinfo!( Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), - "contract has been evicted" + Error::::NotCallable ); // Calling a contract that is about to evict shall emit an event. assert_eq!(System::events(), vec![ @@ -1001,7 +1013,7 @@ fn call_removed_contract() { // Subsequent contract calls should also fail. assert_err_ignore_postinfo!( Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), - "contract has been evicted" + Error::::NotCallable ); }) } @@ -1128,7 +1140,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: // we expect that it will get removed leaving tombstone. assert_err_ignore_postinfo!( Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), - "contract has been evicted" + Error::::NotCallable ); assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); assert_eq!(System::events(), vec![ @@ -1181,7 +1193,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_err_ignore_postinfo!( perform_the_restoration(), - "contract trapped during execution" + Error::::ContractTrapped, ); assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); @@ -1309,7 +1321,7 @@ fn storage_max_value_limit() { GAS_LIMIT, Encode::encode(&(self::MaxValueSize::get() + 1)), ), - "contract trapped during execution" + Error::::ContractTrapped, ); }); } @@ -1373,17 +1385,16 @@ fn cannot_self_destruct_through_draning() { Some(ContractInfo::Alive(_)) ); - // Call BOB with no input data, forcing it to run until out-of-balance - // and eventually trapping because below existential deposit. - assert_err_ignore_postinfo!( + // Call BOB which makes it send all funds to the zero address + // The contract code asserts that the correct error value is returned. + assert_ok!( Contracts::call( Origin::signed(ALICE), BOB, 0, GAS_LIMIT, vec![], - ), - "contract trapped during execution" + ) ); }); } @@ -1423,7 +1434,7 @@ fn cannot_self_destruct_while_live() { GAS_LIMIT, vec![0], ), - "contract trapped during execution" + Error::::ContractTrapped, ); // Check that BOB is still alive. @@ -1535,8 +1546,7 @@ fn cannot_self_destruct_in_constructor() { let _ = Balances::deposit_creating(&ALICE, 1_000_000); assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - // Fail to instantiate the BOB because the call that is issued in the deploy - // function exhausts all balances which puts it below the existential deposit. + // Fail to instantiate the BOB because the contructor calls ext_terminate. assert_err_ignore_postinfo!( Contracts::instantiate( Origin::signed(ALICE), @@ -1545,7 +1555,7 @@ fn cannot_self_destruct_in_constructor() { code_hash.into(), vec![], ), - "contract trapped during execution" + Error::::NewContractNotFunded, ); }); } @@ -1603,3 +1613,216 @@ fn crypto_hashes() { } }) } + +#[test] +fn transfer_return_code() { + let (wasm, code_hash) = compile_module::("transfer_return_code").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let subsistence = Config::::subsistence_threshold_uncached(); + let _ = Balances::deposit_creating(&ALICE, 10 * subsistence); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + + assert_ok!( + Contracts::instantiate( + Origin::signed(ALICE), + subsistence, + GAS_LIMIT, + code_hash.into(), + vec![], + ), + ); + + // Contract has only the minimal balance so any transfer will return BelowSubsistence. + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::BelowSubsistenceThreshold); + + // Contract has enough total balance in order to not go below the subsistence + // threshold when transfering 100 balance but this balance is reserved so + // the transfer still fails but with another return code. + Balances::make_free_balance_be(&BOB, subsistence + 100); + Balances::reserve(&BOB, subsistence + 100).unwrap(); + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); + }); +} + +#[test] +fn call_return_code() { + let (caller_code, caller_hash) = compile_module::("call_return_code").unwrap(); + let (callee_code, callee_hash) = compile_module::("ok_trap_revert").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let subsistence = Config::::subsistence_threshold_uncached(); + let _ = Balances::deposit_creating(&ALICE, 10 * subsistence); + let _ = Balances::deposit_creating(&CHARLIE, 10 * subsistence); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_code)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_code)); + + assert_ok!( + Contracts::instantiate( + Origin::signed(ALICE), + subsistence, + GAS_LIMIT, + caller_hash.into(), + vec![0], + ), + ); + + // Contract calls into Django which is no valid contract + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![0], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::NotCallable); + + assert_ok!( + Contracts::instantiate( + Origin::signed(CHARLIE), + subsistence, + GAS_LIMIT, + callee_hash.into(), + vec![0], + ), + ); + + // Contract has only the minimal balance so any transfer will return BelowSubsistence. + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![0], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::BelowSubsistenceThreshold); + + // Contract has enough total balance in order to not go below the subsistence + // threshold when transfering 100 balance but this balance is reserved so + // the transfer still fails but with another return code. + Balances::make_free_balance_be(&BOB, subsistence + 100); + Balances::reserve(&BOB, subsistence + 100).unwrap(); + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![0], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); + + // Contract has enough balance but callee reverts because "1" is passed. + Balances::make_free_balance_be(&BOB, subsistence + 1000); + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![1], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::CalleeReverted); + + // Contract has enough balance but callee traps because "2" is passed. + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![2], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::CalleeTrapped); + + }); +} + +#[test] +fn instantiate_return_code() { + let (caller_code, caller_hash) = compile_module::("instantiate_return_code").unwrap(); + let (callee_code, callee_hash) = compile_module::("ok_trap_revert").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let subsistence = Config::::subsistence_threshold_uncached(); + let _ = Balances::deposit_creating(&ALICE, 10 * subsistence); + let _ = Balances::deposit_creating(&CHARLIE, 10 * subsistence); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_code)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_code)); + let callee_hash = callee_hash.as_ref().to_vec(); + + assert_ok!( + Contracts::instantiate( + Origin::signed(ALICE), + subsistence, + GAS_LIMIT, + caller_hash.into(), + vec![], + ), + ); + + // Contract has only the minimal balance so any transfer will return BelowSubsistence. + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![0; 33], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::BelowSubsistenceThreshold); + + // Contract has enough total balance in order to not go below the subsistence + // threshold when transfering 100 balance but this balance is reserved so + // the transfer still fails but with another return code. + Balances::make_free_balance_be(&BOB, subsistence + 100); + Balances::reserve(&BOB, subsistence + 100).unwrap(); + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![0; 33], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); + + // Contract has enough balance but the passed code hash is invalid + Balances::make_free_balance_be(&BOB, subsistence + 1000); + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + vec![0; 33], + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::CodeNotFound); + + // Contract has enough balance but callee reverts because "1" is passed. + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + callee_hash.iter().cloned().chain(sp_std::iter::once(1)).collect(), + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::CalleeReverted); + + // Contract has enough balance but callee traps because "2" is passed. + let result = Contracts::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + callee_hash.iter().cloned().chain(sp_std::iter::once(2)).collect(), + ).0.unwrap(); + assert_return_code!(result, RuntimeReturnCode::CalleeTrapped); + + }); +} diff --git a/frame/contracts/src/wasm/mod.rs b/frame/contracts/src/wasm/mod.rs index 68dbae896b0c6..7f985e90b66b6 100644 --- a/frame/contracts/src/wasm/mod.rs +++ b/frame/contracts/src/wasm/mod.rs @@ -36,6 +36,7 @@ use self::runtime::{to_execution_result, Runtime}; use self::code_cache::load as load_code; pub use self::code_cache::save as save_code; +pub use self::runtime::ReturnCode; /// A prepared wasm module ready for execution. #[derive(Clone, Encode, Decode)] @@ -152,13 +153,12 @@ mod tests { use super::*; use std::collections::HashMap; use sp_core::H256; - use crate::exec::{Ext, StorageKey, ExecReturnValue, ReturnFlags}; + use crate::exec::{Ext, StorageKey, ExecReturnValue, ReturnFlags, ExecError, ErrorOrigin}; use crate::gas::{Gas, GasMeter}; use crate::tests::{Test, Call}; use crate::wasm::prepare::prepare_contract; - use crate::{CodeHash, BalanceOf}; + use crate::{CodeHash, BalanceOf, Error}; use hex_literal::hex; - use assert_matches::assert_matches; use sp_runtime::DispatchError; use frame_support::weights::Weight; @@ -225,7 +225,7 @@ mod tests { endowment: u64, gas_meter: &mut GasMeter, data: Vec, - ) -> Result<(u64, ExecReturnValue), DispatchError> { + ) -> Result<(u64, ExecReturnValue), ExecError> { self.instantiates.push(InstantiateEntry { code_hash: code_hash.clone(), endowment, @@ -365,7 +365,7 @@ mod tests { value: u64, gas_meter: &mut GasMeter, input_data: Vec, - ) -> Result<(u64, ExecReturnValue), DispatchError> { + ) -> Result<(u64, ExecReturnValue), ExecError> { (**self).instantiate(code, value, gas_meter, input_data) } fn transfer( @@ -483,14 +483,16 @@ mod tests { ;; value_ptr: u32, ;; value_len: u32, ;;) -> u32 - (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32))) + (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32) (result i32))) (import "env" "memory" (memory 1 1)) (func (export "call") - (call $ext_transfer - (i32.const 4) ;; Pointer to "account" address. - (i32.const 8) ;; Length of "account" address. - (i32.const 12) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. + (drop + (call $ext_transfer + (i32.const 4) ;; Pointer to "account" address. + (i32.const 8) ;; Length of "account" address. + (i32.const 12) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + ) ) ) (func (export "deploy")) @@ -521,7 +523,7 @@ mod tests { to: 7, value: 153, data: Vec::new(), - gas_left: 9989500000, + gas_left: 9989000000, }] ); } @@ -1503,14 +1505,17 @@ mod tests { // Checks that the runtime traps if there are more than `max_topic_events` topics. let mut gas_meter = GasMeter::new(GAS_LIMIT); - assert_matches!( + assert_eq!( execute( CODE_DEPOSIT_EVENT_MAX_TOPICS, vec![], MockExt::default(), &mut gas_meter ), - Err(DispatchError::Other("contract trapped during execution")) + Err(ExecError { + error: Error::::ContractTrapped.into(), + origin: ErrorOrigin::Caller, + }) ); } @@ -1545,14 +1550,17 @@ mod tests { // Checks that the runtime traps if there are duplicates. let mut gas_meter = GasMeter::new(GAS_LIMIT); - assert_matches!( + assert_eq!( execute( CODE_DEPOSIT_EVENT_DUPLICATES, vec![], MockExt::default(), &mut gas_meter ), - Err(DispatchError::Other("contract trapped during execution")) + Err(ExecError { + error: Error::::ContractTrapped.into(), + origin: ErrorOrigin::Caller, + }) ); } @@ -1666,4 +1674,75 @@ mod tests { assert_eq!(output, ExecReturnValue { flags: ReturnFlags::REVERT, data: hex!("5566778899").to_vec() }); assert!(!output.is_success()); } + + const CODE_OUT_OF_BOUNDS_ACCESS: &str = r#" +(module + (import "env" "ext_terminate" (func $ext_terminate (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "deploy")) + + (func (export "call") + (call $ext_terminate + (i32.const 65536) ;; Pointer to "account" address (out of bound). + (i32.const 8) ;; Length of "account" address. + ) + ) +) +"#; + + #[test] + fn contract_out_of_bounds_access() { + let mut mock_ext = MockExt::default(); + let result = execute( + CODE_OUT_OF_BOUNDS_ACCESS, + vec![], + &mut mock_ext, + &mut GasMeter::new(GAS_LIMIT), + ); + + assert_eq!( + result, + Err(ExecError { + error: Error::::OutOfBounds.into(), + origin: ErrorOrigin::Caller, + }) + ); + } + + const CODE_DECODE_FAILURE: &str = r#" +(module + (import "env" "ext_terminate" (func $ext_terminate (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "deploy")) + + (func (export "call") + (call $ext_terminate + (i32.const 0) ;; Pointer to "account" address. + (i32.const 4) ;; Length of "account" address (too small -> decode fail). + ) + ) +) +"#; + + #[test] + fn contract_decode_failure() { + let mut mock_ext = MockExt::default(); + let result = execute( + CODE_DECODE_FAILURE, + vec![], + &mut mock_ext, + &mut GasMeter::new(GAS_LIMIT), + ); + + assert_eq!( + result, + Err(ExecError { + error: Error::::DecodingFailed.into(), + origin: ErrorOrigin::Caller, + }) + ); + } + } diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index ab93076f57b2a..ed97a4dae3c9f 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -18,7 +18,7 @@ use crate::{Schedule, Trait, CodeHash, BalanceOf, Error}; use crate::exec::{ - Ext, ExecResult, ExecReturnValue, StorageKey, TopicOf, ReturnFlags, + Ext, ExecResult, ExecReturnValue, StorageKey, TopicOf, ReturnFlags, ExecError }; use crate::gas::{Gas, GasMeter, Token, GasMeterResult}; use crate::wasm::env_def::ConvertibleToWasm; @@ -36,21 +36,33 @@ use sp_io::hashing::{ sha2_256, }; -/// Every error that can be returned from a runtime API call. +/// Every error that can be returned to a contract when it calls any of the host functions. #[repr(u32)] pub enum ReturnCode { /// API call successful. Success = 0, /// The called function trapped and has its state changes reverted. /// In this case no output buffer is returned. - /// Can only be returned from `ext_call` and `ext_instantiate`. CalleeTrapped = 1, /// The called function ran to completion but decided to revert its state. /// An output buffer is returned when one was supplied. - /// Can only be returned from `ext_call` and `ext_instantiate`. CalleeReverted = 2, /// The passed key does not exist in storage. KeyNotFound = 3, + /// Transfer failed because it would have brought the sender's total balance below the + /// subsistence threshold. + BelowSubsistenceThreshold = 4, + /// Transfer failed for other reasons. Most probably reserved or locked balance of the + /// sender prevents the transfer. + TransferFailed = 5, + /// The newly created contract is below the subsistence threshold after executing + /// its constructor. + NewContractNotFunded = 6, + /// No code could be found at the supplied code hash. + CodeNotFound = 7, + /// The contract that was called is either no contract at all (a plain account) + /// or is a tombstone. + NotCallable = 8, } impl ConvertibleToWasm for ReturnCode { @@ -66,7 +78,7 @@ impl ConvertibleToWasm for ReturnCode { } impl From for ReturnCode { - fn from(from: ExecReturnValue) -> ReturnCode { + fn from(from: ExecReturnValue) -> Self { if from.flags.contains(ReturnFlags::REVERT) { Self::CalleeReverted } else { @@ -96,7 +108,7 @@ enum TrapReason { SupervisorError(DispatchError), /// Signals that trap was generated in response to call `ext_return` host function. Return(ReturnData), - /// Signals that a trap was generated in response to a succesful call to the + /// Signals that a trap was generated in response to a successful call to the /// `ext_terminate` host function. Termination, /// Signals that a trap was generated because of a successful restoration. @@ -131,35 +143,42 @@ impl<'a, E: Ext + 'a> Runtime<'a, E> { } } +/// Converts the sandbox result and the runtime state into the execution outcome. +/// +/// It evaluates information stored in the `trap_reason` variable of the runtime and +/// bases the outcome on the value if this variable. Only if `trap_reason` is `None` +/// the result of the sandbox is evaluated. pub(crate) fn to_execution_result( runtime: Runtime, sandbox_result: Result, ) -> ExecResult { - match runtime.trap_reason { - // The trap was the result of the execution `return` host function. - Some(TrapReason::Return(ReturnData{ flags, data })) => { - let flags = ReturnFlags::from_bits(flags).ok_or_else(|| - "used reserved bit in return flags" - )?; - return Ok(ExecReturnValue { - flags, - data, - }) - }, - Some(TrapReason::Termination) => { - return Ok(ExecReturnValue { - flags: ReturnFlags::empty(), - data: Vec::new(), - }) - }, - Some(TrapReason::Restoration) => { - return Ok(ExecReturnValue { - flags: ReturnFlags::empty(), - data: Vec::new(), - }) + // If a trap reason is set we base our decision solely on that. + if let Some(trap_reason) = runtime.trap_reason { + return match trap_reason { + // The trap was the result of the execution `return` host function. + TrapReason::Return(ReturnData{ flags, data }) => { + let flags = ReturnFlags::from_bits(flags).ok_or_else(|| + "used reserved bit in return flags" + )?; + Ok(ExecReturnValue { + flags, + data, + }) + }, + TrapReason::Termination => { + Ok(ExecReturnValue { + flags: ReturnFlags::empty(), + data: Vec::new(), + }) + }, + TrapReason::Restoration => { + Ok(ExecReturnValue { + flags: ReturnFlags::empty(), + data: Vec::new(), + }) + }, + TrapReason::SupervisorError(error) => Err(error)?, } - Some(TrapReason::SupervisorError(error)) => Err(error)?, - None => (), } // Check the exact type of the error. @@ -178,7 +197,7 @@ pub(crate) fn to_execution_result( Err("validation error")?, // Any other kind of a trap should result in a failure. Err(sp_sandbox::Error::Execution) | Err(sp_sandbox::Error::OutOfBounds) => - Err("contract trapped during execution")?, + Err(Error::::ContractTrapped)? } } @@ -280,7 +299,8 @@ fn read_sandbox_memory( )?; let mut buf = vec![0u8; len as usize]; - ctx.memory.get(ptr, buf.as_mut_slice()).map_err(|_| sp_sandbox::HostError)?; + ctx.memory.get(ptr, buf.as_mut_slice()) + .map_err(|_| store_err(ctx, Error::::OutOfBounds))?; Ok(buf) } @@ -304,7 +324,7 @@ fn read_sandbox_memory_into_buf( RuntimeToken::ReadMemory(buf.len() as u32), )?; - ctx.memory.get(ptr, buf).map_err(Into::into) + ctx.memory.get(ptr, buf).map_err(|_| store_err(ctx, Error::::OutOfBounds)) } /// Read designated chunk from the sandbox memory, consuming an appropriate amount of @@ -322,7 +342,7 @@ fn read_sandbox_memory_as( len: u32, ) -> Result { let buf = read_sandbox_memory(ctx, ptr, len)?; - D::decode(&mut &buf[..]).map_err(|_| sp_sandbox::HostError) + D::decode(&mut &buf[..]).map_err(|_| store_err(ctx, Error::::DecodingFailed)) } /// Write the given buffer to the designated location in the sandbox memory, consuming @@ -345,9 +365,8 @@ fn write_sandbox_memory( RuntimeToken::WriteMemory(buf.len() as u32), )?; - ctx.memory.set(ptr, buf)?; - - Ok(()) + ctx.memory.set(ptr, buf) + .map_err(|_| store_err(ctx, Error::::OutOfBounds)) } /// Write the given buffer and its length to the designated locations in sandbox memory. @@ -379,7 +398,7 @@ fn write_sandbox_output( let len: u32 = read_sandbox_memory_as(ctx, out_len_ptr, 4)?; if len < buf_len { - Err(map_err(ctx, Error::::OutputBufferTooSmall))? + Err(store_err(ctx, Error::::OutputBufferTooSmall))? } charge_gas( @@ -398,7 +417,7 @@ fn write_sandbox_output( /// Stores a DispatchError returned from an Ext function into the trap_reason. /// /// This allows through supervisor generated errors to the caller. -fn map_err(ctx: &mut Runtime, err: Error) -> sp_sandbox::HostError where +fn store_err(ctx: &mut Runtime, err: Error) -> sp_sandbox::HostError where E: Ext, Error: Into, { @@ -406,12 +425,86 @@ fn map_err(ctx: &mut Runtime, err: Error) -> sp_sandbox::HostError sp_sandbox::HostError } +/// Fallible conversion of `DispatchError` to `ReturnCode`. +fn err_into_return_code(from: DispatchError) -> Result { + use ReturnCode::*; + + let below_sub = Error::::BelowSubsistenceThreshold.into(); + let transfer_failed = Error::::TransferFailed.into(); + let not_funded = Error::::NewContractNotFunded.into(); + let no_code = Error::::CodeNotFound.into(); + let invalid_contract = Error::::NotCallable.into(); + + match from { + x if x == below_sub => Ok(BelowSubsistenceThreshold), + x if x == transfer_failed => Ok(TransferFailed), + x if x == not_funded => Ok(NewContractNotFunded), + x if x == no_code => Ok(CodeNotFound), + x if x == invalid_contract => Ok(NotCallable), + err => Err(err) + } +} + +/// Fallible conversion of a `ExecResult` to `ReturnCode`. +fn exec_into_return_code(from: ExecResult) -> Result { + use crate::exec::ErrorOrigin::Callee; + + let ExecError { error, origin } = match from { + Ok(retval) => return Ok(retval.into()), + Err(err) => err, + }; + + match (error, origin) { + (_, Callee) => Ok(ReturnCode::CalleeTrapped), + (err, _) => err_into_return_code::(err) + } +} + +/// Used by Runtime API that calls into other contracts. +/// +/// Those need to transform the the `ExecResult` returned from the execution into +/// a `ReturnCode`. If this conversion fails because the `ExecResult` constitutes a +/// a fatal error then this error is stored in the `ExecutionContext` so it can be +/// extracted for display in the UI. +fn map_exec_result(ctx: &mut Runtime, result: ExecResult) + -> Result +{ + match exec_into_return_code::(result) { + Ok(code) => Ok(code), + Err(err) => Err(store_err(ctx, err)), + } +} + +/// Try to convert an error into a `ReturnCode`. +/// +/// Used to decide between fatal and non-fatal errors. +fn map_dispatch_result(ctx: &mut Runtime, result: Result) + -> Result +{ + let err = if let Err(err) = result { + err + } else { + return Ok(ReturnCode::Success) + }; + + match err_into_return_code::(err) { + Ok(code) => Ok(code), + Err(err) => Err(store_err(ctx, err)), + } +} + // *********************************************************** // * AFTER MAKING A CHANGE MAKE SURE TO UPDATE COMPLEXITY.MD * // *********************************************************** // Define a function `fn init_env() -> HostFunctionSet` that returns // a function set which can be imported by an executed contract. +// +// # Note +// +// Any input that leads to a out of bound error (reading or writing) or failing to decode +// data passed to the supervisor will lead to a trap. This is not documented explicitly +// for every function. define_env!(Env, , // Account for used gas. Traps if gas used is greater than gas limit. @@ -441,7 +534,7 @@ define_env!(Env, , // - `value_ptr`: pointer into the linear memory where the value to set is placed. // - `value_len`: the length of the value in bytes. // - // # Errors + // # Traps // // - If value length exceeds the configured maximum value length of a storage entry. // - Upon trying to set an empty storage entry (value length is 0). @@ -480,12 +573,7 @@ define_env!(Env, , // // # Errors // - // If there is no entry under the given key then this function will return - // `ReturnCode::KeyNotFound`. - // - // # Traps - // - // Traps if the supplied buffer length is smaller than the size of the stored value. + // `ReturnCode::KeyNotFound` ext_get_storage(ctx, key_ptr: u32, out_ptr: u32, out_len_ptr: u32) -> ReturnCode => { let mut key: StorageKey = [0; 32]; read_sandbox_memory_into_buf(ctx, key_ptr, &mut key)?; @@ -508,24 +596,24 @@ define_env!(Env, , // Should be decodable as a `T::Balance`. Traps otherwise. // - value_len: length of the value buffer. // - // # Traps + // # Errors // - // Traps if the transfer wasn't succesful. This can happen when the value transfered - // brings the sender below the existential deposit. Use `ext_terminate` to remove - // the caller contract. + // `ReturnCode::BelowSubsistenceThreshold` + // `ReturnCode::TransferFailed` ext_transfer( ctx, account_ptr: u32, account_len: u32, value_ptr: u32, value_len: u32 - ) => { + ) -> ReturnCode => { let callee: <::T as frame_system::Trait>::AccountId = read_sandbox_memory_as(ctx, account_ptr, account_len)?; let value: BalanceOf<::T> = read_sandbox_memory_as(ctx, value_ptr, value_len)?; - ctx.ext.transfer(&callee, value, ctx.gas_meter).map_err(|e| map_err(ctx, e)) + let result = ctx.ext.transfer(&callee, value, ctx.gas_meter); + map_dispatch_result(ctx, result) }, // Make a call to another contract. @@ -551,17 +639,14 @@ define_env!(Env, , // // # Errors // - // `ReturnCode::CalleeReverted`: The callee ran to completion but decided to have its - // changes reverted. The delivery of the output buffer is still possible. - // `ReturnCode::CalleeTrapped`: The callee trapped during execution. All changes are reverted - // and no output buffer is delivered. - // - // # Traps + // An error means that the call wasn't successful output buffer is returned unless + // stated otherwise. // - // - Transfer of balance failed. This call can not bring the sender below the existential - // deposit. Use `ext_terminate` to remove the caller. - // - Callee does not exist. - // - Supplied output buffer is too small. + // `ReturnCode::CalleeReverted`: Output buffer is returned. + // `ReturnCode::CalleeTrapped` + // `ReturnCode::BelowSubsistenceThreshold` + // `ReturnCode::TransferFailed` + // `ReturnCode::NotCallable` ext_call( ctx, callee_ptr: u32, @@ -594,22 +679,16 @@ define_env!(Env, , nested_meter, input_data, ) - .map_err(|_| ()) } // there is not enough gas to allocate for the nested call. - None => Err(()), + None => Err(Error::<::T>::OutOfGas.into()), } }); - match call_outcome { - Ok(output) => { - write_sandbox_output(ctx, output_ptr, output_len_ptr, &output.data, true)?; - Ok(output.into()) - }, - Err(_) => { - Ok(ReturnCode::CalleeTrapped) - }, + if let Ok(output) = &call_outcome { + write_sandbox_output(ctx, output_ptr, output_len_ptr, &output.data, true)?; } + map_exec_result(ctx, call_outcome) }, // Instantiate a contract with the specified code hash. @@ -643,19 +722,18 @@ define_env!(Env, , // // # Errors // - // `ReturnCode::CalleeReverted`: The callee's constructor ran to completion but decided to have - // its changes reverted. The delivery of the output buffer is still possible but the - // account was not created and no address is returned. - // `ReturnCode::CalleeTrapped`: The callee trapped during execution. All changes are reverted - // and no output buffer is delivered. The accounts was not created and no address is - // returned. + // Please consult the `ReturnCode` enum declaration for more information on those + // errors. Here we only note things specific to this function. // - // # Traps + // An error means that the account wasn't created and no address or output buffer + // is returned unless stated otherwise. // - // - Transfer of balance failed. This call can not bring the sender below the existential - // deposit. Use `ext_terminate` to remove the caller. - // - Code hash does not exist. - // - Supplied output buffers are too small. + // `ReturnCode::CalleeReverted`: Output buffer is returned. + // `ReturnCode::CalleeTrapped` + // `ReturnCode::BelowSubsistenceThreshold` + // `ReturnCode::TransferFailed` + // `ReturnCode::NewContractNotFunded` + // `ReturnCode::CodeNotFound` ext_instantiate( ctx, code_hash_ptr: u32, @@ -690,26 +768,20 @@ define_env!(Env, , nested_meter, input_data ) - .map_err(|_| ()) } // there is not enough gas to allocate for the nested call. - None => Err(()), + None => Err(Error::<::T>::OutOfGas.into()), } }); - match instantiate_outcome { - Ok((address, output)) => { - if !output.flags.contains(ReturnFlags::REVERT) { - write_sandbox_output( - ctx, address_ptr, address_len_ptr, &address.encode(), true - )?; - } - write_sandbox_output(ctx, output_ptr, output_len_ptr, &output.data, true)?; - Ok(output.into()) - }, - Err(_) => { - Ok(ReturnCode::CalleeTrapped) - }, + if let Ok((address, output)) = &instantiate_outcome { + if !output.flags.contains(ReturnFlags::REVERT) { + write_sandbox_output( + ctx, address_ptr, address_len_ptr, &address.encode(), true + )?; + } + write_sandbox_output(ctx, output_ptr, output_len_ptr, &output.data, true)?; } + map_exec_result(ctx, instantiate_outcome.map(|(_id, retval)| retval)) }, // Remove the calling account and transfer remaining balance. @@ -722,6 +794,10 @@ define_env!(Env, , // where all remaining funds of the caller are transfered. // Should be decodable as an `T::AccountId`. Traps otherwise. // - beneficiary_len: length of the address buffer. + // + // # Traps + // + // - The contract is live i.e is already on the call stack. ext_terminate( ctx, beneficiary_ptr: u32, @@ -939,6 +1015,11 @@ define_env!(Env, , // encodes the rent allowance that must be set in the case of successful restoration. // `delta_ptr` is the pointer to the start of a buffer that has `delta_count` storage keys // laid out sequentially. + // + // # Traps + // + // - Tombstone hashes do not match + // - Calling cantract is live i.e is already on the call stack. ext_restore_to( ctx, dest_ptr: u32, From 2bd4f4515fbfd9c7433eb67b5de87549fea0bc89 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Mon, 3 Aug 2020 14:41:54 +0200 Subject: [PATCH 10/32] Improve Benchmark Writer: Remove Unused Components, Remove Multiply by Zero, Files Split by Pallet (#6785) * initial improvements * better file management, ignore unused components * Output warning when components unused * update comment * Write even when base weight is zero * remove unwrap where possible * Dont sort components to dedup * undo delete * improve clarity of unused components * remove unused dep * Update Process.json --- Cargo.lock | 1 - Process.json | 5 + frame/benchmarking/src/utils.rs | 7 + utils/frame/benchmarking-cli/Cargo.toml | 1 - utils/frame/benchmarking-cli/src/command.rs | 3 +- utils/frame/benchmarking-cli/src/writer.rs | 193 +++++++++++--------- 6 files changed, 120 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b86507cf072f2..fe237f2341b23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1572,7 +1572,6 @@ dependencies = [ name = "frame-benchmarking-cli" version = "2.0.0-rc5" dependencies = [ - "Inflector", "frame-benchmarking", "parity-scale-codec", "sc-cli", diff --git a/Process.json b/Process.json index cd15e137df6ed..540bd644311cf 100644 --- a/Process.json +++ b/Process.json @@ -21,4 +21,9 @@ "project_name": "Smart Contracts", "owner": "pepyakin", "matrix_room_id": "!yBKstWVBkwzUkPslsp:matrix.parity.io" +}, +{ + "project_name": "Benchmarking and Weights", + "owner": "shawntabrizi", + "matrix_room_id": "!pZPWqCRLVtORZTEsEf:matrix.parity.io" }] diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index 7ed9a862a0479..5a2bd55ff79aa 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -30,6 +30,13 @@ pub enum BenchmarkParameter { a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, } +#[cfg(feature = "std")] +impl std::fmt::Display for BenchmarkParameter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + /// The results of a single of benchmark. #[derive(Encode, Decode, Clone, PartialEq, Debug)] pub struct BenchmarkBatch { diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 4c522337259ef..c34404575e5fc 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -12,7 +12,6 @@ description = "CLI for benchmarking FRAME" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -Inflector = "0.11.4" frame-benchmarking = { version = "2.0.0-rc5", path = "../../../frame/benchmarking" } sp-core = { version = "2.0.0-rc5", path = "../../../primitives/core" } sc-service = { version = "0.8.0-rc5", default-features = false, path = "../../../client/service" } diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 553b68c453fa7..688e393bd605a 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -95,8 +95,7 @@ impl BenchmarkCmd { let mut file = crate::writer::open_file("traits.rs")?; crate::writer::write_trait(&mut file, batches.clone())?; } else { - let mut file = crate::writer::open_file("benchmarks.rs")?; - crate::writer::write_results(&mut file, batches.clone())?; + crate::writer::write_results(&batches)?; } } diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 199dbb795e581..2bc17aa85bddb 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -20,13 +20,15 @@ use std::fs::{File, OpenOptions}; use std::io::prelude::*; use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; -use inflector::Inflector; +use sp_runtime::traits::Zero; + +const VERSION: &'static str = env!("CARGO_PKG_VERSION"); pub fn open_file(path: &str) -> Result { OpenOptions::new() .create(true) .write(true) - .append(true) + .truncate(true) .open(path) } @@ -47,81 +49,49 @@ pub fn write_trait(file: &mut File, batches: Vec) -> Result<(), if batch.pallet != current_pallet { if !current_pallet.is_empty() { // close trait - write!(file, "}}\n").unwrap(); + write!(file, "}}\n")?; } // trait wrapper - write!(file, "// {}\n", pallet_string).unwrap(); - write!(file, "pub trait WeightInfo {{\n").unwrap(); - - current_pallet = batch.pallet.clone() - } - - // function name - write!(file, "\tfn {}(", benchmark_string).unwrap(); - - // params - let components = &batch.results[0].components; - for component in components { - write!(file, "{:?}: u32, ", component.0).unwrap(); - } - // return value - write!(file, ") -> Weight;\n").unwrap(); - } - - // final close trait - write!(file, "}}\n").unwrap(); - - // Reset - current_pallet = Vec::::new(); - - for batch in &batches { - if batch.results.is_empty() { continue } - - let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); - - // only create new trait definitions when we go to a new pallet - if batch.pallet != current_pallet { - if !current_pallet.is_empty() { - // close trait - write!(file, "}}\n").unwrap(); - } - - // impl trait - write!(file, "\n").unwrap(); - write!(file, "impl WeightInfo for () {{\n").unwrap(); + write!(file, "// {}\n", pallet_string)?; + write!(file, "pub trait WeightInfo {{\n")?; current_pallet = batch.pallet.clone() } // function name - write!(file, "\tfn {}(", benchmark_string).unwrap(); + write!(file, "\tfn {}(", benchmark_string)?; // params let components = &batch.results[0].components; for component in components { - write!(file, "_{:?}: u32, ", component.0).unwrap(); + write!(file, "{:?}: u32, ", component.0)?; } // return value - write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap(); + write!(file, ") -> Weight;\n")?; } // final close trait - write!(file, "}}\n").unwrap(); + write!(file, "}}\n")?; Ok(()) } -pub fn write_results(file: &mut File, batches: Vec) -> Result<(), std::io::Error> { +pub fn write_results(batches: &[BenchmarkBatch]) -> Result<(), std::io::Error> { let mut current_pallet = Vec::::new(); // Skip writing if there are no batches if batches.is_empty() { return Ok(()) } - // general imports - write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap(); + let mut batches_iter = batches.iter().peekable(); - for batch in &batches { + let first_pallet = String::from_utf8( + batches_iter.peek().expect("we checked that batches is not empty").pallet.clone() + ).unwrap(); + let mut file = open_file(&(first_pallet + ".rs"))?; + + + while let Some(batch) = batches_iter.next() { // Skip writing if there are no results if batch.results.is_empty() { continue } @@ -130,69 +100,120 @@ pub fn write_results(file: &mut File, batches: Vec) -> Result<() // only create new trait definitions when we go to a new pallet if batch.pallet != current_pallet { - if !current_pallet.is_empty() { - // close trait - write!(file, "}}\n").unwrap(); - } + // auto-generation note + write!( + file, + "//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {}\n\n", + VERSION, + )?; + + // general imports + write!( + file, + "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n\n" + )?; // struct for weights - write!(file, "pub struct WeightFor{};\n", - pallet_string.to_pascal_case(), - ).unwrap(); + write!(file, "pub struct WeightInfo;\n")?; // trait wrapper - write!(file, "impl {}::WeightInfo for WeightFor{} {{\n", - pallet_string, - pallet_string.to_pascal_case(), - ).unwrap(); + write!(file, "impl {}::WeightInfo for WeightInfo {{\n", pallet_string)?; current_pallet = batch.pallet.clone() } - // function name - write!(file, "\tfn {}(", benchmark_string).unwrap(); + // Analysis results + let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); + let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); + let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); + // Analysis data may include components that are not used, this filters out anything whose value is zero. + let mut used_components = Vec::new(); + let mut used_extrinsic_time = Vec::new(); + let mut used_reads = Vec::new(); + let mut used_writes = Vec::new(); + extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { + if !slope.is_zero() { + if !used_components.contains(&name) { used_components.push(name); } + used_extrinsic_time.push((slope, name)); + } + }); + reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { + if !slope.is_zero() { + if !used_components.contains(&name) { used_components.push(name); } + used_reads.push((slope, name)); + } + }); + writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { + if !slope.is_zero() { + if !used_components.contains(&name) { used_components.push(name); } + used_writes.push((slope, name)); + } + }); + + let all_components = batch.results[0].components + .iter() + .map(|(name, _)| -> String { return name.to_string() }) + .collect::>(); + if all_components.len() != used_components.len() { + let mut unused_components = all_components; + unused_components.retain(|x| !used_components.contains(&x)); + write!(file, "\t// WARNING! Some components were not used: {:?}\n", unused_components)?; + } + + // function name + write!(file, "\tfn {}(", benchmark_string)?; // params - let components = &batch.results[0].components; - for component in components { - write!(file, "{:?}: u32, ", component.0).unwrap(); + for component in used_components { + write!(file, "{}: u32, ", component)?; } // return value - write!(file, ") -> Weight {{\n").unwrap(); + write!(file, ") -> Weight {{\n")?; - let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); - write!(file, "\t\t({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); - extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { + write!(file, "\t\t({} as Weight)\n", extrinsic_time.base.saturating_mul(1000))?; + used_extrinsic_time.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> { write!(file, "\t\t\t.saturating_add(({} as Weight).saturating_mul({} as Weight))\n", slope.saturating_mul(1000), name, - ).unwrap(); - }); + ) + })?; - let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - write!(file, "\t\t\t.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); - reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { + if !reads.base.is_zero() { + write!(file, "\t\t\t.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base)?; + } + used_reads.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> { write!(file, "\t\t\t.saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", slope, name, - ).unwrap(); - }); + ) + })?; - let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - write!(file, "\t\t\t.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); - writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { + if !writes.base.is_zero() { + write!(file, "\t\t\t.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base)?; + } + used_writes.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> { write!(file, "\t\t\t.saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", slope, name, - ).unwrap(); - }); + ) + })?; // close function - write!(file, "\t}}\n").unwrap(); + write!(file, "\t}}\n")?; + + // Check if this is the end of the iterator + if let Some(next) = batches_iter.peek() { + // Next pallet is different than current pallet, so we close up the file and open a new one. + if next.pallet != current_pallet { + write!(file, "}}\n")?; + let next_pallet = String::from_utf8(next.pallet.clone()).unwrap(); + file = open_file(&(next_pallet + ".rs"))?; + } + } else { + // This is the end of the iterator, so we close up the final file. + write!(file, "}}\n")?; + } } - // final close trait - write!(file, "}}\n").unwrap(); - Ok(()) } From 2afbcf7e57c005f5601f6fc40353594923060a88 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Mon, 3 Aug 2020 15:26:09 +0200 Subject: [PATCH 11/32] Add integrity test for slash defer duration (#6782) * Add integrity test for slash defer duration * Wrap in externalities * Update frame/staking/src/lib.rs --- frame/staking/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index cd820051b157f..60943be82f1dc 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1459,6 +1459,17 @@ decl_module! { // `on_finalize` weight is tracked in `on_initialize` } + fn integrity_test() { + sp_io::TestExternalities::new_empty().execute_with(|| + assert!( + T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, + "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", + T::SlashDeferDuration::get(), + T::BondingDuration::get(), + ) + ); + } + /// Take the origin account as a stash and lock up `value` of its balance. `controller` will /// be the account that controls it. /// From 584588b7d9e88a8253ef5f48abd477505714f462 Mon Sep 17 00:00:00 2001 From: Ashley Date: Mon, 3 Aug 2020 15:48:32 +0200 Subject: [PATCH 12/32] Convert spaces to tabs (#6799) --- bin/node/bench/src/core.rs | 24 +- bin/node/bench/src/txpool.rs | 6 +- bin/node/browser-testing/src/lib.rs | 44 +-- client/proposer-metrics/src/lib.rs | 22 +- client/service/test/src/client/light.rs | 4 +- frame/contracts/src/benchmarking.rs | 358 +++++++++--------- frame/staking/src/tests.rs | 6 +- primitives/consensus/common/src/metrics.rs | 2 +- primitives/core/src/sr25519.rs | 12 +- .../runtime/src/offchain/storage_lock.rs | 44 +-- utils/fork-tree/src/lib.rs | 2 +- 11 files changed, 262 insertions(+), 262 deletions(-) diff --git a/bin/node/bench/src/core.rs b/bin/node/bench/src/core.rs index c1b1711549be1..6faa7b72721f4 100644 --- a/bin/node/bench/src/core.rs +++ b/bin/node/bench/src/core.rs @@ -93,26 +93,26 @@ pub enum Mode { } impl std::str::FromStr for Mode { - type Err = &'static str; - fn from_str(day: &str) -> Result { - match day { - "regular" => Ok(Mode::Regular), - "profile" => Ok(Mode::Profile), - _ => Err("Could not parse mode"), - } - } + type Err = &'static str; + fn from_str(day: &str) -> Result { + match day { + "regular" => Ok(Mode::Regular), + "profile" => Ok(Mode::Profile), + _ => Err("Could not parse mode"), + } + } } impl fmt::Display for BenchmarkOutput { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( f, "{}: avg {}, w_avg {}", self.name, NsFormatter(self.raw_average), NsFormatter(self.average), ) - } + } } pub fn run_benchmark( @@ -159,4 +159,4 @@ macro_rules! matrix( } }; () => { vec![] } -); \ No newline at end of file +); diff --git a/bin/node/bench/src/txpool.rs b/bin/node/bench/src/txpool.rs index d6e5578192e01..7ea13fc15ec68 100644 --- a/bin/node/bench/src/txpool.rs +++ b/bin/node/bench/src/txpool.rs @@ -32,7 +32,7 @@ use sp_transaction_pool::{TransactionPool, TransactionSource}; use crate::core::{self, Path, Mode}; pub struct PoolBenchmarkDescription { - pub database_type: DatabaseType, + pub database_type: DatabaseType, } pub struct PoolBenchmark { @@ -41,7 +41,7 @@ pub struct PoolBenchmark { impl core::BenchmarkDescription for PoolBenchmarkDescription { fn path(&self) -> Path { - Path::new(&["node", "txpool"]) + Path::new(&["node", "txpool"]) } fn setup(self: Box) -> Box { @@ -55,7 +55,7 @@ impl core::BenchmarkDescription for PoolBenchmarkDescription { } fn name(&self) -> Cow<'static, str> { - "Transaction pool benchmark".into() + "Transaction pool benchmark".into() } } diff --git a/bin/node/browser-testing/src/lib.rs b/bin/node/browser-testing/src/lib.rs index c943a383aefad..777e5ea9f132e 100644 --- a/bin/node/browser-testing/src/lib.rs +++ b/bin/node/browser-testing/src/lib.rs @@ -37,34 +37,34 @@ use serde::de::DeserializeOwned; wasm_bindgen_test_configure!(run_in_browser); fn rpc_call(method: &str) -> String { - serde_json::to_string(&MethodCall { - jsonrpc: Some(Version::V2), - method: method.into(), - params: Params::None, - id: Id::Num(1) - }).unwrap() + serde_json::to_string(&MethodCall { + jsonrpc: Some(Version::V2), + method: method.into(), + params: Params::None, + id: Id::Num(1) + }).unwrap() } fn deserialize_rpc_result(js_value: JsValue) -> T { - let string = js_value.as_string().unwrap(); - let value = serde_json::from_str::(&string).unwrap().result; - // We need to convert a `Value::Object` into a proper type. - let value_string = serde_json::to_string(&value).unwrap(); - serde_json::from_str(&value_string).unwrap() + let string = js_value.as_string().unwrap(); + let value = serde_json::from_str::(&string).unwrap().result; + // We need to convert a `Value::Object` into a proper type. + let value_string = serde_json::to_string(&value).unwrap(); + serde_json::from_str(&value_string).unwrap() } #[wasm_bindgen_test] async fn runs() { - let mut client = node_cli::start_client(None, "info".into()) - .await - .unwrap(); + let mut client = node_cli::start_client(None, "info".into()) + .await + .unwrap(); - // Check that the node handles rpc calls. - // TODO: Re-add the code that checks if the node is syncing. - let chain_name: String = deserialize_rpc_result( - JsFuture::from(client.rpc_send(&rpc_call("system_chain"))) - .await - .unwrap() - ); - assert_eq!(chain_name, "Development"); + // Check that the node handles rpc calls. + // TODO: Re-add the code that checks if the node is syncing. + let chain_name: String = deserialize_rpc_result( + JsFuture::from(client.rpc_send(&rpc_call("system_chain"))) + .await + .unwrap() + ); + assert_eq!(chain_name, "Development"); } diff --git a/client/proposer-metrics/src/lib.rs b/client/proposer-metrics/src/lib.rs index 5cb749f4a260a..50498d40b62d5 100644 --- a/client/proposer-metrics/src/lib.rs +++ b/client/proposer-metrics/src/lib.rs @@ -41,8 +41,8 @@ impl MetricsLink { /// Authorship metrics. #[derive(Clone)] pub struct Metrics { - pub block_constructed: Histogram, - pub number_of_transactions: Gauge, + pub block_constructed: Histogram, + pub number_of_transactions: Gauge, } impl Metrics { @@ -54,14 +54,14 @@ impl Metrics { "Histogram of time taken to construct new block", ))?, registry, - )?, - number_of_transactions: register( - Gauge::new( - "proposer_number_of_transactions", - "Number of transactions included in block", - )?, - registry, - )?, + )?, + number_of_transactions: register( + Gauge::new( + "proposer_number_of_transactions", + "Number of transactions included in block", + )?, + registry, + )?, }) - } + } } diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 031c234c1ab35..ffc84ad47b8f3 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -686,7 +686,7 @@ fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() { match local_result == expected_result { true => (), false => panic!(format!("Failed test {}: local = {:?}, expected = {:?}", - index, local_result, expected_result)), + index, local_result, expected_result)), } } } @@ -843,7 +843,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { Box::new(TaskExecutor::new()), ); assert!(local_checker.check_changes_tries_proof(4, &remote_proof.roots, - remote_proof.roots_proof.clone()).is_err()); + remote_proof.roots_proof.clone()).is_err()); // fails when proof is broken let mut local_storage = DummyStorage::new(); diff --git a/frame/contracts/src/benchmarking.rs b/frame/contracts/src/benchmarking.rs index 29f992643ef75..4bdb14576eeed 100644 --- a/frame/contracts/src/benchmarking.rs +++ b/frame/contracts/src/benchmarking.rs @@ -28,21 +28,21 @@ use parity_wasm::elements::FuncBody; use sp_runtime::traits::Hash; macro_rules! load_module { - ($name:expr) => {{ - let code = include_bytes!(concat!("../fixtures/benchmarks/", $name, ".wat")); - compile_module::(code) - }}; + ($name:expr) => {{ + let code = include_bytes!(concat!("../fixtures/benchmarks/", $name, ".wat")); + compile_module::(code) + }}; } fn compile_module(code: &[u8]) -> (Vec, ::Output) { - let code = sp_std::str::from_utf8(code).expect("Invalid utf8 in wat file."); - let binary = wat::parse_str(code).expect("Failed to compile wat file."); - let hash = T::Hashing::hash(&binary); - (binary, hash) + let code = sp_std::str::from_utf8(code).expect("Invalid utf8 in wat file."); + let binary = wat::parse_str(code).expect("Failed to compile wat file."); + let hash = T::Hashing::hash(&binary); + (binary, hash) } fn funding() -> BalanceOf { - T::Currency::minimum_balance() * 10_000.into() + T::Currency::minimum_balance() * 10_000.into() } fn create_funded_user(string: &'static str, n: u32) -> T::AccountId { @@ -52,55 +52,55 @@ fn create_funded_user(string: &'static str, n: u32) -> T::AccountId { } fn contract_with_call_body(body: FuncBody) -> (Vec, ::Output) { - use parity_wasm::elements::{ - Instructions, Instruction::End, - }; - let contract = parity_wasm::builder::ModuleBuilder::new() - // deploy function (idx 0) - .function() - .signature().with_params(vec![]).with_return_type(None).build() - .body().with_instructions(Instructions::new(vec![End])).build() - .build() - // call function (idx 1) - .function() - .signature().with_params(vec![]).with_return_type(None).build() - .with_body(body) - .build() - .export().field("deploy").internal().func(0).build() - .export().field("call").internal().func(1).build() - .build(); - let bytes = contract.to_bytes().unwrap(); - let hash = T::Hashing::hash(&bytes); - (bytes, hash) + use parity_wasm::elements::{ + Instructions, Instruction::End, + }; + let contract = parity_wasm::builder::ModuleBuilder::new() + // deploy function (idx 0) + .function() + .signature().with_params(vec![]).with_return_type(None).build() + .body().with_instructions(Instructions::new(vec![End])).build() + .build() + // call function (idx 1) + .function() + .signature().with_params(vec![]).with_return_type(None).build() + .with_body(body) + .build() + .export().field("deploy").internal().func(0).build() + .export().field("call").internal().func(1).build() + .build(); + let bytes = contract.to_bytes().unwrap(); + let hash = T::Hashing::hash(&bytes); + (bytes, hash) } fn expanded_contract(target_bytes: u32) -> (Vec, ::Output) { - use parity_wasm::elements::{ - Instruction::{self, If, I32Const, Return, End}, - BlockType, Instructions, - }; - // Base size of a contract is 47 bytes and each expansion adds 6 bytes. - // We do one expansion less to account for the code section and function body - // size fields inside the binary wasm module representation which are leb128 encoded - // and therefore grow in size when the contract grows. We are not allowed to overshoot - // because of the maximum code size that is enforced by `put_code`. - let expansions = (target_bytes.saturating_sub(47) / 6).saturating_sub(1) as usize; - const EXPANSION: [Instruction; 4] = [ - I32Const(0), - If(BlockType::NoResult), - Return, - End, - ]; - let instructions = Instructions::new( - EXPANSION - .iter() - .cycle() - .take(EXPANSION.len() * expansions) - .cloned() - .chain(sp_std::iter::once(End)) - .collect() - ); - contract_with_call_body::(FuncBody::new(Vec::new(), instructions)) + use parity_wasm::elements::{ + Instruction::{self, If, I32Const, Return, End}, + BlockType, Instructions, + }; + // Base size of a contract is 47 bytes and each expansion adds 6 bytes. + // We do one expansion less to account for the code section and function body + // size fields inside the binary wasm module representation which are leb128 encoded + // and therefore grow in size when the contract grows. We are not allowed to overshoot + // because of the maximum code size that is enforced by `put_code`. + let expansions = (target_bytes.saturating_sub(47) / 6).saturating_sub(1) as usize; + const EXPANSION: [Instruction; 4] = [ + I32Const(0), + If(BlockType::NoResult), + Return, + End, + ]; + let instructions = Instructions::new( + EXPANSION + .iter() + .cycle() + .take(EXPANSION.len() * expansions) + .cloned() + .chain(sp_std::iter::once(End)) + .collect() + ); + contract_with_call_body::(FuncBody::new(Vec::new(), instructions)) } fn advance_block(num: ::BlockNumber) { @@ -109,161 +109,161 @@ fn advance_block(num: ::BlockNumber) { } benchmarks! { - _ { - } + _ { + } - // This extrinsic is pretty much constant as it is only a simple setter. - update_schedule { - let schedule = Schedule { - version: 1, - .. Default::default() - }; - }: _(RawOrigin::Root, schedule) + // This extrinsic is pretty much constant as it is only a simple setter. + update_schedule { + let schedule = Schedule { + version: 1, + .. Default::default() + }; + }: _(RawOrigin::Root, schedule) - // This constructs a contract that is maximal expensive to instrument. - // It creates a maximum number of metering blocks per byte. - put_code { - let n in 0 .. Contracts::::current_schedule().max_code_size; - let caller = create_funded_user::("caller", 0); - let (binary, hash) = expanded_contract::(n); - }: _(RawOrigin::Signed(caller), binary) + // This constructs a contract that is maximal expensive to instrument. + // It creates a maximum number of metering blocks per byte. + put_code { + let n in 0 .. Contracts::::current_schedule().max_code_size; + let caller = create_funded_user::("caller", 0); + let (binary, hash) = expanded_contract::(n); + }: _(RawOrigin::Signed(caller), binary) - // Instantiate uses a dummy contract constructor to measure the overhead of the instantiate. - // The size of the data has no influence on the costs of this extrinsic as long as the contract - // won't call `ext_input` in its constructor to copy the data to contract memory. - // The dummy contract used here does not do this. The costs for the data copy is billed as - // part of `ext_input`. - instantiate { - let data = vec![0u8; 128]; - let endowment = Config::::subsistence_threshold_uncached(); - let caller = create_funded_user::("caller", 0); - let (binary, hash) = load_module!("dummy"); - Contracts::::put_code(RawOrigin::Signed(caller.clone()).into(), binary.to_vec()) - .unwrap(); + // Instantiate uses a dummy contract constructor to measure the overhead of the instantiate. + // The size of the data has no influence on the costs of this extrinsic as long as the contract + // won't call `ext_input` in its constructor to copy the data to contract memory. + // The dummy contract used here does not do this. The costs for the data copy is billed as + // part of `ext_input`. + instantiate { + let data = vec![0u8; 128]; + let endowment = Config::::subsistence_threshold_uncached(); + let caller = create_funded_user::("caller", 0); + let (binary, hash) = load_module!("dummy"); + Contracts::::put_code(RawOrigin::Signed(caller.clone()).into(), binary.to_vec()) + .unwrap(); - }: _( - RawOrigin::Signed(caller.clone()), - endowment, - Weight::max_value(), - hash, - data - ) - verify { - assert_eq!( - funding::() - endowment, - T::Currency::free_balance(&caller), - ) - } + }: _( + RawOrigin::Signed(caller.clone()), + endowment, + Weight::max_value(), + hash, + data + ) + verify { + assert_eq!( + funding::() - endowment, + T::Currency::free_balance(&caller), + ) + } - // We just call a dummy contract to measure to overhead of the call extrinsic. - // As for instantiate the size of the data does not influence the costs. - call { - let data = vec![0u8; 128]; - let endowment = Config::::subsistence_threshold_uncached(); - let value = T::Currency::minimum_balance() * 100.into(); - let caller = create_funded_user::("caller", 0); - let (binary, hash) = load_module!("dummy"); - let addr = T::DetermineContractAddress::contract_address_for(&hash, &[], &caller); - Contracts::::put_code(RawOrigin::Signed(caller.clone()).into(), binary.to_vec()) - .unwrap(); - Contracts::::instantiate( - RawOrigin::Signed(caller.clone()).into(), - endowment, - Weight::max_value(), - hash, - vec![], - ).unwrap(); - }: _( - RawOrigin::Signed(caller.clone()), - T::Lookup::unlookup(addr), - value, - Weight::max_value(), - data - ) - verify { - assert_eq!( - funding::() - endowment - value, - T::Currency::free_balance(&caller), - ) - } + // We just call a dummy contract to measure to overhead of the call extrinsic. + // As for instantiate the size of the data does not influence the costs. + call { + let data = vec![0u8; 128]; + let endowment = Config::::subsistence_threshold_uncached(); + let value = T::Currency::minimum_balance() * 100.into(); + let caller = create_funded_user::("caller", 0); + let (binary, hash) = load_module!("dummy"); + let addr = T::DetermineContractAddress::contract_address_for(&hash, &[], &caller); + Contracts::::put_code(RawOrigin::Signed(caller.clone()).into(), binary.to_vec()) + .unwrap(); + Contracts::::instantiate( + RawOrigin::Signed(caller.clone()).into(), + endowment, + Weight::max_value(), + hash, + vec![], + ).unwrap(); + }: _( + RawOrigin::Signed(caller.clone()), + T::Lookup::unlookup(addr), + value, + Weight::max_value(), + data + ) + verify { + assert_eq!( + funding::() - endowment - value, + T::Currency::free_balance(&caller), + ) + } - // We benchmark the costs for sucessfully evicting an empty contract. - // The actual costs are depending on how many storage items the evicted contract - // does have. However, those costs are not to be payed by the sender but - // will be distributed over multiple blocks using a scheduler. Otherwise there is - // no incentive to remove large contracts when the removal is more expensive than - // the reward for removing them. - claim_surcharge { - let endowment = Config::::subsistence_threshold_uncached(); - let value = T::Currency::minimum_balance() * 100.into(); - let caller = create_funded_user::("caller", 0); - let (binary, hash) = load_module!("dummy"); - let addr = T::DetermineContractAddress::contract_address_for(&hash, &[], &caller); - Contracts::::put_code(RawOrigin::Signed(caller.clone()).into(), binary.to_vec()) - .unwrap(); - Contracts::::instantiate( - RawOrigin::Signed(caller.clone()).into(), - endowment, - Weight::max_value(), - hash, - vec![], - ).unwrap(); + // We benchmark the costs for sucessfully evicting an empty contract. + // The actual costs are depending on how many storage items the evicted contract + // does have. However, those costs are not to be payed by the sender but + // will be distributed over multiple blocks using a scheduler. Otherwise there is + // no incentive to remove large contracts when the removal is more expensive than + // the reward for removing them. + claim_surcharge { + let endowment = Config::::subsistence_threshold_uncached(); + let value = T::Currency::minimum_balance() * 100.into(); + let caller = create_funded_user::("caller", 0); + let (binary, hash) = load_module!("dummy"); + let addr = T::DetermineContractAddress::contract_address_for(&hash, &[], &caller); + Contracts::::put_code(RawOrigin::Signed(caller.clone()).into(), binary.to_vec()) + .unwrap(); + Contracts::::instantiate( + RawOrigin::Signed(caller.clone()).into(), + endowment, + Weight::max_value(), + hash, + vec![], + ).unwrap(); - // instantiate should leave us with an alive contract - ContractInfoOf::::get(addr.clone()).unwrap().get_alive().unwrap(); + // instantiate should leave us with an alive contract + ContractInfoOf::::get(addr.clone()).unwrap().get_alive().unwrap(); - // generate some rent - advance_block::(::SignedClaimHandicap::get() + 1.into()); + // generate some rent + advance_block::(::SignedClaimHandicap::get() + 1.into()); - }: _(RawOrigin::Signed(caller.clone()), addr.clone(), None) - verify { - // the claim surcharge should have evicted the contract - ContractInfoOf::::get(addr.clone()).unwrap().get_tombstone().unwrap(); + }: _(RawOrigin::Signed(caller.clone()), addr.clone(), None) + verify { + // the claim surcharge should have evicted the contract + ContractInfoOf::::get(addr.clone()).unwrap().get_tombstone().unwrap(); - // the caller should get the reward for being a good snitch - assert_eq!( - funding::() - endowment + ::SurchargeReward::get(), - T::Currency::free_balance(&caller), - ); - } + // the caller should get the reward for being a good snitch + assert_eq!( + funding::() - endowment + ::SurchargeReward::get(), + T::Currency::free_balance(&caller), + ); + } } #[cfg(test)] mod tests { - use super::*; - use crate::tests::{ExtBuilder, Test}; - use frame_support::assert_ok; + use super::*; + use crate::tests::{ExtBuilder, Test}; + use frame_support::assert_ok; - #[test] - fn update_schedule() { + #[test] + fn update_schedule() { ExtBuilder::default().build().execute_with(|| { assert_ok!(test_benchmark_update_schedule::()); }); - } + } - #[test] - fn put_code() { + #[test] + fn put_code() { ExtBuilder::default().build().execute_with(|| { assert_ok!(test_benchmark_put_code::()); }); - } + } - #[test] - fn instantiate() { + #[test] + fn instantiate() { ExtBuilder::default().build().execute_with(|| { assert_ok!(test_benchmark_instantiate::()); }); - } + } - #[test] - fn call() { + #[test] + fn call() { ExtBuilder::default().build().execute_with(|| { assert_ok!(test_benchmark_call::()); }); - } + } - #[test] - fn claim_surcharge() { + #[test] + fn claim_surcharge() { ExtBuilder::default().build().execute_with(|| { assert_ok!(test_benchmark_claim_surcharge::()); }); diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index a957b6ef33a79..e5015cbdc9283 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -2861,9 +2861,9 @@ mod offchain_phragmen { let (offchain, offchain_state) = TestOffchainExt::new(); let (pool, pool_state) = TestTransactionPoolExt::new(); - let mut seed = [0_u8; 32]; - seed[0..4].copy_from_slice(&iterations.to_le_bytes()); - offchain_state.write().seed = seed; + let mut seed = [0_u8; 32]; + seed[0..4].copy_from_slice(&iterations.to_le_bytes()); + offchain_state.write().seed = seed; ext.register_extension(OffchainExt::new(offchain)); ext.register_extension(TransactionPoolExt::new(pool)); diff --git a/primitives/consensus/common/src/metrics.rs b/primitives/consensus/common/src/metrics.rs index 90df85a2948ef..f9326fac062dc 100644 --- a/primitives/consensus/common/src/metrics.rs +++ b/primitives/consensus/common/src/metrics.rs @@ -48,7 +48,7 @@ impl Metrics { &["result"], )?, registry, - )?, + )?, }) } diff --git a/primitives/core/src/sr25519.rs b/primitives/core/src/sr25519.rs index 9e9aaf53bbf1f..b015347e9aa28 100644 --- a/primitives/core/src/sr25519.rs +++ b/primitives/core/src/sr25519.rs @@ -399,15 +399,15 @@ impl TraitPublic for Public { } impl From for CryptoTypePublicPair { - fn from(key: Public) -> Self { - (&key).into() - } + fn from(key: Public) -> Self { + (&key).into() + } } impl From<&Public> for CryptoTypePublicPair { - fn from(key: &Public) -> Self { - CryptoTypePublicPair(CRYPTO_ID, key.to_raw_vec()) - } + fn from(key: &Public) -> Self { + CryptoTypePublicPair(CRYPTO_ID, key.to_raw_vec()) + } } #[cfg(feature = "std")] diff --git a/primitives/runtime/src/offchain/storage_lock.rs b/primitives/runtime/src/offchain/storage_lock.rs index 9d4e671db6ea1..a3838f21fd13d 100644 --- a/primitives/runtime/src/offchain/storage_lock.rs +++ b/primitives/runtime/src/offchain/storage_lock.rs @@ -560,34 +560,34 @@ mod tests { offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(200))); // the lock is still active, extend it successfully - assert_eq!(guard.extend_lock().is_ok(), true); + assert_eq!(guard.extend_lock().is_ok(), true); - // sleep_until < deadline - offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(200))); + // sleep_until < deadline + offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(200))); - // the lock is still active, try_lock will fail - let mut lock = StorageLock::<'_, Time>::with_deadline(b"lock_4", lock_expiration); - let res = lock.try_lock(); - assert_eq!(res.is_ok(), false); + // the lock is still active, try_lock will fail + let mut lock = StorageLock::<'_, Time>::with_deadline(b"lock_4", lock_expiration); + let res = lock.try_lock(); + assert_eq!(res.is_ok(), false); - // sleep again untill sleep_until > deadline - offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(200))); + // sleep again untill sleep_until > deadline + offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(200))); - // the lock has expired, failed to extend it - assert_eq!(guard.extend_lock().is_ok(), false); - guard.forget(); + // the lock has expired, failed to extend it + assert_eq!(guard.extend_lock().is_ok(), false); + guard.forget(); - // try_lock will succeed - let mut lock = StorageLock::<'_, Time>::with_deadline(b"lock_4", lock_expiration); - let res = lock.try_lock(); - assert!(res.is_ok()); - let guard = res.unwrap(); + // try_lock will succeed + let mut lock = StorageLock::<'_, Time>::with_deadline(b"lock_4", lock_expiration); + let res = lock.try_lock(); + assert!(res.is_ok()); + let guard = res.unwrap(); - guard.forget(); - }); + guard.forget(); + }); - // lock must have been cleared at this point - let opt = state.read().persistent_storage.get(b"", b"lock_4"); - assert_eq!(opt.unwrap(), vec![132_u8, 3u8, 0, 0, 0, 0, 0, 0]); // 132 + 256 * 3 = 900 + // lock must have been cleared at this point + let opt = state.read().persistent_storage.get(b"", b"lock_4"); + assert_eq!(opt.unwrap(), vec![132_u8, 3u8, 0, 0, 0, 0, 0, 0]); // 132 + 256 * 3 = 900 } } diff --git a/utils/fork-tree/src/lib.rs b/utils/fork-tree/src/lib.rs index e11c1138f49db..1d01c53417649 100644 --- a/utils/fork-tree/src/lib.rs +++ b/utils/fork-tree/src/lib.rs @@ -132,7 +132,7 @@ impl ForkTree where let mut root = root .expect("find_node_index_where will return array with at least one index; \ - this results in at least one item in removed; qed"); + this results in at least one item in removed; qed"); let mut removed = old_roots; From 0330f629dd1aed9d7dbe44e9b89d995444aa99d0 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 3 Aug 2020 16:55:32 +0200 Subject: [PATCH 13/32] Add details to legacy requests (#6747) --- client/network/src/protocol.rs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index d3a729cc8d587..630471414b223 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -48,7 +48,10 @@ use sp_runtime::traits::{ use sp_arithmetic::traits::SaturatedConversion; use message::{BlockAnnounce, Message}; use message::generic::{Message as GenericMessage, Roles}; -use prometheus_endpoint::{Registry, Gauge, Counter, GaugeVec, PrometheusError, Opts, register, U64}; +use prometheus_endpoint::{ + Registry, Gauge, Counter, CounterVec, GaugeVec, + PrometheusError, Opts, register, U64 +}; use sync::{ChainSync, SyncState}; use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque, hash_map::Entry}; @@ -142,7 +145,7 @@ struct Metrics { finality_proofs: GaugeVec, justifications: GaugeVec, propagated_transactions: Counter, - legacy_requests_received: Counter, + legacy_requests_received: CounterVec, } impl Metrics { @@ -188,9 +191,12 @@ impl Metrics { "sync_propagated_transactions", "Number of transactions propagated to at least one peer", )?, r)?, - legacy_requests_received: register(Counter::new( - "sync_legacy_requests_received", - "Number of block/finality/light-client requests received on the legacy substream", + legacy_requests_received: register(CounterVec::new( + Opts::new( + "sync_legacy_requests_received", + "Number of block/finality/light-client requests received on the legacy substream", + ), + &["kind"] )?, r)?, }) } @@ -719,7 +725,7 @@ impl Protocol { fn on_block_request(&mut self, peer: PeerId, request: message::BlockRequest) { if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["block-request"]).inc(); } trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?} for {:?}", @@ -1395,7 +1401,7 @@ impl Protocol { ); if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["remote-call"]).inc(); } let proof = match self.context_data.chain.execution_proof( @@ -1519,7 +1525,7 @@ impl Protocol { request: message::RemoteReadRequest, ) { if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["remote-read"]).inc(); } if request.keys.is_empty() { @@ -1572,7 +1578,7 @@ impl Protocol { request: message::RemoteReadChildRequest, ) { if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["remote-child"]).inc(); } if request.keys.is_empty() { @@ -1632,7 +1638,7 @@ impl Protocol { request: message::RemoteHeaderRequest>, ) { if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["remote-header"]).inc(); } trace!(target: "sync", "Remote header proof request {} from {} ({})", @@ -1666,7 +1672,7 @@ impl Protocol { request: message::RemoteChangesRequest, ) { if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["remote-changes"]).inc(); } trace!(target: "sync", "Remote changes proof request {} from {} for key {} ({}..{})", @@ -1733,7 +1739,7 @@ impl Protocol { request: message::FinalityProofRequest, ) { if let Some(metrics) = &self.metrics { - metrics.legacy_requests_received.inc(); + metrics.legacy_requests_received.with_label_values(&["finality-proof"]).inc(); } trace!(target: "sync", "Finality proof request from {} for {}", who, request.block); From de8a0d5a2697b0c43700536b9854ccfa50ca93c8 Mon Sep 17 00:00:00 2001 From: Alex Siman Date: Mon, 3 Aug 2020 18:04:56 +0300 Subject: [PATCH 14/32] Add ss58 address for Subsocial (#6800) --- primitives/core/src/crypto.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs index 0cdbebde9f26a..2e71e676b3e95 100644 --- a/primitives/core/src/crypto.rs +++ b/primitives/core/src/crypto.rs @@ -446,6 +446,8 @@ ss58_address_format!( (18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).") StafiAccount => (20, "stafi", "Stafi mainnet, standard account (*25519).") + SubsocialAccount => + (28, "subsocial", "Subsocial network, standard account (*25519).") RobonomicsAccount => (32, "robonomics", "Any Robonomics network standard account (*25519).") DataHighwayAccount => From 270a992bf299cd41b2ed4e211814d2a958e3bfc8 Mon Sep 17 00:00:00 2001 From: Shaopeng Wang Date: Tue, 4 Aug 2020 20:02:24 +1200 Subject: [PATCH 15/32] Add mutate_exists to StorageDoubleMap. (#6704) --- frame/support/src/lib.rs | 17 +++++++++++++++++ .../support/src/storage/generator/double_map.rs | 9 +++++++++ frame/support/src/storage/mod.rs | 7 +++++++ 3 files changed, 33 insertions(+) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index bb8aacd1a4886..f0ffdc90a74e0 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -629,6 +629,23 @@ mod tests { }); } + #[test] + fn double_map_mutate_exists_should_work() { + new_test_ext().execute_with(|| { + type DoubleMap = DataDM; + + let (key1, key2) = (11, 13); + + // mutated + DoubleMap::mutate_exists(key1, key2, |v| *v = Some(1)); + assert_eq!(DoubleMap::get(&key1, key2), 1); + + // removed if mutated to `None` + DoubleMap::mutate_exists(key1, key2, |v| *v = None); + assert!(!DoubleMap::contains_key(&key1, key2)); + }); + } + #[test] fn double_map_try_mutate_exists_should_work() { new_test_ext().execute_with(|| { diff --git a/frame/support/src/storage/generator/double_map.rs b/frame/support/src/storage/generator/double_map.rs index 8fbef16204f4e..3c82f4156a271 100644 --- a/frame/support/src/storage/generator/double_map.rs +++ b/frame/support/src/storage/generator/double_map.rs @@ -228,6 +228,15 @@ impl storage::StorageDoubleMap for G where Self::try_mutate(k1, k2, |v| Ok::(f(v))).expect("`Never` can not be constructed; qed") } + fn mutate_exists(k1: KArg1, k2: KArg2, f: F) -> R + where + KArg1: EncodeLike, + KArg2: EncodeLike, + F: FnOnce(&mut Option) -> R, + { + Self::try_mutate_exists(k1, k2, |v| Ok::(f(v))).expect("`Never` can not be constructed; qed") + } + fn try_mutate(k1: KArg1, k2: KArg2, f: F) -> Result where KArg1: EncodeLike, KArg2: EncodeLike, diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 4623f81859b35..347fd814136d7 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -366,6 +366,13 @@ pub trait StorageDoubleMap { KArg2: EncodeLike, F: FnOnce(&mut Self::Query) -> Result; + /// Mutate the value under the given keys. Deletes the item if mutated to a `None`. + fn mutate_exists(k1: KArg1, k2: KArg2, f: F) -> R + where + KArg1: EncodeLike, + KArg2: EncodeLike, + F: FnOnce(&mut Option) -> R; + /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. fn try_mutate_exists(k1: KArg1, k2: KArg2, f: F) -> Result where From 549050b7f1740c90855e777daf3f9700750ad7ff Mon Sep 17 00:00:00 2001 From: Guillaume Thiolliere Date: Tue, 4 Aug 2020 19:58:03 +0200 Subject: [PATCH 16/32] pallet-democracy use of weightinfo (#6783) * democracy use of weightinfo * fix some doc and benchs * todo generate from parity machine * factorize and add license * use final weights * add slightly more sensible default weight * refactor * rename benchmark to avoid confusion * just make remove_other_vote benchmark being the worst case of the extrinsic --- bin/node/runtime/src/lib.rs | 2 +- bin/node/runtime/src/weights/mod.rs | 1 + .../runtime/src/weights/pallet_democracy.rs | 155 ++++++++++++ frame/democracy/src/benchmarking.rs | 86 ++----- frame/democracy/src/default_weight.rs | 158 +++++++++++++ frame/democracy/src/lib.rs | 223 +++++------------- 6 files changed, 395 insertions(+), 230 deletions(-) create mode 100644 bin/node/runtime/src/weights/pallet_democracy.rs create mode 100644 frame/democracy/src/default_weight.rs diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fff0dd3427fee..acc1b07281834 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -495,7 +495,7 @@ impl pallet_democracy::Trait for Runtime { type Scheduler = Scheduler; type PalletsOrigin = OriginCaller; type MaxVotes = MaxVotes; - type WeightInfo = (); + type WeightInfo = weights::pallet_democracy::WeightInfo; } parameter_types! { diff --git a/bin/node/runtime/src/weights/mod.rs b/bin/node/runtime/src/weights/mod.rs index 70e10d5342f36..70bae879ce05c 100644 --- a/bin/node/runtime/src/weights/mod.rs +++ b/bin/node/runtime/src/weights/mod.rs @@ -16,3 +16,4 @@ //! A list of the different weight modules for our runtime. pub mod pallet_balances; +pub mod pallet_democracy; diff --git a/bin/node/runtime/src/weights/pallet_democracy.rs b/bin/node/runtime/src/weights/pallet_democracy.rs new file mode 100644 index 0000000000000..2c55a848061a3 --- /dev/null +++ b/bin/node/runtime/src/weights/pallet_democracy.rs @@ -0,0 +1,155 @@ +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Weights for the Democracy Pallet +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc5 + +use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight}; + +pub struct WeightInfo; +impl pallet_democracy::WeightInfo for WeightInfo { + fn propose() -> Weight { + (49113000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn second(s: u32, ) -> Weight { + (42067000 as Weight) + .saturating_add((220000 as Weight).saturating_mul(s as Weight)) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn vote_new(r: u32, ) -> Weight { + (54159000 as Weight) + .saturating_add((252000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn vote_existing(r: u32, ) -> Weight { + (54145000 as Weight) + .saturating_add((262000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn emergency_cancel() -> Weight { + (31071000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn external_propose(v: u32, ) -> Weight { + (14282000 as Weight) + .saturating_add((109000 as Weight).saturating_mul(v as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn external_propose_majority() -> Weight { + (3478000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn external_propose_default() -> Weight { + (3442000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn fast_track() -> Weight { + (30820000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn veto_external(v: u32, ) -> Weight { + (30971000 as Weight) + .saturating_add((184000 as Weight).saturating_mul(v as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn cancel_referendum() -> Weight { + (20431000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn cancel_queued(r: u32, ) -> Weight { + (42438000 as Weight) + .saturating_add((3284000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn on_initialize_base(r: u32, ) -> Weight { + (70826000 as Weight) + .saturating_add((10716000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(6 as Weight)) + .saturating_add(DbWeight::get().reads((2 as Weight).saturating_mul(r as Weight))) + .saturating_add(DbWeight::get().writes(5 as Weight)) + } + fn delegate(r: u32, ) -> Weight { + (72046000 as Weight) + .saturating_add((7837000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(4 as Weight)) + .saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(DbWeight::get().writes(4 as Weight)) + .saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(r as Weight))) + } + fn undelegate(r: u32, ) -> Weight { + (41028000 as Weight) + .saturating_add((7810000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(DbWeight::get().writes(2 as Weight)) + .saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(r as Weight))) + } + fn clear_public_proposals() -> Weight { + (3643000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn note_preimage(b: u32, ) -> Weight { + (46629000 as Weight) + .saturating_add((4000 as Weight).saturating_mul(b as Weight)) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn note_imminent_preimage(b: u32, ) -> Weight { + (31147000 as Weight) + .saturating_add((3000 as Weight).saturating_mul(b as Weight)) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn reap_preimage(b: u32, ) -> Weight { + (42848000 as Weight) + .saturating_add((3000 as Weight).saturating_mul(b as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn unlock_remove(r: u32, ) -> Weight { + (45333000 as Weight) + .saturating_add((171000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn unlock_set(r: u32, ) -> Weight { + (44424000 as Weight) + .saturating_add((291000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn remove_vote(r: u32, ) -> Weight { + (28250000 as Weight) + .saturating_add((283000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn remove_other_vote(r: u32, ) -> Weight { + (28250000 as Weight) + .saturating_add((283000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } +} diff --git a/frame/democracy/src/benchmarking.rs b/frame/democracy/src/benchmarking.rs index 421eb07e32c4a..1fa0988fbbd4d 100644 --- a/frame/democracy/src/benchmarking.rs +++ b/frame/democracy/src/benchmarking.rs @@ -31,7 +31,6 @@ use crate::Module as Democracy; const SEED: u32 = 0; const MAX_REFERENDUMS: u32 = 100; -const MAX_PROPOSALS: u32 = 100; const MAX_SECONDERS: u32 = 100; const MAX_BYTES: u32 = 16_384; @@ -101,21 +100,12 @@ benchmarks! { _ { } propose { - let p in 1 .. MAX_PROPOSALS; - - // Add p proposals - for i in 0 .. p { - add_proposal::(i)?; - } - - assert_eq!(Democracy::::public_props().len(), p as usize, "Proposals not created."); - let caller = funded_account::("caller", 0); - let proposal_hash: T::Hash = T::Hashing::hash_of(&p); + let proposal_hash: T::Hash = T::Hashing::hash_of(&0); let value = T::MinimumDeposit::get(); }: _(RawOrigin::Signed(caller), proposal_hash, value.into()) verify { - assert_eq!(Democracy::::public_props().len(), (p + 1) as usize, "Proposals not created."); + assert_eq!(Democracy::::public_props().len(), 1, "Proposals not created."); } second { @@ -206,18 +196,8 @@ benchmarks! { } emergency_cancel { - let r in 1 .. MAX_REFERENDUMS; let origin = T::CancellationOrigin::successful_origin(); - - // Create and cancel a bunch of referendums - for i in 0 .. r { - let ref_idx = add_referendum::(i)?; - let call = Call::::emergency_cancel(ref_idx); - call.dispatch_bypass_filter(origin.clone())?; - } - - // Lets now measure one more - let referendum_index = add_referendum::(r)?; + let referendum_index = add_referendum::(0)?; let call = Call::::emergency_cancel(referendum_index); assert!(Democracy::::referendum_status(referendum_index).is_ok()); }: { call.dispatch_bypass_filter(origin)? } @@ -228,11 +208,10 @@ benchmarks! { // Worst case scenario, we external propose a previously blacklisted proposal external_propose { - let p in 1 .. MAX_PROPOSALS; let v in 1 .. MAX_VETOERS as u32; let origin = T::ExternalOrigin::successful_origin(); - let proposal_hash = T::Hashing::hash_of(&p); + let proposal_hash = T::Hashing::hash_of(&0); // Add proposal to blacklist with block number 0 Blacklist::::insert( proposal_hash, @@ -247,10 +226,8 @@ benchmarks! { } external_propose_majority { - let p in 1 .. MAX_PROPOSALS; - let origin = T::ExternalMajorityOrigin::successful_origin(); - let proposal_hash = T::Hashing::hash_of(&p); + let proposal_hash = T::Hashing::hash_of(&0); let call = Call::::external_propose_majority(proposal_hash); }: { call.dispatch_bypass_filter(origin)? } verify { @@ -259,10 +236,8 @@ benchmarks! { } external_propose_default { - let p in 1 .. MAX_PROPOSALS; - let origin = T::ExternalDefaultOrigin::successful_origin(); - let proposal_hash = T::Hashing::hash_of(&p); + let proposal_hash = T::Hashing::hash_of(&0); let call = Call::::external_propose_default(proposal_hash); }: { call.dispatch_bypass_filter(origin)? } verify { @@ -271,10 +246,8 @@ benchmarks! { } fast_track { - let p in 1 .. MAX_PROPOSALS; - let origin_propose = T::ExternalDefaultOrigin::successful_origin(); - let proposal_hash: T::Hash = T::Hashing::hash_of(&p); + let proposal_hash: T::Hash = T::Hashing::hash_of(&0); Democracy::::external_propose_default(origin_propose, proposal_hash.clone())?; // NOTE: Instant origin may invoke a little bit more logic, but may not always succeed. @@ -315,24 +288,21 @@ benchmarks! { } cancel_referendum { - let r in 0 .. MAX_REFERENDUMS; - // Should have no effect on the execution time. - for i in 0..r { - add_referendum::(i)?; - } - let referendum_index = add_referendum::(r)?; + let referendum_index = add_referendum::(0)?; }: _(RawOrigin::Root, referendum_index) cancel_queued { let r in 1 .. MAX_REFERENDUMS; - // Should have no effect on the execution time. + for i in 0..r { - add_referendum::(i)?; + add_referendum::(i)?; // This add one element in the scheduler } + let referendum_index = add_referendum::(r)?; }: _(RawOrigin::Root, referendum_index) // Note that we have a separate benchmark for `launch_next` + #[extra] on_initialize_external { let r in 0 .. MAX_REFERENDUMS; @@ -371,6 +341,7 @@ benchmarks! { } } + #[extra] on_initialize_public { let r in 1 .. MAX_REFERENDUMS; @@ -401,7 +372,8 @@ benchmarks! { } } - on_initialize_no_launch_no_maturing { + // No launch no maturing referenda. + on_initialize_base { let r in 1 .. MAX_REFERENDUMS; for i in 0..r { @@ -526,11 +498,7 @@ benchmarks! { } clear_public_proposals { - let p in 0 .. MAX_PROPOSALS; - - for i in 0 .. p { - add_proposal::(i)?; - } + add_proposal::(0)?; }: _(RawOrigin::Root) @@ -687,41 +655,36 @@ benchmarks! { assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); } + // Worst case is when target == caller and referendum is ongoing remove_other_vote { let r in 1 .. MAX_REFERENDUMS; - let other = funded_account::("other", r); + let caller = funded_account::("caller", r); let account_vote = account_vote::(100.into()); for i in 0 .. r { let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; + Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; } - let votes = match VotingOf::::get(&other) { + let votes = match VotingOf::::get(&caller) { Voting::Direct { votes, .. } => votes, _ => return Err("Votes are not direct"), }; assert_eq!(votes.len(), r as usize, "Votes not created"); let referendum_index = r - 1; - ReferendumInfoOf::::insert( - referendum_index, - ReferendumInfo::Finished { end: T::BlockNumber::zero(), approved: true } - ); - let caller = funded_account::("caller", 0); - - System::::set_block_number(T::EnactmentPeriod::get() * 10u32.into()); - }: _(RawOrigin::Signed(caller), other.clone(), referendum_index) + }: _(RawOrigin::Signed(caller.clone()), caller.clone(), referendum_index) verify { - let votes = match VotingOf::::get(&other) { + let votes = match VotingOf::::get(&caller) { Voting::Direct { votes, .. } => votes, _ => return Err("Votes are not direct"), }; assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); } + #[extra] enact_proposal_execute { // Num of bytes in encoded proposal let b in 0 .. MAX_BYTES; @@ -743,6 +706,7 @@ benchmarks! { assert_last_event::(RawEvent::Executed(0, false).into()); } + #[extra] enact_proposal_slash { // Num of bytes in encoded proposal let b in 0 .. MAX_BYTES; @@ -788,7 +752,7 @@ mod tests { assert_ok!(test_benchmark_cancel_queued::()); assert_ok!(test_benchmark_on_initialize_external::()); assert_ok!(test_benchmark_on_initialize_public::()); - assert_ok!(test_benchmark_on_initialize_no_launch_no_maturing::()); + assert_ok!(test_benchmark_on_initialize_base::()); assert_ok!(test_benchmark_delegate::()); assert_ok!(test_benchmark_undelegate::()); assert_ok!(test_benchmark_clear_public_proposals::()); diff --git a/frame/democracy/src/default_weight.rs b/frame/democracy/src/default_weight.rs new file mode 100644 index 0000000000000..2c74a4af2020f --- /dev/null +++ b/frame/democracy/src/default_weight.rs @@ -0,0 +1,158 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Default weights for the Democracy Pallet +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 2.0.0-rc5 + +use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight}; + +/// Default implementation of weight, this is just from an example return, values may change +/// depending on the runtime. This is not meant to be used in production. +impl crate::WeightInfo for () { + fn propose() -> Weight { + (49113000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn second(s: u32, ) -> Weight { + (42067000 as Weight) + .saturating_add((220000 as Weight).saturating_mul(s as Weight)) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn vote_new(r: u32, ) -> Weight { + (54159000 as Weight) + .saturating_add((252000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn vote_existing(r: u32, ) -> Weight { + (54145000 as Weight) + .saturating_add((262000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn emergency_cancel() -> Weight { + (31071000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn external_propose(v: u32, ) -> Weight { + (14282000 as Weight) + .saturating_add((109000 as Weight).saturating_mul(v as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn external_propose_majority() -> Weight { + (3478000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn external_propose_default() -> Weight { + (3442000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn fast_track() -> Weight { + (30820000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn veto_external(v: u32, ) -> Weight { + (30971000 as Weight) + .saturating_add((184000 as Weight).saturating_mul(v as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn cancel_referendum() -> Weight { + (20431000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn cancel_queued(r: u32, ) -> Weight { + (42438000 as Weight) + .saturating_add((3284000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn on_initialize_base(r: u32, ) -> Weight { + (70826000 as Weight) + .saturating_add((10716000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(6 as Weight)) + .saturating_add(DbWeight::get().reads((2 as Weight).saturating_mul(r as Weight))) + .saturating_add(DbWeight::get().writes(5 as Weight)) + } + fn delegate(r: u32, ) -> Weight { + (72046000 as Weight) + .saturating_add((7837000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(4 as Weight)) + .saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(DbWeight::get().writes(4 as Weight)) + .saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(r as Weight))) + } + fn undelegate(r: u32, ) -> Weight { + (41028000 as Weight) + .saturating_add((7810000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(DbWeight::get().writes(2 as Weight)) + .saturating_add(DbWeight::get().writes((1 as Weight).saturating_mul(r as Weight))) + } + fn clear_public_proposals() -> Weight { + (3643000 as Weight) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn note_preimage(b: u32, ) -> Weight { + (46629000 as Weight) + .saturating_add((4000 as Weight).saturating_mul(b as Weight)) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn note_imminent_preimage(b: u32, ) -> Weight { + (31147000 as Weight) + .saturating_add((3000 as Weight).saturating_mul(b as Weight)) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn reap_preimage(b: u32, ) -> Weight { + (42848000 as Weight) + .saturating_add((3000 as Weight).saturating_mul(b as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn unlock_remove(r: u32, ) -> Weight { + (45333000 as Weight) + .saturating_add((171000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn unlock_set(r: u32, ) -> Weight { + (44424000 as Weight) + .saturating_add((291000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(3 as Weight)) + .saturating_add(DbWeight::get().writes(3 as Weight)) + } + fn remove_vote(r: u32, ) -> Weight { + (28250000 as Weight) + .saturating_add((283000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } + fn remove_other_vote(r: u32, ) -> Weight { + (28250000 as Weight) + .saturating_add((283000 as Weight).saturating_mul(r as Weight)) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } +} diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index f546b87dc603b..f3a5960eb2fe6 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -173,6 +173,7 @@ mod vote_threshold; mod vote; mod conviction; mod types; +mod default_weight; pub use vote_threshold::{Approved, VoteThreshold}; pub use vote::{Vote, AccountVote, Voting}; pub use conviction::Conviction; @@ -189,7 +190,7 @@ const DEMOCRACY_ID: LockIdentifier = *b"democrac"; /// The maximum number of vetoers on a single proposal used to compute Weight. /// /// NOTE: This is not enforced by any logic. -pub const MAX_VETOERS: Weight = 100; +pub const MAX_VETOERS: u32 = 100; /// A proposal index. pub type PropIndex = u32; @@ -202,24 +203,22 @@ type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; pub trait WeightInfo { - fn propose(p: u32, ) -> Weight; + fn propose() -> Weight; fn second(s: u32, ) -> Weight; fn vote_new(r: u32, ) -> Weight; fn vote_existing(r: u32, ) -> Weight; - fn emergency_cancel(r: u32, ) -> Weight; - fn external_propose(p: u32, v: u32, ) -> Weight; - fn external_propose_majority(p: u32, ) -> Weight; - fn external_propose_default(p: u32, ) -> Weight; - fn fast_track(p: u32, ) -> Weight; + fn emergency_cancel() -> Weight; + fn external_propose(v: u32, ) -> Weight; + fn external_propose_majority() -> Weight; + fn external_propose_default() -> Weight; + fn fast_track() -> Weight; fn veto_external(v: u32, ) -> Weight; - fn cancel_referendum(r: u32, ) -> Weight; + fn cancel_referendum() -> Weight; fn cancel_queued(r: u32, ) -> Weight; - fn on_initialize_external(r: u32, ) -> Weight; - fn on_initialize_public(r: u32, ) -> Weight; - fn on_initialize_no_launch_no_maturing(r: u32, ) -> Weight; + fn on_initialize_base(r: u32, ) -> Weight; fn delegate(r: u32, ) -> Weight; fn undelegate(r: u32, ) -> Weight; - fn clear_public_proposals(p: u32, ) -> Weight; + fn clear_public_proposals() -> Weight; fn note_preimage(b: u32, ) -> Weight; fn note_imminent_preimage(b: u32, ) -> Weight; fn reap_preimage(b: u32, ) -> Weight; @@ -227,38 +226,6 @@ pub trait WeightInfo { fn unlock_set(r: u32, ) -> Weight; fn remove_vote(r: u32, ) -> Weight; fn remove_other_vote(r: u32, ) -> Weight; - fn enact_proposal_execute(b: u32, ) -> Weight; - fn enact_proposal_slash(b: u32, ) -> Weight; -} - -impl WeightInfo for () { - fn propose(_p: u32, ) -> Weight { 1_000_000_000 } - fn second(_s: u32, ) -> Weight { 1_000_000_000 } - fn vote_new(_r: u32, ) -> Weight { 1_000_000_000 } - fn vote_existing(_r: u32, ) -> Weight { 1_000_000_000 } - fn emergency_cancel(_r: u32, ) -> Weight { 1_000_000_000 } - fn external_propose(_p: u32, _v: u32, ) -> Weight { 1_000_000_000 } - fn external_propose_majority(_p: u32, ) -> Weight { 1_000_000_000 } - fn external_propose_default(_p: u32, ) -> Weight { 1_000_000_000 } - fn fast_track(_p: u32, ) -> Weight { 1_000_000_000 } - fn veto_external(_v: u32, ) -> Weight { 1_000_000_000 } - fn cancel_referendum(_r: u32, ) -> Weight { 1_000_000_000 } - fn cancel_queued(_r: u32, ) -> Weight { 1_000_000_000 } - fn on_initialize_external(_r: u32, ) -> Weight { 1_000_000_000 } - fn on_initialize_public(_r: u32, ) -> Weight { 1_000_000_000 } - fn on_initialize_no_launch_no_maturing(_r: u32, ) -> Weight { 1_000_000_000 } - fn delegate(_r: u32, ) -> Weight { 1_000_000_000 } - fn undelegate(_r: u32, ) -> Weight { 1_000_000_000 } - fn clear_public_proposals(_p: u32, ) -> Weight { 1_000_000_000 } - fn note_preimage(_b: u32, ) -> Weight { 1_000_000_000 } - fn note_imminent_preimage(_b: u32, ) -> Weight { 1_000_000_000 } - fn reap_preimage(_b: u32, ) -> Weight { 1_000_000_000 } - fn unlock_remove(_r: u32, ) -> Weight { 1_000_000_000 } - fn unlock_set(_r: u32, ) -> Weight { 1_000_000_000 } - fn remove_vote(_r: u32, ) -> Weight { 1_000_000_000 } - fn remove_other_vote(_r: u32, ) -> Weight { 1_000_000_000 } - fn enact_proposal_execute(_b: u32, ) -> Weight { 1_000_000_000 } - fn enact_proposal_slash(_b: u32, ) -> Weight { 1_000_000_000 } } pub trait Trait: frame_system::Trait + Sized { @@ -578,63 +545,6 @@ decl_error! { } } -/// Functions for calcuating the weight of some dispatchables. -mod weight_for { - use frame_support::{traits::Get, weights::Weight}; - use super::Trait; - - /// Calculate the weight for `delegate`. - /// - Db reads: 2*`VotingOf`, `balances locks` - /// - Db writes: 2*`VotingOf`, `balances locks` - /// - Db reads per votes: `ReferendumInfoOf` - /// - Db writes per votes: `ReferendumInfoOf` - /// - Base Weight: 65.78 + 8.229 * R µs - // NOTE: weight must cover an incorrect voting of origin with 100 votes. - pub fn delegate(votes: Weight) -> Weight { - T::DbWeight::get().reads_writes(votes.saturating_add(3), votes.saturating_add(3)) - .saturating_add(66_000_000) - .saturating_add(votes.saturating_mul(8_100_000)) - } - - /// Calculate the weight for `undelegate`. - /// - Db reads: 2*`VotingOf` - /// - Db writes: 2*`VotingOf` - /// - Db reads per votes: `ReferendumInfoOf` - /// - Db writes per votes: `ReferendumInfoOf` - /// - Base Weight: 33.29 + 8.104 * R µs - pub fn undelegate(votes: Weight) -> Weight { - T::DbWeight::get().reads_writes(votes.saturating_add(2), votes.saturating_add(2)) - .saturating_add(33_000_000) - .saturating_add(votes.saturating_mul(8_000_000)) - } - - /// Calculate the weight for `note_preimage`. - /// # - /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). - /// - Db reads: `Preimages` - /// - Db writes: `Preimages` - /// - Base Weight: 37.93 + .004 * b µs - /// # - pub fn note_preimage(encoded_proposal_len: Weight) -> Weight { - T::DbWeight::get().reads_writes(1, 1) - .saturating_add(38_000_000) - .saturating_add(encoded_proposal_len.saturating_mul(4_000)) - } - - /// Calculate the weight for `note_imminent_preimage`. - /// # - /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). - /// - Db reads: `Preimages` - /// - Db writes: `Preimages` - /// - Base Weight: 28.04 + .003 * b µs - /// # - pub fn note_imminent_preimage(encoded_proposal_len: Weight) -> Weight { - T::DbWeight::get().reads_writes(1, 1) - .saturating_add(28_000_000) - .saturating_add(encoded_proposal_len.saturating_mul(3_000)) - } -} - decl_module! { pub struct Module for enum Call where origin: T::Origin { type Error = Error; @@ -683,10 +593,8 @@ decl_module! { /// - Complexity: `O(1)` /// - Db reads: `PublicPropCount`, `PublicProps` /// - Db writes: `PublicPropCount`, `PublicProps`, `DepositOf` - /// ------------------- - /// Base Weight: 42.58 + .127 * P µs with `P` the number of proposals `PublicProps` /// # - #[weight = 50_000_000 + T::DbWeight::get().reads_writes(2, 3)] + #[weight = T::WeightInfo::propose()] fn propose(origin, proposal_hash: T::Hash, #[compact] value: BalanceOf) { let who = ensure_signed(origin)?; ensure!(value >= T::MinimumDeposit::get(), Error::::ValueLow); @@ -715,13 +623,8 @@ decl_module! { /// - Complexity: `O(S)` where S is the number of seconds a proposal already has. /// - Db reads: `DepositOf` /// - Db writes: `DepositOf` - /// --------- - /// - Base Weight: 22.28 + .229 * S µs /// # - #[weight = 23_000_000 - .saturating_add(230_000.saturating_mul(Weight::from(*seconds_upper_bound))) - .saturating_add(T::DbWeight::get().reads_writes(1, 1)) - ] + #[weight = T::WeightInfo::second(*seconds_upper_bound)] fn second(origin, #[compact] proposal: PropIndex, #[compact] seconds_upper_bound: u32) { let who = ensure_signed(origin)?; @@ -748,12 +651,9 @@ decl_module! { /// weight is charged as if maximum votes. /// - Db reads: `ReferendumInfoOf`, `VotingOf`, `balances locks` /// - Db writes: `ReferendumInfoOf`, `VotingOf`, `balances locks` - /// -------------------- - /// - Base Weight: - /// - Vote New: 49.24 + .333 * R µs - /// - Vote Existing: 49.94 + .343 * R µs /// # - #[weight = 50_000_000 + 350_000 * Weight::from(T::MaxVotes::get()) + T::DbWeight::get().reads_writes(3, 3)] + #[weight = T::WeightInfo::vote_new(T::MaxVotes::get()) + .max(T::WeightInfo::vote_existing(T::MaxVotes::get()))] fn vote(origin, #[compact] ref_index: ReferendumIndex, vote: AccountVote>, @@ -773,10 +673,8 @@ decl_module! { /// - Complexity: `O(1)`. /// - Db reads: `ReferendumInfoOf`, `Cancellations` /// - Db writes: `ReferendumInfoOf`, `Cancellations` - /// ------------- - /// - Base Weight: 34.25 µs /// # - #[weight = (35_000_000 + T::DbWeight::get().reads_writes(2, 2), DispatchClass::Operational)] + #[weight = (T::WeightInfo::emergency_cancel(), DispatchClass::Operational)] fn emergency_cancel(origin, ref_index: ReferendumIndex) { T::CancellationOrigin::ensure_origin(origin)?; @@ -800,9 +698,8 @@ decl_module! { /// Decoding vec of length V. Charged as maximum /// - Db reads: `NextExternal`, `Blacklist` /// - Db writes: `NextExternal` - /// - Base Weight: 13.8 + .106 * V µs /// # - #[weight = 15_000_000 + 110_000 * MAX_VETOERS + T::DbWeight::get().reads_writes(2, 1)] + #[weight = T::WeightInfo::external_propose(MAX_VETOERS)] fn external_propose(origin, proposal_hash: T::Hash) { T::ExternalOrigin::ensure_origin(origin)?; ensure!(!>::exists(), Error::::DuplicateProposal); @@ -828,9 +725,8 @@ decl_module! { /// # /// - Complexity: `O(1)` /// - Db write: `NextExternal` - /// - Base Weight: 3.065 µs /// # - #[weight = 3_100_000 + T::DbWeight::get().writes(1)] + #[weight = T::WeightInfo::external_propose_majority()] fn external_propose_majority(origin, proposal_hash: T::Hash) { T::ExternalMajorityOrigin::ensure_origin(origin)?; >::put((proposal_hash, VoteThreshold::SimpleMajority)); @@ -849,9 +745,8 @@ decl_module! { /// # /// - Complexity: `O(1)` /// - Db write: `NextExternal` - /// - Base Weight: 3.087 µs /// # - #[weight = 3_100_000 + T::DbWeight::get().writes(1)] + #[weight = T::WeightInfo::external_propose_default()] fn external_propose_default(origin, proposal_hash: T::Hash) { T::ExternalDefaultOrigin::ensure_origin(origin)?; >::put((proposal_hash, VoteThreshold::SuperMajorityAgainst)); @@ -877,7 +772,7 @@ decl_module! { /// - Db writes: `NextExternal`, `ReferendumCount`, `ReferendumInfoOf` /// - Base Weight: 30.1 µs /// # - #[weight = 30_000_000 + T::DbWeight::get().reads_writes(2, 3)] + #[weight = T::WeightInfo::fast_track()] fn fast_track(origin, proposal_hash: T::Hash, voting_period: T::BlockNumber, @@ -926,9 +821,8 @@ decl_module! { /// Performs a binary search on `existing_vetoers` which should not be very large. /// - Db reads: `NextExternal`, `Blacklist` /// - Db writes: `NextExternal`, `Blacklist` - /// - Base Weight: 29.87 + .188 * V µs /// # - #[weight = 30_000_000 + 180_000 * MAX_VETOERS + T::DbWeight::get().reads_writes(2, 2)] + #[weight = T::WeightInfo::veto_external(MAX_VETOERS)] fn veto_external(origin, proposal_hash: T::Hash) { let who = T::VetoOrigin::ensure_origin(origin)?; @@ -961,9 +855,8 @@ decl_module! { /// # /// - Complexity: `O(1)`. /// - Db writes: `ReferendumInfoOf` - /// - Base Weight: 21.57 µs /// # - #[weight = (22_000_000 + T::DbWeight::get().writes(1), DispatchClass::Operational)] + #[weight = T::WeightInfo::cancel_referendum()] fn cancel_referendum(origin, #[compact] ref_index: ReferendumIndex) { ensure_root(origin)?; Self::internal_cancel_referendum(ref_index); @@ -979,9 +872,8 @@ decl_module! { /// - `O(D)` where `D` is the items in the dispatch queue. Weighted as `D = 10`. /// - Db reads: `scheduler lookup`, scheduler agenda` /// - Db writes: `scheduler lookup`, scheduler agenda` - /// - Base Weight: 36.78 + 3.277 * D µs /// # - #[weight = (68_000_000 + T::DbWeight::get().reads_writes(2, 2), DispatchClass::Operational)] + #[weight = (T::WeightInfo::cancel_queued(10), DispatchClass::Operational)] fn cancel_queued(origin, which: ReferendumIndex) { ensure_root(origin)?; T::Scheduler::cancel_named((DEMOCRACY_ID, which).encode()) @@ -1017,14 +909,14 @@ decl_module! { /// # /// - Complexity: `O(R)` where R is the number of referendums the voter delegating to has /// voted on. Weight is charged as if maximum votes. - /// - Db reads: 2*`VotingOf`, `balances locks` - /// - Db writes: 2*`VotingOf`, `balances locks` + /// - Db reads: 3*`VotingOf`, `origin account locks` + /// - Db writes: 3*`VotingOf`, `origin account locks` /// - Db reads per votes: `ReferendumInfoOf` /// - Db writes per votes: `ReferendumInfoOf` - /// - Base Weight: 65.78 + 8.229 * R µs - // NOTE: weight must cover an incorrect voting of origin with 100 votes. + // NOTE: weight must cover an incorrect voting of origin with max votes, this is ensure + // because a valid delegation cover decoding a direct voting with max votes. /// # - #[weight = weight_for::delegate::(T::MaxVotes::get().into())] + #[weight = T::WeightInfo::delegate(T::MaxVotes::get())] pub fn delegate( origin, to: T::AccountId, @@ -1034,7 +926,7 @@ decl_module! { let who = ensure_signed(origin)?; let votes = Self::try_delegate(who, to, conviction, balance)?; - Ok(Some(weight_for::delegate::(votes.into())).into()) + Ok(Some(T::WeightInfo::delegate(votes)).into()) } /// Undelegate the voting power of the sending account. @@ -1054,14 +946,14 @@ decl_module! { /// - Db writes: 2*`VotingOf` /// - Db reads per votes: `ReferendumInfoOf` /// - Db writes per votes: `ReferendumInfoOf` - /// - Base Weight: 33.29 + 8.104 * R µs - // NOTE: weight must cover an incorrect voting of origin with 100 votes. + // NOTE: weight must cover an incorrect voting of origin with max votes, this is ensure + // because a valid delegation cover decoding a direct voting with max votes. /// # - #[weight = weight_for::undelegate::(T::MaxVotes::get().into())] + #[weight = T::WeightInfo::undelegate(T::MaxVotes::get().into())] fn undelegate(origin) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; let votes = Self::try_undelegate(who)?; - Ok(Some(weight_for::undelegate::(votes.into())).into()) + Ok(Some(T::WeightInfo::undelegate(votes)).into()) } /// Clears all public proposals. @@ -1071,9 +963,8 @@ decl_module! { /// # /// - `O(1)`. /// - Db writes: `PublicProps` - /// - Base Weight: 2.505 µs /// # - #[weight = 2_500_000 + T::DbWeight::get().writes(1)] + #[weight = T::WeightInfo::clear_public_proposals()] fn clear_public_proposals(origin) { ensure_root(origin)?; >::kill(); @@ -1089,16 +980,18 @@ decl_module! { /// Emits `PreimageNoted`. /// /// # - /// see `weight_for::note_preimage` + /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). + /// - Db reads: `Preimages` + /// - Db writes: `Preimages` /// # - #[weight = weight_for::note_preimage::((encoded_proposal.len() as u32).into())] + #[weight = T::WeightInfo::note_preimage(encoded_proposal.len() as u32)] fn note_preimage(origin, encoded_proposal: Vec) { Self::note_preimage_inner(ensure_signed(origin)?, encoded_proposal)?; } /// Same as `note_preimage` but origin is `OperationalPreimageOrigin`. #[weight = ( - weight_for::note_preimage::((encoded_proposal.len() as u32).into()), + T::WeightInfo::note_preimage(encoded_proposal.len() as u32), DispatchClass::Operational, )] fn note_preimage_operational(origin, encoded_proposal: Vec) { @@ -1116,16 +1009,18 @@ decl_module! { /// Emits `PreimageNoted`. /// /// # - /// see `weight_for::note_preimage` + /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). + /// - Db reads: `Preimages` + /// - Db writes: `Preimages` /// # - #[weight = weight_for::note_imminent_preimage::((encoded_proposal.len() as u32).into())] + #[weight = T::WeightInfo::note_imminent_preimage(encoded_proposal.len() as u32)] fn note_imminent_preimage(origin, encoded_proposal: Vec) { Self::note_imminent_preimage_inner(ensure_signed(origin)?, encoded_proposal)?; } /// Same as `note_imminent_preimage` but origin is `OperationalPreimageOrigin`. #[weight = ( - weight_for::note_imminent_preimage::((encoded_proposal.len() as u32).into()), + T::WeightInfo::note_imminent_preimage(encoded_proposal.len() as u32), DispatchClass::Operational, )] fn note_imminent_preimage_operational(origin, encoded_proposal: Vec) { @@ -1149,12 +1044,10 @@ decl_module! { /// /// # /// - Complexity: `O(D)` where D is length of proposal. - /// - Db reads: `Preimages` - /// - Db writes: `Preimages` - /// - Base Weight: 39.31 + .003 * b µs + /// - Db reads: `Preimages`, provider account data + /// - Db writes: `Preimages` provider account data /// # - #[weight = (39_000_000 + T::DbWeight::get().reads_writes(1, 1)) - .saturating_add(3_000.saturating_mul(Weight::from(*proposal_len_upper_bound)))] + #[weight = T::WeightInfo::reap_preimage(*proposal_len_upper_bound)] fn reap_preimage(origin, proposal_hash: T::Hash, #[compact] proposal_len_upper_bound: u32) { let who = ensure_signed(origin)?; @@ -1191,12 +1084,9 @@ decl_module! { /// - Complexity `O(R)` with R number of vote of target. /// - Db reads: `VotingOf`, `balances locks`, `target account` /// - Db writes: `VotingOf`, `balances locks`, `target account` - /// - Base Weight: - /// - Unlock Remove: 42.96 + .048 * R - /// - Unlock Set: 37.63 + .327 * R /// # - #[weight = 43_000_000 + 330_000 * Weight::from(T::MaxVotes::get()) - + T::DbWeight::get().reads_writes(3, 3)] + #[weight = T::WeightInfo::unlock_set(T::MaxVotes::get()) + .max(T::WeightInfo::unlock_remove(T::MaxVotes::get()))] fn unlock(origin, target: T::AccountId) { ensure_signed(origin)?; Self::update_lock(&target); @@ -1232,9 +1122,8 @@ decl_module! { /// Weight is calculated for the maximum number of vote. /// - Db reads: `ReferendumInfoOf`, `VotingOf` /// - Db writes: `ReferendumInfoOf`, `VotingOf` - /// - Base Weight: 21.03 + .359 * R /// # - #[weight = 21_000_000 + 360_000 * Weight::from(T::MaxVotes::get()) + T::DbWeight::get().reads_writes(2, 2)] + #[weight = T::WeightInfo::remove_vote(T::MaxVotes::get())] fn remove_vote(origin, index: ReferendumIndex) -> DispatchResult { let who = ensure_signed(origin)?; Self::try_remove_vote(&who, index, UnvoteScope::Any) @@ -1258,9 +1147,8 @@ decl_module! { /// Weight is calculated for the maximum number of vote. /// - Db reads: `ReferendumInfoOf`, `VotingOf` /// - Db writes: `ReferendumInfoOf`, `VotingOf` - /// - Base Weight: 19.15 + .372 * R /// # - #[weight = 19_000_000 + 370_000 * Weight::from(T::MaxVotes::get()) + T::DbWeight::get().reads_writes(2, 2)] + #[weight = T::WeightInfo::remove_other_vote(T::MaxVotes::get())] fn remove_other_vote(origin, target: T::AccountId, index: ReferendumIndex) -> DispatchResult { let who = ensure_signed(origin)?; let scope = if target == who { UnvoteScope::Any } else { UnvoteScope::OnlyExpired }; @@ -1716,10 +1604,9 @@ impl Module { /// `ReferendumCount`, `LowestUnbaked` /// - Db writes: `PublicProps`, `account`, `ReferendumCount`, `DepositOf`, `ReferendumInfoOf` /// - Db reads per R: `DepositOf`, `ReferendumInfoOf` - /// - Base Weight: 58.58 + 10.9 * R µs /// # fn begin_block(now: T::BlockNumber) -> Result { - let mut weight = 60_000_000 + T::DbWeight::get().reads_writes(6, 5); + let mut weight = 0; // pick out another public referendum if it's time. if (now % T::LaunchPeriod::get()).is_zero() { @@ -1729,11 +1616,11 @@ impl Module { weight = T::MaximumBlockWeight::get(); } - // tally up votes for any expiring referenda. let next = Self::lowest_unbaked(); let last = Self::referendum_count(); - let r = Weight::from(last.saturating_sub(next)); - weight += 11_000_000 * r + T::DbWeight::get().reads(2 * r); + let r = last.saturating_sub(next); + weight = weight.saturating_add(T::WeightInfo::on_initialize_base(r)); + // tally up votes for any expiring referenda. for (index, info) in Self::maturing_referenda_at_inner(now, next..last).into_iter() { let approved = Self::bake_referendum(now, index, info)?; ReferendumInfoOf::::insert(index, ReferendumInfo::Finished { end: now, approved }); From f71a5725cb7bc5c474eb20c32519e514f18c9872 Mon Sep 17 00:00:00 2001 From: ddorgan Date: Wed, 5 Aug 2020 10:39:13 +0200 Subject: [PATCH 17/32] Use DNS hostnames for flaming fir bootnodes (#6807) * Use dns hostnames for flaming fir bootnodes * Remove newline --- bin/node/cli/res/flaming-fir.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/node/cli/res/flaming-fir.json b/bin/node/cli/res/flaming-fir.json index 376c6a0e6e7e5..e2ecac2b44880 100644 --- a/bin/node/cli/res/flaming-fir.json +++ b/bin/node/cli/res/flaming-fir.json @@ -3,14 +3,14 @@ "id": "flamingfir8", "chainType": "Live", "bootNodes": [ - "/ip4/35.246.224.91/tcp/30333/p2p/12D3KooWLK2gMLhWsYJzjW3q35zAs9FDDVqfqVfVuskiGZGRSMvR", - "/ip4/35.246.224.91/tcp/30334/ws/p2p/12D3KooWLK2gMLhWsYJzjW3q35zAs9FDDVqfqVfVuskiGZGRSMvR", - "/ip4/35.246.210.11/tcp/30333/p2p/12D3KooWHyUSQkoL1WtnhLUYHuKbowZEZW1NNJe7TePKYZf9ucBY", - "/ip4/35.246.210.11/tcp/30334/ws/p2p/12D3KooWHyUSQkoL1WtnhLUYHuKbowZEZW1NNJe7TePKYZf9ucBY", - "/ip4/35.198.110.45/tcp/30333/p2p/12D3KooWFcry65ShtPT6roTTEPXD9H89A1iA2wPKgJCgXW1yZwyy", - "/ip4/35.198.110.45/tcp/30334/ws/p2p/12D3KooWFcry65ShtPT6roTTEPXD9H89A1iA2wPKgJCgXW1yZwyy", - "/ip4/35.198.114.154/tcp/30333/p2p/12D3KooWDfFapccu3KgvWyVMdXhMGPPpKiJ1yEhSMEupBZppfi9U", - "/ip4/35.198.114.154/tcp/30334/ws/p2p/12D3KooWDfFapccu3KgvWyVMdXhMGPPpKiJ1yEhSMEupBZppfi9U" + "/dns/0.flamingfir.paritytech.net/tcp/30333/p2p/12D3KooWLK2gMLhWsYJzjW3q35zAs9FDDVqfqVfVuskiGZGRSMvR", + "/dns/0.flamingfir.paritytech.net/tcp/30334/ws/p2p/12D3KooWLK2gMLhWsYJzjW3q35zAs9FDDVqfqVfVuskiGZGRSMvR", + "/dns/1.flamingfir.paritytech.net/tcp/30333/p2p/12D3KooWHyUSQkoL1WtnhLUYHuKbowZEZW1NNJe7TePKYZf9ucBY", + "/dns/1.flamingfir.paritytech.net/tcp/30334/ws/p2p/12D3KooWHyUSQkoL1WtnhLUYHuKbowZEZW1NNJe7TePKYZf9ucBY", + "/dns/2.flamingfir.paritytech.net/tcp/30333/p2p/12D3KooWFcry65ShtPT6roTTEPXD9H89A1iA2wPKgJCgXW1yZwyy", + "/dns/2.flamingfir.paritytech.net/tcp/30334/ws/p2p/12D3KooWFcry65ShtPT6roTTEPXD9H89A1iA2wPKgJCgXW1yZwyy", + "/dns/3.flamingfir.paritytech.net/tcp/30333/p2p/12D3KooWDfFapccu3KgvWyVMdXhMGPPpKiJ1yEhSMEupBZppfi9U", + "/dns/3.flamingfir.paritytech.net/tcp/30334/ws/p2p/12D3KooWDfFapccu3KgvWyVMdXhMGPPpKiJ1yEhSMEupBZppfi9U" ], "telemetryEndpoints": [ [ From f4bd80c4ca75e20ac028dfc57b1829b937976740 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Wed, 5 Aug 2020 11:16:19 +0200 Subject: [PATCH 18/32] Fix warning being printed by authority-discovery (#6820) --- client/network/src/service.rs | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/client/network/src/service.rs b/client/network/src/service.rs index e4ba36be587fe..230af3fb8e185 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -30,7 +30,7 @@ use crate::{ ExHashT, NetworkStateInfo, behaviour::{Behaviour, BehaviourOut}, - config::{parse_addr, parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig}, + config::{parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig}, DhtEvent, discovery::DiscoveryConfig, error::Error, @@ -43,7 +43,7 @@ use crate::{ transport, ReputationChange, }; use futures::prelude::*; -use libp2p::{PeerId, Multiaddr}; +use libp2p::{PeerId, multiaddr, Multiaddr}; use libp2p::core::{ConnectedPoint, Executor, connection::{ConnectionError, PendingConnectionError}, either::EitherError}; use libp2p::kad::record; use libp2p::ping::handler::PingFailure; @@ -879,21 +879,27 @@ impl NetworkService { /// Modify a peerset priority group. /// - /// Returns an `Err` if one of the given addresses contains an invalid - /// peer ID (which includes the local peer ID). + /// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. + /// + /// Returns an `Err` if one of the given addresses is invalid or contains an + /// invalid peer ID (which includes the local peer ID). pub fn set_priority_group(&self, group_id: String, peers: HashSet) -> Result<(), String> { let peers = peers.into_iter() - .map(|p| match parse_addr(p) { - Err(e) => Err(format!("{:?}", e)), - Ok((peer, addr)) => - // Make sure the local peer ID is never added to the PSM - // or added as a "known address", even if given. - if peer == self.local_peer_id { - Err("Local peer ID in priority group.".to_string()) - } else { - Ok((peer, addr)) - } - }) + .map(|mut addr| { + let peer = match addr.pop() { + Some(multiaddr::Protocol::P2p(key)) => PeerId::from_multihash(key) + .map_err(|_| "Invalid PeerId format".to_string())?, + _ => return Err("Missing PeerId from address".to_string()), + }; + + // Make sure the local peer ID is never added to the PSM + // or added as a "known address", even if given. + if peer == self.local_peer_id { + Err("Local peer ID in priority group.".to_string()) + } else { + Ok((peer, addr)) + } + }) .collect::, String>>()?; let peer_ids = peers.iter().map(|(peer_id, _addr)| peer_id.clone()).collect(); From f92a86a9506f331a867d36b0aa6b0bff3e462536 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Wed, 5 Aug 2020 11:40:31 +0200 Subject: [PATCH 19/32] Better default balances weights (#6813) --- frame/balances/src/default_weight.rs | 46 ++++++++++++++++++++++++++++ frame/balances/src/lib.rs | 9 +----- 2 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 frame/balances/src/default_weight.rs diff --git a/frame/balances/src/default_weight.rs b/frame/balances/src/default_weight.rs new file mode 100644 index 0000000000000..47a9199600564 --- /dev/null +++ b/frame/balances/src/default_weight.rs @@ -0,0 +1,46 @@ +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Weights for the Balances Pallet + +use frame_support::weights::{Weight, constants::RocksDbWeight as DbWeight}; + +impl crate::WeightInfo for () { + fn transfer() -> Weight { + (65949000 as Weight) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn transfer_keep_alive() -> Weight { + (46665000 as Weight) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn set_balance_creating() -> Weight { + (27086000 as Weight) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn set_balance_killing() -> Weight { + (33424000 as Weight) + .saturating_add(DbWeight::get().reads(1 as Weight)) + .saturating_add(DbWeight::get().writes(1 as Weight)) + } + fn force_transfer() -> Weight { + (65343000 as Weight) + .saturating_add(DbWeight::get().reads(2 as Weight)) + .saturating_add(DbWeight::get().writes(2 as Weight)) + } +} diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index cc9d9d179fa9c..c6b43677f2e59 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -154,6 +154,7 @@ mod tests; mod tests_local; mod tests_composite; mod benchmarking; +mod default_weight; use sp_std::prelude::*; use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible}; @@ -187,14 +188,6 @@ pub trait WeightInfo { fn force_transfer() -> Weight; } -impl WeightInfo for () { - fn transfer() -> Weight { 1_000_000_000 } - fn transfer_keep_alive() -> Weight { 1_000_000_000 } - fn set_balance_creating() -> Weight { 1_000_000_000 } - fn set_balance_killing() -> Weight { 1_000_000_000 } - fn force_transfer() -> Weight { 1_000_000_000 } -} - pub trait Subtrait: frame_system::Trait { /// The balance of an account. type Balance: Parameter + Member + AtLeast32BitUnsigned + Codec + Default + Copy + From b8ffd568b3cf7a0f5bcd0776398767082d18792d Mon Sep 17 00:00:00 2001 From: Guillaume Thiolliere Date: Wed, 5 Aug 2020 13:37:01 +0200 Subject: [PATCH 20/32] Remove generation of instance trait by decl_storage. (#6812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove generation of instance trait, no breaking change * doc * doc * Update frame/support/src/traits.rs Co-authored-by: Bastian Köcher * Update frame/support/procedural/src/storage/instance_trait.rs Co-authored-by: Bastian Köcher --- .../src/storage/genesis_config/mod.rs | 9 ++-- .../procedural/src/storage/instance_trait.rs | 42 +++++++++---------- frame/support/procedural/src/storage/parse.rs | 11 ++++- .../procedural/src/storage/storage_struct.rs | 10 ++--- frame/support/src/traits.rs | 11 +++++ frame/support/test/tests/final_keys.rs | 4 +- frame/support/test/tests/instance.rs | 7 ++-- 7 files changed, 54 insertions(+), 40 deletions(-) diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index 7cc0f7c3befd4..27fbdd2cd38b5 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -20,7 +20,7 @@ use proc_macro2::{TokenStream, Span}; use quote::quote; -use super::{DeclStorageDefExt, instance_trait::DEFAULT_INSTANTIABLE_TRAIT_NAME}; +use super::DeclStorageDefExt; use genesis_config_def::GenesisConfigDef; use builder_def::BuilderDef; @@ -104,10 +104,9 @@ fn impl_build_storage( let name = syn::Ident::new(DEFAULT_INSTANCE_NAME, Span::call_site()); quote!( #name ) }); - let inherent_instance_bound = def.optional_instance_bound.clone().unwrap_or_else(|| { - let bound = syn::Ident::new(DEFAULT_INSTANTIABLE_TRAIT_NAME, Span::call_site()); - quote!( #inherent_instance: #bound ) - }); + let inherent_instance_bound = quote!( + #inherent_instance: #scrate::traits::Instance + ); let build_storage_impl = quote!( <#runtime_generic: #runtime_trait, #inherent_instance_bound> diff --git a/frame/support/procedural/src/storage/instance_trait.rs b/frame/support/procedural/src/storage/instance_trait.rs index 1e5e198a8c527..a28c3ae622082 100644 --- a/frame/support/procedural/src/storage/instance_trait.rs +++ b/frame/support/procedural/src/storage/instance_trait.rs @@ -24,7 +24,6 @@ use super::DeclStorageDefExt; const NUMBER_OF_INSTANCE: usize = 16; pub(crate) const INHERENT_INSTANCE_NAME: &str = "__InherentHiddenInstance"; -pub(crate) const DEFAULT_INSTANTIABLE_TRAIT_NAME: &str = "__GeneratedInstantiable"; // Used to generate an instance implementation. struct InstanceDef { @@ -36,7 +35,7 @@ struct InstanceDef { pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStream { let mut impls = TokenStream::new(); - impls.extend(create_instance_trait(def)); + impls.extend(reexport_instance_trait(scrate, def)); // Implementation of instances. if let Some(module_instance) = &def.module_instance { @@ -70,6 +69,8 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre .and_then(|i| i.instance_default.as_ref()) { impls.extend(quote! { + /// Hidden instance generated to be internally used when module is used without + /// instance. #[doc(hidden)] pub type #inherent_instance = #default_instance; }); @@ -77,7 +78,11 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre let instance_def = InstanceDef { prefix: String::new(), instance_struct: inherent_instance, - doc: quote!(#[doc(hidden)]), + doc: quote!( + /// Hidden instance generated to be internally used when module is used without + /// instance. + #[doc(hidden)] + ), }; impls.extend(create_and_impl_instance_struct(scrate, &instance_def, def)); } @@ -85,27 +90,19 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre impls } -fn create_instance_trait( +fn reexport_instance_trait( + scrate: &TokenStream, def: &DeclStorageDefExt, ) -> TokenStream { - let instance_trait = def.module_instance.as_ref().map(|i| i.instance_trait.clone()) - .unwrap_or_else(|| syn::Ident::new(DEFAULT_INSTANTIABLE_TRAIT_NAME, Span::call_site())); - - let optional_hide = if def.module_instance.is_some() { - quote!() + if let Some(i) = def.module_instance.as_ref() { + let instance_trait = &i.instance_trait; + quote!( + /// Local import of frame_support::traits::Instance + // This import is not strictly needed but made in order not to have breaking change. + use #scrate::traits::Instance as #instance_trait; + ) } else { - quote!(#[doc(hidden)]) - }; - - quote! { - /// Tag a type as an instance of a module. - /// - /// Defines storage prefixes, they must be unique. - #optional_hide - pub trait #instance_trait: 'static { - /// The prefix used by any storage entry of an instance. - const PREFIX: &'static str; - } + quote!() } } @@ -114,8 +111,7 @@ fn create_and_impl_instance_struct( instance_def: &InstanceDef, def: &DeclStorageDefExt, ) -> TokenStream { - let instance_trait = def.module_instance.as_ref().map(|i| i.instance_trait.clone()) - .unwrap_or_else(|| syn::Ident::new(DEFAULT_INSTANTIABLE_TRAIT_NAME, Span::call_site())); + let instance_trait = quote!( #scrate::traits::Instance ); let instance_struct = &instance_def.instance_struct; let prefix = format!("{}{}", instance_def.prefix, def.crate_name.to_string()); diff --git a/frame/support/procedural/src/storage/parse.rs b/frame/support/procedural/src/storage/parse.rs index b1ef2916ad886..504af6d0ffcad 100644 --- a/frame/support/procedural/src/storage/parse.rs +++ b/frame/support/procedural/src/storage/parse.rs @@ -324,7 +324,16 @@ fn get_module_instance( instantiable: Option, default_instance: Option, ) -> syn::Result> { - let right_syntax = "Should be $Instance: $Instantiable = $DefaultInstance"; + let right_syntax = "Should be $I: $Instance = $DefaultInstance"; + + if instantiable.as_ref().map_or(false, |i| i != "Instance") { + let msg = format!( + "Instance trait must be named `Instance`, other names are no longer supported, because \ + it is now defined at frame_support::traits::Instance. Expect `Instance` found `{}`", + instantiable.as_ref().unwrap(), + ); + return Err(syn::Error::new(instantiable.span(), msg)); + } match (instance, instantiable, default_instance) { (Some(instance), Some(instantiable), default_instance) => { diff --git a/frame/support/procedural/src/storage/storage_struct.rs b/frame/support/procedural/src/storage/storage_struct.rs index 4cacb35c49d9a..e89b06770a6c5 100644 --- a/frame/support/procedural/src/storage/storage_struct.rs +++ b/frame/support/procedural/src/storage/storage_struct.rs @@ -106,7 +106,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre type Query = #query_type; fn module_prefix() -> &'static [u8] { - #instance_or_inherent::PREFIX.as_bytes() + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() } fn storage_prefix() -> &'static [u8] { @@ -130,7 +130,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre for #storage_struct #optional_storage_where_clause { fn module_prefix() -> &'static [u8] { - #instance_or_inherent::PREFIX.as_bytes() + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() } fn storage_prefix() -> &'static [u8] { @@ -145,7 +145,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre type Hasher = #scrate::#hasher; fn module_prefix() -> &'static [u8] { - #instance_or_inherent::PREFIX.as_bytes() + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() } fn storage_prefix() -> &'static [u8] { @@ -170,7 +170,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre for #storage_struct #optional_storage_where_clause { fn module_prefix() -> &'static [u8] { - #instance_or_inherent::PREFIX.as_bytes() + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() } fn storage_prefix() -> &'static [u8] { @@ -188,7 +188,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre type Hasher2 = #scrate::#hasher2; fn module_prefix() -> &'static [u8] { - #instance_or_inherent::PREFIX.as_bytes() + <#instance_or_inherent as #scrate::traits::Instance>::PREFIX.as_bytes() } fn storage_prefix() -> &'static [u8] { diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 752725ab46608..72a3850d2d37e 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1653,6 +1653,17 @@ impl IsType for T { fn into_mut(&mut self) -> &mut T { self } } +/// An instance of a pallet in the storage. +/// +/// It is required that these instances are unique, to support multiple instances per pallet in the same runtime! +/// +/// E.g. for module MyModule default instance will have prefix "MyModule" and other instances +/// "InstanceNMyModule". +pub trait Instance: 'static { + /// Unique module prefix. E.g. "InstanceNMyModule" or "MyModule" + const PREFIX: &'static str ; +} + #[cfg(test)] mod tests { use super::*; diff --git a/frame/support/test/tests/final_keys.rs b/frame/support/test/tests/final_keys.rs index e88389ade772b..34da1752da052 100644 --- a/frame/support/test/tests/final_keys.rs +++ b/frame/support/test/tests/final_keys.rs @@ -53,12 +53,12 @@ mod instance { pub trait Trait: super::no_instance::Trait {} frame_support::decl_module! { - pub struct Module, I: Instantiable = DefaultInstance> + pub struct Module, I: Instance = DefaultInstance> for enum Call where origin: T::Origin {} } frame_support::decl_storage!{ - trait Store for Module, I: Instantiable = DefaultInstance> + trait Store for Module, I: Instance = DefaultInstance> as FinalKeysSome { pub Value config(value): u32; diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 08389eed3aa83..33e8cc1fd6c0f 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -36,7 +36,6 @@ pub trait Currency {} // Test for: // * No default instance -// * Custom InstantiableTrait // * Origin, Inherent, Event mod module1 { use super::*; @@ -49,7 +48,7 @@ mod module1 { } frame_support::decl_module! { - pub struct Module, I: InstantiableThing> for enum Call where + pub struct Module, I: Instance> for enum Call where origin: ::Origin, system = system, T::BlockNumber: From @@ -67,7 +66,7 @@ mod module1 { } frame_support::decl_storage! { - trait Store for Module, I: InstantiableThing> as Module1 where + trait Store for Module, I: Instance> as Module1 where T::BlockNumber: From + std::fmt::Display { pub Value config(value): T::GenericType; @@ -97,7 +96,7 @@ mod module1 { pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"12345678"; - impl, I: InstantiableThing> ProvideInherent for Module where + impl, I: Instance> ProvideInherent for Module where T::BlockNumber: From { type Call = Call; From be40a78490b31ce82cb3f175f218e745a3e88833 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Wed, 5 Aug 2020 16:22:21 +0200 Subject: [PATCH 21/32] Successful `note_imminent_preimage` is free (#6793) * Successful `note_imminent_preimage` is free * update docs * Add test for duplicate preimage --- frame/democracy/src/lib.rs | 20 +++++++++++------ frame/democracy/src/tests/preimage.rs | 31 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index f3a5960eb2fe6..e298b1e4508c2 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -160,7 +160,7 @@ use sp_runtime::{ use codec::{Encode, Decode, Input}; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, ensure, Parameter, - weights::{Weight, DispatchClass}, + weights::{Weight, DispatchClass, Pays}, traits::{ Currency, ReservableCurrency, LockableCurrency, WithdrawReason, LockIdentifier, Get, OnUnbalanced, BalanceStatus, schedule::{Named as ScheduleNamed, DispatchTime}, EnsureOrigin @@ -458,14 +458,14 @@ decl_event! { Vetoed(AccountId, Hash, BlockNumber), /// A proposal's preimage was noted, and the deposit taken. [proposal_hash, who, deposit] PreimageNoted(Hash, AccountId, Balance), - /// A proposal preimage was removed and used (the deposit was returned). + /// A proposal preimage was removed and used (the deposit was returned). /// [proposal_hash, provider, deposit] PreimageUsed(Hash, AccountId, Balance), /// A proposal could not be executed because its preimage was invalid. [proposal_hash, ref_index] PreimageInvalid(Hash, ReferendumIndex), /// A proposal could not be executed because its preimage was missing. [proposal_hash, ref_index] PreimageMissing(Hash, ReferendumIndex), - /// A registered preimage was removed and the deposit collected by the reaper. + /// A registered preimage was removed and the deposit collected by the reaper. /// [proposal_hash, provider, deposit, reaper] PreimageReaped(Hash, AccountId, Balance, AccountId), /// An [account] has been unlocked successfully. @@ -1000,7 +1000,9 @@ decl_module! { } /// Register the preimage for an upcoming proposal. This requires the proposal to be - /// in the dispatch queue. No deposit is needed. + /// in the dispatch queue. No deposit is needed. When this call is successful, i.e. + /// the preimage has not been uploaded before and matches some imminent proposal, + /// no fee is paid. /// /// The dispatch origin of this call must be _Signed_. /// @@ -1014,8 +1016,11 @@ decl_module! { /// - Db writes: `Preimages` /// # #[weight = T::WeightInfo::note_imminent_preimage(encoded_proposal.len() as u32)] - fn note_imminent_preimage(origin, encoded_proposal: Vec) { + fn note_imminent_preimage(origin, encoded_proposal: Vec) -> DispatchResultWithPostInfo { Self::note_imminent_preimage_inner(ensure_signed(origin)?, encoded_proposal)?; + // We check that this preimage was not uploaded before in `note_imminent_preimage_inner`, + // thus this call can only be successful once. If successful, user does not pay a fee. + Ok(Pays::No.into()) } /// Same as `note_imminent_preimage` but origin is `OperationalPreimageOrigin`. @@ -1023,9 +1028,12 @@ decl_module! { T::WeightInfo::note_imminent_preimage(encoded_proposal.len() as u32), DispatchClass::Operational, )] - fn note_imminent_preimage_operational(origin, encoded_proposal: Vec) { + fn note_imminent_preimage_operational(origin, encoded_proposal: Vec) -> DispatchResultWithPostInfo { let who = T::OperationalPreimageOrigin::ensure_origin(origin)?; Self::note_imminent_preimage_inner(who, encoded_proposal)?; + // We check that this preimage was not uploaded before in `note_imminent_preimage_inner`, + // thus this call can only be successful once. If successful, user does not pay a fee. + Ok(Pays::No.into()) } /// Remove an expired proposal preimage and collect the deposit. diff --git a/frame/democracy/src/tests/preimage.rs b/frame/democracy/src/tests/preimage.rs index 4100a6a6b6375..8a2cbaf534032 100644 --- a/frame/democracy/src/tests/preimage.rs +++ b/frame/democracy/src/tests/preimage.rs @@ -164,3 +164,34 @@ fn reaping_imminent_preimage_should_fail() { assert_noop!(Democracy::reap_preimage(Origin::signed(6), h, u32::max_value()), Error::::Imminent); }); } + +#[test] +fn note_imminent_preimage_can_only_be_successful_once() { + new_test_ext().execute_with(|| { + PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); + + let r = Democracy::inject_referendum( + 2, + set_balance_proposal_hash(2), + VoteThreshold::SuperMajorityApprove, + 1 + ); + assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); + next_block(); + + // First time works + assert_ok!(Democracy::note_imminent_preimage(Origin::signed(6), set_balance_proposal(2))); + + // Second time fails + assert_noop!( + Democracy::note_imminent_preimage(Origin::signed(6), set_balance_proposal(2)), + Error::::DuplicatePreimage + ); + + // Fails from any user + assert_noop!( + Democracy::note_imminent_preimage(Origin::signed(5), set_balance_proposal(2)), + Error::::DuplicatePreimage + ); + }); +} From 5ce8f093a5c0dc077977a042d34a51e68de7b85e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= <123550+andresilva@users.noreply.github.com> Date: Wed, 5 Aug 2020 16:58:07 +0100 Subject: [PATCH 22/32] service: remove collection of system/process metrics (#6822) --- Cargo.lock | 346 +++++++++++----------------------- client/service/Cargo.toml | 7 - client/service/src/metrics.rs | 299 ++++------------------------- 3 files changed, 148 insertions(+), 504 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe237f2341b23..fb3cd395cf2f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,7 +151,7 @@ checksum = "4f823d037a7ec6ea2197046bafd4ae150e6bc36f9ca347404f46a46823fa84f2" dependencies = [ "approx", "num-complex", - "num-traits 0.2.11", + "num-traits", ] [[package]] @@ -184,7 +184,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" dependencies = [ - "num-traits 0.2.11", + "num-traits", ] [[package]] @@ -235,8 +235,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -382,7 +382,7 @@ dependencies = [ "log", "peeking_take_while", "proc-macro2", - "quote 1.0.6", + "quote", "regex", "rustc-hash", "shlex", @@ -686,7 +686,7 @@ checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" dependencies = [ "js-sys", "num-integer", - "num-traits 0.2.11", + "num-traits", "time", "wasm-bindgen", ] @@ -927,7 +927,7 @@ dependencies = [ "itertools 0.8.2", "lazy_static", "libc", - "num-traits 0.2.11", + "num-traits", "rand_core 0.3.1", "rand_os", "rand_xoshiro", @@ -953,7 +953,7 @@ dependencies = [ "csv", "itertools 0.8.2", "lazy_static", - "num-traits 0.2.11", + "num-traits", "oorandom", "plotters", "rayon", @@ -1096,8 +1096,8 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47c5e5ac752e18207b12e16b10631ae5f7f68f8805f335f9b817ead83d9ffce1" dependencies = [ - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1136,8 +1136,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2323f3f47db9a0e77ce7a300605d8d2098597fc451ed1a97bb1f6411bb550a7" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1219,8 +1219,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "558e40ea573c374cf53507fd240b7ee2f5477df7cfebdb97323ec61c719399c5" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1258,17 +1258,6 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" -[[package]] -name = "enum-primitive-derive" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b90e520ec62c1864c8c78d637acbfe8baf5f63240f2fb8165b8325c07812dd" -dependencies = [ - "num-traits 0.1.43", - "quote 0.3.15", - "syn 0.11.11", -] - [[package]] name = "enumflags2" version = "0.6.3" @@ -1285,8 +1274,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ed9afacaea0301eefb738c9deea725e6d53938004597cdc518a8cf9a7aa2f03" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1450,8 +1439,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "synstructure", ] @@ -1502,7 +1491,7 @@ dependencies = [ "futures 0.3.5", "futures-timer 2.0.2", "log", - "num-traits 0.2.11", + "num-traits", "parity-scale-codec", "parking_lot 0.9.0", "rand 0.6.5", @@ -1648,8 +1637,8 @@ version = "2.0.0-rc5" dependencies = [ "frame-support-procedural-tools", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1659,8 +1648,8 @@ dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1668,8 +1657,8 @@ name = "frame-support-procedural-tools-derive" version = "2.0.0-rc5" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -1885,8 +1874,8 @@ checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -2431,8 +2420,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -2571,8 +2560,8 @@ checksum = "0fadf6945e227246825a583514534d864554e9f23d80b3c77d034b10983db5ef" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -2784,18 +2773,6 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" -[[package]] -name = "libflate" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9135df43b1f5d0e333385cb6e7897ecd1a43d7d11b91ac003f4d2c2d2401fdd" -dependencies = [ - "adler32", - "crc32fast", - "rle-decode-fast", - "take_mut", -] - [[package]] name = "libloading" version = "0.5.2" @@ -2884,8 +2861,8 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "515c4a7cba5d321bb88ed3ed803997bdd5634ce35c9c5e8e9ace9c512e57eceb" dependencies = [ - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -3467,7 +3444,7 @@ dependencies = [ "matrixmultiply", "num-complex", "num-rational", - "num-traits 0.2.11", + "num-traits", "rand 0.6.5", "typenum", ] @@ -3492,20 +3469,6 @@ dependencies = [ "winapi 0.3.8", ] -[[package]] -name = "netstat2" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29449d242064c48d3057a194b049a2bdcccadda16faa18a91468677b44e8d422" -dependencies = [ - "bitflags", - "byteorder", - "enum-primitive-derive", - "libc", - "num-traits 0.2.11", - "thiserror", -] - [[package]] name = "nix" version = "0.17.0" @@ -3946,15 +3909,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "ntapi" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26e041cd983acbc087e30fcba770380cfa352d0e392e175b2344ebaf7ea0602" -dependencies = [ - "winapi 0.3.8", -] - [[package]] name = "num-bigint" version = "0.2.6" @@ -3963,7 +3917,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg 1.0.0", "num-integer", - "num-traits 0.2.11", + "num-traits", ] [[package]] @@ -3973,7 +3927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ "autocfg 1.0.0", - "num-traits 0.2.11", + "num-traits", ] [[package]] @@ -3983,7 +3937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" dependencies = [ "autocfg 1.0.0", - "num-traits 0.2.11", + "num-traits", ] [[package]] @@ -3995,16 +3949,7 @@ dependencies = [ "autocfg 1.0.0", "num-bigint", "num-integer", - "num-traits 0.2.11", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.11", + "num-traits", ] [[package]] @@ -4824,9 +4769,9 @@ version = "2.0.0-rc5" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.6", + "quote", "sp-runtime", - "syn 1.0.33", + "syn", ] [[package]] @@ -5026,8 +4971,8 @@ checksum = "cd20ff7e0399b274a5f5bb37b712fccb5b3a64b9128200d1c3cc40fe709cb073" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -5093,7 +5038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ "proc-macro2", - "syn 1.0.33", + "syn", "synstructure", ] @@ -5186,8 +5131,8 @@ checksum = "a62486e111e571b1e93b710b61e8f493c0013be39629b714cb166bdb06aa5a8a" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -5250,8 +5195,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -5285,7 +5230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3bb8da247d27ae212529352020f3e5ee16e83c0c258061d27b08ab92675eeb" dependencies = [ "js-sys", - "num-traits 0.2.11", + "num-traits", "wasm-bindgen", "web-sys", ] @@ -5383,8 +5328,8 @@ checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" dependencies = [ "proc-macro-error-attr", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "version_check", ] @@ -5395,8 +5340,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "syn-mid", "version_check", ] @@ -5419,22 +5364,7 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ - "unicode-xid 0.2.0", -] - -[[package]] -name = "procfs" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe50036aa1b71e553a4a0c48ab7baabf8aa8c7a5a61aae06bf38c2eab7430475" -dependencies = [ - "bitflags", - "byteorder", - "chrono", - "hex", - "lazy_static", - "libc", - "libflate", + "unicode-xid", ] [[package]] @@ -5487,8 +5417,8 @@ dependencies = [ "anyhow", "itertools 0.8.2", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -5541,12 +5471,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "quote" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" - [[package]] name = "quote" version = "1.0.6" @@ -5849,8 +5773,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602eb59cda66fcb9aec25841fb76bc01d2b34282dcdd705028da297db6f3eec8" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -5929,8 +5853,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "475e68978dc5b743f2f40d8e0a8fdc83f1c5e78cbf4b8fa5e74e73beebc340de" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -5965,12 +5889,6 @@ dependencies = [ "opaque-debug 0.3.0", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" - [[package]] name = "rlp" version = "0.4.5" @@ -6071,8 +5989,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -6205,8 +6123,8 @@ version = "2.0.0-rc5" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -6383,7 +6301,7 @@ dependencies = [ "merlin", "num-bigint", "num-rational", - "num-traits 0.2.11", + "num-traits", "parity-scale-codec", "parking_lot 0.10.2", "pdqselect", @@ -7015,12 +6933,10 @@ dependencies = [ "jsonrpc-pubsub", "lazy_static", "log", - "netstat2", "parity-scale-codec", "parity-util-mem 0.7.0", "parking_lot 0.10.2", "pin-project", - "procfs", "rand 0.7.3", "sc-block-builder", "sc-chain-spec", @@ -7061,7 +6977,6 @@ dependencies = [ "sp-version", "substrate-prometheus-endpoint", "substrate-test-runtime-client", - "sysinfo", "tempfile", "tokio 0.2.21", "tracing", @@ -7267,8 +7182,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -7372,8 +7287,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -7505,8 +7420,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a945ec7f7ce853e89ffa36be1e27dce9a43e82ff9093bf3461c30d5da74ed11b" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -7624,8 +7539,8 @@ dependencies = [ "blake2-rfc", "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -7675,7 +7590,7 @@ version = "2.0.0-rc5" dependencies = [ "criterion 0.3.1", "integer-sqrt", - "num-traits 0.2.11", + "num-traits", "parity-scale-codec", "primitive-types", "rand 0.7.3", @@ -7691,7 +7606,7 @@ version = "2.0.0-rc5" dependencies = [ "honggfuzz", "num-bigint", - "num-traits 0.2.11", + "num-traits", "primitive-types", "sp-arithmetic", ] @@ -7860,7 +7775,7 @@ dependencies = [ "libsecp256k1", "log", "merlin", - "num-traits 0.2.11", + "num-traits", "parity-scale-codec", "parity-util-mem 0.7.0", "parking_lot 0.10.2", @@ -7901,8 +7816,8 @@ name = "sp-debug-derive" version = "2.0.0-rc5" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -8000,8 +7915,8 @@ version = "2.0.0-rc5" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -8092,8 +8007,8 @@ dependencies = [ "Inflector", "proc-macro-crate", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -8184,7 +8099,7 @@ dependencies = [ "hex-literal", "itertools 0.9.0", "log", - "num-traits 0.2.11", + "num-traits", "parity-scale-codec", "parking_lot 0.10.2", "pretty_assertions", @@ -8392,8 +8307,8 @@ dependencies = [ "heck", "proc-macro-error", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -8413,8 +8328,8 @@ checksum = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" dependencies = [ "heck", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -8708,17 +8623,6 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" -[[package]] -name = "syn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" -dependencies = [ - "quote 0.3.15", - "synom", - "unicode-xid 0.0.4", -] - [[package]] name = "syn" version = "1.0.33" @@ -8726,8 +8630,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" dependencies = [ "proc-macro2", - "quote 1.0.6", - "unicode-xid 0.2.0", + "quote", + "unicode-xid", ] [[package]] @@ -8737,17 +8641,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", -] - -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" -dependencies = [ - "unicode-xid 0.0.4", + "quote", + "syn", ] [[package]] @@ -8757,24 +8652,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", - "unicode-xid 0.2.0", -] - -[[package]] -name = "sysinfo" -version = "0.14.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2983daff11a197c7c406b130579bc362177aa54cf2cc1f34d6ac88fccaa6a5e1" -dependencies = [ - "cfg-if", - "doc-comment", - "libc", - "ntapi", - "once_cell", - "rayon", - "winapi 0.3.8", + "quote", + "syn", + "unicode-xid", ] [[package]] @@ -8820,8 +8700,8 @@ checksum = "a605baa797821796a751f4a959e1206079b24a4b7e1ed302b7d785d81a9276c9" dependencies = [ "lazy_static", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "version_check", ] @@ -8850,8 +8730,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -9056,8 +8936,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -9257,8 +9137,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] @@ -9421,12 +9301,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" -[[package]] -name = "unicode-xid" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" - [[package]] name = "unicode-xid" version = "0.2.0" @@ -9611,8 +9485,8 @@ dependencies = [ "lazy_static", "log", "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "wasm-bindgen-shared", ] @@ -9634,7 +9508,7 @@ version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd85aa2c579e8892442954685f0d801f9129de24fa2136b2c6a539c76b65776" dependencies = [ - "quote 1.0.6", + "quote", "wasm-bindgen-macro-support", ] @@ -9645,8 +9519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eb197bd3a47553334907ffd2f16507b4f4f01bbec3ac921a7719e0decdfe72a" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9678,7 +9552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2f86cd78a2aa7b1fb4bb6ed854eccb7f9263089c79542dca1576a1518a8467" dependencies = [ "proc-macro2", - "quote 1.0.6", + "quote", ] [[package]] @@ -9718,7 +9592,7 @@ dependencies = [ "libc", "memory_units", "num-rational", - "num-traits 0.2.11", + "num-traits", "parity-wasm 0.41.0", "wasmi-validation", ] @@ -10060,8 +9934,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ "proc-macro2", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", "synstructure", ] diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index 32f6532e7e0d2..212d4e4b59edc 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -39,7 +39,6 @@ pin-project = "0.4.8" hash-db = "0.15.2" serde = "1.0.101" serde_json = "1.0.41" -sysinfo = "0.14.15" sc-keystore = { version = "2.0.0-rc5", path = "../keystore" } sp-io = { version = "2.0.0-rc5", path = "../../primitives/io" } sp-runtime = { version = "2.0.0-rc5", path = "../../primitives/runtime" } @@ -76,12 +75,6 @@ sc-tracing = { version = "2.0.0-rc5", path = "../tracing" } tracing = "0.1.10" parity-util-mem = { version = "0.7.0", default-features = false, features = ["primitive-types"] } -[target.'cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))'.dependencies] -netstat2 = "0.8.1" - -[target.'cfg(target_os = "linux")'.dependencies] -procfs = '0.7.8' - [target.'cfg(not(target_os = "unknown"))'.dependencies] tempfile = "3.1.0" directories = "2.0.2" diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index bac8b38d42390..7336d3862a65e 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -19,7 +19,7 @@ use std::{convert::TryFrom, time::SystemTime}; use crate::{NetworkStatus, config::Configuration}; -use prometheus_endpoint::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; +use prometheus_endpoint::{register, Gauge, U64, Registry, PrometheusError, Opts, GaugeVec}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::traits::{NumberFor, Block, SaturatedConversion, UniqueSaturatedInto}; use sp_transaction_pool::PoolStatus; @@ -27,28 +27,7 @@ use sp_utils::metrics::register_globals; use sc_client_api::ClientInfo; use sc_network::config::Role; -use sysinfo::{self, ProcessExt, SystemExt}; - -#[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] -use netstat2::{ - TcpState, ProtocolSocketInfo, iterate_sockets_info, AddressFamilyFlags, ProtocolFlags, -}; - struct PrometheusMetrics { - // system - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - load_avg: GaugeVec, - - // process - cpu_usage_percentage: Gauge, - memory_usage_bytes: Gauge, - threads: Gauge, - open_files: GaugeVec, - - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - netstat: GaugeVec, - - // -- inner counters // generic info block_height: GaugeVec, number_leaves: Gauge, @@ -62,9 +41,12 @@ struct PrometheusMetrics { } impl PrometheusMetrics { - fn setup(registry: &Registry, name: &str, version: &str, roles: u64) - -> Result - { + fn setup( + registry: &Registry, + name: &str, + version: &str, + roles: u64, + ) -> Result { register(Gauge::::with_opts( Opts::new( "build_info", @@ -88,39 +70,6 @@ impl PrometheusMetrics { )?, registry)?.set(start_time_since_epoch.as_secs()); Ok(Self { - // system - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - load_avg: register(GaugeVec::new( - Opts::new("load_avg", "System load average"), - &["over"] - )?, registry)?, - - // process - memory_usage_bytes: register(Gauge::new( - "memory_usage_bytes", "Process memory (resident set size) usage", - )?, registry)?, - - cpu_usage_percentage: register(Gauge::new( - "cpu_usage_percentage", "Process CPU usage, percentage per core summed over all cores", - )?, registry)?, - - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - netstat: register(GaugeVec::new( - Opts::new("netstat_tcp", "Number of TCP sockets of the process"), - &["status"] - )?, registry)?, - - threads: register(Gauge::new( - "threads", "Number of threads used by the process", - )?, registry)?, - - open_files: register(GaugeVec::new( - Opts::new("open_file_handles", "Number of open file handlers held by the process"), - &["fd_type"] - )?, registry)?, - - // --- internal - // generic internals block_height: register(GaugeVec::new( Opts::new("block_height", "Block height info of the chain"), @@ -154,116 +103,19 @@ impl PrometheusMetrics { } } -#[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] -#[derive(Default)] -struct ConnectionsCount { - listen: u64, - established: u64, - starting: u64, - closing: u64, - closed: u64, - other: u64 -} - -#[derive(Default)] -struct FdCounter { - paths: u64, - sockets: u64, - net: u64, - pipes: u64, - anon_inode: u64, - mem: u64, - other: u64, -} - -#[derive(Default)] -struct ProcessInfo { - cpu_usage: f64, - memory: u64, - threads: Option, - open_fd: Option, -} - pub struct MetricsService { metrics: Option, - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - system: sysinfo::System, - pid: Option, -} - -#[cfg(target_os = "linux")] -impl MetricsService { - fn inner_new(metrics: Option) -> Self { - let process = procfs::process::Process::myself() - .expect("Procfs doesn't fail on unix. qed"); - - Self { - metrics, - system: sysinfo::System::new_with_specifics(sysinfo::RefreshKind::new().with_processes()), - pid: Some(process.pid), - } - } - - fn process_info(&mut self) -> ProcessInfo { - let pid = self.pid.clone().expect("unix always has a pid. qed"); - let mut info = self.process_info_for(&pid); - let process = procfs::process::Process::new(pid).expect("Our process exists. qed."); - info.threads = process.stat().ok().map(|s| - u64::try_from(s.num_threads).expect("There are no negative thread counts. qed"), - ); - info.open_fd = process.fd().ok().map(|i| - i.into_iter().fold(FdCounter::default(), |mut f, info| { - match info.target { - procfs::process::FDTarget::Path(_) => f.paths += 1, - procfs::process::FDTarget::Socket(_) => f.sockets += 1, - procfs::process::FDTarget::Net(_) => f.net += 1, - procfs::process::FDTarget::Pipe(_) => f.pipes += 1, - procfs::process::FDTarget::AnonInode(_) => f.anon_inode += 1, - procfs::process::FDTarget::MemFD(_) => f.mem += 1, - procfs::process::FDTarget::Other(_,_) => f.other += 1, - }; - f - }) - ); - info - } } -#[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios"), not(target_os = "linux")))] impl MetricsService { - fn inner_new(metrics: Option) -> Self { - Self { - metrics, - system: sysinfo::System::new_with_specifics(sysinfo::RefreshKind::new().with_processes()), - pid: sysinfo::get_current_pid().ok(), - } - } - - fn process_info(&mut self) -> ProcessInfo { - self.pid.map(|pid| self.process_info_for(&pid)).unwrap_or_default() - } -} - - -#[cfg(not(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios"))))] -impl MetricsService { - fn inner_new(metrics: Option) -> Self { - Self { - metrics, - pid: None, - } - } - - fn process_info(&mut self) -> ProcessInfo { - ProcessInfo::default() + pub fn new() -> Self { + MetricsService { metrics: None } } -} - -impl MetricsService { - pub fn with_prometheus(registry: &Registry, config: &Configuration) - -> Result - { + pub fn with_prometheus( + registry: &Registry, + config: &Configuration, + ) -> Result { let role_bits = match config.role { Role::Full => 1u64, Role::Light => 2u64, @@ -271,57 +123,13 @@ impl MetricsService { Role::Authority { .. } => 4u64, }; - PrometheusMetrics::setup(registry, &config.network.node_name, &config.impl_version, role_bits).map(|p| { - Self::inner_new(Some(p)) - }) - } - - pub fn new() -> Self { - Self::inner_new(None) - } - - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - fn process_info_for(&mut self, pid: &sysinfo::Pid) -> ProcessInfo { - let mut info = ProcessInfo::default(); - self.system.refresh_process(*pid); - self.system.get_process(*pid).map(|prc| { - info.cpu_usage = prc.cpu_usage().into(); - info.memory = prc.memory(); - }); - info - } - - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - fn connections_info(&self) -> Option { - self.pid.as_ref().and_then(|pid| { - let af_flags = AddressFamilyFlags::IPV4 | AddressFamilyFlags::IPV6; - let proto_flags = ProtocolFlags::TCP; - let netstat_pid = *pid as u32; - - iterate_sockets_info(af_flags, proto_flags).ok().map(|iter| - iter.filter_map(|r| - r.ok().and_then(|s| { - match s.protocol_socket_info { - ProtocolSocketInfo::Tcp(info) - if s.associated_pids.contains(&netstat_pid) => Some(info.state), - _ => None - } - }) - ).fold(ConnectionsCount::default(), |mut counter, socket_state| { - match socket_state { - TcpState::Listen => counter.listen += 1, - TcpState::Established => counter.established += 1, - TcpState::Closed => counter.closed += 1, - TcpState::SynSent | TcpState::SynReceived => counter.starting += 1, - TcpState::FinWait1 | TcpState::FinWait2 | TcpState::CloseWait - | TcpState::Closing | TcpState::LastAck => counter.closing += 1, - _ => counter.other += 1 - } - - counter - }) - ) - }) + PrometheusMetrics::setup( + registry, + &config.network.node_name, + &config.impl_version, + role_bits, + ) + .map(|p| MetricsService { metrics: Some(p) }) } pub fn tick( @@ -330,16 +138,15 @@ impl MetricsService { txpool_status: &PoolStatus, net_status: &NetworkStatus, ) { - let best_number = info.chain.best_number.saturated_into::(); let best_hash = info.chain.best_hash; let num_peers = net_status.num_connected_peers; let finalized_number: u64 = info.chain.finalized_number.saturated_into::(); let bandwidth_download = net_status.average_download_per_sec; let bandwidth_upload = net_status.average_upload_per_sec; - let best_seen_block = net_status.best_seen_block + let best_seen_block = net_status + .best_seen_block .map(|num: NumberFor| num.unique_saturated_into() as u64); - let process_info = self.process_info(); telemetry!( SUBSTRATE_INFO; @@ -348,8 +155,6 @@ impl MetricsService { "height" => best_number, "best" => ?best_hash, "txcount" => txpool_status.ready, - "cpu" => process_info.cpu_usage, - "memory" => process_info.memory, "finalized_height" => finalized_number, "finalized_hash" => ?info.chain.finalized_hash, "bandwidth_download" => bandwidth_download, @@ -369,34 +174,23 @@ impl MetricsService { ); if let Some(metrics) = self.metrics.as_ref() { - metrics.cpu_usage_percentage.set(process_info.cpu_usage as f64); - // `sysinfo::Process::memory` returns memory usage in KiB and not bytes. - metrics.memory_usage_bytes.set(process_info.memory * 1024); - - if let Some(threads) = process_info.threads { - metrics.threads.set(threads); - } - - if let Some(fd_info) = process_info.open_fd { - metrics.open_files.with_label_values(&["paths"]).set(fd_info.paths); - metrics.open_files.with_label_values(&["mem"]).set(fd_info.mem); - metrics.open_files.with_label_values(&["sockets"]).set(fd_info.sockets); - metrics.open_files.with_label_values(&["net"]).set(fd_info.net); - metrics.open_files.with_label_values(&["pipe"]).set(fd_info.pipes); - metrics.open_files.with_label_values(&["anon_inode"]).set(fd_info.anon_inode); - metrics.open_files.with_label_values(&["other"]).set(fd_info.other); - } - - - metrics.network_per_sec_bytes.with_label_values(&["download"]).set( - net_status.average_download_per_sec, - ); - metrics.network_per_sec_bytes.with_label_values(&["upload"]).set( - net_status.average_upload_per_sec, - ); + metrics + .network_per_sec_bytes + .with_label_values(&["download"]) + .set(net_status.average_download_per_sec); + metrics + .network_per_sec_bytes + .with_label_values(&["upload"]) + .set(net_status.average_upload_per_sec); + metrics + .block_height + .with_label_values(&["finalized"]) + .set(finalized_number); + metrics + .block_height + .with_label_values(&["best"]) + .set(best_number); - metrics.block_height.with_label_values(&["finalized"]).set(finalized_number); - metrics.block_height.with_label_values(&["best"]).set(best_number); if let Ok(leaves) = u64::try_from(info.chain.number_leaves) { metrics.number_leaves.set(leaves); } @@ -421,23 +215,6 @@ impl MetricsService { info.memory.state_db.pinned.as_bytes() as u64, ); } - - #[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "ios")))] - { - let load = self.system.get_load_average(); - metrics.load_avg.with_label_values(&["1min"]).set(load.one); - metrics.load_avg.with_label_values(&["5min"]).set(load.five); - metrics.load_avg.with_label_values(&["15min"]).set(load.fifteen); - - if let Some(conns) = self.connections_info() { - metrics.netstat.with_label_values(&["listen"]).set(conns.listen); - metrics.netstat.with_label_values(&["established"]).set(conns.established); - metrics.netstat.with_label_values(&["starting"]).set(conns.starting); - metrics.netstat.with_label_values(&["closing"]).set(conns.closing); - metrics.netstat.with_label_values(&["closed"]).set(conns.closed); - metrics.netstat.with_label_values(&["other"]).set(conns.other); - } - } } } } From 35f3ba44819c2e0de54d01beddb24b3f0e24e8ef Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Wed, 5 Aug 2020 18:20:09 +0200 Subject: [PATCH 23/32] Fix legacy substream fallback not working (#6826) * Fix legacy substream fallback not working * Make it nicer --- .../network/src/protocol/generic_proto/handler/group.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs index 2826f7a19c8c2..dfa066936985e 100644 --- a/client/network/src/protocol/generic_proto/handler/group.rs +++ b/client/network/src/protocol/generic_proto/handler/group.rs @@ -600,15 +600,10 @@ impl ProtocolsHandler for NotifsHandler { message } => { for (handler, _) in &mut self.out_handlers { - if handler.protocol_name() != &protocol_name[..] { - continue; - } - - if handler.is_open() { + if handler.protocol_name() == &protocol_name[..] && handler.is_open() { handler.send_or_discard(message); + continue 'poll_notifs_sink; } - - continue 'poll_notifs_sink; } self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { From 60d67dcf02f09001ab99a3a0511d5bf906d71146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= <123550+andresilva@users.noreply.github.com> Date: Thu, 6 Aug 2020 10:30:29 +0100 Subject: [PATCH 24/32] grandpa: never overwrite current rounds voter state (#6823) * grandpa: never overwrite current rounds voter state * grandpa: add test for voter state overwrite --- client/finality-grandpa/Cargo.toml | 2 +- client/finality-grandpa/src/environment.rs | 7 +- client/finality-grandpa/src/tests.rs | 202 ++++++++++++++------- 3 files changed, 148 insertions(+), 63 deletions(-) diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 7b2e58b8be9a6..7cd3548a7628c 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -21,7 +21,6 @@ futures-timer = "3.0.1" log = "0.4.8" parking_lot = "0.10.0" rand = "0.7.2" -assert_matches = "1.3.0" parity-scale-codec = { version = "1.3.4", features = ["derive"] } sp-application-crypto = { version = "2.0.0-rc5", path = "../../primitives/application-crypto" } sp-arithmetic = { version = "2.0.0-rc5", path = "../../primitives/arithmetic" } @@ -47,6 +46,7 @@ finality-grandpa = { version = "0.12.3", features = ["derive-codec"] } pin-project = "0.4.6" [dev-dependencies] +assert_matches = "1.3.0" finality-grandpa = { version = "0.12.3", features = ["derive-codec", "test-helpers"] } sc-network = { version = "0.8.0-rc5", path = "../network" } sc-network-test = { version = "0.8.0-rc5", path = "../network/test" } diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index 0cfab13a6fa11..ca47e5e2cc4c5 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -930,7 +930,12 @@ where // remove the round from live rounds and start tracking the next round let mut current_rounds = current_rounds.clone(); current_rounds.remove(&round); - current_rounds.insert(round + 1, HasVoted::No); + + // NOTE: this condition should always hold as GRANDPA rounds are always + // started in increasing order, still it's better to play it safe. + if !current_rounds.contains_key(&(round + 1)) { + current_rounds.insert(round + 1, HasVoted::No); + } let set_state = VoterSetState::::Live { completed_rounds, diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index e2b9671f04df3..e2cdd7653a64f 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -19,10 +19,11 @@ //! Tests and test helpers for GRANDPA. use super::*; +use assert_matches::assert_matches; use environment::HasVoted; use sc_network_test::{ - Block, Hash, TestNetFactory, BlockImportAdapter, Peer, - PeersClient, PassThroughVerifier, PeersFullClient, + Block, BlockImportAdapter, Hash, PassThroughVerifier, Peer, PeersClient, PeersFullClient, + TestClient, TestNetFactory, }; use sc_network::config::{ProtocolConfig, BoxFinalityProofRequestBuilder}; use parking_lot::Mutex; @@ -53,16 +54,9 @@ use consensus_changes::ConsensusChanges; use sc_block_builder::BlockBuilderProvider; use sc_consensus::LongestChain; -type PeerData = - Mutex< - Option< - LinkHalf< - Block, - PeersFullClient, - LongestChain - > - > - >; +type TestLinkHalf = + LinkHalf>; +type PeerData = Mutex>; type GrandpaPeer = Peer; struct GrandpaTestNet { @@ -1519,10 +1513,67 @@ fn voter_catches_up_to_latest_round_when_behind() { ); } +type TestEnvironment = Environment< + substrate_test_runtime_client::Backend, + Block, + TestClient, + N, + LongestChain, + VR, +>; + +fn test_environment( + link: &TestLinkHalf, + keystore: Option, + network_service: N, + voting_rule: VR, +) -> TestEnvironment +where + N: NetworkT, + VR: VotingRule, +{ + let PersistentData { + ref authority_set, + ref consensus_changes, + ref set_state, + .. + } = link.persistent_data; + + let config = Config { + gossip_duration: TEST_GOSSIP_DURATION, + justification_period: 32, + keystore, + name: None, + is_authority: true, + observer_enabled: true, + }; + + let network = NetworkBridge::new( + network_service.clone(), + config.clone(), + set_state.clone(), + None, + ); + + Environment { + authority_set: authority_set.clone(), + config: config.clone(), + consensus_changes: consensus_changes.clone(), + client: link.client.clone(), + select_chain: link.select_chain.clone(), + set_id: authority_set.set_id(), + voter_set_state: set_state.clone(), + voters: Arc::new(authority_set.current_authorities()), + network, + voting_rule, + metrics: None, + _phantom: PhantomData, + } +} + #[test] fn grandpa_environment_respects_voting_rules() { use finality_grandpa::Chain; - use sc_network_test::TestClient; let peers = &[Ed25519Keyring::Alice]; let voters = make_ids(peers); @@ -1532,63 +1583,28 @@ fn grandpa_environment_respects_voting_rules() { let network_service = peer.network_service().clone(); let link = peer.data.lock().take().unwrap(); - // create a voter environment with a given voting rule - let environment = |voting_rule: Box>| { - let PersistentData { - ref authority_set, - ref consensus_changes, - ref set_state, - .. - } = link.persistent_data; - - let config = Config { - gossip_duration: TEST_GOSSIP_DURATION, - justification_period: 32, - keystore: None, - name: None, - is_authority: true, - observer_enabled: true, - }; - - let network = NetworkBridge::new( - network_service.clone(), - config.clone(), - set_state.clone(), - None, - ); - - Environment { - authority_set: authority_set.clone(), - config: config.clone(), - consensus_changes: consensus_changes.clone(), - client: link.client.clone(), - select_chain: link.select_chain.clone(), - set_id: authority_set.set_id(), - voter_set_state: set_state.clone(), - voters: Arc::new(authority_set.current_authorities()), - network, - voting_rule, - metrics: None, - _phantom: PhantomData, - } - }; - // add 21 blocks peer.push_blocks(21, false); // create an environment with no voting rule restrictions - let unrestricted_env = environment(Box::new(())); + let unrestricted_env = test_environment(&link, None, network_service.clone(), ()); // another with 3/4 unfinalized chain voting rule restriction - let three_quarters_env = environment(Box::new( - voting_rule::ThreeQuartersOfTheUnfinalizedChain - )); + let three_quarters_env = test_environment( + &link, + None, + network_service.clone(), + voting_rule::ThreeQuartersOfTheUnfinalizedChain, + ); // and another restricted with the default voting rules: i.e. 3/4 rule and // always below best block - let default_env = environment(Box::new( - VotingRulesBuilder::default().build() - )); + let default_env = test_environment( + &link, + None, + network_service.clone(), + VotingRulesBuilder::default().build(), + ); // the unrestricted environment should just return the best block assert_eq!( @@ -1648,6 +1664,70 @@ fn grandpa_environment_respects_voting_rules() { ); } +#[test] +fn grandpa_environment_never_overwrites_round_voter_state() { + use finality_grandpa::voter::Environment; + + let peers = &[Ed25519Keyring::Alice]; + let voters = make_ids(peers); + + let mut net = GrandpaTestNet::new(TestApi::new(voters), 1); + let peer = net.peer(0); + let network_service = peer.network_service().clone(); + let link = peer.data.lock().take().unwrap(); + + let (keystore, _keystore_path) = create_keystore(peers[0]); + let environment = test_environment(&link, Some(keystore), network_service.clone(), ()); + + let round_state = || finality_grandpa::round::State::genesis(Default::default()); + let base = || Default::default(); + let historical_votes = || finality_grandpa::HistoricalVotes::new(); + + let get_current_round = |n| { + let current_rounds = environment + .voter_set_state + .read() + .with_current_round(n) + .map(|(_, current_rounds)| current_rounds.clone()) + .ok()?; + + Some(current_rounds.get(&n).unwrap().clone()) + }; + + // round 2 should not be tracked + assert_eq!(get_current_round(2), None); + + // after completing round 1 we should start tracking round 2 + environment + .completed(1, round_state(), base(), &historical_votes()) + .unwrap(); + + assert_eq!(get_current_round(2).unwrap(), HasVoted::No); + + let info = peer.client().info(); + + let prevote = finality_grandpa::Prevote { + target_hash: info.best_hash, + target_number: info.best_number, + }; + + // we prevote for round 2 which should lead to us updating the voter state + environment.prevoted(2, prevote.clone()).unwrap(); + + let has_voted = get_current_round(2).unwrap(); + + assert_matches!(has_voted, HasVoted::Yes(_, _)); + assert_eq!(*has_voted.prevote().unwrap(), prevote); + + // if we report round 1 as completed again we should not overwrite the + // voter state for round 2 + environment + .completed(1, round_state(), base(), &historical_votes()) + .unwrap(); + + assert_matches!(get_current_round(2).unwrap(), HasVoted::Yes(_, _)); +} + #[test] fn imports_justification_for_regular_blocks_on_import() { // NOTE: this is a regression test since initially we would only import From bcf9dc57ac9b5f29f5668d596ca525547eadba85 Mon Sep 17 00:00:00 2001 From: Hamza Tokuchi Date: Thu, 6 Aug 2020 12:31:03 +0200 Subject: [PATCH 25/32] Add RPC Builder to Substrate Node Template (#6808) * Pulled RPC from node and populated the node-template's RPC builder with one example implementation * surpress build errror * dead_code * Fixed module usage, removed copyright, removed rpc builder for light client + some comments * added a comment for rpc extension * Update bin/node-template/node/src/rpc.rs Co-authored-by: Shawn Tabrizi * Update rpc.rs * fix spacing * more space to tabs * more space to tabs * Documenation nitpick * Documentation nitpick * Documentation nitpick * Documentation nitpick * Documentation nitpick * pre-format * Updated transaction payment API implemented for node template * fix space and commented code * fix long line Co-authored-by: Shawn Tabrizi Co-authored-by: Dan Forbes --- Cargo.lock | 10 +++++ bin/node-template/node/Cargo.toml | 12 ++++- bin/node-template/node/src/lib.rs | 1 + bin/node-template/node/src/main.rs | 1 + bin/node-template/node/src/rpc.rs | 64 +++++++++++++++++++++++++++ bin/node-template/node/src/service.rs | 35 ++++++++++----- bin/node-template/runtime/Cargo.toml | 7 ++- bin/node-template/runtime/src/lib.rs | 15 +++++++ 8 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 bin/node-template/node/src/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index fb3cd395cf2f0..20ed87326b36c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3789,7 +3789,9 @@ dependencies = [ name = "node-template" version = "2.0.0-rc5" dependencies = [ + "jsonrpc-core", "node-template-runtime", + "pallet-transaction-payment-rpc", "sc-basic-authorship", "sc-cli", "sc-client-api", @@ -3797,8 +3799,13 @@ dependencies = [ "sc-consensus-aura", "sc-executor", "sc-finality-grandpa", + "sc-rpc", + "sc-rpc-api", "sc-service", "sc-transaction-pool", + "sp-api", + "sp-block-builder", + "sp-blockchain", "sp-consensus", "sp-consensus-aura", "sp-core", @@ -3808,6 +3815,7 @@ dependencies = [ "sp-transaction-pool", "structopt", "substrate-build-script-utils", + "substrate-frame-rpc-system", ] [[package]] @@ -3817,6 +3825,7 @@ dependencies = [ "frame-executive", "frame-support", "frame-system", + "frame-system-rpc-runtime-api", "pallet-aura", "pallet-balances", "pallet-grandpa", @@ -3825,6 +3834,7 @@ dependencies = [ "pallet-template", "pallet-timestamp", "pallet-transaction-payment", + "pallet-transaction-payment-rpc-runtime-api", "parity-scale-codec", "serde", "sp-api", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 82c2d7ad43bb6..0c988ebd1a22b 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -33,7 +33,17 @@ sc-finality-grandpa = { version = "0.8.0-rc5", path = "../../../client/finality- sp-finality-grandpa = { version = "2.0.0-rc5", path = "../../../primitives/finality-grandpa" } sc-client-api = { version = "2.0.0-rc5", path = "../../../client/api" } sp-runtime = { version = "2.0.0-rc5", path = "../../../primitives/runtime" } -sc-basic-authorship = { path = "../../../client/basic-authorship", version = "0.8.0-rc5"} + +# These dependencies are used for the node template's RPCs +jsonrpc-core = "14.0.3" +sc-rpc = { version = "2.0.0-rc5", path = "../../../client/rpc" } +sp-api = { version = "2.0.0-rc5", path = "../../../primitives/api" } +sc-rpc-api = { version = "0.8.0-rc5", path = "../../../client/rpc-api" } +sp-blockchain = { version = "2.0.0-rc5", path = "../../../primitives/blockchain" } +sp-block-builder = { version = "2.0.0-rc5", path = "../../../primitives/block-builder" } +sc-basic-authorship = { version = "0.8.0-rc5", path = "../../../client/basic-authorship" } +substrate-frame-rpc-system = { version = "2.0.0-rc5", path = "../../../utils/frame/rpc/system" } +pallet-transaction-payment-rpc = { version = "2.0.0-rc5", path = "../../../frame/transaction-payment/rpc/" } node-template-runtime = { version = "2.0.0-rc5", path = "../runtime" } diff --git a/bin/node-template/node/src/lib.rs b/bin/node-template/node/src/lib.rs index 38e43372ca3ff..777c4f0a77147 100644 --- a/bin/node-template/node/src/lib.rs +++ b/bin/node-template/node/src/lib.rs @@ -1,2 +1,3 @@ pub mod chain_spec; pub mod service; +pub mod rpc; diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 369e6932a0308..4449d28b9fa41 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -6,6 +6,7 @@ mod chain_spec; mod service; mod cli; mod command; +mod rpc; fn main() -> sc_cli::Result<()> { command::run() diff --git a/bin/node-template/node/src/rpc.rs b/bin/node-template/node/src/rpc.rs new file mode 100644 index 0000000000000..c1f0e0a8457bc --- /dev/null +++ b/bin/node-template/node/src/rpc.rs @@ -0,0 +1,64 @@ +//! A collection of node-specific RPC methods. +//! Substrate provides the `sc-rpc` crate, which defines the core RPC layer +//! used by Substrate nodes. This file extends those RPC definitions with +//! capabilities that are specific to this project's runtime configuration. + +#![warn(missing_docs)] + +use std::sync::Arc; + +use node_template_runtime::{opaque::Block, AccountId, Balance, Index}; +use sp_api::ProvideRuntimeApi; +use sp_blockchain::{Error as BlockChainError, HeaderMetadata, HeaderBackend}; +use sp_block_builder::BlockBuilder; +pub use sc_rpc_api::DenyUnsafe; +use sp_transaction_pool::TransactionPool; + + +/// Full client dependencies. +pub struct FullDeps { + /// The client instance to use. + pub client: Arc, + /// Transaction pool instance. + pub pool: Arc

, + /// Whether to deny unsafe calls + pub deny_unsafe: DenyUnsafe, +} + +/// Instantiate all full RPC extensions. +pub fn create_full( + deps: FullDeps, +) -> jsonrpc_core::IoHandler where + C: ProvideRuntimeApi, + C: HeaderBackend + HeaderMetadata + 'static, + C: Send + Sync + 'static, + C::Api: substrate_frame_rpc_system::AccountNonceApi, + C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, + C::Api: BlockBuilder, + P: TransactionPool + 'static, +{ + use substrate_frame_rpc_system::{FullSystem, SystemApi}; + use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi}; + + let mut io = jsonrpc_core::IoHandler::default(); + let FullDeps { + client, + pool, + deny_unsafe, + } = deps; + + io.extend_with( + SystemApi::to_delegate(FullSystem::new(client.clone(), pool, deny_unsafe)) + ); + + io.extend_with( + TransactionPaymentApi::to_delegate(TransactionPayment::new(client.clone())) + ); + + // Extend this RPC with a custom API by using the following syntax. + // `YourRpcStruct` should have a reference to a client, which is needed + // to call into the runtime. + // `io.extend_with(YourRpcTrait::to_delegate(YourRpcStruct::new(ReferenceToClient, ...)));` + + io +} diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 0de17103b05a4..339aafbefcdf4 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -73,7 +73,7 @@ pub fn new_partial(config: &Configuration) -> Result Result { +pub fn new_full(config: Configuration) -> Result { let sc_service::PartialComponents { client, backend, mut task_manager, import_queue, keystore, select_chain, transaction_pool, inherent_data_providers, @@ -93,7 +93,7 @@ pub fn new_full(config: Configuration) -> Result { on_demand: None, block_announce_validator_builder: None, finality_proof_request_builder: None, - finality_proof_provider: Some(finality_proof_provider.clone()), + finality_proof_provider: Some(finality_proof_provider.clone()), })?; if config.offchain_worker.enabled { @@ -109,6 +109,21 @@ pub fn new_full(config: Configuration) -> Result { let prometheus_registry = config.prometheus_registry().cloned(); let telemetry_connection_sinks = sc_service::TelemetryConnectionSinks::default(); + let rpc_extensions_builder = { + let client = client.clone(); + let pool = transaction_pool.clone(); + + Box::new(move |deny_unsafe| { + let deps = crate::rpc::FullDeps { + client: client.clone(), + pool: pool.clone(), + deny_unsafe, + }; + + crate::rpc::create_full(deps) + }) + }; + sc_service::spawn_tasks(sc_service::SpawnTasksParams { network: network.clone(), client: client.clone(), @@ -116,7 +131,7 @@ pub fn new_full(config: Configuration) -> Result { task_manager: &mut task_manager, transaction_pool: transaction_pool.clone(), telemetry_connection_sinks: telemetry_connection_sinks.clone(), - rpc_extensions_builder: Box::new(|_| ()), + rpc_extensions_builder: rpc_extensions_builder, on_demand: None, remote_blockchain: None, backend, network_status_sinks, system_rpc_tx, config, @@ -256,7 +271,7 @@ pub fn new_light(config: Configuration) -> Result { &config, backend.clone(), task_manager.spawn_handle(), client.clone(), network.clone(), ); } - + sc_service::spawn_tasks(sc_service::SpawnTasksParams { remote_blockchain: Some(backend.remote_blockchain()), transaction_pool, @@ -264,12 +279,12 @@ pub fn new_light(config: Configuration) -> Result { on_demand: Some(on_demand), rpc_extensions_builder: Box::new(|_| ()), telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), - config, - client, - keystore, - backend, - network, - network_status_sinks, + config, + client, + keystore, + backend, + network, + network_status_sinks, system_rpc_tx, })?; diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 2bf3182542837..011916880bf8d 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -12,7 +12,6 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] } - aura = { version = "2.0.0-rc5", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } balances = { version = "2.0.0-rc5", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } frame-support = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/support" } @@ -36,6 +35,10 @@ sp-std = { version = "2.0.0-rc5", default-features = false, path = "../../../pri sp-transaction-pool = { version = "2.0.0-rc5", default-features = false, path = "../../../primitives/transaction-pool" } sp-version = { version = "2.0.0-rc5", default-features = false, path = "../../../primitives/version" } +# Used for the node template's RPCs +frame-system-rpc-runtime-api = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } + template = { version = "2.0.0-rc5", default-features = false, path = "../pallets/template", package = "pallet-template" } [build-dependencies] @@ -67,5 +70,7 @@ std = [ "system/std", "timestamp/std", "transaction-payment/std", + "frame-system-rpc-runtime-api/std", + "pallet-transaction-payment-rpc-runtime-api/std", "template/std", ] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index c46d515a3efdc..c7cd80868bffa 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -417,4 +417,19 @@ impl_runtime_apis! { None } } + + impl frame_system_rpc_runtime_api::AccountNonceApi for Runtime { + fn account_nonce(account: AccountId) -> Index { + System::account_nonce(account) + } + } + + impl pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi for Runtime { + fn query_info( + uxt: ::Extrinsic, + len: u32, + ) -> pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo { + TransactionPayment::query_info(uxt, len) + } + } } From 52494a465ab68904343c0525e86261ebcb166808 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Thu, 6 Aug 2020 14:46:34 +0200 Subject: [PATCH 26/32] Delay network startup to after complete initialization (#6833) * Delay network startup to after complete initialization * Update client/service/src/builder.rs Co-authored-by: Ashley Co-authored-by: Ashley --- bin/node-template/node/src/service.rs | 7 ++-- bin/node/cli/src/service.rs | 18 +++++----- client/service/src/builder.rs | 50 ++++++++++++++++++++++++--- client/service/src/lib.rs | 2 +- 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 339aafbefcdf4..021d0ac8f7d1e 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -83,7 +83,7 @@ pub fn new_full(config: Configuration) -> Result { let finality_proof_provider = GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); - let (network, network_status_sinks, system_rpc_tx) = + let (network, network_status_sinks, system_rpc_tx, network_starter) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), @@ -215,6 +215,7 @@ pub fn new_full(config: Configuration) -> Result { )?; } + network_starter.start_network(); Ok(task_manager) } @@ -253,7 +254,7 @@ pub fn new_light(config: Configuration) -> Result { let finality_proof_provider = GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); - let (network, network_status_sinks, system_rpc_tx) = + let (network, network_status_sinks, system_rpc_tx, network_starter) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), @@ -288,5 +289,7 @@ pub fn new_light(config: Configuration) -> Result { system_rpc_tx, })?; + network_starter.start_network(); + Ok(task_manager) } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index a47869ed83232..fdfa6816296aa 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -164,8 +164,8 @@ pub fn new_full_base( let finality_proof_provider = GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); - - let (network, network_status_sinks, system_rpc_tx) = + + let (network, network_status_sinks, system_rpc_tx, network_starter) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), @@ -206,7 +206,7 @@ pub fn new_full_base( network_status_sinks, system_rpc_tx, })?; - + let (block_import, grandpa_link, babe_link) = import_setup; let shared_voter_state = rpc_setup; @@ -322,6 +322,7 @@ pub fn new_full_base( )?; } + network_starter.start_network(); Ok((task_manager, inherent_data_providers, client, network, transaction_pool)) } @@ -383,7 +384,7 @@ pub fn new_light_base(config: Configuration) -> Result<( let finality_proof_provider = GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); - let (network, network_status_sinks, system_rpc_tx) = + let (network, network_status_sinks, system_rpc_tx, network_starter) = sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), @@ -395,7 +396,8 @@ pub fn new_light_base(config: Configuration) -> Result<( finality_proof_request_builder: Some(finality_proof_request_builder), finality_proof_provider: Some(finality_proof_provider), })?; - + network_starter.start_network(); + if config.offchain_worker.enabled { sc_service::build_offchain_workers( &config, backend.clone(), task_manager.spawn_handle(), client.clone(), network.clone(), @@ -412,7 +414,7 @@ pub fn new_light_base(config: Configuration) -> Result<( let rpc_extensions = node_rpc::create_light(light_deps); let rpc_handlers = - sc_service::spawn_tasks(sc_service::SpawnTasksParams { + sc_service::spawn_tasks(sc_service::SpawnTasksParams { on_demand: Some(on_demand), remote_blockchain: Some(backend.remote_blockchain()), rpc_extensions_builder: Box::new(sc_service::NoopRpcExtensionBuilder(rpc_extensions)), @@ -423,7 +425,7 @@ pub fn new_light_base(config: Configuration) -> Result<( telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), task_manager: &mut task_manager, })?; - + Ok((task_manager, rpc_handlers, client, network, transaction_pool)) } @@ -498,7 +500,7 @@ mod tests { setup_handles = Some((block_import.clone(), babe_link.clone())); } )?; - + let node = sc_service_test::TestNetComponents::new( keep_alive, client, network, transaction_pool ); diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 4c7c1f57ee04b..fc9303498d674 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -33,7 +33,7 @@ use sp_consensus::{ block_validation::{BlockAnnounceValidator, DefaultBlockAnnounceValidator, Chain}, import_queue::ImportQueue, }; -use futures::{FutureExt, StreamExt, future::ready}; +use futures::{FutureExt, StreamExt, future::ready, channel::oneshot}; use jsonrpc_pubsub::manager::SubscriptionManager; use sc_keystore::Store as Keystore; use log::{info, warn, error}; @@ -668,7 +668,7 @@ fn build_telemetry( let startup_time = SystemTime::UNIX_EPOCH.elapsed() .map(|dur| dur.as_millis()) .unwrap_or(0); - + spawn_handle.spawn( "telemetry-worker", telemetry.clone() @@ -822,6 +822,7 @@ pub fn build_network( Arc::Hash>>, NetworkStatusSinks, TracingUnboundedSender>, + NetworkStarter, ), Error > @@ -900,6 +901,22 @@ pub fn build_network( config.announce_block, ); + // TODO: Normally, one is supposed to pass a list of notifications protocols supported by the + // node through the `NetworkConfiguration` struct. But because this function doesn't know in + // advance which components, such as GrandPa or Polkadot, will be plugged on top of the + // service, it is unfortunately not possible to do so without some deep refactoring. To bypass + // this problem, the `NetworkService` provides a `register_notifications_protocol` method that + // can be called even after the network has been initialized. However, we want to avoid the + // situation where `register_notifications_protocol` is called *after* the network actually + // connects to other peers. For this reason, we delay the process of the network future until + // the user calls `NetworkStarter::start_network`. + // + // This entire hack should eventually be removed in favour of passing the list of protocols + // through the configuration. + // + // See also https://github.com/paritytech/substrate/issues/6827 + let (network_start_tx, network_start_rx) = oneshot::channel(); + // The network worker is responsible for gathering all network messages and processing // them. This is quite a heavy task, and at the time of the writing of this comment it // frequently happens that this future takes several seconds or in some situations @@ -907,7 +924,32 @@ pub fn build_network( // issue, and ideally we would like to fix the network future to take as little time as // possible, but we also take the extra harm-prevention measure to execute the networking // future using `spawn_blocking`. - spawn_handle.spawn_blocking("network-worker", future); + spawn_handle.spawn_blocking("network-worker", async move { + if network_start_rx.await.is_err() { + debug_assert!(false); + log::warn!( + "The NetworkStart returned as part of `build_network` has been silently dropped" + ); + // This `return` might seem unnecessary, but we don't want to make it look like + // everything is working as normal even though the user is clearly misusing the API. + return; + } + + future.await + }); + + Ok((network, network_status_sinks, system_rpc_tx, NetworkStarter(network_start_tx))) +} - Ok((network, network_status_sinks, system_rpc_tx)) +/// Object used to start the network. +#[must_use] +pub struct NetworkStarter(oneshot::Sender<()>); + +impl NetworkStarter { + /// Start the network. Call this after all sub-components have been initialized. + /// + /// > **Note**: If you don't call this function, the networking will not work. + pub fn start_network(self) { + let _ = self.0.send(()); + } } diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 40826a70d4501..415a5de4f930d 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -53,7 +53,7 @@ use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver, pub use self::error::Error; pub use self::builder::{ new_full_client, new_client, new_full_parts, new_light_parts, - spawn_tasks, build_network, BuildNetworkParams, build_offchain_workers, + spawn_tasks, build_network, BuildNetworkParams, NetworkStarter, build_offchain_workers, SpawnTasksParams, TFullClient, TLightClient, TFullBackend, TLightBackend, TLightBackendWithHash, TLightClientWithBackend, TFullCallExecutor, TLightCallExecutor, RpcExtensionBuilder, NoopRpcExtensionBuilder, From e2589a862c69ec487df6c1954f208205c9e6117c Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Thu, 6 Aug 2020 12:18:35 -0400 Subject: [PATCH 27/32] De-alias frame_system in node template runtime (#6829) * de-alias frame_system in node template * Fix line length * Fix chainspec --- bin/node-template/node/src/chain_spec.rs | 2 +- bin/node-template/runtime/Cargo.toml | 4 ++-- bin/node-template/runtime/src/lib.rs | 24 +++++++++++++++--------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index e49457b0b9a2e..bfa1b8b8ce60b 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -134,7 +134,7 @@ fn testnet_genesis( _enable_println: bool, ) -> GenesisConfig { GenesisConfig { - system: Some(SystemConfig { + frame_system: Some(SystemConfig { // Add Wasm runtime to storage. code: wasm_binary.to_vec(), changes_trie_config: Default::default(), diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 011916880bf8d..7523ad776ddd4 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -18,7 +18,7 @@ frame-support = { version = "2.0.0-rc5", default-features = false, path = "../.. grandpa = { version = "2.0.0-rc5", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } randomness-collective-flip = { version = "2.0.0-rc5", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } sudo = { version = "2.0.0-rc5", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } -system = { version = "2.0.0-rc5", default-features = false, package = "frame-system", path = "../../../frame/system" } +frame-system = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/system" } timestamp = { version = "2.0.0-rc5", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } transaction-payment = { version = "2.0.0-rc5", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } frame-executive = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/executive" } @@ -67,7 +67,7 @@ std = [ "sp-transaction-pool/std", "sp-version/std", "sudo/std", - "system/std", + "frame-system/std", "timestamp/std", "transaction-payment/std", "frame-system-rpc-runtime-api/std", diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index c7cd80868bffa..eebbfe2c5de1e 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -133,7 +133,7 @@ parameter_types! { // Configure FRAME pallets to include in runtime. -impl system::Trait for Runtime { +impl frame_system::Trait for Runtime { /// The basic call filter to use in dispatchable. type BaseCallFilter = (); /// The identifier used to distinguish between accounts. @@ -269,7 +269,7 @@ construct_runtime!( NodeBlock = opaque::Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Module, Call, Config, Storage, Event}, + System: frame_system::{Module, Call, Config, Storage, Event}, RandomnessCollectiveFlip: randomness_collective_flip::{Module, Call, Storage}, Timestamp: timestamp::{Module, Call, Storage, Inherent}, Aura: aura::{Module, Config, Inherent}, @@ -294,12 +294,12 @@ pub type SignedBlock = generic::SignedBlock; pub type BlockId = generic::BlockId; /// The SignedExtension to the basic transaction logic. pub type SignedExtra = ( - system::CheckSpecVersion, - system::CheckTxVersion, - system::CheckGenesis, - system::CheckEra, - system::CheckNonce, - system::CheckWeight, + frame_system::CheckSpecVersion, + frame_system::CheckTxVersion, + frame_system::CheckGenesis, + frame_system::CheckEra, + frame_system::CheckNonce, + frame_system::CheckWeight, transaction_payment::ChargeTransactionPayment ); /// Unchecked extrinsic type as expected by this runtime. @@ -307,7 +307,13 @@ pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; /// Executive: handles dispatch to the various modules. -pub type Executive = frame_executive::Executive, Runtime, AllModules>; +pub type Executive = frame_executive::Executive< + Runtime, + Block, + frame_system::ChainContext, + Runtime, + AllModules, +>; impl_runtime_apis! { impl sp_api::Core for Runtime { From bd1a16c318a5f42f1a0176f33bad61bad491c771 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Thu, 6 Aug 2020 21:20:46 +0200 Subject: [PATCH 28/32] Allow task manager to have children (#6771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial commit Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * WIP Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * WIP Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * WIP Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * WIP Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * WIP Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * WIP Forked at: 19c1d9028d8d6eabef41693433b56e14da025247 Parent branch: origin/master * changelog * Remove Box * Make future nicer * Revert "Make future nicer" This reverts commit 49fb8fb6f245c3ca2c384468df14b34f34616736. * Simplify * Additional check * Simplify more Co-authored-by: Bastian Köcher --- client/service/src/task_manager/mod.rs | 39 ++++++++- client/service/src/task_manager/tests.rs | 106 ++++++++++++++++++++++- docs/CHANGELOG.md | 5 ++ 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index e0e8699ce1d37..729b43bce94c1 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -18,7 +18,7 @@ use exit_future::Signal; use log::{debug, error}; use futures::{ Future, FutureExt, StreamExt, - future::{select, Either, BoxFuture}, + future::{select, Either, BoxFuture, join_all, try_join_all, pending}, sink::SinkExt, }; use prometheus_endpoint::{ @@ -214,8 +214,14 @@ pub struct TaskManager { essential_failed_rx: TracingUnboundedReceiver<()>, /// Things to keep alive until the task manager is dropped. keep_alive: Box, + /// A sender to a stream of background tasks. This is used for the completion future. task_notifier: TracingUnboundedSender, + /// This future will complete when all the tasks are joined and the stream is closed. completion_future: JoinFuture, + /// A list of other `TaskManager`'s to terminate and gracefully shutdown when the parent + /// terminates and gracefully shutdown. Also ends the parent `future()` if a child's essential + /// task fails. + children: Vec, } impl TaskManager { @@ -251,6 +257,7 @@ impl TaskManager { keep_alive: Box::new(()), task_notifier, completion_future, + children: Vec::new(), }) } @@ -271,12 +278,21 @@ impl TaskManager { /// Send the signal for termination, prevent new tasks to be created, await for all the existing /// tasks to be finished and drop the object. You can consider this as an async drop. + /// + /// It's always better to call and await this function before exiting the process as background + /// tasks may be running in the background. If the process exit and the background tasks are not + /// cancelled, this will lead to objects not getting dropped properly. + /// + /// This is an issue in some cases as some of our dependencies do require that we drop all the + /// objects properly otherwise it triggers a SIGABRT on exit. pub fn clean_shutdown(mut self) -> Pin + Send>> { self.terminate(); + let children_shutdowns = self.children.into_iter().map(|x| x.clean_shutdown()); let keep_alive = self.keep_alive; let completion_future = self.completion_future; Box::pin(async move { + join_all(children_shutdowns).await; completion_future.await; drop(keep_alive); }) @@ -293,10 +309,17 @@ impl TaskManager { Box::pin(async move { let mut t1 = self.essential_failed_rx.next().fuse(); let mut t2 = self.on_exit.clone().fuse(); + let mut t3 = try_join_all( + self.children.iter_mut().map(|x| x.future()) + // Never end this future if there is no error because if there is no children, + // it must not stop + .chain(std::iter::once(pending().boxed())) + ).fuse(); futures::select! { _ = t1 => Err(Error::Other("Essential task failed.".into())), _ = t2 => Ok(()), + res = t3 => Err(res.map(|_| ()).expect_err("this future never ends; qed")), } }) } @@ -305,15 +328,25 @@ impl TaskManager { pub fn terminate(&mut self) { if let Some(signal) = self.signal.take() { let _ = signal.fire(); - // NOTE: task will prevent new tasks to be spawned + // NOTE: this will prevent new tasks to be spawned self.task_notifier.close_channel(); + for child in self.children.iter_mut() { + child.terminate(); + } } } - /// Set what the task manager should keep alivei + /// Set what the task manager should keep alive. pub(super) fn keep_alive(&mut self, to_keep_alive: T) { self.keep_alive = Box::new(to_keep_alive); } + + /// Register another TaskManager to terminate and gracefully shutdown when the parent + /// terminates and gracefully shutdown. Also ends the parent `future()` if a child's essential + /// task fails. (But don't end the parent if a child's normal task fails.) + pub fn add_children(&mut self, child: TaskManager) { + self.children.push(child); + } } #[derive(Clone)] diff --git a/client/service/src/task_manager/tests.rs b/client/service/src/task_manager/tests.rs index c60d15b3394c3..a2bd84802aa17 100644 --- a/client/service/src/task_manager/tests.rs +++ b/client/service/src/task_manager/tests.rs @@ -18,7 +18,7 @@ use crate::config::TaskExecutor; use crate::task_manager::TaskManager; -use futures::future::FutureExt; +use futures::{future::FutureExt, pin_mut, select}; use parking_lot::Mutex; use std::any::Any; use std::sync::Arc; @@ -82,7 +82,7 @@ async fn run_background_task_blocking(duration: Duration, _keep_alive: impl Any) } #[test] -fn ensure_futures_are_awaited_on_shutdown() { +fn ensure_tasks_are_awaited_on_shutdown() { let mut runtime = tokio::runtime::Runtime::new().unwrap(); let handle = runtime.handle().clone(); let task_executor: TaskExecutor = (move |future, _| handle.spawn(future).map(|_| ())).into(); @@ -187,7 +187,7 @@ fn ensure_task_manager_future_ends_when_task_manager_terminated() { } #[test] -fn ensure_task_manager_future_ends_with_error_when_essential_task_ends() { +fn ensure_task_manager_future_ends_with_error_when_essential_task_fails() { let mut runtime = tokio::runtime::Runtime::new().unwrap(); let handle = runtime.handle().clone(); let task_executor: TaskExecutor = (move |future, _| handle.spawn(future).map(|_| ())).into(); @@ -208,3 +208,103 @@ fn ensure_task_manager_future_ends_with_error_when_essential_task_ends() { runtime.block_on(task_manager.clean_shutdown()); assert_eq!(drop_tester, 0); } + +#[test] +fn ensure_children_tasks_ends_when_task_manager_terminated() { + let mut runtime = tokio::runtime::Runtime::new().unwrap(); + let handle = runtime.handle().clone(); + let task_executor: TaskExecutor = (move |future, _| handle.spawn(future).map(|_| ())).into(); + + let mut task_manager = TaskManager::new(task_executor.clone(), None).unwrap(); + let child_1 = TaskManager::new(task_executor.clone(), None).unwrap(); + let spawn_handle_child_1 = child_1.spawn_handle(); + let child_2 = TaskManager::new(task_executor.clone(), None).unwrap(); + let spawn_handle_child_2 = child_2.spawn_handle(); + task_manager.add_children(child_1); + task_manager.add_children(child_2); + let spawn_handle = task_manager.spawn_handle(); + let drop_tester = DropTester::new(); + spawn_handle.spawn("task1", run_background_task(drop_tester.new_ref())); + spawn_handle.spawn("task2", run_background_task(drop_tester.new_ref())); + spawn_handle_child_1.spawn("task3", run_background_task(drop_tester.new_ref())); + spawn_handle_child_2.spawn("task4", run_background_task(drop_tester.new_ref())); + assert_eq!(drop_tester, 4); + // allow the tasks to even start + runtime.block_on(async { tokio::time::delay_for(Duration::from_secs(1)).await }); + assert_eq!(drop_tester, 4); + task_manager.terminate(); + runtime.block_on(task_manager.future()).expect("future has ended without error"); + runtime.block_on(task_manager.clean_shutdown()); + assert_eq!(drop_tester, 0); +} + +#[test] +fn ensure_task_manager_future_ends_with_error_when_childs_essential_task_fails() { + let mut runtime = tokio::runtime::Runtime::new().unwrap(); + let handle = runtime.handle().clone(); + let task_executor: TaskExecutor = (move |future, _| handle.spawn(future).map(|_| ())).into(); + + let mut task_manager = TaskManager::new(task_executor.clone(), None).unwrap(); + let child_1 = TaskManager::new(task_executor.clone(), None).unwrap(); + let spawn_handle_child_1 = child_1.spawn_handle(); + let spawn_essential_handle_child_1 = child_1.spawn_essential_handle(); + let child_2 = TaskManager::new(task_executor.clone(), None).unwrap(); + let spawn_handle_child_2 = child_2.spawn_handle(); + task_manager.add_children(child_1); + task_manager.add_children(child_2); + let spawn_handle = task_manager.spawn_handle(); + let drop_tester = DropTester::new(); + spawn_handle.spawn("task1", run_background_task(drop_tester.new_ref())); + spawn_handle.spawn("task2", run_background_task(drop_tester.new_ref())); + spawn_handle_child_1.spawn("task3", run_background_task(drop_tester.new_ref())); + spawn_handle_child_2.spawn("task4", run_background_task(drop_tester.new_ref())); + assert_eq!(drop_tester, 4); + // allow the tasks to even start + runtime.block_on(async { tokio::time::delay_for(Duration::from_secs(1)).await }); + assert_eq!(drop_tester, 4); + spawn_essential_handle_child_1.spawn("task5", async { panic!("task failed") }); + runtime.block_on(task_manager.future()).expect_err("future()'s Result must be Err"); + assert_eq!(drop_tester, 4); + runtime.block_on(task_manager.clean_shutdown()); + assert_eq!(drop_tester, 0); +} + +#[test] +fn ensure_task_manager_future_continues_when_childs_not_essential_task_fails() { + let mut runtime = tokio::runtime::Runtime::new().unwrap(); + let handle = runtime.handle().clone(); + let task_executor: TaskExecutor = (move |future, _| handle.spawn(future).map(|_| ())).into(); + + let mut task_manager = TaskManager::new(task_executor.clone(), None).unwrap(); + let child_1 = TaskManager::new(task_executor.clone(), None).unwrap(); + let spawn_handle_child_1 = child_1.spawn_handle(); + let child_2 = TaskManager::new(task_executor.clone(), None).unwrap(); + let spawn_handle_child_2 = child_2.spawn_handle(); + task_manager.add_children(child_1); + task_manager.add_children(child_2); + let spawn_handle = task_manager.spawn_handle(); + let drop_tester = DropTester::new(); + spawn_handle.spawn("task1", run_background_task(drop_tester.new_ref())); + spawn_handle.spawn("task2", run_background_task(drop_tester.new_ref())); + spawn_handle_child_1.spawn("task3", run_background_task(drop_tester.new_ref())); + spawn_handle_child_2.spawn("task4", run_background_task(drop_tester.new_ref())); + assert_eq!(drop_tester, 4); + // allow the tasks to even start + runtime.block_on(async { tokio::time::delay_for(Duration::from_secs(1)).await }); + assert_eq!(drop_tester, 4); + spawn_handle_child_1.spawn("task5", async { panic!("task failed") }); + runtime.block_on(async { + let t1 = task_manager.future().fuse(); + let t2 = tokio::time::delay_for(Duration::from_secs(3)).fuse(); + + pin_mut!(t1, t2); + + select! { + res = t1 => panic!("task should not have stopped: {:?}", res), + _ = t2 => {}, + } + }); + assert_eq!(drop_tester, 4); + runtime.block_on(task_manager.clean_shutdown()); + assert_eq!(drop_tester, 0); +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 333733d1aee01..719059477e13e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog]. ## Unreleased +Client +------ + +* Child nodes can be handled by adding a child `TaskManager` to the parent's `TaskManager` (#6771) + ## 2.0.0-rc4 -> 2.0.0-rc5 – River Dolphin Runtime From f17569fa4c7fa930eab1dd07a359fde7e3019093 Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Thu, 6 Aug 2020 15:23:31 -0400 Subject: [PATCH 29/32] De-alias pallets in node template runtime (#6836) * dealias pallets * Restore accidentally deleted code blocks --- bin/node-template/node/src/chain_spec.rs | 8 ++--- bin/node-template/runtime/Cargo.toml | 31 ++++++++--------- bin/node-template/runtime/src/lib.rs | 42 ++++++++++++------------ 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index bfa1b8b8ce60b..41f582fb64a46 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -139,17 +139,17 @@ fn testnet_genesis( code: wasm_binary.to_vec(), changes_trie_config: Default::default(), }), - balances: Some(BalancesConfig { + pallet_balances: Some(BalancesConfig { // Configure endowed accounts with initial balance of 1 << 60. balances: endowed_accounts.iter().cloned().map(|k|(k, 1 << 60)).collect(), }), - aura: Some(AuraConfig { + pallet_aura: Some(AuraConfig { authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(), }), - grandpa: Some(GrandpaConfig { + pallet_grandpa: Some(GrandpaConfig { authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(), }), - sudo: Some(SudoConfig { + pallet_sudo: Some(SudoConfig { // Assign network admin rights. key: root_key, }), diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 7523ad776ddd4..f4e8697a47d0d 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -12,15 +12,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] } -aura = { version = "2.0.0-rc5", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } -balances = { version = "2.0.0-rc5", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } + +pallet-aura = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/aura" } +pallet-balances = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/balances" } frame-support = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/support" } -grandpa = { version = "2.0.0-rc5", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } -randomness-collective-flip = { version = "2.0.0-rc5", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } -sudo = { version = "2.0.0-rc5", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } +pallet-grandpa = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/grandpa" } +pallet-randomness-collective-flip = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/randomness-collective-flip" } +pallet-sudo = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/sudo" } frame-system = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/system" } -timestamp = { version = "2.0.0-rc5", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } -transaction-payment = { version = "2.0.0-rc5", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } +pallet-timestamp = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/transaction-payment" } frame-executive = { version = "2.0.0-rc5", default-features = false, path = "../../../frame/executive" } serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-api = { version = "2.0.0-rc5", default-features = false, path = "../../../primitives/api" } @@ -47,13 +48,17 @@ wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-run [features] default = ["std"] std = [ - "aura/std", - "balances/std", "codec/std", "frame-executive/std", "frame-support/std", - "grandpa/std", - "randomness-collective-flip/std", + "pallet-aura/std", + "pallet-balances/std", + "pallet-grandpa/std", + "pallet-randomness-collective-flip/std", + "pallet-sudo/std", + "pallet-timestamp/std", + "pallet-transaction-payment/std", + "pallet-transaction-payment-rpc-runtime-api/std", "serde", "sp-api/std", "sp-block-builder/std", @@ -66,11 +71,7 @@ std = [ "sp-std/std", "sp-transaction-pool/std", "sp-version/std", - "sudo/std", "frame-system/std", - "timestamp/std", - "transaction-payment/std", "frame-system-rpc-runtime-api/std", - "pallet-transaction-payment-rpc-runtime-api/std", "template/std", ] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index eebbfe2c5de1e..06e34e4551673 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -17,8 +17,8 @@ use sp_runtime::traits::{ }; use sp_api::impl_runtime_apis; use sp_consensus_aura::sr25519::AuthorityId as AuraId; -use grandpa::{AuthorityId as GrandpaId, AuthorityList as GrandpaAuthorityList}; -use grandpa::fg_primitives; +use pallet_grandpa::{AuthorityId as GrandpaId, AuthorityList as GrandpaAuthorityList}; +use pallet_grandpa::fg_primitives; use sp_version::RuntimeVersion; #[cfg(feature = "std")] use sp_version::NativeVersion; @@ -26,8 +26,8 @@ use sp_version::NativeVersion; // A few exports that help ease life for downstream crates. #[cfg(any(feature = "std", test))] pub use sp_runtime::BuildStorage; -pub use timestamp::Call as TimestampCall; -pub use balances::Call as BalancesCall; +pub use pallet_timestamp::Call as TimestampCall; +pub use pallet_balances::Call as BalancesCall; pub use sp_runtime::{Permill, Perbill}; pub use frame_support::{ construct_runtime, parameter_types, StorageValue, @@ -187,16 +187,16 @@ impl frame_system::Trait for Runtime { /// What to do if an account is fully reaped from the system. type OnKilledAccount = (); /// The data to be stored in an account. - type AccountData = balances::AccountData; + type AccountData = pallet_balances::AccountData; /// Weight information for the extrinsics of this pallet. type SystemWeightInfo = (); } -impl aura::Trait for Runtime { +impl pallet_aura::Trait for Runtime { type AuthorityId = AuraId; } -impl grandpa::Trait for Runtime { +impl pallet_grandpa::Trait for Runtime { type Event = Event; type Call = Call; @@ -217,7 +217,7 @@ parameter_types! { pub const MinimumPeriod: u64 = SLOT_DURATION / 2; } -impl timestamp::Trait for Runtime { +impl pallet_timestamp::Trait for Runtime { /// A timestamp: milliseconds since the unix epoch. type Moment = u64; type OnTimestampSet = Aura; @@ -229,7 +229,7 @@ parameter_types! { pub const ExistentialDeposit: u128 = 500; } -impl balances::Trait for Runtime { +impl pallet_balances::Trait for Runtime { /// The type for recording an account's balance. type Balance = Balance; /// The ubiquitous event type. @@ -244,15 +244,15 @@ parameter_types! { pub const TransactionByteFee: Balance = 1; } -impl transaction_payment::Trait for Runtime { - type Currency = balances::Module; +impl pallet_transaction_payment::Trait for Runtime { + type Currency = Balances; type OnTransactionPayment = (); type TransactionByteFee = TransactionByteFee; type WeightToFee = IdentityFee; type FeeMultiplierUpdate = (); } -impl sudo::Trait for Runtime { +impl pallet_sudo::Trait for Runtime { type Event = Event; type Call = Call; } @@ -270,13 +270,13 @@ construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic { System: frame_system::{Module, Call, Config, Storage, Event}, - RandomnessCollectiveFlip: randomness_collective_flip::{Module, Call, Storage}, - Timestamp: timestamp::{Module, Call, Storage, Inherent}, - Aura: aura::{Module, Config, Inherent}, - Grandpa: grandpa::{Module, Call, Storage, Config, Event}, - Balances: balances::{Module, Call, Storage, Config, Event}, - TransactionPayment: transaction_payment::{Module, Storage}, - Sudo: sudo::{Module, Call, Config, Storage, Event}, + RandomnessCollectiveFlip: pallet_randomness_collective_flip::{Module, Call, Storage}, + Timestamp: pallet_timestamp::{Module, Call, Storage, Inherent}, + Aura: pallet_aura::{Module, Config, Inherent}, + Grandpa: pallet_grandpa::{Module, Call, Storage, Config, Event}, + Balances: pallet_balances::{Module, Call, Storage, Config, Event}, + TransactionPayment: pallet_transaction_payment::{Module, Storage}, + Sudo: pallet_sudo::{Module, Call, Config, Storage, Event}, // Include the custom logic from the template pallet in the runtime. TemplateModule: template::{Module, Call, Storage, Event}, } @@ -300,7 +300,7 @@ pub type SignedExtra = ( frame_system::CheckEra, frame_system::CheckNonce, frame_system::CheckWeight, - transaction_payment::ChargeTransactionPayment + pallet_transaction_payment::ChargeTransactionPayment ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; @@ -423,7 +423,7 @@ impl_runtime_apis! { None } } - + impl frame_system_rpc_runtime_api::AccountNonceApi for Runtime { fn account_nonce(account: AccountId) -> Index { System::account_nonce(account) From 5719b8cab597e04ee0d4c30f8aed9acc3921e7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= <123550+andresilva@users.noreply.github.com> Date: Thu, 6 Aug 2020 20:43:36 +0100 Subject: [PATCH 30/32] grandpa: fix enacting forced changes with no delay (#6828) * grandpa: fix enacting forced changes with no delay * grandpa: fix formatting --- client/finality-grandpa/src/authorities.rs | 42 ++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/client/finality-grandpa/src/authorities.rs b/client/finality-grandpa/src/authorities.rs index 117f5cad5e3d2..7a064d7a6224b 100644 --- a/client/finality-grandpa/src/authorities.rs +++ b/client/finality-grandpa/src/authorities.rs @@ -356,8 +356,9 @@ where .take_while(|c| c.effective_number() <= best_number) // to prevent iterating too far .filter(|c| c.effective_number() == best_number) { - // check if the given best block is in the same branch as the block that signaled the change. - if is_descendent_of(&change.canon_hash, &best_hash)? { + // check if the given best block is in the same branch as + // the block that signaled the change. + if change.canon_hash == best_hash || is_descendent_of(&change.canon_hash, &best_hash)? { // apply this change: make the set canonical afg_log!(initial_sync, "👴 Applying authority set change forced at block #{:?}", @@ -984,6 +985,43 @@ mod tests { ); } + #[test] + fn forced_changes_with_no_delay() { + // NOTE: this is a regression test + let mut authorities = AuthoritySet { + current_authorities: Vec::new(), + set_id: 0, + pending_standard_changes: ForkTree::new(), + pending_forced_changes: Vec::new(), + }; + + let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)]; + + // we create a forced change with no delay + let change_a = PendingChange { + next_authorities: set_a.clone(), + delay: 0, + canon_height: 5, + canon_hash: "hash_a", + delay_kind: DelayKind::Best { + median_last_finalized: 0, + }, + }; + + // and import it + authorities + .add_pending_change(change_a, &static_is_descendent_of(false)) + .unwrap(); + + // it should be enacted at the same block that signaled it + assert!( + authorities + .apply_forced_changes("hash_a", 5, &static_is_descendent_of(false), false) + .unwrap() + .is_some() + ); + } + #[test] fn next_change_works() { let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; From 2e9b63b1f1fc4d5a018e9801cdef4a54e9839857 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Fri, 7 Aug 2020 10:56:32 +0200 Subject: [PATCH 31/32] Renamed add_children to add_child since it adds only one child (#6838) --- client/service/src/task_manager/mod.rs | 2 +- client/service/src/task_manager/tests.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index 729b43bce94c1..6925d27f4e5ca 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -344,7 +344,7 @@ impl TaskManager { /// Register another TaskManager to terminate and gracefully shutdown when the parent /// terminates and gracefully shutdown. Also ends the parent `future()` if a child's essential /// task fails. (But don't end the parent if a child's normal task fails.) - pub fn add_children(&mut self, child: TaskManager) { + pub fn add_child(&mut self, child: TaskManager) { self.children.push(child); } } diff --git a/client/service/src/task_manager/tests.rs b/client/service/src/task_manager/tests.rs index a2bd84802aa17..27d9b0b9e9ad9 100644 --- a/client/service/src/task_manager/tests.rs +++ b/client/service/src/task_manager/tests.rs @@ -220,8 +220,8 @@ fn ensure_children_tasks_ends_when_task_manager_terminated() { let spawn_handle_child_1 = child_1.spawn_handle(); let child_2 = TaskManager::new(task_executor.clone(), None).unwrap(); let spawn_handle_child_2 = child_2.spawn_handle(); - task_manager.add_children(child_1); - task_manager.add_children(child_2); + task_manager.add_child(child_1); + task_manager.add_child(child_2); let spawn_handle = task_manager.spawn_handle(); let drop_tester = DropTester::new(); spawn_handle.spawn("task1", run_background_task(drop_tester.new_ref())); @@ -250,8 +250,8 @@ fn ensure_task_manager_future_ends_with_error_when_childs_essential_task_fails() let spawn_essential_handle_child_1 = child_1.spawn_essential_handle(); let child_2 = TaskManager::new(task_executor.clone(), None).unwrap(); let spawn_handle_child_2 = child_2.spawn_handle(); - task_manager.add_children(child_1); - task_manager.add_children(child_2); + task_manager.add_child(child_1); + task_manager.add_child(child_2); let spawn_handle = task_manager.spawn_handle(); let drop_tester = DropTester::new(); spawn_handle.spawn("task1", run_background_task(drop_tester.new_ref())); @@ -280,8 +280,8 @@ fn ensure_task_manager_future_continues_when_childs_not_essential_task_fails() { let spawn_handle_child_1 = child_1.spawn_handle(); let child_2 = TaskManager::new(task_executor.clone(), None).unwrap(); let spawn_handle_child_2 = child_2.spawn_handle(); - task_manager.add_children(child_1); - task_manager.add_children(child_2); + task_manager.add_child(child_1); + task_manager.add_child(child_2); let spawn_handle = task_manager.spawn_handle(); let drop_tester = DropTester::new(); spawn_handle.spawn("task1", run_background_task(drop_tester.new_ref())); From 74804b5649eccfb83c90aec87bdca58e5d5c8789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 7 Aug 2020 13:58:51 +0200 Subject: [PATCH 32/32] Revalidate transactions only on latest best block (#6824) * Revalidate transactions only on latest best block We should revalidate transactions only on the latest best block and not on any arbitrary block. The revalidation before failed when there were multiple blocks on the height given to the revalidation function, but no block was imported as best block. * Update test-utils/runtime/transaction-pool/src/lib.rs Co-authored-by: Jaco Greeff * Fix tests * Only process best blocks in the transaction pool Co-authored-by: Jaco Greeff --- bin/node/cli/src/service.rs | 4 +- client/api/src/client.rs | 20 +- .../basic-authorship/src/basic_authorship.rs | 4 +- client/consensus/manual-seal/src/lib.rs | 14 +- client/transaction-pool/src/lib.rs | 17 +- client/transaction-pool/src/revalidation.rs | 21 +- client/transaction-pool/src/testing/pool.rs | 227 +++++++----------- primitives/transaction-pool/src/pool.rs | 8 +- .../runtime/transaction-pool/src/lib.rs | 96 ++++++-- 9 files changed, 205 insertions(+), 206 deletions(-) diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index fdfa6816296aa..fd2c240a44e3b 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -523,11 +523,9 @@ mod tests { futures::executor::block_on( service.transaction_pool().maintain( - ChainEvent::NewBlock { - is_new_best: true, + ChainEvent::NewBestBlock { hash: parent_header.hash(), tree_route: None, - header: parent_header.clone(), }, ) ); diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 35d40965e6425..f97daa487638f 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -16,7 +16,7 @@ //! A set of APIs supported by the client along with their primitives. -use std::{fmt, collections::HashSet, sync::Arc}; +use std::{fmt, collections::HashSet, sync::Arc, convert::TryFrom}; use sp_core::storage::StorageKey; use sp_runtime::{ traits::{Block as BlockT, NumberFor}, @@ -252,13 +252,17 @@ pub struct FinalityNotification { pub header: Block::Header, } -impl From> for sp_transaction_pool::ChainEvent { - fn from(n: BlockImportNotification) -> Self { - Self::NewBlock { - is_new_best: n.is_new_best, - hash: n.hash, - header: n.header, - tree_route: n.tree_route, +impl TryFrom> for sp_transaction_pool::ChainEvent { + type Error = (); + + fn try_from(n: BlockImportNotification) -> Result { + if n.is_new_best { + Ok(Self::NewBestBlock { + hash: n.hash, + tree_route: n.tree_route, + }) + } else { + Err(()) } } } diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 3c56bdd33db04..41d12970464f4 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -346,11 +346,9 @@ mod tests { fn chain_event(header: B::Header) -> ChainEvent where NumberFor: From { - ChainEvent::NewBlock { + ChainEvent::NewBestBlock { hash: header.hash(), tree_route: None, - is_new_best: true, - header, } } diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 2799a498c1fb8..36aeffd9794f0 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -207,6 +207,7 @@ mod tests { use sp_consensus::ImportedAux; use sp_inherents::InherentDataProviders; use sc_basic_authorship::ProposerFactory; + use sc_client_api::BlockBackend; fn api() -> Arc { Arc::new(TestApi::empty()) @@ -415,15 +416,13 @@ mod tests { } } ); - // assert that there's a new block in the db. - assert!(client.header(&BlockId::Number(0)).unwrap().is_some()); + let block = client.block(&BlockId::Number(1)).unwrap().unwrap().block; + pool_api.add_block(block, true); assert!(pool.submit_one(&BlockId::Number(1), SOURCE, uxt(Alice, 1)).await.is_ok()); let header = client.header(&BlockId::Number(1)).expect("db error").expect("imported above"); - pool.maintain(sp_transaction_pool::ChainEvent::NewBlock { + pool.maintain(sp_transaction_pool::ChainEvent::NewBestBlock { hash: header.hash(), - header, - is_new_best: true, tree_route: None, }).await; @@ -438,10 +437,11 @@ mod tests { rx1.await.expect("should be no error receiving"), Ok(_) ); - assert!(client.header(&BlockId::Number(1)).unwrap().is_some()); + let block = client.block(&BlockId::Number(2)).unwrap().unwrap().block; + pool_api.add_block(block, true); pool_api.increment_nonce(Alice.into()); - assert!(pool.submit_one(&BlockId::Number(2), SOURCE, uxt(Alice, 2)).await.is_ok()); + assert!(pool.submit_one(&BlockId::Number(1), SOURCE, uxt(Alice, 2)).await.is_ok()); let (tx2, rx2) = futures::channel::oneshot::channel(); assert!(sink.send(EngineCommand::SealNewBlock { parent_hash: Some(created_block.hash), diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 6255fd478b756..0b6a1e935b9d0 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -34,7 +34,7 @@ pub mod testing; pub use sc_transaction_graph as txpool; pub use crate::api::{FullChainApi, LightChainApi}; -use std::{collections::{HashMap, HashSet}, sync::Arc, pin::Pin}; +use std::{collections::{HashMap, HashSet}, sync::Arc, pin::Pin, convert::TryInto}; use futures::{prelude::*, future::{self, ready}, channel::oneshot}; use parking_lot::Mutex; @@ -549,7 +549,7 @@ impl MaintainedTransactionPool for BasicPool { fn maintain(&self, event: ChainEvent) -> Pin + Send>> { match event { - ChainEvent::NewBlock { hash, tree_route, is_new_best, .. } => { + ChainEvent::NewBestBlock { hash, tree_route } => { let pool = self.pool.clone(); let api = self.api.clone(); @@ -608,10 +608,7 @@ impl MaintainedTransactionPool for BasicPool }) } - // If this is a new best block, we need to prune its transactions from the pool. - if is_new_best { - pruned_log.extend(prune_known_txs_for_block(id.clone(), &*api, &*pool).await); - } + pruned_log.extend(prune_known_txs_for_block(id.clone(), &*api, &*pool).await); metrics.report( |metrics| metrics.block_transactions_pruned.inc_by(pruned_log.len() as u64) @@ -690,9 +687,9 @@ impl MaintainedTransactionPool for BasicPool .map(|tx| tx.hash.clone()) .collect(); revalidation_queue.revalidate_later(block_number, hashes).await; - } - revalidation_strategy.lock().clear(); + revalidation_strategy.lock().clear(); + } }.boxed() } ChainEvent::Finalized { hash } => { @@ -721,7 +718,9 @@ pub async fn notification_future( Client: sc_client_api::BlockchainEvents, Pool: MaintainedTransactionPool, { - let import_stream = client.import_notification_stream().map(Into::into).fuse(); + let import_stream = client.import_notification_stream() + .filter_map(|n| ready(n.try_into().ok())) + .fuse(); let finality_stream = client.finality_notification_stream() .map(Into::into) .fuse(); diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index af9a76c055b6b..7be8688eaea5d 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -211,8 +211,7 @@ impl RevalidationWorker { mut self, from_queue: TracingUnboundedReceiver>, interval: R, - ) where R: Send, R::Guard: Send - { + ) where R: Send, R::Guard: Send { let interval = interval.into_stream().fuse(); let from_queue = from_queue.fuse(); futures::pin_mut!(interval, from_queue); @@ -253,7 +252,7 @@ impl RevalidationWorker { if this.members.len() > 0 { log::debug!( target: "txpool", - "Updated revalidation queue at {}. Transactions: {:?}", + "Updated revalidation queue at {:?}. Transactions: {:?}", this.best_block, this.members, ); @@ -298,9 +297,7 @@ where api: Arc, pool: Arc>, interval: R, - ) -> (Self, Pin + Send>>) - where R: Send + 'static, R::Guard: Send - { + ) -> (Self, Pin + Send>>) where R: Send + 'static, R::Guard: Send { let (to_worker, from_queue) = tracing_unbounded("mpsc_revalidation_queue"); let worker = RevalidationWorker::new(api.clone(), pool.clone()); @@ -338,16 +335,22 @@ where /// If queue configured with background worker, this will return immediately. /// If queue configured without background worker, this will resolve after /// revalidation is actually done. - pub async fn revalidate_later(&self, at: NumberFor, transactions: Vec>) { + pub async fn revalidate_later( + &self, + at: NumberFor, + transactions: Vec>, + ) { if transactions.len() > 0 { - log::debug!(target: "txpool", "Sent {} transactions to revalidation queue", transactions.len()); + log::debug!( + target: "txpool", "Sent {} transactions to revalidation queue", + transactions.len(), + ); } if let Some(ref to_worker) = self.background { if let Err(e) = to_worker.unbounded_send(WorkerPayload { at, transactions }) { log::warn!(target: "txpool", "Failed to update background worker: {:?}", e); } - return; } else { let pool = self.pool.clone(); let api = self.api.clone(); diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index a938313733ecd..8fa742cd419a3 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -106,7 +106,7 @@ fn prune_tags_should_work() { let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); - pool.validated_pool().api().push_block(1, Vec::new()); + pool.validated_pool().api().push_block(1, Vec::new(), true); block_on( pool.prune_tags( &BlockId::number(1), @@ -141,25 +141,14 @@ fn only_prune_on_new_best() { let uxt = uxt(Alice, 209); let _ = block_on( - pool.submit_and_watch(&BlockId::number(1), SOURCE, uxt.clone()) + pool.submit_and_watch(&BlockId::number(0), SOURCE, uxt.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(1, vec![uxt.clone()]); + pool.api.push_block(1, vec![uxt.clone()], true); assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { + let header = pool.api.push_block(2, vec![uxt], true); + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: false, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); - assert_eq!(pool.status().ready, 1); - - let header = pool.api.push_block(2, vec![uxt]); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; block_on(pool.maintain(event)); @@ -179,7 +168,7 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // remove the transaction that just got imported. api.increment_nonce(Alice.into()); - api.push_block(1, Vec::new()); + api.push_block(1, Vec::new(), true); block_on(pool.prune_tags(&BlockId::number(1), vec![vec![209]], vec![])).expect("1. Pruned"); assert_eq!(pool.validated_pool().status().ready, 0); // it's re-imported to future @@ -187,7 +176,7 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // so now let's insert another transaction that also provides the 155 api.increment_nonce(Alice.into()); - api.push_block(2, Vec::new()); + api.push_block(2, Vec::new(), true); let xt = uxt(Alice, 211); block_on(pool.submit_one(&BlockId::number(2), SOURCE, xt.clone())).expect("2. Imported"); assert_eq!(pool.validated_pool().status().ready, 1); @@ -197,18 +186,16 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // prune it and make sure the pool is empty api.increment_nonce(Alice.into()); - api.push_block(3, Vec::new()); + api.push_block(3, Vec::new(), true); block_on(pool.prune_tags(&BlockId::number(3), vec![vec![155]], vec![])).expect("2. Pruned"); assert_eq!(pool.validated_pool().status().ready, 0); assert_eq!(pool.validated_pool().status().future, 2); } fn block_event(header: Header) -> ChainEvent { - ChainEvent::NewBlock { + ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, tree_route: None, - header, } } @@ -219,11 +206,9 @@ fn block_event_with_retracted( ) -> ChainEvent { let tree_route = api.tree_route(retracted_start, header.parent_hash).expect("Tree route exists"); - ChainEvent::NewBlock { + ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, tree_route: Some(Arc::new(tree_route)), - header, } } @@ -236,7 +221,7 @@ fn should_prune_old_during_maintenance() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(1, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![xt.clone()], true); block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 0); @@ -253,7 +238,7 @@ fn should_revalidate_during_maintenance() { assert_eq!(pool.status().ready, 2); assert_eq!(pool.api.validation_requests().len(), 2); - let header = pool.api.push_block(1, vec![xt1.clone()]); + let header = pool.api.push_block(1, vec![xt1.clone()], true); block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 1); @@ -272,8 +257,8 @@ fn should_resubmit_from_retracted_during_maintenance() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(1, vec![]); - let fork_header = pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![], true); + let fork_header = pool.api.push_block(1, vec![], false); let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api); @@ -291,8 +276,8 @@ fn should_not_resubmit_from_retracted_during_maintenance_if_tx_is_also_in_enacte block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(1, vec![xt.clone()]); - let fork_header = pool.api.push_block(1, vec![xt]); + let header = pool.api.push_block(1, vec![xt.clone()], true); + let fork_header = pool.api.push_block(1, vec![xt], false); let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api); @@ -309,8 +294,8 @@ fn should_not_retain_invalid_hashes_from_retracted() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(1, vec![]); - let fork_header = pool.api.push_block(1, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![], true); + let fork_header = pool.api.push_block(1, vec![xt.clone()], false); pool.api.add_invalid(&xt); let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api); @@ -330,14 +315,14 @@ fn should_revalidate_transaction_multiple_times() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(1, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![xt.clone()], true); block_on(pool.maintain(block_event(header))); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(2, vec![]); + let header = pool.api.push_block(2, vec![], true); pool.api.add_invalid(&xt); block_on(pool.maintain(block_event(header))); @@ -354,18 +339,18 @@ fn should_revalidate_across_many_blocks() { let (pool, _guard, mut notifier) = maintained_pool(); - block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("1. Imported"); - block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt2.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt2.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 2); - let header = pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![], true); block_on(pool.maintain(block_event(header))); block_on(notifier.next()); - block_on(pool.submit_one(&BlockId::number(2), SOURCE, xt3.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt3.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 3); - let header = pool.api.push_block(2, vec![xt1.clone()]); + let header = pool.api.push_block(2, vec![xt1.clone()], true); block_on(pool.maintain(block_event(header))); block_on(notifier.next()); @@ -411,7 +396,7 @@ fn should_push_watchers_during_maintaince() { pool.api.add_invalid(&tx4); // clear timer events if any - let header = pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![], true); block_on(pool.maintain(block_event(header))); block_on(notifier.next()); @@ -429,7 +414,7 @@ fn should_push_watchers_during_maintaince() { ); // when - let header = pool.api.push_block(2, vec![tx0, tx1, tx2]); + let header = pool.api.push_block(2, vec![tx0, tx1, tx2], true); let header_hash = header.hash(); block_on(pool.maintain(block_event(header))); @@ -478,18 +463,16 @@ fn can_track_heap_size() { fn finalization() { let xt = uxt(Alice, 209); let api = TestApi::with_alice_nonce(209); - api.push_block(1, vec![]); + api.push_block(1, vec![], true); let (pool, _background, _) = BasicPool::new_test(api.into()); let watcher = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, xt.clone()) ).expect("1. Imported"); - pool.api.push_block(2, vec![xt.clone()]); + pool.api.push_block(2, vec![xt.clone()], true); - let header = pool.api.chain().read().block_by_number.get(&2).unwrap()[0].header().clone(); - let event = ChainEvent::NewBlock { + let header = pool.api.chain().read().block_by_number.get(&2).unwrap()[0].0.header().clone(); + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; block_on(pool.maintain(event)); @@ -508,7 +491,7 @@ fn finalization() { fn fork_aware_finalization() { let api = TestApi::empty(); // starting block A1 (last finalized.) - api.push_block(1, vec![]); + api.push_block(1, vec![], true); let (pool, _background, _) = BasicPool::new_test(api.into()); let mut canon_watchers = vec![]; @@ -534,14 +517,12 @@ fn fork_aware_finalization() { let watcher = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, from_alice.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(2, vec![from_alice.clone()]); + let header = pool.api.push_block(2, vec![from_alice.clone()], true); canon_watchers.push((watcher, header.hash())); assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; b1 = header.hash(); @@ -553,15 +534,13 @@ fn fork_aware_finalization() { // block C2 { - let header = pool.api.push_block_with_parent(b1, vec![from_dave.clone()]); + let header = pool.api.push_block_with_parent(b1, vec![from_dave.clone()], true); from_dave_watcher = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, from_dave.clone()) ).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; c2 = header.hash(); @@ -575,12 +554,10 @@ fn fork_aware_finalization() { pool.submit_and_watch(&BlockId::number(1), SOURCE, from_bob.clone()) ).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block_with_parent(c2, vec![from_bob.clone()]); + let header = pool.api.push_block_with_parent(c2, vec![from_bob.clone()], true); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; d2 = header.hash(); @@ -594,7 +571,7 @@ fn fork_aware_finalization() { pool.submit_and_watch(&BlockId::number(1), SOURCE, from_charlie.clone()) ).expect("1.Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_block(3, vec![from_charlie.clone()]); + let header = pool.api.push_block(3, vec![from_charlie.clone()], true); canon_watchers.push((watcher, header.hash())); let event = block_event_with_retracted(header.clone(), d2, &*pool.api); @@ -612,13 +589,11 @@ fn fork_aware_finalization() { pool.submit_and_watch(&BlockId::number(1), SOURCE, xt.clone()) ).expect("1. Imported"); assert_eq!(pool.status().ready, 3); - let header = pool.api.push_block(4, vec![xt.clone()]); + let header = pool.api.push_block(4, vec![xt.clone()], true); canon_watchers.push((w, header.hash())); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; d1 = header.hash(); @@ -632,12 +607,10 @@ fn fork_aware_finalization() { // block e1 { - let header = pool.api.push_block(5, vec![from_dave, from_bob]); + let header = pool.api.push_block(5, vec![from_dave, from_bob], true); e1 = header.hash(); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; block_on(pool.maintain(event)); @@ -684,7 +657,7 @@ fn fork_aware_finalization() { fn prune_and_retract_tx_at_same_time() { let api = TestApi::empty(); // starting block A1 (last finalized.) - api.push_block(1, vec![]); + api.push_block(1, vec![], true); let (pool, _background, _) = BasicPool::new_test(api.into()); @@ -697,13 +670,11 @@ fn prune_and_retract_tx_at_same_time() { // Block B1 let b1 = { - let header = pool.api.push_block(2, vec![from_alice.clone()]); + let header = pool.api.push_block(2, vec![from_alice.clone()], true); assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; block_on(pool.maintain(event)); @@ -713,7 +684,7 @@ fn prune_and_retract_tx_at_same_time() { // Block B2 let b2 = { - let header = pool.api.push_block(2, vec![from_alice.clone()]); + let header = pool.api.push_block(2, vec![from_alice.clone()], false); assert_eq!(pool.status().ready, 0); let event = block_event_with_retracted(header.clone(), b1, &*pool.api); @@ -757,7 +728,7 @@ fn prune_and_retract_tx_at_same_time() { fn resubmit_tx_of_fork_that_is_not_part_of_retracted() { let api = TestApi::empty(); // starting block A1 (last finalized.) - api.push_block(1, vec![]); + api.push_block(1, vec![], true); let (pool, _background, _) = BasicPool::new_test(api.into()); @@ -773,13 +744,11 @@ fn resubmit_tx_of_fork_that_is_not_part_of_retracted() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx0.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(2, vec![tx0.clone()]); + let header = pool.api.push_block(2, vec![tx0.clone()], true); assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { + let event = ChainEvent::NewBestBlock { hash: header.hash(), - is_new_best: true, - header: header.clone(), tree_route: None, }; d0 = header.hash(); @@ -792,23 +761,13 @@ fn resubmit_tx_of_fork_that_is_not_part_of_retracted() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx1.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(2, vec![tx1.clone()]); - assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: false, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); - - // Only transactions from new best should be pruned + pool.api.push_block(2, vec![tx1.clone()], false); assert_eq!(pool.status().ready, 1); } // Block D2 { - let header = pool.api.push_block(2, vec![]); + let header = pool.api.push_block(2, vec![], false); let event = block_event_with_retracted(header, d0, &*pool.api); block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 2); @@ -819,7 +778,7 @@ fn resubmit_tx_of_fork_that_is_not_part_of_retracted() { fn resubmit_from_retracted_fork() { let api = TestApi::empty(); // starting block A1 (last finalized.) - api.push_block(1, vec![]); + api.push_block(1, vec![], true); let (pool, _background, _) = BasicPool::new_test(api.into()); @@ -844,16 +803,10 @@ fn resubmit_from_retracted_fork() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx0.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(2, vec![tx0.clone()]); + let header = pool.api.push_block(2, vec![tx0.clone()], true); assert_eq!(pool.status().ready, 1); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: true, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); + block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 0); } @@ -862,14 +815,8 @@ fn resubmit_from_retracted_fork() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx1.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(3, vec![tx1.clone()]); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: true, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); + let header = pool.api.push_block(3, vec![tx1.clone()], true); + block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 0); } @@ -878,14 +825,8 @@ fn resubmit_from_retracted_fork() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx2.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(4, vec![tx2.clone()]); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: true, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); + let header = pool.api.push_block(4, vec![tx2.clone()], true); + block_on(pool.maintain(block_event(header.clone()))); assert_eq!(pool.status().ready, 0); header.hash() }; @@ -895,14 +836,7 @@ fn resubmit_from_retracted_fork() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx3.clone()) ).expect("1. Imported"); - let header = pool.api.push_block(2, vec![tx3.clone()]); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: false, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); + let header = pool.api.push_block(2, vec![tx3.clone()], true); assert_eq!(pool.status().ready, 1); header.hash() }; @@ -912,14 +846,7 @@ fn resubmit_from_retracted_fork() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx4.clone()) ).expect("1. Imported"); - let header = pool.api.push_block_with_parent(d1.clone(), vec![tx4.clone()]); - let event = ChainEvent::NewBlock { - hash: header.hash(), - is_new_best: false, - header: header.clone(), - tree_route: None, - }; - block_on(pool.maintain(event)); + let header = pool.api.push_block_with_parent(d1.clone(), vec![tx4.clone()], true); assert_eq!(pool.status().ready, 2); header.hash() }; @@ -929,7 +856,7 @@ fn resubmit_from_retracted_fork() { let _ = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, tx5.clone()) ).expect("1. Imported"); - let header = pool.api.push_block_with_parent(e1.clone(), vec![tx5.clone()]); + let header = pool.api.push_block_with_parent(e1.clone(), vec![tx5.clone()], true); // Don't announce the block event to the pool directly, because we will // re-org to this block. assert_eq!(pool.status().ready, 3); @@ -953,7 +880,7 @@ fn resubmit_from_retracted_fork() { fn ready_set_should_not_resolve_before_block_update() { let (pool, _guard, _notifier) = maintained_pool(); let xt1 = uxt(Alice, 209); - block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("1. Imported"); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); assert!(pool.ready_at(1).now_or_never().is_none()); } @@ -961,7 +888,7 @@ fn ready_set_should_not_resolve_before_block_update() { #[test] fn ready_set_should_resolve_after_block_update() { let (pool, _guard, _notifier) = maintained_pool(); - let header = pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![], true); let xt1 = uxt(Alice, 209); @@ -974,7 +901,7 @@ fn ready_set_should_resolve_after_block_update() { #[test] fn ready_set_should_eventually_resolve_when_block_update_arrives() { let (pool, _guard, _notifier) = maintained_pool(); - let header = pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![], true); let xt1 = uxt(Alice, 209); @@ -1063,7 +990,7 @@ fn import_notification_to_pool_maintain_works() { // Get the notification of the block import and maintain the pool with it, // Now, the pool should not contain any transactions. let evt = import_stream.next().expect("Importing a block leads to an event"); - block_on(pool.maintain(evt.into())); + block_on(pool.maintain(evt.try_into().expect("Imported as new best block"))); assert_eq!(pool.status().ready, 0); } @@ -1075,7 +1002,7 @@ fn pruning_a_transaction_should_remove_it_from_best_transaction() { let xt1 = Extrinsic::IncludeData(Vec::new()); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); - let header = pool.api.push_block(1, vec![xt1.clone()]); + let header = pool.api.push_block(1, vec![xt1.clone()], true); // This will prune `xt1`. block_on(pool.maintain(block_event(header))); @@ -1091,3 +1018,23 @@ fn pruning_a_transaction_should_remove_it_from_best_transaction() { // returned a second time by the iterator. assert!(iterator.next().is_none()); } + +#[test] +fn only_revalidate_on_best_block() { + let xt = uxt(Alice, 209); + + let (pool, _guard, mut notifier) = maintained_pool(); + + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + + let header = pool.api.push_block(1, vec![], true); + + pool.api.push_block(2, vec![], false); + pool.api.push_block(2, vec![], false); + + block_on(pool.maintain(block_event(header))); + block_on(notifier.next()); + + assert_eq!(pool.status().ready, 1); +} diff --git a/primitives/transaction-pool/src/pool.rs b/primitives/transaction-pool/src/pool.rs index 7d1d5537dc9bb..6235ca7cdfcf3 100644 --- a/primitives/transaction-pool/src/pool.rs +++ b/primitives/transaction-pool/src/pool.rs @@ -248,14 +248,10 @@ pub trait TransactionPool: Send + Sync { /// Events that the transaction pool listens for. pub enum ChainEvent { - /// New blocks have been added to the chain - NewBlock { - /// Is this the new best block. - is_new_best: bool, + /// New best block have been added to the chain + NewBestBlock { /// Hash of the block. hash: B::Hash, - /// Header of the just imported block - header: B::Header, /// Tree route from old best to new best parent that was calculated on import. /// /// If `None`, no re-org happened on import. diff --git a/test-utils/runtime/transaction-pool/src/lib.rs b/test-utils/runtime/transaction-pool/src/lib.rs index 17cecd394ab91..f772ba9b02d5c 100644 --- a/test-utils/runtime/transaction-pool/src/lib.rs +++ b/test-utils/runtime/transaction-pool/src/lib.rs @@ -35,6 +35,7 @@ use substrate_test_runtime_client::{ AccountKeyring::{self, *}, }; use sp_blockchain::CachedHeaderMetadata; +use futures::future::ready; /// Error type used by [`TestApi`]. #[derive(Debug, derive_more::From, derive_more::Display)] @@ -52,9 +53,30 @@ impl std::error::Error for Error { } } +pub enum IsBestBlock { + Yes, + No, +} + +impl IsBestBlock { + pub fn is_best(&self) -> bool { + matches!(self, Self::Yes) + } +} + +impl From for IsBestBlock { + fn from(is_best: bool) -> Self { + if is_best { + Self::Yes + } else { + Self::No + } + } +} + #[derive(Default)] pub struct ChainState { - pub block_by_number: BTreeMap>, + pub block_by_number: BTreeMap>, pub block_by_hash: HashMap, pub nonces: HashMap, pub invalid_hashes: HashSet, @@ -86,7 +108,7 @@ impl TestApi { }; // Push genesis block - api.push_block(0, Vec::new()); + api.push_block(0, Vec::new(), true); api } @@ -97,10 +119,12 @@ impl TestApi { } /// Push block under given number. - /// - /// If multiple blocks exists with the same block number, the first inserted block will be - /// interpreted as part of the canonical chain. - pub fn push_block(&self, block_number: BlockNumber, xts: Vec) -> Header { + pub fn push_block( + &self, + block_number: BlockNumber, + xts: Vec, + is_best_block: bool, + ) -> Header { let parent_hash = { let chain = self.chain.read(); block_number @@ -109,12 +133,12 @@ impl TestApi { chain.block_by_number .get(&num) .map(|blocks| { - blocks[0].header.hash() + blocks[0].0.header.hash() }) }).unwrap_or_default() }; - self.push_block_with_parent(parent_hash, xts) + self.push_block_with_parent(parent_hash, xts, is_best_block) } /// Push a block using the given `parent`. @@ -124,14 +148,14 @@ impl TestApi { &self, parent: Hash, xts: Vec, + is_best_block: bool, ) -> Header { - let mut chain = self.chain.write(); - // `Hash::default()` is the genesis parent hash let block_number = if parent == Hash::default() { 0 } else { - *chain.block_by_hash + *self.chain.read() + .block_by_hash .get(&parent) .expect("`parent` exists") .header() @@ -146,14 +170,21 @@ impl TestApi { state_root: Default::default(), }; - let hash = header.hash(); - let block = Block::new(header.clone(), xts); - chain.block_by_hash.insert(hash, block.clone()); - chain.block_by_number.entry(block_number).or_default().push(block); + self.add_block(Block::new(header.clone(), xts), is_best_block); header } + /// Add a block to the internal state. + pub fn add_block(&self, block: Block, is_best_block: bool) { + let hash = block.header.hash(); + let block_number = block.header.number().clone(); + + let mut chain = self.chain.write(); + chain.block_by_hash.insert(hash, block.clone()); + chain.block_by_number.entry(block_number).or_default().push((block, is_best_block.into())); + } + fn hash_and_length_inner(ex: &Extrinsic) -> (Hash, usize) { let encoded = ex.encode(); (BlakeTwo256::hash(&encoded), encoded.len()) @@ -203,12 +234,36 @@ impl sc_transaction_graph::ChainApi for TestApi { fn validate_transaction( &self, - _at: &BlockId, + at: &BlockId, _source: TransactionSource, uxt: sc_transaction_graph::ExtrinsicFor, ) -> Self::ValidationFuture { self.validation_requests.write().push(uxt.clone()); + match self.block_id_to_number(at) { + Ok(Some(number)) => { + let found_best = self.chain + .read() + .block_by_number + .get(&number) + .map(|blocks| blocks.iter().any(|b| b.1.is_best())) + .unwrap_or(false); + + // If there is no best block, we don't know based on which block we should validate + // the transaction. (This is not required for this test function, but in real + // environment it would fail because of this). + if !found_best { + return ready(Ok( + Err(TransactionValidityError::Invalid(InvalidTransaction::Custom(1)).into()) + )) + } + }, + Ok(None) => return ready(Ok( + Err(TransactionValidityError::Invalid(InvalidTransaction::Custom(2)).into()) + )), + Err(e) => return ready(Err(e)), + } + let (requires, provides) = if let Some(transfer) = uxt.try_transfer() { let chain_nonce = self.chain.read().nonces.get(&transfer.from).cloned().unwrap_or(0); let requires = if chain_nonce == transfer.nonce { @@ -224,7 +279,7 @@ impl sc_transaction_graph::ChainApi for TestApi { }; if self.chain.read().invalid_hashes.contains(&self.hash_and_length(&uxt).0) { - return futures::future::ready(Ok( + return ready(Ok( Err(TransactionValidityError::Invalid(InvalidTransaction::Custom(0)).into()) )) } @@ -239,7 +294,7 @@ impl sc_transaction_graph::ChainApi for TestApi { (self.valid_modifier.read())(&mut validity); - futures::future::ready(Ok(Ok(validity))) + ready(Ok(Ok(validity))) } fn block_id_to_number( @@ -266,7 +321,7 @@ impl sc_transaction_graph::ChainApi for TestApi { .read() .block_by_number .get(num) - .map(|blocks| blocks[0].header().hash()), + .and_then(|blocks| blocks.iter().find(|b| b.1.is_best()).map(|b| b.0.header().hash())), }) } @@ -283,7 +338,7 @@ impl sc_transaction_graph::ChainApi for TestApi { .read() .block_by_number .get(num) - .map(|b| b[0].extrinsics().to_vec()), + .map(|b| b[0].0.extrinsics().to_vec()), BlockId::Hash(hash) => self.chain .read() .block_by_hash @@ -332,4 +387,3 @@ pub fn uxt(who: AccountKeyring, nonce: Index) -> Extrinsic { let signature = transfer.using_encoded(|e| who.sign(e)).into(); Extrinsic::Transfer { transfer, signature, exhaust_resources_when_not_first: false } } -