From f0735bce2d8d9b36ef5683bae3a37a39da222a2c Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 14 Jun 2021 09:53:07 +0200 Subject: [PATCH 1/4] childstate_getKeysPaged rpc --- client/api/src/backend.rs | 37 ++++++++++++++++++++++--- client/rpc-api/src/child_state/mod.rs | 14 ++++++++++ client/rpc/src/state/mod.rs | 21 +++++++++++++++ client/rpc/src/state/state_full.rs | 23 ++++++++++++++++ client/rpc/src/state/state_light.rs | 11 ++++++++ client/service/src/client/client.rs | 15 ++++++++++- client/service/test/src/client/mod.rs | 39 ++++++++++++++++++++++++--- 7 files changed, 151 insertions(+), 9 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 09e9e0cb2e173..be4fcdca865f3 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -275,6 +275,7 @@ pub trait AuxStore { /// An `Iterator` that iterates keys in a given block under a prefix. pub struct KeyIterator<'a, State, Block> { state: State, + child_storage: Option, prefix: Option<&'a StorageKey>, current_key: Vec, _phantom: PhantomData, @@ -285,6 +286,23 @@ impl <'a, State, Block> KeyIterator<'a, State, Block> { pub fn new(state: State, prefix: Option<&'a StorageKey>, current_key: Vec) -> Self { Self { state, + child_storage: None, + prefix, + current_key, + _phantom: PhantomData, + } + } + + /// create a KeyIterator instance for a child storage + pub fn new_child( + state: State, + child_info: ChildInfo, + prefix: Option<&'a StorageKey>, + current_key: Vec, + ) -> Self { + Self { + state, + child_storage: Some(child_info), prefix, current_key, _phantom: PhantomData, @@ -299,10 +317,11 @@ impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where type Item = StorageKey; fn next(&mut self) -> Option { - let next_key = self.state - .next_storage_key(&self.current_key) - .ok() - .flatten()?; + let next_key = if let Some(child_info) = self.child_storage.as_ref() { + self.state.next_child_storage_key(child_info, &self.current_key) + } else { + self.state.next_storage_key(&self.current_key) + }.ok().flatten()?; // this terminates the iterator the first time it fails. if let Some(prefix) = self.prefix { if !next_key.starts_with(&prefix.0[..]) { @@ -356,6 +375,16 @@ pub trait StorageProvider> { key_prefix: &StorageKey ) -> sp_blockchain::Result>; + /// Given a `BlockId` and a key prefix and a chidl storage key, + /// return a `KeyIterator` iterates matching storage keys in that block. + fn child_storage_keys_iter<'a>( + &self, + id: &BlockId, + child_info: ChildInfo, + prefix: Option<&'a StorageKey>, + start_key: Option<&StorageKey> + ) -> sp_blockchain::Result>; + /// Given a `BlockId`, a key and a child storage key, return the hash under the key in that block. fn child_storage_hash( &self, diff --git a/client/rpc-api/src/child_state/mod.rs b/client/rpc-api/src/child_state/mod.rs index 7ab897d6174a4..99990017fd826 100644 --- a/client/rpc-api/src/child_state/mod.rs +++ b/client/rpc-api/src/child_state/mod.rs @@ -34,6 +34,7 @@ pub trait ChildStateApi { /// RPC Metadata type Metadata; + /// DEPRECATED: Please use `childstate_getKeysPaged` with proper paging support. /// Returns the keys with prefix from a child storage, leave empty to get all the keys #[rpc(name = "childstate_getKeys")] fn storage_keys( @@ -43,6 +44,19 @@ pub trait ChildStateApi { hash: Option ) -> FutureResult>; + /// Returns the keys with prefix from a child storage with pagination support. + /// Up to `count` keys will be returned. + /// If `start_key` is passed, return next keys in storage in lexicographic order. + #[rpc(name = "childstate_getKeysPaged", alias("childstate_getKeysPagedAt"))] + fn storage_keys_paged( + &self, + child_storage_key: PrefixedStorageKey, + prefix: Option, + count: u32, + start_key: Option, + hash: Option, + ) -> FutureResult>; + /// Returns a child storage entry at a specific block's state. #[rpc(name = "childstate_getStorage")] fn storage( diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 803fc6797ee9a..58940da58cbc6 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -399,6 +399,16 @@ pub trait ChildStateBackend: Send + Sync + 'static prefix: StorageKey, ) -> FutureResult>; + /// Returns the keys with prefix from a child storage with pagination support. + fn storage_keys_paged( + &self, + block: Option, + storage_key: PrefixedStorageKey, + prefix: Option, + count: u32, + start_key: Option, + ) -> FutureResult>; + /// Returns a child storage entry at a specific block's state. fn storage( &self, @@ -466,6 +476,17 @@ impl ChildStateApi for ChildState self.backend.storage_keys(block, storage_key, key_prefix) } + fn storage_keys_paged( + &self, + storage_key: PrefixedStorageKey, + prefix: Option, + count: u32, + start_key: Option, + block: Option, + ) -> FutureResult> { + self.backend.storage_keys_paged(block, storage_key, prefix, count, start_key) + } + fn storage_hash( &self, storage_key: PrefixedStorageKey, diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index bea7ddfbb3b76..5007f24254d87 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -607,6 +607,29 @@ impl ChildStateBackend for FullState, + storage_key: PrefixedStorageKey, + prefix: Option, + count: u32, + start_key: Option, + ) -> FutureResult> { + Box::new(result( + self.block_or_best(block) + .and_then(|block| { + let child_info = match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => ChildInfo::new_default(storage_key), + None => return Err(sp_blockchain::Error::InvalidChildStorageKey), + }; + self.client.child_storage_keys_iter( + &BlockId::Hash(block), child_info, prefix.as_ref(), start_key.as_ref(), + ) + }) + .map(|v| v.take(count as usize).collect()) + .map_err(client_err))) + } + fn storage( &self, block: Option, diff --git a/client/rpc/src/state/state_light.rs b/client/rpc/src/state/state_light.rs index 09fefd2e02c42..a2f69df9d0271 100644 --- a/client/rpc/src/state/state_light.rs +++ b/client/rpc/src/state/state_light.rs @@ -509,6 +509,17 @@ impl ChildStateBackend for LightState, + _storage_key: PrefixedStorageKey, + _prefix: Option, + _count: u32, + _start_key: Option, + ) -> FutureResult> { + Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) + } + fn storage( &self, block: Option, diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 06d9aec4e4fd3..d1da934da52e6 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1392,7 +1392,6 @@ impl StorageProvider for Client wher Ok(keys) } - fn storage_keys_iter<'a>( &self, id: &BlockId, @@ -1407,6 +1406,20 @@ impl StorageProvider for Client wher Ok(KeyIterator::new(state, prefix, start_key)) } + fn child_storage_keys_iter<'a>( + &self, + id: &BlockId, + child_info: ChildInfo, + prefix: Option<&'a StorageKey>, + start_key: Option<&StorageKey> + ) -> sp_blockchain::Result> { + let state = self.state_at(id)?; + let start_key = start_key + .or(prefix) + .map(|key| key.0.clone()) + .unwrap_or_else(Vec::new); + Ok(KeyIterator::new_child(state, child_info, prefix, start_key)) + } fn storage( &self, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 3852ab2d61b5f..72de473028680 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -50,7 +50,7 @@ use sp_consensus::{ BlockOrigin, SelectChain, BlockImport, Error as ConsensusError, BlockCheckParams, ImportResult, BlockStatus, BlockImportParams, ForkChoiceStrategy, }; -use sp_storage::StorageKey; +use sp_storage::{StorageKey, ChildInfo}; use sp_trie::{TrieConfiguration, trie_types::Layout}; use sp_runtime::{generic::BlockId, DigestItem, Justifications}; use hex_literal::hex; @@ -1741,15 +1741,26 @@ fn imports_blocks_with_changes_tries_config_change() { #[test] fn storage_keys_iter_prefix_and_start_key_works() { - let client = substrate_test_runtime_client::new(); - + let child_info = ChildInfo::new_default(b"child"); + let client = TestClientBuilder::new() + .add_extra_child_storage(&child_info, b"first".to_vec(), vec![0u8; 32]) + .add_extra_child_storage(&child_info, b"second".to_vec(), vec![0u8; 32]) + .add_extra_child_storage(&child_info, b"third".to_vec(), vec![0u8; 32]) + .build(); + + let child_root = b":child_storage:default:child".to_vec(); let prefix = StorageKey(hex!("3a").to_vec()); + let child_prefix = StorageKey(b"sec".to_vec()); let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), None) .unwrap() .map(|x| x.0) .collect(); - assert_eq!(res, [hex!("3a636f6465").to_vec(), hex!("3a686561707061676573").to_vec()]); + assert_eq!(res, [ + child_root.clone(), + hex!("3a636f6465").to_vec(), + hex!("3a686561707061676573").to_vec(), + ]); let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a636f6465").to_vec()))) .unwrap() @@ -1762,6 +1773,26 @@ fn storage_keys_iter_prefix_and_start_key_works() { .map(|x| x.0) .collect(); assert_eq!(res, Vec::>::new()); + + let res: Vec<_> = client.child_storage_keys_iter( + &BlockId::Number(0), + child_info.clone(), + Some(&child_prefix), + None, + ).unwrap() + .map(|x| x.0) + .collect(); + assert_eq!(res, [b"second".to_vec()]); + + let res: Vec<_> = client.child_storage_keys_iter( + &BlockId::Number(0), + child_info, + None, + Some(&StorageKey(b"second".to_vec())), + ).unwrap() + .map(|x| x.0) + .collect(); + assert_eq!(res, [b"third".to_vec()]); } #[test] From f197fd51452c8ac6fe5a1cca0103ff13eb97b4d8 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 15 Jun 2021 09:08:32 +0200 Subject: [PATCH 2/4] Rename `v` to `iter`. --- client/rpc/src/state/state_full.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 5007f24254d87..3fd83d89751fe 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -291,7 +291,7 @@ impl StateBackend for FullState ChildStateBackend for FullState Date: Wed, 16 Jun 2021 11:45:55 +0200 Subject: [PATCH 3/4] Update client/api/src/backend.rs Co-authored-by: Alexander Popiak --- client/api/src/backend.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index be4fcdca865f3..63b7ed5c6ba82 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -375,8 +375,8 @@ pub trait StorageProvider> { key_prefix: &StorageKey ) -> sp_blockchain::Result>; - /// Given a `BlockId` and a key prefix and a chidl storage key, - /// return a `KeyIterator` iterates matching storage keys in that block. + /// Given a `BlockId` and a key `prefix` and a child storage key, + /// return a `KeyIterator` that iterates matching storage keys in that block. fn child_storage_keys_iter<'a>( &self, id: &BlockId, From 05c8bd9dd4268d654af32b9ee7de0890fa8c9a2a Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 16 Jun 2021 11:46:03 +0200 Subject: [PATCH 4/4] Update client/api/src/backend.rs Co-authored-by: Alexander Popiak --- client/api/src/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 63b7ed5c6ba82..368d7b87edde4 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -293,7 +293,7 @@ impl <'a, State, Block> KeyIterator<'a, State, Block> { } } - /// create a KeyIterator instance for a child storage + /// Create a `KeyIterator` instance for a child storage. pub fn new_child( state: State, child_info: ChildInfo,