diff --git a/test-support/keccak-hasher/src/lib.rs b/test-support/keccak-hasher/src/lib.rs index 5c06ba6a..b4692f77 100644 --- a/test-support/keccak-hasher/src/lib.rs +++ b/test-support/keccak-hasher/src/lib.rs @@ -18,11 +18,14 @@ use hash256_std_hasher::Hash256StdHasher; use hash_db::Hasher; use tiny_keccak::{Hasher as _, Keccak}; +/// The `Keccak` hash output type. +pub type KeccakHash = [u8; 32]; + /// Concrete `Hasher` impl for the Keccak-256 hash #[derive(Default, Debug, Clone, PartialEq)] pub struct KeccakHasher; impl Hasher for KeccakHasher { - type Out = [u8; 32]; + type Out = KeccakHash; type StdHasher = Hash256StdHasher; diff --git a/test-support/reference-trie/Cargo.toml b/test-support/reference-trie/Cargo.toml index 7181166d..617485bf 100644 --- a/test-support/reference-trie/Cargo.toml +++ b/test-support/reference-trie/Cargo.toml @@ -14,6 +14,7 @@ keccak-hasher = { path = "../keccak-hasher", version = "0.15.3" } trie-db = { path = "../../trie-db", default-features = false, version = "0.23.0" } trie-root = { path = "../../trie-root", default-features = false, version = "0.17.0" } parity-scale-codec = { version = "3.0.0", features = ["derive"] } +hashbrown = { version = "0.12.0", default-features = false, features = ["ahash"] } [dev-dependencies] trie-bench = { path = "../trie-bench" } diff --git a/test-support/reference-trie/src/lib.rs b/test-support/reference-trie/src/lib.rs index 52a9246b..2356e104 100644 --- a/test-support/reference-trie/src/lib.rs +++ b/test-support/reference-trie/src/lib.rs @@ -14,23 +14,21 @@ //! Reference implementation of a streamer. -mod substrate_like; - +use hashbrown::{hash_map::Entry, HashMap}; use parity_scale_codec::{Compact, Decode, Encode, Error as CodecError, Input, Output}; use std::{borrow::Borrow, fmt, iter::once, marker::PhantomData, ops::Range}; use trie_db::{ - node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, + nibble_ops, + node::{NibbleSlicePlan, NodeHandlePlan, NodeOwned, NodePlan, Value, ValuePlan}, trie_visit, triedbmut::ChildReference, - DBValue, Partial, TrieBuilder, TrieRoot, -}; -use trie_root::Hasher; - -use trie_db::{ - nibble_ops, NodeCodec, Trie, TrieConfiguration, TrieDB, TrieDBMut, TrieLayout, TrieMut, + DBValue, NodeCodec, Trie, TrieBuilder, TrieConfiguration, TrieDBBuilder, TrieDBMutBuilder, + TrieHash, TrieLayout, TrieMut, TrieRoot, }; pub use trie_root::TrieStream; -use trie_root::Value as TrieStreamValue; +use trie_root::{Hasher, Value as TrieStreamValue}; + +mod substrate_like; pub mod node { pub use trie_db::node::Node; } @@ -49,10 +47,14 @@ macro_rules! test_layouts { ($test:ident, $test_internal:ident) => { #[test] fn $test() { - $test_internal::(); - $test_internal::(); - $test_internal::(); - $test_internal::(); + eprintln!("Running with layout `HashedValueNoExtThreshold`"); + $test_internal::<$crate::HashedValueNoExtThreshold>(); + eprintln!("Running with layout `HashedValueNoExt`"); + $test_internal::<$crate::HashedValueNoExt>(); + eprintln!("Running with layout `NoExtensionLayout`"); + $test_internal::<$crate::NoExtensionLayout>(); + eprintln!("Running with layout `ExtensionLayout`"); + $test_internal::<$crate::ExtensionLayout>(); } }; } @@ -63,8 +65,8 @@ macro_rules! test_layouts_no_meta { ($test:ident, $test_internal:ident) => { #[test] fn $test() { - $test_internal::(); - $test_internal::(); + $test_internal::<$crate::NoExtensionLayout>(); + $test_internal::<$crate::ExtensionLayout>(); } }; } @@ -152,16 +154,22 @@ impl Bitmap { } } -pub type RefTrieDB<'a> = trie_db::TrieDB<'a, ExtensionLayout>; +pub type RefTrieDB<'a, 'cache> = trie_db::TrieDB<'a, 'cache, ExtensionLayout>; +pub type RefTrieDBBuilder<'a, 'cache> = trie_db::TrieDBBuilder<'a, 'cache, ExtensionLayout>; pub type RefTrieDBMut<'a> = trie_db::TrieDBMut<'a, ExtensionLayout>; +pub type RefTrieDBMutBuilder<'a> = trie_db::TrieDBMutBuilder<'a, ExtensionLayout>; pub type RefTrieDBMutNoExt<'a> = trie_db::TrieDBMut<'a, NoExtensionLayout>; +pub type RefTrieDBMutNoExtBuilder<'a> = trie_db::TrieDBMutBuilder<'a, NoExtensionLayout>; pub type RefTrieDBMutAllowEmpty<'a> = trie_db::TrieDBMut<'a, AllowEmptyLayout>; -pub type RefFatDB<'a> = trie_db::FatDB<'a, ExtensionLayout>; +pub type RefTrieDBMutAllowEmptyBuilder<'a> = trie_db::TrieDBMutBuilder<'a, AllowEmptyLayout>; +pub type RefTestTrieDBCache = TestTrieCache; +pub type RefTestTrieDBCacheNoExt = TestTrieCache; +pub type RefFatDB<'a, 'cache> = trie_db::FatDB<'a, 'cache, ExtensionLayout>; pub type RefFatDBMut<'a> = trie_db::FatDBMut<'a, ExtensionLayout>; -pub type RefSecTrieDB<'a> = trie_db::SecTrieDB<'a, ExtensionLayout>; +pub type RefSecTrieDB<'a, 'cache> = trie_db::SecTrieDB<'a, 'cache, ExtensionLayout>; pub type RefSecTrieDBMut<'a> = trie_db::SecTrieDBMut<'a, ExtensionLayout>; -pub type RefLookup<'a, Q> = trie_db::Lookup<'a, ExtensionLayout, Q>; -pub type RefLookupNoExt<'a, Q> = trie_db::Lookup<'a, NoExtensionLayout, Q>; +pub type RefLookup<'a, 'cache, Q> = trie_db::Lookup<'a, 'cache, ExtensionLayout, Q>; +pub type RefLookupNoExt<'a, 'cache, Q> = trie_db::Lookup<'a, 'cache, NoExtensionLayout, Q>; pub fn reference_trie_root(input: I) -> ::Out where @@ -170,11 +178,11 @@ where B: AsRef<[u8]> + fmt::Debug, { if T::USE_EXTENSION { - trie_root::trie_root::(input, Default::default()) + trie_root::trie_root::(input, T::MAX_INLINE_VALUE) } else { trie_root::trie_root_no_extension::( input, - Default::default(), + T::MAX_INLINE_VALUE, ) } } @@ -488,18 +496,6 @@ pub struct ReferenceNodeCodec(PhantomData); #[derive(Default, Clone)] pub struct ReferenceNodeCodecNoExt(PhantomData); -fn partial_to_key(partial: Partial, offset: u8, over: u8) -> Vec { - let number_nibble_encoded = (partial.0).0 as usize; - let nibble_count = partial.1.len() * nibble_ops::NIBBLE_PER_BYTE + number_nibble_encoded; - assert!(nibble_count < over as usize); - let mut output = vec![offset + nibble_count as u8]; - if number_nibble_encoded > 0 { - output.push(nibble_ops::pad_right((partial.0).1)); - } - output.extend_from_slice(&partial.1[..]); - output -} - fn partial_from_iterator_to_key>( partial: I, nibble_count: usize, @@ -532,27 +528,6 @@ fn partial_from_iterator_encode>( output } -fn partial_encode(partial: Partial, node_kind: NodeKindNoExt) -> Vec { - let number_nibble_encoded = (partial.0).0 as usize; - let nibble_count = partial.1.len() * nibble_ops::NIBBLE_PER_BYTE + number_nibble_encoded; - - let nibble_count = ::std::cmp::min(NIBBLE_SIZE_BOUND_NO_EXT, nibble_count); - - let mut output = Vec::with_capacity(3 + partial.1.len()); - match node_kind { - NodeKindNoExt::Leaf => NodeHeaderNoExt::Leaf(nibble_count).encode_to(&mut output), - NodeKindNoExt::BranchWithValue => - NodeHeaderNoExt::Branch(true, nibble_count).encode_to(&mut output), - NodeKindNoExt::BranchNoValue => - NodeHeaderNoExt::Branch(false, nibble_count).encode_to(&mut output), - }; - if number_nibble_encoded > 0 { - output.push(nibble_ops::pad_right((partial.0).1)); - } - output.extend_from_slice(&partial.1[..]); - output -} - struct ByteSliceInput<'a> { data: &'a [u8], offset: usize, @@ -684,8 +659,9 @@ impl NodeCodec for ReferenceNodeCodec { &[EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: Value) -> Vec { - let mut output = partial_to_key(partial, LEAF_NODE_OFFSET, LEAF_NODE_OVER); + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + let mut output = + partial_from_iterator_to_key(partial, number_nibble, LEAF_NODE_OFFSET, LEAF_NODE_OVER); match value { Value::Inline(value) => { Compact(value.len() as u32).encode_to(&mut output); @@ -839,8 +815,8 @@ impl NodeCodec for ReferenceNodeCodecNoExt { &[EMPTY_TRIE_NO_EXT] } - fn leaf_node(partial: Partial, value: Value) -> Vec { - let mut output = partial_encode(partial, NodeKindNoExt::Leaf); + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { + let mut output = partial_from_iterator_encode(partial, number_nibble, NodeKindNoExt::Leaf); match value { Value::Inline(value) => { Compact(value.len() as u32).encode_to(&mut output); @@ -918,7 +894,7 @@ where let root_new = calc_root_build::(data.clone(), &mut hashdb); let root = { let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } @@ -928,7 +904,7 @@ where if root_new != root { { let db: &dyn hash_db::HashDB<_, _> = &hashdb; - let t = TrieDB::::new(&db, &root_new); + let t = TrieDBBuilder::::new(&db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); @@ -936,7 +912,7 @@ where } { let db: &dyn hash_db::HashDB<_, _> = &memdb; - let t = TrieDB::::new(&db, &root); + let t = TrieDBBuilder::::new(&db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:x?}", a); @@ -957,7 +933,7 @@ pub fn compare_root>( let root_new = reference_trie_root_iter_build::(data.clone()); let root = { let mut root = Default::default(); - let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); } @@ -1032,7 +1008,7 @@ pub fn compare_implementations_unordered( let mut b_map = std::collections::btree_map::BTreeMap::new(); let root = { let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for i in 0..data.len() { t.insert(&data[i].0[..], &data[i].1[..]).unwrap(); b_map.insert(data[i].0.clone(), data[i].1.clone()); @@ -1048,7 +1024,7 @@ pub fn compare_implementations_unordered( if root != root_new { { let db: &dyn hash_db::HashDB<_, _> = &memdb; - let t = TrieDB::::new(&db, &root); + let t = TrieDBBuilder::::new(&db, &root).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); @@ -1056,7 +1032,7 @@ pub fn compare_implementations_unordered( } { let db: &dyn hash_db::HashDB<_, _> = &hashdb; - let t = TrieDB::::new(&db, &root_new); + let t = TrieDBBuilder::::new(&db, &root_new).build(); println!("{:?}", t); for a in t.iter().unwrap() { println!("a:{:?}", a); @@ -1080,13 +1056,13 @@ pub fn compare_insert_remove>( let mut root = Default::default(); let mut a = 0; { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.commit(); } while a < data.len() { // new triemut every 3 element root = { - let mut t = TrieDBMut::::from_existing(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); for _ in 0..3 { if data[a].0 { // remove @@ -1107,16 +1083,75 @@ pub fn compare_insert_remove>( *t.root() }; } - let mut t = TrieDBMut::::from_existing(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); // we are testing the RefTrie code here so we do not sort or check uniqueness // before. assert_eq!(*t.root(), calc_root::(data2)); } +/// Example trie cache implementation. +/// +/// Should not be used for anything in production. +pub struct TestTrieCache { + /// In a real implementation we need to make sure that this is unique per trie root. + value_cache: HashMap, trie_db::CachedValue>>, + node_cache: HashMap, NodeOwned>>, +} + +impl TestTrieCache { + /// Clear the value cache. + pub fn clear_value_cache(&mut self) { + self.value_cache.clear(); + } + + /// Clear the node cache. + pub fn clear_node_cache(&mut self) { + self.node_cache.clear(); + } +} + +impl Default for TestTrieCache { + fn default() -> Self { + Self { value_cache: Default::default(), node_cache: Default::default() } + } +} + +impl trie_db::TrieCache for TestTrieCache { + fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&trie_db::CachedValue>> { + self.value_cache.get(key) + } + + fn cache_value_for_key(&mut self, key: &[u8], value: trie_db::CachedValue>) { + self.value_cache.insert(key.to_vec(), value); + } + + fn get_or_insert_node( + &mut self, + hash: TrieHash, + fetch_node: &mut dyn FnMut() -> trie_db::Result< + NodeOwned>, + TrieHash, + trie_db::CError, + >, + ) -> trie_db::Result<&NodeOwned>, TrieHash, trie_db::CError> { + match self.node_cache.entry(hash) { + Entry::Occupied(e) => Ok(e.into_mut()), + Entry::Vacant(e) => { + let node = (*fetch_node)()?; + Ok(e.insert(node)) + }, + } + } + + fn get_node(&mut self, hash: &TrieHash) -> Option<&NodeOwned>> { + self.node_cache.get(hash) + } +} + #[cfg(test)] mod tests { use super::*; - use trie_db::node::Node; + use trie_db::{nibble_ops::NIBBLE_PER_BYTE, node::Node}; #[test] fn test_encoding_simple_trie() { @@ -1140,7 +1175,8 @@ mod tests { // + 1 for 0 added byte of nibble encode let input = vec![0u8; (NIBBLE_SIZE_BOUND_NO_EXT as usize + 1) / 2 + 1]; let enc = as NodeCodec>::leaf_node( - ((0, 0), &input), + input.iter().cloned(), + input.len() * NIBBLE_PER_BYTE, Value::Inline(&[1]), ); let dec = as NodeCodec>::decode(&enc).unwrap(); diff --git a/test-support/reference-trie/src/substrate_like.rs b/test-support/reference-trie/src/substrate_like.rs index 0180f57d..e9b67489 100644 --- a/test-support/reference-trie/src/substrate_like.rs +++ b/test-support/reference-trie/src/substrate_like.rs @@ -169,19 +169,19 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: Value) -> Vec { + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec { let contains_hash = matches!(&value, Value::Node(..)); let mut output = if contains_hash { - partial_encode(partial, NodeKind::HashedValueLeaf) + partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueLeaf) } else { - partial_encode(partial, NodeKind::Leaf) + partial_from_iterator_encode(partial, number_nibble, NodeKind::Leaf) }; match value { Value::Inline(value) => { Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Value::Node(hash, _) => { + Value::Node(hash) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -228,7 +228,7 @@ where Compact(value.len() as u32).encode_to(&mut output); output.extend_from_slice(value); }, - Some(Value::Node(hash, _)) => { + Some(Value::Node(hash)) => { debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, @@ -279,31 +279,6 @@ fn partial_from_iterator_encode>( output } -/// Encode and allocate node type header (type and size), and partial value. -/// Same as `partial_from_iterator_encode` but uses non encoded `Partial` as input. -fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { - let number_nibble_encoded = (partial.0).0 as usize; - let nibble_count = partial.1.len() * nibble_ops::NIBBLE_PER_BYTE + number_nibble_encoded; - - let nibble_count = std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - - let mut output = Vec::with_capacity(3 + partial.1.len()); - match node_kind { - NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), - NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), - NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::HashedValueLeaf => - NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), - NodeKind::HashedValueBranch => - NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), - }; - if number_nibble_encoded > 0 { - output.push(nibble_ops::pad_right((partial.0).1)); - } - output.extend_from_slice(&partial.1[..]); - output -} - /// A node header. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub(crate) enum NodeHeader { diff --git a/test-support/trie-bench/src/lib.rs b/test-support/trie-bench/src/lib.rs index f54f9ce2..afa45849 100644 --- a/test-support/trie-bench/src/lib.rs +++ b/test-support/trie-bench/src/lib.rs @@ -20,7 +20,7 @@ use keccak_hasher::KeccakHasher; use memory_db::{HashKey, MemoryDB}; use parity_scale_codec::{Compact, Encode}; use std::default::Default; -use trie_db::{NodeCodec, Trie, TrieDB, TrieDBMut, TrieHash, TrieLayout, TrieMut}; +use trie_db::{NodeCodec, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieHash, TrieLayout, TrieMut}; use trie_root::{trie_root, TrieStream}; use trie_standardmap::*; @@ -61,7 +61,7 @@ fn benchmark( b.iter(&mut || { let mut memdb = MemoryDB::<_, HashKey, _>::new(L::Codec::empty_node()); let mut root = >::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for i in d.0.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -75,13 +75,13 @@ fn benchmark( let mut memdb = MemoryDB::<_, HashKey<_>, _>::new(L::Codec::empty_node()); let mut root = >::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for i in d.0.iter() { t.insert(&i.0, &i.1).unwrap(); } } b.iter(&mut || { - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); for n in t.iter().unwrap() { black_box(n).unwrap(); } diff --git a/trie-db/src/fatdb.rs b/trie-db/src/fatdb.rs index 06abe1b1..16883394 100644 --- a/trie-db/src/fatdb.rs +++ b/trie-db/src/fatdb.rs @@ -18,20 +18,20 @@ use super::{ }; use hash_db::{HashDBRef, Hasher}; -use crate::rstd::boxed::Box; +use crate::{rstd::boxed::Box, TrieDBBuilder}; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. /// /// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDB<'db, L> +pub struct FatDB<'db, 'cache, L> where L: TrieLayout, { - raw: TrieDB<'db, L>, + raw: TrieDB<'db, 'cache, L>, } -impl<'db, L> FatDB<'db, L> +impl<'db, 'cache, L> FatDB<'db, 'cache, L> where L: TrieLayout, { @@ -39,7 +39,7 @@ where /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - FatDB { raw: TrieDB::new(db, root) } + FatDB { raw: TrieDBBuilder::new(db, root).build() } } /// Get the backing database. @@ -48,7 +48,7 @@ where } } -impl<'db, L> Trie for FatDB<'db, L> +impl<'db, 'cache, L> Trie for FatDB<'db, 'cache, L> where L: TrieLayout, { @@ -60,14 +60,15 @@ where self.raw.contains(L::Hash::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>( - &'a self, - key: &'key [u8], + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { + self.raw.get_hash(key) + } + + fn get_with>( + &self, + key: &[u8], query: Q, - ) -> Result, TrieHash, CError> - where - 'a: 'key, - { + ) -> Result, TrieHash, CError> { self.raw.get_with(L::Hash::hash(key).as_ref(), query) } @@ -93,25 +94,25 @@ where } /// Iterator over inserted pairs of key values. -pub struct FatDBIterator<'db, L> +pub struct FatDBIterator<'db, 'cache, L> where L: TrieLayout, { - trie_iterator: TrieDBIterator<'db, L>, - trie: &'db TrieDB<'db, L>, + trie_iterator: TrieDBIterator<'db, 'cache, L>, + trie: &'db TrieDB<'db, 'cache, L>, } -impl<'db, L> FatDBIterator<'db, L> +impl<'db, 'cache, L> FatDBIterator<'db, 'cache, L> where L: TrieLayout, { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> Result, CError> { + pub fn new(trie: &'db TrieDB<'db, 'cache, L>) -> Result, CError> { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie }) } } -impl<'db, L> TrieIterator for FatDBIterator<'db, L> +impl<'db, 'cache, L> TrieIterator for FatDBIterator<'db, 'cache, L> where L: TrieLayout, { @@ -121,11 +122,11 @@ where } } -impl<'db, L> Iterator for FatDBIterator<'db, L> +impl<'db, 'cache, L> Iterator for FatDBIterator<'db, 'cache, L> where L: TrieLayout, { - type Item = TrieItem<'db, TrieHash, CError>; + type Item = TrieItem, CError>; fn next(&mut self) -> Option { self.trie_iterator.next().map(|res| { @@ -141,25 +142,25 @@ where } /// Iterator over inserted keys. -pub struct FatDBKeyIterator<'db, L> +pub struct FatDBKeyIterator<'db, 'cache, L> where L: TrieLayout, { - trie_iterator: TrieDBKeyIterator<'db, L>, - trie: &'db TrieDB<'db, L>, + trie_iterator: TrieDBKeyIterator<'db, 'cache, L>, + trie: &'db TrieDB<'db, 'cache, L>, } -impl<'db, L> FatDBKeyIterator<'db, L> +impl<'db, 'cache, L> FatDBKeyIterator<'db, 'cache, L> where L: TrieLayout, { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> Result, CError> { + pub fn new(trie: &'db TrieDB<'db, 'cache, L>) -> Result, CError> { Ok(FatDBKeyIterator { trie_iterator: TrieDBKeyIterator::new(trie)?, trie }) } } -impl<'db, L> TrieIterator for FatDBKeyIterator<'db, L> +impl<'db, 'cache, L> TrieIterator for FatDBKeyIterator<'db, 'cache, L> where L: TrieLayout, { @@ -169,11 +170,11 @@ where } } -impl<'db, L> Iterator for FatDBKeyIterator<'db, L> +impl<'db, 'cache, L> Iterator for FatDBKeyIterator<'db, 'cache, L> where L: TrieLayout, { - type Item = TrieKeyItem<'db, TrieHash, CError>; + type Item = TrieKeyItem, CError>; fn next(&mut self) -> Option { self.trie_iterator.next().map(|res| { diff --git a/trie-db/src/fatdbmut.rs b/trie-db/src/fatdbmut.rs index dbd9bbae..fa7a8f07 100644 --- a/trie-db/src/fatdbmut.rs +++ b/trie-db/src/fatdbmut.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, Value}; +use crate::{ + triedbmut::{TrieDBMutBuilder, Value}, + CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, +}; use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -34,7 +37,7 @@ where /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - FatDBMut { raw: TrieDBMut::new(db, root) } + FatDBMut { raw: TrieDBMutBuilder::new(db, root).build() } } /// Create a new trie with the backing database `db` and `root`. @@ -44,7 +47,7 @@ where db: &'db mut dyn HashDB, root: &'db mut TrieHash, ) -> Self { - FatDBMut { raw: TrieDBMut::from_existing(db, root) } + FatDBMut { raw: TrieDBMutBuilder::from_existing(db, root).build() } } /// Get the backing database. diff --git a/trie-db/src/iter_build.rs b/trie-db/src/iter_build.rs index a48964ea..c843c3f1 100644 --- a/trie-db/src/iter_build.rs +++ b/trie-db/src/iter_build.rs @@ -134,9 +134,9 @@ where value } else { hashed = callback.process_inner_hashed_value((k2.as_ref(), None), v2.as_ref()); - Value::Node(hashed.as_ref(), None) + Value::Node(hashed.as_ref()) }; - let encoded = T::Codec::leaf_node(nkey.right(), value); + let encoded = T::Codec::leaf_node(nkey.right_iter(), nkey.len(), value); let hash = callback.process(pr.left(), encoded, false); // insert hash in branch (first level branch only at this point) @@ -201,7 +201,7 @@ where let mut prefix = NibbleSlice::new_offset(&key_branch, 0); prefix.advance(branch_d); hashed = callback.process_inner_hashed_value(prefix.left(), v.as_ref()); - Value::Node(hashed.as_ref(), None) + Value::Node(hashed.as_ref()) }) } else { None @@ -244,7 +244,7 @@ where let mut prefix = NibbleSlice::new_offset(&key_branch, 0); prefix.advance(branch_d); hashed = callback.process_inner_hashed_value(prefix.left(), v.as_ref()); - Value::Node(hashed.as_ref(), None) + Value::Node(hashed.as_ref()) }) } else { None @@ -318,10 +318,10 @@ where value } else { hashed = callback.process_inner_hashed_value((k2.as_ref(), None), v2.as_ref()); - Value::Node(hashed.as_ref(), None) + Value::Node(hashed.as_ref()) }; - let encoded = T::Codec::leaf_node(nkey.right(), value); + let encoded = T::Codec::leaf_node(nkey.right_iter(), nkey.len(), value); callback.process(pr.left(), encoded, true); } else { depth_queue.flush_value(callback, last_depth, &previous_value); diff --git a/trie-db/src/iterator.rs b/trie-db/src/iterator.rs index 5b3716e7..77ae04e2 100644 --- a/trie-db/src/iterator.rs +++ b/trie-db/src/iterator.rs @@ -58,8 +58,8 @@ impl Crumb { } /// Iterator for going through all nodes in the trie in pre-order traversal order. -pub struct TrieDBNodeIterator<'a, L: TrieLayout> { - db: &'a TrieDB<'a, L>, +pub struct TrieDBNodeIterator<'a, 'cache, L: TrieLayout> { + db: &'a TrieDB<'a, 'cache, L>, trail: Vec>, key_nibbles: NibbleVec, } @@ -73,18 +73,25 @@ pub struct SuspendedTrieDBNodeIterator { impl SuspendedTrieDBNodeIterator { /// Restore iterator. - pub fn unsafe_restore<'a>(self, db: &'a TrieDB<'a, L>) -> TrieDBNodeIterator<'a, L> { + pub fn unsafe_restore<'a, 'cache>( + self, + db: &'a TrieDB<'a, 'cache, L>, + ) -> TrieDBNodeIterator<'a, 'cache, L> { TrieDBNodeIterator { db, trail: self.trail, key_nibbles: self.key_nibbles } } } -impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> Result, TrieHash, CError> { + pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { let mut r = TrieDBNodeIterator { db, trail: Vec::with_capacity(8), key_nibbles: NibbleVec::new() }; - let (root_node, root_hash) = - db.get_raw_or_lookup(*db.root(), NodeHandle::Hash(db.root().as_ref()), EMPTY_PREFIX)?; + let (root_node, root_hash) = db.get_raw_or_lookup( + *db.root(), + NodeHandle::Hash(db.root().as_ref()), + EMPTY_PREFIX, + true, + )?; r.descend(root_node, root_hash); Ok(r) } @@ -102,14 +109,18 @@ impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { } /// Fetch value by hash at a current node height - pub fn fetch_value(&self, key: &[u8], prefix: Prefix) -> Option { + pub fn fetch_value( + &self, + key: &[u8], + prefix: Prefix, + ) -> Result, CError> { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(key); - self.db.db().get(&res, prefix) + self.db.fetch_value(res, prefix) } } -impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieDBNodeIterator<'a, 'cache, L> { /// Seek a node position at 'key' for iterator. /// Returns true if the cursor is at or after the key, but still shares /// a common prefix with the key, return false if the key do not @@ -125,6 +136,7 @@ impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { >::default(), NodeHandle::Hash(self.db.root().as_ref()), EMPTY_PREFIX, + true, )?; let mut partial = key; let mut full_key_nibbles = 0; @@ -167,6 +179,7 @@ impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { node_hash.unwrap_or_default(), child.build(node_data), prefix.left(), + true, )? }, NodePlan::Branch { value: _, children } => { @@ -187,6 +200,7 @@ impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { node_hash.unwrap_or_default(), child.build(node_data), prefix.left(), + true, )? } else { return Ok(false) @@ -225,6 +239,7 @@ impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { node_hash.unwrap_or_default(), child.build(node_data), prefix.left(), + true, )? } else { return Ok(false) @@ -306,13 +321,13 @@ impl<'a, L: TrieLayout> TrieDBNodeIterator<'a, L> { } } -impl<'a, L: TrieLayout> TrieIterator for TrieDBNodeIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBNodeIterator<'a, 'cache, L> { fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { self.seek_prefix(key).map(|_| ()) } } -impl<'a, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, 'cache, L> { type Item = Result<(NibbleVec, Option>, Rc>), TrieHash, CError>; @@ -352,6 +367,7 @@ impl<'a, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, L> { b.hash.unwrap_or_default(), child.build(node_data), self.key_nibbles.as_prefix(), + true, )) }, (Status::At, NodePlan::Branch { .. }) => { @@ -373,6 +389,7 @@ impl<'a, L: TrieLayout> Iterator for TrieDBNodeIterator<'a, L> { b.hash.unwrap_or_default(), child.build(node_data), self.key_nibbles.as_prefix(), + true, )) } else { IterStep::Continue diff --git a/trie-db/src/lib.rs b/trie-db/src/lib.rs index 4e3982eb..aa020447 100644 --- a/trie-db/src/lib.rs +++ b/trie-db/src/lib.rs @@ -22,14 +22,14 @@ extern crate alloc; mod rstd { pub use std::{ borrow, boxed, cmp, collections::VecDeque, convert, error::Error, fmt, hash, iter, marker, - mem, ops, rc, result, vec, + mem, ops, rc, result, sync, vec, }; } #[cfg(not(feature = "std"))] mod rstd { - pub use alloc::{boxed, collections::VecDeque, rc, vec}; - pub use core::{borrow, cmp, convert, fmt, hash, iter, marker, mem, ops, result}; + pub use alloc::{borrow, boxed, collections::VecDeque, rc, sync, vec}; + pub use core::{cmp, convert, fmt, hash, iter, marker, mem, ops, result}; pub trait Error {} impl Error for T {} } @@ -39,6 +39,7 @@ use self::rstd::{fmt, Error}; use self::rstd::{boxed::Box, vec::Vec}; use hash_db::MaybeDebug; +use node::NodeOwned; pub mod node; pub mod proof; @@ -62,11 +63,11 @@ pub use self::{ fatdbmut::FatDBMut, lookup::Lookup, nibble::{nibble_ops, NibbleSlice, NibbleVec}, - recorder::{Record, Recorder}, + recorder::Recorder, sectriedb::SecTrieDB, sectriedbmut::SecTrieDBMut, - triedb::{TrieDB, TrieDBIterator, TrieDBKeyIterator}, - triedbmut::{ChildReference, TrieDBMut, Value}, + triedb::{TrieDB, TrieDBBuilder, TrieDBIterator, TrieDBKeyIterator}, + triedbmut::{ChildReference, TrieDBMut, TrieDBMutBuilder, Value}, }; pub use crate::{ iter_build::{trie_visit, ProcessEncodedNode, TrieBuilder, TrieRoot, TrieRootUnhashed}, @@ -141,37 +142,100 @@ where pub type Result = crate::rstd::result::Result>>; /// Trie-Item type used for iterators over trie data. -pub type TrieItem<'a, U, E> = Result<(Vec, DBValue), U, E>; +pub type TrieItem = Result<(Vec, DBValue), U, E>; /// Trie-Item type used for iterators over trie key only. -pub type TrieKeyItem<'a, U, E> = Result, U, E>; +pub type TrieKeyItem = Result, U, E>; /// Description of what kind of query will be made to the trie. -/// -/// This is implemented for any &mut recorder (where the query will return -/// a DBValue), any function taking raw bytes (where no recording will be made), -/// or any tuple of (&mut Recorder, FnOnce(&[u8])) pub trait Query { /// Output item. type Item; /// Decode a byte-slice into the desired item. fn decode(self, data: &[u8]) -> Self::Item; +} - /// Record that a node has been passed through. - fn record(&mut self, _hash: &H::Out, _data: &[u8], _depth: u32) {} +/// Used to report the trie access to the [`TrieRecorder`]. +/// +/// As the trie can use a [`TrieCache`], there are multiple kinds of accesses. +/// If a cache is used, [`Self::Key`] and [`Self::NodeOwned`] are possible +/// values. Otherwise only [`Self::EncodedNode`] is a possible value. +#[cfg_attr(feature = "std", derive(Debug))] +pub enum TrieAccess<'a, H> { + /// The given [`NodeOwned`] was accessed using its `hash`. + NodeOwned { hash: H, node_owned: &'a NodeOwned }, + /// The given `encoded_node` was accessed using its `hash`. + EncodedNode { hash: H, encoded_node: rstd::borrow::Cow<'a, [u8]> }, + /// The given `value` was accessed using its `hash`. + /// + /// The given `full_key` is the key to access this value in the trie. + /// + /// Should map to [`RecordedForKey::Value`] when checking the recorder. + Value { hash: H, value: rstd::borrow::Cow<'a, [u8]>, full_key: &'a [u8] }, + /// The hash of the value for the given `full_key` was accessed. + /// + /// Should map to [`RecordedForKey::Hash`] when checking the recorder. + Hash { full_key: &'a [u8] }, + /// The value/hash for `full_key` was accessed, but it couldn't be found in the trie. + /// + /// Should map to [`RecordedForKey::Value`] when checking the recorder. + NonExisting { full_key: &'a [u8] }, } -impl<'a, H: Hasher> Query for &'a mut Recorder { - type Item = DBValue; - fn decode(self, value: &[u8]) -> DBValue { - value.to_vec() - } - fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) { - (&mut **self).record(hash, data, depth); +/// Result of [`TrieRecorder::trie_nodes_recorded_for_key`]. +#[derive(Debug, Clone, Copy)] +pub enum RecordedForKey { + /// We recorded all trie nodes up to the value for a storage key. + /// + /// This should be returned when the recorder has seen the following [`TrieAccess`]: + /// + /// - [`TrieAccess::Value`]: If we see this [`TrieAccess`], it means we have recorded all the + /// trie nodes up to the value. + /// - [`TrieAccess::NonExisting`]: If we see this [`TrieAccess`], it means we have recorded all + /// the necessary trie nodes to prove that the value doesn't exist in the trie. + Value, + /// We recorded all trie nodes up to the value hash for a storage key. + /// + /// If we have a [`RecordedForKey::Value`], it means that we also have the hash of this value. + /// This also means that if we first have recorded the hash of a value and then also record the + /// value, the access should be upgraded to [`RecordedForKey::Value`]. + /// + /// This should be returned when the recorder has seen the following [`TrieAccess`]: + /// + /// - [`TrieAccess::Hash`]: If we see this [`TrieAccess`], it means we have recorded all trie + /// nodes to have the hash of the value. + Hash, + /// We haven't recorded any trie nodes yet for a storage key. + /// + /// This means we have not seen any [`TrieAccess`] referencing the searched key. + None, +} + +impl RecordedForKey { + /// Is `self` equal to [`Self::None`]? + pub fn is_none(&self) -> bool { + matches!(self, Self::None) } } +/// A trie recorder that can be used to record all kind of [`TrieAccess`]'s. +/// +/// To build a trie proof a recorder is required that records all trie accesses. These recorded trie +/// accesses can then be used to create the proof. +pub trait TrieRecorder { + /// Record the given [`TrieAccess`]. + /// + /// Depending on the [`TrieAccess`] a call of [`Self::trie_nodes_recorded_for_key`] afterwards + /// must return the correct recorded state. + fn record<'a>(&mut self, access: TrieAccess<'a, H>); + + /// Check if we have recorded any trie nodes for the given `key`. + /// + /// Returns [`RecordedForKey`] to express the state of the recorded trie nodes. + fn trie_nodes_recorded_for_key(&self, key: &[u8]) -> RecordedForKey; +} + impl Query for F where F: for<'a> FnOnce(&'a [u8]) -> T, @@ -182,19 +246,6 @@ where } } -impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) -where - F: FnOnce(&[u8]) -> T, -{ - type Item = T; - fn decode(self, value: &[u8]) -> T { - (self.1)(value) - } - fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) { - self.0.record(hash, data, depth) - } -} - /// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait Trie { /// Return the root of the trie. @@ -210,23 +261,21 @@ pub trait Trie { self.get(key).map(|x| x.is_some()) } + /// Returns the hash of the value for `key`. + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError>; + /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, TrieHash, CError> - where - 'a: 'key, - { + fn get(&self, key: &[u8]) -> Result, TrieHash, CError> { self.get_with(key, |v: &[u8]| v.to_vec()) } /// Search for the key with the given query parameter. See the docs of the `Query` /// trait for more details. - fn get_with<'a, 'key, Q: Query>( - &'a self, - key: &'key [u8], + fn get_with>( + &self, + key: &[u8], query: Q, - ) -> Result, TrieHash, CError> - where - 'a: 'key; + ) -> Result, TrieHash, CError>; /// Returns a depth-first iterator over the elements of trie. fn iter<'a>( @@ -310,13 +359,13 @@ pub struct TrieFactory { /// All different kinds of tries. /// This is used to prevent a heap allocation for every created trie. -pub enum TrieKinds<'db, L: TrieLayout> { +pub enum TrieKinds<'db, 'cache, L: TrieLayout> { /// A generic trie db. - Generic(TrieDB<'db, L>), + Generic(TrieDB<'db, 'cache, L>), /// A secure trie db. - Secure(SecTrieDB<'db, L>), + Secure(SecTrieDB<'db, 'cache, L>), /// A fat trie db. - Fat(FatDB<'db, L>), + Fat(FatDB<'db, 'cache, L>), } // wrapper macro for making the match easier to deal with. @@ -330,7 +379,7 @@ macro_rules! wrapper { } } -impl<'db, L: TrieLayout> Trie for TrieKinds<'db, L> { +impl<'db, 'cache, L: TrieLayout> Trie for TrieKinds<'db, 'cache, L> { fn root(&self) -> &TrieHash { wrapper!(self, root,) } @@ -343,14 +392,15 @@ impl<'db, L: TrieLayout> Trie for TrieKinds<'db, L> { wrapper!(self, contains, key) } - fn get_with<'a, 'key, Q: Query>( - &'a self, - key: &'key [u8], + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { + wrapper!(self, get_hash, key) + } + + fn get_with>( + &self, + key: &[u8], query: Q, - ) -> Result, TrieHash, CError> - where - 'a: 'key, - { + ) -> Result, TrieHash, CError> { wrapper!(self, get_with, key, query) } @@ -382,13 +432,13 @@ impl TrieFactory { } /// Create new immutable instance of Trie. - pub fn readonly<'db, L: TrieLayout>( + pub fn readonly<'db, 'cache, L: TrieLayout>( &self, db: &'db dyn HashDBRef, root: &'db TrieHash, - ) -> TrieKinds<'db, L> { + ) -> TrieKinds<'db, 'cache, L> { match self.spec { - TrieSpec::Generic => TrieKinds::Generic(TrieDB::new(db, root)), + TrieSpec::Generic => TrieKinds::Generic(TrieDBBuilder::new(db, root).build()), TrieSpec::Secure => TrieKinds::Secure(SecTrieDB::new(db, root)), TrieSpec::Fat => TrieKinds::Fat(FatDB::new(db, root)), } @@ -401,7 +451,7 @@ impl TrieFactory { root: &'db mut TrieHash, ) -> Box + 'db> { match self.spec { - TrieSpec::Generic => Box::new(TrieDBMut::::new(db, root)), + TrieSpec::Generic => Box::new(TrieDBMutBuilder::::new(db, root).build()), TrieSpec::Secure => Box::new(SecTrieDBMut::::new(db, root)), TrieSpec::Fat => Box::new(FatDBMut::::new(db, root)), } @@ -414,7 +464,7 @@ impl TrieFactory { root: &'db mut TrieHash, ) -> Box + 'db> { match self.spec { - TrieSpec::Generic => Box::new(TrieDBMut::::from_existing(db, root)), + TrieSpec::Generic => Box::new(TrieDBMutBuilder::::from_existing(db, root).build()), TrieSpec::Secure => Box::new(SecTrieDBMut::::from_existing(db, root)), TrieSpec::Fat => Box::new(FatDBMut::::from_existing(db, root)), } @@ -507,3 +557,184 @@ pub trait TrieConfiguration: Sized + TrieLayout { pub type TrieHash = <::Hash as Hasher>::Out; /// Alias accessor to `NodeCodec` associated `Error` type from a `TrieLayout`. pub type CError = <::Codec as NodeCodec>::Error; + +/// A value as cached by the [`TrieCache`]. +#[derive(Clone, Debug)] +pub enum CachedValue { + /// The value doesn't exist in the trie. + NonExisting, + /// We cached the hash, because we did not yet accessed the data. + ExistingHash(H), + /// The value exists in the trie. + Existing { + /// The hash of the value. + hash: H, + /// The actual data of the value stored as [`BytesWeak`]. + /// + /// The original data [`Bytes`] is stored in the trie node + /// that is also cached by the [`TrieCache`]. If this node is dropped, + /// this data will also not be "upgradeable" anymore. + data: BytesWeak, + }, +} + +impl CachedValue { + /// Returns the data of the value. + /// + /// If a value doesn't exist in the trie or only the value hash is cached, this function returns + /// `None`. If the reference to the data couldn't be upgraded (see [`Bytes::upgrade`]), this + /// function returns `Some(None)`, aka the data needs to be fetched again from the trie. + pub fn data(&self) -> Option> { + match self { + Self::Existing { data, .. } => Some(data.upgrade()), + _ => None, + } + } + + /// Returns the hash of the value. + /// + /// Returns only `None` when the value doesn't exist. + pub fn hash(&self) -> Option { + match self { + Self::ExistingHash(hash) | Self::Existing { hash, .. } => Some(*hash), + Self::NonExisting => None, + } + } +} + +impl From<(Bytes, H)> for CachedValue { + fn from(value: (Bytes, H)) -> Self { + Self::Existing { hash: value.1, data: value.0.into() } + } +} + +impl From for CachedValue { + fn from(value: H) -> Self { + Self::ExistingHash(value) + } +} + +impl From> for CachedValue { + fn from(value: Option<(Bytes, H)>) -> Self { + value.map_or(Self::NonExisting, |v| Self::Existing { hash: v.1, data: v.0.into() }) + } +} + +impl From> for CachedValue { + fn from(value: Option) -> Self { + value.map_or(Self::NonExisting, |v| Self::ExistingHash(v)) + } +} + +/// A cache that can be used to speed-up certain operations when accessing the trie. +/// +/// The [`TrieDB`]/[`TrieDBMut`] by default are working with the internal hash-db in a non-owning +/// mode. This means that for every lookup in the trie, every node is always fetched and decoded on +/// the fly. Fetching and decoding a node always takes some time and can kill the performance of any +/// application that is doing quite a lot of trie lookups. To circumvent this performance +/// degradation, a cache can be used when looking up something in the trie. Any cache that should be +/// used with the [`TrieDB`]/[`TrieDBMut`] needs to implement this trait. +/// +/// The trait is laying out a two level cache, first the trie nodes cache and then the value cache. +/// The trie nodes cache, as the name indicates, is for caching trie nodes as [`NodeOwned`]. These +/// trie nodes are referenced by their hash. The value cache is caching [`CachedValue`]'s and these +/// are referenced by the key to look them up in the trie. As multiple different tries can have +/// different values under the same key, it up to the cache implementation to ensure that the +/// correct value is returned. As each trie has a different root, this root can be used to +/// differentiate values under the same key. +pub trait TrieCache { + /// Lookup value for the given `key`. + /// + /// Returns the `None` if the `key` is unknown or otherwise `Some(_)` with the associated + /// value. + /// + /// [`Self::cache_data_for_key`] is used to make the cache aware of data that is associated + /// to a `key`. + /// + /// # Attention + /// + /// The cache can be used for different tries, aka with different roots. This means + /// that the cache implementation needs to take care of always returning the correct value + /// for the current trie root. + fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&CachedValue>; + + /// Cache the given `value` for the given `key`. + /// + /// # Attention + /// + /// The cache can be used for different tries, aka with different roots. This means + /// that the cache implementation needs to take care of caching `value` for the current + /// trie root. + fn cache_value_for_key(&mut self, key: &[u8], value: CachedValue); + + /// Get or insert a [`NodeOwned`]. + /// + /// The cache implementation should look up based on the given `hash` if the node is already + /// known. If the node is not yet known, the given `fetch_node` function can be used to fetch + /// the particular node. + /// + /// Returns the [`NodeOwned`] or an error that happened on fetching the node. + fn get_or_insert_node( + &mut self, + hash: NC::HashOut, + fetch_node: &mut dyn FnMut() -> Result, NC::HashOut, NC::Error>, + ) -> Result<&NodeOwned, NC::HashOut, NC::Error>; + + /// Get the [`NodeOwned`] that corresponds to the given `hash`. + fn get_node(&mut self, hash: &NC::HashOut) -> Option<&NodeOwned>; +} + +/// A container for storing bytes. +/// +/// This uses a reference counted pointer internally, so it is cheap to clone this object. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Bytes(rstd::sync::Arc<[u8]>); + +impl rstd::ops::Deref for Bytes { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl From> for Bytes { + fn from(bytes: Vec) -> Self { + Self(bytes.into()) + } +} + +impl From<&[u8]> for Bytes { + fn from(bytes: &[u8]) -> Self { + Self(bytes.into()) + } +} + +impl> PartialEq for Bytes { + fn eq(&self, other: &T) -> bool { + self.as_ref() == other.as_ref() + } +} + +/// A weak reference of [`Bytes`]. +/// +/// A weak reference means that it doesn't prevent [`Bytes`] from being dropped because +/// it holds a non-owning reference to the associated [`Bytes`] object. With [`Self::upgrade`] it +/// is possible to upgrade it again to [`Bytes`] if the reference is still valid. +#[derive(Clone, Debug)] +pub struct BytesWeak(rstd::sync::Weak<[u8]>); + +impl BytesWeak { + /// Upgrade to [`Bytes`]. + /// + /// Returns `None` when the inner value was already dropped. + pub fn upgrade(&self) -> Option { + self.0.upgrade().map(Bytes) + } +} + +impl From for BytesWeak { + fn from(bytes: Bytes) -> Self { + Self(rstd::sync::Arc::downgrade(&bytes.0)) + } +} diff --git a/trie-db/src/lookup.rs b/trie-db/src/lookup.rs index 275fe379..ee321797 100644 --- a/trie-db/src/lookup.rs +++ b/trie-db/src/lookup.rs @@ -14,45 +14,64 @@ //! Trie lookup via HashDB. -use super::{CError, DBValue, Query, Result, TrieError, TrieHash, TrieLayout}; use crate::{ nibble::NibbleSlice, - node::{decode_hash, Node, NodeHandle, Value}, + node::{decode_hash, Node, NodeHandle, NodeHandleOwned, NodeOwned, Value, ValueOwned}, node_codec::NodeCodec, rstd::boxed::Box, + Bytes, CError, CachedValue, DBValue, Query, RecordedForKey, Result, TrieAccess, TrieCache, + TrieError, TrieHash, TrieLayout, TrieRecorder, }; -use hash_db::{HashDBRef, Prefix}; +use hash_db::{HashDBRef, Hasher, Prefix}; /// Trie lookup helper object. -pub struct Lookup<'a, L: TrieLayout, Q: Query> { +pub struct Lookup<'a, 'cache, L: TrieLayout, Q: Query> { /// database to query from. pub db: &'a dyn HashDBRef, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at pub hash: TrieHash, + /// Optional cache that should be used to speed up the lookup. + pub cache: Option<&'cache mut dyn TrieCache>, + /// Optional recorder that will be called to record all trie accesses. + pub recorder: Option<&'cache mut dyn TrieRecorder>>, } -impl<'a, L, Q> Lookup<'a, L, Q> +impl<'a, 'cache, L, Q> Lookup<'a, 'cache, L, Q> where L: TrieLayout, Q: Query, { - fn decode( - mut self, + /// Load the given value. + /// + /// This will access the `db` if the value is not already in memory, but then it will put it + /// into the given `cache` as `NodeOwned::Value`. + /// + /// Returns the bytes representing the value. + fn load_value( v: Value, prefix: Prefix, - depth: u32, + full_key: &[u8], + db: &dyn HashDBRef, + recorder: &mut Option<&mut dyn TrieRecorder>>, + query: Q, ) -> Result, CError> { match v { - Value::Inline(value) => Ok(self.query.decode(value)), - Value::Node(_, Some(value)) => Ok(self.query.decode(value.as_slice())), - Value::Node(hash, None) => { + Value::Inline(value) => Ok(query.decode(&value)), + Value::Node(hash) => { let mut res = TrieHash::::default(); res.as_mut().copy_from_slice(hash); - if let Some(value) = self.db.get(&res, prefix) { - self.query.record(&res, &value, depth); - Ok(self.query.decode(value.as_slice())) + if let Some(value) = db.get(&res, prefix) { + if let Some(recorder) = recorder { + recorder.record(TrieAccess::Value { + hash: res, + value: value.as_slice().into(), + full_key, + }); + } + + Ok(query.decode(&value)) } else { Err(Box::new(TrieError::IncompleteDatabase(res))) } @@ -60,20 +79,463 @@ where } } + /// Load the given value. + /// + /// This will access the `db` if the value is not already in memory, but then it will put it + /// into the given `cache` as `NodeOwned::Value`. + /// + /// Returns the bytes representing the value and its hash. + fn load_owned_value( + v: ValueOwned>, + prefix: Prefix, + full_key: &[u8], + cache: &mut dyn crate::TrieCache, + db: &dyn HashDBRef, + recorder: &mut Option<&mut dyn TrieRecorder>>, + ) -> Result<(Bytes, TrieHash), TrieHash, CError> { + match v { + ValueOwned::Inline(value, hash) => Ok((value.clone(), hash)), + ValueOwned::Node(hash) => { + let node = cache.get_or_insert_node(hash, &mut || { + let value = db + .get(&hash, prefix) + .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + + Ok(NodeOwned::Value(value.into(), hash)) + })?; + + let value = node + .data() + .expect( + "We are caching a `NodeOwned::Value` for a value node \ + hash and this cached node has always data attached; qed", + ) + .clone(); + + if let Some(recorder) = recorder { + recorder.record(TrieAccess::Value { + hash, + value: value.as_ref().into(), + full_key, + }); + } + + Ok((value, hash)) + }, + } + } + + fn record<'b>(&mut self, get_access: impl FnOnce() -> TrieAccess<'b, TrieHash>) + where + TrieHash: 'b, + { + if let Some(recorder) = self.recorder.as_mut() { + recorder.record(get_access()); + } + } + + /// Look up the given `nibble_key`. + /// + /// If the value is found, it will be passed to the given function to decode or copy. + /// + /// The given `full_key` should be the full key to the data that is requested. This will + /// be used when there is a cache to potentially speed up the lookup. + pub fn look_up( + mut self, + full_key: &[u8], + nibble_key: NibbleSlice, + ) -> Result, TrieHash, CError> { + match self.cache.take() { + Some(cache) => self.look_up_with_cache(full_key, nibble_key, cache), + None => self.look_up_without_cache(nibble_key, full_key, Self::load_value), + } + } + + /// Look up the value hash for the given `nibble_key`. + /// + /// The given `full_key` should be the full key to the data that is requested. This will + /// be used when there is a cache to potentially speed up the lookup. + pub fn look_up_hash( + mut self, + full_key: &[u8], + nibble_key: NibbleSlice, + ) -> Result>, TrieHash, CError> { + match self.cache.take() { + Some(cache) => self.look_up_hash_with_cache(full_key, nibble_key, cache), + None => self.look_up_without_cache( + nibble_key, + full_key, + |v, _, full_key, _, recorder, _| { + Ok(match v { + Value::Inline(v) => { + let hash = L::Hash::hash(&v); + + if let Some(recoder) = recorder.as_mut() { + recoder.record(TrieAccess::Value { + hash, + value: v.into(), + full_key, + }); + } + + hash + }, + Value::Node(hash_bytes) => { + if let Some(recoder) = recorder.as_mut() { + recoder.record(TrieAccess::Hash { full_key }); + } + + let mut hash = TrieHash::::default(); + hash.as_mut().copy_from_slice(hash_bytes); + hash + }, + }) + }, + ), + } + } + + /// Look up the value hash for the given key. + /// + /// It uses the given cache to speed-up lookups. + fn look_up_hash_with_cache( + mut self, + full_key: &[u8], + nibble_key: NibbleSlice, + cache: &mut dyn crate::TrieCache, + ) -> Result>, TrieHash, CError> { + let value_cache_allowed = self + .recorder + .as_ref() + // Check if the recorder has the trie nodes already recorded for this key. + .map(|r| !r.trie_nodes_recorded_for_key(full_key).is_none()) + // If there is no recorder, we can always use the value cache. + .unwrap_or(true); + + let res = if let Some(hash) = value_cache_allowed + .then(|| cache.lookup_value_for_key(full_key).map(|v| v.hash())) + .flatten() + { + hash + } else { + let hash_and_value = self.look_up_with_cache_internal( + nibble_key, + full_key, + cache, + |value, _, full_key, _, _, recorder| match value { + ValueOwned::Inline(value, hash) => { + if let Some(recorder) = recorder.as_mut() { + recorder.record(TrieAccess::Value { + hash, + value: value.as_ref().into(), + full_key, + }); + } + + Ok((hash, Some(value.clone()))) + }, + ValueOwned::Node(hash) => { + if let Some(recoder) = recorder.as_mut() { + recoder.record(TrieAccess::Hash { full_key }); + } + + Ok((hash, None)) + }, + }, + )?; + + match &hash_and_value { + Some((hash, Some(value))) => + cache.cache_value_for_key(full_key, (value.clone(), *hash).into()), + Some((hash, None)) => cache.cache_value_for_key(full_key, (*hash).into()), + None => cache.cache_value_for_key(full_key, CachedValue::NonExisting), + } + + hash_and_value.map(|v| v.0) + }; + + Ok(res) + } + /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. - pub fn look_up(mut self, key: NibbleSlice) -> Result, TrieHash, CError> { - let mut partial = key; + /// + /// It uses the given cache to speed-up lookups. + fn look_up_with_cache( + mut self, + full_key: &[u8], + nibble_key: NibbleSlice, + cache: &mut dyn crate::TrieCache, + ) -> Result, TrieHash, CError> { + let trie_nodes_recorded = + self.recorder.as_ref().map(|r| r.trie_nodes_recorded_for_key(full_key)); + + let (value_cache_allowed, value_recording_required) = match trie_nodes_recorded { + // If we already have the trie nodes recorded up to the value, we are allowed + // to use the value cache. + Some(RecordedForKey::Value) | None => (true, false), + // If we only have recorded the hash, we are allowed to use the value cache, but + // we may need to have the value recorded. + Some(RecordedForKey::Hash) => (true, true), + // As we don't allow the value cache, the second value can be actually anything. + Some(RecordedForKey::None) => (false, true), + }; + + let lookup_data = |lookup: &mut Self, + cache: &mut dyn crate::TrieCache| + -> Result, TrieHash, CError> { + let data = lookup.look_up_with_cache_internal( + nibble_key, + full_key, + cache, + Self::load_owned_value, + )?; + + cache.cache_value_for_key(full_key, data.clone().into()); + + Ok(data.map(|d| d.0)) + }; + + let res = match value_cache_allowed.then(|| cache.lookup_value_for_key(full_key)).flatten() + { + Some(CachedValue::NonExisting) => None, + Some(CachedValue::ExistingHash(hash)) => { + let data = Self::load_owned_value( + // If we only have the hash cached, this can only be a value node. + // For inline nodes we cache them directly as `CachedValue::Existing`. + ValueOwned::Node(*hash), + nibble_key.original_data_as_prefix(), + full_key, + cache, + self.db, + &mut self.recorder, + )?; + + cache.cache_value_for_key(full_key, data.clone().into()); + + Some(data.0) + }, + Some(CachedValue::Existing { data, hash, .. }) => + if let Some(data) = data.upgrade() { + if value_recording_required { + // As a value is only raw data, we can directly record it. + self.record(|| TrieAccess::Value { + hash: *hash, + value: data.as_ref().into(), + full_key, + }); + } + + Some(data) + } else { + lookup_data(&mut self, cache)? + }, + None => lookup_data(&mut self, cache)?, + }; + + Ok(res.map(|v| self.query.decode(&v))) + } + + /// When modifying any logic inside this function, you also need to do the same in + /// [`Self::lookup_without_cache`]. + fn look_up_with_cache_internal( + &mut self, + nibble_key: NibbleSlice, + full_key: &[u8], + cache: &mut dyn crate::TrieCache, + load_value_owned: impl Fn( + ValueOwned>, + Prefix, + &[u8], + &mut dyn crate::TrieCache, + &dyn HashDBRef, + &mut Option<&mut dyn TrieRecorder>>, + ) -> Result, CError>, + ) -> Result, TrieHash, CError> { + let mut partial = nibble_key; + let mut hash = self.hash; let mut key_nibbles = 0; - let mut full_key = key.clone(); - full_key.advance(key.len()); - let full_key = full_key.left(); + // this loop iterates through non-inline nodes. + for depth in 0.. { + let mut node = cache.get_or_insert_node(hash, &mut || { + let node_data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) { + Some(value) => value, + None => + return Err(Box::new(match depth { + 0 => TrieError::InvalidStateRoot(hash), + _ => TrieError::IncompleteDatabase(hash), + })), + }; + + let decoded = match L::Codec::decode(&node_data[..]) { + Ok(node) => node, + Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), + }; + + decoded.to_owned_node::() + })?; + + self.record(|| TrieAccess::NodeOwned { hash, node_owned: node }); + + // this loop iterates through all inline children (usually max 1) + // without incrementing the depth. + loop { + let next_node = match node { + NodeOwned::Leaf(slice, value) => + return if partial == *slice { + let value = (*value).clone(); + drop(node); + load_value_owned( + value, + nibble_key.original_data_as_prefix(), + full_key, + cache, + self.db, + &mut self.recorder, + ) + .map(Some) + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + Ok(None) + }, + NodeOwned::Extension(slice, item) => + if partial.starts_with_vec(&slice) { + partial = partial.mid(slice.len()); + key_nibbles += slice.len(); + item + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + NodeOwned::Branch(children, value) => + if partial.is_empty() { + return if let Some(value) = value.clone() { + drop(node); + load_value_owned( + value, + nibble_key.original_data_as_prefix(), + full_key, + cache, + self.db, + &mut self.recorder, + ) + .map(Some) + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + Ok(None) + } + } else { + match &children[partial.at(0) as usize] { + Some(x) => { + partial = partial.mid(1); + key_nibbles += 1; + x + }, + None => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + } + }, + NodeOwned::NibbledBranch(slice, children, value) => { + if !partial.starts_with_vec(&slice) { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + } + + if partial.len() == slice.len() { + return if let Some(value) = value.clone() { + drop(node); + load_value_owned( + value, + nibble_key.original_data_as_prefix(), + full_key, + cache, + self.db, + &mut self.recorder, + ) + .map(Some) + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + Ok(None) + } + } else { + match &children[partial.at(slice.len()) as usize] { + Some(x) => { + partial = partial.mid(slice.len() + 1); + key_nibbles += slice.len() + 1; + x + }, + None => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + } + } + }, + NodeOwned::Empty => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + NodeOwned::Value(_, _) => { + unreachable!( + "`NodeOwned::Value` can not be reached by using the hash of a node. \ + `NodeOwned::Value` is only constructed when loading a value into memory, \ + which needs to have a different hash than any node; qed", + ) + }, + }; + + // check if new node data is inline or hash. + match next_node { + NodeHandleOwned::Hash(new_hash) => { + hash = *new_hash; + break + }, + NodeHandleOwned::Inline(inline_node) => { + node = &inline_node; + }, + } + } + } + + Ok(None) + } + + /// Look up the given key. If the value is found, it will be passed to the given + /// function to decode or copy. + /// + /// When modifying any logic inside this function, you also need to do the same in + /// [`Self::lookup_with_cache_internal`]. + fn look_up_without_cache( + mut self, + nibble_key: NibbleSlice, + full_key: &[u8], + load_value: impl Fn( + Value, + Prefix, + &[u8], + &dyn HashDBRef, + &mut Option<&mut dyn TrieRecorder>>, + Q, + ) -> Result, CError>, + ) -> Result, TrieHash, CError> { + let mut partial = nibble_key; + let mut hash = self.hash; + let mut key_nibbles = 0; // this loop iterates through non-inline nodes. for depth in 0.. { - let hash = self.hash; - let node_data = match self.db.get(&hash, key.mid(key_nibbles).left()) { + let node_data = match self.db.get(&hash, nibble_key.mid(key_nibbles).left()) { Some(value) => value, None => return Err(Box::new(match depth { @@ -82,7 +544,10 @@ where })), }; - self.query.record(&hash, &node_data, depth); + self.record(|| TrieAccess::EncodedNode { + hash, + encoded_node: node_data.as_slice().into(), + }); // this loop iterates through all inline children (usually max 1) // without incrementing the depth. @@ -92,65 +557,114 @@ where Ok(node) => node, Err(e) => return Err(Box::new(TrieError::DecoderError(hash, e))), }; + let next_node = match decoded { Node::Leaf(slice, value) => - return Ok(match slice == partial { - true => Some(self.decode(value, full_key, depth)?), - false => None, - }), + return if slice == partial { + load_value( + value, + nibble_key.original_data_as_prefix(), + full_key, + self.db, + &mut self.recorder, + self.query, + ) + .map(Some) + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + Ok(None) + }, Node::Extension(slice, item) => if partial.starts_with(&slice) { partial = partial.mid(slice.len()); key_nibbles += slice.len(); item } else { + self.record(|| TrieAccess::NonExisting { full_key }); + return Ok(None) }, - Node::Branch(children, value) => match partial.is_empty() { - true => - if let Some(value) = value { - return Ok(Some(self.decode(value, full_key, depth)?)) + Node::Branch(children, value) => + if partial.is_empty() { + return if let Some(val) = value { + load_value( + val, + nibble_key.original_data_as_prefix(), + full_key, + self.db, + &mut self.recorder, + self.query, + ) + .map(Some) } else { - return Ok(None) - }, - false => match children[partial.at(0) as usize] { - Some(x) => { - partial = partial.mid(1); - key_nibbles += 1; - x - }, - None => return Ok(None), + self.record(|| TrieAccess::NonExisting { full_key }); + + Ok(None) + } + } else { + match children[partial.at(0) as usize] { + Some(x) => { + partial = partial.mid(1); + key_nibbles += 1; + x + }, + None => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + } }, - }, Node::NibbledBranch(slice, children, value) => { if !partial.starts_with(&slice) { + self.record(|| TrieAccess::NonExisting { full_key }); + return Ok(None) } - match partial.len() == slice.len() { - true => - if let Some(value) = value { - return Ok(Some(self.decode(value, full_key, depth)?)) - } else { - return Ok(None) - }, - false => match children[partial.at(slice.len()) as usize] { + if partial.len() == slice.len() { + return if let Some(val) = value { + load_value( + val, + nibble_key.original_data_as_prefix(), + full_key, + self.db, + &mut self.recorder, + self.query, + ) + .map(Some) + } else { + self.record(|| TrieAccess::NonExisting { full_key }); + + Ok(None) + } + } else { + match children[partial.at(slice.len()) as usize] { Some(x) => { partial = partial.mid(slice.len() + 1); key_nibbles += slice.len() + 1; x }, - None => return Ok(None), - }, + None => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, + } } }, - Node::Empty => return Ok(None), + Node::Empty => { + self.record(|| TrieAccess::NonExisting { full_key }); + + return Ok(None) + }, }; // check if new node data is inline or hash. match next_node { NodeHandle::Hash(data) => { - self.hash = decode_hash::(data) + hash = decode_hash::(data) .ok_or_else(|| Box::new(TrieError::InvalidHash(hash, data.to_vec())))?; break }, diff --git a/trie-db/src/nibble/mod.rs b/trie-db/src/nibble/mod.rs index 9c23d2fb..4fc0d68e 100644 --- a/trie-db/src/nibble/mod.rs +++ b/trie-db/src/nibble/mod.rs @@ -79,8 +79,8 @@ pub mod nibble_ops { into | if ix == 1 { v } else { v << BIT_PER_NIBBLE } } - #[inline] /// Calculate the number of needed padding a array of nibble length `i`. + #[inline] pub fn number_padding(i: usize) -> usize { i % NIBBLE_PER_BYTE } diff --git a/trie-db/src/nibble/nibbleslice.rs b/trie-db/src/nibble/nibbleslice.rs index 21b3bae9..f91fad7f 100644 --- a/trie-db/src/nibble/nibbleslice.rs +++ b/trie-db/src/nibble/nibbleslice.rs @@ -14,7 +14,7 @@ //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. -use super::{nibble_ops, BackingByteVec, NibbleSlice, NibbleSliceIterator}; +use super::{nibble_ops, BackingByteVec, NibbleSlice, NibbleSliceIterator, NibbleVec}; #[cfg(feature = "std")] use crate::rstd::fmt; use crate::{node::NodeKey, node_codec::Partial, rstd::cmp::*}; @@ -133,15 +133,35 @@ impl<'a> NibbleSlice<'a> { /// How many of the same nibbles at the beginning do we match with `them`? pub fn common_prefix(&self, them: &Self) -> usize { - let s = min(self.len(), them.len()); - let mut i = 0usize; - while i < s { - if self.at(i) != them.at(i) { - break + let self_align = self.offset % nibble_ops::NIBBLE_PER_BYTE; + let them_align = them.offset % nibble_ops::NIBBLE_PER_BYTE; + if self_align == them_align { + let mut self_start = self.offset / nibble_ops::NIBBLE_PER_BYTE; + let mut them_start = them.offset / nibble_ops::NIBBLE_PER_BYTE; + let mut first = 0; + if self_align != 0 { + if nibble_ops::pad_right(self.data[self_start]) != + nibble_ops::pad_right(them.data[them_start]) + { + // warning only for radix 16 + return 0 + } + self_start += 1; + them_start += 1; + first += 1; } - i += 1; + nibble_ops::biggest_depth(&self.data[self_start..], &them.data[them_start..]) + first + } else { + let s = min(self.len(), them.len()); + let mut i = 0usize; + while i < s { + if self.at(i) != them.at(i) { + break + } + i += 1; + } + i } - i } /// Return `Partial` representation of this slice: @@ -227,16 +247,42 @@ impl<'a> NibbleSlice<'a> { } } + /// Get [`Prefix`] representation of the inner data. + /// + /// This means the entire inner data will be returned as [`Prefix`], ignoring any `offset`. + pub fn original_data_as_prefix(&self) -> Prefix { + (&self.data, None) + } + /// Owned version of a `Prefix` from a `left` method call. pub fn left_owned(&'a self) -> (BackingByteVec, Option) { let (a, b) = self.left(); (a.into(), b) } + + /// Same as [`Self::starts_with`] but using [`NibbleVec`]. + pub fn starts_with_vec(&self, other: &NibbleVec) -> bool { + if self.len() < other.len() { + return false + } + + match other.as_nibbleslice() { + Some(other) => self.starts_with(&other), + None => { + for i in 0..other.len() { + if self.at(i) != other.at(i) { + return false + } + } + true + }, + } + } } -impl<'a> Into for NibbleSlice<'a> { - fn into(self) -> NodeKey { - (self.offset, self.data.into()) +impl<'a> From> for NodeKey { + fn from(slice: NibbleSlice<'a>) -> NodeKey { + (slice.offset, slice.data.into()) } } @@ -246,6 +292,19 @@ impl<'a> PartialEq for NibbleSlice<'a> { } } +impl<'a> PartialEq for NibbleSlice<'a> { + fn eq(&self, other: &NibbleVec) -> bool { + if self.len() != other.len() { + return false + } + + match other.as_nibbleslice() { + Some(other) => *self == other, + None => self.iter().enumerate().all(|(index, l)| l == other.at(index)), + } + } +} + impl<'a> Eq for NibbleSlice<'a> {} impl<'a> PartialOrd for NibbleSlice<'a> { diff --git a/trie-db/src/nibble/nibblevec.rs b/trie-db/src/nibble/nibblevec.rs index 9bc39114..f46f919f 100644 --- a/trie-db/src/nibble/nibblevec.rs +++ b/trie-db/src/nibble/nibblevec.rs @@ -17,6 +17,7 @@ use super::NibbleVec; use crate::{ nibble::{nibble_ops, BackingByteVec, NibbleSlice}, + node::NodeKey, node_codec::Partial, }; use hash_db::Prefix; @@ -39,7 +40,7 @@ impl NibbleVec { self.len } - /// Retrurns true if `NibbleVec` has zero length. + /// Returns true if `NibbleVec` has zero length. pub fn is_empty(&self) -> bool { self.len == 0 } @@ -121,6 +122,7 @@ impl NibbleVec { if v.len == 0 { return } + let final_len = self.len + v.len; let offset = self.len % nibble_ops::NIBBLE_PER_BYTE; let final_offset = final_len % nibble_ops::NIBBLE_PER_BYTE; @@ -231,6 +233,34 @@ impl NibbleVec { } true } + + /// Return an iterator over `Partial` bytes representation. + pub fn right_iter<'a>(&'a self) -> impl Iterator + 'a { + let require_padding = self.len % nibble_ops::NIBBLE_PER_BYTE != 0; + let mut ix = 0; + let inner = &self.inner; + + let (left_s, right_s) = nibble_ops::SPLIT_SHIFTS; + + crate::rstd::iter::from_fn(move || { + if require_padding && ix < inner.len() { + if ix == 0 { + ix += 1; + Some(inner[ix - 1] >> nibble_ops::BIT_PER_NIBBLE) + } else { + ix += 1; + + Some(inner[ix - 2] << left_s | inner[ix - 1] >> right_s) + } + } else if ix < inner.len() { + ix += 1; + + Some(inner[ix - 1]) + } else { + None + } + }) + } } impl<'a> From> for NibbleVec { @@ -243,9 +273,20 @@ impl<'a> From> for NibbleVec { } } +impl From<&NibbleVec> for NodeKey { + fn from(nibble: &NibbleVec) -> NodeKey { + if let Some(slice) = nibble.as_nibbleslice() { + slice.into() + } else { + (1, nibble.right_iter().collect()) + } + } +} + #[cfg(test)] mod tests { - use crate::nibble::{nibble_ops, NibbleVec}; + use super::*; + use crate::{nibble::nibble_ops, NibbleSlice}; #[test] fn push_pop() { @@ -304,4 +345,28 @@ mod tests { test_trun(&[1, 2, 3], 3, (&[], 0)); test_trun(&[1, 2, 3], 4, (&[], 0)); } + + #[test] + fn right_iter_works() { + let data = vec![1, 2, 3, 4, 5, 234, 78, 99]; + + let nibble = NibbleSlice::new(&data); + let vec = NibbleVec::from(nibble); + + nibble + .right_iter() + .zip(vec.right_iter()) + .enumerate() + .for_each(|(i, (l, r))| assert_eq!(l, r, "Don't match at {}", i)); + + // Also try with using an offset. + let nibble = NibbleSlice::new_offset(&data, 3); + let vec = NibbleVec::from(nibble); + + nibble + .right_iter() + .zip(vec.right_iter()) + .enumerate() + .for_each(|(i, (l, r))| assert_eq!(l, r, "Don't match at {}", i)); + } } diff --git a/trie-db/src/node.rs b/trie-db/src/node.rs index fe696b60..19ed9162 100644 --- a/trie-db/src/node.rs +++ b/trie-db/src/node.rs @@ -13,13 +13,15 @@ // limitations under the License. use crate::{ - nibble::{self, nibble_ops, NibbleSlice}, + nibble::{self, nibble_ops, NibbleSlice, NibbleVec}, node_codec::NodeCodec, - DBValue, + Bytes, CError, ChildReference, Result, TrieError, TrieHash, TrieLayout, }; +#[cfg(not(feature = "std"))] +use alloc::{boxed::Box, vec::Vec}; use hash_db::Hasher; -use crate::rstd::{borrow::Borrow, ops::Range}; +use crate::rstd::{borrow::Borrow, mem, ops::Range}; /// Partial node key type: offset and owned value of a nibbleslice. /// Offset is applied on first byte of array (bytes are right aligned). @@ -32,6 +34,66 @@ pub enum NodeHandle<'a> { Inline(&'a [u8]), } +impl NodeHandle<'_> { + /// Converts this node handle into a [`NodeHandleOwned`]. + pub fn to_owned_handle( + &self, + ) -> Result>, TrieHash, CError> { + match self { + Self::Hash(h) => decode_hash::(h) + .ok_or_else(|| Box::new(TrieError::InvalidHash(Default::default(), h.to_vec()))) + .map(NodeHandleOwned::Hash), + Self::Inline(i) => match L::Codec::decode(i) { + Ok(node) => Ok(NodeHandleOwned::Inline(Box::new(node.to_owned_node::()?))), + Err(e) => Err(Box::new(TrieError::DecoderError(Default::default(), e))), + }, + } + } +} + +/// Owned version of [`NodeHandleOwned`]. +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum NodeHandleOwned { + Hash(H), + Inline(Box>), +} + +impl NodeHandleOwned +where + H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, +{ + /// Returns `self` as a [`ChildReference`]. + /// + /// # Panic + /// + /// This function panics if `self == Self::Inline(_)` and the inline node encoded length is + /// greater then the length of the hash. + fn as_child_reference>(&self) -> ChildReference { + match self { + NodeHandleOwned::Hash(h) => ChildReference::Hash(*h), + NodeHandleOwned::Inline(n) => { + let encoded = n.to_encoded::(); + let mut store = H::default(); + assert!(store.as_ref().len() > encoded.len(), "Invalid inline node handle"); + + store.as_mut()[..encoded.len()].copy_from_slice(&encoded); + ChildReference::Inline(store, encoded.len()) + }, + } + } +} + +impl NodeHandleOwned { + /// Returns `self` as inline node. + pub fn as_inline(&self) -> Option<&NodeOwned> { + match self { + Self::Hash(_) => None, + Self::Inline(node) => Some(&*node), + } + } +} + /// Read a hash from a slice into a Hasher output. Returns None if the slice is the wrong length. pub fn decode_hash(data: &[u8]) -> Option { if data.len() != H::LENGTH { @@ -48,9 +110,8 @@ pub fn decode_hash(data: &[u8]) -> Option { pub enum Value<'a> { /// Value byte slice as stored in a trie node. Inline(&'a [u8]), - /// Hash byte slice as stored in a trie node, - /// and the actual value when accessed. - Node(&'a [u8], Option), + /// Hash byte slice as stored in a trie node. + Node(&'a [u8]), } impl<'a> Value<'a> { @@ -65,6 +126,56 @@ impl<'a> Value<'a> { Some(Value::Inline(value)) } } + + pub fn to_owned_value(&self) -> ValueOwned> { + match self { + Self::Inline(data) => ValueOwned::Inline(Bytes::from(*data), L::Hash::hash(data)), + Self::Node(hash) => { + let mut res = TrieHash::::default(); + res.as_mut().copy_from_slice(hash); + + ValueOwned::Node(res) + }, + } + } +} + +/// Owned value representation in `Node`. +#[derive(Eq, PartialEq, Clone)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum ValueOwned { + /// Value bytes as stored in a trie node and its hash. + Inline(Bytes, H), + /// Hash byte slice as stored in a trie node. + Node(H), +} + +impl + Copy> ValueOwned { + /// Returns self as [`Value`]. + pub fn as_value(&self) -> Value { + match self { + Self::Inline(data, _) => Value::Inline(&data), + Self::Node(hash) => Value::Node(hash.as_ref()), + } + } + + /// Returns the hash of the data stored in self. + pub fn data_hash(&self) -> Option { + match self { + Self::Inline(_, hash) => Some(*hash), + Self::Node(hash) => Some(*hash), + } + } +} + +impl ValueOwned { + /// Returns the data stored in self. + pub fn data(&self) -> Option<&Bytes> { + match self { + Self::Inline(data, _) => Some(data), + Self::Node(_) => None, + } + } } /// Type of node in the trie and essential information thereof. @@ -88,6 +199,234 @@ pub enum Node<'a> { ), } +impl Node<'_> { + /// Converts this node into a [`NodeOwned`]. + pub fn to_owned_node( + &self, + ) -> Result>, TrieHash, CError> { + match self { + Self::Empty => Ok(NodeOwned::Empty), + Self::Leaf(n, d) => Ok(NodeOwned::Leaf((*n).into(), d.to_owned_value::())), + Self::Extension(n, h) => + Ok(NodeOwned::Extension((*n).into(), h.to_owned_handle::()?)), + Self::Branch(childs, data) => { + let mut childs_owned = [(); nibble_ops::NIBBLE_LENGTH].map(|_| None); + childs + .iter() + .enumerate() + .map(|(i, c)| { + childs_owned[i] = + c.as_ref().map(|c| c.to_owned_handle::()).transpose()?; + Ok(()) + }) + .collect::>()?; + + Ok(NodeOwned::Branch(childs_owned, data.as_ref().map(|d| d.to_owned_value::()))) + }, + Self::NibbledBranch(n, childs, data) => { + let mut childs_owned = [(); nibble_ops::NIBBLE_LENGTH].map(|_| None); + childs + .iter() + .enumerate() + .map(|(i, c)| { + childs_owned[i] = + c.as_ref().map(|c| c.to_owned_handle::()).transpose()?; + Ok(()) + }) + .collect::>()?; + + Ok(NodeOwned::NibbledBranch( + (*n).into(), + childs_owned, + data.as_ref().map(|d| d.to_owned_value::()), + )) + }, + } + } +} + +/// Owned version of [`Node`]. +#[derive(Eq, PartialEq, Clone)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum NodeOwned { + /// Null trie node; could be an empty root or an empty branch entry. + Empty, + /// Leaf node; has key slice and value. Value may not be empty. + Leaf(NibbleVec, ValueOwned), + /// Extension node; has key slice and node data. Data may not be null. + Extension(NibbleVec, NodeHandleOwned), + /// Branch node; has slice of child nodes (each possibly null) + /// and an optional immediate node data. + Branch([Option>; nibble_ops::NIBBLE_LENGTH], Option>), + /// Branch node with support for a nibble (when extension nodes are not used). + NibbledBranch( + NibbleVec, + [Option>; nibble_ops::NIBBLE_LENGTH], + Option>, + ), + /// Node that represents a value. + /// + /// This variant is only constructed when working with a [`crate::TrieCache`]. It is only + /// used to cache a raw value. + Value(Bytes, H), +} + +impl NodeOwned +where + H: Default + AsRef<[u8]> + AsMut<[u8]> + Copy, +{ + /// Convert to its encoded format. + pub fn to_encoded(&self) -> Vec + where + C: NodeCodec, + { + match self { + Self::Empty => C::empty_node().to_vec(), + Self::Leaf(partial, value) => + C::leaf_node(partial.right_iter(), partial.len(), value.as_value()), + Self::Extension(partial, child) => C::extension_node( + partial.right_iter(), + partial.len(), + child.as_child_reference::(), + ), + Self::Branch(children, value) => C::branch_node( + children.iter().map(|child| child.as_ref().map(|c| c.as_child_reference::())), + value.as_ref().map(|v| v.as_value()), + ), + Self::NibbledBranch(partial, children, value) => C::branch_node_nibbled( + partial.right_iter(), + partial.len(), + children.iter().map(|child| child.as_ref().map(|c| c.as_child_reference::())), + value.as_ref().map(|v| v.as_value()), + ), + Self::Value(data, _) => data.to_vec(), + } + } + + /// Returns an iterator over all existing children with their optional nibble. + pub fn child_iter(&self) -> impl Iterator, &NodeHandleOwned)> { + enum ChildIter<'a, H> { + Empty, + Single(&'a NodeHandleOwned, bool), + Array(&'a [Option>; nibble_ops::NIBBLE_LENGTH], usize), + } + + impl<'a, H> Iterator for ChildIter<'a, H> { + type Item = (Option, &'a NodeHandleOwned); + + fn next(&mut self) -> Option { + loop { + match self { + Self::Empty => break None, + Self::Single(child, returned) => + break if *returned { + None + } else { + *returned = true; + Some((None, child)) + }, + Self::Array(childs, index) => + if *index >= childs.len() { + break None + } else { + *index += 1; + + // Ignore non-existing childs. + if let Some(ref child) = childs[*index - 1] { + break Some((Some(*index as u8 - 1), child)) + } + }, + } + } + } + } + + match self { + Self::Leaf(_, _) | Self::Empty | Self::Value(_, _) => ChildIter::Empty, + Self::Extension(_, child) => ChildIter::Single(child, false), + Self::Branch(children, _) | Self::NibbledBranch(_, children, _) => + ChildIter::Array(children, 0), + } + } + + /// Returns the hash of the data attached to this node. + pub fn data_hash(&self) -> Option { + match &self { + Self::Empty => None, + Self::Leaf(_, value) => value.data_hash(), + Self::Extension(_, _) => None, + Self::Branch(_, value) => value.as_ref().and_then(|v| v.data_hash()), + Self::NibbledBranch(_, _, value) => value.as_ref().and_then(|v| v.data_hash()), + Self::Value(_, hash) => Some(*hash), + } + } +} + +impl NodeOwned { + /// Returns the data attached to this node. + pub fn data(&self) -> Option<&Bytes> { + match &self { + Self::Empty => None, + Self::Leaf(_, value) => value.data(), + Self::Extension(_, _) => None, + Self::Branch(_, value) => value.as_ref().and_then(|v| v.data()), + Self::NibbledBranch(_, _, value) => value.as_ref().and_then(|v| v.data()), + Self::Value(data, _) => Some(data), + } + } + + /// Returns the partial key of this node. + pub fn partial_key(&self) -> Option<&NibbleVec> { + match self { + Self::Branch(_, _) | Self::Value(_, _) | Self::Empty => None, + Self::Extension(partial, _) | + Self::Leaf(partial, _) | + Self::NibbledBranch(partial, _, _) => Some(partial), + } + } + + /// Returns the size in bytes of this node in memory. + /// + /// This also includes the size of any inline child nodes. + pub fn size_in_bytes(&self) -> usize { + let self_size = mem::size_of::(); + + fn childs_size<'a, H: 'a>( + childs: impl Iterator>>, + ) -> usize { + // If a `child` isn't an inline node, its size is already taken account for by + // `self_size`. + childs + .filter_map(|c| c.as_ref()) + .map(|c| c.as_inline().map_or(0, |n| n.size_in_bytes())) + .sum() + } + + // As `self_size` only represents the static size of `Self`, we also need + // to add the size of any dynamically allocated data. + let dynamic_size = match self { + Self::Empty => 0, + Self::Leaf(nibbles, value) => + nibbles.inner().len() + value.data().map_or(0, |b| b.len()), + Self::Value(bytes, _) => bytes.len(), + Self::Extension(nibbles, child) => { + // If the `child` isn't an inline node, its size is already taken account for by + // `self_size`. + nibbles.inner().len() + child.as_inline().map_or(0, |n| n.size_in_bytes()) + }, + Self::Branch(childs, value) => + childs_size(childs.iter()) + + value.as_ref().and_then(|v| v.data()).map_or(0, |b| b.len()), + Self::NibbledBranch(nibbles, childs, value) => + nibbles.inner().len() + + childs_size(childs.iter()) + + value.as_ref().and_then(|v| v.data()).map_or(0, |b| b.len()), + }; + + self_size + dynamic_size + } +} + /// A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie /// node. This is used as a substructure of `NodePlan`. See `NodePlan` for details. #[derive(Debug, Clone, PartialEq, Eq)] @@ -152,7 +491,7 @@ impl ValuePlan { pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Value<'b> { match self { ValuePlan::Inline(range) => Value::Inline(&data[range.clone()]), - ValuePlan::Node(range) => Value::Node(&data[range.clone()], None), + ValuePlan::Node(range) => Value::Node(&data[range.clone()]), } } } @@ -251,7 +590,7 @@ pub struct OwnedNode> { impl> OwnedNode { /// Construct an `OwnedNode` by decoding an owned data source according to some codec. - pub fn new(data: D) -> Result { + pub fn new(data: D) -> core::result::Result { let plan = C::decode_plan(data.borrow())?; Ok(OwnedNode { data, plan }) } diff --git a/trie-db/src/node_codec.rs b/trie-db/src/node_codec.rs index ccc41bbc..eb9b1f67 100644 --- a/trie-db/src/node_codec.rs +++ b/trie-db/src/node_codec.rs @@ -70,9 +70,14 @@ pub trait NodeCodec: Sized { fn empty_node() -> &'static [u8]; /// Returns an encoded leaf node - fn leaf_node(partial: Partial, value: Value) -> Vec; + /// + /// Note that number_nibble is the number of element of the iterator + /// it can possibly be obtain by `Iterator` `size_hint`, but + /// for simplicity it is used directly as a parameter. + fn leaf_node(partial: impl Iterator, number_nibble: usize, value: Value) -> Vec; /// Returns an encoded extension node + /// /// Note that number_nibble is the number of element of the iterator /// it can possibly be obtain by `Iterator` `size_hint`, but /// for simplicity it is used directly as a parameter. diff --git a/trie-db/src/proof/generate.rs b/trie-db/src/proof/generate.rs index d9d2108e..2db1eafa 100644 --- a/trie-db/src/proof/generate.rs +++ b/trie-db/src/proof/generate.rs @@ -16,14 +16,15 @@ use crate::rstd::{boxed::Box, convert::TryInto, marker::PhantomData, vec, vec::Vec}; -use hash_db::Hasher; +use hash_db::{HashDBRef, Hasher}; use crate::{ nibble::LeftNibbleSlice, nibble_ops::NIBBLE_LENGTH, node::{NodeHandle, NodeHandlePlan, NodePlan, OwnedNode, Value, ValuePlan}, - CError, ChildReference, NibbleSlice, NodeCodec, Record, Recorder, Result as TrieResult, Trie, - TrieError, TrieHash, TrieLayout, + recorder::Record, + CError, ChildReference, DBValue, NibbleSlice, NodeCodec, Recorder, Result as TrieResult, Trie, + TrieDBBuilder, TrieError, TrieHash, TrieLayout, }; struct StackEntry<'a, C: NodeCodec> { @@ -87,7 +88,7 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { NodePlan::Leaf { .. } if !omit_value => node_data.to_vec(), NodePlan::Leaf { partial, value: _ } => { let partial = partial.build(node_data); - C::leaf_node(partial.right(), Value::Inline(&[])) + C::leaf_node(partial.right_iter(), partial.len(), Value::Inline(&[])) }, NodePlan::Extension { .. } if self.child_index == 0 => node_data.to_vec(), NodePlan::Extension { partial: partial_plan, child: _ } => { @@ -217,17 +218,18 @@ impl<'a, C: NodeCodec> StackEntry<'a, C> { /// Generate a compact proof for key-value pairs in a trie given a set of keys. /// /// Assumes inline nodes have only inline children. -pub fn generate_proof<'a, T, L, I, K>( - trie: &T, +pub fn generate_proof<'a, D, L, I, K>( + db: &D, + root: &TrieHash, keys: I, ) -> TrieResult>, TrieHash, CError> where - T: Trie, + D: HashDBRef, L: TrieLayout, I: IntoIterator, K: 'a + AsRef<[u8]>, { - // Sort and deduplicate keys. + // Sort and de-duplicate keys. let mut keys = keys.into_iter().map(|key| key.as_ref()).collect::>(); keys.sort(); keys.dedup(); @@ -246,9 +248,12 @@ where unwind_stack(&mut stack, &mut proof_nodes, Some(&key))?; // Perform the trie lookup for the next key, recording the sequence of nodes traversed. - let mut recorder = Recorder::new(); - let expected_value = trie.get_with(key_bytes, &mut recorder)?; - // Remember recorded entry is node query order and hashed value after its node. + let mut recorder = Recorder::::new(); + let expected_value = { + let trie = TrieDBBuilder::::new(db, root).with_recorder(&mut recorder).build(); + trie.get(key_bytes)? + }; + let mut recorded_nodes = recorder.drain().into_iter().peekable(); // Skip over recorded nodes already on the stack. Their indexes into the respective vector @@ -280,10 +285,8 @@ where &mut recorded_nodes, )?, // If stack is empty, descend into the root node. - None => Step::Descend { - child_prefix_len: 0, - child: NodeHandle::Hash(trie.root().as_ref()), - }, + None => + Step::Descend { child_prefix_len: 0, child: NodeHandle::Hash(root.as_ref()) }, }; match step { diff --git a/trie-db/src/proof/verify.rs b/trie-db/src/proof/verify.rs index d481264f..e4dcfe34 100644 --- a/trie-db/src/proof/verify.rs +++ b/trie-db/src/proof/verify.rs @@ -134,7 +134,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { fn value(&self) -> Option { if let Some(hash) = self.next_value_hash.as_ref() { - Some(Value::Node(hash.as_ref(), None)) + Some(Value::Node(hash.as_ref())) } else { self.value.clone() } @@ -151,7 +151,7 @@ impl<'a, L: TrieLayout> StackEntry<'a, L> { value is only ever reassigned in the ValueMatch::MatchesLeaf match \ clause, which assigns only to Some", ); - L::Codec::leaf_node(partial.right(), value) + L::Codec::leaf_node(partial.right_iter(), partial.len(), value) }, Node::Extension(partial, _) => { let child = diff --git a/trie-db/src/recorder.rs b/trie-db/src/recorder.rs index 6891ea7c..ba4c77e5 100644 --- a/trie-db/src/recorder.rs +++ b/trie-db/src/recorder.rs @@ -14,56 +14,69 @@ //! Trie query recorder. -use crate::rstd::vec::Vec; +use crate::{rstd::vec::Vec, RecordedForKey, TrieAccess, TrieHash, TrieLayout, TrieRecorder}; +use hashbrown::HashMap; -/// A record of a visited node. +/// The record of a visited node. #[cfg_attr(feature = "std", derive(Debug))] #[derive(PartialEq, Eq, Clone)] pub struct Record { - /// The depth of this node. - pub depth: u32, - - /// The raw data of the node. - pub data: Vec, - - /// The hash of the data. + /// The hash of the node. pub hash: HO, + /// The data representing the node. + pub data: Vec, } /// Records trie nodes as they pass it. #[cfg_attr(feature = "std", derive(Debug))] -pub struct Recorder { - nodes: Vec>, - min_depth: u32, +pub struct Recorder { + nodes: Vec>>, + recorded_keys: HashMap, RecordedForKey>, } -impl Default for Recorder { +impl Default for Recorder { fn default() -> Self { Recorder::new() } } -impl Recorder { +impl Recorder { /// Create a new `Recorder` which records all given nodes. - #[inline] pub fn new() -> Self { - Recorder::with_depth(0) + Self { nodes: Default::default(), recorded_keys: Default::default() } } - /// Create a `Recorder` which only records nodes beyond a given depth. - pub fn with_depth(depth: u32) -> Self { - Recorder { nodes: Vec::new(), min_depth: depth } + /// Drain all visited records. + pub fn drain(&mut self) -> Vec>> { + self.recorded_keys.clear(); + crate::rstd::mem::take(&mut self.nodes) } +} - /// Record a visited node, given its hash, data, and depth. - pub fn record(&mut self, hash: &HO, data: &[u8], depth: u32) { - if depth >= self.min_depth { - self.nodes.push(Record { depth, data: data.into(), hash: *hash }) +impl TrieRecorder> for Recorder { + fn record<'a>(&mut self, access: TrieAccess<'a, TrieHash>) { + match access { + TrieAccess::EncodedNode { hash, encoded_node, .. } => { + self.nodes.push(Record { hash, data: encoded_node.to_vec() }); + }, + TrieAccess::NodeOwned { hash, node_owned, .. } => { + self.nodes.push(Record { hash, data: node_owned.to_encoded::() }); + }, + TrieAccess::Value { hash, value, full_key } => { + self.nodes.push(Record { hash, data: value.to_vec() }); + self.recorded_keys.entry(full_key.to_vec()).insert(RecordedForKey::Value); + }, + TrieAccess::Hash { full_key } => { + self.recorded_keys.entry(full_key.to_vec()).or_insert(RecordedForKey::Hash); + }, + TrieAccess::NonExisting { full_key } => { + // We handle the non existing value/hash like having recorded the value. + self.recorded_keys.entry(full_key.to_vec()).insert(RecordedForKey::Value); + }, } } - /// Drain all visited records. - pub fn drain(&mut self) -> Vec> { - crate::rstd::mem::replace(&mut self.nodes, Vec::new()) + fn trie_nodes_recorded_for_key(&self, key: &[u8]) -> RecordedForKey { + self.recorded_keys.get(key).copied().unwrap_or(RecordedForKey::None) } } diff --git a/trie-db/src/sectriedb.rs b/trie-db/src/sectriedb.rs index 03aa2162..ccfbd971 100644 --- a/trie-db/src/sectriedb.rs +++ b/trie-db/src/sectriedb.rs @@ -12,24 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{ - triedb::TrieDB, CError, Query, Result, Trie, TrieHash, TrieItem, TrieIterator, TrieKeyItem, - TrieLayout, +use crate::{ + rstd::boxed::Box, triedb::TrieDB, CError, DBValue, Query, Result, Trie, TrieDBBuilder, + TrieHash, TrieItem, TrieIterator, TrieKeyItem, TrieLayout, }; -use crate::{rstd::boxed::Box, DBValue}; use hash_db::{HashDBRef, Hasher}; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` trait object. You can use `raw()` to get the backing `TrieDB` object. -pub struct SecTrieDB<'db, L> +pub struct SecTrieDB<'db, 'cache, L> where L: TrieLayout, { - raw: TrieDB<'db, L>, + raw: TrieDB<'db, 'cache, L>, } -impl<'db, L> SecTrieDB<'db, L> +impl<'db, 'cache, L> SecTrieDB<'db, 'cache, L> where L: TrieLayout, { @@ -38,21 +37,21 @@ where /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - SecTrieDB { raw: TrieDB::new(db, root) } + SecTrieDB { raw: TrieDBBuilder::new(db, root).build() } } /// Get a reference to the underlying raw `TrieDB` struct. - pub fn raw(&self) -> &TrieDB { + pub fn raw(&self) -> &TrieDB<'db, 'cache, L> { &self.raw } /// Get a mutable reference to the underlying raw `TrieDB` struct. - pub fn raw_mut(&mut self) -> &mut TrieDB<'db, L> { + pub fn raw_mut(&mut self) -> &mut TrieDB<'db, 'cache, L> { &mut self.raw } } -impl<'db, L> Trie for SecTrieDB<'db, L> +impl<'db, 'cache, L> Trie for SecTrieDB<'db, 'cache, L> where L: TrieLayout, { @@ -64,14 +63,15 @@ where self.raw.contains(L::Hash::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>( - &'a self, - key: &'key [u8], + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { + self.raw.get_hash(key) + } + + fn get_with>( + &self, + key: &[u8], query: Q, - ) -> Result, TrieHash, CError> - where - 'a: 'key, - { + ) -> Result, TrieHash, CError> { self.raw.get_with(L::Hash::hash(key).as_ref(), query) } diff --git a/trie-db/src/sectriedbmut.rs b/trie-db/src/sectriedbmut.rs index 24b1af51..b56dc55a 100644 --- a/trie-db/src/sectriedbmut.rs +++ b/trie-db/src/sectriedbmut.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, Value}; +use crate::{ + triedbmut::TrieDBMutBuilder, CError, DBValue, Result, TrieDBMut, TrieHash, TrieLayout, TrieMut, + Value, +}; use hash_db::{HashDB, Hasher}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -31,10 +34,10 @@ where L: TrieLayout, { /// Create a new trie with the backing database `db` and empty `root` - /// Initialise to the state entailed by the genesis block. + /// Initialize to the state entailed by the genesis block. /// This guarantees the trie is built correctly. pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { - SecTrieDBMut { raw: TrieDBMut::new(db, root) } + SecTrieDBMut { raw: TrieDBMutBuilder::new(db, root).build() } } /// Create a new trie with the backing database `db` and `root`. @@ -42,7 +45,7 @@ where db: &'db mut dyn HashDB, root: &'db mut TrieHash, ) -> Self { - SecTrieDBMut { raw: TrieDBMut::from_existing(db, root) } + SecTrieDBMut { raw: TrieDBMutBuilder::from_existing(db, root).build() } } /// Get the backing database. diff --git a/trie-db/src/trie_codec.rs b/trie-db/src/trie_codec.rs index 988795e2..6c39388e 100644 --- a/trie-db/src/trie_codec.rs +++ b/trie-db/src/trie_codec.rs @@ -173,7 +173,7 @@ impl EncoderStackEntry { } } -/// Dettached value if included does write a reserved header, +/// Detached value if included does write a reserved header, /// followed by node encoded with 0 length value and the value /// as a standalone vec. fn detached_value( @@ -185,8 +185,7 @@ fn detached_value( let fetched; match value { ValuePlan::Node(hash_plan) => { - if let Some(value) = val_fetcher.fetch_value(&node_data[hash_plan.clone()], node_prefix) - { + if let Ok(value) = val_fetcher.fetch_value(&node_data[hash_plan.clone()], node_prefix) { fetched = value; } else { return None @@ -397,11 +396,11 @@ impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> { /// Preconditions: /// - if node is an extension node, then `children[0]` is Some. fn encode_node(self, attached_hash: Option<&[u8]>) -> Vec { - let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h, None)); + let attached_hash = attached_hash.map(|h| crate::node::Value::Node(h)); match self.node { Node::Empty => C::empty_node().to_vec(), Node::Leaf(partial, value) => - C::leaf_node(partial.right(), attached_hash.unwrap_or(value)), + C::leaf_node(partial.right_iter(), partial.len(), attached_hash.unwrap_or(value)), Node::Extension(partial, _) => C::extension_node( partial.right_iter(), partial.len(), diff --git a/trie-db/src/triedb.rs b/trie-db/src/triedb.rs index 57c26202..ad9c3593 100644 --- a/trie-db/src/triedb.rs +++ b/trie-db/src/triedb.rs @@ -13,19 +13,82 @@ // limitations under the License. #[cfg(feature = "std")] -use super::nibble::NibbleVec; -use super::{ +use crate::nibble::NibbleVec; +use crate::{ + iterator::TrieDBNodeIterator, lookup::Lookup, + nibble::NibbleSlice, node::{decode_hash, Node, NodeHandle, OwnedNode, Value}, - CError, Query, Result, Trie, TrieError, TrieHash, TrieItem, TrieIterator, TrieKeyItem, - TrieLayout, + rstd::boxed::Box, + CError, DBValue, Query, Result, Trie, TrieAccess, TrieCache, TrieError, TrieHash, TrieItem, + TrieIterator, TrieKeyItem, TrieLayout, TrieRecorder, }; -use crate::{iterator::TrieDBNodeIterator, nibble::NibbleSlice, rstd::boxed::Box, DBValue}; use hash_db::{HashDBRef, Prefix, EMPTY_PREFIX}; #[cfg(feature = "std")] use crate::rstd::{fmt, vec::Vec}; +/// A builder for creating a [`TrieDB`]. +pub struct TrieDBBuilder<'db, 'cache, L: TrieLayout> { + db: &'db dyn HashDBRef, + root: &'db TrieHash, + cache: Option<&'cache mut dyn TrieCache>, + recorder: Option<&'cache mut dyn TrieRecorder>>, +} + +impl<'db, 'cache, L: TrieLayout> TrieDBBuilder<'db, 'cache, L> { + /// Create a new trie-db builder with the backing database `db` and `root`. + /// + /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail + /// when trying to lookup any key. + pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { + Self { db, root, cache: None, recorder: None } + } + + /// Use the given `cache` for the db. + pub fn with_cache(mut self, cache: &'cache mut dyn TrieCache) -> Self { + self.cache = Some(cache); + self + } + + /// Use the given optional `cache` for the db. + pub fn with_optional_cache<'ocache: 'cache>( + mut self, + cache: Option<&'ocache mut dyn TrieCache>, + ) -> Self { + // Make the compiler happy by "converting" the lifetime + self.cache = cache.map(|c| c as _); + self + } + + /// Use the given `recorder` to record trie accesses. + pub fn with_recorder(mut self, recorder: &'cache mut dyn TrieRecorder>) -> Self { + self.recorder = Some(recorder); + self + } + + /// Use the given optional `recorder` to record trie accesses. + pub fn with_optional_recorder<'recorder: 'cache>( + mut self, + recorder: Option<&'recorder mut dyn TrieRecorder>>, + ) -> Self { + // Make the compiler happy by "converting" the lifetime + self.recorder = recorder.map(|r| r as _); + self + } + + /// Build the [`TrieDB`]. + pub fn build(self) -> TrieDB<'db, 'cache, L> { + TrieDB { + db: self.db, + root: self.root, + cache: self.cache.map(core::cell::RefCell::new), + recorder: self.recorder.map(core::cell::RefCell::new), + hash_count: 0, + } + } +} + /// A `Trie` implementation using a generic `HashDB` backing database, a `Hasher` /// implementation to generate keys and a `NodeCodec` implementation to encode/decode /// the nodes. @@ -48,7 +111,7 @@ use crate::rstd::{fmt, vec::Vec}; /// assert!(t.contains(b"foo").unwrap()); /// assert_eq!(t.get(b"foo").unwrap().unwrap(), b"bar".to_vec()); /// ``` -pub struct TrieDB<'db, L> +pub struct TrieDB<'db, 'cache, L> where L: TrieLayout, { @@ -56,20 +119,14 @@ where root: &'db TrieHash, /// The number of hashes performed so far in operations on this trie. hash_count: usize, + cache: Option>>, + recorder: Option>>>, } -impl<'db, L> TrieDB<'db, L> +impl<'db, 'cache, L> TrieDB<'db, 'cache, L> where L: TrieLayout, { - /// Create a new trie with the backing database `db` and `root`. - /// - /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail - /// when trying to lookup any key. - pub fn new(db: &'db dyn HashDBRef, root: &'db TrieHash) -> Self { - TrieDB { db, root, hash_count: 0 } - } - /// Get the backing database. pub fn db(&'db self) -> &'db dyn HashDBRef { self.db @@ -83,11 +140,15 @@ where /// or None if it was returned raw. /// /// `partial_key` is encoded nibble slice that addresses the node. + /// + /// `record_access` should be set to `true` when the access to the trie should be recorded. + /// However, this will only be done when there is a recorder set. pub(crate) fn get_raw_or_lookup( &self, parent_hash: TrieHash, node_handle: NodeHandle, partial_key: Prefix, + record_access: bool, ) -> Result<(OwnedNode, Option>), TrieHash, CError> { let (node_hash, node_data) = match node_handle { NodeHandle::Hash(data) => { @@ -107,11 +168,47 @@ where }; let owned_node = OwnedNode::new::(node_data) .map_err(|e| Box::new(TrieError::DecoderError(node_hash.unwrap_or(parent_hash), e)))?; + + if record_access { + if let Some((hash, recorder)) = + node_hash.as_ref().and_then(|h| self.recorder.as_ref().map(|r| (h, r))) + { + recorder.borrow_mut().record(TrieAccess::EncodedNode { + hash: *hash, + encoded_node: owned_node.data().into(), + }); + } + } + Ok((owned_node, node_hash)) } + + /// Fetch a value under the given `hash`. + pub(crate) fn fetch_value( + &self, + hash: TrieHash, + prefix: Prefix, + ) -> Result, CError> { + let value = self + .db + .get(&hash, prefix) + .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + + if let Some(recorder) = self.recorder.as_ref() { + debug_assert!(prefix.1.is_none(), "A value has never a partial key; qed"); + + recorder.borrow_mut().record(TrieAccess::Value { + hash, + value: value.as_slice().into(), + full_key: prefix.0, + }); + } + + Ok(value) + } } -impl<'db, L> Trie for TrieDB<'db, L> +impl<'db, 'cache, L> Trie for TrieDB<'db, 'cache, L> where L: TrieLayout, { @@ -119,15 +216,36 @@ where self.root } - fn get_with<'a, 'key, Q: Query>( - &'a self, - key: &'key [u8], + fn get_hash(&self, key: &[u8]) -> Result>, TrieHash, CError> { + let mut cache = self.cache.as_ref().map(|c| c.borrow_mut()); + let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); + + Lookup:: { + db: self.db, + query: |_: &[u8]| (), + hash: *self.root, + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder>), + } + .look_up_hash(key, NibbleSlice::new(key)) + } + + fn get_with>( + &self, + key: &[u8], query: Q, - ) -> Result, TrieHash, CError> - where - 'a: 'key, - { - Lookup:: { db: self.db, query, hash: *self.root }.look_up(NibbleSlice::new(key)) + ) -> Result, TrieHash, CError> { + let mut cache = self.cache.as_ref().map(|c| c.borrow_mut()); + let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); + + Lookup:: { + db: self.db, + query, + hash: *self.root, + cache: cache.as_mut().map(|c| &mut ***c as &mut dyn TrieCache), + recorder: recorder.as_mut().map(|r| &mut ***r as &mut dyn TrieRecorder>), + } + .look_up(key, NibbleSlice::new(key)) } fn iter<'a>( @@ -151,20 +269,20 @@ where } } -#[cfg(feature = "std")] // This is for pretty debug output only -struct TrieAwareDebugNode<'db, 'a, L> +#[cfg(feature = "std")] +struct TrieAwareDebugNode<'db, 'cache, 'a, L> where L: TrieLayout, { - trie: &'db TrieDB<'db, L>, + trie: &'db TrieDB<'db, 'cache, L>, node_key: NodeHandle<'a>, partial_key: NibbleVec, index: Option, } #[cfg(feature = "std")] -impl<'db, 'a, L> fmt::Debug for TrieAwareDebugNode<'db, 'a, L> +impl<'db, 'cache, 'a, L> fmt::Debug for TrieAwareDebugNode<'db, 'cache, 'a, L> where L: TrieLayout, { @@ -173,6 +291,7 @@ where >::default(), self.node_key, self.partial_key.as_prefix(), + false, ) { Ok((owned_node, _node_hash)) => match owned_node.node() { Node::Leaf(slice, value) => { @@ -260,7 +379,7 @@ where } #[cfg(feature = "std")] -impl<'db, L> fmt::Debug for TrieDB<'db, L> +impl<'db, 'cache, L> fmt::Debug for TrieDB<'db, 'cache, L> where L: TrieLayout, { @@ -281,13 +400,13 @@ where } /// Iterator for going through all values in the trie in pre-order traversal order. -pub struct TrieDBIterator<'a, L: TrieLayout> { - inner: TrieDBNodeIterator<'a, L>, +pub struct TrieDBIterator<'a, 'cache, L: TrieLayout> { + inner: TrieDBNodeIterator<'a, 'cache, L>, } /// Iterator for going through all of key with values in the trie in pre-order traversal order. -pub struct TrieDBKeyIterator<'a, L: TrieLayout> { - inner: TrieDBNodeIterator<'a, L>, +pub struct TrieDBKeyIterator<'a, 'cache, L: TrieLayout> { + inner: TrieDBNodeIterator<'a, 'cache, L>, } /// When there is guaranties the storage backend do not change, @@ -298,23 +417,26 @@ pub struct SuspendedTrieDBKeyIterator { impl SuspendedTrieDBKeyIterator { /// Restore iterator. - pub fn unsafe_restore<'a>(self, db: &'a TrieDB<'a, L>) -> TrieDBKeyIterator<'a, L> { + pub fn unsafe_restore<'a, 'cache>( + self, + db: &'a TrieDB<'a, 'cache, L>, + ) -> TrieDBKeyIterator<'a, 'cache, L> { TrieDBKeyIterator { inner: self.inner.unsafe_restore(db) } } } -impl<'a, L: TrieLayout> TrieDBIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieDBIterator<'a, 'cache, L> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> Result, TrieHash, CError> { + pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { let inner = TrieDBNodeIterator::new(db)?; Ok(TrieDBIterator { inner }) } /// Create a new iterator, but limited to a given prefix. pub fn new_prefixed( - db: &'a TrieDB, + db: &'a TrieDB<'a, 'cache, L>, prefix: &[u8], - ) -> Result, TrieHash, CError> { + ) -> Result, CError> { let mut inner = TrieDBNodeIterator::new(db)?; inner.prefix(prefix)?; @@ -325,10 +447,10 @@ impl<'a, L: TrieLayout> TrieDBIterator<'a, L> { /// It then do a seek operation from prefixed context (using `seek` lose /// prefix context by default). pub fn new_prefixed_then_seek( - db: &'a TrieDB, + db: &'a TrieDB<'a, 'cache, L>, prefix: &[u8], start_at: &[u8], - ) -> Result, TrieHash, CError> { + ) -> Result, CError> { let mut inner = TrieDBNodeIterator::new(db)?; inner.prefix_then_seek(prefix, start_at)?; @@ -336,16 +458,16 @@ impl<'a, L: TrieLayout> TrieDBIterator<'a, L> { } } -impl<'a, L: TrieLayout> TrieIterator for TrieDBIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBIterator<'a, 'cache, L> { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { TrieIterator::seek(&mut self.inner, key) } } -impl<'a, L: TrieLayout> TrieDBKeyIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieDBKeyIterator<'a, 'cache, L> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> Result, TrieHash, CError> { + pub fn new(db: &'a TrieDB<'a, 'cache, L>) -> Result, CError> { let inner = TrieDBNodeIterator::new(db)?; Ok(TrieDBKeyIterator { inner }) } @@ -358,9 +480,9 @@ impl<'a, L: TrieLayout> TrieDBKeyIterator<'a, L> { /// Create a new iterator, but limited to a given prefix. pub fn new_prefixed( - db: &'a TrieDB, + db: &'a TrieDB<'a, 'cache, L>, prefix: &[u8], - ) -> Result, TrieHash, CError> { + ) -> Result, TrieHash, CError> { let mut inner = TrieDBNodeIterator::new(db)?; inner.prefix(prefix)?; @@ -371,10 +493,10 @@ impl<'a, L: TrieLayout> TrieDBKeyIterator<'a, L> { /// It then do a seek operation from prefixed context (using `seek` lose /// prefix context by default). pub fn new_prefixed_then_seek( - db: &'a TrieDB, + db: &'a TrieDB<'a, 'cache, L>, prefix: &[u8], start_at: &[u8], - ) -> Result, TrieHash, CError> { + ) -> Result, TrieHash, CError> { let mut inner = TrieDBNodeIterator::new(db)?; inner.prefix_then_seek(prefix, start_at)?; @@ -382,15 +504,15 @@ impl<'a, L: TrieLayout> TrieDBKeyIterator<'a, L> { } } -impl<'a, L: TrieLayout> TrieIterator for TrieDBKeyIterator<'a, L> { +impl<'a, 'cache, L: TrieLayout> TrieIterator for TrieDBKeyIterator<'a, 'cache, L> { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> Result<(), TrieHash, CError> { TrieIterator::seek(&mut self.inner, key) } } -impl<'a, L: TrieLayout> Iterator for TrieDBIterator<'a, L> { - type Item = TrieItem<'a, TrieHash, CError>; +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBIterator<'a, 'cache, L> { + type Item = TrieItem, CError>; fn next(&mut self) -> Option { while let Some(item) = self.inner.next() { @@ -420,17 +542,13 @@ impl<'a, L: TrieLayout> Iterator for TrieDBIterator<'a, L> { )))) } let value = match maybe_value.expect("None checked above.") { - Value::Node(hash, None) => { - if let Some(value) = self.inner.fetch_value(&hash, (key_slice, None)) { - value - } else { - let mut res = TrieHash::::default(); - res.as_mut().copy_from_slice(hash); - return Some(Err(Box::new(TrieError::IncompleteDatabase(res)))) + Value::Node(hash) => { + match self.inner.fetch_value(&hash, (key_slice, None)) { + Ok(value) => value, + Err(err) => return Some(Err(err)), } }, Value::Inline(value) => value.to_vec(), - Value::Node(_hash, Some(value)) => value, }; return Some(Ok((key, value))) }, @@ -441,8 +559,8 @@ impl<'a, L: TrieLayout> Iterator for TrieDBIterator<'a, L> { } } -impl<'a, L: TrieLayout> Iterator for TrieDBKeyIterator<'a, L> { - type Item = TrieKeyItem<'a, TrieHash, CError>; +impl<'a, 'cache, L: TrieLayout> Iterator for TrieDBKeyIterator<'a, 'cache, L> { + type Item = TrieKeyItem, CError>; fn next(&mut self) -> Option { while let Some(item) = self.inner.next() { diff --git a/trie-db/src/triedbmut.rs b/trie-db/src/triedbmut.rs index 08df4145..bf3edbcb 100644 --- a/trie-db/src/triedbmut.rs +++ b/trie-db/src/triedbmut.rs @@ -14,24 +14,22 @@ //! In-memory trie representation. -use super::{ +use crate::{ lookup::Lookup, + nibble::{nibble_ops, BackingByteVec, NibbleSlice, NibbleVec}, node::{ - decode_hash, Node as EncodedNode, NodeHandle as EncodedNodeHandle, NodeKey, - Value as EncodedValue, + decode_hash, Node as EncodedNode, NodeHandle as EncodedNodeHandle, NodeHandleOwned, + NodeKey, NodeOwned, Value as EncodedValue, ValueOwned, }, - CError, DBValue, Result, TrieError, TrieHash, TrieLayout, TrieMut, + node_codec::NodeCodec, + rstd::{boxed::Box, convert::TryFrom, mem, ops::Index, result, vec::Vec, VecDeque}, + Bytes, CError, CachedValue, DBValue, Result, TrieAccess, TrieCache, TrieError, TrieHash, + TrieLayout, TrieMut, TrieRecorder, }; use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use hashbrown::HashSet; -use crate::{ - nibble::{nibble_ops, BackingByteVec, NibbleSlice, NibbleVec}, - node_codec::NodeCodec, - rstd::{boxed::Box, convert::TryFrom, mem, ops::Index, result, vec::Vec, VecDeque}, -}; - #[cfg(feature = "std")] use log::trace; @@ -73,20 +71,20 @@ type NibbleFullKey<'key> = NibbleSlice<'key>; #[derive(Clone, Eq)] pub enum Value { /// Value bytes inlined in a trie node. - Inline(DBValue), - /// Hash of value bytes and value bytes when accessed. - Node(TrieHash, Option), + Inline(Bytes), + /// Hash of the value. + Node(TrieHash), /// Hash of value bytes if calculated and value bytes. /// The hash may be undefined until it node is added /// to the db. - NewNode(Option>, DBValue), + NewNode(Option>, Bytes), } impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { match (self, other) { (Value::Inline(v), Value::Inline(ov)) => v == ov, - (Value::Node(h, _), Value::Node(oh, _)) => h == oh, + (Value::Node(h), Value::Node(oh)) => h == oh, (Value::NewNode(Some(h), _), Value::NewNode(Some(oh), _)) => h == oh, (Value::NewNode(_, v), Value::NewNode(_, ov)) => v == ov, // Note that for uncalculated hash we do not calculate it and default to true. @@ -99,24 +97,33 @@ impl PartialEq for Value { impl<'a, L: TrieLayout> From> for Value { fn from(v: EncodedValue<'a>) -> Self { match v { - EncodedValue::Inline(value) => Value::Inline(value.to_vec()), - EncodedValue::Node(hash, value) => { + EncodedValue::Inline(value) => Value::Inline(value.into()), + EncodedValue::Node(hash) => { let mut h = TrieHash::::default(); h.as_mut().copy_from_slice(hash); - Value::Node(h, value) + Value::Node(h) }, } } } -impl From<(DBValue, Option)> for Value { - fn from((v, threshold): (DBValue, Option)) -> Self { +impl From<&ValueOwned>> for Value { + fn from(val: &ValueOwned>) -> Self { + match val { + ValueOwned::Inline(data, _) => Self::Inline(data.clone()), + ValueOwned::Node(hash) => Self::Node(*hash), + } + } +} + +impl From<(Bytes, Option)> for Value { + fn from((v, threshold): (Bytes, Option)) -> Self { match v { value => if threshold.map(|threshold| value.len() >= threshold as usize).unwrap_or(false) { - Value::NewNode(None, value.to_vec()) + Value::NewNode(None, value) } else { - Value::Inline(value.to_vec()) + Value::Inline(value) }, } } @@ -128,7 +135,7 @@ enum NodeToEncode<'a, H> { } impl Value { - fn new(value: DBValue, new_threshold: Option) -> Self { + fn new(value: Bytes, new_threshold: Option) -> Self { (value, new_threshold).into() } @@ -145,13 +152,12 @@ impl Value { ) -> ChildReference>, { if let Value::NewNode(hash, value) = self { - let new_hash = if let ChildReference::Hash(hash) = - f(NodeToEncode::Node(value.as_slice()), partial, None) - { - hash - } else { - unreachable!("Value node can never be inlined; qed") - }; + let new_hash = + if let ChildReference::Hash(hash) = f(NodeToEncode::Node(&value), partial, None) { + hash + } else { + unreachable!("Value node can never be inlined; qed") + }; if let Some(h) = hash.as_ref() { debug_assert!(h == &new_hash); } else { @@ -159,9 +165,9 @@ impl Value { } } let value = match &*self { - Value::Inline(value) => EncodedValue::Inline(value.as_slice()), - Value::Node(hash, _value) => EncodedValue::Node(hash.as_ref(), None), - Value::NewNode(Some(hash), _value) => EncodedValue::Node(hash.as_ref(), None), + Value::Inline(value) => EncodedValue::Inline(&value), + Value::Node(hash) => EncodedValue::Node(hash.as_ref()), + Value::NewNode(Some(hash), _value) => EncodedValue::Node(hash.as_ref()), Value::NewNode(None, _value) => unreachable!("New external value are always added before encoding anode"), }; @@ -172,13 +178,22 @@ impl Value { &self, prefix: Prefix, db: &dyn HashDB, + recorder: &Option>>>, + full_key: &[u8], ) -> Result, TrieHash, CError> { Ok(Some(match self { - Value::Inline(value) => value.clone(), - Value::NewNode(_, value) => value.clone(), - Value::Node(_, Some(value)) => value.clone(), - Value::Node(hash, None) => + Value::Inline(value) => value.to_vec(), + Value::NewNode(_, value) => value.to_vec(), + Value::Node(hash) => if let Some(value) = db.get(hash, prefix) { + recorder.as_ref().map(|r| { + r.borrow_mut().record(TrieAccess::Value { + hash: *hash, + value: value.as_slice().into(), + full_key, + }) + }); + value } else { return Err(Box::new(TrieError::IncompleteDatabase(hash.clone()))) @@ -228,7 +243,7 @@ impl Debug for Value { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self { Self::Inline(value) => write!(fmt, "Some({:?})", ToHex(value)), - Self::Node(hash, _) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), + Self::Node(hash) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), Self::NewNode(Some(hash), _) => write!(fmt, "Hash({:?})", ToHex(hash.as_ref())), Self::NewNode(_hash, value) => write!(fmt, "Some({:?})", ToHex(value)), } @@ -259,7 +274,6 @@ impl Node { fn inline_or_hash( parent_hash: TrieHash, child: EncodedNodeHandle, - db: &dyn HashDB, storage: &mut NodeStorage, ) -> Result>, TrieHash, CError> { let handle = match child { @@ -269,18 +283,31 @@ impl Node { NodeHandle::Hash(hash) }, EncodedNodeHandle::Inline(data) => { - let child = Node::from_encoded(parent_hash, data, db, storage)?; + let child = Node::from_encoded(parent_hash, data, storage)?; NodeHandle::InMemory(storage.alloc(Stored::New(child))) }, }; Ok(handle) } + // load an inline node into memory or get the hash to do the lookup later. + fn inline_or_hash_owned( + child: &NodeHandleOwned>, + storage: &mut NodeStorage, + ) -> NodeHandle> { + match child { + NodeHandleOwned::Hash(hash) => NodeHandle::Hash(*hash), + NodeHandleOwned::Inline(node) => { + let child = Node::from_node_owned(&**node, storage); + NodeHandle::InMemory(storage.alloc(Stored::New(child))) + }, + } + } + // Decode a node from encoded bytes. fn from_encoded<'a, 'b>( node_hash: TrieHash, data: &'a [u8], - db: &dyn HashDB, storage: &'b mut NodeStorage, ) -> Result, CError> { let encoded_node = @@ -289,10 +316,10 @@ impl Node { EncodedNode::Empty => Node::Empty, EncodedNode::Leaf(k, v) => Node::Leaf(k.into(), v.into()), EncodedNode::Extension(key, cb) => - Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, db, storage)?), + Node::Extension(key.into(), Self::inline_or_hash(node_hash, cb, storage)?), EncodedNode::Branch(encoded_children, val) => { let mut child = |i: usize| match encoded_children[i] { - Some(child) => Self::inline_or_hash(node_hash, child, db, storage).map(Some), + Some(child) => Self::inline_or_hash(node_hash, child, storage).map(Some), None => Ok(None), }; @@ -319,7 +346,7 @@ impl Node { }, EncodedNode::NibbledBranch(k, encoded_children, val) => { let mut child = |i: usize| match encoded_children[i] { - Some(child) => Self::inline_or_hash(node_hash, child, db, storage).map(Some), + Some(child) => Self::inline_or_hash(node_hash, child, storage).map(Some), None => Ok(None), }; @@ -348,6 +375,74 @@ impl Node { Ok(node) } + /// Decode a node from a [`NodeOwned`]. + fn from_node_owned(node_owned: &NodeOwned>, storage: &mut NodeStorage) -> Self { + match node_owned { + NodeOwned::Empty => Node::Empty, + NodeOwned::Leaf(k, v) => Node::Leaf(k.into(), v.into()), + NodeOwned::Extension(key, cb) => + Node::Extension(key.into(), Self::inline_or_hash_owned(cb, storage)), + NodeOwned::Branch(encoded_children, val) => { + let mut child = |i: usize| { + encoded_children[i] + .as_ref() + .map(|child| Self::inline_or_hash_owned(child, storage)) + }; + + let children = Box::new([ + child(0), + child(1), + child(2), + child(3), + child(4), + child(5), + child(6), + child(7), + child(8), + child(9), + child(10), + child(11), + child(12), + child(13), + child(14), + child(15), + ]); + + Node::Branch(children, val.as_ref().map(Into::into)) + }, + NodeOwned::NibbledBranch(k, encoded_children, val) => { + let mut child = |i: usize| { + encoded_children[i] + .as_ref() + .map(|child| Self::inline_or_hash_owned(child, storage)) + }; + + let children = Box::new([ + child(0), + child(1), + child(2), + child(3), + child(4), + child(5), + child(6), + child(7), + child(8), + child(9), + child(10), + child(11), + child(12), + child(13), + child(14), + child(15), + ]); + + Node::NibbledBranch(k.into(), children, val.as_ref().map(Into::into)) + }, + NodeOwned::Value(_, _) => + unreachable!("`NodeOwned::Value` can only be returned for the hash of a value."), + } + } + // TODO: parallelize /// Here `child_cb` should process the first parameter to either insert an external /// node value or to encode and add a new branch child node. @@ -364,7 +459,7 @@ impl Node { Node::Leaf(partial, mut value) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); let value = value.into_encoded::(Some(&pr), &mut child_cb); - L::Codec::leaf_node(pr.right(), value) + L::Codec::leaf_node(pr.right_iter(), pr.len(), value) }, Node::Extension(partial, child) => { let pr = NibbleSlice::new_offset(&partial.1[..], partial.0); @@ -404,6 +499,17 @@ impl Node { }, } } + + /// Returns the key partial key of this node. + fn partial_key(&self) -> Option<&NodeKey> { + match &self { + Self::Empty => None, + Self::Leaf(key, _) => Some(key), + Self::Branch(_, _) => None, + Self::NibbledBranch(key, _, _) => Some(key), + Self::Extension(key, _) => Some(key), + } + } } // post-inspect action. @@ -528,6 +634,83 @@ impl<'a, L: TrieLayout> Index<&'a StorageHandle> for NodeStorage { } } +/// A builder for creating a [`TrieDBMut`]. +pub struct TrieDBMutBuilder<'db, L: TrieLayout> { + db: &'db mut dyn HashDB, + root: &'db mut TrieHash, + cache: Option<&'db mut dyn TrieCache>, + recorder: Option<&'db mut dyn TrieRecorder>>, +} + +impl<'db, L: TrieLayout> TrieDBMutBuilder<'db, L> { + /// Create a builder for constructing a new trie with the backing database `db` and empty + /// `root`. + pub fn new(db: &'db mut dyn HashDB, root: &'db mut TrieHash) -> Self { + *root = L::Codec::hashed_null_node(); + + Self { root, db, cache: None, recorder: None } + } + + /// Create a builder for constructing a new trie with the backing database `db` and `root`. + /// + /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail + /// when trying to lookup any key. + pub fn from_existing( + db: &'db mut dyn HashDB, + root: &'db mut TrieHash, + ) -> Self { + Self { db, root, cache: None, recorder: None } + } + + /// Use the given `cache` for the db. + pub fn with_cache(mut self, cache: &'db mut dyn TrieCache) -> Self { + self.cache = Some(cache); + self + } + + /// Use the given optional `cache` for the db. + pub fn with_optional_cache<'cache: 'db>( + mut self, + cache: Option<&'cache mut dyn TrieCache>, + ) -> Self { + // Make the compiler happy by "converting" the lifetime + self.cache = cache.map(|c| c as _); + self + } + + /// Use the given `recorder` to record trie accesses. + pub fn with_recorder(mut self, recorder: &'db mut dyn TrieRecorder>) -> Self { + self.recorder = Some(recorder); + self + } + + /// Use the given optional `recorder` to record trie accesses. + pub fn with_optional_recorder<'recorder: 'db>( + mut self, + recorder: Option<&'recorder mut dyn TrieRecorder>>, + ) -> Self { + // Make the compiler happy by "converting" the lifetime + self.recorder = recorder.map(|r| r as _); + self + } + + /// Build the [`TrieDBMut`]. + pub fn build(self) -> TrieDBMut<'db, L> { + let root_handle = NodeHandle::Hash(*self.root); + + TrieDBMut { + db: self.db, + root: self.root, + cache: self.cache, + recorder: self.recorder.map(core::cell::RefCell::new), + hash_count: 0, + storage: NodeStorage::empty(), + death_row: Default::default(), + root_handle, + } + } +} + /// A `Trie` implementation using a generic `HashDB` backing database. /// /// Use it as a `TrieMut` trait object. You can use `db()` to get the backing database object. @@ -567,46 +750,16 @@ where /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, + /// Optional cache for speeding up the lookup of nodes. + cache: Option<&'a mut dyn TrieCache>, + /// Optional trie recorder for recording trie accesses. + recorder: Option>>>, } impl<'a, L> TrieDBMut<'a, L> where L: TrieLayout, { - /// Create a new trie with backing database `db` and empty `root`. - pub fn new(db: &'a mut dyn HashDB, root: &'a mut TrieHash) -> Self { - *root = L::Codec::hashed_null_node(); - let root_handle = NodeHandle::Hash(L::Codec::hashed_null_node()); - - TrieDBMut { - storage: NodeStorage::empty(), - db, - root, - root_handle, - death_row: HashSet::new(), - hash_count: 0, - } - } - - /// Create a new trie with the backing database `db` and `root`. - /// - /// This doesn't check if `root` exists in the given `db`. If `root` doesn't exist it will fail - /// when trying to lookup any key. - pub fn from_existing( - db: &'a mut dyn HashDB, - root: &'a mut TrieHash, - ) -> Self { - let root_handle = NodeHandle::Hash(*root); - TrieDBMut { - storage: NodeStorage::empty(), - db, - root, - root_handle, - death_row: HashSet::new(), - hash_count: 0, - } - } - /// Get the backing database. pub fn db(&self) -> &dyn HashDB { self.db @@ -623,11 +776,35 @@ where hash: TrieHash, key: Prefix, ) -> Result, CError> { - let node_encoded = self - .db - .get(&hash, key) - .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - let node = Node::from_encoded(hash, &node_encoded, &*self.db, &mut self.storage)?; + // We only check the `cache` for a node with `get_node` and don't insert + // the node if it wasn't there, because in substrate we only access the node while computing + // a new trie (aka some branch). We assume that this node isn't that important + // to have it being cached. + let node = match self.cache.as_mut().and_then(|c| c.get_node(&hash)) { + Some(node) => { + if let Some(recorder) = self.recorder.as_mut() { + recorder.borrow_mut().record(TrieAccess::NodeOwned { hash, node_owned: &node }); + } + + Node::from_node_owned(&node, &mut self.storage) + }, + None => { + let node_encoded = self + .db + .get(&hash, key) + .ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + + if let Some(recorder) = self.recorder.as_mut() { + recorder.borrow_mut().record(TrieAccess::EncodedNode { + hash, + encoded_node: node_encoded.as_slice().into(), + }); + } + + Node::from_encoded(hash, &node_encoded, &mut self.storage)? + }, + }; + Ok(self.storage.alloc(Stored::Cached(node, hash))) } @@ -668,31 +845,40 @@ where } // Walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>( - &'x self, - mut partial: NibbleSlice<'key>, - full_key: &'key [u8], + fn lookup( + &self, + full_key: &[u8], + mut partial: NibbleSlice, handle: &NodeHandle>, - ) -> Result, TrieHash, CError> - where - 'x: 'key, - { + ) -> Result, TrieHash, CError> { let mut handle = handle; let prefix = (full_key, None); loop { let (mid, child) = match handle { - NodeHandle::Hash(hash) => + NodeHandle::Hash(hash) => { + let mut recorder = self.recorder.as_ref().map(|r| r.borrow_mut()); + return Lookup:: { db: &self.db, query: |v: &[u8]| v.to_vec(), hash: *hash, + cache: None, + recorder: recorder + .as_mut() + .map(|r| &mut ***r as &mut dyn TrieRecorder>), } - .look_up(partial), + .look_up(full_key, partial) + }, NodeHandle::InMemory(handle) => match &self.storage[handle] { Node::Empty => return Ok(None), Node::Leaf(key, value) => if NibbleSlice::from_stored(key) == partial { - return Ok(value.in_memory_fetched_value(prefix, self.db)?) + return Ok(value.in_memory_fetched_value( + prefix, + self.db, + &self.recorder, + full_key, + )?) } else { return Ok(None) }, @@ -707,7 +893,12 @@ where Node::Branch(children, value) => if partial.is_empty() { return Ok(if let Some(v) = value.as_ref() { - v.in_memory_fetched_value(prefix, self.db)? + v.in_memory_fetched_value( + prefix, + self.db, + &self.recorder, + full_key, + )? } else { None }) @@ -722,7 +913,12 @@ where let slice = NibbleSlice::from_stored(slice); if slice == partial { return Ok(if let Some(v) = value.as_ref() { - v.in_memory_fetched_value(prefix, self.db)? + v.in_memory_fetched_value( + prefix, + self.db, + &self.recorder, + full_key, + )? } else { None }) @@ -749,7 +945,7 @@ where &mut self, handle: NodeHandle>, key: &mut NibbleFullKey, - value: DBValue, + value: Bytes, old_val: &mut Option>, ) -> Result<(StorageHandle, bool), TrieHash, CError> { let h = match handle { @@ -775,7 +971,7 @@ where ) { match &stored_value { Some(Value::NewNode(Some(hash), _)) // also removing new node in case commit is called multiple times - | Some(Value::Node(hash, _)) => { + | Some(Value::Node(hash)) => { self.death_row.insert(( hash.clone(), (prefix.0.into(), prefix.1), @@ -791,7 +987,7 @@ where &mut self, node: Node, key: &mut NibbleFullKey, - value: DBValue, + value: Bytes, old_val: &mut Option>, ) -> Result, TrieHash, CError> { let partial = *key; @@ -818,9 +1014,10 @@ where self.replace_old_value(old_val, stored_value, key.left()); - match unchanged { - true => InsertAction::Restore(branch), - false => InsertAction::Replace(branch), + if unchanged { + InsertAction::Restore(branch) + } else { + InsertAction::Replace(branch) } } else { let idx = partial.at(0) as usize; @@ -861,9 +1058,10 @@ where key_val.advance(existing_key.len()); self.replace_old_value(old_val, stored_value, key_val.left()); - match unchanged { - true => InsertAction::Restore(branch), - false => InsertAction::Replace(branch), + if unchanged { + InsertAction::Restore(branch) + } else { + InsertAction::Replace(branch) } } else if common < existing_key.len() { // insert a branch value in between @@ -951,10 +1149,11 @@ where let mut key_val = key.clone(); key_val.advance(existing_key.len()); self.replace_old_value(old_val, Some(stored_value), key_val.left()); - match unchanged { + if unchanged { // unchanged. restore - true => InsertAction::Restore(Node::Leaf(encoded.clone(), value)), - false => InsertAction::Replace(Node::Leaf(encoded.clone(), value)), + InsertAction::Restore(Node::Leaf(encoded.clone(), value)) + } else { + InsertAction::Replace(Node::Leaf(encoded.clone(), value)) } } else if (L::USE_EXTENSION && common == 0) || (!L::USE_EXTENSION && common < existing_key.len()) @@ -1097,9 +1296,10 @@ where let new_ext = Node::Extension(existing_key.to_stored(), new_child.into()); // if the child branch wasn't changed, meaning this extension remains the same. - match changed { - true => InsertAction::Replace(new_ext), - false => InsertAction::Restore(new_ext), + if changed { + InsertAction::Replace(new_ext) + } else { + InsertAction::Restore(new_ext) } } else { #[cfg(feature = "std")] @@ -1371,8 +1571,9 @@ where } match (used_index, value) { - (UsedIndex::None, None) => - panic!("Branch with no subvalues. Something went wrong."), + (UsedIndex::None, None) => { + panic!("Branch with no subvalues. Something went wrong.") + }, (UsedIndex::One(a), None) => { // only one onward node. make an extension. @@ -1418,8 +1619,9 @@ where } match (used_index, value) { - (UsedIndex::None, None) => - panic!("Branch with no subvalues. Something went wrong."), + (UsedIndex::None, None) => { + panic!("Branch with no subvalues. Something went wrong.") + }, (UsedIndex::One(a), None) => { // only one onward node. use child instead let child = children[a as usize] @@ -1616,6 +1818,11 @@ where match self.storage.destroy(handle) { Stored::New(node) => { + // Reconstructs the full key for root node. + let full_key = self.cache.as_ref().and_then(|_| { + node.partial_key().and_then(|k| Some(NibbleSlice::from_stored(k).into())) + }); + let mut k = NibbleVec::new(); let encoded_root = node.into_encoded(|node, o_slice, o_index| { @@ -1623,6 +1830,7 @@ where match node { NodeToEncode::Node(value) => { let value_hash = self.db.insert(k.as_prefix(), value); + self.cache_value(k.inner(), value, value_hash); k.drop_lasts(mov); ChildReference::Hash(value_hash) }, @@ -1636,9 +1844,11 @@ where #[cfg(feature = "std")] trace!(target: "trie", "encoded root node: {:?}", ToHex(&encoded_root[..])); - *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root[..]); + *self.root = self.db.insert(EMPTY_PREFIX, &encoded_root); self.hash_count += 1; + self.cache_node(*self.root, &encoded_root, full_key); + self.root_handle = NodeHandle::Hash(*self.root); }, Stored::Cached(node, hash) => { @@ -1650,6 +1860,85 @@ where } } + /// Cache the given `encoded` node. + fn cache_node(&mut self, hash: TrieHash, encoded: &[u8], full_key: Option) { + // If we have a cache, cache our node directly. + if let Some(cache) = self.cache.as_mut() { + let node = cache.get_or_insert_node(hash, &mut || { + Ok(L::Codec::decode(&encoded) + .ok() + .and_then(|n| n.to_owned_node::().ok()) + .expect("Just encoded the node, so it should decode without any errors; qed")) + }); + + // `node` should always be `OK`, but let's play it safe. + let node = if let Ok(node) = node { node } else { return }; + + let mut values_to_cache = Vec::new(); + + // If the given node has data attached, the `full_key` is the full key to this node. + if let Some(full_key) = full_key { + node.data().and_then(|v| node.data_hash().map(|h| (&full_key, v, h))).map( + |(k, v, h)| { + values_to_cache.push((k.inner().to_vec(), (v.clone(), h).into())); + }, + ); + + fn cache_child_values( + node: &NodeOwned>, + values_to_cache: &mut Vec<(Vec, CachedValue>)>, + full_key: NibbleVec, + ) { + node.child_iter().flat_map(|(n, c)| c.as_inline().map(|c| (n, c))).for_each( + |(n, c)| { + let mut key = full_key.clone(); + n.map(|n| key.push(n)); + c.partial_key().map(|p| key.append(p)); + + if let Some((hash, data)) = + c.data().and_then(|d| c.data_hash().map(|h| (h, d))) + { + values_to_cache + .push((key.inner().to_vec(), (data.clone(), hash).into())); + } + + cache_child_values::(c, values_to_cache, key); + }, + ); + } + + // Also cache values of inline nodes. + cache_child_values::(&node, &mut values_to_cache, full_key.clone()); + } + + drop(node); + values_to_cache.into_iter().for_each(|(k, v)| cache.cache_value_for_key(&k, v)); + } + } + + /// Cache the given `value`. + /// + /// `hash` is the hash of `value`. + fn cache_value(&mut self, full_key: &[u8], value: impl Into, hash: TrieHash) { + if let Some(cache) = self.cache.as_mut() { + let value = value.into(); + + // `get_or_insert` should always return `Ok`, but be safe. + let value = if let Ok(value) = cache + .get_or_insert_node(hash, &mut || Ok(NodeOwned::Value(value.clone(), hash))) + .map(|n| n.data().cloned()) + { + value + } else { + None + }; + + if let Some(value) = value { + cache.cache_value_for_key(full_key, (value, hash).into()) + } + } + } + /// Commit a node by hashing it and writing it to the db. Returns a /// `ChildReference` which in most cases carries a normal hash but for the /// case where we can fit the actual data in the `Hasher`s output type, we @@ -1666,6 +1955,15 @@ where match self.storage.destroy(storage_handle) { Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { + // Reconstructs the full key + let full_key = self.cache.as_ref().and_then(|_| { + let mut prefix = prefix.clone(); + if let Some(partial) = node.partial_key() { + prefix.append_partial(NibbleSlice::from_stored(partial).right()); + } + Some(prefix) + }); + let encoded = { let commit_child = |node: NodeToEncode>, o_slice: Option<&NibbleSlice>, @@ -1674,6 +1972,9 @@ where match node { NodeToEncode::Node(value) => { let value_hash = self.db.insert(prefix.as_prefix(), value); + + self.cache_value(prefix.inner(), value, value_hash); + prefix.drop_lasts(mov); ChildReference::Hash(value_hash) }, @@ -1687,8 +1988,11 @@ where node.into_encoded(commit_child) }; if encoded.len() >= L::Hash::LENGTH { - let hash = self.db.insert(prefix.as_prefix(), &encoded[..]); + let hash = self.db.insert(prefix.as_prefix(), &encoded); self.hash_count += 1; + + self.cache_node(hash, &encoded, full_key); + ChildReference::Hash(hash) } else { // it's a small value, so we cram it into a `TrieHash` @@ -1696,6 +2000,7 @@ where let mut h = >::default(); let len = encoded.len(); h.as_mut()[..len].copy_from_slice(&encoded[..len]); + ChildReference::Inline(h, len) } }, @@ -1736,7 +2041,7 @@ where where 'x: 'key, { - self.lookup(NibbleSlice::new(key), key, &self.root_handle) + self.lookup(key, NibbleSlice::new(key), &self.root_handle) } fn insert( @@ -1753,9 +2058,10 @@ where #[cfg(feature = "std")] trace!(target: "trie", "insert: key={:?}, value={:?}", ToHex(key), ToHex(&value)); + let value = Bytes::from(value); let root_handle = self.root_handle(); let (new_handle, _changed) = - self.insert_at(root_handle, &mut NibbleSlice::new(key), value.to_vec(), &mut old_val)?; + self.insert_at(root_handle, &mut NibbleSlice::new(key), value, &mut old_val)?; #[cfg(feature = "std")] trace!(target: "trie", "insert: altered trie={}", _changed); diff --git a/trie-db/test/benches/bench.rs b/trie-db/test/benches/bench.rs index 1f16c34a..670a07fb 100644 --- a/trie-db/test/benches/bench.rs +++ b/trie-db/test/benches/bench.rs @@ -365,7 +365,8 @@ fn trie_mut_a(c: &mut Criterion) { let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = trie_db::TrieDBMut::::new(&mut mdb, &mut root); + let mut trie = + trie_db::TrieDBMutBuilder::::new(&mut mdb, &mut root).build(); for (key, value) in datac { trie.insert(&key, &value).expect( "changes trie: insertion to trie is not allowed to fail within runtime", @@ -393,7 +394,8 @@ fn trie_mut_b(c: &mut Criterion) { let mut root = Default::default(); let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); - let mut trie = trie_db::TrieDBMut::::new(&mut mdb, &mut root); + let mut trie = + trie_db::TrieDBMutBuilder::::new(&mut mdb, &mut root).build(); for (key, value) in datac { trie.insert(&key, &value).expect( "changes trie: insertion to trie is not allowed to fail within runtime", @@ -464,7 +466,7 @@ fn trie_iteration(c: &mut Criterion) { c.bench_function("trie_iteration", move |b: &mut Bencher| { b.iter(|| { - let trie = trie_db::TrieDB::::new(&mdb, &root); + let trie = trie_db::TrieDBBuilder::::new(&mdb, &root).build(); let mut iter = trie_db::TrieDBNodeIterator::new(&trie).unwrap(); assert!(iter.all(|result| result.is_ok())); }) @@ -485,8 +487,8 @@ fn trie_proof_verification(c: &mut Criterion) { let mut mdb = memory_db::MemoryDB::<_, HashKey<_>, _>::default(); let root = reference_trie::calc_root_build::(data, &mut mdb); - let trie = trie_db::TrieDB::::new(&mdb, &root); - let proof = generate_proof(&trie, keys.iter()).unwrap(); + let proof = generate_proof::<_, Layout, _, _>(&mdb, &root, keys.iter()).unwrap(); + let trie = trie_db::TrieDBBuilder::::new(&mdb, &root).build(); let items = keys .into_iter() .map(|key| { diff --git a/trie-db/test/src/fatdbmut.rs b/trie-db/test/src/fatdbmut.rs index 5d5fa73d..519f5f0f 100644 --- a/trie-db/test/src/fatdbmut.rs +++ b/trie-db/test/src/fatdbmut.rs @@ -14,7 +14,7 @@ use hash_db::{Hasher, EMPTY_PREFIX}; use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefFatDBMut, RefHasher, RefTrieDB}; +use reference_trie::{RefFatDBMut, RefHasher, RefTrieDBBuilder}; use trie_db::{Trie, TrieMut}; #[test] @@ -25,7 +25,8 @@ fn fatdbmut_to_trie() { let mut t = RefFatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = RefTrieDB::new(&memdb, &root); + + let t = RefTrieDBBuilder::new(&memdb, &root).build(); assert_eq!(t.get(&RefHasher::hash(&[0x01u8, 0x23])), Ok(Some(vec![0x01u8, 0x23])),); } diff --git a/trie-db/test/src/iter_build.rs b/trie-db/test/src/iter_build.rs index 477205cf..636a4e86 100644 --- a/trie-db/test/src/iter_build.rs +++ b/trie-db/test/src/iter_build.rs @@ -17,7 +17,7 @@ use reference_trie::{ test_layouts, ExtensionLayout, HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, RefHasher, }; -use trie_db::{DBValue, TrieLayout}; +use trie_db::{DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut}; #[test] fn trie_root_empty() { @@ -38,19 +38,17 @@ fn root_extension_one() { } fn test_iter(data: Vec<(Vec, Vec)>) { - use trie_db::{Trie, TrieDB, TrieDBMut, TrieMut}; - let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut db, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); for i in 0..data.len() { let key: &[u8] = &data[i].0; let value: &[u8] = &data[i].1; t.insert(key, value).unwrap(); } } - let t = TrieDB::::new(&db, &root); + let t = TrieDBBuilder::::new(&db, &root).build(); for (i, kv) in t.iter().unwrap().enumerate() { let (k, v) = kv.unwrap(); let key: &[u8] = &data[i].0; diff --git a/trie-db/test/src/iterator.rs b/trie-db/test/src/iterator.rs index ad989048..e36ef6d0 100644 --- a/trie-db/test/src/iterator.rs +++ b/trie-db/test/src/iterator.rs @@ -17,7 +17,7 @@ use hex_literal::hex; use reference_trie::test_layouts; use trie_db::{ node::{Node, Value}, - DBValue, NibbleSlice, NibbleVec, TrieDB, TrieDBNodeIterator, TrieError, TrieIterator, + DBValue, NibbleSlice, NibbleVec, TrieDBBuilder, TrieDBNodeIterator, TrieError, TrieIterator, TrieLayout, TrieMut, }; @@ -33,7 +33,7 @@ fn build_trie_db( let mut memdb = MemoryDB::::default(); let mut root = Default::default(); { - let mut t = trie_db::TrieDBMut::::new(&mut memdb, &mut root); + let mut t = trie_db::TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for (x, y) in pairs.iter() { t.insert(x, y).unwrap(); } @@ -60,7 +60,7 @@ fn iterator_works_internal() { ]; let (memdb, root) = build_trie_db::(&pairs); - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); if T::USE_EXTENSION { @@ -186,7 +186,7 @@ fn iterator_works_internal() { test_layouts!(iterator_over_empty_works, iterator_over_empty_works_internal); fn iterator_over_empty_works_internal() { let (memdb, root) = build_trie_db::(&[]); - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); match iter.next() { @@ -212,7 +212,7 @@ fn seek_works_internal() { ]; let (memdb, root) = build_trie_db::(&pairs); - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("")[..]).unwrap(); @@ -246,7 +246,7 @@ fn seek_works_internal() { test_layouts!(seek_over_empty_works, seek_over_empty_works_internal); fn seek_over_empty_works_internal() { let (memdb, root) = build_trie_db::(&[]); - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("")[..]).unwrap(); @@ -278,7 +278,7 @@ fn iterate_over_incomplete_db_internal() { // Look up the leaf node with prefix "02". let leaf_hash = { - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("02")[..]).unwrap(); @@ -297,7 +297,7 @@ fn iterate_over_incomplete_db_internal() { // Seek to missing node returns error. { - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); match TrieIterator::seek(&mut iter, &hex!("02")[..]) { @@ -311,7 +311,7 @@ fn iterate_over_incomplete_db_internal() { // Iterate over missing node works. { - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); TrieIterator::seek(&mut iter, &hex!("0130")[..]).unwrap(); @@ -347,7 +347,7 @@ fn prefix_works_internal() { ]; let (memdb, root) = build_trie_db::(&pairs); - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("01").to_vec()[..]).unwrap(); @@ -387,8 +387,9 @@ fn prefix_works_internal() { } assert_eq!(prefix, nibble_vec(hex!("0120"), 3)); match node.node() { - Node::Leaf(partial, _) => - assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)), + Node::Leaf(partial, _) => { + assert_eq!(partial, NibbleSlice::new_offset(&hex!("03")[..], 1)) + }, _ => panic!("unexpected node"), } }, @@ -408,7 +409,7 @@ fn prefix_works_internal() { test_layouts!(prefix_over_empty_works, prefix_over_empty_works_internal); fn prefix_over_empty_works_internal() { let (memdb, root) = build_trie_db::(&[]); - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = TrieDBNodeIterator::new(&trie).unwrap(); iter.prefix(&hex!("")[..]).unwrap(); match iter.next() { diff --git a/trie-db/test/src/proof.rs b/trie-db/test/src/proof.rs index eaf31d88..f84ed8cd 100644 --- a/trie-db/test/src/proof.rs +++ b/trie-db/test/src/proof.rs @@ -17,7 +17,7 @@ use reference_trie::{test_layouts, NoExtensionLayout}; use trie_db::{ proof::{generate_proof, verify_proof, VerifyError}, - DBValue, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut, + DBValue, Trie, TrieDBBuilder, TrieDBMutBuilder, TrieLayout, TrieMut, }; type MemoryDB = memory_db::MemoryDB< @@ -53,7 +53,7 @@ fn test_generate_proof( let mut db = >::default(); let mut root = Default::default(); { - let mut trie = >::new(&mut db, &mut root); + let mut trie = >::new(&mut db, &mut root).build(); for (key, value) in entries.iter() { trie.insert(key, value).unwrap(); } @@ -62,8 +62,8 @@ fn test_generate_proof( }; // Generate proof for the given keys.. - let trie = >::new(&db, &root); - let proof = generate_proof::<_, L, _, _>(&trie, keys.iter()).unwrap(); + let proof = generate_proof::<_, L, _, _>(&db, &root, keys.iter()).unwrap(); + let trie = >::new(&db, &root).build(); let items = keys.into_iter().map(|key| (key, trie.get(key).unwrap())).collect(); (root, proof, items) diff --git a/trie-db/test/src/recorder.rs b/trie-db/test/src/recorder.rs index 94784ed0..485d0153 100644 --- a/trie-db/test/src/recorder.rs +++ b/trie-db/test/src/recorder.rs @@ -14,54 +14,16 @@ //! Trie query recorder. -use hash_db::Hasher; use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefTrieDB, RefTrieDBMut}; -use trie_db::{Record, Recorder, Trie, TrieMut}; - -#[test] -fn basic_recorder() { - let mut basic = Recorder::new(); - - let node1 = vec![1, 2, 3, 4]; - let node2 = vec![4, 5, 6, 7, 8, 9, 10]; - - let (hash1, hash2) = (RefHasher::hash(&node1), RefHasher::hash(&node2)); - basic.record(&hash1, &node1, 0); - basic.record(&hash2, &node2, 456); - - let record1 = Record { data: node1, hash: hash1, depth: 0 }; - - let record2 = Record { data: node2, hash: hash2, depth: 456 }; - - assert_eq!(basic.drain(), vec![record1, record2]); -} - -#[test] -fn basic_recorder_min_depth() { - let mut basic = Recorder::with_depth(400); - - let node1 = vec![1, 2, 3, 4]; - let node2 = vec![4, 5, 6, 7, 8, 9, 10]; - - let hash1 = RefHasher::hash(&node1); - let hash2 = RefHasher::hash(&node2); - basic.record(&hash1, &node1, 0); - basic.record(&hash2, &node2, 456); - - let records = basic.drain(); - - assert_eq!(records.len(), 1); - - assert_eq!(records[0].clone(), Record { data: node2, hash: hash2, depth: 456 }); -} +use reference_trie::{NoExtensionLayout, RefHasher, RefTrieDBBuilder, RefTrieDBMutBuilder}; +use trie_db::{Recorder, Trie, TrieMut}; #[test] fn trie_record() { let mut db = MemoryDB::, _>::default(); let mut root = Default::default(); { - let mut x = RefTrieDBMut::new(&mut db, &mut root); + let mut x = RefTrieDBMutBuilder::new(&mut db, &mut root).build(); x.insert(b"dog", b"cat").unwrap(); x.insert(b"lunch", b"time").unwrap(); @@ -73,48 +35,54 @@ fn trie_record() { x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); } - let trie = RefTrieDB::new(&db, &root); - let mut recorder = Recorder::new(); + { + let mut recorder = Recorder::::new(); + let trie = RefTrieDBBuilder::new(&db, &root).with_recorder(&mut recorder).build(); - trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); + trie.get(b"pirate").unwrap().unwrap(); - let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); - assert_eq!( - nodes, - vec![ + let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); + assert_eq!( + nodes, vec![ - 254, 192, 0, 128, 32, 27, 87, 5, 125, 163, 0, 90, 117, 142, 28, 67, 189, 82, 249, - 72, 103, 181, 28, 167, 216, 106, 191, 152, 9, 255, 42, 59, 75, 199, 172, 190, 128, - 227, 98, 5, 56, 103, 215, 106, 0, 144, 78, 159, 78, 163, 198, 13, 159, 226, 112, - 82, 132, 211, 79, 143, 4, 16, 109, 253, 182, 34, 196, 39, 13 - ], - vec![ - 254, 1, 2, 52, 11, 105, 114, 97, 116, 101, 24, 97, 97, 114, 103, 104, 33, 112, 15, - 111, 32, 104, 111, 32, 104, 111, 76, 97, 110, 100, 32, 97, 32, 98, 111, 116, 116, - 108, 101, 32, 111, 102, 32, 114, 117, 109 + vec![ + 254, 192, 0, 128, 32, 27, 87, 5, 125, 163, 0, 90, 117, 142, 28, 67, 189, 82, + 249, 72, 103, 181, 28, 167, 216, 106, 191, 152, 9, 255, 42, 59, 75, 199, 172, + 190, 128, 227, 98, 5, 56, 103, 215, 106, 0, 144, 78, 159, 78, 163, 198, 13, + 159, 226, 112, 82, 132, 211, 79, 143, 4, 16, 109, 253, 182, 34, 196, 39, 13 + ], + vec![ + 254, 1, 2, 52, 11, 105, 114, 97, 116, 101, 24, 97, 97, 114, 103, 104, 33, 112, + 15, 111, 32, 104, 111, 32, 104, 111, 76, 97, 110, 100, 32, 97, 32, 98, 111, + 116, 116, 108, 101, 32, 111, 102, 32, 114, 117, 109 + ] ] - ] - ); + ); + } - trie.get_with(b"letter", &mut recorder).unwrap().unwrap(); + { + let mut recorder = Recorder::::new(); + let trie = RefTrieDBBuilder::new(&db, &root).with_recorder(&mut recorder).build(); + trie.get(b"letter").unwrap().unwrap(); - let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); - assert_eq!( - nodes, - vec![ + let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); + assert_eq!( + nodes, vec![ - 254, 192, 0, 128, 32, 27, 87, 5, 125, 163, 0, 90, 117, 142, 28, 67, 189, 82, 249, - 72, 103, 181, 28, 167, 216, 106, 191, 152, 9, 255, 42, 59, 75, 199, 172, 190, 128, - 227, 98, 5, 56, 103, 215, 106, 0, 144, 78, 159, 78, 163, 198, 13, 159, 226, 112, - 82, 132, 211, 79, 143, 4, 16, 109, 253, 182, 34, 196, 39, 13 - ], - vec![ - 254, 16, 83, 28, 5, 111, 103, 12, 99, 97, 116, 52, 11, 111, 116, 100, 111, 103, 24, - 104, 111, 116, 99, 97, 116, 52, 11, 110, 115, 101, 114, 116, 24, 114, 101, 109, - 111, 118, 101, 124, 254, 192, 0, 64, 10, 5, 116, 116, 101, 114, 36, 99, 111, 110, - 102, 117, 115, 105, 111, 110, 40, 8, 5, 110, 99, 104, 16, 116, 105, 109, 101, 52, - 11, 111, 116, 100, 111, 103, 24, 110, 111, 116, 99, 97, 116 + vec![ + 254, 192, 0, 128, 32, 27, 87, 5, 125, 163, 0, 90, 117, 142, 28, 67, 189, 82, + 249, 72, 103, 181, 28, 167, 216, 106, 191, 152, 9, 255, 42, 59, 75, 199, 172, + 190, 128, 227, 98, 5, 56, 103, 215, 106, 0, 144, 78, 159, 78, 163, 198, 13, + 159, 226, 112, 82, 132, 211, 79, 143, 4, 16, 109, 253, 182, 34, 196, 39, 13 + ], + vec![ + 254, 16, 83, 28, 5, 111, 103, 12, 99, 97, 116, 52, 11, 111, 116, 100, 111, 103, + 24, 104, 111, 116, 99, 97, 116, 52, 11, 110, 115, 101, 114, 116, 24, 114, 101, + 109, 111, 118, 101, 124, 254, 192, 0, 64, 10, 5, 116, 116, 101, 114, 36, 99, + 111, 110, 102, 117, 115, 105, 111, 110, 40, 8, 5, 110, 99, 104, 16, 116, 105, + 109, 101, 52, 11, 111, 116, 100, 111, 103, 24, 110, 111, 116, 99, 97, 116 + ] ] - ] - ); + ); + } } diff --git a/trie-db/test/src/sectriedb.rs b/trie-db/test/src/sectriedb.rs index b1c5e75b..bc04a5c3 100644 --- a/trie-db/test/src/sectriedb.rs +++ b/trie-db/test/src/sectriedb.rs @@ -14,7 +14,7 @@ use hash_db::Hasher; use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefSecTrieDB, RefTrieDBMut}; +use reference_trie::{RefHasher, RefSecTrieDB, RefTrieDBMutBuilder}; use trie_db::{DBValue, Trie, TrieMut}; #[test] @@ -22,7 +22,7 @@ fn trie_to_sectrie() { let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = RefTrieDBMut::new(&mut db, &mut root); + let mut t = RefTrieDBMutBuilder::new(&mut db, &mut root).build(); t.insert(&RefHasher::hash(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); } let t = RefSecTrieDB::new(&db, &root); diff --git a/trie-db/test/src/sectriedbmut.rs b/trie-db/test/src/sectriedbmut.rs index 53d1991f..e984d2e2 100644 --- a/trie-db/test/src/sectriedbmut.rs +++ b/trie-db/test/src/sectriedbmut.rs @@ -14,7 +14,7 @@ use hash_db::Hasher; use memory_db::{HashKey, MemoryDB}; -use reference_trie::{RefHasher, RefSecTrieDBMut, RefTrieDB}; +use reference_trie::{RefHasher, RefSecTrieDBMut, RefTrieDBBuilder}; use trie_db::{DBValue, Trie, TrieMut}; #[test] @@ -25,6 +25,6 @@ fn sectrie_to_trie() { let mut t = RefSecTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = RefTrieDB::new(&memdb, &root); + let t = RefTrieDBBuilder::new(&memdb, &root).build(); assert_eq!(t.get(&RefHasher::hash(&[0x01u8, 0x23])).unwrap().unwrap(), vec![0x01u8, 0x23],); } diff --git a/trie-db/test/src/trie_codec.rs b/trie-db/test/src/trie_codec.rs index b35ec639..17b52cb5 100644 --- a/trie-db/test/src/trie_codec.rs +++ b/trie-db/test/src/trie_codec.rs @@ -13,10 +13,10 @@ // limitations under the License. use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use reference_trie::test_layouts; +use reference_trie::{test_layouts, ExtensionLayout}; use trie_db::{ - decode_compact, encode_compact, DBValue, Recorder, Trie, TrieDB, TrieDBMut, TrieError, - TrieLayout, TrieMut, + decode_compact, encode_compact, DBValue, NodeCodec, Recorder, Trie, TrieDBBuilder, + TrieDBMutBuilder, TrieError, TrieLayout, TrieMut, }; type MemoryDB = memory_db::MemoryDB< @@ -34,7 +34,7 @@ fn test_encode_compact( let mut db = >::default(); let mut root = Default::default(); { - let mut trie = >::new(&mut db, &mut root); + let mut trie = >::new(&mut db, &mut root).build(); for (key, value) in entries.iter() { trie.insert(key, value).unwrap(); } @@ -43,12 +43,12 @@ fn test_encode_compact( }; // Lookup items in trie while recording traversed nodes. - let mut recorder = Recorder::new(); + let mut recorder = Recorder::::new(); let items = { let mut items = Vec::with_capacity(keys.len()); - let trie = >::new(&db, &root); + let trie = >::new(&db, &root).with_recorder(&mut recorder).build(); for key in keys { - let value = trie.get_with(key, &mut recorder).unwrap(); + let value = trie.get(key).unwrap(); items.push((key, value)); } items @@ -62,7 +62,7 @@ fn test_encode_compact( // Compactly encode the partial trie DB. let compact_trie = { - let trie = >::new(&partial_db, &root); + let trie = >::new(&partial_db, &root).build(); encode_compact::(&trie).unwrap() }; @@ -82,7 +82,7 @@ fn test_decode_compact( assert_eq!(used, expected_used); // Check that lookups for all items succeed. - let trie = >::new(&db, &root); + let trie = >::new(&db, &root).build(); for (key, expected_value) in items { assert_eq!(trie.get(key).unwrap(), expected_value); } @@ -138,3 +138,52 @@ fn trie_decoding_fails_with_incomplete_database_internal() { _ => panic!("decode was unexpectedly successful"), } } + +#[test] +fn encoding_node_owned_and_decoding_node_works() { + let entries: Vec<(&[u8], &[u8])> = vec![ + // "alfa" is at a hash-referenced leaf node. + (b"alfa", &[0; 32]), + // "bravo" is at an inline leaf node. + (b"bravo", b"bravo"), + // "do" is at a hash-referenced branch node. + (b"do", b"verb"), + // "dog" is at an inline leaf node. + (b"dog", b"puppy"), + // "doge" is at a hash-referenced leaf node. + (b"doge", &[0; 32]), + // extension node "o" (plus nibble) to next branch. + (b"horse", b"stallion"), + (b"house", b"building"), + ]; + + // Populate DB with full trie from entries. + let mut recorder = { + let mut db = >::default(); + let mut root = Default::default(); + let mut recorder = Recorder::::new(); + { + let mut trie = >::new(&mut db, &mut root).build(); + for (key, value) in entries.iter() { + trie.insert(key, value).unwrap(); + } + } + + let trie = TrieDBBuilder::::new(&db, &root) + .with_recorder(&mut recorder) + .build(); + for (key, _) in entries.iter() { + trie.get(key).unwrap(); + } + + recorder + }; + + for record in recorder.drain() { + let node = + <::Codec as NodeCodec>::decode(&record.data).unwrap(); + let node_owned = node.to_owned_node::().unwrap(); + + assert_eq!(record.data, node_owned.to_encoded::<::Codec>()); + } +} diff --git a/trie-db/test/src/triedb.rs b/trie-db/test/src/triedb.rs index 04e0f710..9b54d836 100644 --- a/trie-db/test/src/triedb.rs +++ b/trie-db/test/src/triedb.rs @@ -12,13 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::ops::Deref; + +use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use hex_literal::hex; -use memory_db::{MemoryDB, PrefixedKey}; -use reference_trie::test_layouts; -use trie_db::{DBValue, Lookup, NibbleSlice, Trie, TrieDB, TrieDBMut, TrieLayout, TrieMut}; +use memory_db::{HashKey, MemoryDB, PrefixedKey}; +use reference_trie::{test_layouts, TestTrieCache}; +use trie_db::{ + CachedValue, DBValue, Lookup, NibbleSlice, Recorder, Trie, TrieCache, TrieDBBuilder, + TrieDBMutBuilder, TrieLayout, TrieMut, +}; type PrefixedMemoryDB = MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; +type MemoryDBProof = + MemoryDB<::Hash, HashKey<::Hash>, DBValue>; test_layouts!(iterator_works, iterator_works_internal); fn iterator_works_internal() { @@ -30,13 +38,13 @@ fn iterator_works_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for (x, y) in &pairs { t.insert(x, y).unwrap(); } } - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); @@ -58,13 +66,13 @@ fn iterator_seek_works_internal() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for (x, y) in &pairs { t.insert(x, y).unwrap(); } } - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = t.iter().unwrap(); assert_eq!( @@ -91,13 +99,13 @@ fn iterator_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for x in &d { t.insert(x, x).unwrap(); } } - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!( d.iter().map(|i| i.clone()).collect::>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::>() @@ -113,13 +121,13 @@ fn iterator_seek_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for (k, val) in d.iter().zip(vals.iter()) { t.insert(k, val.as_slice()).unwrap(); } } - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); let mut iter = t.iter().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (b"A".to_vec(), vals[0].clone())); iter.seek(b"!").unwrap(); @@ -166,12 +174,12 @@ fn get_length_with_extension_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); } - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()).unwrap(), Some(32)); assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()).unwrap(), None); @@ -184,13 +192,13 @@ fn debug_output_supports_pretty_print_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); let root = { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); for x in &d { t.insert(x, x).unwrap(); } t.root().clone() }; - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); if T::USE_EXTENSION { assert_eq!( @@ -267,16 +275,361 @@ fn test_lookup_with_corrupt_data_returns_decoder_error_internal() let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBA").unwrap(); } - let t = TrieDB::::new(&memdb, &root); + let t = TrieDBBuilder::::new(&memdb, &root).build(); // query for an invalid data type to trigger an error let q = |x: &[u8]| x.len() < 64; - let lookup = Lookup:: { db: t.db(), query: q, hash: root }; - let query_result = lookup.look_up(NibbleSlice::new(b"A")); + let lookup = Lookup:: { db: t.db(), query: q, hash: root, cache: None, recorder: None }; + let query_result = lookup.look_up(&b"A"[..], NibbleSlice::new(b"A")); assert_eq!(query_result.unwrap().unwrap(), true); } + +test_layouts!(test_recorder, test_recorder_internal); +fn test_recorder_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 4]), + (b"B".to_vec(), vec![4; 64]), + ]; + + let mut memdb = MemoryDB::, DBValue>::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + } + + let mut recorder = Recorder::::new(); + { + let trie = TrieDBBuilder::::new(&memdb, &root).with_recorder(&mut recorder).build(); + + for (key, value) in key_value.iter().take(3) { + assert_eq!(*value, trie.get(key).unwrap().unwrap()); + } + } + + let mut partial_db = MemoryDB::, DBValue>::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + { + let trie = TrieDBBuilder::::new(&partial_db, &root).build(); + + for (key, value) in key_value.iter().take(3) { + assert_eq!(*value, trie.get(key).unwrap().unwrap()); + } + assert!(trie.get(&key_value[3].0).is_err()); + } +} + +test_layouts!(test_recorder_with_cache, test_recorder_with_cache_internal); +fn test_recorder_with_cache_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 4]), + (b"B".to_vec(), vec![4; 64]), + ]; + + let mut memdb = MemoryDB::, DBValue>::default(); + let mut root = Default::default(); + + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + } + + let mut cache = TestTrieCache::::default(); + + { + let trie = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + + // Only read one entry. + assert_eq!(key_value[1].1, trie.get(&key_value[1].0).unwrap().unwrap()); + } + + // Root should now be cached. + assert!(cache.get_node(&root).is_some()); + // Also the data should be cached. + let value = cache.lookup_value_for_key(&key_value[1].0).unwrap(); + + assert_eq!(key_value[1].1, value.data().unwrap().unwrap().deref()); + assert_eq!(T::Hash::hash(&key_value[1].1), value.hash().unwrap()); + + // And the rest not + assert!(cache.lookup_value_for_key(&key_value[0].0).is_none()); + assert!(cache.lookup_value_for_key(&key_value[2].0).is_none()); + assert!(cache.lookup_value_for_key(&key_value[3].0).is_none()); + + // Run this multiple times to ensure that the cache is not interfering the recording. + for i in 0..6 { + eprintln!("Round: {}", i); + + // Ensure that it works with a filled value/node cache and without it. + if i < 2 { + cache.clear_value_cache(); + } else if i < 4 { + cache.clear_node_cache(); + } + + let mut recorder = Recorder::::new(); + { + let trie = TrieDBBuilder::::new(&memdb, &root) + .with_cache(&mut cache) + .with_recorder(&mut recorder) + .build(); + + for (key, value) in key_value.iter().take(2) { + assert_eq!(*value, trie.get(key).unwrap().unwrap()); + } + + assert_eq!( + T::Hash::hash(&key_value[2].1), + trie.get_hash(&key_value[2].0).unwrap().unwrap() + ); + assert_eq!(key_value[2].1, trie.get(&key_value[2].0).unwrap().unwrap()); + } + + let mut partial_db = MemoryDB::, DBValue>::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + { + let trie = TrieDBBuilder::::new(&partial_db, &root).build(); + + for (key, value) in key_value.iter().take(3) { + assert_eq!(*value, trie.get(key).unwrap().unwrap()); + } + + assert!(trie.get(&key_value[3].0).is_err()); + } + } +} + +test_layouts!(test_recorder_with_cache_get_hash, test_recorder_with_cache_get_hash_internal); +fn test_recorder_with_cache_get_hash_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 4]), + (b"B".to_vec(), vec![4; 64]), + ]; + + let mut memdb = MemoryDB::, DBValue>::default(); + let mut root = Default::default(); + + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + } + + let mut cache = TestTrieCache::::default(); + + { + let trie = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + + // Only read one entry. + assert_eq!( + T::Hash::hash(&key_value[1].1), + trie.get_hash(&key_value[1].0).unwrap().unwrap() + ); + } + + // Root should now be cached. + assert!(cache.get_node(&root).is_some()); + // Also the data should be cached. + + if T::MAX_INLINE_VALUE.map_or(true, |l| l as usize >= key_value[1].1.len()) { + assert!(matches!( + cache.lookup_value_for_key(&key_value[1].0).unwrap(), + CachedValue::Existing { hash, .. } if *hash == T::Hash::hash(&key_value[1].1) + )); + } else { + assert!(matches!( + cache.lookup_value_for_key(&key_value[1].0).unwrap(), + CachedValue::ExistingHash(hash) if *hash == T::Hash::hash(&key_value[1].1) + )); + } + + // Run this multiple times to ensure that the cache is not interfering the recording. + for i in 0..6 { + // Ensure that it works with a filled value/node cache and without it. + if i < 2 { + cache.clear_value_cache(); + } else if i < 4 { + cache.clear_node_cache(); + } + + let mut recorder = Recorder::::new(); + { + let trie = TrieDBBuilder::::new(&memdb, &root) + .with_cache(&mut cache) + .with_recorder(&mut recorder) + .build(); + + assert_eq!( + T::Hash::hash(&key_value[2].1), + trie.get_hash(&key_value[2].0).unwrap().unwrap() + ); + assert_eq!( + T::Hash::hash(&key_value[1].1), + trie.get_hash(&key_value[1].0).unwrap().unwrap() + ); + } + + let mut partial_db = MemoryDB::, DBValue>::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + { + let trie = TrieDBBuilder::::new(&partial_db, &root).build(); + + assert_eq!( + T::Hash::hash(&key_value[2].1), + trie.get_hash(&key_value[2].0).unwrap().unwrap() + ); + assert_eq!( + T::Hash::hash(&key_value[1].1), + trie.get_hash(&key_value[1].0).unwrap().unwrap() + ); + + // Check if the values are part of the proof or not, based on the layout. + if T::MAX_INLINE_VALUE.map_or(true, |l| l as usize >= key_value[2].1.len()) { + assert_eq!(key_value[2].1, trie.get(&key_value[2].0).unwrap().unwrap()); + } else { + assert!(trie.get(&key_value[2].0).is_err()); + } + + if T::MAX_INLINE_VALUE.map_or(true, |l| l as usize >= key_value[1].1.len()) { + assert_eq!(key_value[1].1, trie.get(&key_value[1].0).unwrap().unwrap()); + } else { + assert!(trie.get(&key_value[1].0).is_err()); + } + } + } +} + +test_layouts!(iterator_seek_with_recorder, iterator_seek_with_recorder_internal); +fn iterator_seek_with_recorder_internal() { + let d = vec![b"A".to_vec(), b"AA".to_vec(), b"AB".to_vec(), b"B".to_vec()]; + let vals = vec![vec![0; 64], vec![1; 64], vec![2; 64], vec![3; 64]]; + + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (k, val) in d.iter().zip(vals.iter()) { + t.insert(k, val.as_slice()).unwrap(); + } + } + + let mut recorder = Recorder::::new(); + { + let t = TrieDBBuilder::::new(&memdb, &root).with_recorder(&mut recorder).build(); + let mut iter = t.iter().unwrap(); + iter.seek(b"AA").unwrap(); + assert_eq!(&vals[1..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + } + + let mut partial_db = MemoryDBProof::::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + // Replay with from the proof. + { + let trie = TrieDBBuilder::::new(&partial_db, &root).build(); + + let mut iter = trie.iter().unwrap(); + iter.seek(b"AA").unwrap(); + assert_eq!(&vals[1..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + } +} + +test_layouts!(test_cache, test_cache_internal); +fn test_cache_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 4]), + (b"B".to_vec(), vec![4; 64]), + (b"BC".to_vec(), vec![4; 64]), + ]; + + let mut memdb = MemoryDB::, DBValue>::default(); + let mut root = Default::default(); + let mut cache = TestTrieCache::::default(); + + { + let mut t = + TrieDBMutBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + } + + // Ensure that when we cache the same value multiple times under different keys, + // the first cached key is still working. + assert_eq!( + cache.lookup_value_for_key(&b"B"[..]).unwrap().data().flatten().unwrap(), + vec![4u8; 64] + ); + assert_eq!( + cache.lookup_value_for_key(&b"BC"[..]).unwrap().data().flatten().unwrap(), + vec![4u8; 64] + ); + + // Ensure that we don't insert the same node multiple times, which would result in invalidating + // cached values. + let cached_value = cache.lookup_value_for_key(&b"AB"[..]).unwrap().clone(); + assert_eq!(cached_value.data().flatten().unwrap(), vec![3u8; 4]); + + { + let mut t = + TrieDBMutBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + for (key, value) in &key_value { + t.insert(key, value).unwrap(); + } + } + + assert_eq!( + cache.lookup_value_for_key(&b"AB"[..]).unwrap().data().flatten().unwrap(), + vec![3u8; 4] + ); + assert_eq!(cached_value.data().flatten().unwrap(), vec![3u8; 4]); + + // Clear all nodes and ensure that the value cache works flawlessly. + cache.clear_node_cache(); + + { + let t = TrieDBBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + for (key, value) in &key_value { + assert_eq!(*value, t.get(key).unwrap().unwrap()); + } + } + + // Ensure `get_hash` is also working properly + cache.clear_node_cache(); + + { + let t = TrieDBBuilder::::new(&mut memdb, &mut root).with_cache(&mut cache).build(); + for (key, value) in &key_value { + assert_eq!(T::Hash::hash(value), t.get_hash(key).unwrap().unwrap()); + } + } +} diff --git a/trie-db/test/src/triedbmut.rs b/trie-db/test/src/triedbmut.rs index 6079b888..cb7c0d28 100644 --- a/trie-db/test/src/triedbmut.rs +++ b/trie-db/test/src/triedbmut.rs @@ -12,27 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::ops::Deref; + use env_logger; -use hash_db::{HashDB, Hasher}; +use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use log::debug; -use memory_db::{MemoryDB, PrefixedKey}; +use memory_db::{HashKey, MemoryDB, PrefixedKey}; use reference_trie::{ - reference_trie_root_iter_build as reference_trie_root, test_layouts, ExtensionLayout, - HashedValueNoExt, HashedValueNoExtThreshold, NoExtensionLayout, RefHasher, ReferenceNodeCodec, - ReferenceNodeCodecNoExt, + reference_trie_root, test_layouts, ExtensionLayout, HashedValueNoExt, + HashedValueNoExtThreshold, NoExtensionLayout, RefHasher, ReferenceNodeCodec, + ReferenceNodeCodecNoExt, TestTrieCache, +}; +use trie_db::{ + DBValue, NodeCodec, Recorder, Trie, TrieCache, TrieDBBuilder, TrieDBMut, TrieDBMutBuilder, + TrieError, TrieLayout, TrieMut, Value, }; -use trie_db::{DBValue, NodeCodec, TrieDBMut, TrieError, TrieLayout, TrieMut, Value}; use trie_standardmap::*; type PrefixedMemoryDB = MemoryDB<::Hash, PrefixedKey<::Hash>, DBValue>; +type MemoryDBProof = + MemoryDB<::Hash, HashKey<::Hash>, DBValue>; fn populate_trie<'db, T: TrieLayout>( db: &'db mut dyn HashDB, root: &'db mut ::Out, v: &[(Vec, Vec)], ) -> TrieDBMut<'db, T> { - let mut t = TrieDBMut::::new(db, root); + let mut t = TrieDBMutBuilder::::new(db, root).build(); for i in 0..v.len() { let key: &[u8] = &v[i].0; @@ -121,7 +128,7 @@ test_layouts!(init, init_internal); fn init_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); let hashed_null_node = reference_hashed_null_node::(); assert_eq!(*t.root(), hashed_null_node); } @@ -130,7 +137,7 @@ test_layouts!(insert_on_empty, insert_on_empty_internal); fn insert_on_empty_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!( *t.root(), @@ -145,7 +152,7 @@ fn remove_to_empty_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01], big_value).unwrap(); t.insert(&[0x01, 0x23], big_value).unwrap(); @@ -164,7 +171,7 @@ fn remove_to_empty_checked_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01], big_value).unwrap(); t.insert(&[0x01, 0x23], big_value).unwrap(); @@ -189,7 +196,7 @@ fn remove_to_empty_no_extension_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01, 0x23], big_value3).unwrap(); t.insert(&[0x01], big_value2).unwrap(); @@ -210,7 +217,7 @@ test_layouts!(insert_replace_root, insert_replace_root_internal); fn insert_replace_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); assert_eq!( @@ -223,7 +230,7 @@ test_layouts!(insert_make_branch_root, insert_make_branch_root_internal); fn insert_make_branch_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); assert_eq!( @@ -239,7 +246,7 @@ test_layouts!(insert_into_branch_root, insert_into_branch_root_internal); fn insert_into_branch_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -257,7 +264,7 @@ test_layouts!(insert_value_into_branch_root, insert_value_into_branch_root_inter fn insert_value_into_branch_root_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); assert_eq!( @@ -273,7 +280,7 @@ test_layouts!(insert_split_leaf, insert_split_leaf_internal); fn insert_split_leaf_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); assert_eq!( @@ -289,7 +296,7 @@ test_layouts!(insert_split_extenstion, insert_split_extenstion_internal); fn insert_split_extenstion_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); @@ -310,7 +317,7 @@ fn insert_big_value_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); assert_eq!( @@ -328,7 +335,7 @@ fn insert_duplicate_value_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); assert_eq!( @@ -344,7 +351,7 @@ test_layouts!(test_at_empty, test_at_empty_internal); fn test_at_empty_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let t = TrieDBMut::::new(&mut memdb, &mut root); + let t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); assert_eq!(t.get(&[0x5]).unwrap(), None); } @@ -353,14 +360,14 @@ fn test_at_one_and_two_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); t.commit(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), vec![0x1u8, 0x23]); t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x24]).unwrap(); } - let mut t = TrieDBMut::::from_existing(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23, 0x00], &[0x01u8, 0x25]).unwrap(); // This test that middle node get resolved correctly (modified // triedbmut node due to change of child node). @@ -371,7 +378,7 @@ test_layouts!(test_at_three, test_at_three_internal); fn test_at_three_internal() { let mut memdb = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -390,7 +397,7 @@ fn test_at_three_internal() { fn test_nibbled_branch_changed_value() { let mut memdb = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = reference_trie::RefTrieDBMutNoExt::new(&mut memdb, &mut root); + let mut t = reference_trie::RefTrieDBMutNoExtBuilder::new(&mut memdb, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23, 0x11], &[0xf1u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), Some(vec![0x01u8, 0x23])); @@ -440,12 +447,12 @@ fn test_trie_existing_internal() { let mut db = PrefixedMemoryDB::::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut db, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } { - let _ = TrieDBMut::::from_existing(&mut db, &mut root); + let _ = TrieDBMutBuilder::::from_existing(&mut db, &mut root); } } @@ -463,7 +470,7 @@ fn insert_empty_internal() { let mut db = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut db, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } @@ -494,18 +501,18 @@ fn return_old_values_internal() { let mut db = PrefixedMemoryDB::::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut db, &mut root); + let mut t = TrieDBMutBuilder::::new(&mut db, &mut root).build(); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap() == None); if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { - assert_eq!(t.insert(key, value).unwrap(), Some(Value::Inline(value.clone()))); + assert_eq!(t.insert(key, value).unwrap(), Some(Value::Inline(value.clone().into()))); } else { assert!(matches!(t.insert(key, value).unwrap(), Some(Value::NewNode(..)))); } } for (key, value) in x { if threshold.map(|t| value.len() < t as usize).unwrap_or(true) { - assert_eq!(t.remove(&key).unwrap(), Some(Value::Inline(value))); + assert_eq!(t.remove(&key).unwrap(), Some(Value::Inline(value.into()))); } else { assert!(matches!(t.remove(&key).unwrap(), Some(Value::NewNode(..)))); } @@ -517,7 +524,7 @@ fn return_old_values_internal() { fn insert_empty_allowed() { let mut db = MemoryDB::, DBValue>::default(); let mut root = Default::default(); - let mut t = reference_trie::RefTrieDBMutAllowEmpty::new(&mut db, &mut root); + let mut t = reference_trie::RefTrieDBMutAllowEmptyBuilder::new(&mut db, &mut root).build(); t.insert(b"test", &[]).unwrap(); assert_eq!( @@ -535,7 +542,6 @@ fn register_proof_without_value() { use hash_db::{AsHashDB, Prefix}; use reference_trie::HashedValueNoExtThreshold; use std::{cell::RefCell, collections::HashMap}; - use trie_db::TrieDB; type Layout = HashedValueNoExtThreshold; type MemoryDB = memory_db::MemoryDB, DBValue>; @@ -549,7 +555,7 @@ fn register_proof_without_value() { let mut root = Default::default(); let _ = populate_trie::(&mut memdb, &mut root, &x); { - let trie = TrieDB::::new(&memdb, &root); + let trie = TrieDBBuilder::::new(&memdb, &root).build(); println!("{:?}", trie); } @@ -600,7 +606,7 @@ fn register_proof_without_value() { let root_proof = root.clone(); { - let mut trie = TrieDBMut::::from_existing(&mut memdb, &mut root); + let mut trie = TrieDBMutBuilder::::from_existing(&mut memdb, &mut root).build(); // touch te value (test1 remains untouch). trie.get(b"te").unwrap(); // cut test_1234 prefix @@ -623,7 +629,9 @@ fn register_proof_without_value() { let mut memdb_from_proof = db_unpacked.clone(); let mut root_proof = root_unpacked.clone(); { - let mut trie = TrieDBMut::::from_existing(&mut memdb_from_proof, &mut root_proof); + let mut trie = + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, &mut root_proof) + .build(); trie.get(b"te").unwrap(); trie.insert(b"test12", &[2u8; 36][..]).unwrap(); trie.remove(b"test1234").unwrap(); @@ -633,7 +641,7 @@ fn register_proof_without_value() { let mut root_proof = root_unpacked.clone(); { use trie_db::Trie; - let trie = TrieDB::::new(&memdb_from_proof, &root_proof); + let trie = TrieDBBuilder::::new(&memdb_from_proof, &root_proof).build(); assert!(trie.get(b"te").unwrap().is_some()); assert!(matches!( trie.get(b"test1").map_err(|e| *e), @@ -642,7 +650,9 @@ fn register_proof_without_value() { } { - let trie = TrieDBMut::::from_existing(&mut memdb_from_proof, &mut root_proof); + let trie = + TrieDBMutBuilder::::from_existing(&mut memdb_from_proof, &mut root_proof) + .build(); assert!(trie.get(b"te").unwrap().is_some()); assert!(matches!( trie.get(b"test1").map_err(|e| *e), @@ -650,3 +660,179 @@ fn register_proof_without_value() { )); } } + +test_layouts!(test_recorder, test_recorder_internal); +fn test_recorder_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 64]), + (b"B".to_vec(), vec![4; 64]), + ]; + + // Add some initial data to the trie + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (key, value) in key_value.iter().take(1) { + t.insert(key, value).unwrap(); + } + } + + // Add more data, but this time only to the overlay. + // While doing that we record all trie accesses to replay this operation. + let mut recorder = Recorder::::new(); + let mut overlay = memdb.clone(); + let mut new_root = root; + { + let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, &mut new_root) + .with_recorder(&mut recorder) + .build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + } + + let mut partial_db = MemoryDBProof::::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + // Replay the it, but this time we use the proof. + let mut validated_root = root; + { + let mut trie = + TrieDBMutBuilder::::from_existing(&mut partial_db, &mut validated_root).build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + } + + assert_eq!(new_root, validated_root); +} + +test_layouts!(test_recorder_witch_cache, test_recorder_with_cache_internal); +fn test_recorder_with_cache_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + (b"AB".to_vec(), vec![3; 64]), + (b"B".to_vec(), vec![4; 64]), + ]; + + // Add some initial data to the trie + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + { + let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + for (key, value) in key_value.iter().take(1) { + t.insert(key, value).unwrap(); + } + } + + let mut cache = TestTrieCache::::default(); + + { + let trie = TrieDBBuilder::::new(&memdb, &root).with_cache(&mut cache).build(); + + // Only read one entry. + assert_eq!(key_value[0].1, trie.get(&key_value[0].0).unwrap().unwrap()); + } + + // Root should now be cached. + assert!(cache.get_node(&root).is_some()); + + // Add more data, but this time only to the overlay. + // While doing that we record all trie accesses to replay this operation. + let mut recorder = Recorder::::new(); + let mut overlay = memdb.clone(); + let mut new_root = root; + { + let mut trie = TrieDBMutBuilder::::from_existing(&mut overlay, &mut new_root) + .with_recorder(&mut recorder) + .with_cache(&mut cache) + .build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + } + + for (key, value) in key_value.iter().skip(1) { + let cached_value = cache.lookup_value_for_key(key).unwrap(); + + assert_eq!(value, cached_value.data().flatten().unwrap().deref()); + assert_eq!(T::Hash::hash(&value), cached_value.hash().unwrap()); + } + + let mut partial_db = MemoryDBProof::::default(); + for record in recorder.drain() { + partial_db.insert(EMPTY_PREFIX, &record.data); + } + + // Replay the it, but this time we use the proof. + let mut validated_root = root; + { + let mut trie = + TrieDBMutBuilder::::from_existing(&mut partial_db, &mut validated_root).build(); + + for (key, value) in key_value.iter().skip(1) { + trie.insert(key, value).unwrap(); + } + } + + assert_eq!(new_root, validated_root); +} + +test_layouts!(test_insert_remove_data_with_cache, test_insert_remove_data_with_cache_internal); +fn test_insert_remove_data_with_cache_internal() { + let key_value = vec![ + (b"A".to_vec(), vec![1; 64]), + (b"AA".to_vec(), vec![2; 64]), + // Should be inlined + (b"AC".to_vec(), vec![7; 4]), + (b"AB".to_vec(), vec![3; 64]), + (b"B".to_vec(), vec![4; 64]), + ]; + + let mut cache = TestTrieCache::::default(); + let mut recorder = Recorder::::new(); + let mut memdb = PrefixedMemoryDB::::default(); + let mut root = Default::default(); + { + let mut trie = TrieDBMutBuilder::::new(&mut memdb, &mut root) + .with_recorder(&mut recorder) + .with_cache(&mut cache) + .build(); + + // Add all values + for (key, value) in key_value.iter() { + trie.insert(key, value).unwrap(); + } + + // Remove only the last 2 elements + for (key, _) in key_value.iter().skip(3) { + let _ = trie.remove(key); + } + } + + // Then only the first 3 elements should be in the cache and the last + // two ones should not be there. + for (key, value) in key_value.iter().take(3) { + let key_str = String::from_utf8_lossy(key); + + let cached_value = cache + .lookup_value_for_key(key) + .unwrap_or_else(|| panic!("Failed to lookup `{}`", key_str)); + + assert_eq!(value, cached_value.data().flatten().unwrap().deref(), "{:?}", key_str); + assert_eq!(T::Hash::hash(&value), cached_value.hash().unwrap()); + } + + for (key, _) in key_value.iter().skip(3) { + assert!(cache.lookup_value_for_key(key).is_none()); + } +} diff --git a/trie-eip1186/src/eip1186.rs b/trie-eip1186/src/eip1186.rs index 83cd827c..37ad106d 100644 --- a/trie-eip1186/src/eip1186.rs +++ b/trie-eip1186/src/eip1186.rs @@ -1,22 +1,28 @@ use crate::rstd::{result::Result, vec::Vec}; -use hash_db::Hasher; +use hash_db::{HashDBRef, Hasher}; use trie_db::{ node::{decode_hash, Node, NodeHandle, Value}, recorder::Recorder, - CError, NibbleSlice, NodeCodec, Result as TrieResult, Trie, TrieHash, TrieLayout, + CError, DBValue, NibbleSlice, NodeCodec, Result as TrieResult, Trie, TrieDBBuilder, TrieHash, + TrieLayout, }; /// Generate an eip-1186 compatible proof for key-value pairs in a trie given a key. -pub fn generate_proof( - trie: &T, +pub fn generate_proof( + db: &dyn HashDBRef, + root: &TrieHash, key: &[u8], ) -> TrieResult<(Vec>, Option>), TrieHash, CError> where - T: Trie, L: TrieLayout, { - let mut recorder = Recorder::new(); - let item = trie.get_with(key, &mut recorder)?; + let mut recorder = Recorder::::new(); + + let item = { + let trie = TrieDBBuilder::::new(db, root).with_recorder(&mut recorder).build(); + trie.get(key)? + }; + let proof: Vec> = recorder.drain().into_iter().map(|r| r.data).collect(); Ok((proof, item)) } @@ -286,7 +292,7 @@ where }, (Some(Value::Inline(inline_data)), _, None) => Err(VerifyError::ExistingValue(inline_data.to_vec())), - (Some(Value::Node(plain_hash, _)), Some(next_proof_item), Some(value)) => { + (Some(Value::Node(plain_hash)), Some(next_proof_item), Some(value)) => { let value_hash = L::Hash::hash(value); let node_hash = decode_hash::(plain_hash) .ok_or(VerifyError::HashDecodeError(plain_hash))?; @@ -298,8 +304,8 @@ where Ok(()) } }, - (Some(Value::Node(_, _)), None, _) => Err(VerifyError::IncompleteProof), - (Some(Value::Node(_, _)), Some(proof_item), None) => + (Some(Value::Node(_)), None, _) => Err(VerifyError::IncompleteProof), + (Some(Value::Node(_)), Some(proof_item), None) => Err(VerifyError::ExistingValue(proof_item.to_vec())), } } diff --git a/trie-eip1186/test/src/eip1186.rs b/trie-eip1186/test/src/eip1186.rs index 2135d4b1..05d89edc 100644 --- a/trie-eip1186/test/src/eip1186.rs +++ b/trie-eip1186/test/src/eip1186.rs @@ -14,7 +14,7 @@ use hash_db::Hasher; use reference_trie::test_layouts; -use trie_db::{DBValue, TrieDB, TrieDBMut, TrieLayout, TrieMut}; +use trie_db::{DBValue, TrieDBMutBuilder, TrieLayout, TrieMut}; use trie_eip1186::{generate_proof, verify_proof, VerifyError}; type MemoryDB = memory_db::MemoryDB< @@ -50,7 +50,7 @@ fn test_generate_proof( let mut db = >::default(); let mut root = Default::default(); { - let mut trie = >::new(&mut db, &mut root); + let mut trie = >::new(&mut db, &mut root).build(); for (key, value) in entries.iter() { trie.insert(key, value).unwrap(); } @@ -58,8 +58,7 @@ fn test_generate_proof( (db, root) }; // Generate proof for the given keys.. - let trie = >::new(&db, &root); - let proof = generate_proof::<_, L>(&trie, key).unwrap(); + let proof = generate_proof::(&db, &root, key).unwrap(); (root, proof.0, proof.1) }