From 59b8b10b77a03baca1f4e477a12f563998e402f7 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Thu, 21 Nov 2019 11:01:49 +0100 Subject: [PATCH] Upgrade trie-db crate to 0.16.0. (#4144) * Upgrade paritytech/trie crates to 0.16.0. * Fix tests and unexpected Cargo.lock change. * Bump trie-bench version. --- Cargo.lock | 24 ++--- primitives/state-machine/Cargo.toml | 2 +- primitives/trie/Cargo.toml | 4 +- primitives/trie/src/lib.rs | 2 +- primitives/trie/src/node_codec.rs | 148 ++++++++++++++++++---------- test/utils/runtime/Cargo.toml | 2 +- 6 files changed, 112 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb89f5fd35a71..e67e0e365bf68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ dependencies = [ [[package]] name = "ahash" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1525,7 +1525,7 @@ name = "hashbrown" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ahash 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", + "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -6312,7 +6312,7 @@ dependencies = [ "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", "substrate-trie 2.0.0", - "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -6403,7 +6403,7 @@ dependencies = [ "substrate-transaction-pool-runtime-api 2.0.0", "substrate-trie 2.0.0", "substrate-wasm-builder-runner 1.0.4", - "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -6476,8 +6476,8 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-primitives 2.0.0", - "trie-bench 0.16.2 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-bench 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -6954,7 +6954,7 @@ dependencies = [ [[package]] name = "trie-bench" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -6962,14 +6962,14 @@ dependencies = [ "keccak-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "trie-db" -version = "0.15.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -7670,7 +7670,7 @@ dependencies = [ "checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" "checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" -"checksum ahash 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "2f00e10d4814aa20900e7948174384f79f1317f24f0ba7494e735111653fc330" +"checksum ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" @@ -8139,8 +8139,8 @@ dependencies = [ "checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" "checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" -"checksum trie-bench 0.16.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3073600c543ed001319d7e092c46dfd8c245af1a218ec5c75eb01582660a2b3e" -"checksum trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b62d27e8aa1c07414549ac872480ac82380bab39e730242ab08d82d7cc098a" +"checksum trie-bench 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d14da20d0549737c88299aee9e7b46098c30dd2af7cbca0518bdd420f3766191" +"checksum trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "784a9813d23f18bccab728ab039c39b8a87d0d6956dcdece39e92f5cffe5076e" "checksum trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" "checksum trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3161ba520ab28cd8e6b68e1126f1009f6e335339d1a73b978139011703264c8" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 3f02056647a00..36557803e10e8 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" log = "0.4.8" parking_lot = "0.9.0" hash-db = "0.15.2" -trie-db = "0.15.2" +trie-db = "0.16.0" trie-root = "0.15.2" trie = { package = "substrate-trie", path = "../trie" } primitives = { package = "substrate-primitives", path = "../core" } diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 202c9066d0917..a8e98bee0e700 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -15,13 +15,13 @@ harness = false codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } rstd = { package = "sr-std", path = "../sr-std", default-features = false } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.15.2", default-features = false } +trie-db = { version = "0.16.0", default-features = false } trie-root = { version = "0.15.2", default-features = false } memory-db = { version = "0.15.2", default-features = false } primitives = { package = "substrate-primitives", path = "../core", default-features = false } [dev-dependencies] -trie-bench = "0.16.2" +trie-bench = "0.16.3" trie-standardmap = "0.15.2" criterion = "0.2.11" hex-literal = "0.2.1" diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index eab05bab28af0..de056cace1b7e 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -323,7 +323,7 @@ mod tests { type Layout = super::Layout; fn hashed_null_node() -> TrieHash { - >::hashed_null_node() + ::hashed_null_node() } fn check_equivalent(input: &Vec<(&[u8], &[u8])>) { diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 03cbdfce528f4..e31ce8cc91245 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -17,105 +17,147 @@ //! `NodeCodec` implementation for Substrate's trie format. use rstd::marker::PhantomData; +use rstd::ops::Range; use rstd::vec::Vec; use rstd::borrow::Borrow; -use codec::{Encode, Decode, Compact}; +use codec::{Encode, Decode, Input, Compact}; use hash_db::Hasher; -use trie_db::{self, NibbleSlice, node::Node, ChildReference, +use trie_db::{self, node::{NibbleSlicePlan, NodePlan, NodeHandlePlan}, ChildReference, nibble_ops, Partial, NodeCodec as NodeCodecT}; use crate::error::Error; use crate::trie_constants; use super::{node_header::{NodeHeader, NodeKind}}; -fn take<'a>(input: &mut &'a[u8], count: usize) -> Option<&'a[u8]> { - if input.len() < count { - return None +/// Helper struct for trie node decoder. This implements `codec::Input` on a byte slice, while +/// tracking the absolute position. This is similar to `std::io::Cursor` but does not implement +/// `Read` and `io` is not in `rstd`. +struct ByteSliceInput<'a> { + data: &'a [u8], + offset: usize, +} + +impl<'a> ByteSliceInput<'a> { + fn new(data: &'a [u8]) -> Self { + ByteSliceInput { + data, + offset: 0, + } + } + + fn take(&mut self, count: usize) -> Result, codec::Error> { + if self.offset + count > self.data.len() { + return Err("out of data".into()); + } + + let range = self.offset..(self.offset + count); + self.offset += count; + Ok(range) + } +} + +impl<'a> Input for ByteSliceInput<'a> { + fn remaining_len(&mut self) -> Result, codec::Error> { + let remaining = if self.offset <= self.data.len() { + Some(self.data.len() - self.offset) + } else { + None + }; + Ok(remaining) + } + + fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { + let range = self.take(into.len())?; + into.copy_from_slice(&self.data[range]); + Ok(()) + } + + fn read_byte(&mut self) -> Result { + if self.offset + 1 > self.data.len() { + return Err("out of data".into()); + } + + let byte = self.data[self.offset]; + self.offset += 1; + Ok(byte) } - let r = &(*input)[..count]; - *input = &(*input)[count..]; - Some(r) } /// Concrete implementation of a `NodeCodec` with Parity Codec encoding, generic over the `Hasher` #[derive(Default, Clone)] pub struct NodeCodec(PhantomData); -impl NodeCodecT for NodeCodec { +impl NodeCodecT for NodeCodec { type Error = Error; + type HashOut = H::Out; fn hashed_null_node() -> ::Out { - H::hash(>::empty_node()) + H::hash(::empty_node()) } - fn decode(data: &[u8]) -> rstd::result::Result { - let input = &mut &*data; - let head = NodeHeader::decode(input)?; - match head { - NodeHeader::Null => Ok(Node::Empty), + fn decode_plan(data: &[u8]) -> rstd::result::Result { + let mut input = ByteSliceInput::new(data); + match NodeHeader::decode(&mut input)? { + NodeHeader::Null => Ok(NodePlan::Empty), NodeHeader::Branch(has_value, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) - if padding && nibble_ops::pad_left(input[0]) != 0 { + if padding && nibble_ops::pad_left(data[input.offset]) != 0 { return Err(Error::BadFormat); } - let nibble_data = take( - input, + let partial = input.take( (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / nibble_ops::NIBBLE_PER_BYTE, - ).ok_or(Error::BadFormat)?; - let nibble_slice = NibbleSlice::new_offset( - nibble_data, - nibble_ops::number_padding(nibble_count), - ); - let bitmap_slice = take(input, BITMAP_LENGTH).ok_or(Error::BadFormat)?; - let bitmap = Bitmap::decode(&bitmap_slice[..])?; + )?; + let partial_padding = nibble_ops::number_padding(nibble_count); + let bitmap_range = input.take(BITMAP_LENGTH)?; + let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if has_value { - let count = >::decode(input)?.0 as usize; - Some(take(input, count).ok_or(Error::BadFormat)?) + let count = >::decode(&mut input)?.0 as usize; + Some(input.take(count)?) } else { None }; - let mut children = [None; 16]; - + let mut children = [ + None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, + ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { - let count = >::decode(input)?.0 as usize; - children[i] = Some(take(input, count).ok_or(Error::BadFormat)?); + let count = >::decode(&mut input)?.0 as usize; + let range = input.take(count)?; + children[i] = Some(if count == H::LENGTH { + NodeHandlePlan::Hash(range) + } else { + NodeHandlePlan::Inline(range) + }); } } - Ok(Node::NibbledBranch(nibble_slice, children, value)) + Ok(NodePlan::NibbledBranch { + partial: NibbleSlicePlan::new(partial, partial_padding), + value, + children, + }) } NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) - if padding && nibble_ops::pad_left(input[0]) != 0 { + if padding && nibble_ops::pad_left(data[input.offset]) != 0 { return Err(Error::BadFormat); } - let nibble_data = take( - input, + let partial = input.take( (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / nibble_ops::NIBBLE_PER_BYTE, - ).ok_or(Error::BadFormat)?; - let nibble_slice = NibbleSlice::new_offset( - nibble_data, - nibble_ops::number_padding(nibble_count), - ); - let count = >::decode(input)?.0 as usize; - Ok(Node::Leaf(nibble_slice, take(input, count).ok_or(Error::BadFormat)?)) + )?; + let partial_padding = nibble_ops::number_padding(nibble_count); + let count = >::decode(&mut input)?.0 as usize; + Ok(NodePlan::Leaf { + partial: NibbleSlicePlan::new(partial, partial_padding), + value: input.take(count)?, + }) } } } - fn try_decode_hash(data: &[u8]) -> Option<::Out> { - if data.len() == H::LENGTH { - let mut r = ::Out::default(); - r.as_mut().copy_from_slice(data); - Some(r) - } else { - None - } - } - fn is_empty_node(data: &[u8]) -> bool { - data == >::empty_node() + data == ::empty_node() } fn empty_node() -> &'static [u8] { diff --git a/test/utils/runtime/Cargo.toml b/test/utils/runtime/Cargo.toml index 4caba58301e4a..580ed816e0c2b 100644 --- a/test/utils/runtime/Cargo.toml +++ b/test/utils/runtime/Cargo.toml @@ -35,7 +35,7 @@ pallet-timestamp = { path = "../../../palette/timestamp", default-features = fal substrate-client = { path = "../../../client", optional = true } substrate-trie = { path = "../../../primitives/trie", default-features = false } transaction-pool-api = { package = "substrate-transaction-pool-runtime-api", path = "../../../primitives/transaction-pool/runtime-api", default-features = false } -trie-db = { version = "0.15.2", default-features = false } +trie-db = { version = "0.16.0", default-features = false } [dev-dependencies] substrate-executor = { path = "../../../client/executor" }