Batch update.

memory-db/src/lib.rs

@@ -48,6 +48,7 @@ use core::{
 	mem,
 	marker::PhantomData,
 	cmp::Eq,
+	cmp,
 	borrow::Borrow,
 };
 
@@ -157,8 +158,13 @@ impl<H, KF, T, M> PartialEq<MemoryDB<H, KF, T, M>> for MemoryDB<H, KF, T, M>
 	M: MemTracker<T> + PartialEq,
 {
 	fn eq(&self, other: &MemoryDB<H, KF, T, M>) -> bool {
-		for a in self.data.iter() {
-			match other.data.get(&a.0) {
+		let (s, other) = if self.data.len() > other.data.len() {
+			(&self.data, &other.data)
+		} else {
+			(&other.data, &self.data)
+		};
+		for a in s.iter() {
+			match other.get(&a.0) {
 				Some(v) if v != a.1 => return false,
 				None => return false,
 				_ => (),
@@ -178,7 +184,7 @@ impl<H, KF, T, M> Eq for MemoryDB<H, KF, T, M>
 {}
 
 pub trait KeyFunction<H: KeyHasher> {
-	type Key: Send + Sync + Clone + hash::Hash + Eq;
+	type Key: Send + Sync + Clone + hash::Hash + Eq + AsRef<[u8]>;
 
 	fn key(hash: &H::Out, prefix: Prefix) -> Self::Key;
 }

test-support/reference-trie/src/lib.rs

@@ -29,9 +29,11 @@ use trie_db::{
 	TrieBuilder,
 	TrieRoot,
 	Partial,
+	OwnedPrefix,
 };
 use std::borrow::Borrow;
 use keccak_hasher::KeccakHasher;
+pub use trie_db::traverse::{batch_update, InputAction, unprefixed_detached_trie, prefixed_detached_trie};
 
 use trie_db::{
 	nibble_ops, NodeCodec,
@@ -1200,6 +1202,31 @@ pub fn compare_no_extension_insert_remove(
 	assert_eq!(*t.root(), calc_root_no_extension(data2));
 }
 
+pub fn trie_traverse_key_no_extension_build<'a, I, K, V, B>(
+	// TODO db to non mutable
+	db: &'a mut dyn hash_db::HashDBRef<keccak_hasher::KeccakHasher, B>,
+	root: &'a <KeccakHasher as Hasher>::Out,
+	elements: I,
+) -> (
+	<KeccakHasher as Hasher>::Out,
+	impl Iterator<Item = (OwnedPrefix, <KeccakHasher as Hasher>::Out, Option<Vec<u8>>)>,
+	impl Iterator<Item = (Vec<u8>, OwnedPrefix, <KeccakHasher as Hasher>::Out)>,
+)
+	where
+		I: IntoIterator<Item = (K, Option<V>)>,
+		K: AsRef<[u8]> + Ord,
+		V: AsRef<[u8]>,
+		B: Borrow<[u8]> + AsRef<[u8]> + for<'b> From<&'b [u8]>,
+{
+	let elements = elements.into_iter().map(|(k, v)| (k, if let Some(v) = v {
+		InputAction::Insert(v)
+	} else {
+		InputAction::Delete
+	}));
+	let (root, values, values_detached, detached_root) = batch_update::<NoExtensionLayout, _, _, _, _>(db, root, elements).unwrap();
+	(root, values.into_iter().chain(values_detached.into_iter()), detached_root.into_iter())
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;

trie-db/fuzz/Cargo.toml

@@ -64,3 +64,11 @@ path = "fuzz_targets/trie_proof_valid.rs"
 [[bin]]
 name = "trie_proof_invalid"
 path = "fuzz_targets/trie_proof_invalid.rs"
+
+[[bin]]
+name = "batch_update"
+path = "fuzz_targets/batch_update.rs"
+
+[[bin]]
+name = "detach_attach"
+path = "fuzz_targets/detach_attach.rs"

trie-db/fuzz/fuzz_targets/batch_update.rs

@@ -0,0 +1,10 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+	// use next line to favor complex structure and inline data
+	trie_db_fuzz::fuzz_batch_update(data, |_v| (), false);
+	// use next line to favor db prefix verification
+	//trie_db_fuzz::fuzz_batch_update(data, |v| v.extend(&[4u8; 32]), true);
+});

trie-db/fuzz/fuzz_targets/detach_attach.rs

@@ -0,0 +1,10 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+fuzz_target!(|data: &[u8]| {
+	// use next line to favor complex structure and inline data
+	//trie_db_fuzz::fuzz_detach_attach(data, |_v| (), false);
+	// use next line to favor db prefix verification
+	trie_db_fuzz::fuzz_detach_attach(data, |v| v.extend(&[4u8; 32]), true);
+});

trie-db/fuzz/src/lib.rs

@@ -20,7 +20,7 @@ use reference_trie::{
 	calc_root_no_extension,
 	compare_no_extension_insert_remove,
 	ExtensionLayout,
-	NoExtensionLayout,
+	NoExtensionLayout, batch_update, InputAction,
 	proof::{generate_proof, verify_proof},
 	reference_trie_root,
 	RefTrieDBMut,
@@ -72,7 +72,7 @@ fn fuzz_to_data(input: &[u8]) -> Vec<(Vec<u8>,Vec<u8>)> {
 	result
 }
 
-fn fuzz_removal(data: Vec<(Vec<u8>,Vec<u8>)>) -> Vec<(bool, Vec<u8>,Vec<u8>)> {
+fn fuzz_removal(data: Vec<(Vec<u8>,Vec<u8>)>) -> Vec<(bool, Vec<u8>, Vec<u8>)> {
 	let mut res = Vec::new();
 	let mut torem = None;
 	for (a, d) in data.into_iter().enumerate() {
@@ -363,3 +363,183 @@ fn test_generate_proof<L: TrieLayout>(
 
 	(root, proof, items)
 }
+
+pub fn fuzz_batch_update(input: &[u8], build_val: fn(&mut Vec<u8>), compare_db: bool) {
+	let data = fuzz_to_data(input);
+	let mut data = fuzz_removal(data);
+	for i in data.iter_mut() {
+		build_val(&mut i.2);
+	}
+	let data = data;
+//println!("{}: {:?}", data.len(), data);
+	let mut db = memory_db::MemoryDB::<_, PrefixedKey<_>, _>::default();
+	let mut root = Default::default();
+	{
+		let mut t = reference_trie::RefTrieDBMutNoExt::new(&mut db, &mut root);
+		for i in 0..data.len() / 2 {
+			let key: &[u8]= &data[i].1;
+			let val: &[u8] = &data[i].2;
+			t.insert(key, val).unwrap();
+		}
+	}
+
+	let initial_root = root.clone();
+	let mut initial_db = db.clone();
+
+	let mut sorted_data = std::collections::BTreeMap::new();
+	{
+		let mut t = reference_trie::RefTrieDBMutNoExt::from_existing(&mut db, &mut root)
+			.unwrap();
+		for i in data.len() / 2..data.len() {
+			let key: &[u8]= &data[i].1;
+			let val: &[u8] = &data[i].2;
+			if !data[i].0 {
+				sorted_data.insert(key, Some(val));
+				t.insert(key, val).unwrap();
+			} else {
+				sorted_data.insert(key, None);
+				// remove overwrite insert from fuzz_removal ordering,
+				// that is important
+				t.remove(key).unwrap();
+			}
+		}
+	}
+//println!("{:?}", sorted_data);
+	let (calc_root, payload, _detached) = reference_trie::trie_traverse_key_no_extension_build(
+		&mut initial_db,
+		&initial_root,
+		sorted_data.into_iter(),
+	);
+	assert!(calc_root == root);
+
+	if compare_db {
+		for (p, h, v) in payload {
+			use hash_db::HashDB;
+			if let Some(v) = v {
+				let prefix = (p.0.as_ref(), p.1);
+				initial_db.emplace(h, prefix, v[..].into());
+			} else {
+				let prefix = (p.0.as_ref(), p.1);
+				initial_db.remove(&h, prefix);
+			}
+		}
+
+		assert!(initial_db == db);
+	}
+}
+
+pub fn fuzz_detach_attach(input: &[u8], build_val: fn(&mut Vec<u8>), compare_db: bool) {
+	//x: &[(Vec<u8>, Vec<u8>)],
+	//d: &Vec<u8>,
+	let mut data = fuzz_to_data(input);
+	if data.len() == 0 {
+		return;
+	}
+	for i in data.iter_mut() {
+		build_val(&mut i.1);
+	}
+	let x = &data[1..];
+	let d = &data[0].0;
+	//println!("{:?}\n d {:?}", x, d);
+	let mut db = MemoryDB::<KeccakHasher, PrefixedKey<_>, DBValue>::default();
+	let mut root = Default::default();
+	{
+		let mut t = reference_trie::RefTrieDBMutNoExt::new(&mut db, &mut root);
+		for i in 1..x.len() {
+			let key: &[u8]= &x[i].0;
+			let val: &[u8] = &x[i].1;
+			t.insert(key, val).unwrap();
+		}
+	}
+	let initial_root = root.clone();
+	let initial_db = db.clone();
+	// reference
+	{
+		let mut t = RefTrieDBMutNoExt::from_existing(&mut db, &mut root).unwrap();
+		for i in 1..x.len() {
+			if x[i].0.starts_with(d) {
+				let key: &[u8]= &x[i].0;
+				t.remove(key).unwrap();
+			}
+		}
+	}
+	let elements = Some(d.clone()).into_iter().map(|k| (k, InputAction::<Vec<u8>, _>::Detach));
+	let (calc_root, payload, payload_det, detached_root) = batch_update::<NoExtensionLayout, _, _, _, _>(
+		&initial_db,
+		&initial_root,
+		elements,
+	).unwrap();
+
+	assert_eq!(calc_root, root);
+
+	let mut batch_delta = initial_db.clone();
+	for (p, h, v) in payload.into_iter().chain(payload_det) {
+		use hash_db::HashDB;
+		if let Some(v) = v {
+			let prefix = (p.0.as_ref(), p.1);
+			batch_delta.emplace(h, prefix, v[..].into());
+		} else {
+			let prefix = (p.0.as_ref(), p.1);
+			batch_delta.remove(&h, prefix);
+		}
+	}
+
+	// attach back
+	let elements = detached_root.into_iter().map(|(k, _prefix, root)| (k, InputAction::<Vec<u8>, _>::Attach(root)));
+	let (calc_root, payload, payload_det, detached_root) = batch_update::<NoExtensionLayout, _, _, _, _>(
+		&batch_delta,
+		&calc_root,
+		elements,
+	).unwrap();
+	if detached_root.is_empty() {
+		if compare_db {
+			for (p, h, v) in payload.into_iter().chain(payload_det) {
+				use hash_db::HashDB;
+				if let Some(v) = v {
+					let prefix = (p.0.as_ref(), p.1);
+					batch_delta.emplace(h, prefix, v[..].into());
+				} else {
+					let prefix = (p.0.as_ref(), p.1);
+					batch_delta.remove(&h, prefix);
+				}
+			}
+			batch_delta.purge();
+			assert!(initial_db == batch_delta);
+		}
+		assert!(calc_root == initial_root);
+	} else {
+		// case where there was a node fuse due to dettach
+		// we could inject manually detached node and compare
+	}
+}
+
+
+#[test]
+fn test() {
+	let tests = [
+		vec![0x1,0x0,0x0,0x0,0x8,0xc,0x8,0x8,0x8,0x0,0x3d,0x0,0x9d,0x4,0x4e],
+		vec![0x1,0x0,0x0,0x0,0x8,0x8,0x8,0x8,0x8,0x0,0x80,0x0,0x9d,0x4,0x4e],
+		vec![0x0,0x80,0xd4,0xd4,0xd4,0xd4,0x3a,0x3a,0x3f,0x0,0x3f,0x0],
+		vec![0x0,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x9,0x1,0x8d,0x2c,0xd4,0x0,0x0,0x33,0x8a,0x20,0x80,0x9a,0x2c,0xd4,0x0,0x0,0x33,0x8a,0xff,0x8],
+		vec![0xff,0xd1,0x0,0x90,0x40,0xd4,0x8d,0x1,0x0,0x0,0xff,0x90,0x40,0xd4,0x8d,0x1,0x0,0x8d,0xce,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xd0,0xd0,0xd0,0xd0,0xd0,0xd0,0xd0,0xd0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x80,0x6,0x8,0x15,0x1,0x4,0x0,0x8d,0x87,0xcf,0x0,0x3f,0xcb,0xd8,0xb9,0xa2,0x4d,0x9a,0xd6,0xd2,0x0,0x0,0x0,0x0,0x80,0x0,0x6,0x8,0x15,0x1,0x4,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x23,0xc7,0xd7,0xd7,0xfb,0x7f,0x83,0x3,0x37,0x37,0x37,0xb2,0xa8,0xb,0xf5,0x5a,0x50,0xb6,0x0,0xff,0x17,0x21,0x0],
+		vec![0x43,0x19,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x3,0x0,0x0,0x6,0x0,0x0,0x0,0x0,0xaa,0x0,0x0,0x49,0x0,0x0,0xc7,0x8d,0x0,0x5b,0x2d,0xbd,0x20,0x0,0x0,0x0,0x0,0xc7,0x8d,0x0,0x5b,0x2d,0xbd,0x20,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xaa,0x0,0x0,0x49,0x1,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xc,0x0,0x80,0x0,0x0,0x0,0x0,0x0,0x0,0x4,0x40,0x2,0x4,0x0,0x0,0xc7,0x8d,0x0,0x0,0xe0],
+		vec![0x7f,0x2b,0x4,0x3a,0x89,0xfb,0x0,0x2e,0x70,0x0,0x0,0x2e,0x2,0x0,0x0,0x0,0x41,0xd1,0x2e,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x80,0xff,0xcd,0x72,0xfb,0x7f,0xc3,0x0,0x9a,0xff,0xff,0xff,0x72,0xfb,0x7f,0xff,0x0,0x0,0x0,0x0,0x0,0x82,0x82,0x81,0x81,0x29,0x8f,0x7d,0x42,0x12,0xf7,0xb4,0x77,0xd6,0x65,0x91,0xff,0x96,0xa9,0xe0,0x64,0xbc,0xc9,0x8a,0x70,0xc,0x4,0x0,0x0,0xc3,0x0,0x0,0x0,0x0,0x0,0x0],
+		vec![0x0,0x84,0xff,0xfb,0x7f,0xff,0xff,0xff,0xff,0x7f,0x70,0xff,0xff,0x7f,0x72,0xfd,0xc3,0x0,0x4,0xfb,0xff,0x10,0x10,0x10,0x10,0x10,0x9a,0x4],
+		vec![0x0,0x80,0xd4,0x0,0x1,0xc,0x0,0xc,0xd,0x2,0x9,0xd4,0xd4,0x0,0x8,0x8,0x8,0x1,0x0,0x0,0x0,0x0],
+		vec![0x0,0x80,0x0,0xd1,0x0,0x0,0x9a,0x4,0x4,0x0,0x0,0xc],
+		vec![0x0,0x0,0xff,0xff,0x0,0x0,0x4,0x8d,0x87,0xd1],
+		vec![0x0,0x0,0x0,0x0,0x4,0x0,0xff,0xd1,0x0,0x0,0xfe,0x0],
+		vec![0x0,0x0,0xfe,0x0,0xff,0xff,0xd1,0xd1,0x27,0x0],
+		vec![0x0,0xfe,0x41,0x0,0x0,0x80,0x0,0x0,0xff],
+		vec![0x0,0x0,0x0,0x4,0x20,0x1a,0x0,0x0],
+		vec![0x0,0x0,0x0,0x0,0xfe,0xff,0xff,0xd1,0x27],
+		vec![0x0,0x0,0x0,0xb7,0x4,0xf8,0x0,0x0,0x0],
+		vec![0xa,0x0,0x0,0x0,0x0,0x4,0x0,0x0],
+		vec![0x0,0x0,0x0,0x0,0x8d,0x4],
+		vec![0x0,0x0,0x4,0x8d,0x8d,0x4],
+	];
+	for v in tests.iter() {
+		fuzz_batch_update(&v[..], |_v| (), false);
+		fuzz_batch_update(&v[..], |v| v.extend(&[4u8; 32]), true);
+	}
+}

trie-db/src/iter_build.rs

@@ -200,7 +200,7 @@ impl<T, V> CacheAccum<T, V>
 			v.as_ref().map(|v| v.as_ref()),
 		);
 		self.reset_depth(branch_d);
-		let pr = NibbleSlice::new_offset(&key_branch, branch_d);
+		let pr = NibbleSlice::new_offset(key_branch, branch_d);
 		let branch_hash = callback.process(pr.left(), encoded, is_root && nkey.is_none());
 
 		if let Some(nkeyix) = nkey {

trie-db/src/lib.rs

@@ -51,6 +51,7 @@ pub mod recorder;
 mod fatdb;
 mod fatdbmut;
 mod iter_build;
+pub mod traverse;
 mod iterator;
 mod lookup;
 mod nibble;
@@ -66,7 +67,7 @@ pub use self::fatdb::{FatDB, FatDBIterator};
 pub use self::fatdbmut::FatDBMut;
 pub use self::recorder::{Recorder, Record};
 pub use self::lookup::Lookup;
-pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops};
+pub use self::nibble::{NibbleSlice, NibbleVec, nibble_ops, OwnedPrefix};
 pub use crate::node_codec::{NodeCodec, Partial};
 pub use crate::iter_build::{trie_visit, ProcessEncodedNode,
 	 TrieBuilder, TrieRoot, TrieRootUnhashed};
@@ -128,7 +129,6 @@ impl<T, E> Error for TrieError<T, E> where T: fmt::Debug, E: Error {}
 /// Boxed to avoid copying around extra space for the `Hasher`s `Out` on successful queries.
 pub type Result<T, H, E> = crate::rstd::result::Result<T, Box<TrieError<H, E>>>;
 
-
 /// Trie-Item type used for iterators over trie data.
 pub type TrieItem<'a, U, E> = Result<(Vec<u8>, DBValue), U, E>;
 
@@ -450,3 +450,8 @@ pub trait TrieConfiguration: Sized + TrieLayout {
 pub type TrieHash<L> = <<L as TrieLayout>::Hash as Hasher>::Out;
 /// Alias accessor to `NodeCodec` associated `Error` type from a `TrieLayout`.
 pub type CError<L> = <<L as TrieLayout>::Codec as NodeCodec>::Error;
+
+/// This was only implemented for trie without extension, it could
+/// be implemented for trie and extension in the future but is not
+/// at this point.
+const NO_EXTENSION_ONLY: &str = "trie without extension implemented only";

trie-db/src/nibble/leftnibbleslice.rs

@@ -226,4 +226,4 @@ mod tests {
 			Ordering::Equal
 		);
 	}
-}
+}

trie-db/src/nibble/mod.rs

@@ -151,6 +151,9 @@ pub mod nibble_ops {
 /// Backing storage for `NibbleVec`s.
 pub(crate) type BackingByteVec = smallvec::SmallVec<[u8; 36]>;
 
+/// Backing storage for `Prefix`.
+pub type OwnedPrefix = (BackingByteVec, Option<u8>);
+
 /// Owning, nibble-oriented byte vector. Counterpart to `NibbleSlice`.
 /// Nibbles are always left aligned, so making a `NibbleVec` from
 /// a `NibbleSlice` can get costy.