storage doublemap in decl_storage (#1918)

* factorization

* introduce GenericUnhashedStorage

* implement generator and storage

* impl double map in storage macro

* improve StorageDoubleMapXX methods

* remove storage from example and impl test

* remove old comments

* wasm compatible

* improve imports

* rename storages

* update runtime impl version

* make code less verbose

* impl hash config for second key in double map

hash available are all of Hashable trait

* use double map in decl_storage for contract

* fix double map config issue

* add hasher into metadata

* update impl version and build wasm

* doc

* add attrs

* update metadata version

* update runtime version

* fix unused storage

node/runtime/src/lib.rs

@@ -58,8 +58,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("node"),
 	impl_name: create_runtime_str!("substrate-node"),
 	authoring_version: 10,
-	spec_version: 47,
-	impl_version: 47,
+	spec_version: 48,
+	impl_version: 48,
 	apis: RUNTIME_API_VERSIONS,
 };
 

srml/example/src/lib.rs

@@ -66,7 +66,6 @@ decl_storage! {
 		// A map that has enumerable entries.
 		Bar get(bar) config(): linked_map T::AccountId => T::Balance;
 
-
 		// this one uses the default, we'll demonstrate the usage of 'mutate' API.
 		Foo get(foo) config(): T::Balance;
 	}

srml/metadata/src/lib.rs

@@ -263,6 +263,12 @@ pub enum StorageFunctionType {
 		value: DecodeDifferentStr,
 		is_linked: bool,
 	},
+	DoubleMap {
+		key1: DecodeDifferentStr,
+		key2: DecodeDifferentStr,
+		value: DecodeDifferentStr,
+		key2_hasher: DecodeDifferentStr,
+	},
 }
 
 /// A storage function modifier.
@@ -304,8 +310,10 @@ pub enum RuntimeMetadata {
 	V0(RuntimeMetadataDeprecated),
 	/// Version 1 for runtime metadata. No longer used.
 	V1(RuntimeMetadataDeprecated),
-	/// Version 2 for runtime metadata.
-	V2(RuntimeMetadataV2),
+	/// Version 2 for runtime metadata. No longer used.
+	V2(RuntimeMetadataDeprecated),
+	/// Version 3 for runtime metadata.
+	V3(RuntimeMetadataV3),
 }
 
 /// Enum that should fail.
@@ -325,10 +333,10 @@ impl Decode for RuntimeMetadataDeprecated {
 	}
 }
 
-/// The metadata of a runtime version 2.
+/// The metadata of a runtime.
 #[derive(Eq, Encode, PartialEq)]
 #[cfg_attr(feature = "std", derive(Decode, Debug, Serialize))]
-pub struct RuntimeMetadataV2 {
+pub struct RuntimeMetadataV3 {
 	pub modules: DecodeDifferentArray<ModuleMetadata>,
 }
 

srml/support/procedural/src/lib.rs

@@ -49,6 +49,8 @@ use proc_macro::TokenStream;
 /// * storage value: `Foo: type`: implements [StorageValue](https://crates.parity.io/srml_support/storage/trait.StorageValue.html)
 /// * storage map: `Foo: map type => type`: implements [StorageMap](https://crates.parity.io/srml_support/storage/trait.StorageMap.html)
 /// * storage linked map: `Foo: linked_map type => type`: implements [StorageMap](https://crates.parity.io/srml_support/storage/trait.StorageMap.html) and [EnumarableStorageMap](https://crates.parity.io/srml_support/storage/trait.EnumerableStorageMap.html)
+/// * storage double map: Foo: double_map u32, $hash(u32) => u32;` implements `StorageDoubleMap` with hasher $hash one available in `Hashable` trait
+/// /!\ be careful while choosing the Hash, indeed malicious could craft second keys to lower the trie.
 ///
 /// And it can be extended as such:
 ///
@@ -87,7 +89,7 @@ use proc_macro::TokenStream;
 /// ```
 /// This struct can be expose as `Config` by `decl_runtime` macro.
 ///
-/// ## Module with instances
+/// ### Module with instances
 ///
 /// `decl_storage!` macro support building modules with instances with the following syntax: (DefaultInstance type
 /// is optional)

srml/support/procedural/src/storage/impls.rs

@@ -541,4 +541,93 @@ impl<'a, I: Iterator<Item=syn::Meta>> Impls<'a, I> {
 			}
 		}
 	}
+
+	pub fn double_map(self, k1ty: &syn::Type, k2ty: &syn::Type, k2_hasher: TokenStream2) -> TokenStream2 {
+		let Self {
+			scrate,
+			visibility,
+			traitinstance,
+			traittype,
+			type_infos,
+			fielddefault,
+			prefix,
+			name,
+			attrs,
+			instance_opts,
+			..
+		} = self;
+
+		let DeclStorageTypeInfos { typ, value_type, is_option, .. } = type_infos;
+		let option_simple_1 = option_unwrap(is_option);
+
+		let as_double_map = quote!{ <Self as #scrate::storage::unhashed::generator::StorageDoubleMap<#k1ty, #k2ty, #typ>> };
+
+		let mutate_impl = if !is_option {
+			quote!{
+				#as_double_map::insert(key1, key2, &val, storage)
+			}
+		} else {
+			quote!{
+				match val {
+					Some(ref val) => #as_double_map::insert(key1, key2, &val, storage),
+					None => #as_double_map::remove(key1, key2, storage),
+				}
+			}
+		};
+
+		let InstanceOpts {
+			comma_instance,
+			equal_default_instance,
+			bound_instantiable,
+			instance,
+			..
+		} = instance_opts;
+
+		let final_prefix = if let Some(instance) = instance {
+			let const_name = syn::Ident::new(&format!("{}{}", PREFIX_FOR, name.to_string()), proc_macro2::Span::call_site());
+			quote!{ #instance::#const_name.as_bytes() }
+		} else {
+			quote!{ #prefix.as_bytes() }
+		};
+
+		// generator for double map
+		quote!{
+			#( #[ #attrs ] )*
+			#visibility struct #name<#traitinstance: #traittype, #instance #bound_instantiable #equal_default_instance>(#scrate::storage::generator::PhantomData<(#traitinstance #comma_instance)>);
+
+			impl<#traitinstance: #traittype, #instance #bound_instantiable> #scrate::storage::unhashed::generator::StorageDoubleMap<#k1ty, #k2ty, #typ> for #name<#traitinstance, #instance> {
+				type Query = #value_type;
+
+				fn prefix() -> &'static [u8] {
+					#final_prefix
+				}
+
+				fn key_for(k1: &#k1ty, k2: &#k2ty) -> Vec<u8> {
+					let mut key = #as_double_map::prefix_for(k1);
+					key.extend(&#scrate::Hashable::#k2_hasher(k2));
+					key
+				}
+
+				fn get<S: #scrate::GenericUnhashedStorage>(key1: &#k1ty, key2: &#k2ty, storage: &S) -> Self::Query {
+					let key = #as_double_map::key_for(key1, key2);
+					storage.get(&key).#option_simple_1(|| #fielddefault)
+				}
+
+				fn take<S: #scrate::GenericUnhashedStorage>(key1: &#k1ty, key2: &#k2ty, storage: &S) -> Self::Query {
+					let key = #as_double_map::key_for(key1, key2);
+					storage.take(&key).#option_simple_1(|| #fielddefault)
+				}
+
+				fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: #scrate::GenericUnhashedStorage>(key1: &#k1ty, key2: &#k2ty, f: F, storage: &S) -> R {
+					let mut val = #as_double_map::get(key1, key2, storage);
+
+					let ret = f(&mut val);
+					#mutate_impl ;
+					ret
+				}
+
+			}
+		}
+
+	}
 }

srml/support/procedural/src/storage/mod.rs

@@ -131,6 +131,7 @@ struct DeclStorageBuild {
 enum DeclStorageType {
 	Map(DeclStorageMap),
 	LinkedMap(DeclStorageLinkedMap),
+	DoubleMap(DeclStorageDoubleMap),
 	Simple(syn::Type),
 }
 
@@ -150,6 +151,24 @@ struct DeclStorageLinkedMap {
 	pub value: syn::Type,
 }
 
+#[derive(Parse, ToTokens, Debug)]
+struct DeclStorageDoubleMap {
+	pub map_keyword: ext::CustomToken<DoubleMapKeyword>,
+	pub key1: syn::Type,
+	pub comma_keyword: Token![,],
+	pub key2_hasher: DeclStorageDoubleMapHasher,
+	pub key2: ext::Parens<syn::Type>,
+	pub ass_keyword: Token![=>],
+	pub value: syn::Type,
+}
+
+#[derive(Parse, ToTokens, Debug)]
+enum DeclStorageDoubleMapHasher {
+	Blake2_256(ext::CustomToken<Blake2_256Keyword>),
+	Twox256(ext::CustomToken<Twox256Keyword>),
+	Twox128(ext::CustomToken<Twox128Keyword>),
+}
+
 #[derive(Parse, ToTokens, Debug)]
 struct DeclStorageDefault {
 	pub equal_token: Token![=],
@@ -165,4 +184,8 @@ custom_keyword_impl!(AddExtraGenesis, "add_extra_genesis", "storage extra genesi
 custom_keyword_impl!(DeclStorageGetter, "get", "storage getter");
 custom_keyword!(MapKeyword, "map", "map as keyword");
 custom_keyword!(LinkedMapKeyword, "linked_map", "linked_map as keyword");
+custom_keyword!(DoubleMapKeyword, "double_map", "double_map as keyword");
+custom_keyword!(Blake2_256Keyword, "blake2_256", "Blake2_256 as keyword");
+custom_keyword!(Twox256Keyword, "twox_256", "Twox_256 as keyword");
+custom_keyword!(Twox128Keyword, "twox_128", "Twox_128 as keyword");
 custom_keyword_impl!(ExtraGenesisSkipPhantomDataField, "extra_genesis_skip_phantom_data_field", "extra_genesis_skip_phantom_data_field as keyword");

srml/support/procedural/src/storage/transformation.rs

@@ -255,6 +255,9 @@ fn decl_store_extra_genesis(
 				DeclStorageTypeInfosKind::Map {key_type, .. } => {
 					quote!( #( #[ #attrs ] )* pub #ident: Vec<(#key_type, #storage_type)>, )
 				},
+				DeclStorageTypeInfosKind::DoubleMap {key1_type, key2_type, .. } => {
+					quote!( #( #[ #attrs ] )* pub #ident: Vec<(#key1_type, #key2_type, #storage_type)>, )
+				},
 			});
 			opt_build = Some(build.as_ref().map(|b| &b.expr.content).map(|b|quote!( #b ))
 				.unwrap_or_else(|| quote!( (|config: &GenesisConfig<#traitinstance, #instance>| config.#ident.clone()) )));
@@ -295,6 +298,17 @@ fn decl_store_extra_genesis(
 						}
 					}}
 				},
+				DeclStorageTypeInfosKind::DoubleMap { key1_type, key2_type, .. } => {
+					quote!{{
+						use #scrate::rstd::{cell::RefCell, marker::PhantomData};
+						use #scrate::codec::{Encode, Decode};
+
+						let data = (#builder)(&self);
+						for (k1, k2, v) in data.into_iter() {
+							<#name<#traitinstance, #instance> as #scrate::storage::unhashed::generator::StorageDoubleMap<#key1_type, #key2_type, #typ>>::insert(&k1, &k2, &v, &storage);
+						}
+					}}
+				},
 			});
 		}
 
@@ -581,6 +595,9 @@ fn decl_storage_items(
 			DeclStorageTypeInfosKind::Map { key_type, is_linked: true } => {
 				i.linked_map(key_type)
 			},
+			DeclStorageTypeInfosKind::DoubleMap { key1_type, key2_type, key2_hasher } => {
+				i.double_map(key1_type, key2_type, key2_hasher)
+			},
 		};
 		impls.extend(implementation)
 	}
@@ -657,6 +674,17 @@ fn impl_store_fns(
 						}
 					}
 				}
+				DeclStorageTypeInfosKind::DoubleMap { key1_type, key2_type, .. } => {
+					quote!{
+						pub fn #get_fn<KArg1, KArg2>(k1: KArg1, k2: KArg2) -> #value_type
+						where
+							KArg1: #scrate::storage::generator::Borrow<#key1_type>,
+							KArg2: #scrate::storage::generator::Borrow<#key2_type>,
+						{
+							<#name<#traitinstance> as #scrate::storage::unhashed::generator::StorageDoubleMap<#key1_type, #key2_type, #typ>> :: get(k1.borrow(), k2.borrow(), &#scrate::storage::RuntimeStorage)
+						}
+					}
+				}
 			};
 			items.extend(item);
 		}
@@ -714,6 +742,19 @@ fn store_functions_to_metadata (
 					}
 				}
 			},
+			DeclStorageTypeInfosKind::DoubleMap { key1_type, key2_type, key2_hasher } => {
+				let k1ty = clean_type_string(&quote!(#key1_type).to_string());
+				let k2ty = clean_type_string(&quote!(#key2_type).to_string());
+				let k2_hasher = clean_type_string(&key2_hasher.to_string());
+				quote!{
+					#scrate::storage::generator::StorageFunctionType::DoubleMap {
+						key1: #scrate::storage::generator::DecodeDifferent::Encode(#k1ty),
+						key2: #scrate::storage::generator::DecodeDifferent::Encode(#k2ty),
+						value: #scrate::storage::generator::DecodeDifferent::Encode(#styp),
+						key2_hasher: #scrate::storage::generator::DecodeDifferent::Encode(#k2_hasher),
+					}
+				}
+			},
 		};
 		let modifier = if type_infos.is_option {
 			quote!{
@@ -810,6 +851,11 @@ enum DeclStorageTypeInfosKind<'a> {
 		key_type: &'a syn::Type,
 		is_linked: bool,
 	},
+	DoubleMap {
+		key1_type: &'a syn::Type,
+		key2_type: &'a syn::Type,
+		key2_hasher: TokenStream2,
+	}
 }
 
 impl<'a> DeclStorageTypeInfosKind<'a> {
@@ -832,6 +878,11 @@ fn get_type_infos(storage_type: &DeclStorageType) -> DeclStorageTypeInfos {
 			key_type: &map.key,
 			is_linked: true,
 		}),
+		DeclStorageType::DoubleMap(ref map) => (&map.value, DeclStorageTypeInfosKind::DoubleMap {
+			key1_type: &map.key1,
+			key2_type: &map.key2.content,
+			key2_hasher: { let h = &map.key2_hasher; quote! { #h } },
+		}),
 	};
 
 	let extracted_type = ext::extract_type_option(value_type);

srml/support/src/double_map.rs

@@ -31,7 +31,9 @@ use sr_std::borrow::Borrow;
 /// The storage key (i.e. the key under which the `Value` will be stored) is created from two parts.
 /// The first part is a hash of a concatenation of the `PREFIX` and `Key1`. And the second part
 /// is a hash of a `Key2`.
-pub trait StorageDoubleMap {
+///
+/// Hasher are implemented in derive_key* methods.
+pub trait StorageDoubleMapWithHasher {
 	type Key1: Codec;
 	type Key2: Codec;
 	type Value: Codec + Default;

srml/support/src/hashable.rs

@@ -27,12 +27,12 @@ pub trait Hashable: Sized {
 
 impl<T: Codec> Hashable for T {
 	fn blake2_256(&self) -> [u8; 32] {
-		blake2_256(&self.encode())
+		self.using_encoded(blake2_256)
 	}
 	fn twox_128(&self) -> [u8; 16] {
-		twox_128(&self.encode())
+		self.using_encoded(twox_128)
 	}
 	fn twox_256(&self) -> [u8; 32] {
-		twox_256(&self.encode())
+		self.using_encoded(twox_256)
 	}
 }