remote-externalities: batch insert key/values (paritytech#14004)

* add batch inserting into remote externalities

* use into_iter

* remove redundant comment

* redundant batch insert

* avoid extra vec allocations

primitives/state-machine/src/testing.rs

@@ -132,6 +132,17 @@ where
 		self.offchain_db.clone()
 	}
 
+	/// Batch insert key/values into backend
+	pub fn batch_insert<I>(&mut self, kvs: I)
+	where
+		I: IntoIterator<Item = (StorageKey, StorageValue)>,
+	{
+		self.backend.insert(
+			Some((None, kvs.into_iter().map(|(k, v)| (k, Some(v))).collect())),
+			self.state_version,
+		);
+	}
+
 	/// Insert key/value into backend
 	pub fn insert(&mut self, k: StorageKey, v: StorageValue) {
 		self.backend.insert(vec![(None, vec![(k, Some(v))])], self.state_version);

utils/frame/remote-externalities/src/lib.rs

@@ -638,24 +638,20 @@ where
 		let mut processed = 0usize;
 		loop {
 			match rx.next().await.unwrap() {
-				Message::Batch(kv) => {
-					for (k, v) in kv {
-						processed += 1;
-						if processed % 50_000 == 0 || processed == keys.len() || processed == 1 {
-							log::info!(
-								target: LOG_TARGET,
-								"inserting keys progress = {:.0}% [{} / {}]",
-								(processed as f32 / keys.len() as f32) * 100f32,
-								processed,
-								keys.len(),
-							);
-						}
-						// skip writing the child root data.
-						if is_default_child_storage_key(k.as_ref()) {
-							continue
-						}
-						pending_ext.insert(k, v);
-					}
+				Message::Batch(kvs) => {
+					let kvs = kvs
+						.into_iter()
+						.filter(|(k, _)| !is_default_child_storage_key(k))
+						.collect::<Vec<_>>();
+					processed += kvs.len();
+					pending_ext.batch_insert(kvs);
+					log::info!(
+						target: LOG_TARGET,
+						"inserting keys progress = {:.0}% [{} / {}]",
+						(processed as f32 / keys.len() as f32) * 100f32,
+						processed,
+						keys.len(),
+					);
 				},
 				Message::BatchFailed(error) => {
 					log::error!(target: LOG_TARGET, "Batch processing failed: {:?}", error);
@@ -1010,13 +1006,12 @@ where
 		);
 
 		info!(target: LOG_TARGET, "injecting a total of {} top keys", top.len());
-		for (k, v) in top {
-			// skip writing the child root data.
-			if is_default_child_storage_key(k.as_ref()) {
-				continue
-			}
-			inner_ext.insert(k.0, v.0);
-		}
+		let top = top
+			.into_iter()
+			.filter(|(k, _)| !is_default_child_storage_key(k.as_ref()))
+			.map(|(k, v)| (k.0, v.0))
+			.collect::<Vec<_>>();
+		inner_ext.batch_insert(top);
 
 		info!(
 			target: LOG_TARGET,
@@ -1052,9 +1047,7 @@ where
 				"extending externalities with {} manually injected key-values",
 				self.hashed_key_values.len()
 			);
-			for (k, v) in self.hashed_key_values {
-				ext.insert(k.0, v.0);
-			}
+			ext.batch_insert(self.hashed_key_values.into_iter().map(|(k, v)| (k.0, v.0)));
 		}
 
 		// exclude manual key values.