network: Use "one shot" protocol handler.

Add two new `NetworkBehaviour`s, one handling remote block requests
and another one to handle light client requests (both local and from
remote). The change is motivated by the desire to use multiple
substreams of a single connection for different protocols. To achieve
this, libp2p's `OneShotHandler` is used as a protocol handler in each
behaviour. It will open a fresh substream for the duration of the
request and close it afterwards. For block requests, we currently only
handle incoming requests from remote and tests are missing. For light
client handling we support incoming requests from remote and also
ported a substantial amount of functionality over from
`light_dispatch.rs` (including several tests). However the result lacks
in at least two aspects:

(1) We require external updates w.r.t. the best block per peer and
currently nothing updates this information.
(2) We carry a lot of peer-related state around.

Both aspects could be simplified by externalising peer selection and
just requiring a specific peer ID where the request should be sent to.
We still have to maintain some peer related state due to the way
libp2p's swarm and network behaviour work (e.g. we must make sure to
always issue `NetworkBehaviourAction::SendEvent`s to peers we are
connected to, otherwise the actions die a silent death.

Another change implemented here is the use of protocol buffers as the
encoding for network messages. Certain individual fields of messages
are still SCALE encoded. There has been some discussion about this
in another PR (paritytech#3452), so
far without resolution.

core/network/Cargo.toml

@@ -6,6 +6,9 @@ license = "GPL-3.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 
+[build-dependencies]
+prost-build = "0.5"
+
 [dependencies]
 bytes = "0.4"
 derive_more = "0.14.0"
@@ -30,6 +33,7 @@ sr-primitives = { path = "../../core/sr-primitives" }
 primitives = { package = "substrate-primitives", path = "../../core/primitives" }
 codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
 peerset = { package = "substrate-peerset", path = "../../core/peerset" }
+prost = "0.5"
 serde = { version = "1.0.70", features = ["derive"] }
 serde_json = "1.0.24"
 slog = { version = "^2", features = ["nested-values"] }
@@ -47,6 +51,7 @@ zeroize = "0.9.0"
 babe-primitives = { package = "substrate-consensus-babe-primitives", path = "../consensus/babe/primitives" }
 
 [dev-dependencies]
+assert_matches = "1.3.0"
 env_logger = { version = "0.6" }
 keyring = { package = "substrate-keyring", path = "../../core/keyring" }
 quickcheck = "0.8.5"

core/network/build.rs

@@ -0,0 +1,8 @@
+const PROTOS: &[&str] = &[
+	"src/protocol/schema/api.v1.proto",
+	"src/protocol/schema/light.v1.proto"
+];
+
+fn main() {
+	prost_build::compile_protos(PROTOS, &["src/protocol"]).unwrap();
+}

core/network/src/behaviour.rs

@@ -19,7 +19,7 @@ use crate::{
 	protocol::event::DhtEvent
 };
 use crate::{ExHashT, specialization::NetworkSpecialization};
-use crate::protocol::{CustomMessageOutcome, Protocol};
+use crate::protocol::{self, CustomMessageOutcome, Protocol};
 use futures::prelude::*;
 use libp2p::NetworkBehaviour;
 use libp2p::core::{Multiaddr, PeerId, PublicKey};
@@ -42,6 +42,10 @@ pub struct Behaviour<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> {
 	debug_info: debug_info::DebugInfoBehaviour<Substream<StreamMuxerBox>>,
 	/// Discovers nodes of the network.
 	discovery: DiscoveryBehaviour<Substream<StreamMuxerBox>>,
+	/// Block request handling.
+	block_requests: protocol::BlockRequests<Substream<StreamMuxerBox>, B>,
+	/// Light client request handling.
+	light_client_handler: protocol::LightClientHandler<Substream<StreamMuxerBox>, B>,
 
 	/// Queue of events to produce for the outside.
 	#[behaviour(ignore)]
@@ -62,12 +66,16 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Behaviour<B, S, H> {
 		local_public_key: PublicKey,
 		known_addresses: Vec<(PeerId, Multiaddr)>,
 		enable_mdns: bool,
+		block_requests: protocol::BlockRequests<Substream<StreamMuxerBox>, B>,
+		light_client_handler: protocol::LightClientHandler<Substream<StreamMuxerBox>, B>
 	) -> Self {
 		Behaviour {
 			substrate,
 			debug_info: debug_info::DebugInfoBehaviour::new(user_agent, local_public_key.clone()),
 			discovery: DiscoveryBehaviour::new(local_public_key, known_addresses, enable_mdns),
-			events: Vec::new(),
+			block_requests,
+			light_client_handler,
+			events: Vec::new()
 		}
 	}
 
@@ -109,6 +117,11 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Behaviour<B, S, H> {
 	pub fn put_value(&mut self, key: record::Key, value: Vec<u8>) {
 		self.discovery.put_value(key, value);
 	}
+
+	/// Get unique access to the light client handler.
+	pub fn light_client_handler(&mut self) -> &mut protocol::LightClientHandler<Substream<StreamMuxerBox>, B> {
+		&mut self.light_client_handler
+	}
 }
 
 impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> NetworkBehaviourEventProcess<void::Void> for

core/network/src/protocol.rs

@@ -49,14 +49,29 @@ use client::light::fetcher::{FetchChecker, ChangesProof};
 use crate::error;
 use util::LruHashSet;
 
+// Include sources generated from protobuf definitions.
+pub mod api {
+	pub mod v1 {
+		include!(concat!(env!("OUT_DIR"), "/api.v1.rs"));
+		pub mod light {
+			include!(concat!(env!("OUT_DIR"), "/api.v1.light.rs"));
+		}
+	}
+}
+
 mod util;
+pub mod block_requests;
 pub mod consensus_gossip;
 pub mod message;
 pub mod event;
+pub mod light_client_handler;
 pub mod light_dispatch;
 pub mod specialization;
 pub mod sync;
 
+pub use block_requests::BlockRequests;
+pub use light_client_handler::LightClientHandler;
+
 const REQUEST_TIMEOUT_SEC: u64 = 40;
 /// Interval at which we perform time based maintenance
 const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100);