From 69ef3fabdfd63f48c41efb260f478ed07fafc99f Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 26 Oct 2020 18:57:41 +0100 Subject: [PATCH 01/48] client/network: Add scaffolding for finality req to use req resp #sc --- client/network/src/behaviour.rs | 41 ++++++-- client/network/src/protocol.rs | 17 +++- client/network/src/protocol/sync.rs | 38 ++++---- .../src/protocol/sync/extra_requests.rs | 94 ++++++++++++++++--- client/network/src/request_responses.rs | 9 ++ client/network/src/service.rs | 43 ++++++++- 6 files changed, 198 insertions(+), 44 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index c8684eba625c5..e8a460f904c6c 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -29,6 +29,7 @@ use libp2p::identify::IdentifyInfo; use libp2p::kad::record; use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters}; use log::debug; +use prost::Message; use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}}; use sp_runtime::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId, Justification}; use std::{ @@ -359,7 +360,23 @@ Behaviour { } }, CustomMessageOutcome::FinalityProofRequest { target, block_hash, request } => { - self.finality_proof_requests.send_request(&target, block_hash, request); + let protobuf_rq = crate::schema::v1::finality::FinalityProofRequest { + block_hash: block_hash.encode(), + request, + }; + + let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); + if let Err(err) = protobuf_rq.encode(&mut buf) { + log::warn!("failed to encode finality proof request {:?}: {:?}", protobuf_rq, err); + return; + } + self.request_responses.send_request(&target, "abc", buf).unwrap(); + + + + + + // self.finality_proof_requests.send_request(&target, block_hash, request); }, CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles, notifications_sink } => { let role = reported_roles_to_observed_role(&self.role, &remote, roles); @@ -408,12 +425,18 @@ impl NetworkBehaviourEventProcess { - self.events.push_back(BehaviourOut::RequestFinished { - request_id, - result, - }); + request_responses::Event::RequestFinished { peer, protocol, request_id, result } => { + if protocol == "abc" { + let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) + .unwrap(); + let ev = self.substrate.on_finality_proof_response(peer, request_id, proof_response.proof); + self.inject_event(ev); + } else { + self.events.push_back(BehaviourOut::RequestFinished { + request_id, + result, + }); + } }, } } @@ -466,8 +489,8 @@ impl NetworkBehaviourEventProcess Protocol { pub fn on_finality_proof_response( &mut self, who: PeerId, - response: message::FinalityProofResponse, + request_id: libp2p::request_response::RequestId, + proof: Vec, ) -> CustomMessageOutcome { - trace!(target: "sync", "Finality proof response from {} for {}", who, response.block); - match self.sync.on_block_finality_proof(who, response) { + match self.sync.on_block_finality_proof(who, request_id, proof) { Ok(sync::OnBlockFinalityProof::Nothing) => CustomMessageOutcome::None, Ok(sync::OnBlockFinalityProof::Import { peer, hash, number, proof }) => CustomMessageOutcome::FinalityProofImport(peer, hash, number, proof), @@ -1384,6 +1384,17 @@ impl Protocol { } } + // TODO: Comment + // Informs sync of the request id. + pub fn on_finality_proof_request_started( + &mut self, + who: PeerId, + block_hash: B::Hash, + request_id: libp2p::request_response::RequestId, + ) { + self.sync.on_finality_proof_request_started(who, block_hash, request_id) + } + fn format_stats(&self) -> String { let mut out = String::new(); for (id, stats) in &self.context_data.stats { diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 03714b05ace0d..28f527da68ed9 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -1033,10 +1033,22 @@ impl ChainSync { Ok(OnBlockJustification::Nothing) } + pub fn on_finality_proof_request_started( + &mut self, + who: PeerId, + block_hash: B::Hash, + request_id: libp2p::request_response::RequestId, + ) { + self.extra_finality_proofs.on_request_started(who, block_hash, request_id) + } + /// Handle new finality proof data. - pub fn on_block_finality_proof - (&mut self, who: PeerId, resp: FinalityProofResponse) -> Result, BadPeer> - { + pub fn on_block_finality_proof( + &mut self, + who: PeerId, + request_id: libp2p::request_response::RequestId, + proof: Vec, + ) -> Result, BadPeer> { let peer = if let Some(peer) = self.peers.get_mut(&who) { peer @@ -1046,22 +1058,14 @@ impl ChainSync { }; self.pending_requests.add(&who); - if let PeerSyncState::DownloadingFinalityProof(hash) = peer.state { + if let PeerSyncState::DownloadingFinalityProof(state_hash) = peer.state { peer.state = PeerSyncState::Available; - // We only request one finality proof at a time. - if hash != resp.block { - info!( - target: "sync", - "💔 Invalid block finality proof provided: requested: {:?} got: {:?}", - hash, - resp.block - ); - return Err(BadPeer(who, rep::BAD_FINALITY_PROOF)); - } - - if let Some((peer, hash, number, p)) = self.extra_finality_proofs.on_response(who, resp.proof) { - return Ok(OnBlockFinalityProof::Import { peer, hash, number, proof: p }) + if let Some((peer, hash, number, p)) = self.extra_finality_proofs.on_response(who, if proof.is_empty() { None } else { Some(proof) }) { + trace!(target: "sync", "Finality proof response from {} for {}", peer, hash); + // TODO: Safe assumption to make? + assert_eq!(hash, state_hash); + return Ok(OnBlockFinalityProof::Import { peer, hash, number, proof: p}) } } diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index df336c25339fd..8d5569734da9c 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -46,7 +46,7 @@ pub(crate) struct ExtraRequests { /// requests which have been queued for later processing pending_requests: VecDeque>, /// requests which are currently underway to some peer - active_requests: HashMap>, + active_requests: HashMap>, /// previous requests without response failed_requests: HashMap, Vec<(PeerId, Instant)>>, /// successful requests @@ -55,6 +55,13 @@ pub(crate) struct ExtraRequests { request_type_name: &'static str, } +#[derive(Debug)] +struct ActiveRequest { + request_id: Option, + block_hash: ::Hash, + number: NumberFor, +} + #[derive(Debug)] pub(crate) struct Metrics { pub(crate) pending_requests: u32, @@ -114,8 +121,21 @@ impl ExtraRequests { /// Retry any pending request if a peer disconnected. pub(crate) fn peer_disconnected(&mut self, who: &PeerId) { - if let Some(request) = self.active_requests.remove(who) { - self.pending_requests.push_front(request); + if let Some(ActiveRequest { block_hash, number, ..}) = self.active_requests.remove(who) { + self.pending_requests.push_front((block_hash, number)); + } + } + + pub(crate) fn on_request_started( + &mut self, + who: PeerId, + block_hash: B::Hash, + request_id: libp2p::request_response::RequestId, + ) { + if let Some(req) = self.active_requests.get_mut(&who) { + if req.block_hash == block_hash { + req.request_id = Some(request_id); + } } } @@ -124,25 +144,69 @@ impl ExtraRequests { // we assume that the request maps to the given response, this is // currently enforced by the outer network protocol before passing on // messages to chain sync. - if let Some(request) = self.active_requests.remove(&who) { + if let Some(ActiveRequest { block_hash, number, ..}) = self.active_requests.remove(&who) { if let Some(r) = resp { trace!(target: "sync", "Queuing import of {} from {:?} for {:?}", self.request_type_name, who, - request, + (block_hash, number), ); - self.importing_requests.insert(request); - return Some((who, request.0, request.1, r)) + self.importing_requests.insert((block_hash, number)); + return Some((who, block_hash, number, r)) } else { trace!(target: "sync", "Empty {} response from {:?} for {:?}", self.request_type_name, who, - request, + (block_hash, number), + ); + } + self.failed_requests.entry((block_hash, number)).or_default().push((who, Instant::now())); + self.pending_requests.push_front((block_hash, number)); + } else { + trace!(target: "sync", "No active {} request to {:?}", + self.request_type_name, + who, + ); + } + None + } + + /// Processes the response for the request previously sent to the given peer. + pub(crate) fn on_response_with_request_id( + &mut self, + who: PeerId, + request_id: libp2p::request_response::RequestId, + // TODO: Is it needed to send the response through this function? + resp: Option, + // TODO: Why return the peer id once more? + ) -> Option<(PeerId, B::Hash, NumberFor, R)> { + if let Some(req) = self.active_requests.get(&who) { + if req.request_id != Some(request_id) { + // TODO: Should we log something? + return None; + } + } + + if let Some(ActiveRequest { request_id, block_hash, number }) = self.active_requests.remove(&who) { + if let Some(r) = resp { + trace!(target: "sync", "Queuing import of {} from {:?} for {:?}", + self.request_type_name, + who, + (block_hash, number), + ); + + self.importing_requests.insert((block_hash, number)); + return Some((who, block_hash, number, r)) + } else { + trace!(target: "sync", "Empty {} response from {:?} for {:?}", + self.request_type_name, + who, + (block_hash, number), ); } - self.failed_requests.entry(request).or_default().push((who, Instant::now())); - self.pending_requests.push_front(request); + self.failed_requests.entry((block_hash, number)).or_default().push((who, Instant::now())); + self.pending_requests.push_front((block_hash, number)); } else { trace!(target: "sync", "No active {} request to {:?}", self.request_type_name, @@ -190,7 +254,7 @@ impl ExtraRequests { let roots = self.tree.roots().collect::>(); self.pending_requests.retain(|(h, n)| roots.contains(&(h, n, &()))); - self.active_requests.retain(|_, (h, n)| roots.contains(&(h, n, &()))); + self.active_requests.retain(|_, ActiveRequest { block_hash, number, ..}| roots.contains(&(block_hash, number, &()))); self.failed_requests.retain(|(h, n), _| roots.contains(&(h, n, &()))); Ok(()) @@ -240,7 +304,7 @@ impl ExtraRequests { /// Returns an iterator over all active (in-flight) requests and associated peer id. #[cfg(test)] - pub(crate) fn active_requests(&self) -> impl Iterator)> { + pub(crate) fn active_requests(&self) -> impl Iterator)> { self.active_requests.iter() } @@ -318,7 +382,11 @@ impl<'a, B: BlockT> Matcher<'a, B> { if self.extras.failed_requests.get(&request).map(|rr| rr.iter().any(|i| &i.0 == peer)).unwrap_or(false) { continue } - self.extras.active_requests.insert(peer.clone(), request); + self.extras.active_requests.insert(peer.clone(), ActiveRequest{ + request_id: None, + block_hash: request.0, + number: request.1, + }); trace!(target: "sync", "Sending {} request to {:?} for {:?}", self.extras.request_type_name, diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 5141e6db70141..76d9098dc1ed5 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -134,6 +134,9 @@ pub enum Event { /// A request initiated using [`RequestResponsesBehaviour::send_request`] has succeeded or /// failed. RequestFinished { + peer: PeerId, + /// Name of the protocol in question. + protocol: Cow<'static, str>, /// Request that has succeeded. request_id: RequestId, /// Response sent by the remote or reason for failure. @@ -445,6 +448,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour { // Received a response from a remote to one of our requests. RequestResponseEvent::Message { + peer, message: RequestResponseMessage::Response { request_id, @@ -453,6 +457,8 @@ impl NetworkBehaviour for RequestResponsesBehaviour { .. } => { let out = Event::RequestFinished { + peer, + protocol: protocol.clone(), request_id, result: response.map_err(|()| RequestFailure::Refused), }; @@ -461,11 +467,14 @@ impl NetworkBehaviour for RequestResponsesBehaviour { // One of our requests has failed. RequestResponseEvent::OutboundFailure { + peer, request_id, error, .. } => { let out = Event::RequestFinished { + peer, + protocol: protocol.clone(), request_id, result: Err(RequestFailure::Network(error)), }; diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 9cb37e7700f3a..d6acbf768956c 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -51,6 +51,7 @@ use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent, protocols_handle use log::{error, info, trace, warn}; use metrics::{Metrics, MetricSources, Histogram, HistogramVec}; use parking_lot::Mutex; +use prost::Message; use sc_peerset::PeersetHandle; use sp_consensus::import_queue::{BlockImportError, BlockImportResult, ImportQueue, Link}; use sp_runtime::{ @@ -117,7 +118,7 @@ impl NetworkWorker { /// Returns a `NetworkWorker` that implements `Future` and must be regularly polled in order /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. - pub fn new(params: Params) -> Result, Error> { + pub fn new(mut params: Params) -> Result, Error> { // Ensure the listen addresses are consistent with the transport. ensure_addresses_consistent_with_transport( params.network_config.listen_addresses.iter(), @@ -286,6 +287,44 @@ impl NetworkWorker { ) }; + // TODO: Should finality request protocol be added here? + let mut request_response_protocols = params.network_config.request_response_protocols; + let (tx, mut rx) = futures::channel::mpsc::channel(10); + request_response_protocols.push(crate::request_responses::ProtocolConfig { + name: std::borrow::Cow::Borrowed(&"abc"), + // TODO: Change. + max_request_size: 4096, + // TODO: Change. + max_response_size: 4096, + request_timeout: std::time::Duration::from_secs(10), + inbound_queue: Some(tx), + }); + + let finality_proof_provider = params.finality_proof_provider.clone(); + let finality_proof_server = async move { + let finality_proof_provider = finality_proof_provider.unwrap(); + while let Some(crate::request_responses::IncomingRequest { peer, payload, pending_response }) = rx.next().await { + let req = crate::schema::v1::finality::FinalityProofRequest::decode(&payload[..]).unwrap(); + + let block_hash = codec::Decode::decode(&mut req.block_hash.as_ref()).unwrap(); + + log::trace!(target: "sync", "Finality proof request from {} for {}", peer, block_hash); + + let finality_proof = finality_proof_provider + .prove_finality(block_hash, &req.request) + .unwrap() + .unwrap_or_default(); + + let resp = crate::schema::v1::finality::FinalityProofResponse { proof: finality_proof }; + let mut data = Vec::with_capacity(resp.encoded_len()); + resp.encode(&mut data).unwrap(); + + pending_response.send(data); + } + }.boxed(); + + params.executor.as_mut().unwrap()(finality_proof_server); + let discovery_config = { let mut config = DiscoveryConfig::new(local_public.clone()); config.with_user_defined(known_addresses); @@ -318,7 +357,7 @@ impl NetworkWorker { finality_proof_requests, light_client_handler, discovery_config, - params.network_config.request_response_protocols, + request_response_protocols, ); match result { From e8df53516640c6e1ff06952d65a5712a925e9f12 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 26 Oct 2020 19:20:08 +0100 Subject: [PATCH 02/48] client/network/src/finality_requests: Remove --- client/network/src/behaviour.rs | 28 +- client/network/src/finality_requests.rs | 403 ------------------------ client/network/src/lib.rs | 1 - client/network/src/protocol/sync.rs | 2 +- client/network/src/service.rs | 21 +- 5 files changed, 11 insertions(+), 444 deletions(-) delete mode 100644 client/network/src/finality_requests.rs diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index e8a460f904c6c..cf6b60df5ce77 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -15,8 +15,8 @@ // along with Substrate. If not, see . use crate::{ - config::{ProtocolId, Role}, block_requests, light_client_handler, finality_requests, - peer_info, request_responses, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, + config::{ProtocolId, Role}, block_requests, light_client_handler, peer_info, request_responses, + discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, protocol::{message::{self, Roles}, CustomMessageOutcome, NotificationsSink, Protocol}, ObservedRole, DhtEvent, ExHashT, }; @@ -59,8 +59,6 @@ pub struct Behaviour { request_responses: request_responses::RequestResponsesBehaviour, /// Block request handling. block_requests: block_requests::BlockRequests, - /// Finality proof request handling. - finality_proof_requests: finality_requests::FinalityProofRequests, /// Light client request handling. light_client_handler: light_client_handler::LightClientHandler, @@ -183,7 +181,6 @@ impl Behaviour { user_agent: String, local_public_key: PublicKey, block_requests: block_requests::BlockRequests, - finality_proof_requests: finality_requests::FinalityProofRequests, light_client_handler: light_client_handler::LightClientHandler, disco_config: DiscoveryConfig, request_response_protocols: Vec, @@ -195,7 +192,6 @@ impl Behaviour { request_responses: request_responses::RequestResponsesBehaviour::new(request_response_protocols.into_iter())?, block_requests, - finality_proof_requests, light_client_handler, events: VecDeque::new(), role, @@ -476,26 +472,6 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess> for Behaviour { - fn inject_event(&mut self, event: finality_requests::Event) { - match event { - finality_requests::Event::Response { peer, block_hash, proof } => { - let response = message::FinalityProofResponse { - id: 0, - block: block_hash, - proof: if !proof.is_empty() { - Some(proof) - } else { - None - }, - }; - // let ev = self.substrate.on_finality_proof_response(peer, response); - // self.inject_event(ev); - } - } - } -} - impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: peer_info::PeerInfoEvent) { diff --git a/client/network/src/finality_requests.rs b/client/network/src/finality_requests.rs deleted file mode 100644 index 55f56b9a0cc25..0000000000000 --- a/client/network/src/finality_requests.rs +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. -// -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! `NetworkBehaviour` implementation which handles incoming finality proof requests. -//! -//! Every request is coming in on a separate connection substream which gets -//! closed after we have sent the response back. Incoming requests are encoded -//! as protocol buffers (cf. `finality.v1.proto`). - -#![allow(unused)] - -use bytes::Bytes; -use codec::{Encode, Decode}; -use crate::{ - chain::FinalityProofProvider, - config::ProtocolId, - protocol::message, - schema, -}; -use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; -use libp2p::{ - core::{ - ConnectedPoint, - Multiaddr, - PeerId, - connection::ConnectionId, - upgrade::{InboundUpgrade, OutboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, - upgrade::{DeniedUpgrade, read_one, write_one} - }, - swarm::{ - NegotiatedSubstream, - NetworkBehaviour, - NetworkBehaviourAction, - NotifyHandler, - OneShotHandler, - OneShotHandlerConfig, - PollParameters, - SubstreamProtocol - } -}; -use prost::Message; -use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; -use std::{ - cmp::min, - collections::VecDeque, - io, - iter, - marker::PhantomData, - sync::Arc, - time::Duration, - task::{Context, Poll} -}; -use void::{Void, unreachable}; - -// Type alias for convenience. -pub type Error = Box; - -/// Event generated by the finality proof requests behaviour. -#[derive(Debug)] -pub enum Event { - /// A response to a finality proof request has arrived. - Response { - peer: PeerId, - /// Block hash originally passed to `send_request`. - block_hash: B::Hash, - /// Finality proof returned by the remote. - proof: Vec, - }, -} - -/// Configuration options for `FinalityProofRequests`. -#[derive(Debug, Clone)] -pub struct Config { - max_request_len: usize, - max_response_len: usize, - inactivity_timeout: Duration, - protocol: Bytes, -} - -impl Config { - /// Create a fresh configuration with the following options: - /// - /// - max. request size = 1 MiB - /// - max. response size = 1 MiB - /// - inactivity timeout = 15s - pub fn new(id: &ProtocolId) -> Self { - let mut c = Config { - max_request_len: 1024 * 1024, - max_response_len: 1024 * 1024, - inactivity_timeout: Duration::from_secs(15), - protocol: Bytes::new(), - }; - c.set_protocol(id); - c - } - - /// Limit the max. length of incoming finality proof request bytes. - pub fn set_max_request_len(&mut self, v: usize) -> &mut Self { - self.max_request_len = v; - self - } - - /// Limit the max. length of incoming finality proof response bytes. - pub fn set_max_response_len(&mut self, v: usize) -> &mut Self { - self.max_response_len = v; - self - } - - /// Limit the max. duration the substream may remain inactive before closing it. - pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { - self.inactivity_timeout = v; - self - } - - /// Set protocol to use for upgrade negotiation. - pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { - let mut v = Vec::new(); - v.extend_from_slice(b"/"); - v.extend_from_slice(id.as_ref().as_bytes()); - v.extend_from_slice(b"/finality-proof/1"); - self.protocol = v.into(); - self - } -} - -/// The finality proof request handling behaviour. -pub struct FinalityProofRequests { - /// This behaviour's configuration. - config: Config, - /// How to construct finality proofs. - finality_proof_provider: Option>>, - /// Futures sending back the finality proof request responses. - outgoing: FuturesUnordered>, - /// Events to return as soon as possible from `poll`. - pending_events: VecDeque, Event>>, -} - -impl FinalityProofRequests -where - B: Block, -{ - /// Initializes the behaviour. - /// - /// If the proof provider is `None`, then the behaviour will not support the finality proof - /// requests protocol. - pub fn new(cfg: Config, finality_proof_provider: Option>>) -> Self { - FinalityProofRequests { - config: cfg, - finality_proof_provider, - outgoing: FuturesUnordered::new(), - pending_events: VecDeque::new(), - } - } - - /// Issue a new finality proof request. - /// - /// If the response doesn't arrive in time, or if the remote answers improperly, the target - /// will be disconnected. - pub fn send_request(&mut self, target: &PeerId, block_hash: B::Hash, request: Vec) { - let protobuf_rq = schema::v1::finality::FinalityProofRequest { - block_hash: block_hash.encode(), - request, - }; - - let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); - if let Err(err) = protobuf_rq.encode(&mut buf) { - log::warn!("failed to encode finality proof request {:?}: {:?}", protobuf_rq, err); - return; - } - - log::trace!("enqueueing finality proof request to {:?}: {:?}", target, protobuf_rq); - self.pending_events.push_back(NetworkBehaviourAction::NotifyHandler { - peer_id: target.clone(), - handler: NotifyHandler::Any, - event: OutboundProtocol { - request: buf, - block_hash, - max_response_size: self.config.max_response_len, - protocol: self.config.protocol.clone(), - }, - }); - } - - /// Callback, invoked when a new finality request has been received from remote. - fn on_finality_request(&mut self, peer: &PeerId, request: &schema::v1::finality::FinalityProofRequest) - -> Result - { - let block_hash = Decode::decode(&mut request.block_hash.as_ref())?; - - log::trace!(target: "sync", "Finality proof request from {} for {}", peer, block_hash); - - // Note that an empty Vec is sent if no proof is available. - let finality_proof = if let Some(provider) = &self.finality_proof_provider { - provider - .prove_finality(block_hash, &request.request)? - .unwrap_or_default() - } else { - log::error!("Answering a finality proof request while finality provider is empty"); - return Err(From::from("Empty finality proof provider".to_string())) - }; - - Ok(schema::v1::finality::FinalityProofResponse { proof: finality_proof }) - } -} - -impl NetworkBehaviour for FinalityProofRequests -where - B: Block -{ - type ProtocolsHandler = OneShotHandler, OutboundProtocol, NodeEvent>; - type OutEvent = Event; - - fn new_handler(&mut self) -> Self::ProtocolsHandler { - let p = InboundProtocol { - max_request_len: self.config.max_request_len, - protocol: if self.finality_proof_provider.is_some() { - Some(self.config.protocol.clone()) - } else { - None - }, - marker: PhantomData, - }; - let mut cfg = OneShotHandlerConfig::default(); - cfg.keep_alive_timeout = self.config.inactivity_timeout; - OneShotHandler::new(SubstreamProtocol::new(p, ()), cfg) - } - - fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { - Vec::new() - } - - fn inject_connected(&mut self, _peer: &PeerId) { - } - - fn inject_disconnected(&mut self, _peer: &PeerId) { - } - - fn inject_event( - &mut self, - peer: PeerId, - connection: ConnectionId, - event: NodeEvent - ) { - match event { - NodeEvent::Request(request, mut stream) => { - match self.on_finality_request(&peer, &request) { - Ok(res) => { - log::trace!("enqueueing finality response for peer {}", peer); - let mut data = Vec::with_capacity(res.encoded_len()); - if let Err(e) = res.encode(&mut data) { - log::debug!("error encoding finality response for peer {}: {}", peer, e) - } else { - let future = async move { - if let Err(e) = write_one(&mut stream, data).await { - log::debug!("error writing finality response: {}", e) - } - }; - self.outgoing.push(future.boxed()) - } - } - Err(e) => log::debug!("error handling finality request from peer {}: {}", peer, e) - } - } - NodeEvent::Response(response, block_hash) => { - let ev = Event::Response { - peer, - block_hash, - proof: response.proof, - }; - self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); - } - } - } - - fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) - -> Poll, Event>> - { - if let Some(ev) = self.pending_events.pop_front() { - return Poll::Ready(ev); - } - - while let Poll::Ready(Some(_)) = self.outgoing.poll_next_unpin(cx) {} - Poll::Pending - } -} - -/// Output type of inbound and outbound substream upgrades. -#[derive(Debug)] -pub enum NodeEvent { - /// Incoming request from remote and substream to use for the response. - Request(schema::v1::finality::FinalityProofRequest, T), - /// Incoming response from remote. - Response(schema::v1::finality::FinalityProofResponse, B::Hash), -} - -/// Substream upgrade protocol. -/// -/// We attempt to parse an incoming protobuf encoded request (cf. `Request`) -/// which will be handled by the `FinalityProofRequests` behaviour, i.e. the request -/// will become visible via `inject_node_event` which then dispatches to the -/// relevant callback to process the message and prepare a response. -#[derive(Debug, Clone)] -pub struct InboundProtocol { - /// The max. request length in bytes. - max_request_len: usize, - /// The protocol to use during upgrade negotiation. If `None`, then the incoming protocol - /// is simply disabled. - protocol: Option, - /// Marker to pin the block type. - marker: PhantomData, -} - -impl UpgradeInfo for InboundProtocol { - type Info = Bytes; - // This iterator will return either 0 elements if `self.protocol` is `None`, or 1 element if - // it is `Some`. - type InfoIter = std::option::IntoIter; - - fn protocol_info(&self) -> Self::InfoIter { - self.protocol.clone().into_iter() - } -} - -impl InboundUpgrade for InboundProtocol -where - B: Block, - T: AsyncRead + AsyncWrite + Unpin + Send + 'static -{ - type Output = NodeEvent; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; - - fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { - async move { - let len = self.max_request_len; - let vec = read_one(&mut s, len).await?; - match schema::v1::finality::FinalityProofRequest::decode(&vec[..]) { - Ok(r) => Ok(NodeEvent::Request(r, s)), - Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) - } - }.boxed() - } -} - -/// Substream upgrade protocol. -/// -/// Sends a request to remote and awaits the response. -#[derive(Debug, Clone)] -pub struct OutboundProtocol { - /// The serialized protobuf request. - request: Vec, - /// Block hash that has been requested. - block_hash: B::Hash, - /// The max. response length in bytes. - max_response_size: usize, - /// The protocol to use for upgrade negotiation. - protocol: Bytes, -} - -impl UpgradeInfo for OutboundProtocol { - type Info = Bytes; - type InfoIter = iter::Once; - - fn protocol_info(&self) -> Self::InfoIter { - iter::once(self.protocol.clone()) - } -} - -impl OutboundUpgrade for OutboundProtocol -where - B: Block, - T: AsyncRead + AsyncWrite + Unpin + Send + 'static -{ - type Output = NodeEvent; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; - - fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { - async move { - write_one(&mut s, &self.request).await?; - let vec = read_one(&mut s, self.max_response_size).await?; - - schema::v1::finality::FinalityProofResponse::decode(&vec[..]) - .map(|r| NodeEvent::Response(r, self.block_hash)) - .map_err(|e| { - ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) - }) - }.boxed() - } -} diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 3fd01c33dcf5f..b050db8785ac1 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -249,7 +249,6 @@ mod block_requests; mod chain; mod peer_info; mod discovery; -mod finality_requests; mod light_client_handler; mod on_demand_layer; mod protocol; diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 28f527da68ed9..d837dd787555a 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -1061,7 +1061,7 @@ impl ChainSync { if let PeerSyncState::DownloadingFinalityProof(state_hash) = peer.state { peer.state = PeerSyncState::Available; - if let Some((peer, hash, number, p)) = self.extra_finality_proofs.on_response(who, if proof.is_empty() { None } else { Some(proof) }) { + if let Some((peer, hash, number, p)) = self.extra_finality_proofs.on_response_with_request_id(who, request_id, if proof.is_empty() { None } else { Some(proof) }) { trace!(target: "sync", "Finality proof response from {} for {}", peer, hash); // TODO: Safe assumption to make? assert_eq!(hash, state_hash); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index d6acbf768956c..0f9f539c03dd1 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -38,7 +38,7 @@ use crate::{ NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer, }, on_demand_layer::AlwaysBadChecker, - light_client_handler, block_requests, finality_requests, + light_client_handler, block_requests, protocol::{self, event::Event, NotifsHandlerError, LegacyConnectionKillError, NotificationsSink, Ready, sync::SyncState, PeerInfo, Protocol}, transport, ReputationChange, }; @@ -273,10 +273,6 @@ impl NetworkWorker { let config = block_requests::Config::new(¶ms.protocol_id); block_requests::BlockRequests::new(config, params.chain.clone()) }; - let finality_proof_requests = { - let config = finality_requests::Config::new(¶ms.protocol_id); - finality_requests::FinalityProofRequests::new(config, params.finality_proof_provider.clone()) - }; let light_client_handler = { let config = light_client_handler::Config::new(¶ms.protocol_id); light_client_handler::LightClientHandler::new( @@ -319,7 +315,7 @@ impl NetworkWorker { let mut data = Vec::with_capacity(resp.encoded_len()); resp.encode(&mut data).unwrap(); - pending_response.send(data); + pending_response.send(data).unwrap(); } }.boxed(); @@ -354,7 +350,6 @@ impl NetworkWorker { user_agent, local_public, block_requests, - finality_proof_requests, light_client_handler, discovery_config, request_response_protocols, @@ -1621,14 +1616,14 @@ impl Future for NetworkWorker { let reason = match cause { Some(ConnectionError::IO(_)) => "transport-error", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A(EitherError::B( - EitherError::A(PingFailure::Timeout)))))))))) => "ping-timeout", + EitherError::A(EitherError::A(EitherError::B(EitherError::A( + PingFailure::Timeout))))))))) => "ping-timeout", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A(EitherError::A( - NotifsHandlerError::Legacy(LegacyConnectionKillError)))))))))) => "force-closed", + EitherError::A(EitherError::A(EitherError::A( + NotifsHandlerError::Legacy(LegacyConnectionKillError))))))))) => "force-closed", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A(EitherError::A( - NotifsHandlerError::SyncNotificationsClogged))))))))) => "sync-notifications-clogged", + EitherError::A(EitherError::A(EitherError::A( + NotifsHandlerError::SyncNotificationsClogged)))))))) => "sync-notifications-clogged", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(_))) => "protocol-error", Some(ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout)) => "keep-alive-timeout", None => "actively-closed", From 3207c537fc43d2e9e82ed9d5ab3a7002f4aa7dde Mon Sep 17 00:00:00 2001 From: Max Inden Date: Thu, 29 Oct 2020 14:55:08 +0100 Subject: [PATCH 03/48] client/network/src/behaviour: Pass request id down to sync --- client/network/src/behaviour.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index cf6b60df5ce77..2d1ed055b3c34 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -366,13 +366,9 @@ Behaviour { log::warn!("failed to encode finality proof request {:?}: {:?}", protobuf_rq, err); return; } - self.request_responses.send_request(&target, "abc", buf).unwrap(); + let request_id = self.request_responses.send_request(&target, "abc", buf).unwrap(); - - - - - // self.finality_proof_requests.send_request(&target, block_hash, request); + self.substrate.on_finality_proof_request_started(target, block_hash, request_id); }, CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles, notifications_sink } => { let role = reported_roles_to_observed_role(&self.role, &remote, roles); From e3ef56ed91532014159b11a8d353ffa68ed98d1c Mon Sep 17 00:00:00 2001 From: Max Inden Date: Thu, 29 Oct 2020 18:26:12 +0100 Subject: [PATCH 04/48] client/network: Use request response for block requests --- client/network/src/behaviour.rs | 121 ++++++++++++++++------- client/network/src/protocol.rs | 169 ++++++++++++++++++++++++-------- 2 files changed, 212 insertions(+), 78 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 2d1ed055b3c34..316d79e2de4d3 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -333,27 +333,66 @@ Behaviour { CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) => self.events.push_back(BehaviourOut::FinalityProofImport(origin, hash, nb, proof)), CustomMessageOutcome::BlockRequest { target, request } => { - match self.block_requests.send_request(&target, request) { - block_requests::SendRequestOutcome::Ok => { - self.events.push_back(BehaviourOut::OpaqueRequestStarted { - peer: target, - protocol: self.block_requests.protocol_name().to_owned(), - }); + + let protobuf_req = crate::schema::v1::BlockRequest { + fields: request.fields.to_be_u32(), + from_block: match request.from { + message::FromBlock::Hash(h) => + Some(crate::schema::v1::block_request::FromBlock::Hash(h.encode())), + message::FromBlock::Number(n) => + Some(crate::schema::v1::block_request::FromBlock::Number(n.encode())), }, - block_requests::SendRequestOutcome::Replaced { request_duration, .. } => { - self.events.push_back(BehaviourOut::OpaqueRequestFinished { - peer: target.clone(), - protocol: self.block_requests.protocol_name().to_owned(), - request_duration, - }); - self.events.push_back(BehaviourOut::OpaqueRequestStarted { - peer: target, - protocol: self.block_requests.protocol_name().to_owned(), - }); - } - block_requests::SendRequestOutcome::NotConnected | - block_requests::SendRequestOutcome::EncodeError(_) => {}, + to_block: request.to.map(|h| h.encode()).unwrap_or_default(), + direction: match request.direction { + message::Direction::Ascending => crate::schema::v1::Direction::Ascending as i32, + message::Direction::Descending => crate::schema::v1::Direction::Descending as i32, + }, + max_blocks: request.max.unwrap_or(0), + }; + + let mut buf = Vec::with_capacity(protobuf_req.encoded_len()); + if let Err(err) = protobuf_req.encode(&mut buf) { + log::warn!( + target: "sync", + "Failed to encode block request {:?}: {:?}", + protobuf_req, + err + ); + panic!(); + // return SendRequestOutcome::EncodeError(err); } + + let request_id = self.request_responses.send_request(&target, "def", buf).unwrap(); + // TODO: differentiate between the two ids. + self.substrate.on_block_request_started(target, request.id, request_id); + + + + + + + // TODO: Entirely ignoring any previous requests now. Fine? + // match self.block_requests.send_request(&target, request) { + // block_requests::SendRequestOutcome::Ok => { + // self.events.push_back(BehaviourOut::OpaqueRequestStarted { + // peer: target, + // protocol: self.block_requests.protocol_name().to_owned(), + // }); + // }, + // block_requests::SendRequestOutcome::Replaced { request_duration, .. } => { + // self.events.push_back(BehaviourOut::OpaqueRequestFinished { + // peer: target.clone(), + // protocol: self.block_requests.protocol_name().to_owned(), + // request_duration, + // }); + // self.events.push_back(BehaviourOut::OpaqueRequestStarted { + // peer: target, + // protocol: self.block_requests.protocol_name().to_owned(), + // }); + // } + // block_requests::SendRequestOutcome::NotConnected | + // block_requests::SendRequestOutcome::EncodeError(_) => {}, + // } }, CustomMessageOutcome::FinalityProofRequest { target, block_hash, request } => { let protobuf_rq = crate::schema::v1::finality::FinalityProofRequest { @@ -418,16 +457,25 @@ impl NetworkBehaviourEventProcess { - if protocol == "abc" { - let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) - .unwrap(); - let ev = self.substrate.on_finality_proof_response(peer, request_id, proof_response.proof); - self.inject_event(ev); - } else { - self.events.push_back(BehaviourOut::RequestFinished { - request_id, - result, - }); + match protocol { + Cow::Borrowed("abc") => { + let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) + .unwrap(); + let ev = self.substrate.on_finality_proof_response(peer, request_id, proof_response.proof); + self.inject_event(ev); + } + Cow::Borrowed("def") => { + let protobuf_response = crate::schema::v1::BlockResponse::decode(&result.unwrap()[..]).unwrap(); + + let ev = self.substrate.on_block_response(peer, request_id, protobuf_response); + self.inject_event(ev); + } + _ => { + self.events.push_back(BehaviourOut::RequestFinished { + request_id, + result, + }); + } } }, } @@ -445,13 +493,14 @@ impl NetworkBehaviourEventProcess { - self.events.push_back(BehaviourOut::OpaqueRequestFinished { - peer: peer.clone(), - protocol: self.block_requests.protocol_name().to_owned(), - request_duration, - }); - let ev = self.substrate.on_block_response(peer, response); - self.inject_event(ev); + panic!(); + // self.events.push_back(BehaviourOut::OpaqueRequestFinished { + // peer: peer.clone(), + // protocol: self.block_requests.protocol_name().to_owned(), + // request_duration, + // }); + // let ev = self.substrate.on_block_response(peer, response); + // self.inject_event(ev); } block_requests::Event::RequestCancelled { peer, request_duration, .. } | block_requests::Event::RequestTimeout { peer, request_duration, .. } => { diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index f891e48b9d48b..723248de135db 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -257,8 +257,9 @@ struct PacketStats { struct Peer { info: PeerInfo, /// Current block request, if any. - block_request: Option<(Instant, message::BlockRequest)>, + block_request: Option<(Instant, message::BlockRequest, Option)>, /// Requests we are no longer interested in. + // TODO: Do we still need this at all? obsolete_requests: HashMap, /// Holds a set of transactions known to this peer. known_transactions: LruHashSet, @@ -701,52 +702,120 @@ impl Protocol { /// Must contain the same `PeerId` and request that have been emitted. pub fn on_block_response( &mut self, - peer: PeerId, - response: message::BlockResponse, + peer_id: PeerId, + request_id: libp2p::request_response::RequestId, + response: crate::schema::v1::BlockResponse, ) -> CustomMessageOutcome { - let request = if let Some(ref mut p) = self.context_data.peers.get_mut(&peer) { - if p.obsolete_requests.remove(&response.id).is_some() { - trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); + let peer = match self.context_data.peers.get_mut(&peer_id) { + Some(peer) => peer, + None => { + // TODO: Bring back. + // trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); return CustomMessageOutcome::None; } - // Clear the request. If the response is invalid peer will be disconnected anyway. - match p.block_request.take() { - Some((_, request)) if request.id == response.id => request, - Some(_) => { - trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); - return CustomMessageOutcome::None; - } - None => { - trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); - self.behaviour.disconnect_peer(&peer); - self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); - return CustomMessageOutcome::None; - } + }; + + let block_request = match peer.block_request.take() { + Some(req) => req, + None => { + trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer_id); + self.behaviour.disconnect_peer(&peer_id); + self.peerset_handle.report_peer(peer_id, rep::UNEXPECTED_RESPONSE); + return CustomMessageOutcome::None; } - } else { - trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); - self.behaviour.disconnect_peer(&peer); - self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); + }; + + if block_request.2 != Some(request_id) { + // TODO: Properly handle this. + panic!("request id didn't match or was none"); return CustomMessageOutcome::None; + } + + let block_request = block_request.1; + + let blocks = response.blocks.into_iter().map(|block_data| { + Ok(message::BlockData:: { + hash: Decode::decode(&mut block_data.hash.as_ref())?, + header: if !block_data.header.is_empty() { + Some(Decode::decode(&mut block_data.header.as_ref())?) + } else { + None + }, + body: if block_request.fields.contains(message::BlockAttributes::BODY) { + Some(block_data.body.iter().map(|body| { + Decode::decode(&mut body.as_ref()) + }).collect::, _>>()?) + } else { + None + }, + receipt: if !block_data.message_queue.is_empty() { + Some(block_data.receipt) + } else { + None + }, + message_queue: if !block_data.message_queue.is_empty() { + Some(block_data.message_queue) + } else { + None + }, + justification: if !block_data.justification.is_empty() { + Some(block_data.justification) + } else if block_data.is_empty_justification { + Some(Vec::new()) + } else { + None + }, + }) + }).collect::, codec::Error>>().unwrap(); + + let block_response = message::BlockResponse:: { + id: block_request.id, + blocks, }; + // let request = if let Some(ref mut p) = self.context_data.peers.get_mut(&peer) { + // if p.obsolete_requests.remove(&response.id).is_some() { + // trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); + // return CustomMessageOutcome::None; + // } + // // Clear the request. If the response is invalid peer will be disconnected anyway. + // match p.block_request.take() { + // Some((_, request)) if request.id == response.id => request, + // Some(_) => { + // trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); + // return CustomMessageOutcome::None; + // } + // None => { + // trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); + // self.behaviour.disconnect_peer(&peer); + // self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); + // return CustomMessageOutcome::None; + // } + // } + // } else { + // trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); + // self.behaviour.disconnect_peer(&peer); + // self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); + // return CustomMessageOutcome::None; + // }; + let blocks_range = || match ( - response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), - response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), + block_response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), + block_response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), ) { (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), (Some(first), Some(_)) => format!(" ({})", first), _ => Default::default(), }; trace!(target: "sync", "BlockResponse {} from {} with {} blocks {}", - response.id, - peer, - response.blocks.len(), + block_response.id, + peer_id, + block_response.blocks.len(), blocks_range(), ); - if request.fields == message::BlockAttributes::JUSTIFICATION { - match self.sync.on_block_justification(peer, response) { + if block_request.fields == message::BlockAttributes::JUSTIFICATION { + match self.sync.on_block_justification(peer_id, block_response) { Ok(sync::OnBlockJustification::Nothing) => CustomMessageOutcome::None, Ok(sync::OnBlockJustification::Import { peer, hash, number, justification }) => CustomMessageOutcome::JustificationImport(peer, hash, number, justification), @@ -758,26 +827,26 @@ impl Protocol { } } else { // Validate fields against the request. - if request.fields.contains(message::BlockAttributes::HEADER) && response.blocks.iter().any(|b| b.header.is_none()) { - self.behaviour.disconnect_peer(&peer); - self.peerset_handle.report_peer(peer, rep::BAD_RESPONSE); + if block_request.fields.contains(message::BlockAttributes::HEADER) && block_response.blocks.iter().any(|b| b.header.is_none()) { + self.behaviour.disconnect_peer(&peer_id); + self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); trace!(target: "sync", "Missing header for a block"); return CustomMessageOutcome::None } - if request.fields.contains(message::BlockAttributes::BODY) && response.blocks.iter().any(|b| b.body.is_none()) { - self.behaviour.disconnect_peer(&peer); - self.peerset_handle.report_peer(peer, rep::BAD_RESPONSE); + if block_request.fields.contains(message::BlockAttributes::BODY) && block_response.blocks.iter().any(|b| b.body.is_none()) { + self.behaviour.disconnect_peer(&peer_id); + self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); trace!(target: "sync", "Missing body for a block"); return CustomMessageOutcome::None } - match self.sync.on_block_data(&peer, Some(request), response) { + match self.sync.on_block_data(&peer_id, Some(block_request), block_response) { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), Ok(sync::OnBlockData::Request(peer, mut req)) => { - self.update_peer_request(&peer, &mut req); + self.update_peer_request(&peer_id, &mut req); CustomMessageOutcome::BlockRequest { - target: peer, + target: peer_id, request: req, } } @@ -812,7 +881,7 @@ impl Protocol { let mut aborting = Vec::new(); { for (who, peer) in self.context_data.peers.iter() { - if peer.block_request.as_ref().map_or(false, |(t, _)| (tick - *t).as_secs() > REQUEST_TIMEOUT_SEC) { + if peer.block_request.as_ref().map_or(false, |(t, _, _request_id)| (tick - *t).as_secs() > REQUEST_TIMEOUT_SEC) { log!( target: "sync", if self.important_peers.contains(who) { Level::Warn } else { Level::Trace }, @@ -1395,6 +1464,22 @@ impl Protocol { self.sync.on_finality_proof_request_started(who, block_hash, request_id) } + pub fn on_block_request_started( + &mut self, + who: PeerId, + request_id: u64, + request_response_request_id: libp2p::request_response::RequestId, + ) { + let peer = self.context_data.peers.get_mut(&who).unwrap(); + + if peer.block_request.as_mut().unwrap().1.id != request_id { + // TODO: Handle. likely fine. Some other request replaced the current one. + panic!(); + } + + peer.block_request.as_mut().unwrap().2 = Some(request_response_request_id); + } + fn format_stats(&self) -> String { let mut out = String::new(); for (id, stats) in &self.context_data.stats { @@ -1501,11 +1586,11 @@ fn update_peer_request( if let Some(ref mut peer) = peers.get_mut(who) { request.id = peer.next_request_id; peer.next_request_id += 1; - if let Some((timestamp, request)) = peer.block_request.take() { + if let Some((timestamp, request, _request_id)) = peer.block_request.take() { trace!(target: "sync", "Request {} for {} is now obsolete.", request.id, who); peer.obsolete_requests.insert(request.id, timestamp); } - peer.block_request = Some((Instant::now(), request.clone())); + peer.block_request = Some((Instant::now(), request.clone(), None)); } } From 544ebfe86fcd54fc9edbc4816a1c290c7e24b277 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 30 Oct 2020 14:18:26 +0100 Subject: [PATCH 05/48] client/network: Move handler logic into *_*_handler.rs --- client/network/src/behaviour.rs | 142 ++- client/network/src/block_request_handler.rs | 177 ++++ client/network/src/block_requests.rs | 857 ------------------ client/network/src/config.rs | 6 - .../network/src/finality_request_handler.rs | 67 ++ client/network/src/lib.rs | 3 +- client/network/src/light_client_handler.rs | 16 +- client/network/src/protocol.rs | 21 +- client/network/src/protocol/sync.rs | 7 +- .../src/protocol/sync/extra_requests.rs | 2 +- client/network/src/service.rs | 97 +- client/service/src/builder.rs | 17 +- 12 files changed, 379 insertions(+), 1033 deletions(-) create mode 100644 client/network/src/block_request_handler.rs delete mode 100644 client/network/src/block_requests.rs create mode 100644 client/network/src/finality_request_handler.rs diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 316d79e2de4d3..1d446144f569d 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -15,7 +15,7 @@ // along with Substrate. If not, see . use crate::{ - config::{ProtocolId, Role}, block_requests, light_client_handler, peer_info, request_responses, + config::{ProtocolId, Role}, light_client_handler, peer_info, request_responses, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, protocol::{message::{self, Roles}, CustomMessageOutcome, NotificationsSink, Protocol}, ObservedRole, DhtEvent, ExHashT, @@ -57,8 +57,6 @@ pub struct Behaviour { discovery: DiscoveryBehaviour, /// Generic request-reponse protocols. request_responses: request_responses::RequestResponsesBehaviour, - /// Block request handling. - block_requests: block_requests::BlockRequests, /// Light client request handling. light_client_handler: light_client_handler::LightClientHandler, @@ -69,6 +67,13 @@ pub struct Behaviour { /// Role of our local node, as originally passed from the configuration. #[behaviour(ignore)] role: Role, + + // TODO: Does this really belong here? Why require matching on these in the first place? Why not + // offer a oneshot when sending a request? + #[behaviour(ignore)] + block_request_protocol_name: String, + #[behaviour(ignore)] + finality_request_protocol_name: String, } /// Event generated by `Behaviour`. @@ -101,27 +106,30 @@ pub enum BehaviourOut { result: Result, RequestFailure>, }, - /// Started a new request with the given node. - /// - /// This event is for statistics purposes only. The request and response handling are entirely - /// internal to the behaviour. - OpaqueRequestStarted { - peer: PeerId, - /// Protocol name of the request. - protocol: String, - }, - /// Finished, successfully or not, a previously-started request. - /// - /// This event is for statistics purposes only. The request and response handling are entirely - /// internal to the behaviour. - OpaqueRequestFinished { - /// Who we were requesting. - peer: PeerId, - /// Protocol name of the request. - protocol: String, - /// How long before the response came or the request got cancelled. - request_duration: Duration, - }, + // TODO: Do we still need these? Can we not just depend on the request reponse behaviour to + // provide these metrics? + // + // /// Started a new request with the given node. + // /// + // /// This event is for statistics purposes only. The request and response handling are entirely + // /// internal to the behaviour. + // OpaqueRequestStarted { + // peer: PeerId, + // /// Protocol name of the request. + // protocol: String, + // }, + // /// Finished, successfully or not, a previously-started request. + // /// + // /// This event is for statistics purposes only. The request and response handling are entirely + // /// internal to the behaviour. + // OpaqueRequestFinished { + // /// Who we were requesting. + // peer: PeerId, + // /// Protocol name of the request. + // protocol: String, + // /// How long before the response came or the request got cancelled. + // request_duration: Duration, + // }, /// Opened a substream with the given node with the given notifications protocol. /// @@ -180,10 +188,13 @@ impl Behaviour { role: Role, user_agent: String, local_public_key: PublicKey, - block_requests: block_requests::BlockRequests, light_client_handler: light_client_handler::LightClientHandler, disco_config: DiscoveryConfig, request_response_protocols: Vec, + + // TODO: Not so fancy, see definition. + block_request_protocol_name: String, + finality_request_protocol_name: String, ) -> Result { Ok(Behaviour { substrate, @@ -191,10 +202,12 @@ impl Behaviour { discovery: disco_config.finish(), request_responses: request_responses::RequestResponsesBehaviour::new(request_response_protocols.into_iter())?, - block_requests, light_client_handler, events: VecDeque::new(), role, + + block_request_protocol_name, + finality_request_protocol_name, }) } @@ -362,7 +375,7 @@ Behaviour { // return SendRequestOutcome::EncodeError(err); } - let request_id = self.request_responses.send_request(&target, "def", buf).unwrap(); + let request_id = self.request_responses.send_request(&target, &self.block_request_protocol_name, buf).unwrap(); // TODO: differentiate between the two ids. self.substrate.on_block_request_started(target, request.id, request_id); @@ -405,7 +418,7 @@ Behaviour { log::warn!("failed to encode finality proof request {:?}: {:?}", protobuf_rq, err); return; } - let request_id = self.request_responses.send_request(&target, "abc", buf).unwrap(); + let request_id = self.request_responses.send_request(&target, &self.finality_request_protocol_name, buf).unwrap(); self.substrate.on_finality_proof_request_started(target, block_hash, request_id); }, @@ -448,6 +461,10 @@ Behaviour { impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: request_responses::Event) { + // Can we do prettier? + let finality_request_protocol_name = Cow::Borrowed(&self.finality_request_protocol_name); + let block_request_protocol_name = Cow::Borrowed(&self.block_request_protocol_name); + match event { request_responses::Event::InboundRequest { peer, protocol, result } => { self.events.push_back(BehaviourOut::InboundRequest { @@ -457,66 +474,27 @@ impl NetworkBehaviourEventProcess { - match protocol { - Cow::Borrowed("abc") => { - let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) - .unwrap(); - let ev = self.substrate.on_finality_proof_response(peer, request_id, proof_response.proof); - self.inject_event(ev); - } - Cow::Borrowed("def") => { - let protobuf_response = crate::schema::v1::BlockResponse::decode(&result.unwrap()[..]).unwrap(); - - let ev = self.substrate.on_block_response(peer, request_id, protobuf_response); - self.inject_event(ev); - } - _ => { - self.events.push_back(BehaviourOut::RequestFinished { - request_id, - result, - }); - } + if protocol == self.finality_request_protocol_name { + let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) + .unwrap(); + let ev = self.substrate.on_finality_proof_response(peer, request_id, proof_response.proof); + self.inject_event(ev); + } else if protocol == self.block_request_protocol_name { + let protobuf_response = crate::schema::v1::BlockResponse::decode(&result.unwrap()[..]).unwrap(); + + let ev = self.substrate.on_block_response(peer, request_id, protobuf_response); + self.inject_event(ev); + } else { + self.events.push_back(BehaviourOut::RequestFinished { + request_id, + result, + }); } }, } } } -impl NetworkBehaviourEventProcess> for Behaviour { - fn inject_event(&mut self, event: block_requests::Event) { - match event { - block_requests::Event::AnsweredRequest { peer, total_handling_time } => { - self.events.push_back(BehaviourOut::InboundRequest { - peer, - protocol: self.block_requests.protocol_name().to_owned().into(), - result: Ok(total_handling_time), - }); - }, - block_requests::Event::Response { peer, response, request_duration } => { - panic!(); - // self.events.push_back(BehaviourOut::OpaqueRequestFinished { - // peer: peer.clone(), - // protocol: self.block_requests.protocol_name().to_owned(), - // request_duration, - // }); - // let ev = self.substrate.on_block_response(peer, response); - // self.inject_event(ev); - } - block_requests::Event::RequestCancelled { peer, request_duration, .. } | - block_requests::Event::RequestTimeout { peer, request_duration, .. } => { - // There doesn't exist any mechanism to report cancellations or timeouts yet, so - // we process them by disconnecting the node. - self.events.push_back(BehaviourOut::OpaqueRequestFinished { - peer: peer.clone(), - protocol: self.block_requests.protocol_name().to_owned(), - request_duration, - }); - self.substrate.on_block_request_failed(&peer); - } - } - } -} - impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: peer_info::PeerInfoEvent) { diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs new file mode 100644 index 0000000000000..57b44c887136b --- /dev/null +++ b/client/network/src/block_request_handler.rs @@ -0,0 +1,177 @@ +use crate::request_responses::ProtocolConfig; +use futures::channel::mpsc; +use futures::stream::StreamExt; +use std::sync::{Arc}; +pub use crate::{ + protocol::{message::{self, BlockAttributes}}, + chain::{Client, FinalityProofProvider}, + config::ProtocolId, +}; +use sp_runtime::{traits::Block as BlockT}; +use crate::request_responses::IncomingRequest; +use prost::Message; +use codec::{Encode, Decode}; +use sp_runtime::{generic::BlockId, traits::{Header, One, Zero}}; + +pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { + let mut s = String::new(); + s.push_str("/"); + s.push_str(protocol_id.as_ref()); + s.push_str("/sync/2"); + s +} + +pub struct BlockRequestHandler { + // TODO: Rename? + chain: Arc>, + request_receiver: mpsc::Receiver, +} + +impl BlockRequestHandler { + pub fn new(protocol_id: ProtocolId, client: Arc>) -> (Self, ProtocolConfig) { + let (tx, rx) = mpsc::channel(0); + + let protocol_config = ProtocolConfig { + name: generate_protocol_name(protocol_id).into(), + // TODO: Change. + max_request_size: 4096, + // TODO: Change. + max_response_size: 4096, + request_timeout: std::time::Duration::from_secs(10), + inbound_queue: Some(tx), + }; + + let handler = Self { + chain: client, + request_receiver: rx, + }; + + (handler, protocol_config) + } + + pub async fn run(mut self) { + while let Some(crate::request_responses::IncomingRequest { peer: _, payload, pending_response }) = self.request_receiver.next().await { + + let request = crate::schema::v1::BlockRequest::decode(&payload[..]).unwrap(); + + + + let from_block_id = match request.from_block { + Some(crate::schema::v1::block_request::FromBlock::Hash(ref h)) => { + let h = Decode::decode(&mut h.as_ref()).unwrap(); + BlockId::::Hash(h) + } + Some(crate::schema::v1::block_request::FromBlock::Number(ref n)) => { + let n = Decode::decode(&mut n.as_ref()).unwrap(); + BlockId::::Number(n) + } + None => { + panic!(); + // let msg = "missing `BlockRequest::from_block` field"; + // return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + } + }; + + // let max_blocks = + // if request.max_blocks == 0 { + // self.config.max_block_data_response + // } else { + // min(request.max_blocks, self.config.max_block_data_response) + // }; + + let direction = + if request.direction == crate::schema::v1::Direction::Ascending as i32 { + crate::schema::v1::Direction::Ascending + } else if request.direction == crate::schema::v1::Direction::Descending as i32 { + crate::schema::v1::Direction::Descending + } else { + panic!(); + // let msg = format!("invalid `BlockRequest::direction` value: {}", request.direction); + // return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + }; + + let attributes = BlockAttributes::from_be_u32(request.fields).unwrap(); + let get_header = attributes.contains(BlockAttributes::HEADER); + let get_body = attributes.contains(BlockAttributes::BODY); + let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); + + let mut blocks = Vec::new(); + let mut block_id = from_block_id; + // TODO: Still needed? + // let mut total_size = 0; + while let Some(header) = self.chain.header(block_id).unwrap_or(None) { + // if blocks.len() >= max_blocks as usize + // || (blocks.len() >= 1 && total_size > self.config.max_block_body_bytes) + // { + // break + // } + + let number = *header.number(); + let hash = header.hash(); + let parent_hash = *header.parent_hash(); + let justification = if get_justification { + self.chain.justification(&BlockId::Hash(hash)).unwrap() + } else { + None + }; + let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false); + + let body = if get_body { + match self.chain.block_body(&BlockId::Hash(hash)).unwrap() { + Some(mut extrinsics) => extrinsics.iter_mut() + .map(|extrinsic| extrinsic.encode()) + .collect(), + None => { + log::trace!(target: "sync", "Missing data for block request."); + break; + } + } + } else { + Vec::new() + }; + + let block_data = crate::schema::v1::BlockData { + hash: hash.encode(), + header: if get_header { + header.encode() + } else { + Vec::new() + }, + body, + receipt: Vec::new(), + message_queue: Vec::new(), + justification: justification.unwrap_or_default(), + is_empty_justification, + }; + + // TODO: Still needed? + // total_size += block_data.body.len(); + blocks.push(block_data); + + match direction { + crate::schema::v1::Direction::Ascending => { + block_id = BlockId::Number(number + One::one()) + } + crate::schema::v1::Direction::Descending => { + if number.is_zero() { + break + } + block_id = BlockId::Hash(parent_hash) + } + } + } + + let res = crate::schema::v1::BlockResponse { blocks }; + + let mut data = Vec::with_capacity(res.encoded_len()); + res.encode(&mut data).unwrap(); + // log::debug!( + // target: "sync", + // "Error encoding block response for peer {}: {}", + // peer, e + // ) + + pending_response.send(data).unwrap(); + } + } +} diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs deleted file mode 100644 index ace63e6e1cdd4..0000000000000 --- a/client/network/src/block_requests.rs +++ /dev/null @@ -1,857 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. -// -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! `NetworkBehaviour` implementation which handles incoming block requests. -//! -//! Every request is coming in on a separate connection substream which gets -//! closed after we have sent the response back. Incoming requests are encoded -//! as protocol buffers (cf. `api.v1.proto`). - -#![allow(unused)] - -use bytes::Bytes; -use codec::{Encode, Decode}; -use crate::{ - chain::Client, - config::ProtocolId, - protocol::{message::{self, BlockAttributes}}, - schema, -}; -use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; -use futures_timer::Delay; -use libp2p::{ - core::{ - ConnectedPoint, - Multiaddr, - PeerId, - connection::ConnectionId, - upgrade::{InboundUpgrade, OutboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, - upgrade::{DeniedUpgrade, read_one, write_one} - }, - swarm::{ - NegotiatedSubstream, - NetworkBehaviour, - NetworkBehaviourAction, - NotifyHandler, - OneShotHandler, - OneShotHandlerConfig, - PollParameters, - SubstreamProtocol - } -}; -use prost::Message; -use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; -use std::{ - cmp::min, - collections::{HashMap, VecDeque}, - io, - iter, - marker::PhantomData, - pin::Pin, - sync::Arc, - time::Duration, - task::{Context, Poll} -}; -use void::{Void, unreachable}; -use wasm_timer::Instant; - -// Type alias for convenience. -pub type Error = Box; - -/// Event generated by the block requests behaviour. -#[derive(Debug)] -pub enum Event { - /// A request came and we have successfully answered it. - AnsweredRequest { - /// Peer which has emitted the request. - peer: PeerId, - /// Time elapsed between when we received the request and when we sent back the response. - total_handling_time: Duration, - }, - - /// A response to a block request has arrived. - Response { - peer: PeerId, - response: message::BlockResponse, - /// Time elapsed between the start of the request and the response. - request_duration: Duration, - }, - - /// A request has been cancelled because the peer has disconnected. - /// Disconnects can also happen as a result of violating the network protocol. - /// - /// > **Note**: This event is NOT emitted if a request is overridden by calling `send_request`. - /// > For that, you must check the value returned by `send_request`. - RequestCancelled { - peer: PeerId, - /// Time elapsed between the start of the request and the cancellation. - request_duration: Duration, - }, - - /// A request has timed out. - RequestTimeout { - peer: PeerId, - /// Time elapsed between the start of the request and the timeout. - request_duration: Duration, - } -} - -/// Configuration options for `BlockRequests`. -#[derive(Debug, Clone)] -pub struct Config { - max_block_data_response: u32, - max_block_body_bytes: usize, - max_request_len: usize, - max_response_len: usize, - inactivity_timeout: Duration, - request_timeout: Duration, - protocol: String, -} - -impl Config { - /// Create a fresh configuration with the following options: - /// - /// - max. block data in response = 128 - /// - max. request size = 1 MiB - /// - max. response size = 16 MiB - /// - inactivity timeout = 15s - /// - request timeout = 40s - pub fn new(id: &ProtocolId) -> Self { - let mut c = Config { - max_block_data_response: 128, - max_block_body_bytes: 8 * 1024 * 1024, - max_request_len: 1024 * 1024, - max_response_len: 16 * 1024 * 1024, - inactivity_timeout: Duration::from_secs(15), - request_timeout: Duration::from_secs(40), - protocol: String::new(), - }; - c.set_protocol(id); - c - } - - /// Limit the max. number of block data in a response. - pub fn set_max_block_data_response(&mut self, v: u32) -> &mut Self { - self.max_block_data_response = v; - self - } - - /// Limit the max. length of incoming block request bytes. - pub fn set_max_request_len(&mut self, v: usize) -> &mut Self { - self.max_request_len = v; - self - } - - /// Limit the max. size of responses to our block requests. - pub fn set_max_response_len(&mut self, v: usize) -> &mut Self { - self.max_response_len = v; - self - } - - /// Limit the max. duration the substream may remain inactive before closing it. - pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { - self.inactivity_timeout = v; - self - } - - /// Set the maximum total bytes of block bodies that are send in the response. - /// Note that at least one block is always sent regardless of the limit. - /// This should be lower than the value specified in `set_max_response_len` - /// accounting for headers, justifications and encoding overhead. - pub fn set_max_block_body_bytes(&mut self, v: usize) -> &mut Self { - self.max_block_body_bytes = v; - self - } - - /// Set protocol to use for upgrade negotiation. - pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { - let mut s = String::new(); - s.push_str("/"); - s.push_str(id.as_ref()); - s.push_str("/sync/2"); - self.protocol = s; - self - } -} - -/// The block request handling behaviour. -pub struct BlockRequests { - /// This behaviour's configuration. - config: Config, - /// Blockchain client. - chain: Arc>, - /// List of all active connections and the requests we've sent. - peers: HashMap>>, - /// Futures sending back the block request response. Returns the `PeerId` we sent back to, and - /// the total time the handling of this request took. - outgoing: FuturesUnordered>, - /// Events to return as soon as possible from `poll`. - pending_events: VecDeque, Event>>, -} - -/// Local tracking of a libp2p connection. -#[derive(Debug)] -struct Connection { - id: ConnectionId, - ongoing_request: Option>, -} - -#[derive(Debug)] -struct OngoingRequest { - /// `Instant` when the request has been emitted. Used for diagnostic purposes. - emitted: Instant, - request: message::BlockRequest, - timeout: Delay, -} - -/// Outcome of calling `send_request`. -#[derive(Debug)] -#[must_use] -pub enum SendRequestOutcome { - /// Request has been emitted. - Ok, - /// The request has been emitted and has replaced an existing request. - Replaced { - /// The previously-emitted request. - previous: message::BlockRequest, - /// Time that had elapsed since `previous` has been emitted. - request_duration: Duration, - }, - /// Didn't start a request because we have no connection to this node. - /// If `send_request` returns that, it is as if the function had never been called. - NotConnected, - /// Error while serializing the request. - EncodeError(prost::EncodeError), -} - -impl BlockRequests -where - B: Block, -{ - pub fn new(cfg: Config, chain: Arc>) -> Self { - BlockRequests { - config: cfg, - chain, - peers: HashMap::new(), - outgoing: FuturesUnordered::new(), - pending_events: VecDeque::new(), - } - } - - /// Returns the libp2p protocol name used on the wire (e.g. `/foo/sync/2`). - pub fn protocol_name(&self) -> &str { - &self.config.protocol - } - - /// Issue a new block request. - /// - /// Cancels any existing request targeting the same `PeerId`. - /// - /// If the response doesn't arrive in time, or if the remote answers improperly, the target - /// will be disconnected. - pub fn send_request(&mut self, target: &PeerId, req: message::BlockRequest) -> SendRequestOutcome { - // Determine which connection to send the request to. - let connection = if let Some(peer) = self.peers.get_mut(target) { - // We don't want to have multiple requests for any given node, so in priority try to - // find a connection with an existing request, to override it. - if let Some(entry) = peer.iter_mut().find(|c| c.ongoing_request.is_some()) { - entry - } else if let Some(entry) = peer.get_mut(0) { - entry - } else { - log::error!( - target: "sync", - "State inconsistency: empty list of peer connections" - ); - return SendRequestOutcome::NotConnected; - } - } else { - return SendRequestOutcome::NotConnected; - }; - - let protobuf_rq = build_protobuf_block_request( - req.fields, - req.from.clone(), - req.to.clone(), - req.direction, - req.max, - ); - - let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); - if let Err(err) = protobuf_rq.encode(&mut buf) { - log::warn!( - target: "sync", - "Failed to encode block request {:?}: {:?}", - protobuf_rq, - err - ); - return SendRequestOutcome::EncodeError(err); - } - - let previous_request = connection.ongoing_request.take(); - connection.ongoing_request = Some(OngoingRequest { - emitted: Instant::now(), - request: req.clone(), - timeout: Delay::new(self.config.request_timeout), - }); - - log::trace!(target: "sync", "Enqueueing block request to {:?}: {:?}", target, protobuf_rq); - self.pending_events.push_back(NetworkBehaviourAction::NotifyHandler { - peer_id: target.clone(), - handler: NotifyHandler::One(connection.id), - event: OutboundProtocol { - request: buf, - original_request: req, - max_response_size: self.config.max_response_len, - protocol: self.config.protocol.as_bytes().to_vec().into(), - }, - }); - - if let Some(previous_request) = previous_request { - log::debug!( - target: "sync", - "Replacing existing block request on connection {:?}", - connection.id - ); - SendRequestOutcome::Replaced { - previous: previous_request.request, - request_duration: previous_request.emitted.elapsed(), - } - } else { - SendRequestOutcome::Ok - } - } - - /// Callback, invoked when a new block request has been received from remote. - fn on_block_request - ( &mut self - , peer: &PeerId - , request: &schema::v1::BlockRequest - ) -> Result - { - log::trace!( - target: "sync", - "Block request from peer {}: from block {:?} to block {:?}, max blocks {:?}", - peer, - request.from_block, - request.to_block, - request.max_blocks); - - let from_block_id = - match request.from_block { - Some(schema::v1::block_request::FromBlock::Hash(ref h)) => { - let h = Decode::decode(&mut h.as_ref())?; - BlockId::::Hash(h) - } - Some(schema::v1::block_request::FromBlock::Number(ref n)) => { - let n = Decode::decode(&mut n.as_ref())?; - BlockId::::Number(n) - } - None => { - let msg = "missing `BlockRequest::from_block` field"; - return Err(io::Error::new(io::ErrorKind::Other, msg).into()) - } - }; - - let max_blocks = - if request.max_blocks == 0 { - self.config.max_block_data_response - } else { - min(request.max_blocks, self.config.max_block_data_response) - }; - - let direction = - if request.direction == schema::v1::Direction::Ascending as i32 { - schema::v1::Direction::Ascending - } else if request.direction == schema::v1::Direction::Descending as i32 { - schema::v1::Direction::Descending - } else { - let msg = format!("invalid `BlockRequest::direction` value: {}", request.direction); - return Err(io::Error::new(io::ErrorKind::Other, msg).into()) - }; - - let attributes = BlockAttributes::from_be_u32(request.fields)?; - let get_header = attributes.contains(BlockAttributes::HEADER); - let get_body = attributes.contains(BlockAttributes::BODY); - let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); - - let mut blocks = Vec::new(); - let mut block_id = from_block_id; - let mut total_size = 0; - while let Some(header) = self.chain.header(block_id).unwrap_or(None) { - if blocks.len() >= max_blocks as usize - || (blocks.len() >= 1 && total_size > self.config.max_block_body_bytes) - { - break - } - - let number = *header.number(); - let hash = header.hash(); - let parent_hash = *header.parent_hash(); - let justification = if get_justification { - self.chain.justification(&BlockId::Hash(hash))? - } else { - None - }; - let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false); - - let body = if get_body { - match self.chain.block_body(&BlockId::Hash(hash))? { - Some(mut extrinsics) => extrinsics.iter_mut() - .map(|extrinsic| extrinsic.encode()) - .collect(), - None => { - log::trace!(target: "sync", "Missing data for block request."); - break; - } - } - } else { - Vec::new() - }; - - let block_data = schema::v1::BlockData { - hash: hash.encode(), - header: if get_header { - header.encode() - } else { - Vec::new() - }, - body, - receipt: Vec::new(), - message_queue: Vec::new(), - justification: justification.unwrap_or_default(), - is_empty_justification, - }; - - total_size += block_data.body.len(); - blocks.push(block_data); - - match direction { - schema::v1::Direction::Ascending => { - block_id = BlockId::Number(number + One::one()) - } - schema::v1::Direction::Descending => { - if number.is_zero() { - break - } - block_id = BlockId::Hash(parent_hash) - } - } - } - - Ok(schema::v1::BlockResponse { blocks }) - } -} - -impl NetworkBehaviour for BlockRequests -where - B: Block -{ - type ProtocolsHandler = OneShotHandler, OutboundProtocol, NodeEvent>; - type OutEvent = Event; - - fn new_handler(&mut self) -> Self::ProtocolsHandler { - let p = InboundProtocol { - max_request_len: self.config.max_request_len, - protocol: self.config.protocol.as_bytes().to_owned().into(), - marker: PhantomData, - }; - let mut cfg = OneShotHandlerConfig::default(); - cfg.keep_alive_timeout = self.config.inactivity_timeout; - cfg.outbound_substream_timeout = self.config.request_timeout; - OneShotHandler::new(SubstreamProtocol::new(p, ()), cfg) - } - - fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { - Vec::new() - } - - fn inject_connected(&mut self, _peer: &PeerId) { - } - - fn inject_disconnected(&mut self, _peer: &PeerId) { - } - - fn inject_connection_established(&mut self, peer_id: &PeerId, id: &ConnectionId, _: &ConnectedPoint) { - self.peers.entry(peer_id.clone()) - .or_default() - .push(Connection { - id: *id, - ongoing_request: None, - }); - } - - fn inject_connection_closed(&mut self, peer_id: &PeerId, id: &ConnectionId, _: &ConnectedPoint) { - let mut needs_remove = false; - if let Some(entry) = self.peers.get_mut(peer_id) { - if let Some(pos) = entry.iter().position(|i| i.id == *id) { - let ongoing_request = entry.remove(pos).ongoing_request; - if let Some(ongoing_request) = ongoing_request { - log::debug!( - target: "sync", - "Connection {:?} with {} closed with ongoing sync request: {:?}", - id, - peer_id, - ongoing_request - ); - let ev = Event::RequestCancelled { - peer: peer_id.clone(), - request_duration: ongoing_request.emitted.elapsed(), - }; - self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); - } - if entry.is_empty() { - needs_remove = true; - } - } else { - log::error!( - target: "sync", - "State inconsistency: connection id not found in list" - ); - } - } else { - log::error!( - target: "sync", - "State inconsistency: peer_id not found in list of connections" - ); - } - if needs_remove { - self.peers.remove(peer_id); - } - } - - fn inject_event( - &mut self, - peer: PeerId, - connection_id: ConnectionId, - node_event: NodeEvent - ) { - match node_event { - NodeEvent::Request(request, mut stream, handling_start) => { - match self.on_block_request(&peer, &request) { - Ok(res) => { - log::trace!( - target: "sync", - "Enqueueing block response for peer {} with {} blocks", - peer, res.blocks.len() - ); - let mut data = Vec::with_capacity(res.encoded_len()); - if let Err(e) = res.encode(&mut data) { - log::debug!( - target: "sync", - "Error encoding block response for peer {}: {}", - peer, e - ) - } else { - self.outgoing.push(async move { - if let Err(e) = write_one(&mut stream, data).await { - log::debug!( - target: "sync", - "Error writing block response: {}", - e - ); - } - (peer, handling_start.elapsed()) - }.boxed()); - } - } - Err(e) => log::debug!( - target: "sync", - "Error handling block request from peer {}: {}", peer, e - ) - } - } - NodeEvent::Response(original_request, response) => { - log::trace!( - target: "sync", - "Received block response from peer {} with {} blocks", - peer, response.blocks.len() - ); - let request_duration = if let Some(connections) = self.peers.get_mut(&peer) { - if let Some(connection) = connections.iter_mut().find(|c| c.id == connection_id) { - if let Some(ongoing_request) = &mut connection.ongoing_request { - if ongoing_request.request == original_request { - let request_duration = ongoing_request.emitted.elapsed(); - connection.ongoing_request = None; - request_duration - } else { - // We're no longer interested in that request. - log::debug!( - target: "sync", - "Received response from {} to obsolete block request {:?}", - peer, - original_request - ); - return; - } - } else { - // We remove from `self.peers` requests we're no longer interested in, - // so this can legitimately happen. - log::trace!( - target: "sync", - "Response discarded because it concerns an obsolete request" - ); - return; - } - } else { - log::error!( - target: "sync", - "State inconsistency: response on non-existing connection {:?}", - connection_id - ); - return; - } - } else { - log::error!( - target: "sync", - "State inconsistency: response on non-connected peer {}", - peer - ); - return; - }; - - let blocks = response.blocks.into_iter().map(|block_data| { - Ok(message::BlockData:: { - hash: Decode::decode(&mut block_data.hash.as_ref())?, - header: if !block_data.header.is_empty() { - Some(Decode::decode(&mut block_data.header.as_ref())?) - } else { - None - }, - body: if original_request.fields.contains(message::BlockAttributes::BODY) { - Some(block_data.body.iter().map(|body| { - Decode::decode(&mut body.as_ref()) - }).collect::, _>>()?) - } else { - None - }, - receipt: if !block_data.message_queue.is_empty() { - Some(block_data.receipt) - } else { - None - }, - message_queue: if !block_data.message_queue.is_empty() { - Some(block_data.message_queue) - } else { - None - }, - justification: if !block_data.justification.is_empty() { - Some(block_data.justification) - } else if block_data.is_empty_justification { - Some(Vec::new()) - } else { - None - }, - }) - }).collect::, codec::Error>>(); - - match blocks { - Ok(blocks) => { - let id = original_request.id; - let ev = Event::Response { - peer, - response: message::BlockResponse:: { id, blocks }, - request_duration, - }; - self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); - } - Err(err) => { - log::debug!( - target: "sync", - "Failed to decode block response from peer {}: {}", peer, err - ); - } - } - } - } - } - - fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) - -> Poll, Event>> - { - if let Some(ev) = self.pending_events.pop_front() { - return Poll::Ready(ev); - } - - // Check the request timeouts. - for (peer, connections) in &mut self.peers { - for connection in connections { - let ongoing_request = match &mut connection.ongoing_request { - Some(rq) => rq, - None => continue, - }; - - if let Poll::Ready(_) = Pin::new(&mut ongoing_request.timeout).poll(cx) { - let original_request = ongoing_request.request.clone(); - let request_duration = ongoing_request.emitted.elapsed(); - connection.ongoing_request = None; - log::debug!( - target: "sync", - "Request timeout for {}: {:?}", - peer, original_request - ); - let ev = Event::RequestTimeout { - peer: peer.clone(), - request_duration, - }; - return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); - } - } - } - - if let Poll::Ready(Some((peer, total_handling_time))) = self.outgoing.poll_next_unpin(cx) { - let ev = Event::AnsweredRequest { - peer, - total_handling_time, - }; - return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); - } - - Poll::Pending - } -} - -/// Output type of inbound and outbound substream upgrades. -#[derive(Debug)] -pub enum NodeEvent { - /// Incoming request from remote, substream to use for the response, and when we started - /// handling this request. - Request(schema::v1::BlockRequest, T, Instant), - /// Incoming response from remote. - Response(message::BlockRequest, schema::v1::BlockResponse), -} - -/// Substream upgrade protocol. -/// -/// We attempt to parse an incoming protobuf encoded request (cf. `Request`) -/// which will be handled by the `BlockRequests` behaviour, i.e. the request -/// will become visible via `inject_node_event` which then dispatches to the -/// relevant callback to process the message and prepare a response. -#[derive(Debug, Clone)] -pub struct InboundProtocol { - /// The max. request length in bytes. - max_request_len: usize, - /// The protocol to use during upgrade negotiation. - protocol: Bytes, - /// Type of the block. - marker: PhantomData, -} - -impl UpgradeInfo for InboundProtocol { - type Info = Bytes; - type InfoIter = iter::Once; - - fn protocol_info(&self) -> Self::InfoIter { - iter::once(self.protocol.clone()) - } -} - -impl InboundUpgrade for InboundProtocol -where - B: Block, - T: AsyncRead + AsyncWrite + Unpin + Send + 'static -{ - type Output = NodeEvent; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; - - fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { - // This `Instant` will be passed around until the processing of this request is done. - let handling_start = Instant::now(); - - let future = async move { - let len = self.max_request_len; - let vec = read_one(&mut s, len).await?; - match schema::v1::BlockRequest::decode(&vec[..]) { - Ok(r) => Ok(NodeEvent::Request(r, s, handling_start)), - Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) - } - }; - future.boxed() - } -} - -/// Substream upgrade protocol. -/// -/// Sends a request to remote and awaits the response. -#[derive(Debug, Clone)] -pub struct OutboundProtocol { - /// The serialized protobuf request. - request: Vec, - /// The original request. Passed back through the API when the response comes back. - original_request: message::BlockRequest, - /// The max. response length in bytes. - max_response_size: usize, - /// The protocol to use for upgrade negotiation. - protocol: Bytes, -} - -impl UpgradeInfo for OutboundProtocol { - type Info = Bytes; - type InfoIter = iter::Once; - - fn protocol_info(&self) -> Self::InfoIter { - iter::once(self.protocol.clone()) - } -} - -impl OutboundUpgrade for OutboundProtocol -where - B: Block, - T: AsyncRead + AsyncWrite + Unpin + Send + 'static -{ - type Output = NodeEvent; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; - - fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { - async move { - write_one(&mut s, &self.request).await?; - let vec = read_one(&mut s, self.max_response_size).await?; - - schema::v1::BlockResponse::decode(&vec[..]) - .map(|r| NodeEvent::Response(self.original_request, r)) - .map_err(|e| { - ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) - }) - }.boxed() - } -} - -/// Build protobuf block request message. -pub(crate) fn build_protobuf_block_request( - attributes: BlockAttributes, - from_block: message::FromBlock, - to_block: Option, - direction: message::Direction, - max_blocks: Option, -) -> schema::v1::BlockRequest { - schema::v1::BlockRequest { - fields: attributes.to_be_u32(), - from_block: match from_block { - message::FromBlock::Hash(h) => - Some(schema::v1::block_request::FromBlock::Hash(h.encode())), - message::FromBlock::Number(n) => - Some(schema::v1::block_request::FromBlock::Number(n.encode())), - }, - to_block: to_block.map(|h| h.encode()).unwrap_or_default(), - direction: match direction { - message::Direction::Ascending => schema::v1::Direction::Ascending as i32, - message::Direction::Descending => schema::v1::Direction::Descending as i32, - }, - max_blocks: max_blocks.unwrap_or(0), - } -} diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 86450dc6e79bf..03e4a799cf6d0 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -70,12 +70,6 @@ pub struct Params { /// Client that contains the blockchain. pub chain: Arc>, - /// Finality proof provider. - /// - /// This object, if `Some`, is used when a node on the network requests a proof of finality - /// from us. - pub finality_proof_provider: Option>>, - /// How to build requests for proofs of finality. /// /// This object, if `Some`, is used when we need a proof of finality from another node. diff --git a/client/network/src/finality_request_handler.rs b/client/network/src/finality_request_handler.rs new file mode 100644 index 0000000000000..c451e60e495de --- /dev/null +++ b/client/network/src/finality_request_handler.rs @@ -0,0 +1,67 @@ +use crate::request_responses::ProtocolConfig; +use futures::channel::mpsc; +use futures::stream::StreamExt; +use std::sync::{Arc}; +pub use crate::{ + config::ProtocolId, + chain::{Client, FinalityProofProvider}}; +use sp_runtime::{traits::Block as BlockT}; +use crate::request_responses::IncomingRequest; +use prost::Message; + +pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { + let mut s = String::new(); + s.push_str("/"); + s.push_str(protocol_id.as_ref()); + s.push_str("/finality-proof/1"); + s +} + +pub struct FinalityRequestHandler { + proof_provider: Arc>, + request_receiver: mpsc::Receiver, +} + +impl FinalityRequestHandler { + pub fn new(protocol_id: ProtocolId, proof_provider: Arc>) -> (Self, ProtocolConfig){ + let (tx, rx) = mpsc::channel(0); + + let protocol_config = ProtocolConfig { + name: generate_protocol_name(protocol_id).into(), + // TODO: Change. + max_request_size: 4096, + // TODO: Change. + max_response_size: 4096, + request_timeout: std::time::Duration::from_secs(10), + inbound_queue: Some(tx), + }; + + let handler = FinalityRequestHandler { + proof_provider, + request_receiver: rx, + }; + + (handler, protocol_config) + } + + pub async fn run(mut self) { + while let Some(crate::request_responses::IncomingRequest { peer, payload, pending_response }) = self.request_receiver.next().await { + let req = crate::schema::v1::finality::FinalityProofRequest::decode(&payload[..]).unwrap(); + + let block_hash = codec::Decode::decode(&mut req.block_hash.as_ref()).unwrap(); + + log::trace!(target: "sync", "Finality proof request from {} for {}", peer, block_hash); + + let finality_proof = self.proof_provider + .prove_finality(block_hash, &req.request) + .unwrap() + .unwrap_or_default(); + + let resp = crate::schema::v1::finality::FinalityProofResponse { proof: finality_proof }; + let mut data = Vec::with_capacity(resp.encoded_len()); + resp.encode(&mut data).unwrap(); + + pending_response.send(data).unwrap(); + } + } +} diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index b050db8785ac1..99eb847866f38 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -245,7 +245,6 @@ //! mod behaviour; -mod block_requests; mod chain; mod peer_info; mod discovery; @@ -260,6 +259,8 @@ mod utils; pub mod config; pub mod error; +pub mod finality_request_handler; +pub mod block_request_handler; pub mod gossip; pub mod network_state; diff --git a/client/network/src/light_client_handler.rs b/client/network/src/light_client_handler.rs index d1c407c99695b..383a03f0b0675 100644 --- a/client/network/src/light_client_handler.rs +++ b/client/network/src/light_client_handler.rs @@ -27,7 +27,6 @@ use bytes::Bytes; use codec::{self, Encode, Decode}; use crate::{ - block_requests::build_protobuf_block_request, chain::Client, config::ProtocolId, protocol::message::{BlockAttributes, Direction, FromBlock}, @@ -1063,13 +1062,14 @@ fn retries(request: &Request) -> usize { fn serialize_request(request: &Request) -> Result, prost::EncodeError> { let request = match request { Request::Body { request, .. } => { - let rq = build_protobuf_block_request::<_, NumberFor>( - BlockAttributes::BODY, - FromBlock::Hash(request.header.hash()), - None, - Direction::Ascending, - Some(1), - ); + let rq = crate::schema::v1::BlockRequest { + fields: BlockAttributes::BODY.to_be_u32(), + from_block: Some(crate::schema::v1::block_request::FromBlock::Hash(request.header.hash().encode())), + to_block: Default::default(), + direction:crate::schema::v1::Direction::Ascending as i32, + max_blocks: 1, + }; + let mut buf = Vec::with_capacity(rq.encoded_len()); rq.encode(&mut buf)?; return Ok(buf); diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 723248de135db..3c26663e13fd2 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -728,7 +728,7 @@ impl Protocol { if block_request.2 != Some(request_id) { // TODO: Properly handle this. panic!("request id didn't match or was none"); - return CustomMessageOutcome::None; + // return CustomMessageOutcome::None; } let block_request = block_request.1; @@ -844,7 +844,7 @@ impl Protocol { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), Ok(sync::OnBlockData::Request(peer, mut req)) => { - self.update_peer_request(&peer_id, &mut req); + self.update_peer_request(&peer, &mut req); CustomMessageOutcome::BlockRequest { target: peer_id, request: req, @@ -859,14 +859,15 @@ impl Protocol { } } - /// Must be called in response to a [`CustomMessageOutcome::BlockRequest`] if it has failed. - pub fn on_block_request_failed( - &mut self, - peer: &PeerId, - ) { - self.peerset_handle.report_peer(peer.clone(), rep::TIMEOUT); - self.behaviour.disconnect_peer(peer); - } + // TODO: Bring back? + // /// Must be called in response to a [`CustomMessageOutcome::BlockRequest`] if it has failed. + // pub fn on_block_request_failed( + // &mut self, + // peer: &PeerId, + // ) { + // self.peerset_handle.report_peer(peer.clone(), rep::TIMEOUT); + // self.behaviour.disconnect_peer(peer); + // } /// Perform time based maintenance. /// diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index d837dd787555a..7207f0fb5fd05 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -37,7 +37,7 @@ use sp_consensus::{BlockOrigin, BlockStatus, use crate::{ config::BoxFinalityProofRequestBuilder, protocol::message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, - FinalityProofResponse, Roles}, + Roles}, }; use either::Either; use extra_requests::ExtraRequests; @@ -117,7 +117,8 @@ mod rep { pub const BAD_JUSTIFICATION: Rep = Rep::new(-(1 << 16), "Bad justification"); /// Reputation change for peers which send us a block with bad finality proof. - pub const BAD_FINALITY_PROOF: Rep = Rep::new(-(1 << 16), "Bad finality proof"); + // TODO: bring back? + // pub const BAD_FINALITY_PROOF: Rep = Rep::new(-(1 << 16), "Bad finality proof"); /// Reputation change when a peer sent us invlid ancestry result. pub const UNKNOWN_ANCESTOR:Rep = Rep::new(-(1 << 16), "DB Error"); @@ -1870,7 +1871,7 @@ mod test { // there's one in-flight extra request to the expected peer assert!( - sync.extra_justifications.active_requests().any(|(who, (hash, number))| { + sync.extra_justifications.active_requests().any(|(who, ActiveRequest { block_hash, number, .. })| { *who == peer_id && *hash == a1_hash && *number == a1_number }) ); diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 8d5569734da9c..f278634a50f91 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -188,7 +188,7 @@ impl ExtraRequests { } } - if let Some(ActiveRequest { request_id, block_hash, number }) = self.active_requests.remove(&who) { + if let Some(ActiveRequest { block_hash, number, .. }) = self.active_requests.remove(&who) { if let Some(r) = resp { trace!(target: "sync", "Queuing import of {} from {:?} for {:?}", self.request_type_name, diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 0f9f539c03dd1..15d22d368aa59 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -38,7 +38,7 @@ use crate::{ NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer, }, on_demand_layer::AlwaysBadChecker, - light_client_handler, block_requests, + light_client_handler, protocol::{self, event::Event, NotifsHandlerError, LegacyConnectionKillError, NotificationsSink, Ready, sync::SyncState, PeerInfo, Protocol}, transport, ReputationChange, }; @@ -51,7 +51,6 @@ use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent, protocols_handle use log::{error, info, trace, warn}; use metrics::{Metrics, MetricSources, Histogram, HistogramVec}; use parking_lot::Mutex; -use prost::Message; use sc_peerset::PeersetHandle; use sp_consensus::import_queue::{BlockImportError, BlockImportResult, ImportQueue, Link}; use sp_runtime::{ @@ -118,7 +117,7 @@ impl NetworkWorker { /// Returns a `NetworkWorker` that implements `Future` and must be regularly polled in order /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. - pub fn new(mut params: Params) -> Result, Error> { + pub fn new(params: Params) -> Result, Error> { // Ensure the listen addresses are consistent with the transport. ensure_addresses_consistent_with_transport( params.network_config.listen_addresses.iter(), @@ -269,10 +268,6 @@ impl NetworkWorker { params.network_config.client_version, params.network_config.node_name ); - let block_requests = { - let config = block_requests::Config::new(¶ms.protocol_id); - block_requests::BlockRequests::new(config, params.chain.clone()) - }; let light_client_handler = { let config = light_client_handler::Config::new(¶ms.protocol_id); light_client_handler::LightClientHandler::new( @@ -284,42 +279,14 @@ impl NetworkWorker { }; // TODO: Should finality request protocol be added here? - let mut request_response_protocols = params.network_config.request_response_protocols; - let (tx, mut rx) = futures::channel::mpsc::channel(10); - request_response_protocols.push(crate::request_responses::ProtocolConfig { - name: std::borrow::Cow::Borrowed(&"abc"), - // TODO: Change. - max_request_size: 4096, - // TODO: Change. - max_response_size: 4096, - request_timeout: std::time::Duration::from_secs(10), - inbound_queue: Some(tx), - }); - - let finality_proof_provider = params.finality_proof_provider.clone(); - let finality_proof_server = async move { - let finality_proof_provider = finality_proof_provider.unwrap(); - while let Some(crate::request_responses::IncomingRequest { peer, payload, pending_response }) = rx.next().await { - let req = crate::schema::v1::finality::FinalityProofRequest::decode(&payload[..]).unwrap(); - - let block_hash = codec::Decode::decode(&mut req.block_hash.as_ref()).unwrap(); + // let mut request_response_protocols = params.network_config.request_response_protocols; + // request_response_protocols.push(); - log::trace!(target: "sync", "Finality proof request from {} for {}", peer, block_hash); - - let finality_proof = finality_proof_provider - .prove_finality(block_hash, &req.request) - .unwrap() - .unwrap_or_default(); - - let resp = crate::schema::v1::finality::FinalityProofResponse { proof: finality_proof }; - let mut data = Vec::with_capacity(resp.encoded_len()); - resp.encode(&mut data).unwrap(); - - pending_response.send(data).unwrap(); - } - }.boxed(); + // let finality_proof_provider = params.finality_proof_provider.clone(); + // let finality_proof_server = async move { + // }.boxed(); - params.executor.as_mut().unwrap()(finality_proof_server); + // params.executor.as_mut().unwrap()(finality_proof_server); let discovery_config = { let mut config = DiscoveryConfig::new(local_public.clone()); @@ -349,10 +316,11 @@ impl NetworkWorker { params.role, user_agent, local_public, - block_requests, light_client_handler, discovery_config, - request_response_protocols, + params.network_config.request_response_protocols, + crate::block_request_handler::generate_protocol_name(params.protocol_id.clone()), + crate::finality_request_handler::generate_protocol_name(params.protocol_id.clone()), ); match result { @@ -1482,20 +1450,23 @@ impl Future for NetworkWorker { error!("Request not in pending_requests"); } }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestStarted { protocol, .. })) => { - if let Some(metrics) = this.metrics.as_ref() { - metrics.requests_out_started_total - .with_label_values(&[&protocol]) - .inc(); - } - }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestFinished { protocol, request_duration, .. })) => { - if let Some(metrics) = this.metrics.as_ref() { - metrics.requests_out_success_total - .with_label_values(&[&protocol]) - .observe(request_duration.as_secs_f64()); - } - }, + // TODO: Is this still needed? Can we not do this within the substrate + // requests_response behaviour? + // + // Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestStarted { protocol, .. })) => { + // if let Some(metrics) = this.metrics.as_ref() { + // metrics.requests_out_started_total + // .with_label_values(&[&protocol]) + // .inc(); + // } + // }, + // Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestFinished { protocol, request_duration, .. })) => { + // if let Some(metrics) = this.metrics.as_ref() { + // metrics.requests_out_success_total + // .with_label_values(&[&protocol]) + // .observe(request_duration.as_secs_f64()); + // } + // }, Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted(protocol))) => { if let Some(metrics) = this.metrics.as_ref() { metrics.kademlia_random_queries_total @@ -1616,14 +1587,14 @@ impl Future for NetworkWorker { let reason = match cause { Some(ConnectionError::IO(_)) => "transport-error", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::B(EitherError::A( - PingFailure::Timeout))))))))) => "ping-timeout", + EitherError::A(EitherError::B(EitherError::A( + PingFailure::Timeout)))))))) => "ping-timeout", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A( - NotifsHandlerError::Legacy(LegacyConnectionKillError))))))))) => "force-closed", + EitherError::A(EitherError::A( + NotifsHandlerError::Legacy(LegacyConnectionKillError)))))))) => "force-closed", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A( - NotifsHandlerError::SyncNotificationsClogged)))))))) => "sync-notifications-clogged", + EitherError::A(EitherError::A( + NotifsHandlerError::SyncNotificationsClogged))))))) => "sync-notifications-clogged", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(_))) => "protocol-error", Some(ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout)) => "keep-alive-timeout", None => "actively-closed", diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 3b60db7ec5854..2dfd93401b59f 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -870,6 +870,20 @@ pub fn build_network( Box::new(DefaultBlockAnnounceValidator) }; + let mut network_config = config.network.clone(); + + if let Some(proof_provider) = finality_proof_provider { + let (handler, protocol_config) = sc_network::finality_request_handler::FinalityRequestHandler::new(protocol_id.clone(), proof_provider); + network_config.request_response_protocols.push(protocol_config); + spawn_handle.spawn("finality_request_handler", handler.run()); + } + + { + let (handler, protocol_config) = sc_network::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + network_config.request_response_protocols.push(protocol_config); + spawn_handle.spawn("block_request_handler", handler.run()); + } + let network_params = sc_network::config::Params { role: config.role.clone(), executor: { @@ -878,9 +892,8 @@ pub fn build_network( spawn_handle.spawn("libp2p-node", fut); })) }, - network_config: config.network.clone(), + network_config, chain: client.clone(), - finality_proof_provider, finality_proof_request_builder, on_demand: on_demand, transaction_pool: transaction_pool_adapter as _, From 8abd813f110efa03478687ac54dcddb6956b5bc8 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 2 Nov 2020 15:32:12 +0100 Subject: [PATCH 06/48] client/network: Track ongoing finality requests in protocol.rs --- .maintain/sentry-node/docker-compose.yml | 14 +-- client/network/src/behaviour.rs | 17 +++- client/network/src/protocol.rs | 37 +++++++- client/network/src/protocol/sync.rs | 45 ++++----- .../src/protocol/sync/extra_requests.rs | 94 +++---------------- 5 files changed, 87 insertions(+), 120 deletions(-) diff --git a/.maintain/sentry-node/docker-compose.yml b/.maintain/sentry-node/docker-compose.yml index 2af9449853c77..a4cc8f1ebb92e 100644 --- a/.maintain/sentry-node/docker-compose.yml +++ b/.maintain/sentry-node/docker-compose.yml @@ -47,9 +47,9 @@ services: - "--validator" - "--alice" - "--sentry-nodes" - - "/dns/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" + - "/dns/sentry-a/tcp/30333/p2p/12D3KooWSCufgHzV4fCwRijfH2k3abrpAJxTKxEvN1FDuRXA2U9x" - "--reserved-nodes" - - "/dns/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" + - "/dns/sentry-a/tcp/30333/p2p/12D3KooWSCufgHzV4fCwRijfH2k3abrpAJxTKxEvN1FDuRXA2U9x" # Not only bind to localhost. - "--unsafe-ws-external" - "--unsafe-rpc-external" @@ -83,11 +83,11 @@ services: - "--port" - "30333" - "--sentry" - - "/dns/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "/dns/validator-a/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" - "--reserved-nodes" - - "/dns/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "/dns/validator-a/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" - "--bootnodes" - - "/dns/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk" + - "/dns/validator-b/tcp/30333/p2p/12D3KooWHdiAxVd8uMQR1hGWXccidmfCwLqcMpGwR6QcTP6QRMuD" - "--no-telemetry" - "--rpc-cors" - "all" @@ -118,9 +118,9 @@ services: - "--validator" - "--bob" - "--bootnodes" - - "/dns/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "/dns/validator-a/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp" - "--bootnodes" - - "/dns/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" + - "/dns/sentry-a/tcp/30333/p2p/12D3KooWSCufgHzV4fCwRijfH2k3abrpAJxTKxEvN1FDuRXA2U9x" - "--no-telemetry" - "--rpc-cors" - "all" diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 1d446144f569d..d3a6732e8998a 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -461,10 +461,6 @@ Behaviour { impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: request_responses::Event) { - // Can we do prettier? - let finality_request_protocol_name = Cow::Borrowed(&self.finality_request_protocol_name); - let block_request_protocol_name = Cow::Borrowed(&self.block_request_protocol_name); - match event { request_responses::Event::InboundRequest { peer, protocol, result } => { self.events.push_back(BehaviourOut::InboundRequest { @@ -480,7 +476,18 @@ impl NetworkBehaviourEventProcess resp, + Err(e) => { + debug!( + target: "sub-libp2p", + "Block request to {:?} failed with: {:?}", + peer, e, + ); + return + } + }; + let protobuf_response = crate::schema::v1::BlockResponse::decode(&resp[..]).unwrap(); let ev = self.substrate.on_block_response(peer, request_id, protobuf_response); self.inject_event(ev); diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 3c26663e13fd2..e9a02cb01a3fe 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -257,7 +257,10 @@ struct PacketStats { struct Peer { info: PeerInfo, /// Current block request, if any. + // TODO: Is Instant stil needed? block_request: Option<(Instant, message::BlockRequest, Option)>, + // TODO: Document + finality_request: Option<(message::FinalityProofRequest, Option)>, /// Requests we are no longer interested in. // TODO: Do we still need this at all? obsolete_requests: HashMap, @@ -976,6 +979,7 @@ impl Protocol { best_number: status.best_number }, block_request: None, + finality_request: None, known_transactions: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS) .expect("Constant is nonzero")), known_blocks: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_BLOCKS) @@ -1442,7 +1446,25 @@ impl Protocol { request_id: libp2p::request_response::RequestId, proof: Vec, ) -> CustomMessageOutcome { - match self.sync.on_block_finality_proof(who, request_id, proof) { + let peer = match self.context_data.peers.get_mut(&who) { + Some(peer) => peer, + None => return CustomMessageOutcome::None, + }; + + let req = match &peer.finality_request { + Some((_req, Some(id))) if *id != request_id => peer.finality_request.take().unwrap().0, + Some(_) => return CustomMessageOutcome::None, + None => return CustomMessageOutcome::None, + }; + + let resp = message::FinalityProofResponse:: { + // TODO: correct? + id: 0, + block: req.block, + proof: if proof.is_empty() { None } else { Some(proof) }, + }; + + match self.sync.on_block_finality_proof(who, resp) { Ok(sync::OnBlockFinalityProof::Nothing) => CustomMessageOutcome::None, Ok(sync::OnBlockFinalityProof::Import { peer, hash, number, proof }) => CustomMessageOutcome::FinalityProofImport(peer, hash, number, proof), @@ -1462,7 +1484,12 @@ impl Protocol { block_hash: B::Hash, request_id: libp2p::request_response::RequestId, ) { - self.sync.on_finality_proof_request_started(who, block_hash, request_id) + if let Some(Peer { finality_request: Some(req), ..}) = self.context_data.peers.get_mut(&who) { + if req.0.block == block_hash { + debug_assert!(req.1.is_none()); + req.1 = Some(request_id); + } + } } pub fn on_block_request_started( @@ -1671,6 +1698,12 @@ impl NetworkBehaviour for Protocol { self.pending_messages.push_back(event); } for (id, r) in self.sync.finality_proof_requests() { + match self.context_data.peers.get_mut(&id) { + Some(peer) => { + peer.finality_request = Some((r.clone(), None)) + }, + None => unimplemented!(), + } let event = CustomMessageOutcome::FinalityProofRequest { target: id, block_hash: r.block, diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 7207f0fb5fd05..03714b05ace0d 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -37,7 +37,7 @@ use sp_consensus::{BlockOrigin, BlockStatus, use crate::{ config::BoxFinalityProofRequestBuilder, protocol::message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, - Roles}, + FinalityProofResponse, Roles}, }; use either::Either; use extra_requests::ExtraRequests; @@ -117,8 +117,7 @@ mod rep { pub const BAD_JUSTIFICATION: Rep = Rep::new(-(1 << 16), "Bad justification"); /// Reputation change for peers which send us a block with bad finality proof. - // TODO: bring back? - // pub const BAD_FINALITY_PROOF: Rep = Rep::new(-(1 << 16), "Bad finality proof"); + pub const BAD_FINALITY_PROOF: Rep = Rep::new(-(1 << 16), "Bad finality proof"); /// Reputation change when a peer sent us invlid ancestry result. pub const UNKNOWN_ANCESTOR:Rep = Rep::new(-(1 << 16), "DB Error"); @@ -1034,22 +1033,10 @@ impl ChainSync { Ok(OnBlockJustification::Nothing) } - pub fn on_finality_proof_request_started( - &mut self, - who: PeerId, - block_hash: B::Hash, - request_id: libp2p::request_response::RequestId, - ) { - self.extra_finality_proofs.on_request_started(who, block_hash, request_id) - } - /// Handle new finality proof data. - pub fn on_block_finality_proof( - &mut self, - who: PeerId, - request_id: libp2p::request_response::RequestId, - proof: Vec, - ) -> Result, BadPeer> { + pub fn on_block_finality_proof + (&mut self, who: PeerId, resp: FinalityProofResponse) -> Result, BadPeer> + { let peer = if let Some(peer) = self.peers.get_mut(&who) { peer @@ -1059,14 +1046,22 @@ impl ChainSync { }; self.pending_requests.add(&who); - if let PeerSyncState::DownloadingFinalityProof(state_hash) = peer.state { + if let PeerSyncState::DownloadingFinalityProof(hash) = peer.state { peer.state = PeerSyncState::Available; - if let Some((peer, hash, number, p)) = self.extra_finality_proofs.on_response_with_request_id(who, request_id, if proof.is_empty() { None } else { Some(proof) }) { - trace!(target: "sync", "Finality proof response from {} for {}", peer, hash); - // TODO: Safe assumption to make? - assert_eq!(hash, state_hash); - return Ok(OnBlockFinalityProof::Import { peer, hash, number, proof: p}) + // We only request one finality proof at a time. + if hash != resp.block { + info!( + target: "sync", + "💔 Invalid block finality proof provided: requested: {:?} got: {:?}", + hash, + resp.block + ); + return Err(BadPeer(who, rep::BAD_FINALITY_PROOF)); + } + + if let Some((peer, hash, number, p)) = self.extra_finality_proofs.on_response(who, resp.proof) { + return Ok(OnBlockFinalityProof::Import { peer, hash, number, proof: p }) } } @@ -1871,7 +1866,7 @@ mod test { // there's one in-flight extra request to the expected peer assert!( - sync.extra_justifications.active_requests().any(|(who, ActiveRequest { block_hash, number, .. })| { + sync.extra_justifications.active_requests().any(|(who, (hash, number))| { *who == peer_id && *hash == a1_hash && *number == a1_number }) ); diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index f278634a50f91..df336c25339fd 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -46,7 +46,7 @@ pub(crate) struct ExtraRequests { /// requests which have been queued for later processing pending_requests: VecDeque>, /// requests which are currently underway to some peer - active_requests: HashMap>, + active_requests: HashMap>, /// previous requests without response failed_requests: HashMap, Vec<(PeerId, Instant)>>, /// successful requests @@ -55,13 +55,6 @@ pub(crate) struct ExtraRequests { request_type_name: &'static str, } -#[derive(Debug)] -struct ActiveRequest { - request_id: Option, - block_hash: ::Hash, - number: NumberFor, -} - #[derive(Debug)] pub(crate) struct Metrics { pub(crate) pending_requests: u32, @@ -121,21 +114,8 @@ impl ExtraRequests { /// Retry any pending request if a peer disconnected. pub(crate) fn peer_disconnected(&mut self, who: &PeerId) { - if let Some(ActiveRequest { block_hash, number, ..}) = self.active_requests.remove(who) { - self.pending_requests.push_front((block_hash, number)); - } - } - - pub(crate) fn on_request_started( - &mut self, - who: PeerId, - block_hash: B::Hash, - request_id: libp2p::request_response::RequestId, - ) { - if let Some(req) = self.active_requests.get_mut(&who) { - if req.block_hash == block_hash { - req.request_id = Some(request_id); - } + if let Some(request) = self.active_requests.remove(who) { + self.pending_requests.push_front(request); } } @@ -144,69 +124,25 @@ impl ExtraRequests { // we assume that the request maps to the given response, this is // currently enforced by the outer network protocol before passing on // messages to chain sync. - if let Some(ActiveRequest { block_hash, number, ..}) = self.active_requests.remove(&who) { - if let Some(r) = resp { - trace!(target: "sync", "Queuing import of {} from {:?} for {:?}", - self.request_type_name, - who, - (block_hash, number), - ); - - self.importing_requests.insert((block_hash, number)); - return Some((who, block_hash, number, r)) - } else { - trace!(target: "sync", "Empty {} response from {:?} for {:?}", - self.request_type_name, - who, - (block_hash, number), - ); - } - self.failed_requests.entry((block_hash, number)).or_default().push((who, Instant::now())); - self.pending_requests.push_front((block_hash, number)); - } else { - trace!(target: "sync", "No active {} request to {:?}", - self.request_type_name, - who, - ); - } - None - } - - /// Processes the response for the request previously sent to the given peer. - pub(crate) fn on_response_with_request_id( - &mut self, - who: PeerId, - request_id: libp2p::request_response::RequestId, - // TODO: Is it needed to send the response through this function? - resp: Option, - // TODO: Why return the peer id once more? - ) -> Option<(PeerId, B::Hash, NumberFor, R)> { - if let Some(req) = self.active_requests.get(&who) { - if req.request_id != Some(request_id) { - // TODO: Should we log something? - return None; - } - } - - if let Some(ActiveRequest { block_hash, number, .. }) = self.active_requests.remove(&who) { + if let Some(request) = self.active_requests.remove(&who) { if let Some(r) = resp { trace!(target: "sync", "Queuing import of {} from {:?} for {:?}", self.request_type_name, who, - (block_hash, number), + request, ); - self.importing_requests.insert((block_hash, number)); - return Some((who, block_hash, number, r)) + self.importing_requests.insert(request); + return Some((who, request.0, request.1, r)) } else { trace!(target: "sync", "Empty {} response from {:?} for {:?}", self.request_type_name, who, - (block_hash, number), + request, ); } - self.failed_requests.entry((block_hash, number)).or_default().push((who, Instant::now())); - self.pending_requests.push_front((block_hash, number)); + self.failed_requests.entry(request).or_default().push((who, Instant::now())); + self.pending_requests.push_front(request); } else { trace!(target: "sync", "No active {} request to {:?}", self.request_type_name, @@ -254,7 +190,7 @@ impl ExtraRequests { let roots = self.tree.roots().collect::>(); self.pending_requests.retain(|(h, n)| roots.contains(&(h, n, &()))); - self.active_requests.retain(|_, ActiveRequest { block_hash, number, ..}| roots.contains(&(block_hash, number, &()))); + self.active_requests.retain(|_, (h, n)| roots.contains(&(h, n, &()))); self.failed_requests.retain(|(h, n), _| roots.contains(&(h, n, &()))); Ok(()) @@ -304,7 +240,7 @@ impl ExtraRequests { /// Returns an iterator over all active (in-flight) requests and associated peer id. #[cfg(test)] - pub(crate) fn active_requests(&self) -> impl Iterator)> { + pub(crate) fn active_requests(&self) -> impl Iterator)> { self.active_requests.iter() } @@ -382,11 +318,7 @@ impl<'a, B: BlockT> Matcher<'a, B> { if self.extras.failed_requests.get(&request).map(|rr| rr.iter().any(|i| &i.0 == peer)).unwrap_or(false) { continue } - self.extras.active_requests.insert(peer.clone(), ActiveRequest{ - request_id: None, - block_hash: request.0, - number: request.1, - }); + self.extras.active_requests.insert(peer.clone(), request); trace!(target: "sync", "Sending {} request to {:?} for {:?}", self.extras.request_type_name, From 45de155416b2aea9fb448b847436c1a0f3bd0db0 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 2 Nov 2020 15:41:58 +0100 Subject: [PATCH 07/48] client/network: Remove commented out finalization initialization --- client/network/src/service.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 9ce7a5c055a7f..7021877b5854e 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -278,16 +278,6 @@ impl NetworkWorker { ) }; - // TODO: Should finality request protocol be added here? - // let mut request_response_protocols = params.network_config.request_response_protocols; - // request_response_protocols.push(); - - // let finality_proof_provider = params.finality_proof_provider.clone(); - // let finality_proof_server = async move { - // }.boxed(); - - // params.executor.as_mut().unwrap()(finality_proof_server); - let discovery_config = { let mut config = DiscoveryConfig::new(local_public.clone()); config.with_user_defined(known_addresses); From 456568feb21d86cb2d4ec74bde0355b7b14d282c Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 2 Nov 2020 16:05:26 +0100 Subject: [PATCH 08/48] client/network: Add docs for request handlers --- client/network/src/behaviour.rs | 54 +------------------ client/network/src/block_request_handler.rs | 23 ++++++++ .../network/src/finality_request_handler.rs | 23 ++++++++ client/network/src/lib.rs | 2 +- 4 files changed, 48 insertions(+), 54 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index d3a6732e8998a..0cf497cd7fe04 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -106,31 +106,6 @@ pub enum BehaviourOut { result: Result, RequestFailure>, }, - // TODO: Do we still need these? Can we not just depend on the request reponse behaviour to - // provide these metrics? - // - // /// Started a new request with the given node. - // /// - // /// This event is for statistics purposes only. The request and response handling are entirely - // /// internal to the behaviour. - // OpaqueRequestStarted { - // peer: PeerId, - // /// Protocol name of the request. - // protocol: String, - // }, - // /// Finished, successfully or not, a previously-started request. - // /// - // /// This event is for statistics purposes only. The request and response handling are entirely - // /// internal to the behaviour. - // OpaqueRequestFinished { - // /// Who we were requesting. - // peer: PeerId, - // /// Protocol name of the request. - // protocol: String, - // /// How long before the response came or the request got cancelled. - // request_duration: Duration, - // }, - /// Opened a substream with the given node with the given notifications protocol. /// /// The protocol is always one of the notification protocols that have been registered. @@ -378,34 +353,6 @@ Behaviour { let request_id = self.request_responses.send_request(&target, &self.block_request_protocol_name, buf).unwrap(); // TODO: differentiate between the two ids. self.substrate.on_block_request_started(target, request.id, request_id); - - - - - - - // TODO: Entirely ignoring any previous requests now. Fine? - // match self.block_requests.send_request(&target, request) { - // block_requests::SendRequestOutcome::Ok => { - // self.events.push_back(BehaviourOut::OpaqueRequestStarted { - // peer: target, - // protocol: self.block_requests.protocol_name().to_owned(), - // }); - // }, - // block_requests::SendRequestOutcome::Replaced { request_duration, .. } => { - // self.events.push_back(BehaviourOut::OpaqueRequestFinished { - // peer: target.clone(), - // protocol: self.block_requests.protocol_name().to_owned(), - // request_duration, - // }); - // self.events.push_back(BehaviourOut::OpaqueRequestStarted { - // peer: target, - // protocol: self.block_requests.protocol_name().to_owned(), - // }); - // } - // block_requests::SendRequestOutcome::NotConnected | - // block_requests::SendRequestOutcome::EncodeError(_) => {}, - // } }, CustomMessageOutcome::FinalityProofRequest { target, block_hash, request } => { let protobuf_rq = crate::schema::v1::finality::FinalityProofRequest { @@ -470,6 +417,7 @@ impl NetworkBehaviourEventProcess { + // TODO: Make sure to get these responses captured in metrics. if protocol == self.finality_request_protocol_name { let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) .unwrap(); diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 57b44c887136b..21b88ed935f22 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -1,3 +1,22 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Helper for handling (i.e. answering) block requests from a remote peer via the +//! [`crate::request_responses::RequestResponsesBehaviour`]. + use crate::request_responses::ProtocolConfig; use futures::channel::mpsc; use futures::stream::StreamExt; @@ -13,6 +32,7 @@ use prost::Message; use codec::{Encode, Decode}; use sp_runtime::{generic::BlockId, traits::{Header, One, Zero}}; +/// Generate the block protocol name from chain specific protocol identifier. pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { let mut s = String::new(); s.push_str("/"); @@ -21,6 +41,7 @@ pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { s } +/// Handler for incoming block requests from a remote peer. pub struct BlockRequestHandler { // TODO: Rename? chain: Arc>, @@ -28,6 +49,7 @@ pub struct BlockRequestHandler { } impl BlockRequestHandler { + /// Create a new [`BlockRequestHandler`]. pub fn new(protocol_id: ProtocolId, client: Arc>) -> (Self, ProtocolConfig) { let (tx, rx) = mpsc::channel(0); @@ -49,6 +71,7 @@ impl BlockRequestHandler { (handler, protocol_config) } + /// Run [`BlockRequestHandler`]. pub async fn run(mut self) { while let Some(crate::request_responses::IncomingRequest { peer: _, payload, pending_response }) = self.request_receiver.next().await { diff --git a/client/network/src/finality_request_handler.rs b/client/network/src/finality_request_handler.rs index c451e60e495de..65516e50f9bbc 100644 --- a/client/network/src/finality_request_handler.rs +++ b/client/network/src/finality_request_handler.rs @@ -1,3 +1,22 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Helper for handling (i.e. answering) finality requests from a remote peer via the +//! [`crate::request_responses::RequestResponsesBehaviour`]. + use crate::request_responses::ProtocolConfig; use futures::channel::mpsc; use futures::stream::StreamExt; @@ -9,6 +28,7 @@ use sp_runtime::{traits::Block as BlockT}; use crate::request_responses::IncomingRequest; use prost::Message; +/// Generate the finality proof protocol name from chain specific protocol identifier. pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { let mut s = String::new(); s.push_str("/"); @@ -17,12 +37,14 @@ pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { s } +/// Handler for incoming finality requests from a remote peer. pub struct FinalityRequestHandler { proof_provider: Arc>, request_receiver: mpsc::Receiver, } impl FinalityRequestHandler { + /// Create a new [`FinalityRequestHandler`]. pub fn new(protocol_id: ProtocolId, proof_provider: Arc>) -> (Self, ProtocolConfig){ let (tx, rx) = mpsc::channel(0); @@ -44,6 +66,7 @@ impl FinalityRequestHandler { (handler, protocol_config) } + /// Run [`FinalityRequestHandler`]. pub async fn run(mut self) { while let Some(crate::request_responses::IncomingRequest { peer, payload, pending_response }) = self.request_receiver.next().await { let req = crate::schema::v1::finality::FinalityProofRequest::decode(&payload[..]).unwrap(); diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 99eb847866f38..dc56fe6a05585 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -257,10 +257,10 @@ mod service; mod transport; mod utils; +pub mod block_request_handler; pub mod config; pub mod error; pub mod finality_request_handler; -pub mod block_request_handler; pub mod gossip; pub mod network_state; From 1a6b00eadf4893bf4fb5bf285c342910cc5695a9 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 2 Nov 2020 17:55:23 +0100 Subject: [PATCH 09/48] client/network/finality_request_handler: Log errors --- .../network/src/finality_request_handler.rs | 102 ++++++++++++------ 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/client/network/src/finality_request_handler.rs b/client/network/src/finality_request_handler.rs index 65516e50f9bbc..1c0b5ed0a4c68 100644 --- a/client/network/src/finality_request_handler.rs +++ b/client/network/src/finality_request_handler.rs @@ -17,16 +17,20 @@ //! Helper for handling (i.e. answering) finality requests from a remote peer via the //! [`crate::request_responses::RequestResponsesBehaviour`]. -use crate::request_responses::ProtocolConfig; -use futures::channel::mpsc; +use codec::Decode; +use crate::chain::FinalityProofProvider; +use crate::config::ProtocolId; +use crate::request_responses::{IncomingRequest, ProtocolConfig}; +use crate::schema::v1::finality::{FinalityProofRequest, FinalityProofResponse}; +use futures::channel::{mpsc, oneshot}; use futures::stream::StreamExt; -use std::sync::{Arc}; -pub use crate::{ - config::ProtocolId, - chain::{Client, FinalityProofProvider}}; -use sp_runtime::{traits::Block as BlockT}; -use crate::request_responses::IncomingRequest; +use log::debug; use prost::Message; +use sp_runtime::{traits::Block as BlockT}; +use std::sync::{Arc}; +use std::time::Duration; + +const LOG_TARGET: &str = "finality-request-handler"; /// Generate the finality proof protocol name from chain specific protocol identifier. pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { @@ -45,16 +49,19 @@ pub struct FinalityRequestHandler { impl FinalityRequestHandler { /// Create a new [`FinalityRequestHandler`]. - pub fn new(protocol_id: ProtocolId, proof_provider: Arc>) -> (Self, ProtocolConfig){ + pub fn new( + protocol_id: ProtocolId, + proof_provider: Arc>, + ) -> (Self, ProtocolConfig){ + // TODO: Likeley we want to allow more than 0 buffered requests. Rethink this value. let (tx, rx) = mpsc::channel(0); let protocol_config = ProtocolConfig { name: generate_protocol_name(protocol_id).into(), - // TODO: Change. - max_request_size: 4096, - // TODO: Change. - max_response_size: 4096, - request_timeout: std::time::Duration::from_secs(10), + max_request_size: 1024 * 1024, + max_response_size: 1024 * 1024, + // TODO: What is a sane value here? + request_timeout: Duration::from_secs(10), inbound_queue: Some(tx), }; @@ -66,25 +73,58 @@ impl FinalityRequestHandler { (handler, protocol_config) } + fn handle_request( + &self, + payload: Vec, + pending_response: oneshot::Sender> + ) -> Result<(), HandleRequestError> { + // Decode request. + let request = FinalityProofRequest::decode(&payload[..])?; + let block_hash = Decode::decode(&mut request.block_hash.as_ref())?; + + // Proof. + let finality_proof = self.proof_provider.prove_finality(block_hash, &request.request) + .map_err(|e| HandleRequestError::ProofFinality(block_hash, e))? + .unwrap_or_default(); + + // Encode response. + let resp = FinalityProofResponse { proof: finality_proof }; + let mut data = Vec::with_capacity(resp.encoded_len()); + resp.encode(&mut data) + .map_err(|e| HandleRequestError::EncodeProto(block_hash, e))?; + + // Respond. + pending_response.send(data) + .map_err(|_| HandleRequestError::SendResponse) + } + /// Run [`FinalityRequestHandler`]. pub async fn run(mut self) { - while let Some(crate::request_responses::IncomingRequest { peer, payload, pending_response }) = self.request_receiver.next().await { - let req = crate::schema::v1::finality::FinalityProofRequest::decode(&payload[..]).unwrap(); - - let block_hash = codec::Decode::decode(&mut req.block_hash.as_ref()).unwrap(); - - log::trace!(target: "sync", "Finality proof request from {} for {}", peer, block_hash); - - let finality_proof = self.proof_provider - .prove_finality(block_hash, &req.request) - .unwrap() - .unwrap_or_default(); - - let resp = crate::schema::v1::finality::FinalityProofResponse { proof: finality_proof }; - let mut data = Vec::with_capacity(resp.encoded_len()); - resp.encode(&mut data).unwrap(); - - pending_response.send(data).unwrap(); + while let Some(request) = self.request_receiver.next().await { + let IncomingRequest { peer, payload, pending_response } = request; + + match self.handle_request(payload, pending_response) { + Ok(()) => debug!(target: LOG_TARGET, "Handled finality proof request from {}.", peer), + Err(e) => debug!( + target: LOG_TARGET, + "Failed to handle finality proof request from {}: {}", + peer, e, + ), + } } } } + +#[derive(derive_more::Display, derive_more::From)] +enum HandleRequestError { + #[display(fmt = "Failed to decode request: {}.", _0)] + DecodeProto(prost::DecodeError), + #[display(fmt = "Failed to encode response for {}: {}.", _0, _1)] + EncodeProto(B::Hash, prost::EncodeError), + #[display(fmt = "Failed to decode block hash: {}.", _0)] + DecodeScale(codec::Error), + #[display(fmt = "Failed to proof finality for {}: {}.", _0, _1)] + ProofFinality(B::Hash, sp_blockchain::Error), + #[display(fmt = "Failed to send response.")] + SendResponse, +} From 06dbd9893295efb8a3e177c5fa67ef7dc79f007c Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 2 Nov 2020 19:13:29 +0100 Subject: [PATCH 10/48] client/network/block_request_handler: Log errors --- client/network/src/block_request_handler.rs | 281 ++++++++++---------- 1 file changed, 144 insertions(+), 137 deletions(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 21b88ed935f22..29377670b14b2 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -17,20 +17,25 @@ //! Helper for handling (i.e. answering) block requests from a remote peer via the //! [`crate::request_responses::RequestResponsesBehaviour`]. -use crate::request_responses::ProtocolConfig; -use futures::channel::mpsc; +use codec::{Encode, Decode}; +use crate::chain::Client; +use crate::config::ProtocolId; +use crate::protocol::{message::BlockAttributes}; +use crate::request_responses::{IncomingRequest, ProtocolConfig}; +use crate::schema::v1::block_request::FromBlock; +use crate::schema::v1::{BlockResponse, Direction}; +use futures::channel::{mpsc, oneshot}; use futures::stream::StreamExt; -use std::sync::{Arc}; -pub use crate::{ - protocol::{message::{self, BlockAttributes}}, - chain::{Client, FinalityProofProvider}, - config::ProtocolId, -}; -use sp_runtime::{traits::Block as BlockT}; -use crate::request_responses::IncomingRequest; +use log::debug; use prost::Message; -use codec::{Encode, Decode}; -use sp_runtime::{generic::BlockId, traits::{Header, One, Zero}}; +use sp_runtime::{traits::Block as BlockT, generic::BlockId, traits::{Header, One, Zero}}; +use std::cmp::min; +use std::sync::{Arc}; +use std::time::Duration; + +const LOG_TARGET: &str = "block-request-handler"; +const MAX_BLOCKS_IN_RESPONSE: usize = 128; +const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; /// Generate the block protocol name from chain specific protocol identifier. pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { @@ -43,158 +48,160 @@ pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { /// Handler for incoming block requests from a remote peer. pub struct BlockRequestHandler { - // TODO: Rename? - chain: Arc>, + client: Arc>, request_receiver: mpsc::Receiver, } impl BlockRequestHandler { /// Create a new [`BlockRequestHandler`]. pub fn new(protocol_id: ProtocolId, client: Arc>) -> (Self, ProtocolConfig) { - let (tx, rx) = mpsc::channel(0); + // TODO: Likeley we want to allow more than 0 buffered requests. Rethink this value. + let (tx, request_receiver) = mpsc::channel(0); let protocol_config = ProtocolConfig { name: generate_protocol_name(protocol_id).into(), - // TODO: Change. - max_request_size: 4096, - // TODO: Change. - max_response_size: 4096, - request_timeout: std::time::Duration::from_secs(10), + max_request_size: 1024 * 1024, + max_response_size: 16 * 1024 * 1024, + request_timeout: Duration::from_secs(40), inbound_queue: Some(tx), }; - let handler = Self { - chain: client, - request_receiver: rx, - }; - - (handler, protocol_config) + (Self { client, request_receiver }, protocol_config) } - /// Run [`BlockRequestHandler`]. - pub async fn run(mut self) { - while let Some(crate::request_responses::IncomingRequest { peer: _, payload, pending_response }) = self.request_receiver.next().await { - - let request = crate::schema::v1::BlockRequest::decode(&payload[..]).unwrap(); - + fn handle_request( + &self, + payload: Vec, + pending_response: oneshot::Sender> + ) -> Result<(), HandleRequestError> { + let request = crate::schema::v1::BlockRequest::decode(&payload[..])?; + + let from_block_id = match request.from_block.ok_or(HandleRequestError::MissingFromField)? { + FromBlock::Hash(ref h) => { + let h = Decode::decode(&mut h.as_ref())?; + BlockId::::Hash(h) + } + FromBlock::Number(ref n) => { + let n = Decode::decode(&mut n.as_ref())?; + BlockId::::Number(n) + } + }; + let max_blocks = if request.max_blocks == 0 { + MAX_BLOCKS_IN_RESPONSE + } else { + min(request.max_blocks as usize, MAX_BLOCKS_IN_RESPONSE) + }; - let from_block_id = match request.from_block { - Some(crate::schema::v1::block_request::FromBlock::Hash(ref h)) => { - let h = Decode::decode(&mut h.as_ref()).unwrap(); - BlockId::::Hash(h) - } - Some(crate::schema::v1::block_request::FromBlock::Number(ref n)) => { - let n = Decode::decode(&mut n.as_ref()).unwrap(); - BlockId::::Number(n) - } - None => { - panic!(); - // let msg = "missing `BlockRequest::from_block` field"; - // return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + let direction = Direction::from_i32(request.direction) + .ok_or(HandleRequestError::ParseDirection)?; + let attributes = BlockAttributes::from_be_u32(request.fields)?; + let get_header = attributes.contains(BlockAttributes::HEADER); + let get_body = attributes.contains(BlockAttributes::BODY); + let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); + + let mut blocks = Vec::new(); + let mut block_id = from_block_id; + + let mut total_size: usize = 0; + while let Some(header) = self.client.header(block_id).unwrap_or(None) { + let number = *header.number(); + let hash = header.hash(); + let parent_hash = *header.parent_hash(); + let justification = if get_justification { + self.client.justification(&BlockId::Hash(hash))? + } else { + None + }; + let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false); + + let body = if get_body { + match self.client.block_body(&BlockId::Hash(hash))? { + Some(mut extrinsics) => extrinsics.iter_mut() + .map(|extrinsic| extrinsic.encode()) + .collect(), + None => { + log::trace!(target: "sync", "Missing data for block request."); + break; + } } + } else { + Vec::new() }; - // let max_blocks = - // if request.max_blocks == 0 { - // self.config.max_block_data_response - // } else { - // min(request.max_blocks, self.config.max_block_data_response) - // }; - - let direction = - if request.direction == crate::schema::v1::Direction::Ascending as i32 { - crate::schema::v1::Direction::Ascending - } else if request.direction == crate::schema::v1::Direction::Descending as i32 { - crate::schema::v1::Direction::Descending - } else { - panic!(); - // let msg = format!("invalid `BlockRequest::direction` value: {}", request.direction); - // return Err(io::Error::new(io::ErrorKind::Other, msg).into()) - }; - - let attributes = BlockAttributes::from_be_u32(request.fields).unwrap(); - let get_header = attributes.contains(BlockAttributes::HEADER); - let get_body = attributes.contains(BlockAttributes::BODY); - let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); - - let mut blocks = Vec::new(); - let mut block_id = from_block_id; - // TODO: Still needed? - // let mut total_size = 0; - while let Some(header) = self.chain.header(block_id).unwrap_or(None) { - // if blocks.len() >= max_blocks as usize - // || (blocks.len() >= 1 && total_size > self.config.max_block_body_bytes) - // { - // break - // } - - let number = *header.number(); - let hash = header.hash(); - let parent_hash = *header.parent_hash(); - let justification = if get_justification { - self.chain.justification(&BlockId::Hash(hash)).unwrap() - } else { - None - }; - let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false); - - let body = if get_body { - match self.chain.block_body(&BlockId::Hash(hash)).unwrap() { - Some(mut extrinsics) => extrinsics.iter_mut() - .map(|extrinsic| extrinsic.encode()) - .collect(), - None => { - log::trace!(target: "sync", "Missing data for block request."); - break; - } - } + let block_data = crate::schema::v1::BlockData { + hash: hash.encode(), + header: if get_header { + header.encode() } else { Vec::new() - }; - - let block_data = crate::schema::v1::BlockData { - hash: hash.encode(), - header: if get_header { - header.encode() - } else { - Vec::new() - }, - body, - receipt: Vec::new(), - message_queue: Vec::new(), - justification: justification.unwrap_or_default(), - is_empty_justification, - }; - - // TODO: Still needed? - // total_size += block_data.body.len(); - blocks.push(block_data); - - match direction { - crate::schema::v1::Direction::Ascending => { - block_id = BlockId::Number(number + One::one()) - } - crate::schema::v1::Direction::Descending => { - if number.is_zero() { - break - } - block_id = BlockId::Hash(parent_hash) + }, + body, + receipt: Vec::new(), + message_queue: Vec::new(), + justification: justification.unwrap_or_default(), + is_empty_justification, + }; + + total_size += block_data.body.len(); + blocks.push(block_data); + + if blocks.len() >= max_blocks as usize || total_size > MAX_BODY_BYTES { + break + } + + match direction { + Direction::Ascending => { + block_id = BlockId::Number(number + One::one()) + } + Direction::Descending => { + if number.is_zero() { + break } + block_id = BlockId::Hash(parent_hash) } } + } - let res = crate::schema::v1::BlockResponse { blocks }; + let res = BlockResponse { blocks }; - let mut data = Vec::with_capacity(res.encoded_len()); - res.encode(&mut data).unwrap(); - // log::debug!( - // target: "sync", - // "Error encoding block response for peer {}: {}", - // peer, e - // ) + let mut data = Vec::with_capacity(res.encoded_len()); + res.encode(&mut data)?; - pending_response.send(data).unwrap(); + pending_response.send(data) + .map_err(|_| HandleRequestError::SendResponse) + } + + /// Run [`BlockRequestHandler`]. + pub async fn run(mut self) { + while let Some(request) = self.request_receiver.next().await { + let IncomingRequest { peer, payload, pending_response } = request; + + match self.handle_request(payload, pending_response) { + Ok(()) => debug!(target: LOG_TARGET, "Handled block request from {}.", peer), + Err(e) => debug!( + target: LOG_TARGET, + "Failed to handle block request from {}: {}", + peer, e, + ), + } } } } + +#[derive(derive_more::Display, derive_more::From)] +enum HandleRequestError { + #[display(fmt = "Failed to decode request: {}.", _0)] + DecodeProto(prost::DecodeError), + #[display(fmt = "Failed to encode response: {}.", _0)] + EncodeProto(prost::EncodeError), + #[display(fmt = "Failed to decode block hash: {}.", _0)] + DecodeScale(codec::Error), + #[display(fmt = "Missing `BlockRequest::from_block` field.")] + MissingFromField, + #[display(fmt = "Failed to parse BlockRequest::direction.")] + ParseDirection, + Client(sp_blockchain::Error), + #[display(fmt = "Failed to send response.")] + SendResponse, +} From 25e298fad54360bb41bf0644e750f015f0821b8d Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 3 Nov 2020 16:31:18 +0100 Subject: [PATCH 11/48] client/network: Format --- client/network/src/behaviour.rs | 4 +++- client/network/src/block_request_handler.rs | 5 +++-- client/network/src/light_client_handler.rs | 8 +++++--- client/network/src/service.rs | 3 +++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 0cf497cd7fe04..4d8489928dbc8 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -417,7 +417,9 @@ impl NetworkBehaviourEventProcess { - // TODO: Make sure to get these responses captured in metrics. + // TODO: Make sure to get these responses captured in metrics by emitting a + // `RequestFinished` event without a result. Then the NetworkWorker can capture this + // in a metric. if protocol == self.finality_request_protocol_name { let proof_response = crate::schema::v1::finality::FinalityProofResponse::decode(&result.unwrap()[..]) .unwrap(); diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 29377670b14b2..de13e5437f119 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -28,7 +28,8 @@ use futures::channel::{mpsc, oneshot}; use futures::stream::StreamExt; use log::debug; use prost::Message; -use sp_runtime::{traits::Block as BlockT, generic::BlockId, traits::{Header, One, Zero}}; +use sp_runtime::generic::BlockId; +use sp_runtime::traits::{Block as BlockT, Header, One, Zero}; use std::cmp::min; use std::sync::{Arc}; use std::time::Duration; @@ -76,7 +77,7 @@ impl BlockRequestHandler { ) -> Result<(), HandleRequestError> { let request = crate::schema::v1::BlockRequest::decode(&payload[..])?; - let from_block_id = match request.from_block.ok_or(HandleRequestError::MissingFromField)? { + let from_block_id = match request.from_block.ok_or(HandleRequestError::MissingFromField)? { FromBlock::Hash(ref h) => { let h = Decode::decode(&mut h.as_ref())?; BlockId::::Hash(h) diff --git a/client/network/src/light_client_handler.rs b/client/network/src/light_client_handler.rs index 94e1c1274cc6c..7ca0c145a96c4 100644 --- a/client/network/src/light_client_handler.rs +++ b/client/network/src/light_client_handler.rs @@ -1062,11 +1062,13 @@ fn retries(request: &Request) -> usize { fn serialize_request(request: &Request) -> Result, prost::EncodeError> { let request = match request { Request::Body { request, .. } => { - let rq = crate::schema::v1::BlockRequest { + let rq = schema::v1::BlockRequest { fields: BlockAttributes::BODY.to_be_u32(), - from_block: Some(crate::schema::v1::block_request::FromBlock::Hash(request.header.hash().encode())), + from_block: Some(schema::v1::block_request::FromBlock::Hash( + request.header.hash().encode(), + )), to_block: Default::default(), - direction:crate::schema::v1::Direction::Ascending as i32, + direction: schema::v1::Direction::Ascending as i32, max_blocks: 1, }; diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 7021877b5854e..704d35317ba52 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1412,6 +1412,9 @@ impl Future for NetworkWorker { } } }, + // TODO: Retrieve started and Protocol from `RequestFinished` and make result + // optional. That way we can also return `RequestFinished` for internal requests + // (block, finality and light-client) thus tracking metrics for them as well. Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { request_id, result })) => { if let Some((send_back, started, protocol)) = this.pending_requests.remove(&request_id) { if let Some(metrics) = this.metrics.as_ref() { From 2ed6dd16a33258fe4d17062636fa174ef860a899 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 3 Nov 2020 16:41:45 +0100 Subject: [PATCH 12/48] client/network: Handle block request failure --- client/network/src/behaviour.rs | 1 + client/network/src/protocol.rs | 49 ++++++--------------------------- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 4d8489928dbc8..79dcd706b384c 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -434,6 +434,7 @@ impl NetworkBehaviourEventProcess Protocol { } } - // TODO: Bring back? - // /// Must be called in response to a [`CustomMessageOutcome::BlockRequest`] if it has failed. - // pub fn on_block_request_failed( - // &mut self, - // peer: &PeerId, - // ) { - // self.peerset_handle.report_peer(peer.clone(), rep::TIMEOUT); - // self.behaviour.disconnect_peer(peer); - // } + /// Must be called in response to a [`CustomMessageOutcome::BlockRequest`] if it has failed. + pub fn on_block_request_failed( + &mut self, + peer: &PeerId, + ) { + // TODO: This can not only happen due to timeouts. Should the `rep::TIMEOUT` be used? + self.peerset_handle.report_peer(peer.clone(), rep::TIMEOUT); + self.behaviour.disconnect_peer(peer); + } /// Perform time based maintenance. /// /// > **Note**: This method normally doesn't have to be called except for testing purposes. pub fn tick(&mut self) { - self.maintain_peers(); self.report_metrics() } - fn maintain_peers(&mut self) { - let tick = Instant::now(); - let mut aborting = Vec::new(); - { - for (who, peer) in self.context_data.peers.iter() { - if peer.block_request.as_ref().map_or(false, |(t, _, _request_id)| (tick - *t).as_secs() > REQUEST_TIMEOUT_SEC) { - log!( - target: "sync", - if self.important_peers.contains(who) { Level::Warn } else { Level::Trace }, - "Request timeout {}", who - ); - aborting.push(who.clone()); - } else if peer.obsolete_requests.values().any(|t| (tick - *t).as_secs() > REQUEST_TIMEOUT_SEC) { - log!( - target: "sync", - if self.important_peers.contains(who) { Level::Warn } else { Level::Trace }, - "Obsolete timeout {}", who - ); - aborting.push(who.clone()); - } - } - } - - for p in aborting { - self.behaviour.disconnect_peer(&p); - self.peerset_handle.report_peer(p, rep::TIMEOUT); - } - } - /// Called on the first connection between two peers, after their exchange of handshake. fn on_peer_connected( &mut self, From aacb507ee9a790586f6fc2d2b7d4e934c7fc0391 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 3 Nov 2020 16:59:41 +0100 Subject: [PATCH 13/48] protocols/network: Fix tests --- client/network/src/gossip/tests.rs | 1 - client/network/src/request_responses.rs | 4 ++++ client/network/src/service/tests.rs | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index 0f01ed81bffcb..ab0800f4ba01c 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -96,7 +96,6 @@ fn build_test_full_node(config: config::NetworkConfiguration) executor: None, network_config: config, chain: client.clone(), - finality_proof_provider: None, finality_proof_request_builder: None, on_demand: None, transaction_pool: Arc::new(crate::config::EmptyTransactionPool), diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index f664714d3a478..494e6a536d8ad 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -762,6 +762,8 @@ mod tests { sent_request_id = Some(id); } SwarmEvent::Behaviour(super::Event::RequestFinished { + peer: _, + protocol: _, request_id, result, }) => { @@ -865,6 +867,8 @@ mod tests { sent_request_id = Some(id); } SwarmEvent::Behaviour(super::Event::RequestFinished { + peer: _, + protocol: _, request_id, result, }) => { diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 4b6f9dd156482..7b981c7345fe3 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -97,7 +97,6 @@ fn build_test_full_node(config: config::NetworkConfiguration) executor: None, network_config: config, chain: client.clone(), - finality_proof_provider: None, finality_proof_request_builder: None, on_demand: None, transaction_pool: Arc::new(crate::config::EmptyTransactionPool), From 1a72e0c85923e25347a5cdb804101dd1f0001b15 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 3 Nov 2020 17:18:30 +0100 Subject: [PATCH 14/48] client/network/src/behaviour: Handle request sending errors --- client/network/src/behaviour.rs | 64 +++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 79dcd706b384c..e0cf9ee61e37e 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -18,7 +18,7 @@ use crate::{ config::{ProtocolId, Role}, light_client_handler, peer_info, request_responses, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, protocol::{message::{self, Roles}, CustomMessageOutcome, NotificationsSink, Protocol}, - ObservedRole, DhtEvent, ExHashT, + ObservedRole, DhtEvent, ExHashT, schema, }; use bytes::Bytes; @@ -321,20 +321,16 @@ Behaviour { CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) => self.events.push_back(BehaviourOut::FinalityProofImport(origin, hash, nb, proof)), CustomMessageOutcome::BlockRequest { target, request } => { - - let protobuf_req = crate::schema::v1::BlockRequest { + let protobuf_req = schema::v1::BlockRequest { fields: request.fields.to_be_u32(), from_block: match request.from { message::FromBlock::Hash(h) => - Some(crate::schema::v1::block_request::FromBlock::Hash(h.encode())), + Some(schema::v1::block_request::FromBlock::Hash(h.encode())), message::FromBlock::Number(n) => - Some(crate::schema::v1::block_request::FromBlock::Number(n.encode())), + Some(schema::v1::block_request::FromBlock::Number(n.encode())), }, to_block: request.to.map(|h| h.encode()).unwrap_or_default(), - direction: match request.direction { - message::Direction::Ascending => crate::schema::v1::Direction::Ascending as i32, - message::Direction::Descending => crate::schema::v1::Direction::Descending as i32, - }, + direction: request.direction as i32, max_blocks: request.max.unwrap_or(0), }; @@ -343,31 +339,55 @@ Behaviour { log::warn!( target: "sync", "Failed to encode block request {:?}: {:?}", - protobuf_req, - err + protobuf_req, err ); - panic!(); - // return SendRequestOutcome::EncodeError(err); + + // TODO: Should we notify protocol.rs or sync.rs? + return } - let request_id = self.request_responses.send_request(&target, &self.block_request_protocol_name, buf).unwrap(); - // TODO: differentiate between the two ids. - self.substrate.on_block_request_started(target, request.id, request_id); + match self.request_responses.send_request( + &target, &self.block_request_protocol_name, buf, + ) { + Ok(request_id) => self.substrate.on_block_request_started( + target, request.id, request_id, + ), + // TODO: Should we notify protocol.rs or sync.rs? + Err(err) => log::warn!( + target: "sync", + "Failed to send block request {:?}: {:?}", + protobuf_req, err + ), + }; }, CustomMessageOutcome::FinalityProofRequest { target, block_hash, request } => { - let protobuf_rq = crate::schema::v1::finality::FinalityProofRequest { + let protobuf_req = crate::schema::v1::finality::FinalityProofRequest { block_hash: block_hash.encode(), request, }; - let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); - if let Err(err) = protobuf_rq.encode(&mut buf) { - log::warn!("failed to encode finality proof request {:?}: {:?}", protobuf_rq, err); + let mut buf = Vec::with_capacity(protobuf_req.encoded_len()); + if let Err(err) = protobuf_req.encode(&mut buf) { + log::warn!( + target: "sync", + "Failed to encode finality proof request {:?}: {:?}", + protobuf_req, err, + ); return; } - let request_id = self.request_responses.send_request(&target, &self.finality_request_protocol_name, buf).unwrap(); - self.substrate.on_finality_proof_request_started(target, block_hash, request_id); + match self.request_responses.send_request( + &target, &self.finality_request_protocol_name, buf, + ) { + Ok(request_id) => self.substrate.on_finality_proof_request_started( + target, block_hash, request_id, + ), + Err(err) => log::warn!( + target: "sync", + "Failed to send block request {:?}: {:?}", + protobuf_req, err + ), + } }, CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles, notifications_sink } => { let role = reported_roles_to_observed_role(&self.role, &remote, roles); From 1385f5c45b437a266636a54d6ed53a6329b2d06e Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 3 Nov 2020 18:15:11 +0100 Subject: [PATCH 15/48] client/network: Move response handling into custom method --- client/network/src/behaviour.rs | 82 +++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index e0cf9ee61e37e..841ba0e6bba7e 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -68,8 +68,8 @@ pub struct Behaviour { #[behaviour(ignore)] role: Role, - // TODO: Does this really belong here? Why require matching on these in the first place? Why not - // offer a oneshot when sending a request? + // TODO: Does this really belong here? Why require matching on these when reponses come in? Why + // not offer a oneshot when sending a request? #[behaviour(ignore)] block_request_protocol_name: String, #[behaviour(ignore)] @@ -285,6 +285,48 @@ impl Behaviour { pub fn light_client_request(&mut self, r: light_client_handler::Request) -> Result<(), light_client_handler::Error> { self.light_client_handler.request(r) } + + fn on_block_response( + &mut self, + peer: &PeerId, + // TODO: shorten + request_id: libp2p::request_response::RequestId, + result: Result, request_responses::RequestFailure>, + ) -> Result, OnBlockResponseError>{ + let protobuf_response = schema::v1::BlockResponse::decode(&result?[..])?; + let ev = self.substrate.on_block_response(peer.clone(), request_id, protobuf_response); + Ok(ev) + } + + fn on_finality_response( + &mut self, + peer: &PeerId, + // TODO: shorten + request_id: libp2p::request_response::RequestId, + result: Result, request_responses::RequestFailure>, + ) -> Result, OnFinalityResponseError>{ + let protobuf_response = schema::v1::finality::FinalityProofResponse::decode( + &result?[..], + )?; + let ev = self.substrate.on_finality_proof_response( + peer.clone(), + request_id, + protobuf_response.proof, + ); + Ok(ev) + } +} + +#[derive(derive_more::Display, derive_more::From)] +enum OnBlockResponseError { + Request(request_responses::RequestFailure), + Decode(prost::DecodeError), +} + +#[derive(derive_more::Display, derive_more::From)] +enum OnFinalityResponseError { + Request(request_responses::RequestFailure), + Decode(prost::DecodeError), } fn reported_roles_to_observed_role(local_role: &Role, remote: &PeerId, roles: Roles) -> ObservedRole { @@ -441,27 +483,31 @@ impl NetworkBehaviourEventProcess self.inject_event(ev), + Err(err) => { + debug!( + target: "sync", + "Handling finality response from {} failed with: {}", + peer, err, + ); + + // TODO: Should protocol.rs be notified of the failure? + } + } } else if protocol == self.block_request_protocol_name { - let resp = match result { - Ok(resp) => resp, - Err(e) => { + match self.on_block_response(&peer, request_id, result) { + Ok(ev) => self.inject_event(ev), + Err(err) => { debug!( - target: "sub-libp2p", - "Block request to {:?} failed with: {:?}", - peer, e, + target: "sync", + "Handling block response from {} failed with: {}", + peer, err, ); + self.substrate.on_block_request_failed(&peer); - return } - }; - let protobuf_response = crate::schema::v1::BlockResponse::decode(&resp[..]).unwrap(); - - let ev = self.substrate.on_block_response(peer, request_id, protobuf_response); - self.inject_event(ev); + } } else { self.events.push_back(BehaviourOut::RequestFinished { request_id, From 8fdf1bde52170c2d8e4e737f499db54ff864596a Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 3 Nov 2020 18:55:15 +0100 Subject: [PATCH 16/48] client/network/protocol: Handle block response errors --- client/network/src/protocol.rs | 68 +++++++++------------------------- 1 file changed, 18 insertions(+), 50 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index aba559b2e74ab..ac7d2c2ae124e 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -109,8 +109,6 @@ mod rep { pub const BAD_TRANSACTION: Rep = Rep::new(-(1 << 12), "Bad transaction"); /// We received a message that failed to decode. pub const BAD_MESSAGE: Rep = Rep::new(-(1 << 12), "Bad message"); - /// We received an unexpected response. - pub const UNEXPECTED_RESPONSE: Rep = Rep::new_fatal("Unexpected response packet"); /// We received an unexpected transaction packet. pub const UNEXPECTED_TRANSACTIONS: Rep = Rep::new_fatal("Unexpected transactions packet"); /// Peer has different genesis. @@ -711,30 +709,18 @@ impl Protocol { let peer = match self.context_data.peers.get_mut(&peer_id) { Some(peer) => peer, None => { - // TODO: Bring back. - // trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); + debug!(target: "sync", "Ignoring obsolete block response packet from {}", peer_id); return CustomMessageOutcome::None; } }; - let block_request = match peer.block_request.take() { - Some(req) => req, - None => { - trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer_id); - self.behaviour.disconnect_peer(&peer_id); - self.peerset_handle.report_peer(peer_id, rep::UNEXPECTED_RESPONSE); - return CustomMessageOutcome::None; - } + let block_request = match &peer.block_request { + Some((_, _req, Some(id))) if *id == request_id => peer.block_request + .take() + .expect("id to be Some").1, + Some(_) | None => return CustomMessageOutcome::None, }; - if block_request.2 != Some(request_id) { - // TODO: Properly handle this. - panic!("request id didn't match or was none"); - // return CustomMessageOutcome::None; - } - - let block_request = block_request.1; - let blocks = response.blocks.into_iter().map(|block_data| { Ok(message::BlockData:: { hash: Decode::decode(&mut block_data.hash.as_ref())?, @@ -768,39 +754,22 @@ impl Protocol { None }, }) - }).collect::, codec::Error>>().unwrap(); + }).collect::, codec::Error>>(); + + let blocks = match blocks { + Ok(blocks) => blocks, + Err(err) => { + debug!(target: "sync", "Failed to decode block response from {}: {}", peer_id, err); + // TODO: Reduce reputation of peer? + return CustomMessageOutcome::None; + } + }; let block_response = message::BlockResponse:: { id: block_request.id, blocks, }; - // let request = if let Some(ref mut p) = self.context_data.peers.get_mut(&peer) { - // if p.obsolete_requests.remove(&response.id).is_some() { - // trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); - // return CustomMessageOutcome::None; - // } - // // Clear the request. If the response is invalid peer will be disconnected anyway. - // match p.block_request.take() { - // Some((_, request)) if request.id == response.id => request, - // Some(_) => { - // trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); - // return CustomMessageOutcome::None; - // } - // None => { - // trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); - // self.behaviour.disconnect_peer(&peer); - // self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); - // return CustomMessageOutcome::None; - // } - // } - // } else { - // trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); - // self.behaviour.disconnect_peer(&peer); - // self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); - // return CustomMessageOutcome::None; - // }; - let blocks_range = || match ( block_response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), block_response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), @@ -1421,9 +1390,8 @@ impl Protocol { }; let req = match &peer.finality_request { - Some((_req, Some(id))) if *id != request_id => peer.finality_request.take().unwrap().0, - Some(_) => return CustomMessageOutcome::None, - None => return CustomMessageOutcome::None, + Some((_req, Some(id))) if *id == request_id => peer.finality_request.take().unwrap().0, + Some(_) | None => return CustomMessageOutcome::None, }; let resp = message::FinalityProofResponse:: { From 7460a21efa7f40ddd056624fa3fa7ca15fbf93cf Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 4 Nov 2020 11:16:57 +0100 Subject: [PATCH 17/48] client/network/protocol: Remove tracking of obsolete requests --- client/network/src/protocol.rs | 21 +-------------------- client/network/src/protocol/message.rs | 1 + 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index ac7d2c2ae124e..dd482c1f73a00 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -124,7 +124,6 @@ mod rep { } struct Metrics { - obsolete_requests: Gauge, peers: Gauge, queued_blocks: Gauge, fork_targets: Gauge, @@ -136,10 +135,6 @@ struct Metrics { impl Metrics { fn register(r: &Registry) -> Result { Ok(Metrics { - obsolete_requests: { - let g = Gauge::new("sync_obsolete_requests", "Number of obsolete requests")?; - register(g, r)? - }, peers: { let g = Gauge::new("sync_peers", "Number of peers we sync with")?; register(g, r)? @@ -258,9 +253,6 @@ struct Peer { block_request: Option<(Instant, message::BlockRequest, Option)>, // TODO: Document finality_request: Option<(message::FinalityProofRequest, Option)>, - /// Requests we are no longer interested in. - // TODO: Do we still need this at all? - obsolete_requests: HashMap, /// Holds a set of transactions known to this peer. known_transactions: LruHashSet, /// Holds a set of blocks known to this peer. @@ -923,7 +915,6 @@ impl Protocol { known_blocks: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_BLOCKS) .expect("Constant is nonzero")), next_request_id: 0, - obsolete_requests: HashMap::new(), }; self.context_data.peers.insert(who.clone(), peer); @@ -1465,13 +1456,6 @@ impl Protocol { use std::convert::TryInto; if let Some(metrics) = &self.metrics { - let mut obsolete_requests: u64 = 0; - for peer in self.context_data.peers.values() { - let n = peer.obsolete_requests.len().try_into().unwrap_or(std::u64::MAX); - obsolete_requests = obsolete_requests.saturating_add(n); - } - metrics.obsolete_requests.set(obsolete_requests); - let n = self.context_data.peers.len().try_into().unwrap_or(std::u64::MAX); metrics.peers.set(n); @@ -1543,6 +1527,7 @@ pub enum CustomMessageOutcome { None, } +// TODO: Should this be inlined? fn update_peer_request( peers: &mut HashMap>, who: &PeerId, @@ -1551,10 +1536,6 @@ fn update_peer_request( if let Some(ref mut peer) = peers.get_mut(who) { request.id = peer.next_request_id; peer.next_request_id += 1; - if let Some((timestamp, request, _request_id)) = peer.block_request.take() { - trace!(target: "sync", "Request {} for {} is now obsolete.", request.id, who); - peer.obsolete_requests.insert(request.id, timestamp); - } peer.block_request = Some((Instant::now(), request.clone(), None)); } } diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 1cd78c0ed1dda..5776777f53381 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -392,6 +392,7 @@ pub mod generic { /// Request block data from a peer. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct BlockRequest { + // TODO: Is this used anywhere? /// Unique request id. pub id: RequestId, /// Bits of block data to request. From f1e769133afe6d2ec60b35e8098ab2c05b970291 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 4 Nov 2020 13:32:15 +0100 Subject: [PATCH 18/48] client/network/protocol: Remove block request start time tracking This will be handled generically via request-responses. --- client/network/src/protocol.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index dd482c1f73a00..07802dad680b0 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -55,7 +55,6 @@ use std::sync::Arc; use std::fmt::Write; use std::{io, iter, num::NonZeroUsize, pin::Pin, task::Poll, time}; use log::{log, Level, trace, debug, warn, error}; -use wasm_timer::Instant; mod generic_proto; @@ -250,7 +249,7 @@ struct Peer { info: PeerInfo, /// Current block request, if any. // TODO: Is Instant stil needed? - block_request: Option<(Instant, message::BlockRequest, Option)>, + block_request: Option<(message::BlockRequest, Option)>, // TODO: Document finality_request: Option<(message::FinalityProofRequest, Option)>, /// Holds a set of transactions known to this peer. @@ -707,9 +706,9 @@ impl Protocol { }; let block_request = match &peer.block_request { - Some((_, _req, Some(id))) if *id == request_id => peer.block_request + Some((_req, Some(id))) if *id == request_id => peer.block_request .take() - .expect("id to be Some").1, + .expect("id to be Some").0, Some(_) | None => return CustomMessageOutcome::None, }; @@ -1428,12 +1427,12 @@ impl Protocol { ) { let peer = self.context_data.peers.get_mut(&who).unwrap(); - if peer.block_request.as_mut().unwrap().1.id != request_id { + if peer.block_request.as_mut().unwrap().0.id != request_id { // TODO: Handle. likely fine. Some other request replaced the current one. panic!(); } - peer.block_request.as_mut().unwrap().2 = Some(request_response_request_id); + peer.block_request.as_mut().unwrap().1 = Some(request_response_request_id); } fn format_stats(&self) -> String { @@ -1536,7 +1535,7 @@ fn update_peer_request( if let Some(ref mut peer) = peers.get_mut(who) { request.id = peer.next_request_id; peer.next_request_id += 1; - peer.block_request = Some((Instant::now(), request.clone(), None)); + peer.block_request = Some((request.clone(), None)); } } From 401a7f836a75e7c65a97dd721b2bb647e623241f Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 4 Nov 2020 13:46:55 +0100 Subject: [PATCH 19/48] client/network/protocol: Refactor on_*_request_started --- client/network/src/protocol.rs | 49 +++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 07802dad680b0..a6b17dc920c12 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1403,36 +1403,61 @@ impl Protocol { } } - // TODO: Comment - // Informs sync of the request id. + /// Notify [`Protocol`] that the finality proof request for the given block hash has been + /// started and that the response for this request will be identified with the given request id. + // + // TODO: This could be simplified by having `RequestResponses::send_request` take a + // `oneshot::Sender` to send the response into. pub fn on_finality_proof_request_started( &mut self, who: PeerId, block_hash: B::Hash, request_id: libp2p::request_response::RequestId, ) { - if let Some(Peer { finality_request: Some(req), ..}) = self.context_data.peers.get_mut(&who) { - if req.0.block == block_hash { - debug_assert!(req.1.is_none()); - req.1 = Some(request_id); + let peer = match self.context_data.peers.get_mut(&who) { + Some(peer) => peer, + None => return, + }; + + if let Peer { finality_request: Some((req, id)), ..} = peer { + if req.block == block_hash { + debug_assert!( + id.is_none(), + "Expect `on_finality_proof_request_started` not to be called twice for the \ + same request.", + ); + *id = Some(request_id); } } } + /// Notify [`Protocol`] that the block request with the block request id `request_id` has been + /// started and that the response for this request will be identified with the given request + /// responses request id. + // + // TODO: This could be simplified by having `RequestResponses::send_request` take a + // `oneshot::Sender` to send the response into. pub fn on_block_request_started( &mut self, who: PeerId, request_id: u64, request_response_request_id: libp2p::request_response::RequestId, ) { - let peer = self.context_data.peers.get_mut(&who).unwrap(); + let peer = match self.context_data.peers.get_mut(&who) { + Some(peer) => peer, + None => return, + }; - if peer.block_request.as_mut().unwrap().0.id != request_id { - // TODO: Handle. likely fine. Some other request replaced the current one. - panic!(); + if let Peer { block_request: Some((req, id)), ..} = peer { + if req.id == request_id { + debug_assert!( + id.is_none(), + "Expect `on_block_request_started` not to be called twice for the \ + same request.", + ); + *id = Some(request_response_request_id); + } } - - peer.block_request.as_mut().unwrap().1 = Some(request_response_request_id); } fn format_stats(&self) -> String { From b375b8444191437eae5b6da0b1650f99e9e2d3d6 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 4 Nov 2020 17:09:03 +0100 Subject: [PATCH 20/48] client/network: Pass protocol config instead of protocol name --- Cargo.lock | 1 + client/network/src/behaviour.rs | 12 +++-- client/network/src/block_request_handler.rs | 22 +++++--- client/network/src/config.rs | 4 ++ .../network/src/finality_request_handler.rs | 24 +++++---- client/network/src/gossip/tests.rs | 25 ++++++++- client/network/src/service.rs | 4 +- client/network/src/service/tests.rs | 25 ++++++++- client/network/test/Cargo.toml | 1 + client/network/test/src/lib.rs | 51 ++++++++++++++++--- client/service/src/builder.rs | 33 +++++++----- 11 files changed, 155 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f3e68a844e30..33ee519ba9b8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7182,6 +7182,7 @@ dependencies = [ name = "sc-network-test" version = "0.8.0" dependencies = [ + "async-std", "futures 0.3.5", "futures-timer 3.0.2", "libp2p", diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 841ba0e6bba7e..c385ded4f5185 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -165,12 +165,18 @@ impl Behaviour { local_public_key: PublicKey, light_client_handler: light_client_handler::LightClientHandler, disco_config: DiscoveryConfig, - request_response_protocols: Vec, + // TODO: Document that these are all others apart from block and finality. + mut request_response_protocols: Vec, // TODO: Not so fancy, see definition. - block_request_protocol_name: String, - finality_request_protocol_name: String, + block_request_protocol_config: request_responses::ProtocolConfig, + finality_request_protocol_config: request_responses::ProtocolConfig, ) -> Result { + let block_request_protocol_name = block_request_protocol_config.name.to_string(); + let finality_request_protocol_name = finality_request_protocol_config.name.to_string(); + request_response_protocols.push(block_request_protocol_config); + request_response_protocols.push(finality_request_protocol_config); + Ok(Behaviour { substrate, peer_info: peer_info::PeerInfoBehaviour::new(user_agent, local_public_key), diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index de13e5437f119..1398d835335bb 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -38,8 +38,19 @@ const LOG_TARGET: &str = "block-request-handler"; const MAX_BLOCKS_IN_RESPONSE: usize = 128; const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; +// TODO: Document that this is for clients not reponding only. +pub fn generate_protocol_config(protocol_id: ProtocolId) -> ProtocolConfig { + ProtocolConfig { + name: generate_protocol_name(protocol_id).into(), + max_request_size: 1024 * 1024, + max_response_size: 16 * 1024 * 1024, + request_timeout: Duration::from_secs(40), + inbound_queue: None, + } +} + /// Generate the block protocol name from chain specific protocol identifier. -pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { +fn generate_protocol_name(protocol_id: ProtocolId) -> String { let mut s = String::new(); s.push_str("/"); s.push_str(protocol_id.as_ref()); @@ -59,13 +70,8 @@ impl BlockRequestHandler { // TODO: Likeley we want to allow more than 0 buffered requests. Rethink this value. let (tx, request_receiver) = mpsc::channel(0); - let protocol_config = ProtocolConfig { - name: generate_protocol_name(protocol_id).into(), - max_request_size: 1024 * 1024, - max_response_size: 16 * 1024 * 1024, - request_timeout: Duration::from_secs(40), - inbound_queue: Some(tx), - }; + let mut protocol_config = generate_protocol_config(protocol_id); + protocol_config.inbound_queue = Some(tx); (Self { client, request_receiver }, protocol_config) } diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 03e4a799cf6d0..9db0a84d3792c 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -100,6 +100,10 @@ pub struct Params { /// Registry for recording prometheus metrics to. pub metrics_registry: Option, + + // TODO: Dcoument that you can only get these by creating a Protocol config or creating a whole handler. + pub block_request_protocol_config: RequestResponseConfig, + pub finality_request_protocol_config: RequestResponseConfig, } /// Role of the local node. diff --git a/client/network/src/finality_request_handler.rs b/client/network/src/finality_request_handler.rs index 1c0b5ed0a4c68..0819bd7ae0dd3 100644 --- a/client/network/src/finality_request_handler.rs +++ b/client/network/src/finality_request_handler.rs @@ -32,8 +32,20 @@ use std::time::Duration; const LOG_TARGET: &str = "finality-request-handler"; +// TODO: Document that this is for clients not reponding only. +pub fn generate_protocol_config(protocol_id: ProtocolId) -> ProtocolConfig { + ProtocolConfig { + name: generate_protocol_name(protocol_id).into(), + max_request_size: 1024 * 1024, + max_response_size: 1024 * 1024, + // TODO: What is a sane value here? + request_timeout: Duration::from_secs(10), + inbound_queue: None, + } +} + /// Generate the finality proof protocol name from chain specific protocol identifier. -pub fn generate_protocol_name(protocol_id: ProtocolId) -> String { +fn generate_protocol_name(protocol_id: ProtocolId) -> String { let mut s = String::new(); s.push_str("/"); s.push_str(protocol_id.as_ref()); @@ -56,14 +68,8 @@ impl FinalityRequestHandler { // TODO: Likeley we want to allow more than 0 buffered requests. Rethink this value. let (tx, rx) = mpsc::channel(0); - let protocol_config = ProtocolConfig { - name: generate_protocol_name(protocol_id).into(), - max_request_size: 1024 * 1024, - max_response_size: 1024 * 1024, - // TODO: What is a sane value here? - request_timeout: Duration::from_secs(10), - inbound_queue: Some(tx), - }; + let mut protocol_config = generate_protocol_config(protocol_id); + protocol_config.inbound_queue = Some(tx); let handler = FinalityRequestHandler { proof_provider, diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index ab0800f4ba01c..8fca328456851 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -33,7 +33,7 @@ type TestNetworkService = NetworkService< /// /// > **Note**: We return the events stream in order to not possibly lose events between the /// > construction of the service and the moment the events stream is grabbed. -fn build_test_full_node(config: config::NetworkConfiguration) +fn build_test_full_node(mut network_config: config::NetworkConfiguration) -> (Arc, impl Stream) { let client = Arc::new( @@ -91,10 +91,29 @@ fn build_test_full_node(config: config::NetworkConfiguration) None, )); + let protocol_id = config::ProtocolId::from("/test-protocol-name"); + + // Add block request handler. + let block_request_protocol_name = { + let (handler, protocol_config) = crate::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + let name = protocol_config.name.to_string(); + network_config.request_response_protocols.push(protocol_config); + async_std::task::spawn(handler.run().boxed()); + name + }; + + // Add finality protocol. + let finality_request_protocol_name = { + let protocol_config = crate::finality_request_handler::generate_protocol_config(protocol_id.clone()); + let name = protocol_config.name.to_string(); + network_config.request_response_protocols.push(protocol_config); + name + }; + let worker = NetworkWorker::new(config::Params { role: config::Role::Full, executor: None, - network_config: config, + network_config, chain: client.clone(), finality_proof_request_builder: None, on_demand: None, @@ -105,6 +124,8 @@ fn build_test_full_node(config: config::NetworkConfiguration) sp_consensus::block_validation::DefaultBlockAnnounceValidator, ), metrics_registry: None, + block_request_protocol_name, + finality_request_protocol_name, }) .unwrap(); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 704d35317ba52..2e2a83ce767b4 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -309,8 +309,8 @@ impl NetworkWorker { light_client_handler, discovery_config, params.network_config.request_response_protocols, - crate::block_request_handler::generate_protocol_name(params.protocol_id.clone()), - crate::finality_request_handler::generate_protocol_name(params.protocol_id.clone()), + params.block_request_protocol_config, + params.finality_request_protocol_config, ); match result { diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 7b981c7345fe3..7ee293861ef2a 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -92,20 +92,41 @@ fn build_test_full_node(config: config::NetworkConfiguration) None, )); + let protocol_id = config::ProtocolId::from("/test-protocol-name"); + + // Add block request handler. + let block_request_protocol_name = { + let (handler, protocol_config) = crate::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + let name = protocol_config.name.to_string(); + network_config.request_response_protocols.push(protocol_config); + async_std::task::spawn(handler.run().boxed()); + name + }; + + // Add finality protocol. + let finality_request_protocol_name = { + let protocol_config = crate::finality_request_handler::generate_protocol_config(protocol_id.clone()); + let name = protocol_config.name.to_string(); + network_config.request_response_protocols.push(protocol_config); + name + }; + let worker = NetworkWorker::new(config::Params { role: config::Role::Full, executor: None, - network_config: config, + network_config, chain: client.clone(), finality_proof_request_builder: None, on_demand: None, transaction_pool: Arc::new(crate::config::EmptyTransactionPool), - protocol_id: config::ProtocolId::from("/test-protocol-name"), + protocol_id, import_queue, block_announce_validator: Box::new( sp_consensus::block_validation::DefaultBlockAnnounceValidator, ), metrics_registry: None, + block_request_protocol_name, + finality_request_protocol_name, }) .unwrap(); diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index a8bf98a75ed61..f9db08379347e 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -13,6 +13,7 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +async-std = "1.6.5" sc-network = { version = "0.8.0", path = "../" } log = "0.4.8" parking_lot = "0.10.0" diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 587feebe55c14..2a60f9433f4bf 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -30,6 +30,8 @@ use std::{ use libp2p::build_multiaddr; use log::trace; use sc_network::config::FinalityProofProvider; +use sc_network::block_request_handler::BlockRequestHandler; +use sc_network::finality_request_handler::FinalityRequestHandler; use sp_blockchain::{ HeaderBackend, Result as ClientResult, well_known_cache_keys::{self, Id as CacheKeyId}, @@ -50,6 +52,7 @@ use sp_consensus::block_import::{BlockImport, ImportResult}; use sp_consensus::Error as ConsensusError; use sp_consensus::{BlockOrigin, ForkChoiceStrategy, BlockImportParams, BlockCheckParams, JustificationImport}; use futures::prelude::*; +use futures::future::BoxFuture; use sc_network::{NetworkWorker, NetworkService, config::ProtocolId}; use sc_network::config::{NetworkConfiguration, TransportConfig, BoxFinalityProofRequestBuilder}; use libp2p::PeerId; @@ -670,22 +673,42 @@ pub trait TestNetFactory: Sized { network_config.allow_non_globals_in_dht = true; network_config.notifications_protocols = config.notifications_protocols; + let protocol_id = ProtocolId::from("test-protocol-name"); + + // Add block request handler. + let block_request_protocol_config = { + let (handler, protocol_config) = BlockRequestHandler::new(protocol_id.clone(), client.clone()); + self.spawn_task(handler.run().boxed()); + protocol_config + }; + + // Add finality request handler. + let finality_request_protocol_config = match self.make_finality_proof_provider(PeersClient::Full(client.clone(), backend.clone())) { + Some(provider) => { + let (handler, protocol_config) = FinalityRequestHandler::new(protocol_id.clone(), provider); + self.spawn_task(handler.run().boxed()); + protocol_config + }, + None => { + sc_network::finality_request_handler::generate_protocol_config(protocol_id.clone()) + } + }; + let network = NetworkWorker::new(sc_network::config::Params { role: Role::Full, executor: None, network_config, chain: client.clone(), - finality_proof_provider: self.make_finality_proof_provider( - PeersClient::Full(client.clone(), backend.clone()), - ), finality_proof_request_builder, on_demand: None, transaction_pool: Arc::new(EmptyTransactionPool), - protocol_id: ProtocolId::from("test-protocol-name"), + protocol_id, import_queue, block_announce_validator: config.block_announce_validator .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)), metrics_registry: None, + block_request_protocol_config, + finality_request_protocol_config, }).unwrap(); self.mut_peers(|peers| { @@ -750,21 +773,28 @@ pub trait TestNetFactory: Sized { network_config.listen_addresses = vec![listen_addr.clone()]; network_config.allow_non_globals_in_dht = true; + let protocol_id = ProtocolId::from("test-protocol-name"); + + // Add block request handler. + let block_request_protocol_config = sc_network::block_request_handler::generate_protocol_config(protocol_id.clone()); + + // Add finality request handler. + let finality_request_protocol_config = sc_network::finality_request_handler::generate_protocol_config(protocol_id.clone()); + let network = NetworkWorker::new(sc_network::config::Params { role: Role::Light, executor: None, network_config, chain: client.clone(), - finality_proof_provider: self.make_finality_proof_provider( - PeersClient::Light(client.clone(), backend.clone()) - ), finality_proof_request_builder, on_demand: None, transaction_pool: Arc::new(EmptyTransactionPool), - protocol_id: ProtocolId::from("test-protocol-name"), + protocol_id, import_queue, block_announce_validator: Box::new(DefaultBlockAnnounceValidator), metrics_registry: None, + block_request_protocol_config, + finality_request_protocol_config, }).unwrap(); self.mut_peers(|peers| { @@ -789,6 +819,11 @@ pub trait TestNetFactory: Sized { }); } + // TODO: Not ideal. Can we do better? + fn spawn_task(&self, f: BoxFuture<'static, ()>) { + async_std::task::spawn(f); + } + /// Polls the testnet until all nodes are in sync. /// /// Must be executed in a task context. diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index af7f611bd3375..84aab8d62259f 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -871,19 +871,24 @@ pub fn build_network( Box::new(DefaultBlockAnnounceValidator) }; - let mut network_config = config.network.clone(); - - if let Some(proof_provider) = finality_proof_provider { - let (handler, protocol_config) = sc_network::finality_request_handler::FinalityRequestHandler::new(protocol_id.clone(), proof_provider); - network_config.request_response_protocols.push(protocol_config); - spawn_handle.spawn("finality_request_handler", handler.run()); - } - - { + let block_request_protocol_config = { + // TODO: Only do this if we are not a light client. let (handler, protocol_config) = sc_network::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); - network_config.request_response_protocols.push(protocol_config); spawn_handle.spawn("block_request_handler", handler.run()); - } + protocol_config + }; + + let finality_request_protocol_config = match finality_proof_provider { + Some(provider) => { + // TODO: Only do this if we are not a light client. + let (handler, protocol_config) = sc_network::finality_request_handler::FinalityRequestHandler::new(protocol_id.clone(), provider); + spawn_handle.spawn("finality_request_handler", handler.run()); + protocol_config + }, + None => { + sc_network::finality_request_handler::generate_protocol_config(protocol_id.clone()) + } + }; let network_params = sc_network::config::Params { role: config.role.clone(), @@ -893,7 +898,7 @@ pub fn build_network( spawn_handle.spawn("libp2p-node", fut); })) }, - network_config, + network_config: config.network.clone(), chain: client.clone(), finality_proof_request_builder, on_demand: on_demand, @@ -901,7 +906,9 @@ pub fn build_network( import_queue: Box::new(import_queue), protocol_id, block_announce_validator, - metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()) + metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), + block_request_protocol_config, + finality_request_protocol_config, }; let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); From 107aa83e12ae2adebcdf02d4eaaa5f3cb4fda184 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 4 Nov 2020 20:00:12 +0100 Subject: [PATCH 21/48] client/network: Pass protocol config in tests --- client/network/src/gossip/tests.rs | 19 ++++++------------- client/network/src/service/tests.rs | 19 ++++++------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index 8fca328456851..cbd179290df51 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -33,7 +33,7 @@ type TestNetworkService = NetworkService< /// /// > **Note**: We return the events stream in order to not possibly lose events between the /// > construction of the service and the moment the events stream is grabbed. -fn build_test_full_node(mut network_config: config::NetworkConfiguration) +fn build_test_full_node(network_config: config::NetworkConfiguration) -> (Arc, impl Stream) { let client = Arc::new( @@ -94,21 +94,14 @@ fn build_test_full_node(mut network_config: config::NetworkConfiguration) let protocol_id = config::ProtocolId::from("/test-protocol-name"); // Add block request handler. - let block_request_protocol_name = { + let block_request_protocol_config = { let (handler, protocol_config) = crate::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); - let name = protocol_config.name.to_string(); - network_config.request_response_protocols.push(protocol_config); async_std::task::spawn(handler.run().boxed()); - name + protocol_config }; // Add finality protocol. - let finality_request_protocol_name = { - let protocol_config = crate::finality_request_handler::generate_protocol_config(protocol_id.clone()); - let name = protocol_config.name.to_string(); - network_config.request_response_protocols.push(protocol_config); - name - }; + let finality_request_protocol_config = crate::finality_request_handler::generate_protocol_config(protocol_id.clone()); let worker = NetworkWorker::new(config::Params { role: config::Role::Full, @@ -124,8 +117,8 @@ fn build_test_full_node(mut network_config: config::NetworkConfiguration) sp_consensus::block_validation::DefaultBlockAnnounceValidator, ), metrics_registry: None, - block_request_protocol_name, - finality_request_protocol_name, + block_request_protocol_config, + finality_request_protocol_config, }) .unwrap(); diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 7ee293861ef2a..01b9b0288d037 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -95,26 +95,19 @@ fn build_test_full_node(config: config::NetworkConfiguration) let protocol_id = config::ProtocolId::from("/test-protocol-name"); // Add block request handler. - let block_request_protocol_name = { + let block_request_protocol_config = { let (handler, protocol_config) = crate::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); - let name = protocol_config.name.to_string(); - network_config.request_response_protocols.push(protocol_config); async_std::task::spawn(handler.run().boxed()); - name + protocol_config }; // Add finality protocol. - let finality_request_protocol_name = { - let protocol_config = crate::finality_request_handler::generate_protocol_config(protocol_id.clone()); - let name = protocol_config.name.to_string(); - network_config.request_response_protocols.push(protocol_config); - name - }; + let finality_request_protocol_config = crate::finality_request_handler::generate_protocol_config(protocol_id.clone()); let worker = NetworkWorker::new(config::Params { role: config::Role::Full, executor: None, - network_config, + network_config: config, chain: client.clone(), finality_proof_request_builder: None, on_demand: None, @@ -125,8 +118,8 @@ fn build_test_full_node(config: config::NetworkConfiguration) sp_consensus::block_validation::DefaultBlockAnnounceValidator, ), metrics_registry: None, - block_request_protocol_name, - finality_request_protocol_name, + block_request_protocol_config, + finality_request_protocol_config, }) .unwrap(); From 0535723da004d790d2aee8d3b646bf79022bb641 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 6 Nov 2020 12:39:20 +0100 Subject: [PATCH 22/48] client/network/config: Document request response configs --- client/network/src/config.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 9db0a84d3792c..09752b6198e83 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -101,8 +101,28 @@ pub struct Params { /// Registry for recording prometheus metrics to. pub metrics_registry: Option, - // TODO: Dcoument that you can only get these by creating a Protocol config or creating a whole handler. + /// Request response configuration for the block request protocol. + /// + /// [`RequestResponseConfig`] [`name`] is used to tag outgoing block requests with the correct + /// protocol name. In addition all of [`RequestResponseConfig`] is used to handle incoming block + /// requests, if enabled. + /// + /// Can be constructed either via [`block_request_handler::generate_protocol_config`] allowing + /// outgoing but not incoming requests, or constructed via + /// [`block_request_handler::BlockRequestHandler::new`] allowing both outgoing and incoming + /// requests. pub block_request_protocol_config: RequestResponseConfig, + + /// Request response configuration for the finality request protocol. + /// + /// [`RequestResponseConfig`] [`name`] is used to tag outgoing finality requests with the + /// correct protocol name. In addition all of [`RequestResponseConfig`] is used to handle + /// incoming finality requests, if enabled. + /// + /// Can be constructed either via [`finality_request_handler::generate_protocol_config`] + /// allowing outgoing but not incoming requests, or constructed via + /// [`finality_request_handler::FinalityRequestHandler::new`] allowing both outgoing and + /// incoming requests. pub finality_request_protocol_config: RequestResponseConfig, } From 7a3d71ebf0c0ad6c52eaa0f09a90453d0ce36b5e Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 6 Nov 2020 12:41:28 +0100 Subject: [PATCH 23/48] client/network/src/_request_handler: Document protocol config gen --- client/network/src/block_request_handler.rs | 2 +- client/network/src/finality_request_handler.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 1398d835335bb..4540009900c2d 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -38,7 +38,7 @@ const LOG_TARGET: &str = "block-request-handler"; const MAX_BLOCKS_IN_RESPONSE: usize = 128; const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; -// TODO: Document that this is for clients not reponding only. +/// Generates a [`ProtocolConfig`] for the block request protocol, refusing incoming requests. pub fn generate_protocol_config(protocol_id: ProtocolId) -> ProtocolConfig { ProtocolConfig { name: generate_protocol_name(protocol_id).into(), diff --git a/client/network/src/finality_request_handler.rs b/client/network/src/finality_request_handler.rs index 0819bd7ae0dd3..7d66c8a5edeae 100644 --- a/client/network/src/finality_request_handler.rs +++ b/client/network/src/finality_request_handler.rs @@ -32,7 +32,7 @@ use std::time::Duration; const LOG_TARGET: &str = "finality-request-handler"; -// TODO: Document that this is for clients not reponding only. +/// Generates a [`ProtocolConfig`] for the finality request protocol, refusing incoming requests. pub fn generate_protocol_config(protocol_id: ProtocolId) -> ProtocolConfig { ProtocolConfig { name: generate_protocol_name(protocol_id).into(), From 64d22229c0c4dccda25f50054d44227470fadc3e Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 6 Nov 2020 12:51:12 +0100 Subject: [PATCH 24/48] client/network/src/protocol: Document Peer request values --- client/network/src/protocol.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index a6b17dc920c12..7d42082f4a12d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -247,11 +247,20 @@ struct PacketStats { #[derive(Debug, Clone)] struct Peer { info: PeerInfo, - /// Current block request, if any. - // TODO: Is Instant stil needed? + /// Current block request, if any. Started by emitting [`CustomMessageOutcome::BlockRequest`]. + /// + /// [`libp2p::request_response::RequestId`] is [`Some`] once the block request is send off and + /// [`Protocol::on_block_request_started`] is called. block_request: Option<(message::BlockRequest, Option)>, - // TODO: Document - finality_request: Option<(message::FinalityProofRequest, Option)>, + /// Current finality request, if any. Started by emitting + /// [`CustomMessageOutcome::FinalityProofRequest`]. + /// + /// [`libp2p::request_response::RequestId`] is [`Some`] once the finality request is send off + /// and [`Protocol::on_finality_proof_request_started`] is called. + finality_request: Option<( + message::FinalityProofRequest, + Option, + )>, /// Holds a set of transactions known to this peer. known_transactions: LruHashSet, /// Holds a set of blocks known to this peer. From 9560e3268cb61026ee78d2a5ac1a554c4f53ccce Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 6 Nov 2020 21:37:15 +0100 Subject: [PATCH 25/48] client/network: Rework request response to always use oneshot --- client/network/src/behaviour.rs | 128 +++--------- client/network/src/protocol.rs | 258 +++++++++++++----------- client/network/src/request_responses.rs | 58 ++++-- client/network/src/service.rs | 119 +++++------ 4 files changed, 262 insertions(+), 301 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index c385ded4f5185..f5af95722a923 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -23,6 +23,7 @@ use crate::{ use bytes::Bytes; use codec::Encode as _; +use futures::channel::oneshot; use libp2p::NetworkBehaviour; use libp2p::core::{Multiaddr, PeerId, PublicKey}; use libp2p::identify::IdentifyInfo; @@ -41,7 +42,7 @@ use std::{ }; pub use crate::request_responses::{ - ResponseFailure, InboundFailure, RequestFailure, OutboundFailure, RequestId, SendRequestError + ResponseFailure, InboundFailure, RequestFailure, OutboundFailure, RequestId, }; /// General behaviour of the network. Combines all protocols together. @@ -68,8 +69,7 @@ pub struct Behaviour { #[behaviour(ignore)] role: Role, - // TODO: Does this really belong here? Why require matching on these when reponses come in? Why - // not offer a oneshot when sending a request? + // TODO: Document. #[behaviour(ignore)] block_request_protocol_name: String, #[behaviour(ignore)] @@ -102,8 +102,6 @@ pub enum BehaviourOut { RequestFinished { /// Request that has succeeded. request_id: RequestId, - /// Response sent by the remote or reason for failure. - result: Result, RequestFailure>, }, /// Opened a substream with the given node with the given notifications protocol. @@ -229,14 +227,19 @@ impl Behaviour { self.peer_info.node(peer_id) } + // TODO Update comment. /// Initiates sending a request. /// /// An error is returned if we are not connected to the target peer of if the protocol doesn't /// match one that has been registered. - pub fn send_request(&mut self, target: &PeerId, protocol: &str, request: Vec) - -> Result - { - self.request_responses.send_request(target, protocol, request) + pub fn send_request( + &mut self, + target: &PeerId, + protocol: &str, + request: Vec, + pending_response: oneshot::Sender, RequestFailure>>, + ) { + self.request_responses.send_request(target, protocol, request, pending_response) } /// Registers a new notifications protocol. @@ -291,36 +294,6 @@ impl Behaviour { pub fn light_client_request(&mut self, r: light_client_handler::Request) -> Result<(), light_client_handler::Error> { self.light_client_handler.request(r) } - - fn on_block_response( - &mut self, - peer: &PeerId, - // TODO: shorten - request_id: libp2p::request_response::RequestId, - result: Result, request_responses::RequestFailure>, - ) -> Result, OnBlockResponseError>{ - let protobuf_response = schema::v1::BlockResponse::decode(&result?[..])?; - let ev = self.substrate.on_block_response(peer.clone(), request_id, protobuf_response); - Ok(ev) - } - - fn on_finality_response( - &mut self, - peer: &PeerId, - // TODO: shorten - request_id: libp2p::request_response::RequestId, - result: Result, request_responses::RequestFailure>, - ) -> Result, OnFinalityResponseError>{ - let protobuf_response = schema::v1::finality::FinalityProofResponse::decode( - &result?[..], - )?; - let ev = self.substrate.on_finality_proof_response( - peer.clone(), - request_id, - protobuf_response.proof, - ); - Ok(ev) - } } #[derive(derive_more::Display, derive_more::From)] @@ -368,7 +341,8 @@ Behaviour { self.events.push_back(BehaviourOut::JustificationImport(origin, hash, nb, justification)), CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) => self.events.push_back(BehaviourOut::FinalityProofImport(origin, hash, nb, proof)), - CustomMessageOutcome::BlockRequest { target, request } => { + CustomMessageOutcome::BlockRequest { target, request, pending_response } => { + // TODO: Move this logic into protocol.rs let protobuf_req = schema::v1::BlockRequest { fields: request.fields.to_be_u32(), from_block: match request.from { @@ -394,21 +368,11 @@ Behaviour { return } - match self.request_responses.send_request( - &target, &self.block_request_protocol_name, buf, - ) { - Ok(request_id) => self.substrate.on_block_request_started( - target, request.id, request_id, - ), - // TODO: Should we notify protocol.rs or sync.rs? - Err(err) => log::warn!( - target: "sync", - "Failed to send block request {:?}: {:?}", - protobuf_req, err - ), - }; + self.request_responses.send_request( + &target, &self.block_request_protocol_name, buf, pending_response, + ); }, - CustomMessageOutcome::FinalityProofRequest { target, block_hash, request } => { + CustomMessageOutcome::FinalityProofRequest { target, block_hash, request, pending_response } => { let protobuf_req = crate::schema::v1::finality::FinalityProofRequest { block_hash: block_hash.encode(), request, @@ -424,18 +388,9 @@ Behaviour { return; } - match self.request_responses.send_request( - &target, &self.finality_request_protocol_name, buf, - ) { - Ok(request_id) => self.substrate.on_finality_proof_request_started( - target, block_hash, request_id, - ), - Err(err) => log::warn!( - target: "sync", - "Failed to send block request {:?}: {:?}", - protobuf_req, err - ), - } + self.request_responses.send_request( + &target, &self.finality_request_protocol_name, buf, pending_response, + ); }, CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles, notifications_sink } => { let role = reported_roles_to_observed_role(&self.role, &remote, roles); @@ -484,42 +439,11 @@ impl NetworkBehaviourEventProcess { - // TODO: Make sure to get these responses captured in metrics by emitting a - // `RequestFinished` event without a result. Then the NetworkWorker can capture this - // in a metric. - if protocol == self.finality_request_protocol_name { - match self.on_finality_response(&peer, request_id, result) { - Ok(ev) => self.inject_event(ev), - Err(err) => { - debug!( - target: "sync", - "Handling finality response from {} failed with: {}", - peer, err, - ); - - // TODO: Should protocol.rs be notified of the failure? - } - } - } else if protocol == self.block_request_protocol_name { - match self.on_block_response(&peer, request_id, result) { - Ok(ev) => self.inject_event(ev), - Err(err) => { - debug!( - target: "sync", - "Handling block response from {} failed with: {}", - peer, err, - ); - - self.substrate.on_block_request_failed(&peer); - } - } - } else { - self.events.push_back(BehaviourOut::RequestFinished { - request_id, - result, - }); - } + request_responses::Event::RequestFinished { peer, protocol, request_id } => { + // TODO: Add everything here we need to capture the event in metrics. E.g. protocol. + self.events.push_back(BehaviourOut::RequestFinished { + request_id, + }); }, } } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 7d42082f4a12d..32dade910cfa9 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -25,36 +25,37 @@ use crate::{ }; use bytes::{Bytes, BytesMut}; -use futures::{prelude::*, stream::FuturesUnordered}; +use codec::{Decode, DecodeAll, Encode}; +use futures::{channel::oneshot, prelude::*, stream::FuturesUnordered}; use generic_proto::{GenericProto, GenericProtoOut}; use libp2p::{Multiaddr, PeerId}; use libp2p::core::{ConnectedPoint, connection::{ConnectionId, ListenerId}}; use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; +use log::{log, Level, trace, debug, warn, error}; +use message::{BlockAnnounce, Message}; +use message::generic::{Message as GenericMessage, Roles}; +use prometheus_endpoint::{ + Registry, Gauge, Counter, GaugeVec, + PrometheusError, Opts, register, U64 +}; +use prost::Message as _; use sp_consensus::{ BlockOrigin, block_validation::BlockAnnounceValidator, import_queue::{BlockImportResult, BlockImportError, IncomingBlock, Origin} }; -use codec::{Decode, DecodeAll, Encode}; use sp_runtime::{generic::BlockId, ConsensusEngineId, Justification}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub }; use sp_arithmetic::traits::SaturatedConversion; -use message::{BlockAnnounce, Message}; -use message::generic::{Message as GenericMessage, Roles}; -use prometheus_endpoint::{ - Registry, Gauge, Counter, GaugeVec, - PrometheusError, Opts, register, U64 -}; use sync::{ChainSync, SyncState}; use std::borrow::Cow; use std::collections::{HashMap, HashSet, VecDeque, hash_map::Entry}; use std::sync::Arc; use std::fmt::Write; use std::{io, iter, num::NonZeroUsize, pin::Pin, task::Poll, time}; -use log::{log, Level, trace, debug, warn, error}; mod generic_proto; @@ -244,22 +245,27 @@ struct PacketStats { count_out: u64, } /// Peer information -#[derive(Debug, Clone)] +#[derive(Debug)] struct Peer { info: PeerInfo, /// Current block request, if any. Started by emitting [`CustomMessageOutcome::BlockRequest`]. /// + // TODO: Update /// [`libp2p::request_response::RequestId`] is [`Some`] once the block request is send off and /// [`Protocol::on_block_request_started`] is called. - block_request: Option<(message::BlockRequest, Option)>, + block_request: Option<( + message::BlockRequest, + oneshot::Receiver, crate::request_responses::RequestFailure>>, + )>, /// Current finality request, if any. Started by emitting /// [`CustomMessageOutcome::FinalityProofRequest`]. /// + // TODO: Update /// [`libp2p::request_response::RequestId`] is [`Some`] once the finality request is send off /// and [`Protocol::on_finality_proof_request_started`] is called. finality_request: Option<( message::FinalityProofRequest, - Option, + oneshot::Receiver, crate::request_responses::RequestFailure>>, )>, /// Holds a set of transactions known to this peer. known_transactions: LruHashSet, @@ -668,8 +674,14 @@ impl Protocol { CustomMessageOutcome::None } - fn update_peer_request(&mut self, who: &PeerId, request: &mut message::BlockRequest) { - update_peer_request::(&mut self.context_data.peers, who, request) + fn update_peer_request( + &mut self, + who: &PeerId, + request: &mut message::BlockRequest, + rx: oneshot::Receiver, crate::request_responses::RequestFailure>>, + ) { + // TODO: Can we not inline this? + update_peer_request::(&mut self.context_data.peers, who, request, rx) } /// Called by peer when it is disconnecting @@ -703,24 +715,9 @@ impl Protocol { pub fn on_block_response( &mut self, peer_id: PeerId, - request_id: libp2p::request_response::RequestId, + request: message::BlockRequest, response: crate::schema::v1::BlockResponse, ) -> CustomMessageOutcome { - let peer = match self.context_data.peers.get_mut(&peer_id) { - Some(peer) => peer, - None => { - debug!(target: "sync", "Ignoring obsolete block response packet from {}", peer_id); - return CustomMessageOutcome::None; - } - }; - - let block_request = match &peer.block_request { - Some((_req, Some(id))) if *id == request_id => peer.block_request - .take() - .expect("id to be Some").0, - Some(_) | None => return CustomMessageOutcome::None, - }; - let blocks = response.blocks.into_iter().map(|block_data| { Ok(message::BlockData:: { hash: Decode::decode(&mut block_data.hash.as_ref())?, @@ -729,7 +726,7 @@ impl Protocol { } else { None }, - body: if block_request.fields.contains(message::BlockAttributes::BODY) { + body: if request.fields.contains(message::BlockAttributes::BODY) { Some(block_data.body.iter().map(|body| { Decode::decode(&mut body.as_ref()) }).collect::, _>>()?) @@ -766,7 +763,7 @@ impl Protocol { }; let block_response = message::BlockResponse:: { - id: block_request.id, + id: request.id, blocks, }; @@ -785,7 +782,7 @@ impl Protocol { blocks_range(), ); - if block_request.fields == message::BlockAttributes::JUSTIFICATION { + if request.fields == message::BlockAttributes::JUSTIFICATION { match self.sync.on_block_justification(peer_id, block_response) { Ok(sync::OnBlockJustification::Nothing) => CustomMessageOutcome::None, Ok(sync::OnBlockJustification::Import { peer, hash, number, justification }) => @@ -798,27 +795,29 @@ impl Protocol { } } else { // Validate fields against the request. - if block_request.fields.contains(message::BlockAttributes::HEADER) && block_response.blocks.iter().any(|b| b.header.is_none()) { + if request.fields.contains(message::BlockAttributes::HEADER) && block_response.blocks.iter().any(|b| b.header.is_none()) { self.behaviour.disconnect_peer(&peer_id); self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); trace!(target: "sync", "Missing header for a block"); return CustomMessageOutcome::None } - if block_request.fields.contains(message::BlockAttributes::BODY) && block_response.blocks.iter().any(|b| b.body.is_none()) { + if request.fields.contains(message::BlockAttributes::BODY) && block_response.blocks.iter().any(|b| b.body.is_none()) { self.behaviour.disconnect_peer(&peer_id); self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); trace!(target: "sync", "Missing body for a block"); return CustomMessageOutcome::None } - match self.sync.on_block_data(&peer_id, Some(block_request), block_response) { + match self.sync.on_block_data(&peer_id, Some(request), block_response) { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), Ok(sync::OnBlockData::Request(peer, mut req)) => { - self.update_peer_request(&peer, &mut req); + let (tx, rx) = oneshot::channel(); + self.update_peer_request(&peer, &mut req, rx); CustomMessageOutcome::BlockRequest { target: peer_id, request: req, + pending_response: tx, } } Err(sync::BadPeer(id, repu)) => { @@ -934,10 +933,12 @@ impl Protocol { match self.sync.new_peer(who.clone(), info.best_hash, info.best_number) { Ok(None) => (), Ok(Some(mut req)) => { - self.update_peer_request(&who, &mut req); + let (tx, rx) = oneshot::channel(); + self.update_peer_request(&who, &mut req, rx); self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { target: who.clone(), request: req, + pending_response: tx, }); }, Err(sync::BadPeer(id, repu)) => { @@ -1269,10 +1270,13 @@ impl Protocol { CustomMessageOutcome::BlockImport(origin, blocks) }, Ok(sync::OnBlockData::Request(peer, mut req)) => { - self.update_peer_request(&peer, &mut req); + // TODO: Dry out this logic as it is repeated quite often. How about a `prepare_block_request`? + let (tx, rx) = oneshot::channel(); + self.update_peer_request(&peer, &mut req, rx); CustomMessageOutcome::BlockRequest { target: peer, request: req, + pending_response: tx, } } Err(sync::BadPeer(id, repu)) => { @@ -1333,10 +1337,12 @@ impl Protocol { for result in results { match result { Ok((id, mut req)) => { - update_peer_request(&mut self.context_data.peers, &id, &mut req); + let (tx, rx) = oneshot::channel(); + update_peer_request(&mut self.context_data.peers, &id, &mut req, rx); self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { target: id, request: req, + pending_response: tx, }); } Err(sync::BadPeer(id, repu)) => { @@ -1380,23 +1386,13 @@ impl Protocol { pub fn on_finality_proof_response( &mut self, who: PeerId, - request_id: libp2p::request_response::RequestId, + request: message::FinalityProofRequest, proof: Vec, ) -> CustomMessageOutcome { - let peer = match self.context_data.peers.get_mut(&who) { - Some(peer) => peer, - None => return CustomMessageOutcome::None, - }; - - let req = match &peer.finality_request { - Some((_req, Some(id))) if *id == request_id => peer.finality_request.take().unwrap().0, - Some(_) | None => return CustomMessageOutcome::None, - }; - let resp = message::FinalityProofResponse:: { // TODO: correct? id: 0, - block: req.block, + block: request.block, proof: if proof.is_empty() { None } else { Some(proof) }, }; @@ -1412,63 +1408,6 @@ impl Protocol { } } - /// Notify [`Protocol`] that the finality proof request for the given block hash has been - /// started and that the response for this request will be identified with the given request id. - // - // TODO: This could be simplified by having `RequestResponses::send_request` take a - // `oneshot::Sender` to send the response into. - pub fn on_finality_proof_request_started( - &mut self, - who: PeerId, - block_hash: B::Hash, - request_id: libp2p::request_response::RequestId, - ) { - let peer = match self.context_data.peers.get_mut(&who) { - Some(peer) => peer, - None => return, - }; - - if let Peer { finality_request: Some((req, id)), ..} = peer { - if req.block == block_hash { - debug_assert!( - id.is_none(), - "Expect `on_finality_proof_request_started` not to be called twice for the \ - same request.", - ); - *id = Some(request_id); - } - } - } - - /// Notify [`Protocol`] that the block request with the block request id `request_id` has been - /// started and that the response for this request will be identified with the given request - /// responses request id. - // - // TODO: This could be simplified by having `RequestResponses::send_request` take a - // `oneshot::Sender` to send the response into. - pub fn on_block_request_started( - &mut self, - who: PeerId, - request_id: u64, - request_response_request_id: libp2p::request_response::RequestId, - ) { - let peer = match self.context_data.peers.get_mut(&who) { - Some(peer) => peer, - None => return, - }; - - if let Peer { block_request: Some((req, id)), ..} = peer { - if req.id == request_id { - debug_assert!( - id.is_none(), - "Expect `on_block_request_started` not to be called twice for the \ - same request.", - ); - *id = Some(request_response_request_id); - } - } - } - fn format_stats(&self) -> String { let mut out = String::new(); for (id, stats) in &self.context_data.stats { @@ -1542,19 +1481,29 @@ pub enum CustomMessageOutcome { NotificationStreamClosed { remote: PeerId, protocols: Vec }, /// Messages have been received on one or more notifications protocols. NotificationsReceived { remote: PeerId, messages: Vec<(ConsensusEngineId, Bytes)> }, + // TODO: Update comment. /// A new block request must be emitted. /// You must later call either [`Protocol::on_block_response`] or /// [`Protocol::on_block_request_failed`]. /// Each peer can only have one active request. If a request already exists for this peer, it /// must be silently discarded. /// It is the responsibility of the handler to ensure that a timeout exists. - BlockRequest { target: PeerId, request: message::BlockRequest }, + BlockRequest { + target: PeerId, + request: message::BlockRequest, + pending_response: oneshot::Sender, crate::request_responses::RequestFailure>>, + }, /// A new finality proof request must be emitted. /// Once you have the response, you must call `Protocol::on_finality_proof_response`. /// It is the responsibility of the handler to ensure that a timeout exists. /// If the request times out, or the peer responds in an invalid way, the peer has to be /// disconnect. This will inform the state machine that the request it has emitted is stale. - FinalityProofRequest { target: PeerId, block_hash: B::Hash, request: Vec }, + FinalityProofRequest { + target: PeerId, + block_hash: B::Hash, + request: Vec, + pending_response: oneshot::Sender, crate::request_responses::RequestFailure>>, + }, /// Peer has a reported a new head of chain. PeerNewBest(PeerId, NumberFor), None, @@ -1565,11 +1514,12 @@ fn update_peer_request( peers: &mut HashMap>, who: &PeerId, request: &mut message::BlockRequest, + rx: oneshot::Receiver, crate::request_responses::RequestFailure>>, ) { if let Some(ref mut peer) = peers.get_mut(who) { request.id = peer.next_request_id; peer.next_request_id += 1; - peer.block_request = Some((request.clone(), None)); + peer.block_request = Some((request.clone(), rx)); } } @@ -1624,6 +1574,70 @@ impl NetworkBehaviour for Protocol { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message)); } + // TODO: rework + let mut finished = Vec::new(); + // Check for finished outgoing requests. + for (id, peer) in self.context_data.peers.iter_mut() { + if let Peer { block_request: Some((req, pending_response)), .. } = peer { + match pending_response.poll_unpin(cx) { + Poll::Ready(Ok(Ok(resp))) => { + let (req, _) = peer.block_request.take().unwrap(); + + let protobuf_response = match crate::schema::v1::BlockResponse::decode(&resp[..]) { + Ok(proto) => proto, + Err(e) => { + todo!("log error"); + } + }; + + let id = id.clone(); + finished.push((id, req, protobuf_response)); + }, + Poll::Ready(Ok(Err(e))) => todo!("call on block request canceled"), + Poll::Ready(Err(e)) => todo!("handle"), + Poll::Pending => continue, + } + } + } + for (id, req, protobuf_response) in finished { + let ev = self.on_block_response(id, req, protobuf_response); + // TODO: Is this correct? Should we check at the very end once more whether + // there have been any events addedd to pending_messages? + self.pending_messages.push_back(ev); + } + + // TODO: rework + let mut finished = Vec::new(); + // Check for finished outgoing requests. + for (id, peer) in self.context_data.peers.iter_mut() { + if let Peer { finality_request: Some((req, pending_response)), .. } = peer { + match pending_response.poll_unpin(cx) { + Poll::Ready(Ok(Ok(resp))) => { + let (req, _) = peer.finality_request.take().unwrap(); + + let protobuf_response = match crate::schema::v1::finality::FinalityProofResponse::decode(&resp[..]) { + Ok(proto) => proto, + Err(e) => { + todo!("log error"); + } + }; + + let id = id.clone(); + finished.push((id, req, protobuf_response)); + }, + Poll::Ready(Ok(Err(e))) => todo!("call on block request canceled"), + Poll::Ready(Err(e)) => todo!("handle"), + Poll::Pending => continue, + } + } + } + for (id, req, protobuf_response) in finished { + let ev = self.on_finality_proof_response(id, req, protobuf_response.proof); + // TODO: Is this correct? Should we check at the very end once more whether + // there have been any events addedd to pending_messages? + self.pending_messages.push_back(ev); + } + while let Poll::Ready(Some(())) = self.tick_timeout.poll_next_unpin(cx) { self.tick(); } @@ -1632,26 +1646,33 @@ impl NetworkBehaviour for Protocol { self.propagate_transactions(); } - for (id, mut r) in self.sync.block_requests() { - update_peer_request(&mut self.context_data.peers, &id, &mut r); + for (id, mut request) in self.sync.block_requests() { + let (tx, rx) = oneshot::channel(); + + update_peer_request(&mut self.context_data.peers, id, &mut request, rx); + let event = CustomMessageOutcome::BlockRequest { target: id.clone(), - request: r, + request: request, + pending_response: tx, }; self.pending_messages.push_back(event); } for (id, mut r) in self.sync.justification_requests() { - update_peer_request(&mut self.context_data.peers, &id, &mut r); + let (tx, rx) = oneshot::channel(); + update_peer_request(&mut self.context_data.peers, &id, &mut r, rx); let event = CustomMessageOutcome::BlockRequest { target: id, request: r, + pending_response: tx, }; self.pending_messages.push_back(event); } for (id, r) in self.sync.finality_proof_requests() { + let (tx, rx) = oneshot::channel(); match self.context_data.peers.get_mut(&id) { Some(peer) => { - peer.finality_request = Some((r.clone(), None)) + peer.finality_request = Some((r.clone(), rx)) }, None => unimplemented!(), } @@ -1659,6 +1680,7 @@ impl NetworkBehaviour for Protocol { target: id, block_hash: r.block, request: r.request, + pending_response: tx, }; self.pending_messages.push_back(event); } @@ -1763,7 +1785,7 @@ impl NetworkBehaviour for Protocol { } } Some(Fallback::Transactions) => { - if let Ok(m) = message::Transactions::decode(&mut message.as_ref()) { + if let Ok(m) = as Decode>::decode(&mut message.as_ref()) { self.on_transactions(peer_id, m); } else { warn!(target: "sub-libp2p", "Failed to decode transactions list"); diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 494e6a536d8ad..896715142aada 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -133,14 +133,14 @@ pub enum Event { /// A request initiated using [`RequestResponsesBehaviour::send_request`] has succeeded or /// failed. + /// + /// This event is generated for statistics purposes. RequestFinished { peer: PeerId, /// Name of the protocol in question. protocol: Cow<'static, str>, /// Request that has succeeded. request_id: RequestId, - /// Response sent by the remote or reason for failure. - result: Result, RequestFailure>, }, } @@ -154,6 +154,9 @@ pub struct RequestResponsesBehaviour { (RequestResponse, Option>) >, + // TODO: Document + pending_requests: HashMap, RequestFailure>>>, + /// Whenever an incoming request arrives, a `Future` is added to this list and will yield the /// response to send back to the remote. pending_responses: stream::FuturesUnordered< @@ -204,7 +207,8 @@ impl RequestResponsesBehaviour { Ok(Self { protocols, - pending_responses: stream::FuturesUnordered::new(), + pending_requests: Default::default(), + pending_responses: Default::default(), }) } @@ -212,17 +216,22 @@ impl RequestResponsesBehaviour { /// /// An error is returned if we are not connected to the target peer or if the protocol doesn't /// match one that has been registered. - pub fn send_request(&mut self, target: &PeerId, protocol: &str, request: Vec) - -> Result - { + pub fn send_request( + &mut self, + target: &PeerId, + protocol: &str, + request: Vec, + pending_response: oneshot::Sender, RequestFailure>>, + ) { if let Some((protocol, _)) = self.protocols.get_mut(protocol) { if protocol.is_connected(target) { - Ok(protocol.send_request(target, request)) + let request_id = protocol.send_request(target, request); + self.pending_requests.insert(request_id, pending_response); } else { - Err(SendRequestError::NotConnected) + pending_response.send(Err(RequestFailure::NotConnected)); } } else { - Err(SendRequestError::UnknownProtocol) + pending_response.send(Err(RequestFailure::UnknownProtocol)); } } } @@ -456,11 +465,20 @@ impl NetworkBehaviour for RequestResponsesBehaviour { }, .. } => { + match self.pending_requests.remove(&request_id) { + Some(pending_response) => { + pending_response.send( + response.map_err(|()| RequestFailure::Refused), + ).unwrap(); + } + None => { + todo!("handle this"); + } + } let out = Event::RequestFinished { peer, protocol: protocol.clone(), request_id, - result: response.map_err(|()| RequestFailure::Refused), }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); } @@ -472,11 +490,20 @@ impl NetworkBehaviour for RequestResponsesBehaviour { error, .. } => { + match self.pending_requests.remove(&request_id) { + Some(pending_response) => { + pending_response.send( + Err(RequestFailure::Network(error)), + ).unwrap(); + } + None => { + todo!("handle this"); + } + } let out = Event::RequestFinished { peer, protocol: protocol.clone(), request_id, - result: Err(RequestFailure::Network(error)), }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); } @@ -506,18 +533,13 @@ pub enum RegisterError { DuplicateProtocol(#[error(ignore)] Cow<'static, str>), } -/// Error when sending a request. +/// Error in a request. #[derive(Debug, derive_more::Display, derive_more::Error)] -pub enum SendRequestError { +pub enum RequestFailure { /// We are not currently connected to the requested peer. NotConnected, /// Given protocol hasn't been registered. UnknownProtocol, -} - -/// Error in a request. -#[derive(Debug, derive_more::Display, derive_more::Error)] -pub enum RequestFailure { /// Remote has closed the substream before answering, thereby signaling that it considers the /// request as valid, but refused to answer it. Refused, diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 2e2a83ce767b4..9545ca214ec36 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -408,7 +408,6 @@ impl NetworkWorker { peers_notifications_sinks, metrics, boot_node_ids, - pending_requests: HashMap::with_capacity(128), }) } @@ -1236,13 +1235,6 @@ pub struct NetworkWorker { metrics: Option, /// The `PeerId`'s of all boot nodes. boot_node_ids: Arc>, - /// Requests started using [`NetworkService::request`]. Includes the channel to send back the - /// response, when the request has started, and the name of the protocol for diagnostic - /// purposes. - pending_requests: HashMap< - behaviour::RequestId, - (oneshot::Sender, RequestFailure>>, Instant, String) - >, /// For each peer and protocol combination, an object that allows sending notifications to /// that peer. Shared with the [`NetworkService`]. peers_notifications_sinks: Arc>>, @@ -1315,29 +1307,30 @@ impl Future for NetworkWorker { ServiceToWorkerMsg::EventStream(sender) => this.event_streams.push(sender), ServiceToWorkerMsg::Request { target, protocol, request, pending_response } => { - // Calling `send_request` can fail immediately in some circumstances. - // This is handled by sending back an error on the channel. - match this.network_service.send_request(&target, &protocol, request) { - Ok(request_id) => { - if let Some(metrics) = this.metrics.as_ref() { - metrics.requests_out_started_total - .with_label_values(&[&protocol]) - .inc(); - } - this.pending_requests.insert( - request_id, - (pending_response, Instant::now(), protocol.to_string()) - ); - }, - Err(behaviour::SendRequestError::NotConnected) => { - let err = RequestFailure::Network(OutboundFailure::ConnectionClosed); - let _ = pending_response.send(Err(err)); - }, - Err(behaviour::SendRequestError::UnknownProtocol) => { - let err = RequestFailure::Network(OutboundFailure::UnsupportedProtocols); - let _ = pending_response.send(Err(err)); - }, - } + this.network_service.send_request(&target, &protocol, request, pending_response); + // TODO: Should this be moved to NetworkService? + // { + // Ok(request_id) => { + // // TODO: Do this on a request started event to also track finality and block request events. + // if let Some(metrics) = this.metrics.as_ref() { + // metrics.requests_out_started_total + // .with_label_values(&[&protocol]) + // .inc(); + // } + // this.pending_requests.insert( + // request_id, + // (pending_response, Instant::now(), protocol.to_string()) + // ); + // }, + // Err(behaviour::SendRequestError::NotConnected) => { + // let err = RequestFailure::Network(OutboundFailure::ConnectionClosed); + // let _ = pending_response.send(Err(err)); + // }, + // Err(behaviour::SendRequestError::UnknownProtocol) => { + // let err = RequestFailure::Network(OutboundFailure::UnsupportedProtocols); + // let _ = pending_response.send(Err(err)); + // }, + // } }, ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, protocol_name } => { this.network_service @@ -1415,38 +1408,38 @@ impl Future for NetworkWorker { // TODO: Retrieve started and Protocol from `RequestFinished` and make result // optional. That way we can also return `RequestFinished` for internal requests // (block, finality and light-client) thus tracking metrics for them as well. - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { request_id, result })) => { - if let Some((send_back, started, protocol)) = this.pending_requests.remove(&request_id) { - if let Some(metrics) = this.metrics.as_ref() { - match &result { - Ok(_) => { - metrics.requests_out_success_total - .with_label_values(&[&protocol]) - .observe(started.elapsed().as_secs_f64()); - } - Err(err) => { - let reason = match err { - RequestFailure::Refused => "refused", - RequestFailure::Network(OutboundFailure::DialFailure) => - "dial-failure", - RequestFailure::Network(OutboundFailure::Timeout) => - "timeout", - RequestFailure::Network(OutboundFailure::ConnectionClosed) => - "connection-closed", - RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => - "unsupported", - }; - - metrics.requests_out_failure_total - .with_label_values(&[&protocol, reason]) - .inc(); - } - } - } - let _ = send_back.send(result); - } else { - error!("Request not in pending_requests"); - } + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { request_id })) => { + // if let Some((send_back, started, protocol)) = this.pending_requests.remove(&request_id) { + // if let Some(metrics) = this.metrics.as_ref() { + // match &result { + // Ok(_) => { + // metrics.requests_out_success_total + // .with_label_values(&[&protocol]) + // .observe(started.elapsed().as_secs_f64()); + // } + // Err(err) => { + // let reason = match err { + // RequestFailure::Refused => "refused", + // RequestFailure::Network(OutboundFailure::DialFailure) => + // "dial-failure", + // RequestFailure::Network(OutboundFailure::Timeout) => + // "timeout", + // RequestFailure::Network(OutboundFailure::ConnectionClosed) => + // "connection-closed", + // RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => + // "unsupported", + // }; + + // metrics.requests_out_failure_total + // .with_label_values(&[&protocol, reason]) + // .inc(); + // } + // } + // } + // let _ = send_back.send(result); + // } else { + // error!("Request not in pending_requests"); + // } }, // TODO: Is this still needed? Can we not do this within the substrate // requests_response behaviour? From 8b7b32dfeb7a6c346b4a4f0f23e6ef7dee86ae35 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 6 Nov 2020 22:06:24 +0100 Subject: [PATCH 26/48] client/network: Unified metric reporting for all request protocols --- client/network/src/behaviour.rs | 12 ++-- client/network/src/protocol.rs | 1 + client/network/src/request_responses.rs | 45 ++++++++++---- client/network/src/service.rs | 81 +++++++++---------------- client/network/src/service/metrics.rs | 1 + 5 files changed, 71 insertions(+), 69 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index f5af95722a923..9bb25957e159f 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -100,8 +100,12 @@ pub enum BehaviourOut { /// A request initiated using [`Behaviour::send_request`] has succeeded or failed. RequestFinished { - /// Request that has succeeded. - request_id: RequestId, + peer: PeerId, + /// Name of the protocol in question. + protocol: Cow<'static, str>, + // TODO: Document. + duration: Duration, + result: Result<(), RequestFailure>, }, /// Opened a substream with the given node with the given notifications protocol. @@ -439,10 +443,10 @@ impl NetworkBehaviourEventProcess { + request_responses::Event::RequestFinished { peer, protocol, duration, result } => { // TODO: Add everything here we need to capture the event in metrics. E.g. protocol. self.events.push_back(BehaviourOut::RequestFinished { - request_id, + peer, protocol, duration, result, }); }, } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 32dade910cfa9..a2515ed20da95 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1575,6 +1575,7 @@ impl NetworkBehaviour for Protocol { } // TODO: rework + // TODO: One could as well merge the block and finality loop. let mut finished = Vec::new(); // Check for finished outgoing requests. for (id, peer) in self.context_data.peers.iter_mut() { diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 896715142aada..50d635c1e5c8f 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -50,7 +50,8 @@ use libp2p::{ }; use std::{ borrow::Cow, collections::{hash_map::Entry, HashMap}, convert::TryFrom as _, io, iter, - pin::Pin, task::{Context, Poll}, time::Duration, + // TODO: Should we use wasm_timer instant here? + pin::Pin, task::{Context, Poll}, time::{Duration, Instant}, }; pub use libp2p::request_response::{InboundFailure, OutboundFailure, RequestId}; @@ -139,8 +140,10 @@ pub enum Event { peer: PeerId, /// Name of the protocol in question. protocol: Cow<'static, str>, - /// Request that has succeeded. - request_id: RequestId, + // TODO: Document. + duration: Duration, + // TODO: Document. + result: Result<(), RequestFailure> }, } @@ -155,7 +158,7 @@ pub struct RequestResponsesBehaviour { >, // TODO: Document - pending_requests: HashMap, RequestFailure>>>, + pending_requests: HashMap, RequestFailure>>)>, /// Whenever an incoming request arrives, a `Future` is added to this list and will yield the /// response to send back to the remote. @@ -226,7 +229,7 @@ impl RequestResponsesBehaviour { if let Some((protocol, _)) = self.protocols.get_mut(protocol) { if protocol.is_connected(target) { let request_id = protocol.send_request(target, request); - self.pending_requests.insert(request_id, pending_response); + self.pending_requests.insert(request_id, (Instant::now(), pending_response)); } else { pending_response.send(Err(RequestFailure::NotConnected)); } @@ -465,20 +468,24 @@ impl NetworkBehaviour for RequestResponsesBehaviour { }, .. } => { - match self.pending_requests.remove(&request_id) { - Some(pending_response) => { + // TODO: Rework. + let started = match self.pending_requests.remove(&request_id) { + Some((started, pending_response)) => { pending_response.send( response.map_err(|()| RequestFailure::Refused), ).unwrap(); + started } None => { todo!("handle this"); } - } + }; let out = Event::RequestFinished { peer, protocol: protocol.clone(), - request_id, + duration: started.elapsed(), + // TODO: Don't send OK when the error was mapped above as refused. + result: Ok(()), }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); } @@ -490,20 +497,32 @@ impl NetworkBehaviour for RequestResponsesBehaviour { error, .. } => { - match self.pending_requests.remove(&request_id) { - Some(pending_response) => { + // TODO: Remove hack + let error_clone = match &error { + OutboundFailure::ConnectionClosed => OutboundFailure::ConnectionClosed, + OutboundFailure::DialFailure => OutboundFailure::DialFailure, + OutboundFailure::Timeout => OutboundFailure::Timeout, + OutboundFailure::UnsupportedProtocols => OutboundFailure::UnsupportedProtocols, + }; + + // TODO: Rework. + let started = match self.pending_requests.remove(&request_id) { + Some((started, pending_response)) => { pending_response.send( Err(RequestFailure::Network(error)), ).unwrap(); + started } None => { todo!("handle this"); } - } + }; + let out = Event::RequestFinished { peer, protocol: protocol.clone(), - request_id, + duration: started.elapsed(), + result: Err(RequestFailure::Network(error_clone)), }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); } diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 9545ca214ec36..1ae3c531c4a50 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1405,59 +1405,36 @@ impl Future for NetworkWorker { } } }, - // TODO: Retrieve started and Protocol from `RequestFinished` and make result - // optional. That way we can also return `RequestFinished` for internal requests - // (block, finality and light-client) thus tracking metrics for them as well. - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { request_id })) => { - // if let Some((send_back, started, protocol)) = this.pending_requests.remove(&request_id) { - // if let Some(metrics) = this.metrics.as_ref() { - // match &result { - // Ok(_) => { - // metrics.requests_out_success_total - // .with_label_values(&[&protocol]) - // .observe(started.elapsed().as_secs_f64()); - // } - // Err(err) => { - // let reason = match err { - // RequestFailure::Refused => "refused", - // RequestFailure::Network(OutboundFailure::DialFailure) => - // "dial-failure", - // RequestFailure::Network(OutboundFailure::Timeout) => - // "timeout", - // RequestFailure::Network(OutboundFailure::ConnectionClosed) => - // "connection-closed", - // RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => - // "unsupported", - // }; - - // metrics.requests_out_failure_total - // .with_label_values(&[&protocol, reason]) - // .inc(); - // } - // } - // } - // let _ = send_back.send(result); - // } else { - // error!("Request not in pending_requests"); - // } + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { peer, protocol, duration, result })) => { + if let Some(metrics) = this.metrics.as_ref() { + match result { + Ok(_) => { + metrics.requests_out_success_total + .with_label_values(&[&protocol]) + .observe(duration.as_secs_f64()); + } + Err(err) => { + let reason = match err { + RequestFailure::NotConnected => "not-connected", + RequestFailure::UnknownProtocol => "unknown-protocol", + RequestFailure::Refused => "refused", + RequestFailure::Network(OutboundFailure::DialFailure) => + "dial-failure", + RequestFailure::Network(OutboundFailure::Timeout) => + "timeout", + RequestFailure::Network(OutboundFailure::ConnectionClosed) => + "connection-closed", + RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => + "unsupported", + }; + + metrics.requests_out_failure_total + .with_label_values(&[&protocol, reason]) + .inc(); + } + } + } }, - // TODO: Is this still needed? Can we not do this within the substrate - // requests_response behaviour? - // - // Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestStarted { protocol, .. })) => { - // if let Some(metrics) = this.metrics.as_ref() { - // metrics.requests_out_started_total - // .with_label_values(&[&protocol]) - // .inc(); - // } - // }, - // Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::OpaqueRequestFinished { protocol, request_duration, .. })) => { - // if let Some(metrics) = this.metrics.as_ref() { - // metrics.requests_out_success_total - // .with_label_values(&[&protocol]) - // .observe(request_duration.as_secs_f64()); - // } - // }, Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted(protocol))) => { if let Some(metrics) = this.metrics.as_ref() { metrics.kademlia_random_queries_total diff --git a/client/network/src/service/metrics.rs b/client/network/src/service/metrics.rs index a63ce7a18a519..220521ae6efcc 100644 --- a/client/network/src/service/metrics.rs +++ b/client/network/src/service/metrics.rs @@ -79,6 +79,7 @@ pub struct Metrics { pub requests_in_success_total: HistogramVec, pub requests_out_failure_total: CounterVec, pub requests_out_success_total: HistogramVec, + // TODO: Is this still used anywhere? pub requests_out_started_total: CounterVec, } From 1cd8a713c96036d70d4cdb80aca99f2a375ff1f4 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 11 Nov 2020 17:23:46 +0100 Subject: [PATCH 27/48] client/network: Move protobuf parsing into protocol.rs --- client/network/src/behaviour.rs | 35 ++----- client/network/src/protocol.rs | 162 ++++++++++++++++---------------- 2 files changed, 88 insertions(+), 109 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 9bb25957e159f..445bacb876919 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -346,26 +346,12 @@ Behaviour { CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) => self.events.push_back(BehaviourOut::FinalityProofImport(origin, hash, nb, proof)), CustomMessageOutcome::BlockRequest { target, request, pending_response } => { - // TODO: Move this logic into protocol.rs - let protobuf_req = schema::v1::BlockRequest { - fields: request.fields.to_be_u32(), - from_block: match request.from { - message::FromBlock::Hash(h) => - Some(schema::v1::block_request::FromBlock::Hash(h.encode())), - message::FromBlock::Number(n) => - Some(schema::v1::block_request::FromBlock::Number(n.encode())), - }, - to_block: request.to.map(|h| h.encode()).unwrap_or_default(), - direction: request.direction as i32, - max_blocks: request.max.unwrap_or(0), - }; - - let mut buf = Vec::with_capacity(protobuf_req.encoded_len()); - if let Err(err) = protobuf_req.encode(&mut buf) { + let mut buf = Vec::with_capacity(request.encoded_len()); + if let Err(err) = request.encode(&mut buf) { log::warn!( target: "sync", "Failed to encode block request {:?}: {:?}", - protobuf_req, err + request, err ); // TODO: Should we notify protocol.rs or sync.rs? @@ -376,19 +362,16 @@ Behaviour { &target, &self.block_request_protocol_name, buf, pending_response, ); }, - CustomMessageOutcome::FinalityProofRequest { target, block_hash, request, pending_response } => { - let protobuf_req = crate::schema::v1::finality::FinalityProofRequest { - block_hash: block_hash.encode(), - request, - }; - - let mut buf = Vec::with_capacity(protobuf_req.encoded_len()); - if let Err(err) = protobuf_req.encode(&mut buf) { + CustomMessageOutcome::FinalityProofRequest { target, request, pending_response } => { + let mut buf = Vec::with_capacity(request.encoded_len()); + if let Err(err) = request.encode(&mut buf) { log::warn!( target: "sync", "Failed to encode finality proof request {:?}: {:?}", - protobuf_req, err, + request, err, ); + + // TODO: Should we notify protocol.rs or sync.rs? return; } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index a2515ed20da95..f391a6e9b0c5d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -674,14 +674,13 @@ impl Protocol { CustomMessageOutcome::None } - fn update_peer_request( + fn prepare_block_request( &mut self, - who: &PeerId, - request: &mut message::BlockRequest, - rx: oneshot::Receiver, crate::request_responses::RequestFailure>>, - ) { + who: PeerId, + request: message::BlockRequest, + ) -> CustomMessageOutcome { // TODO: Can we not inline this? - update_peer_request::(&mut self.context_data.peers, who, request, rx) + prepare_block_request::(&mut self.context_data.peers, who, request) } /// Called by peer when it is disconnecting @@ -812,13 +811,7 @@ impl Protocol { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), Ok(sync::OnBlockData::Request(peer, mut req)) => { - let (tx, rx) = oneshot::channel(); - self.update_peer_request(&peer, &mut req, rx); - CustomMessageOutcome::BlockRequest { - target: peer_id, - request: req, - pending_response: tx, - } + self.prepare_block_request(peer, req) } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -933,13 +926,8 @@ impl Protocol { match self.sync.new_peer(who.clone(), info.best_hash, info.best_number) { Ok(None) => (), Ok(Some(mut req)) => { - let (tx, rx) = oneshot::channel(); - self.update_peer_request(&who, &mut req, rx); - self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { - target: who.clone(), - request: req, - pending_response: tx, - }); + let event = self.prepare_block_request(who.clone(), req); + self.pending_messages.push_back(event); }, Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -1270,14 +1258,7 @@ impl Protocol { CustomMessageOutcome::BlockImport(origin, blocks) }, Ok(sync::OnBlockData::Request(peer, mut req)) => { - // TODO: Dry out this logic as it is repeated quite often. How about a `prepare_block_request`? - let (tx, rx) = oneshot::channel(); - self.update_peer_request(&peer, &mut req, rx); - CustomMessageOutcome::BlockRequest { - target: peer, - request: req, - pending_response: tx, - } + self.prepare_block_request(peer, req) } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -1337,13 +1318,9 @@ impl Protocol { for result in results { match result { Ok((id, mut req)) => { - let (tx, rx) = oneshot::channel(); - update_peer_request(&mut self.context_data.peers, &id, &mut req, rx); - self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { - target: id, - request: req, - pending_response: tx, - }); + self.pending_messages.push_back( + prepare_block_request(&mut self.context_data.peers, id, req) + ); } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -1457,6 +1434,65 @@ impl Protocol { } } +fn prepare_block_request( + peers: &mut HashMap>, + who: PeerId, + mut request: message::BlockRequest, +) -> CustomMessageOutcome { + let (tx, rx) = oneshot::channel(); + + if let Some(ref mut peer) = peers.get_mut(&who) { + request.id = peer.next_request_id; + peer.next_request_id += 1; + peer.block_request = Some((request.clone(), rx)); + } + + let request = crate::schema::v1::BlockRequest { + fields: request.fields.to_be_u32(), + from_block: match request.from { + message::FromBlock::Hash(h) => + Some(crate::schema::v1::block_request::FromBlock::Hash(h.encode())), + message::FromBlock::Number(n) => + Some(crate::schema::v1::block_request::FromBlock::Number(n.encode())), + }, + to_block: request.to.map(|h| h.encode()).unwrap_or_default(), + direction: request.direction as i32, + max_blocks: request.max.unwrap_or(0), + }; + + CustomMessageOutcome::BlockRequest { + target: who, + request: request, + pending_response: tx, + } +} + +fn prepare_finality_request( + peers: &mut HashMap>, + who: PeerId, + request: message::FinalityProofRequest, +) -> CustomMessageOutcome { + let (tx, rx) = oneshot::channel(); + + match peers.get_mut(&who) { + Some(peer) => { + peer.finality_request = Some((request.clone(), rx)) + }, + None => unimplemented!(), + } + + let request = crate::schema::v1::finality::FinalityProofRequest { + block_hash: request.block.encode(), + request: request.request, + }; + + CustomMessageOutcome::FinalityProofRequest { + target: who, + request: request, + pending_response: tx, + } +} + /// Outcome of an incoming custom message. #[derive(Debug)] #[must_use] @@ -1490,9 +1526,10 @@ pub enum CustomMessageOutcome { /// It is the responsibility of the handler to ensure that a timeout exists. BlockRequest { target: PeerId, - request: message::BlockRequest, + request: crate::schema::v1::BlockRequest, pending_response: oneshot::Sender, crate::request_responses::RequestFailure>>, }, + // TODO: Maybe update? /// A new finality proof request must be emitted. /// Once you have the response, you must call `Protocol::on_finality_proof_response`. /// It is the responsibility of the handler to ensure that a timeout exists. @@ -1500,8 +1537,7 @@ pub enum CustomMessageOutcome { /// disconnect. This will inform the state machine that the request it has emitted is stale. FinalityProofRequest { target: PeerId, - block_hash: B::Hash, - request: Vec, + request: crate::schema::v1::finality::FinalityProofRequest, pending_response: oneshot::Sender, crate::request_responses::RequestFailure>>, }, /// Peer has a reported a new head of chain. @@ -1509,20 +1545,6 @@ pub enum CustomMessageOutcome { None, } -// TODO: Should this be inlined? -fn update_peer_request( - peers: &mut HashMap>, - who: &PeerId, - request: &mut message::BlockRequest, - rx: oneshot::Receiver, crate::request_responses::RequestFailure>>, -) { - if let Some(ref mut peer) = peers.get_mut(who) { - request.id = peer.next_request_id; - peer.next_request_id += 1; - peer.block_request = Some((request.clone(), rx)); - } -} - impl NetworkBehaviour for Protocol { type ProtocolsHandler = ::ProtocolsHandler; type OutEvent = CustomMessageOutcome; @@ -1648,41 +1670,15 @@ impl NetworkBehaviour for Protocol { } for (id, mut request) in self.sync.block_requests() { - let (tx, rx) = oneshot::channel(); - - update_peer_request(&mut self.context_data.peers, id, &mut request, rx); - - let event = CustomMessageOutcome::BlockRequest { - target: id.clone(), - request: request, - pending_response: tx, - }; + let event = prepare_block_request(&mut self.context_data.peers, id.clone(), request); self.pending_messages.push_back(event); } - for (id, mut r) in self.sync.justification_requests() { - let (tx, rx) = oneshot::channel(); - update_peer_request(&mut self.context_data.peers, &id, &mut r, rx); - let event = CustomMessageOutcome::BlockRequest { - target: id, - request: r, - pending_response: tx, - }; + for (id, mut request) in self.sync.justification_requests() { + let event = prepare_block_request(&mut self.context_data.peers, id, request); self.pending_messages.push_back(event); } - for (id, r) in self.sync.finality_proof_requests() { - let (tx, rx) = oneshot::channel(); - match self.context_data.peers.get_mut(&id) { - Some(peer) => { - peer.finality_request = Some((r.clone(), rx)) - }, - None => unimplemented!(), - } - let event = CustomMessageOutcome::FinalityProofRequest { - target: id, - block_hash: r.block, - request: r.request, - pending_response: tx, - }; + for (id, request) in self.sync.finality_proof_requests() { + let event = prepare_finality_request(&mut self.context_data.peers, id, request); self.pending_messages.push_back(event); } if let Poll::Ready(Some((tx_hash, result))) = self.pending_transactions.poll_next_unpin(cx) { From d5ea27e17971026fd018d4445a10e25af07462f0 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 11 Nov 2020 17:43:36 +0100 Subject: [PATCH 28/48] client/network/src/protocol: Return pending events after poll --- client/network/src/behaviour.rs | 1 - client/network/src/protocol.rs | 61 +++++++++++++++------------------ 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 445bacb876919..63208e95c283c 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -427,7 +427,6 @@ impl NetworkBehaviourEventProcess { - // TODO: Add everything here we need to capture the event in metrics. E.g. protocol. self.events.push_back(BehaviourOut::RequestFinished { peer, protocol, duration, result, }); diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index f391a6e9b0c5d..7f4d501930d85 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1596,68 +1596,57 @@ impl NetworkBehaviour for Protocol { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message)); } - // TODO: rework - // TODO: One could as well merge the block and finality loop. - let mut finished = Vec::new(); // Check for finished outgoing requests. + let mut finished_block_requests = Vec::new(); + let mut finished_finality_requests = Vec::new(); for (id, peer) in self.context_data.peers.iter_mut() { - if let Peer { block_request: Some((req, pending_response)), .. } = peer { + if let Peer { block_request: Some((_, pending_response)), .. } = peer { match pending_response.poll_unpin(cx) { Poll::Ready(Ok(Ok(resp))) => { let (req, _) = peer.block_request.take().unwrap(); let protobuf_response = match crate::schema::v1::BlockResponse::decode(&resp[..]) { Ok(proto) => proto, - Err(e) => { + Err(_e) => { todo!("log error"); } }; let id = id.clone(); - finished.push((id, req, protobuf_response)); + finished_block_requests.push((id, req, protobuf_response)); }, - Poll::Ready(Ok(Err(e))) => todo!("call on block request canceled"), - Poll::Ready(Err(e)) => todo!("handle"), - Poll::Pending => continue, + Poll::Ready(Ok(Err(_e))) => todo!("call on block request canceled"), + Poll::Ready(Err(_e)) => todo!("handle"), + Poll::Pending => {}, } } - } - for (id, req, protobuf_response) in finished { - let ev = self.on_block_response(id, req, protobuf_response); - // TODO: Is this correct? Should we check at the very end once more whether - // there have been any events addedd to pending_messages? - self.pending_messages.push_back(ev); - } - - // TODO: rework - let mut finished = Vec::new(); - // Check for finished outgoing requests. - for (id, peer) in self.context_data.peers.iter_mut() { - if let Peer { finality_request: Some((req, pending_response)), .. } = peer { + if let Peer { finality_request: Some((_, pending_response)), .. } = peer { match pending_response.poll_unpin(cx) { Poll::Ready(Ok(Ok(resp))) => { let (req, _) = peer.finality_request.take().unwrap(); let protobuf_response = match crate::schema::v1::finality::FinalityProofResponse::decode(&resp[..]) { Ok(proto) => proto, - Err(e) => { + Err(_e) => { todo!("log error"); } }; let id = id.clone(); - finished.push((id, req, protobuf_response)); + finished_finality_requests.push((id, req, protobuf_response)); }, - Poll::Ready(Ok(Err(e))) => todo!("call on block request canceled"), - Poll::Ready(Err(e)) => todo!("handle"), - Poll::Pending => continue, + Poll::Ready(Ok(Err(_e))) => todo!("call on block request canceled"), + Poll::Ready(Err(_e)) => todo!("handle"), + Poll::Pending => {}, } } } - for (id, req, protobuf_response) in finished { + for (id, req, protobuf_response) in finished_block_requests { + let ev = self.on_block_response(id, req, protobuf_response); + self.pending_messages.push_back(ev); + } + for (id, req, protobuf_response) in finished_finality_requests { let ev = self.on_finality_proof_response(id, req, protobuf_response.proof); - // TODO: Is this correct? Should we check at the very end once more whether - // there have been any events addedd to pending_messages? self.pending_messages.push_back(ev); } @@ -1812,11 +1801,15 @@ impl NetworkBehaviour for Protocol { } }; - if let CustomMessageOutcome::None = outcome { - Poll::Pending - } else { - Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome)) + if !matches!(CustomMessageOutcome::::None, outcome) { + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome)); + } + + if let Some(message) = self.pending_messages.pop_front() { + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message)); } + + Poll::Pending } fn inject_addr_reach_failure( From 006fd6e9e870506eb2e36989c9dc8816e1f32b45 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 16 Nov 2020 19:13:13 +0100 Subject: [PATCH 29/48] client/network: Improve error handling and documentation --- client/network/src/behaviour.rs | 33 +++++----- client/network/src/protocol.rs | 82 +++++++++++-------------- client/network/src/request_responses.rs | 72 ++++++++++++++++------ client/network/src/service.rs | 29 +-------- 4 files changed, 108 insertions(+), 108 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 63208e95c283c..8ff9d8e953a8f 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -17,8 +17,8 @@ use crate::{ config::{ProtocolId, Role}, light_client_handler, peer_info, request_responses, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, - protocol::{message::{self, Roles}, CustomMessageOutcome, NotificationsSink, Protocol}, - ObservedRole, DhtEvent, ExHashT, schema, + protocol::{message::Roles, CustomMessageOutcome, NotificationsSink, Protocol}, + ObservedRole, DhtEvent, ExHashT, }; use bytes::Bytes; @@ -69,9 +69,12 @@ pub struct Behaviour { #[behaviour(ignore)] role: Role, - // TODO: Document. + /// Protocol name used to send out block requests via + /// [`request_responses::RequestResponsesBehaviour`]. #[behaviour(ignore)] block_request_protocol_name: String, + /// Protocol name used to send out finality requests via + /// [`request_responses::RequestResponsesBehaviour`]. #[behaviour(ignore)] finality_request_protocol_name: String, } @@ -98,13 +101,17 @@ pub enum BehaviourOut { result: Result, }, - /// A request initiated using [`Behaviour::send_request`] has succeeded or failed. + /// A request has succeeded or failed. + /// + /// This event is generated for statistics purposes. RequestFinished { + /// Peer that we send a request to. peer: PeerId, /// Name of the protocol in question. protocol: Cow<'static, str>, - // TODO: Document. + /// Duration the request took. duration: Duration, + /// Result of the request. result: Result<(), RequestFailure>, }, @@ -167,13 +174,13 @@ impl Behaviour { local_public_key: PublicKey, light_client_handler: light_client_handler::LightClientHandler, disco_config: DiscoveryConfig, - // TODO: Document that these are all others apart from block and finality. - mut request_response_protocols: Vec, - - // TODO: Not so fancy, see definition. + // Block and finality request protocol config. block_request_protocol_config: request_responses::ProtocolConfig, finality_request_protocol_config: request_responses::ProtocolConfig, + // All remaining request protocol configs. + mut request_response_protocols: Vec, ) -> Result { + // Extract protocol name and add to `request_response_protocols`. let block_request_protocol_name = block_request_protocol_config.name.to_string(); let finality_request_protocol_name = finality_request_protocol_config.name.to_string(); request_response_protocols.push(block_request_protocol_config); @@ -231,11 +238,7 @@ impl Behaviour { self.peer_info.node(peer_id) } - // TODO Update comment. /// Initiates sending a request. - /// - /// An error is returned if we are not connected to the target peer of if the protocol doesn't - /// match one that has been registered. pub fn send_request( &mut self, target: &PeerId, @@ -353,8 +356,6 @@ Behaviour { "Failed to encode block request {:?}: {:?}", request, err ); - - // TODO: Should we notify protocol.rs or sync.rs? return } @@ -370,8 +371,6 @@ Behaviour { "Failed to encode finality proof request {:?}: {:?}", request, err, ); - - // TODO: Should we notify protocol.rs or sync.rs? return; } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 22be509f1544a..52d13c9b9c250 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -249,20 +249,12 @@ struct PacketStats { struct Peer { info: PeerInfo, /// Current block request, if any. Started by emitting [`CustomMessageOutcome::BlockRequest`]. - /// - // TODO: Update - /// [`libp2p::request_response::RequestId`] is [`Some`] once the block request is send off and - /// [`Protocol::on_block_request_started`] is called. block_request: Option<( message::BlockRequest, oneshot::Receiver, crate::request_responses::RequestFailure>>, )>, /// Current finality request, if any. Started by emitting /// [`CustomMessageOutcome::FinalityProofRequest`]. - /// - // TODO: Update - /// [`libp2p::request_response::RequestId`] is [`Some`] once the finality request is send off - /// and [`Protocol::on_finality_proof_request_started`] is called. finality_request: Option<( message::FinalityProofRequest, oneshot::Receiver, crate::request_responses::RequestFailure>>, @@ -679,7 +671,6 @@ impl Protocol { who: PeerId, request: message::BlockRequest, ) -> CustomMessageOutcome { - // TODO: Can we not inline this? prepare_block_request::(&mut self.context_data.peers, who, request) } @@ -756,7 +747,7 @@ impl Protocol { Ok(blocks) => blocks, Err(err) => { debug!(target: "sync", "Failed to decode block response from {}: {}", peer_id, err); - // TODO: Reduce reputation of peer? + self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); return CustomMessageOutcome::None; } }; @@ -810,7 +801,7 @@ impl Protocol { match self.sync.on_block_data(&peer_id, Some(request), block_response) { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), - Ok(sync::OnBlockData::Request(peer, mut req)) => { + Ok(sync::OnBlockData::Request(peer, req)) => { self.prepare_block_request(peer, req) } Err(sync::BadPeer(id, repu)) => { @@ -822,16 +813,6 @@ impl Protocol { } } - /// Must be called in response to a [`CustomMessageOutcome::BlockRequest`] if it has failed. - pub fn on_block_request_failed( - &mut self, - peer: &PeerId, - ) { - // TODO: This can not only happen due to timeouts. Should the `rep::TIMEOUT` be used? - self.peerset_handle.report_peer(peer.clone(), rep::TIMEOUT); - self.behaviour.disconnect_peer(peer); - } - /// Perform time based maintenance. /// /// > **Note**: This method normally doesn't have to be called except for testing purposes. @@ -925,7 +906,7 @@ impl Protocol { if info.roles.is_full() { match self.sync.new_peer(who.clone(), info.best_hash, info.best_number) { Ok(None) => (), - Ok(Some(mut req)) => { + Ok(Some(req)) => { let event = self.prepare_block_request(who.clone(), req); self.pending_messages.push_back(event); }, @@ -1257,7 +1238,7 @@ impl Protocol { Ok(sync::OnBlockData::Import(origin, blocks)) => { CustomMessageOutcome::BlockImport(origin, blocks) }, - Ok(sync::OnBlockData::Request(peer, mut req)) => { + Ok(sync::OnBlockData::Request(peer, req)) => { self.prepare_block_request(peer, req) } Err(sync::BadPeer(id, repu)) => { @@ -1317,7 +1298,7 @@ impl Protocol { ); for result in results { match result { - Ok((id, mut req)) => { + Ok((id, req)) => { self.pending_messages.push_back( prepare_block_request(&mut self.context_data.peers, id, req) ); @@ -1517,24 +1498,13 @@ pub enum CustomMessageOutcome { NotificationStreamClosed { remote: PeerId, protocols: Vec }, /// Messages have been received on one or more notifications protocols. NotificationsReceived { remote: PeerId, messages: Vec<(ConsensusEngineId, Bytes)> }, - // TODO: Update comment. /// A new block request must be emitted. - /// You must later call either [`Protocol::on_block_response`] or - /// [`Protocol::on_block_request_failed`]. - /// Each peer can only have one active request. If a request already exists for this peer, it - /// must be silently discarded. - /// It is the responsibility of the handler to ensure that a timeout exists. BlockRequest { target: PeerId, request: crate::schema::v1::BlockRequest, pending_response: oneshot::Sender, crate::request_responses::RequestFailure>>, }, - // TODO: Maybe update? /// A new finality proof request must be emitted. - /// Once you have the response, you must call `Protocol::on_finality_proof_response`. - /// It is the responsibility of the handler to ensure that a timeout exists. - /// If the request times out, or the peer responds in an invalid way, the peer has to be - /// disconnect. This will inform the state machine that the request it has emitted is stale. FinalityProofRequest { target: PeerId, request: crate::schema::v1::finality::FinalityProofRequest, @@ -1607,16 +1577,25 @@ impl NetworkBehaviour for Protocol { let protobuf_response = match crate::schema::v1::BlockResponse::decode(&resp[..]) { Ok(proto) => proto, - Err(_e) => { - todo!("log error"); + Err(e) => { + trace!(target: "sync", "Failed to decode block request to peer {:?}: {:?}.", id, e); + self.peerset_handle.report_peer(id.clone(), rep::BAD_MESSAGE); + self.behaviour.disconnect_peer(id); + continue; } }; - let id = id.clone(); - finished_block_requests.push((id, req, protobuf_response)); + finished_block_requests.push((id.clone(), req, protobuf_response)); + }, + // TODO: Match on the concrete error to know whether to reduce the peers reputation. + Poll::Ready(Ok(Err(e))) => { + peer.block_request.take(); + trace!(target: "sync", "Block request to peer {:?} failed: {:?}.", id, e); + }, + Poll::Ready(Err(e)) => { + peer.block_request.take(); + trace!(target: "sync", "Block request to peer {:?} failed: {:?}.", id, e); }, - Poll::Ready(Ok(Err(_e))) => todo!("call on block request canceled"), - Poll::Ready(Err(_e)) => todo!("handle"), Poll::Pending => {}, } } @@ -1628,15 +1607,24 @@ impl NetworkBehaviour for Protocol { let protobuf_response = match crate::schema::v1::finality::FinalityProofResponse::decode(&resp[..]) { Ok(proto) => proto, Err(_e) => { - todo!("log error"); + self.peerset_handle.report_peer(id.clone(), rep::BAD_MESSAGE); + self.behaviour.disconnect_peer(id); + continue; } }; let id = id.clone(); finished_finality_requests.push((id, req, protobuf_response)); }, - Poll::Ready(Ok(Err(_e))) => todo!("call on block request canceled"), - Poll::Ready(Err(_e)) => todo!("handle"), + // TODO: Match on the concrete error to know whether to reduce the peers reputation. + Poll::Ready(Ok(Err(e))) => { + peer.finality_request.take(); + trace!(target: "sync", "Finality request to peer {:?} failed: {:?}.", id, e); + }, + Poll::Ready(Err(e)) => { + peer.finality_request.take(); + trace!(target: "sync", "Finality request to peer {:?} failed: {:?}.", id, e); + }, Poll::Pending => {}, } } @@ -1658,11 +1646,11 @@ impl NetworkBehaviour for Protocol { self.propagate_transactions(); } - for (id, mut request) in self.sync.block_requests() { + for (id, request) in self.sync.block_requests() { let event = prepare_block_request(&mut self.context_data.peers, id.clone(), request); self.pending_messages.push_back(event); } - for (id, mut request) in self.sync.justification_requests() { + for (id, request) in self.sync.justification_requests() { let event = prepare_block_request(&mut self.context_data.peers, id, request); self.pending_messages.push_back(event); } @@ -1801,7 +1789,7 @@ impl NetworkBehaviour for Protocol { } }; - if !matches!(CustomMessageOutcome::::None, outcome) { + if !matches!(outcome, CustomMessageOutcome::::None) { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(outcome)); } diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 50d635c1e5c8f..5aaa0fabef159 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -137,12 +137,13 @@ pub enum Event { /// /// This event is generated for statistics purposes. RequestFinished { + /// Peer that we send a request to. peer: PeerId, /// Name of the protocol in question. protocol: Cow<'static, str>, - // TODO: Document. + /// Duration the request took. duration: Duration, - // TODO: Document. + /// Result of the request. result: Result<(), RequestFailure> }, } @@ -157,7 +158,7 @@ pub struct RequestResponsesBehaviour { (RequestResponse, Option>) >, - // TODO: Document + /// Pending requests, passed down to a [`RequestResponse`] behaviour, awaiting a reply. pending_requests: HashMap, RequestFailure>>)>, /// Whenever an incoming request arrives, a `Future` is added to this list and will yield the @@ -231,10 +232,24 @@ impl RequestResponsesBehaviour { let request_id = protocol.send_request(target, request); self.pending_requests.insert(request_id, (Instant::now(), pending_response)); } else { - pending_response.send(Err(RequestFailure::NotConnected)); + if pending_response.send(Err(RequestFailure::NotConnected)).is_err() { + log::debug!( + target: "sub-libp2p", + "Not connected to peer {:?}. At the same time local \ + node is no longer interested in the result.", + target, + ); + }; } } else { - pending_response.send(Err(RequestFailure::UnknownProtocol)); + if pending_response.send(Err(RequestFailure::UnknownProtocol)).is_err() { + log::debug!( + target: "sub-libp2p", + "Unknown protocol {:?}. At the same time local \ + node is no longer interested in the result.", + protocol, + ); + }; } } } @@ -468,25 +483,31 @@ impl NetworkBehaviour for RequestResponsesBehaviour { }, .. } => { - // TODO: Rework. - let started = match self.pending_requests.remove(&request_id) { + let (started, delivered) = match self.pending_requests.remove(&request_id) { Some((started, pending_response)) => { - pending_response.send( + let delivered = pending_response.send( response.map_err(|()| RequestFailure::Refused), - ).unwrap(); - started + ).map_err(|_| RequestFailure::Obsolete); + (started, delivered) } None => { - todo!("handle this"); + log::warn!( + target: "sub-libp2p", + "Received `RequestResponseEvent::Message` with unexpected request id {:?}", + request_id, + ); + debug_assert!(false); + continue; } }; + let out = Event::RequestFinished { peer, protocol: protocol.clone(), duration: started.elapsed(), - // TODO: Don't send OK when the error was mapped above as refused. - result: Ok(()), + result: delivered, }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); } @@ -497,7 +518,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour { error, .. } => { - // TODO: Remove hack + // TODO: Remove hack by deriving `Clone` for `OutboundFailure`. let error_clone = match &error { OutboundFailure::ConnectionClosed => OutboundFailure::ConnectionClosed, OutboundFailure::DialFailure => OutboundFailure::DialFailure, @@ -505,16 +526,28 @@ impl NetworkBehaviour for RequestResponsesBehaviour { OutboundFailure::UnsupportedProtocols => OutboundFailure::UnsupportedProtocols, }; - // TODO: Rework. let started = match self.pending_requests.remove(&request_id) { Some((started, pending_response)) => { - pending_response.send( + if pending_response.send( Err(RequestFailure::Network(error)), - ).unwrap(); + ).is_err() { + log::debug!( + target: "sub-libp2p", + "Request with id {:?} failed. At the same time local \ + node is no longer interested in the result.", + request_id, + ); + } started } None => { - todo!("handle this"); + log::warn!( + target: "sub-libp2p", + "Received `RequestResponseEvent::Message` with unexpected request id {:?}", + request_id, + ); + debug_assert!(false); + continue; } }; @@ -524,6 +557,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour { duration: started.elapsed(), result: Err(RequestFailure::Network(error_clone)), }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); } @@ -562,6 +596,8 @@ pub enum RequestFailure { /// Remote has closed the substream before answering, thereby signaling that it considers the /// request as valid, but refused to answer it. Refused, + /// The remote replied, but the local node is no longer interested in the response. + Obsolete, /// Problem on the network. #[display(fmt = "Problem on the network")] Network(#[error(ignore)] OutboundFailure), diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 03fd5f326da4c..e8f0db534017b 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -72,7 +72,6 @@ use std::{ }, task::Poll, }; -use wasm_timer::Instant; pub use behaviour::{ResponseFailure, InboundFailure, RequestFailure, OutboundFailure}; @@ -308,9 +307,9 @@ impl NetworkWorker { local_public, light_client_handler, discovery_config, - params.network_config.request_response_protocols, params.block_request_protocol_config, params.finality_request_protocol_config, + params.network_config.request_response_protocols, ); match result { @@ -1308,29 +1307,6 @@ impl Future for NetworkWorker { this.event_streams.push(sender), ServiceToWorkerMsg::Request { target, protocol, request, pending_response } => { this.network_service.send_request(&target, &protocol, request, pending_response); - // TODO: Should this be moved to NetworkService? - // { - // Ok(request_id) => { - // // TODO: Do this on a request started event to also track finality and block request events. - // if let Some(metrics) = this.metrics.as_ref() { - // metrics.requests_out_started_total - // .with_label_values(&[&protocol]) - // .inc(); - // } - // this.pending_requests.insert( - // request_id, - // (pending_response, Instant::now(), protocol.to_string()) - // ); - // }, - // Err(behaviour::SendRequestError::NotConnected) => { - // let err = RequestFailure::Network(OutboundFailure::ConnectionClosed); - // let _ = pending_response.send(Err(err)); - // }, - // Err(behaviour::SendRequestError::UnknownProtocol) => { - // let err = RequestFailure::Network(OutboundFailure::UnsupportedProtocols); - // let _ = pending_response.send(Err(err)); - // }, - // } }, ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, protocol_name } => { this.network_service @@ -1405,7 +1381,7 @@ impl Future for NetworkWorker { } } }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { peer, protocol, duration, result })) => { + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { protocol, duration, result, .. })) => { if let Some(metrics) = this.metrics.as_ref() { match result { Ok(_) => { @@ -1418,6 +1394,7 @@ impl Future for NetworkWorker { RequestFailure::NotConnected => "not-connected", RequestFailure::UnknownProtocol => "unknown-protocol", RequestFailure::Refused => "refused", + RequestFailure::Obsolete => "obsolete", RequestFailure::Network(OutboundFailure::DialFailure) => "dial-failure", RequestFailure::Network(OutboundFailure::Timeout) => From 1bca3478d819f90cd4695e4a1d0d225be2caa40e Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 18 Nov 2020 13:41:35 +0100 Subject: [PATCH 30/48] client/network/behaviour: Remove outdated error types --- client/network/src/behaviour.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 8ff9d8e953a8f..63c27f4723319 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -303,18 +303,6 @@ impl Behaviour { } } -#[derive(derive_more::Display, derive_more::From)] -enum OnBlockResponseError { - Request(request_responses::RequestFailure), - Decode(prost::DecodeError), -} - -#[derive(derive_more::Display, derive_more::From)] -enum OnFinalityResponseError { - Request(request_responses::RequestFailure), - Decode(prost::DecodeError), -} - fn reported_roles_to_observed_role(local_role: &Role, remote: &PeerId, roles: Roles) -> ObservedRole { if roles.is_authority() { match local_role { From 2797595d45bc253a6f8eaf6043a2a5bf2ea0827c Mon Sep 17 00:00:00 2001 From: Addie Wagenknecht Date: Mon, 23 Nov 2020 16:16:38 +0100 Subject: [PATCH 31/48] Update client/network/src/block_request_handler.rs Co-authored-by: Ashley --- client/network/src/block_request_handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 4540009900c2d..3c49e29c9a9f9 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -67,7 +67,7 @@ pub struct BlockRequestHandler { impl BlockRequestHandler { /// Create a new [`BlockRequestHandler`]. pub fn new(protocol_id: ProtocolId, client: Arc>) -> (Self, ProtocolConfig) { - // TODO: Likeley we want to allow more than 0 buffered requests. Rethink this value. + // TODO: Likely we want to allow more than 0 buffered requests. Rethink this value. let (tx, request_receiver) = mpsc::channel(0); let mut protocol_config = generate_protocol_config(protocol_id); From aaa981c5b26ba23d74cd0e16586a74eca3a71a57 Mon Sep 17 00:00:00 2001 From: Addie Wagenknecht Date: Mon, 23 Nov 2020 16:16:56 +0100 Subject: [PATCH 32/48] Update client/network/src/finality_request_handler.rs Co-authored-by: Ashley --- client/network/src/finality_request_handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/network/src/finality_request_handler.rs b/client/network/src/finality_request_handler.rs index 7d66c8a5edeae..3b825504cdeb3 100644 --- a/client/network/src/finality_request_handler.rs +++ b/client/network/src/finality_request_handler.rs @@ -65,7 +65,7 @@ impl FinalityRequestHandler { protocol_id: ProtocolId, proof_provider: Arc>, ) -> (Self, ProtocolConfig){ - // TODO: Likeley we want to allow more than 0 buffered requests. Rethink this value. + // TODO: Likely we want to allow more than 0 buffered requests. Rethink this value. let (tx, rx) = mpsc::channel(0); let mut protocol_config = generate_protocol_config(protocol_id); From c19885ec8e1f5a93671778316b43ef4336b53dcd Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 10:50:03 +0100 Subject: [PATCH 33/48] client/network/protocol: Reduce reputation on timeout --- client/network/src/gossip/tests.rs | 6 ++++-- client/network/src/protocol.rs | 25 ++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index 257d78dfc0300..c7bca1ae3842a 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -16,7 +16,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{config, gossip::QueuedSender, Event, NetworkService, NetworkWorker}; +use crate::block_request_handler::BlockRequestHandler; +use crate::gossip::QueuedSender; +use crate::{config, Event, NetworkService, NetworkWorker}; use futures::prelude::*; use sp_runtime::traits::{Block as BlockT, Header as _}; @@ -105,7 +107,7 @@ fn build_test_full_node(network_config: config::NetworkConfiguration) chain: client.clone(), on_demand: None, transaction_pool: Arc::new(crate::config::EmptyTransactionPool), - protocol_id: config::ProtocolId::from("/test-protocol-name"), + protocol_id, import_queue, block_announce_validator: Box::new( sp_consensus::block_validation::DefaultBlockAnnounceValidator, diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index c69cf6f48b91b..23d8401d08ba9 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -21,6 +21,7 @@ use crate::{ chain::Client, config::{ProtocolId, TransactionPool, TransactionImportFuture, TransactionImport}, error, + request_responses::RequestFailure, utils::{interval, LruHashSet}, }; @@ -28,10 +29,11 @@ use bytes::{Bytes, BytesMut}; use codec::{Decode, DecodeAll, Encode}; use futures::{channel::oneshot, prelude::*, stream::FuturesUnordered}; use generic_proto::{GenericProto, GenericProtoOut}; -use libp2p::{Multiaddr, PeerId}; use libp2p::core::{ConnectedPoint, connection::{ConnectionId, ListenerId}}; -use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; +use libp2p::request_response::OutboundFailure; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; +use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; +use libp2p::{Multiaddr, PeerId}; use log::{log, Level, trace, debug, warn, error}; use message::{BlockAnnounce, Message}; use message::generic::{Message as GenericMessage, Roles}; @@ -241,7 +243,7 @@ struct Peer { /// Current block request, if any. Started by emitting [`CustomMessageOutcome::BlockRequest`]. block_request: Option<( message::BlockRequest, - oneshot::Receiver, crate::request_responses::RequestFailure>>, + oneshot::Receiver, RequestFailure>>, )>, /// Holds a set of transactions known to this peer. known_transactions: LruHashSet, @@ -1373,7 +1375,7 @@ pub enum CustomMessageOutcome { BlockRequest { target: PeerId, request: crate::schema::v1::BlockRequest, - pending_response: oneshot::Sender, crate::request_responses::RequestFailure>>, + pending_response: oneshot::Sender, RequestFailure>>, }, /// Peer has a reported a new head of chain. PeerNewBest(PeerId, NumberFor), @@ -1451,10 +1453,23 @@ impl NetworkBehaviour for Protocol { finished_block_requests.push((id.clone(), req, protobuf_response)); }, - // TODO: Match on the concrete error to know whether to reduce the peers reputation. Poll::Ready(Ok(Err(e))) => { peer.block_request.take(); trace!(target: "sync", "Block request to peer {:?} failed: {:?}.", id, e); + + match e { + RequestFailure::Network(OutboundFailure::Timeout) => { + self.peerset_handle.report_peer(id.clone(), rep::BAD_RESPONSE); + } + RequestFailure::Obsolete => { + debug_assert!( + false, + "Can not receive `RequestFailure::Obsolete` after dropping the \ + response reciver.", + ); + } + _ => {}, + } }, Poll::Ready(Err(e)) => { peer.block_request.take(); From 81cadfc3abb350ae9b164b27b21e74ae2449fc25 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 11:06:49 +0100 Subject: [PATCH 34/48] client/network/protocol: Refine reputation changes --- client/network/src/protocol.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 23d8401d08ba9..aaaed93006c0f 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1458,8 +1458,14 @@ impl NetworkBehaviour for Protocol { trace!(target: "sync", "Block request to peer {:?} failed: {:?}.", id, e); match e { + RequestFailure::UnknownProtocol => { + debug_assert!(false, "Block request protocol should always be known."); + } RequestFailure::Network(OutboundFailure::Timeout) => { - self.peerset_handle.report_peer(id.clone(), rep::BAD_RESPONSE); + self.peerset_handle.report_peer(id.clone(), rep::TIMEOUT); + } + RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => { + self.peerset_handle.report_peer(id.clone(), rep::BAD_PROTOCOL); } RequestFailure::Obsolete => { debug_assert!( From 923611351b6444785c8fadb9960749eda35a0b1d Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 14:16:35 +0100 Subject: [PATCH 35/48] client/network/block_request_handler: Set and explain queue length --- client/network/src/block_request_handler.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 3c49e29c9a9f9..c88be52ecf0de 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -67,8 +67,14 @@ pub struct BlockRequestHandler { impl BlockRequestHandler { /// Create a new [`BlockRequestHandler`]. pub fn new(protocol_id: ProtocolId, client: Arc>) -> (Self, ProtocolConfig) { - // TODO: Likely we want to allow more than 0 buffered requests. Rethink this value. - let (tx, request_receiver) = mpsc::channel(0); + // Rate of arrival multiplied with the waiting time in the queue equals the queue length. + // + // An average Polkadot sentry node serves less than 5 requests per second. The 95th percentile + // serving a request is less than 2 second. Thus one would estimate the queue length to be + // below 10. + // + // Choosing 20 as the queue length to give some additional buffer. + let (tx, request_receiver) = mpsc::channel(20); let mut protocol_config = generate_protocol_config(protocol_id); protocol_config.inbound_queue = Some(tx); From 4c018f174dba29ed11b3231e0100cb44cbc86615 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 14:31:44 +0100 Subject: [PATCH 36/48] client/service: Deny block requests when light client --- client/network/src/protocol/message.rs | 1 - client/network/src/request_responses.rs | 1 - client/network/src/service/metrics.rs | 9 --------- client/network/test/src/lib.rs | 2 +- client/service/src/builder.rs | 13 +++++++++---- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index e814d9bfa7d66..4213d56bbf022 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -388,7 +388,6 @@ pub mod generic { /// Request block data from a peer. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct BlockRequest { - // TODO: Is this used anywhere? /// Unique request id. pub id: RequestId, /// Bits of block data to request. diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 9d5adc5077f1a..c2a7f103d8919 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -50,7 +50,6 @@ use libp2p::{ }; use std::{ borrow::Cow, collections::{hash_map::Entry, HashMap}, convert::TryFrom as _, io, iter, - // TODO: Should we use wasm_timer instant here? pin::Pin, task::{Context, Poll}, time::{Duration, Instant}, }; diff --git a/client/network/src/service/metrics.rs b/client/network/src/service/metrics.rs index dd40ee1da0dfb..77ca091f3edbb 100644 --- a/client/network/src/service/metrics.rs +++ b/client/network/src/service/metrics.rs @@ -78,8 +78,6 @@ pub struct Metrics { pub requests_in_success_total: HistogramVec, pub requests_out_failure_total: CounterVec, pub requests_out_success_total: HistogramVec, - // TODO: Is this still used anywhere? - pub requests_out_started_total: CounterVec, } impl Metrics { @@ -256,13 +254,6 @@ impl Metrics { }, &["protocol"] )?, registry)?, - requests_out_started_total: prometheus::register(CounterVec::new( - Opts::new( - "sub_libp2p_requests_out_started_total", - "Total number of requests emitted" - ), - &["protocol"] - )?, registry)?, }) } } diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 26d3f3e340aca..050f96d028cdb 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -781,7 +781,7 @@ pub trait TestNetFactory: Sized { }); } - // TODO: Not ideal. Can we do better? + /// Used to spawn background tasks, e.g. the block request protocol handler. fn spawn_task(&self, f: BoxFuture<'static, ()>) { async_std::task::spawn(f); } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index a12ce460d9a64..916d4c4fbaa19 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -883,10 +883,15 @@ pub fn build_network( }; let block_request_protocol_config = { - // TODO: Only do this if we are not a light client. - let (handler, protocol_config) = sc_network::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); - spawn_handle.spawn("block_request_handler", handler.run()); - protocol_config + if config.role == Role::Light { + // Allow outgoing requests but deny incoming requests. + sc_network::block_request_handler::generate_protocol_config(protocol_id.clone()) + } else { + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = sc_network::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + spawn_handle.spawn("block_request_handler", handler.run()); + protocol_config + } }; let network_params = sc_network::config::Params { From 07c0d2f37f8bcc93ef7b9de9b2619cb972b66f82 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 15:24:54 +0100 Subject: [PATCH 37/48] client/service: Fix role matching --- client/network/src/request_responses.rs | 2 ++ client/service/src/builder.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index c2a7f103d8919..b2c11fcaf47dd 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -121,6 +121,8 @@ pub enum Event { /// A remote sent a request and either we have successfully answered it or an error happened. /// /// This event is generated for statistics purposes. + // + // TODO: This is currently only emitted on failure. Also emit on success. InboundRequest { /// Peer which has emitted the request. peer: PeerId, diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 916d4c4fbaa19..3dccc145180f0 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -883,7 +883,7 @@ pub fn build_network( }; let block_request_protocol_config = { - if config.role == Role::Light { + if matches!(config.role, Role::Light) { // Allow outgoing requests but deny incoming requests. sc_network::block_request_handler::generate_protocol_config(protocol_id.clone()) } else { From 8c8bcb25014c0c8305321c7c68ab0db99beb49d3 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 15:36:03 +0100 Subject: [PATCH 38/48] client: Enforce line width --- client/network/src/gossip/tests.rs | 5 ++++- client/network/src/protocol.rs | 26 ++++++++++++++------------ client/network/src/service.rs | 4 +++- client/network/src/service/tests.rs | 6 +++++- client/network/test/src/lib.rs | 6 ++++-- client/service/src/builder.rs | 8 ++++++-- 6 files changed, 36 insertions(+), 19 deletions(-) diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index c7bca1ae3842a..63127f1c6cb28 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -95,7 +95,10 @@ fn build_test_full_node(network_config: config::NetworkConfiguration) let protocol_id = config::ProtocolId::from("/test-protocol-name"); let block_request_protocol_config = { - let (handler, protocol_config) = crate::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + let (handler, protocol_config) = BlockRequestHandler::new( + protocol_id.clone(), + client.clone(), + ); async_std::task::spawn(handler.run().boxed()); protocol_config }; diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index aaaed93006c0f..f84b6b3a7cb71 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -736,18 +736,20 @@ impl Protocol { } } else { // Validate fields against the request. - if request.fields.contains(message::BlockAttributes::HEADER) && block_response.blocks.iter().any(|b| b.header.is_none()) { - self.behaviour.disconnect_peer(&peer_id); - self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); - trace!(target: "sync", "Missing header for a block"); - return CustomMessageOutcome::None - } - if request.fields.contains(message::BlockAttributes::BODY) && block_response.blocks.iter().any(|b| b.body.is_none()) { - self.behaviour.disconnect_peer(&peer_id); - self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); - trace!(target: "sync", "Missing body for a block"); - return CustomMessageOutcome::None - } + if request.fields.contains(message::BlockAttributes::HEADER) + && block_response.blocks.iter().any(|b| b.header.is_none()) { + self.behaviour.disconnect_peer(&peer_id); + self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); + trace!(target: "sync", "Missing header for a block"); + return CustomMessageOutcome::None + } + if request.fields.contains(message::BlockAttributes::BODY) + && block_response.blocks.iter().any(|b| b.body.is_none()) { + self.behaviour.disconnect_peer(&peer_id); + self.peerset_handle.report_peer(peer_id, rep::BAD_RESPONSE); + trace!(target: "sync", "Missing body for a block"); + return CustomMessageOutcome::None + } match self.sync.on_block_data(&peer_id, Some(request), block_response) { Ok(sync::OnBlockData::Import(origin, blocks)) => diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 72a383b2d6f32..3921a8b45c57d 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1345,7 +1345,9 @@ impl Future for NetworkWorker { } } }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { protocol, duration, result, .. })) => { + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { + protocol, duration, result, .. + })) => { if let Some(metrics) = this.metrics.as_ref() { match result { Ok(_) => { diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 1c0685b844957..0ccdc984c71f4 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . use crate::{config, Event, NetworkService, NetworkWorker}; +use crate::block_request_handler::BlockRequestHandler; use libp2p::PeerId; use futures::prelude::*; @@ -94,7 +95,10 @@ fn build_test_full_node(config: config::NetworkConfiguration) let protocol_id = config::ProtocolId::from("/test-protocol-name"); let block_request_protocol_config = { - let (handler, protocol_config) = crate::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + let (handler, protocol_config) = BlockRequestHandler::new( + protocol_id.clone(), + client.clone(), + ); async_std::task::spawn(handler.run().boxed()); protocol_config }; diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 050f96d028cdb..58bf2ed369d73 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -29,7 +29,7 @@ use std::{ use libp2p::build_multiaddr; use log::trace; -use sc_network::block_request_handler::BlockRequestHandler; +use sc_network::block_request_handler::{self, BlockRequestHandler}; use sp_blockchain::{ HeaderBackend, Result as ClientResult, well_known_cache_keys::{self, Id as CacheKeyId}, @@ -743,7 +743,9 @@ pub trait TestNetFactory: Sized { let protocol_id = ProtocolId::from("test-protocol-name"); // Add block request handler. - let block_request_protocol_config = sc_network::block_request_handler::generate_protocol_config(protocol_id.clone()); + let block_request_protocol_config = block_request_handler::generate_protocol_config( + protocol_id.clone(), + ); let network = NetworkWorker::new(sc_network::config::Params { role: Role::Light, diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 3dccc145180f0..e63def0e8e165 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -43,6 +43,7 @@ use sc_keystore::LocalKeystore; use log::{info, warn}; use sc_network::config::{Role, OnDemand}; use sc_network::NetworkService; +use sc_network::block_request_handler::{self, BlockRequestHandler}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ Block as BlockT, SaturatedConversion, HashFor, Zero, BlockIdTo, @@ -885,10 +886,13 @@ pub fn build_network( let block_request_protocol_config = { if matches!(config.role, Role::Light) { // Allow outgoing requests but deny incoming requests. - sc_network::block_request_handler::generate_protocol_config(protocol_id.clone()) + block_request_handler::generate_protocol_config(protocol_id.clone()) } else { // Allow both outgoing and incoming requests. - let (handler, protocol_config) = sc_network::block_request_handler::BlockRequestHandler::new(protocol_id.clone(), client.clone()); + let (handler, protocol_config) = BlockRequestHandler::new( + protocol_id.clone(), + client.clone(), + ); spawn_handle.spawn("block_request_handler", handler.run()); protocol_config } From 9481e6dc41c9ee21f708b1c85ceca9365dc2d5ec Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 25 Nov 2020 15:54:37 +0100 Subject: [PATCH 39/48] client/network/request_responses: Fix unit tests --- client/network/src/request_responses.rs | 63 +++++++++++++------------ 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index b2c11fcaf47dd..cc077178ed598 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -742,7 +742,7 @@ impl RequestResponseCodec for GenericCodec { #[cfg(test)] mod tests { - use futures::{channel::mpsc, prelude::*}; + use futures::{channel::{mpsc, oneshot}, prelude::*}; use libp2p::identity::Keypair; use libp2p::Multiaddr; use libp2p::core::upgrade; @@ -753,7 +753,7 @@ mod tests { #[test] fn basic_request_response_works() { - let protocol_name = "/test/req-rep/1"; + let protocol_name = "/test/req-resp/1"; // Build swarms whose behaviour is `RequestResponsesBehaviour`. let mut swarms = (0..2) @@ -826,39 +826,38 @@ mod tests { // Remove and run the remaining swarm. let (mut swarm, _) = swarms.remove(0); async_std::task::block_on(async move { - let mut sent_request_id = None; + let mut response_receiver = None; loop { match swarm.next_event().await { SwarmEvent::ConnectionEstablished { peer_id, .. } => { - let id = swarm.send_request( + let (sender, receiver) = oneshot::channel(); + swarm.send_request( &peer_id, protocol_name, - b"this is a request".to_vec() - ).unwrap(); - assert!(sent_request_id.is_none()); - sent_request_id = Some(id); + b"this is a request".to_vec(), + sender, + ); + assert!(response_receiver.is_none()); + response_receiver = Some(receiver); } SwarmEvent::Behaviour(super::Event::RequestFinished { - peer: _, - protocol: _, - request_id, - result, + result, .. }) => { - assert_eq!(Some(request_id), sent_request_id); - let result = result.unwrap(); - assert_eq!(result, b"this is a response"); + assert!(result.is_ok()); break; } _ => {} } } + + assert_eq!(response_receiver.unwrap().await.unwrap().unwrap(), b"this is a response"); }); } #[test] fn max_response_size_exceeded() { - let protocol_name = "/test/req-rep/1"; + let protocol_name = "/test/req-resp/1"; // Build swarms whose behaviour is `RequestResponsesBehaviour`. let mut swarms = (0..2) @@ -931,35 +930,37 @@ mod tests { // Remove and run the remaining swarm. let (mut swarm, _) = swarms.remove(0); async_std::task::block_on(async move { - let mut sent_request_id = None; + let mut response_receiver = None; loop { match swarm.next_event().await { SwarmEvent::ConnectionEstablished { peer_id, .. } => { - let id = swarm.send_request( + let (sender, receiver) = oneshot::channel(); + swarm.send_request( &peer_id, protocol_name, - b"this is a request".to_vec() - ).unwrap(); - assert!(sent_request_id.is_none()); - sent_request_id = Some(id); + b"this is a request".to_vec(), + sender, + ); + assert!(response_receiver.is_none()); + response_receiver = Some(receiver); } SwarmEvent::Behaviour(super::Event::RequestFinished { - peer: _, - protocol: _, - request_id, - result, + result, .. }) => { - assert_eq!(Some(request_id), sent_request_id); - match result { - Err(super::RequestFailure::Network(super::OutboundFailure::ConnectionClosed)) => {}, - _ => panic!() - } + assert!(result.is_err()); break; } _ => {} } } + + ; + + match response_receiver.unwrap().await.unwrap().unwrap_err() { + super::RequestFailure::Network(super::OutboundFailure::ConnectionClosed) => {}, + _ => panic!() + } }); } } From 707c1bd6a6e00e79e965a4799e04775f06e8bf82 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Fri, 27 Nov 2020 14:21:29 +0100 Subject: [PATCH 40/48] client/network: Expose time to build response via metrics --- client/network/src/request_responses.rs | 43 ++++++++++++++++++------- client/network/src/service.rs | 2 +- client/network/src/service/metrics.rs | 11 ++++--- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index cc077178ed598..36f7836f35159 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -121,15 +121,18 @@ pub enum Event { /// A remote sent a request and either we have successfully answered it or an error happened. /// /// This event is generated for statistics purposes. - // - // TODO: This is currently only emitted on failure. Also emit on success. + /// + /// Note: Multiple [`InboundRequest`] events can be emitted for the same inbound request. E.g. + /// when response building succeeds, but sending the response later on fails. In that case there + /// will be one [`InboundRequest`] with the time it took to build the response and one + /// [`InboundRequest`] with the reason why later on sending the response failed. InboundRequest { /// Peer which has emitted the request. peer: PeerId, /// Name of the protocol in question. protocol: Cow<'static, str>, /// If `Ok`, contains the time elapsed between when we received the request and when we - /// sent back the response. If `Err`, the error that happened. + /// started sending the response. If `Err`, the error that happened. result: Result, }, @@ -163,15 +166,16 @@ pub struct RequestResponsesBehaviour { pending_requests: HashMap, RequestFailure>>)>, /// Whenever an incoming request arrives, a `Future` is added to this list and will yield the - /// response to send back to the remote. + /// start time and the response to send back to the remote. pending_responses: stream::FuturesUnordered< - Pin + Send>> + Pin + Send>> >, } /// Generated by the response builder and waiting to be processed. enum RequestProcessingOutcome { Response { + peer: PeerId, protocol: Cow<'static, str>, inner_channel: ResponseChannel, ()>>, response: Vec, @@ -378,13 +382,22 @@ impl NetworkBehaviour for RequestResponsesBehaviour { > { 'poll_all: loop { // Poll to see if any response is ready to be sent back. - while let Poll::Ready(Some(result)) = self.pending_responses.poll_next_unpin(cx) { + while let Poll::Ready(Some((start, result))) = self.pending_responses.poll_next_unpin(cx) { match result { RequestProcessingOutcome::Response { - protocol, inner_channel, response + peer, protocol: protocol_name, inner_channel, response } => { - if let Some((protocol, _)) = self.protocols.get_mut(&*protocol) { + if let Some((protocol, _)) = self.protocols.get_mut(&*protocol_name) { protocol.send_response(inner_channel, Ok(response)); + + let out = Event::InboundRequest { + peer, + protocol: protocol_name, + result: Ok(start.elapsed()), + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); + } else { + debug_assert!(false); } } RequestProcessingOutcome::Busy { peer, protocol } => { @@ -442,6 +455,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour { peer, message: RequestResponseMessage::Request { request, channel, .. }, } => { + let start = Instant::now(); let (tx, rx) = oneshot::channel(); // Submit the request to the "response builder" passed by the user at @@ -454,6 +468,8 @@ impl NetworkBehaviour for RequestResponsesBehaviour { payload: request, pending_response: tx, }); + } else { + debug_assert!(false, "Received message on outbound-only protocol."); } let protocol = protocol.clone(); @@ -461,11 +477,14 @@ impl NetworkBehaviour for RequestResponsesBehaviour { // The `tx` created above can be dropped if we are not capable of // processing this request, which is reflected as a "Busy" error. if let Ok(response) = rx.await { - RequestProcessingOutcome::Response { - protocol, inner_channel: channel, response - } + let outcome = RequestProcessingOutcome::Response { + peer, protocol, inner_channel: channel, response + }; + (start, outcome) + } else { - RequestProcessingOutcome::Busy { peer, protocol } + let outcome = RequestProcessingOutcome::Busy { peer, protocol }; + (start, outcome) } })); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 3921a8b45c57d..ae25c0371714e 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1324,7 +1324,7 @@ impl Future for NetworkWorker { if let Some(metrics) = this.metrics.as_ref() { match result { Ok(serve_time) => { - metrics.requests_in_success_total + metrics.requests_in_duration_building_response .with_label_values(&[&protocol]) .observe(serve_time.as_secs_f64()); } diff --git a/client/network/src/service/metrics.rs b/client/network/src/service/metrics.rs index 77ca091f3edbb..859ba82cc0243 100644 --- a/client/network/src/service/metrics.rs +++ b/client/network/src/service/metrics.rs @@ -75,7 +75,7 @@ pub struct Metrics { pub pending_connections: Gauge, pub pending_connections_errors_total: CounterVec, pub requests_in_failure_total: CounterVec, - pub requests_in_success_total: HistogramVec, + pub requests_in_duration_building_response: HistogramVec, pub requests_out_failure_total: CounterVec, pub requests_out_success_total: HistogramVec, } @@ -225,11 +225,12 @@ impl Metrics { ), &["protocol", "reason"] )?, registry)?, - requests_in_success_total: prometheus::register(HistogramVec::new( + requests_in_duration_building_response: prometheus::register(HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "sub_libp2p_requests_in_success_total", - "Total number of requests received and answered" + "sub_libp2p_requests_in_duration_building_response", + "For incoming requests, time between receiving the request and starting to \ + send the response" ), buckets: prometheus::exponential_buckets(0.001, 2.0, 16) .expect("parameters are always valid values; qed"), @@ -247,7 +248,7 @@ impl Metrics { HistogramOpts { common_opts: Opts::new( "sub_libp2p_requests_out_success_total", - "For successful requests, time between a request's start and finish" + "For successful outgoing requests, time between a request's start and finish" ), buckets: prometheus::exponential_buckets(0.001, 2.0, 16) .expect("parameters are always valid values; qed"), From df5907ff0023ccffa69c1183f36807be291c453f Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 30 Nov 2020 15:32:43 +0100 Subject: [PATCH 41/48] client/network/request_responses: Fix early connection closed error --- client/network/src/request_responses.rs | 41 +++++++++++++------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index e4d51994b10b1..862fb626418a8 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -761,7 +761,10 @@ impl RequestResponseCodec for GenericCodec { #[cfg(test)] mod tests { - use futures::{channel::{mpsc, oneshot}, prelude::*}; + use futures::channel::{mpsc, oneshot}; + use futures::executor::LocalPool; + use futures::prelude::*; + use futures::task::Spawn; use libp2p::identity::Keypair; use libp2p::Multiaddr; use libp2p::core::upgrade; @@ -773,6 +776,7 @@ mod tests { #[test] fn basic_request_response_works() { let protocol_name = "/test/req-resp/1"; + let mut pool = LocalPool::new(); // Build swarms whose behaviour is `RequestResponsesBehaviour`. let mut swarms = (0..2) @@ -800,12 +804,12 @@ mod tests { inbound_queue: Some(tx), })).unwrap(); - async_std::task::spawn(async move { + pool.spawner().spawn_obj(async move { while let Some(rq) = rx.next().await { assert_eq!(rq.payload, b"this is a request"); let _ = rq.pending_response.send(b"this is a response".to_vec()); } - }); + }.boxed().into()).unwrap(); b }; @@ -825,26 +829,24 @@ mod tests { Swarm::dial_addr(&mut swarms[0].0, dial_addr).unwrap(); } - // Running `swarm[0]` in the background until a `InboundRequest` event happens, - // which is a hint about the test having ended. - async_std::task::spawn({ + // Running `swarm[0]` in the background. + pool.spawner().spawn_obj({ let (mut swarm, _) = swarms.remove(0); async move { loop { match swarm.next_event().await { SwarmEvent::Behaviour(super::Event::InboundRequest { result, .. }) => { - assert!(result.is_ok()); - break + result.unwrap(); }, _ => {} } } - } - }); + }.boxed().into() + }).unwrap(); // Remove and run the remaining swarm. let (mut swarm, _) = swarms.remove(0); - async_std::task::block_on(async move { + pool.run_until(async move { let mut response_receiver = None; loop { @@ -863,7 +865,7 @@ mod tests { SwarmEvent::Behaviour(super::Event::RequestFinished { result, .. }) => { - assert!(result.is_ok()); + result.unwrap(); break; } _ => {} @@ -877,6 +879,7 @@ mod tests { #[test] fn max_response_size_exceeded() { let protocol_name = "/test/req-resp/1"; + let mut pool = LocalPool::new(); // Build swarms whose behaviour is `RequestResponsesBehaviour`. let mut swarms = (0..2) @@ -904,12 +907,12 @@ mod tests { inbound_queue: Some(tx), })).unwrap(); - async_std::task::spawn(async move { + pool.spawner().spawn_obj(async move { while let Some(rq) = rx.next().await { assert_eq!(rq.payload, b"this is a request"); let _ = rq.pending_response.send(b"this response exceeds the limit".to_vec()); } - }); + }.boxed().into()).unwrap(); b }; @@ -931,7 +934,7 @@ mod tests { // Running `swarm[0]` in the background until a `InboundRequest` event happens, // which is a hint about the test having ended. - async_std::task::spawn({ + pool.spawner().spawn_obj({ let (mut swarm, _) = swarms.remove(0); async move { loop { @@ -943,12 +946,12 @@ mod tests { _ => {} } } - } - }); + }.boxed().into() + }).unwrap(); // Remove and run the remaining swarm. let (mut swarm, _) = swarms.remove(0); - async_std::task::block_on(async move { + pool.run_until(async move { let mut response_receiver = None; loop { @@ -974,8 +977,6 @@ mod tests { } } - ; - match response_receiver.unwrap().await.unwrap().unwrap_err() { super::RequestFailure::Network(super::OutboundFailure::ConnectionClosed) => {}, _ => panic!() From 94a1a790030dfcc1c7550704a925ded989ff8424 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 1 Dec 2020 09:25:59 +0100 Subject: [PATCH 42/48] client/network/protocol: Fix line length --- client/network/src/protocol.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 2df95ba8f3daf..44a315590f25e 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1586,7 +1586,9 @@ impl NetworkBehaviour for Protocol { } } Some(Fallback::Transactions) => { - if let Ok(m) = as Decode>::decode(&mut message.as_ref()) { + if let Ok(m) = as Decode>::decode( + &mut message.as_ref(), + ) { self.on_transactions(peer_id, m); } else { warn!(target: "sub-libp2p", "Failed to decode transactions list"); @@ -1610,7 +1612,11 @@ impl NetworkBehaviour for Protocol { } } None => { - debug!(target: "sub-libp2p", "Received notification from unknown protocol {:?}", protocol_name); + debug!( + target: "sub-libp2p", + "Received notification from unknown protocol {:?}", + protocol_name, + ); CustomMessageOutcome::None } } From 8e846e0051c26b3a1d50416e099fe8ebc449c4cd Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 1 Dec 2020 12:58:29 +0100 Subject: [PATCH 43/48] client/network/protocol: Disconnect on most request failures --- client/network/src/protocol.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 44a315590f25e..f2957554e4f1b 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -96,6 +96,8 @@ mod rep { use sc_peerset::ReputationChange as Rep; /// Reputation change when a peer doesn't respond in time to our messages. pub const TIMEOUT: Rep = Rep::new(-(1 << 10), "Request timeout"); + /// Reputation change when a peer refuses a request. + pub const REFUSED: Rep = Rep::new(-(1 << 10), "Request refused"); /// Reputation change when we are a light client and a peer is behind us. pub const PEER_BEHIND_US_LIGHT: Rep = Rep::new(-(1 << 8), "Useless for a light peer"); /// Reputation change when a peer sends us any transaction. @@ -1437,23 +1439,33 @@ impl NetworkBehaviour for Protocol { trace!(target: "sync", "Block request to peer {:?} failed: {:?}.", id, e); match e { - RequestFailure::UnknownProtocol => { - debug_assert!(false, "Block request protocol should always be known."); - } RequestFailure::Network(OutboundFailure::Timeout) => { self.peerset_handle.report_peer(id.clone(), rep::TIMEOUT); + self.behaviour.disconnect_peer(id); } RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => { self.peerset_handle.report_peer(id.clone(), rep::BAD_PROTOCOL); + self.behaviour.disconnect_peer(id); + } + RequestFailure::Network(OutboundFailure::DialFailure) => { + self.behaviour.disconnect_peer(id); + } + RequestFailure::Refused => { + self.peerset_handle.report_peer(id.clone(), rep::REFUSED); + self.behaviour.disconnect_peer(id); + } + RequestFailure::Network(OutboundFailure::ConnectionClosed) + | RequestFailure::NotConnected => {}, + RequestFailure::UnknownProtocol => { + debug_assert!(false, "Block request protocol should always be known."); } RequestFailure::Obsolete => { debug_assert!( false, "Can not receive `RequestFailure::Obsolete` after dropping the \ - response reciver.", + response receiver.", ); } - _ => {}, } }, Poll::Ready(Err(e)) => { From d2c51615d13a744a335b547271afeb2b4ae77348 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 4 Jan 2021 12:20:49 +0100 Subject: [PATCH 44/48] client/network/protocol: Disconnect peer when oneshot is canceled --- client/network/src/protocol.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 302b51f4d1ba8..545440b6fbd6d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1480,9 +1480,14 @@ impl NetworkBehaviour for Protocol { } } }, - Poll::Ready(Err(e)) => { + Poll::Ready(Err(oneshot::Canceled)) => { peer.block_request.take(); - trace!(target: "sync", "Block request to peer {:?} failed: {:?}.", id, e); + self.behaviour.disconnect_peer(id); + trace!( + target: "sync", + "Block request to peer {:?} failed due to oneshot being canceled.", + id, + ); }, Poll::Pending => {}, } From 34d035133a9afaa2c03ea6253dbc31c038dffcde Mon Sep 17 00:00:00 2001 From: Max Inden Date: Mon, 4 Jan 2021 13:17:25 +0100 Subject: [PATCH 45/48] client/network/protocol: Disconnect peer even when connection closed --- client/network/src/protocol.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 240d92e51ff58..ccc3890e397c7 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1467,7 +1467,19 @@ impl NetworkBehaviour for Protocol { self.behaviour.disconnect_peer(id); } RequestFailure::Network(OutboundFailure::ConnectionClosed) - | RequestFailure::NotConnected => {}, + | RequestFailure::NotConnected => { + // TODO: Log line likely not needed once stalling issue is fixed. + if self.sync.peer_info(id).is_some() { + debug!( + target: "sync", + "Block request to peer {:?} failed due to connection \ + closed or not connected while sync assumes peer is \ + connected.", + id + ); + } + self.behaviour.disconnect_peer(id); + }, RequestFailure::UnknownProtocol => { debug_assert!(false, "Block request protocol should always be known."); } From 2cb76ef637a46c88681258c6da072858ebfa6e0b Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 5 Jan 2021 14:57:45 +0100 Subject: [PATCH 46/48] client/network/protocol: Remove debugging log line --- client/network/src/protocol.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index ccc3890e397c7..e3d6d5e815c3b 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1468,16 +1468,6 @@ impl NetworkBehaviour for Protocol { } RequestFailure::Network(OutboundFailure::ConnectionClosed) | RequestFailure::NotConnected => { - // TODO: Log line likely not needed once stalling issue is fixed. - if self.sync.peer_info(id).is_some() { - debug!( - target: "sync", - "Block request to peer {:?} failed due to connection \ - closed or not connected while sync assumes peer is \ - connected.", - id - ); - } self.behaviour.disconnect_peer(id); }, RequestFailure::UnknownProtocol => { @@ -1494,12 +1484,12 @@ impl NetworkBehaviour for Protocol { }, Poll::Ready(Err(oneshot::Canceled)) => { peer.block_request.take(); - self.behaviour.disconnect_peer(id); trace!( target: "sync", "Block request to peer {:?} failed due to oneshot being canceled.", id, ); + self.behaviour.disconnect_peer(id); }, Poll::Pending => {}, } From d525b967673e60149283c9043c0d544300c68f18 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 5 Jan 2021 14:58:20 +0100 Subject: [PATCH 47/48] client/network/request_response: Use Clone::clone for error --- client/network/src/request_responses.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 94e83ec9639b9..0ec6cbe706ac4 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -541,18 +541,10 @@ impl NetworkBehaviour for RequestResponsesBehaviour { error, .. } => { - // TODO: Remove hack by deriving `Clone` for `OutboundFailure`. - let error_clone = match &error { - OutboundFailure::ConnectionClosed => OutboundFailure::ConnectionClosed, - OutboundFailure::DialFailure => OutboundFailure::DialFailure, - OutboundFailure::Timeout => OutboundFailure::Timeout, - OutboundFailure::UnsupportedProtocols => OutboundFailure::UnsupportedProtocols, - }; - let started = match self.pending_requests.remove(&request_id) { Some((started, pending_response)) => { if pending_response.send( - Err(RequestFailure::Network(error)), + Err(RequestFailure::Network(error.clone())), ).is_err() { log::debug!( target: "sub-libp2p", @@ -578,7 +570,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour { peer, protocol: protocol.clone(), duration: started.elapsed(), - result: Err(RequestFailure::Network(error_clone)), + result: Err(RequestFailure::Network(error)), }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(out)); From 1da9269669a6db2aaf881142f6e995053a6349b0 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 5 Jan 2021 15:04:53 +0100 Subject: [PATCH 48/48] client/network/request_response: Remove outdated comment With libp2p v0.33.0 libp2p-request-response properly sends inbound failures on connections being closed. --- client/network/src/request_responses.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 0ec6cbe706ac4..fbdb1432379ed 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -123,11 +123,6 @@ pub enum Event { /// A remote sent a request and either we have successfully answered it or an error happened. /// /// This event is generated for statistics purposes. - /// - /// Note: Multiple [`InboundRequest`] events can be emitted for the same inbound request. E.g. - /// when response building succeeds, but sending the response later on fails. In that case there - /// will be one [`InboundRequest`] with the time it took to build the response and one - /// [`InboundRequest`] with the reason why later on sending the response failed. InboundRequest { /// Peer which has emitted the request. peer: PeerId,