diff --git a/Cargo.lock b/Cargo.lock index 1f22fccd2c1a..0f386c52b384 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14995,6 +14995,7 @@ dependencies = [ "sc-network", "sc-network-common", "sc-network-gossip", + "sc-network-sync", "sc-network-test", "sc-telemetry", "sc-transaction-pool-api", @@ -15211,6 +15212,7 @@ dependencies = [ "sc-client-api", "sc-network", "sc-network-common", + "sc-network-sync", "sp-blockchain", "sp-runtime", ] @@ -15363,6 +15365,7 @@ dependencies = [ "quickcheck", "sc-network", "sc-network-common", + "sc-network-sync", "schnellru", "sp-runtime", "substrate-prometheus-endpoint", @@ -15403,6 +15406,7 @@ dependencies = [ "parity-scale-codec", "sc-network", "sc-network-common", + "sc-network-sync", "sp-consensus", "sp-statement-store", "substrate-prometheus-endpoint", @@ -15489,6 +15493,7 @@ dependencies = [ "parity-scale-codec", "sc-network", "sc-network-common", + "sc-network-sync", "sc-utils", "sp-consensus", "sp-runtime", diff --git a/substrate/client/consensus/grandpa/Cargo.toml b/substrate/client/consensus/grandpa/Cargo.toml index 472bdd1c5b82..921b9c539e37 100644 --- a/substrate/client/consensus/grandpa/Cargo.toml +++ b/substrate/client/consensus/grandpa/Cargo.toml @@ -37,6 +37,7 @@ sc-consensus = { path = "../common" } sc-network = { path = "../../network" } sc-network-gossip = { path = "../../network-gossip" } sc-network-common = { path = "../../network/common" } +sc-network-sync = { path = "../../network/sync" } sc-telemetry = { path = "../../telemetry" } sc-utils = { path = "../../utils" } sp-api = { path = "../../../primitives/api" } diff --git a/substrate/client/consensus/grandpa/src/communication/mod.rs b/substrate/client/consensus/grandpa/src/communication/mod.rs index c0749858568f..6d9e956b41be 100644 --- a/substrate/client/consensus/grandpa/src/communication/mod.rs +++ b/substrate/client/consensus/grandpa/src/communication/mod.rs @@ -59,7 +59,7 @@ use crate::{ use gossip::{ FullCatchUpMessage, FullCommitMessage, GossipMessage, GossipValidator, PeerReport, VoteMessage, }; -use sc_network_common::sync::SyncEventStream; +use sc_network_sync::SyncEventStream; use sc_utils::mpsc::TracingUnboundedReceiver; use sp_consensus_grandpa::{AuthorityId, AuthoritySignature, RoundNumber, SetId as SetIdNumber}; diff --git a/substrate/client/consensus/grandpa/src/communication/tests.rs b/substrate/client/consensus/grandpa/src/communication/tests.rs index 10c4772fc76d..4a869d0f5152 100644 --- a/substrate/client/consensus/grandpa/src/communication/tests.rs +++ b/substrate/client/consensus/grandpa/src/communication/tests.rs @@ -33,11 +33,9 @@ use sc_network::{ NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, PeerId, ReputationChange, }; -use sc_network_common::{ - role::ObservedRole, - sync::{SyncEvent as SyncStreamEvent, SyncEventStream}, -}; +use sc_network_common::role::ObservedRole; use sc_network_gossip::Validator; +use sc_network_sync::{SyncEvent as SyncStreamEvent, SyncEventStream}; use sc_network_test::{Block, Hash}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_consensus_grandpa::AuthorityList; diff --git a/substrate/client/consensus/grandpa/src/warp_proof.rs b/substrate/client/consensus/grandpa/src/warp_proof.rs index ea8114eafd78..dcd55dcdf3af 100644 --- a/substrate/client/consensus/grandpa/src/warp_proof.rs +++ b/substrate/client/consensus/grandpa/src/warp_proof.rs @@ -23,7 +23,7 @@ use crate::{ BlockNumberOps, GrandpaJustification, SharedAuthoritySet, }; use sc_client_api::Backend as ClientBackend; -use sc_network_common::sync::warp::{EncodedProof, VerificationResult, WarpSyncProvider}; +use sc_network_sync::warp::{EncodedProof, VerificationResult, WarpSyncProvider}; use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend}; use sp_consensus_grandpa::{AuthorityList, SetId, GRANDPA_ENGINE_ID}; use sp_runtime::{ diff --git a/substrate/client/informant/Cargo.toml b/substrate/client/informant/Cargo.toml index e077f4e11a59..47e65df3cc11 100644 --- a/substrate/client/informant/Cargo.toml +++ b/substrate/client/informant/Cargo.toml @@ -19,6 +19,7 @@ futures-timer = "3.0.1" log = "0.4.17" sc-client-api = { path = "../api" } sc-network-common = { path = "../network/common" } +sc-network-sync = { path = "../network/sync" } sc-network = { path = "../network" } sp-blockchain = { path = "../../primitives/blockchain" } sp-runtime = { path = "../../primitives/runtime" } diff --git a/substrate/client/informant/src/display.rs b/substrate/client/informant/src/display.rs index 722cf56d778d..64ddb71d572e 100644 --- a/substrate/client/informant/src/display.rs +++ b/substrate/client/informant/src/display.rs @@ -21,7 +21,7 @@ use ansi_term::Colour; use log::info; use sc_client_api::ClientInfo; use sc_network::NetworkStatus; -use sc_network_common::sync::{ +use sc_network_sync::{ warp::{WarpSyncPhase, WarpSyncProgress}, SyncState, SyncStatus, }; diff --git a/substrate/client/informant/src/lib.rs b/substrate/client/informant/src/lib.rs index 03f9075055e2..b072f8551f9f 100644 --- a/substrate/client/informant/src/lib.rs +++ b/substrate/client/informant/src/lib.rs @@ -24,7 +24,7 @@ use futures_timer::Delay; use log::{debug, info, trace}; use sc_client_api::{BlockchainEvents, UsageProvider}; use sc_network::NetworkStatusProvider; -use sc_network_common::sync::SyncStatusProvider; +use sc_network_sync::SyncStatusProvider; use sp_blockchain::HeaderMetadata; use sp_runtime::traits::{Block as BlockT, Header}; use std::{collections::VecDeque, fmt::Display, sync::Arc, time::Duration}; diff --git a/substrate/client/network-gossip/Cargo.toml b/substrate/client/network-gossip/Cargo.toml index 73d2d3fa051e..95e26a232c1d 100644 --- a/substrate/client/network-gossip/Cargo.toml +++ b/substrate/client/network-gossip/Cargo.toml @@ -24,6 +24,7 @@ tracing = "0.1.29" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } sc-network = { path = "../network" } sc-network-common = { path = "../network/common" } +sc-network-sync = { path = "../network/sync" } sp-runtime = { path = "../../primitives/runtime" } [dev-dependencies] diff --git a/substrate/client/network-gossip/src/bridge.rs b/substrate/client/network-gossip/src/bridge.rs index 6a3790ee2b2b..8f7d490757b3 100644 --- a/substrate/client/network-gossip/src/bridge.rs +++ b/substrate/client/network-gossip/src/bridge.rs @@ -22,7 +22,7 @@ use crate::{ }; use sc_network::{event::Event, types::ProtocolName, ReputationChange}; -use sc_network_common::sync::SyncEvent; +use sc_network_sync::SyncEvent; use futures::{ channel::mpsc::{channel, Receiver, Sender}, @@ -338,7 +338,8 @@ mod tests { config::MultiaddrWithPeerId, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, }; - use sc_network_common::{role::ObservedRole, sync::SyncEventStream}; + use sc_network_common::role::ObservedRole; + use sc_network_sync::SyncEventStream; use sp_runtime::{ testing::H256, traits::{Block as BlockT, NumberFor}, diff --git a/substrate/client/network-gossip/src/lib.rs b/substrate/client/network-gossip/src/lib.rs index 5b02be5c23f6..a77141ec6f63 100644 --- a/substrate/client/network-gossip/src/lib.rs +++ b/substrate/client/network-gossip/src/lib.rs @@ -71,7 +71,7 @@ use libp2p::{multiaddr, PeerId}; use sc_network::{ types::ProtocolName, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, }; -use sc_network_common::sync::SyncEventStream; +use sc_network_sync::SyncEventStream; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::iter; diff --git a/substrate/client/network/common/src/sync.rs b/substrate/client/network/common/src/sync.rs index 4ca21221f87f..a910740aef64 100644 --- a/substrate/client/network/common/src/sync.rs +++ b/substrate/client/network/common/src/sync.rs @@ -19,150 +19,6 @@ //! Abstract interfaces and data structures related to network sync. pub mod message; -pub mod metrics; -pub mod warp; - -use crate::{role::Roles, types::ReputationChange}; -use futures::Stream; - -use libp2p_identity::PeerId; - -use message::{BlockAnnounce, BlockRequest, BlockResponse}; -use sc_consensus::{import_queue::RuntimeOrigin, IncomingBlock}; -use sp_consensus::BlockOrigin; -use sp_runtime::{ - traits::{Block as BlockT, NumberFor}, - Justifications, -}; -use warp::WarpSyncProgress; - -use std::{any::Any, fmt, fmt::Formatter, pin::Pin, sync::Arc}; - -/// The sync status of a peer we are trying to sync with -#[derive(Debug)] -pub struct PeerInfo { - /// Their best block hash. - pub best_hash: Block::Hash, - /// Their best block number. - pub best_number: NumberFor, -} - -/// Info about a peer's known state (both full and light). -#[derive(Clone, Debug)] -pub struct ExtendedPeerInfo { - /// Roles - pub roles: Roles, - /// Peer best block hash - pub best_hash: B::Hash, - /// Peer best block number - pub best_number: NumberFor, -} - -/// Reported sync state. -#[derive(Clone, Eq, PartialEq, Debug)] -pub enum SyncState { - /// Initial sync is complete, keep-up sync is active. - Idle, - /// Actively catching up with the chain. - Downloading { target: BlockNumber }, - /// All blocks are downloaded and are being imported. - Importing { target: BlockNumber }, -} - -impl SyncState { - /// Are we actively catching up with the chain? - pub fn is_major_syncing(&self) -> bool { - !matches!(self, SyncState::Idle) - } -} - -/// Reported state download progress. -#[derive(Clone, Eq, PartialEq, Debug)] -pub struct StateDownloadProgress { - /// Estimated download percentage. - pub percentage: u32, - /// Total state size in bytes downloaded so far. - pub size: u64, -} - -/// Syncing status and statistics. -#[derive(Debug, Clone)] -pub struct SyncStatus { - /// Current global sync state. - pub state: SyncState>, - /// Target sync block number. - pub best_seen_block: Option>, - /// Number of peers participating in syncing. - pub num_peers: u32, - /// Number of peers known to `SyncingEngine` (both full and light). - pub num_connected_peers: u32, - /// Number of blocks queued for import - pub queued_blocks: u32, - /// State sync status in progress, if any. - pub state_sync: Option, - /// Warp sync in progress, if any. - pub warp_sync: Option>, -} - -/// A peer did not behave as expected and should be reported. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BadPeer(pub PeerId, pub ReputationChange); - -impl fmt::Display for BadPeer { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Bad peer {}; Reputation change: {:?}", self.0, self.1) - } -} - -impl std::error::Error for BadPeer {} - -/// Action that the parent of [`ChainSync`] should perform if we want to import blocks. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ImportBlocksAction { - pub origin: BlockOrigin, - pub blocks: Vec>, -} - -/// Result of [`ChainSync::on_block_data`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OnBlockData { - /// The block should be imported. - Import(ImportBlocksAction), - /// A new block request needs to be made to the given peer. - Request(PeerId, BlockRequest), - /// Continue processing events. - Continue, -} - -/// Result of [`ChainSync::on_block_justification`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OnBlockJustification { - /// The justification needs no further handling. - Nothing, - /// The justification should be imported. - Import { - peer_id: PeerId, - hash: Block::Hash, - number: NumberFor, - justifications: Justifications, - }, -} - -/// Result of `ChainSync::on_state_data`. -#[derive(Debug)] -pub enum OnStateData { - /// The block and state that should be imported. - Import(BlockOrigin, IncomingBlock), - /// A new state request needs to be made to the given peer. - Continue, -} - -/// Block or justification request polled from `ChainSync` -#[derive(Debug)] -pub enum ImportResult { - BlockImport(BlockOrigin, Vec>), - JustificationImport(RuntimeOrigin, B::Hash, NumberFor, Justifications), -} /// Sync operation mode. #[derive(Copy, Clone, Debug, Eq, PartialEq)] @@ -197,201 +53,3 @@ impl Default for SyncMode { Self::Full } } -#[derive(Debug)] -pub struct Metrics { - pub queued_blocks: u32, - pub fork_targets: u32, - pub justifications: metrics::Metrics, -} - -#[derive(Debug)] -pub enum PeerRequest { - Block(BlockRequest), - State, - WarpProof, -} - -#[derive(Debug)] -pub enum PeerRequestType { - Block, - State, - WarpProof, -} - -impl PeerRequest { - pub fn get_type(&self) -> PeerRequestType { - match self { - PeerRequest::Block(_) => PeerRequestType::Block, - PeerRequest::State => PeerRequestType::State, - PeerRequest::WarpProof => PeerRequestType::WarpProof, - } - } -} - -/// Wrapper for implementation-specific state request. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueStateRequest(pub Box); - -impl fmt::Debug for OpaqueStateRequest { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueStateRequest").finish() - } -} - -/// Wrapper for implementation-specific state response. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueStateResponse(pub Box); - -impl fmt::Debug for OpaqueStateResponse { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueStateResponse").finish() - } -} - -/// Provides high-level status of syncing. -#[async_trait::async_trait] -pub trait SyncStatusProvider: Send + Sync { - /// Get high-level view of the syncing status. - async fn status(&self) -> Result, ()>; -} - -#[async_trait::async_trait] -impl SyncStatusProvider for Arc -where - T: ?Sized, - T: SyncStatusProvider, - Block: BlockT, -{ - async fn status(&self) -> Result, ()> { - T::status(self).await - } -} - -/// Syncing-related events that other protocols can subscribe to. -pub enum SyncEvent { - /// Peer that the syncing implementation is tracking connected. - PeerConnected(PeerId), - - /// Peer that the syncing implementation was tracking disconnected. - PeerDisconnected(PeerId), -} - -pub trait SyncEventStream: Send + Sync { - /// Subscribe to syncing-related events. - fn event_stream(&self, name: &'static str) -> Pin + Send>>; -} - -impl SyncEventStream for Arc -where - T: ?Sized, - T: SyncEventStream, -{ - fn event_stream(&self, name: &'static str) -> Pin + Send>> { - T::event_stream(self, name) - } -} - -/// Something that represents the syncing strategy to download past and future blocks of the chain. -pub trait ChainSync: Send { - /// Returns the state of the sync of the given peer. - /// - /// Returns `None` if the peer is unknown. - fn peer_info(&self, who: &PeerId) -> Option>; - - /// Returns the current sync status. - fn status(&self) -> SyncStatus; - - /// Number of active forks requests. This includes - /// requests that are pending or could be issued right away. - fn num_sync_requests(&self) -> usize; - - /// Number of downloaded blocks. - fn num_downloaded_blocks(&self) -> usize; - - /// Returns the current number of peers stored within this state machine. - fn num_peers(&self) -> usize; - - /// Handle a new connected peer. - /// - /// Call this method whenever we connect to a new peer. - #[must_use] - fn new_peer( - &mut self, - who: PeerId, - best_hash: Block::Hash, - best_number: NumberFor, - ) -> Result>, BadPeer>; - - /// Signal that a new best block has been imported. - fn update_chain_info(&mut self, best_hash: &Block::Hash, best_number: NumberFor); - - /// Schedule a justification request for the given block. - fn request_justification(&mut self, hash: &Block::Hash, number: NumberFor); - - /// Clear all pending justification requests. - fn clear_justification_requests(&mut self); - - /// Request syncing for the given block from given set of peers. - fn set_sync_fork_request( - &mut self, - peers: Vec, - hash: &Block::Hash, - number: NumberFor, - ); - - /// Handle a response from the remote to a block request that we made. - /// - /// `request` must be the original request that triggered `response`. - /// or `None` if data comes from the block announcement. - /// - /// If this corresponds to a valid block, this outputs the block that - /// must be imported in the import queue. - #[must_use] - fn on_block_data( - &mut self, - who: &PeerId, - request: Option>, - response: BlockResponse, - ) -> Result, BadPeer>; - - /// Handle a response from the remote to a justification request that we made. - /// - /// `request` must be the original request that triggered `response`. - #[must_use] - fn on_block_justification( - &mut self, - who: PeerId, - response: BlockResponse, - ) -> Result, BadPeer>; - - /// Call this when a justification has been processed by the import queue, - /// with or without errors. - fn on_justification_import( - &mut self, - hash: Block::Hash, - number: NumberFor, - success: bool, - ); - - /// Notify about finalization of the given block. - fn on_block_finalized(&mut self, hash: &Block::Hash, number: NumberFor); - - /// Notify about pre-validated block announcement. - fn on_validated_block_announce( - &mut self, - is_best: bool, - who: PeerId, - announce: &BlockAnnounce, - ); - - /// Call when a peer has disconnected. - /// Canceled obsolete block request may result in some blocks being ready for - /// import, so this functions checks for such blocks and returns them. - #[must_use] - fn peer_disconnected(&mut self, who: &PeerId) -> Option>; - - /// Return some key metrics. - fn metrics(&self) -> Metrics; -} diff --git a/substrate/client/network/common/src/sync/warp.rs b/substrate/client/network/common/src/sync/warp.rs deleted file mode 100644 index f4e39f438512..000000000000 --- a/substrate/client/network/common/src/sync/warp.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use codec::{Decode, Encode}; -pub use sp_consensus_grandpa::{AuthorityList, SetId}; -use sp_runtime::traits::{Block as BlockT, NumberFor}; -use std::fmt; - -/// Scale-encoded warp sync proof response. -pub struct EncodedProof(pub Vec); - -/// Warp sync request -#[derive(Encode, Decode, Debug)] -pub struct WarpProofRequest { - /// Start collecting proofs from this block. - pub begin: B::Hash, -} - -/// Proof verification result. -pub enum VerificationResult { - /// Proof is valid, but the target was not reached. - Partial(SetId, AuthorityList, Block::Hash), - /// Target finality is proved. - Complete(SetId, AuthorityList, Block::Header), -} - -/// Warp sync backend. Handles retrieving and verifying warp sync proofs. -pub trait WarpSyncProvider: Send + Sync { - /// Generate proof starting at given block hash. The proof is accumulated until maximum proof - /// size is reached. - fn generate( - &self, - start: Block::Hash, - ) -> Result>; - /// Verify warp proof against current set of authorities. - fn verify( - &self, - proof: &EncodedProof, - set_id: SetId, - authorities: AuthorityList, - ) -> Result, Box>; - /// Get current list of authorities. This is supposed to be genesis authorities when starting - /// sync. - fn current_authorities(&self) -> AuthorityList; -} - -/// Reported warp sync phase. -#[derive(Clone, Eq, PartialEq, Debug)] -pub enum WarpSyncPhase { - /// Waiting for peers to connect. - AwaitingPeers { required_peers: usize }, - /// Waiting for target block to be received. - AwaitingTargetBlock, - /// Downloading and verifying grandpa warp proofs. - DownloadingWarpProofs, - /// Downloading target block. - DownloadingTargetBlock, - /// Downloading state data. - DownloadingState, - /// Importing state. - ImportingState, - /// Downloading block history. - DownloadingBlocks(NumberFor), -} - -impl fmt::Display for WarpSyncPhase { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::AwaitingPeers { required_peers } => - write!(f, "Waiting for {required_peers} peers to be connected"), - Self::AwaitingTargetBlock => write!(f, "Waiting for target block to be received"), - Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"), - Self::DownloadingTargetBlock => write!(f, "Downloading target block"), - Self::DownloadingState => write!(f, "Downloading state"), - Self::ImportingState => write!(f, "Importing state"), - Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n), - } - } -} - -/// Reported warp sync progress. -#[derive(Clone, Eq, PartialEq, Debug)] -pub struct WarpSyncProgress { - /// Estimated download percentage. - pub phase: WarpSyncPhase, - /// Total bytes downloaded so far. - pub total_bytes: u64, -} diff --git a/substrate/client/network/src/config.rs b/substrate/client/network/src/config.rs index d069c3f458ff..124d73a74dbc 100644 --- a/substrate/client/network/src/config.rs +++ b/substrate/client/network/src/config.rs @@ -30,7 +30,11 @@ pub use crate::{ types::ProtocolName, }; -pub use libp2p::{identity::Keypair, multiaddr, Multiaddr, PeerId}; +pub use libp2p::{ + build_multiaddr, + identity::{self, ed25519, Keypair}, + multiaddr, Multiaddr, PeerId, +}; use crate::peer_store::PeerStoreHandle; use codec::Encode; @@ -39,9 +43,10 @@ use zeroize::Zeroize; pub use sc_network_common::{ role::{Role, Roles}, - sync::{warp::WarpSyncProvider, SyncMode}, + sync::SyncMode, ExHashT, }; + use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::Block as BlockT; @@ -58,11 +63,6 @@ use std::{ str::{self, FromStr}, }; -pub use libp2p::{ - build_multiaddr, - identity::{self, ed25519}, -}; - /// Protocol name prefix, transmitted on the wire for legacy protocol names. /// I.e., `dot` in `/dot/sync/2`. Should be unique for each chain. Always UTF-8. /// Deprecated in favour of genesis hash & fork ID based protocol names. diff --git a/substrate/client/network/src/lib.rs b/substrate/client/network/src/lib.rs index ee3075968784..4dc9bdb4cc1c 100644 --- a/substrate/client/network/src/lib.rs +++ b/substrate/client/network/src/lib.rs @@ -266,14 +266,7 @@ pub use event::{DhtEvent, Event, SyncEvent}; #[doc(inline)] pub use libp2p::{multiaddr, Multiaddr, PeerId}; pub use request_responses::{Config, IfDisconnected, RequestFailure}; -pub use sc_network_common::{ - role::ObservedRole, - sync::{ - warp::{WarpSyncPhase, WarpSyncProgress}, - ExtendedPeerInfo, StateDownloadProgress, SyncEventStream, SyncState, SyncStatusProvider, - }, - types::ReputationChange, -}; +pub use sc_network_common::{role::ObservedRole, types::ReputationChange}; pub use service::{ signature::Signature, traits::{ diff --git a/substrate/client/network/statement/Cargo.toml b/substrate/client/network/statement/Cargo.toml index adfb2d6a05fb..ef974b4f33f1 100644 --- a/substrate/client/network/statement/Cargo.toml +++ b/substrate/client/network/statement/Cargo.toml @@ -21,6 +21,7 @@ libp2p = "0.51.3" log = "0.4.17" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus" } sc-network-common = { path = "../common" } +sc-network-sync = { path = "../sync" } sc-network = { path = ".." } sp-consensus = { path = "../../../primitives/consensus/common" } sp-statement-store = { path = "../../../primitives/statement-store" } diff --git a/substrate/client/network/statement/src/lib.rs b/substrate/client/network/statement/src/lib.rs index c5d83b59b260..69d4faa13ef2 100644 --- a/substrate/client/network/statement/src/lib.rs +++ b/substrate/client/network/statement/src/lib.rs @@ -39,10 +39,8 @@ use sc_network::{ utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, }; -use sc_network_common::{ - role::ObservedRole, - sync::{SyncEvent, SyncEventStream}, -}; +use sc_network_common::role::ObservedRole; +use sc_network_sync::{SyncEvent, SyncEventStream}; use sp_statement_store::{ Hash, NetworkPriority, Statement, StatementSource, StatementStore, SubmitResult, }; diff --git a/substrate/client/network/sync/src/block_request_handler.rs b/substrate/client/network/sync/src/block_request_handler.rs index c24083f63287..f363dda3a2d1 100644 --- a/substrate/client/network/sync/src/block_request_handler.rs +++ b/substrate/client/network/sync/src/block_request_handler.rs @@ -24,7 +24,6 @@ use crate::{ BlockResponse as BlockResponseSchema, BlockResponse, Direction, }, service::network::NetworkServiceHandle, - MAX_BLOCKS_IN_RESPONSE, }; use codec::{Decode, DecodeAll, Encode}; @@ -54,6 +53,9 @@ use std::{ time::Duration, }; +/// Maximum blocks per response. +pub(crate) const MAX_BLOCKS_IN_RESPONSE: usize = 128; + const LOG_TARGET: &str = "sync"; const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; const MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER: usize = 2; diff --git a/substrate/client/network/sync/src/chain_sync.rs b/substrate/client/network/sync/src/chain_sync.rs new file mode 100644 index 000000000000..9cf0080e36ac --- /dev/null +++ b/substrate/client/network/sync/src/chain_sync.rs @@ -0,0 +1,2435 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Contains the state of the chain synchronization process +//! +//! At any given point in time, a running node tries as much as possible to be at the head of the +//! chain. This module handles the logic of which blocks to request from remotes, and processing +//! responses. It yields blocks to check and potentially move to the database. +//! +//! # Usage +//! +//! The `ChainSync` struct maintains the state of the block requests. Whenever something happens on +//! the network, or whenever a block has been successfully verified, call the appropriate method in +//! order to update it. + +use crate::{ + blocks::BlockCollection, + extra_requests::ExtraRequests, + schema::v1::StateResponse, + service::network::NetworkServiceHandle, + state::{ImportResult, StateSync}, + types::{ + BadPeer, Metrics, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, SyncMode, SyncState, + SyncStatus, + }, + warp::{ + self, EncodedProof, WarpProofImportResult, WarpProofRequest, WarpSync, WarpSyncConfig, + WarpSyncPhase, WarpSyncProgress, + }, +}; + +use codec::Encode; +use libp2p::PeerId; +use log::{debug, error, info, trace, warn}; + +use sc_client_api::{BlockBackend, ProofProvider}; +use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; +use sc_network::types::ProtocolName; +use sc_network_common::sync::message::{ + BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, Direction, FromBlock, +}; +use sp_arithmetic::traits::Saturating; +use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; +use sp_consensus::{BlockOrigin, BlockStatus}; +use sp_runtime::{ + traits::{ + Block as BlockT, CheckedSub, Hash, HashingFor, Header as HeaderT, NumberFor, One, + SaturatedConversion, Zero, + }, + EncodedJustification, Justifications, +}; + +use std::{ + collections::{HashMap, HashSet}, + ops::Range, + sync::Arc, +}; + +#[cfg(test)] +mod test; + +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + +/// Maximum blocks to store in the import queue. +const MAX_IMPORTING_BLOCKS: usize = 2048; + +/// Maximum blocks to download ahead of any gap. +const MAX_DOWNLOAD_AHEAD: u32 = 2048; + +/// Maximum blocks to look backwards. The gap is the difference between the highest block and the +/// common block of a node. +const MAX_BLOCKS_TO_LOOK_BACKWARDS: u32 = MAX_DOWNLOAD_AHEAD / 2; + +/// Pick the state to sync as the latest finalized number minus this. +const STATE_SYNC_FINALITY_THRESHOLD: u32 = 8; + +/// We use a heuristic that with a high likelihood, by the time +/// `MAJOR_SYNC_BLOCKS` have been imported we'll be on the same +/// chain as (or at least closer to) the peer so we want to delay +/// the ancestor search to not waste time doing that when we are +/// so far behind. +const MAJOR_SYNC_BLOCKS: u8 = 5; + +/// Number of peers that need to be connected before warp sync is started. +const MIN_PEERS_TO_START_WARP_SYNC: usize = 3; + +mod rep { + use sc_network::ReputationChange as Rep; + /// Reputation change when a peer sent us a message that led to a + /// database read error. + pub const BLOCKCHAIN_READ_ERROR: Rep = Rep::new(-(1 << 16), "DB Error"); + + /// Reputation change when a peer sent us a status message with a different + /// genesis than us. + pub const GENESIS_MISMATCH: Rep = Rep::new(i32::MIN, "Genesis mismatch"); + + /// Reputation change for peers which send us a block with an incomplete header. + pub const INCOMPLETE_HEADER: Rep = Rep::new(-(1 << 20), "Incomplete header"); + + /// Reputation change for peers which send us a block which we fail to verify. + pub const VERIFICATION_FAIL: Rep = Rep::new(-(1 << 29), "Block verification failed"); + + /// Reputation change for peers which send us a known bad block. + pub const BAD_BLOCK: Rep = Rep::new(-(1 << 29), "Bad block"); + + /// Peer did not provide us with advertised block data. + pub const NO_BLOCK: Rep = Rep::new(-(1 << 29), "No requested block data"); + + /// Reputation change for peers which send us non-requested block data. + pub const NOT_REQUESTED: Rep = Rep::new(-(1 << 29), "Not requested block data"); + + /// Reputation change for peers which send us a block with bad justifications. + pub const BAD_JUSTIFICATION: Rep = Rep::new(-(1 << 16), "Bad justification"); + + /// Reputation change when a peer sent us invlid ancestry result. + pub const UNKNOWN_ANCESTOR: Rep = Rep::new(-(1 << 16), "DB Error"); + + /// Peer response data does not have requested bits. + pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); +} + +enum AllowedRequests { + Some(HashSet), + All, +} + +impl AllowedRequests { + fn add(&mut self, id: &PeerId) { + if let Self::Some(ref mut set) = self { + set.insert(*id); + } + } + + fn take(&mut self) -> Self { + std::mem::take(self) + } + + fn set_all(&mut self) { + *self = Self::All; + } + + fn contains(&self, id: &PeerId) -> bool { + match self { + Self::Some(set) => set.contains(id), + Self::All => true, + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Some(set) => set.is_empty(), + Self::All => false, + } + } + + fn clear(&mut self) { + std::mem::take(self); + } +} + +impl Default for AllowedRequests { + fn default() -> Self { + Self::Some(HashSet::default()) + } +} + +struct GapSync { + blocks: BlockCollection, + best_queued_number: NumberFor, + target: NumberFor, +} + +/// Action that the parent of [`ChainSync`] should perform after reporting imported blocks with +/// [`ChainSync::on_blocks_processed`]. +pub enum BlockRequestAction { + /// Send block request to peer. Always implies dropping a stale block request to the same peer. + SendRequest { peer_id: PeerId, request: BlockRequest }, + /// Drop stale block request. + RemoveStale { peer_id: PeerId }, +} + +/// Action that the parent of [`ChainSync`] should perform if we want to import blocks. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImportBlocksAction { + pub origin: BlockOrigin, + pub blocks: Vec>, +} + +/// Action that the parent of [`ChainSync`] should perform if we want to import justifications. +pub struct ImportJustificationsAction { + pub peer_id: PeerId, + pub hash: B::Hash, + pub number: NumberFor, + pub justifications: Justifications, +} + +/// Result of [`ChainSync::on_block_data`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OnBlockData { + /// The block should be imported. + Import(ImportBlocksAction), + /// A new block request needs to be made to the given peer. + Request(PeerId, BlockRequest), + /// Continue processing events. + Continue, +} + +/// Result of [`ChainSync::on_block_justification`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OnBlockJustification { + /// The justification needs no further handling. + Nothing, + /// The justification should be imported. + Import { + peer_id: PeerId, + hash: Block::Hash, + number: NumberFor, + justifications: Justifications, + }, +} + +// Result of [`ChainSync::on_state_data`]. +#[derive(Debug)] +pub enum OnStateData { + /// The block and state that should be imported. + Import(BlockOrigin, IncomingBlock), + /// A new state request needs to be made to the given peer. + Continue, +} + +/// Action that the parent of [`ChainSync`] should perform after reporting block response with +/// [`ChainSync::on_block_response`]. +pub enum OnBlockResponse { + /// Nothing to do + Nothing, + /// Perform block request. + SendBlockRequest { peer_id: PeerId, request: BlockRequest }, + /// Import blocks. + ImportBlocks(ImportBlocksAction), + /// Import justifications. + ImportJustifications(ImportJustificationsAction), +} + +/// The main data structure which contains all the state for a chains +/// active syncing strategy. +pub struct ChainSync { + /// Chain client. + client: Arc, + /// The active peers that we are using to sync and their PeerSync status + peers: HashMap>, + /// A `BlockCollection` of blocks that are being downloaded from peers + blocks: BlockCollection, + /// The best block number in our queue of blocks to import + best_queued_number: NumberFor, + /// The best block hash in our queue of blocks to import + best_queued_hash: B::Hash, + /// Current mode (full/light) + mode: SyncMode, + /// Any extra justification requests. + extra_justifications: ExtraRequests, + /// A set of hashes of blocks that are being downloaded or have been + /// downloaded and are queued for import. + queue_blocks: HashSet, + /// Fork sync targets. + fork_targets: HashMap>, + /// A set of peers for which there might be potential block requests + allowed_requests: AllowedRequests, + /// Maximum number of peers to ask the same blocks in parallel. + max_parallel_downloads: u32, + /// Maximum blocks per request. + max_blocks_per_request: u32, + /// Total number of downloaded blocks. + downloaded_blocks: usize, + /// State sync in progress, if any. + state_sync: Option>, + /// Warp sync in progress, if any. + warp_sync: Option>, + /// Warp sync configuration. + /// + /// Will be `None` after `self.warp_sync` is `Some(_)`. + warp_sync_config: Option>, + /// A temporary storage for warp sync target block until warp sync is initialized. + warp_sync_target_block_header: Option, + /// Enable importing existing blocks. This is used used after the state download to + /// catch up to the latest state while re-importing blocks. + import_existing: bool, + /// Gap download process. + gap_sync: Option>, + /// Handle for communicating with `NetworkService` + network_service: NetworkServiceHandle, + /// Protocol name used for block announcements + block_announce_protocol_name: ProtocolName, +} + +/// All the data we have about a Peer that we are trying to sync with +#[derive(Debug, Clone)] +pub(crate) struct PeerSync { + /// Peer id of this peer. + pub peer_id: PeerId, + /// The common number is the block number that is a common point of + /// ancestry for both our chains (as far as we know). + pub common_number: NumberFor, + /// The hash of the best block that we've seen for this peer. + pub best_hash: B::Hash, + /// The number of the best block that we've seen for this peer. + pub best_number: NumberFor, + /// The state of syncing this peer is in for us, generally categories + /// into `Available` or "busy" with something as defined by `PeerSyncState`. + pub state: PeerSyncState, +} + +impl PeerSync { + /// Update the `common_number` iff `new_common > common_number`. + fn update_common_number(&mut self, new_common: NumberFor) { + if self.common_number < new_common { + trace!( + target: LOG_TARGET, + "Updating peer {} common number from={} => to={}.", + self.peer_id, + self.common_number, + new_common, + ); + self.common_number = new_common; + } + } +} + +struct ForkTarget { + number: NumberFor, + parent_hash: Option, + peers: HashSet, +} + +/// The state of syncing between a Peer and ourselves. +/// +/// Generally two categories, "busy" or `Available`. If busy, the enum +/// defines what we are busy with. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub(crate) enum PeerSyncState { + /// Available for sync requests. + Available, + /// Searching for ancestors the Peer has in common with us. + AncestorSearch { start: NumberFor, current: NumberFor, state: AncestorSearchState }, + /// Actively downloading new blocks, starting from the given Number. + DownloadingNew(NumberFor), + /// Downloading a stale block with given Hash. Stale means that it is a + /// block with a number that is lower than our best number. It might be + /// from a fork and not necessarily already imported. + DownloadingStale(B::Hash), + /// Downloading justification for given block hash. + DownloadingJustification(B::Hash), + /// Downloading state. + DownloadingState, + /// Downloading warp proof. + DownloadingWarpProof, + /// Downloading warp sync target block. + DownloadingWarpTargetBlock, + /// Actively downloading block history after warp sync. + DownloadingGap(NumberFor), +} + +impl PeerSyncState { + pub fn is_available(&self) -> bool { + matches!(self, Self::Available) + } +} + +impl ChainSync +where + B: BlockT, + Client: HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, +{ + /// Create a new instance. + pub fn new( + mode: SyncMode, + client: Arc, + block_announce_protocol_name: ProtocolName, + max_parallel_downloads: u32, + max_blocks_per_request: u32, + warp_sync_config: Option>, + network_service: NetworkServiceHandle, + ) -> Result { + let mut sync = Self { + client, + peers: HashMap::new(), + blocks: BlockCollection::new(), + best_queued_hash: Default::default(), + best_queued_number: Zero::zero(), + extra_justifications: ExtraRequests::new("justification"), + mode, + queue_blocks: Default::default(), + fork_targets: Default::default(), + allowed_requests: Default::default(), + max_parallel_downloads, + max_blocks_per_request, + downloaded_blocks: 0, + state_sync: None, + warp_sync: None, + import_existing: false, + gap_sync: None, + network_service, + warp_sync_config, + warp_sync_target_block_header: None, + block_announce_protocol_name, + }; + + sync.reset_sync_start_point()?; + Ok(sync) + } + + /// Get peer's best hash & number. + pub fn peer_info(&self, who: &PeerId) -> Option> { + self.peers + .get(who) + .map(|p| PeerInfo { best_hash: p.best_hash, best_number: p.best_number }) + } + + /// Returns the current sync status. + pub fn status(&self) -> SyncStatus { + let median_seen = self.median_seen(); + let best_seen_block = + median_seen.and_then(|median| (median > self.best_queued_number).then_some(median)); + let sync_state = if let Some(target) = median_seen { + // A chain is classified as downloading if the provided best block is + // more than `MAJOR_SYNC_BLOCKS` behind the best block or as importing + // if the same can be said about queued blocks. + let best_block = self.client.info().best_number; + if target > best_block && target - best_block > MAJOR_SYNC_BLOCKS.into() { + // If target is not queued, we're downloading, otherwise importing. + if target > self.best_queued_number { + SyncState::Downloading { target } + } else { + SyncState::Importing { target } + } + } else { + SyncState::Idle + } + } else { + SyncState::Idle + }; + + let warp_sync_progress = match (&self.warp_sync, &self.mode, &self.gap_sync) { + (_, _, Some(gap_sync)) => Some(WarpSyncProgress { + phase: WarpSyncPhase::DownloadingBlocks(gap_sync.best_queued_number), + total_bytes: 0, + }), + (None, SyncMode::Warp, _) => Some(WarpSyncProgress { + phase: WarpSyncPhase::AwaitingPeers { + required_peers: MIN_PEERS_TO_START_WARP_SYNC, + }, + total_bytes: 0, + }), + (Some(sync), _, _) => Some(sync.progress()), + _ => None, + }; + + SyncStatus { + state: sync_state, + best_seen_block, + num_peers: self.peers.len() as u32, + num_connected_peers: 0u32, + queued_blocks: self.queue_blocks.len() as u32, + state_sync: self.state_sync.as_ref().map(|s| s.progress()), + warp_sync: warp_sync_progress, + } + } + + /// Get an estimate of the number of parallel sync requests. + pub fn num_sync_requests(&self) -> usize { + self.fork_targets + .values() + .filter(|f| f.number <= self.best_queued_number) + .count() + } + + /// Get the total number of downloaded blocks. + pub fn num_downloaded_blocks(&self) -> usize { + self.downloaded_blocks + } + + /// Get the number of peers known to the syncing state machine. + pub fn num_peers(&self) -> usize { + self.peers.len() + } + + /// Notify syncing state machine that a new sync peer has connected. + #[must_use] + pub fn new_peer( + &mut self, + who: PeerId, + best_hash: B::Hash, + best_number: NumberFor, + ) -> Result>, BadPeer> { + // There is nothing sync can get from the node that has no blockchain data. + match self.block_status(&best_hash) { + Err(e) => { + debug!(target:LOG_TARGET, "Error reading blockchain: {e}"); + Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) + }, + Ok(BlockStatus::KnownBad) => { + info!("💔 New peer with known bad best block {} ({}).", best_hash, best_number); + Err(BadPeer(who, rep::BAD_BLOCK)) + }, + Ok(BlockStatus::Unknown) => { + if best_number.is_zero() { + info!("💔 New peer with unknown genesis hash {} ({}).", best_hash, best_number); + return Err(BadPeer(who, rep::GENESIS_MISMATCH)) + } + + // If there are more than `MAJOR_SYNC_BLOCKS` in the import queue then we have + // enough to do in the import queue that it's not worth kicking off + // an ancestor search, which is what we do in the next match case below. + if self.queue_blocks.len() > MAJOR_SYNC_BLOCKS.into() { + debug!( + target:LOG_TARGET, + "New peer with unknown best hash {} ({}), assuming common block.", + self.best_queued_hash, + self.best_queued_number + ); + self.peers.insert( + who, + PeerSync { + peer_id: who, + common_number: self.best_queued_number, + best_hash, + best_number, + state: PeerSyncState::Available, + }, + ); + return Ok(None) + } + + // If we are at genesis, just start downloading. + let (state, req) = if self.best_queued_number.is_zero() { + debug!( + target:LOG_TARGET, + "New peer with best hash {best_hash} ({best_number}).", + ); + + (PeerSyncState::Available, None) + } else { + let common_best = std::cmp::min(self.best_queued_number, best_number); + + debug!( + target:LOG_TARGET, + "New peer with unknown best hash {} ({}), searching for common ancestor.", + best_hash, + best_number + ); + + ( + PeerSyncState::AncestorSearch { + current: common_best, + start: self.best_queued_number, + state: AncestorSearchState::ExponentialBackoff(One::one()), + }, + Some(ancestry_request::(common_best)), + ) + }; + + self.allowed_requests.add(&who); + self.peers.insert( + who, + PeerSync { + peer_id: who, + common_number: Zero::zero(), + best_hash, + best_number, + state, + }, + ); + + if let SyncMode::Warp = self.mode { + if self.peers.len() >= MIN_PEERS_TO_START_WARP_SYNC && self.warp_sync.is_none() + { + log::debug!(target: LOG_TARGET, "Starting warp state sync."); + + if let Some(config) = self.warp_sync_config.take() { + let mut warp_sync = WarpSync::new(self.client.clone(), config); + if let Some(header) = self.warp_sync_target_block_header.take() { + warp_sync.set_target_block(header); + } + self.warp_sync = Some(warp_sync); + } + } + } + Ok(req) + }, + Ok(BlockStatus::Queued) | + Ok(BlockStatus::InChainWithState) | + Ok(BlockStatus::InChainPruned) => { + debug!( + target: LOG_TARGET, + "New peer with known best hash {best_hash} ({best_number}).", + ); + self.peers.insert( + who, + PeerSync { + peer_id: who, + common_number: std::cmp::min(self.best_queued_number, best_number), + best_hash, + best_number, + state: PeerSyncState::Available, + }, + ); + self.allowed_requests.add(&who); + Ok(None) + }, + } + } + + /// Inform sync about a new best imported block. + pub fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { + self.on_block_queued(best_hash, best_number); + } + + /// Request extra justification. + pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { + let client = &self.client; + self.extra_justifications + .schedule((*hash, number), |base, block| is_descendent_of(&**client, base, block)) + } + + /// Clear extra justification requests. + pub fn clear_justification_requests(&mut self) { + self.extra_justifications.reset(); + } + + /// Configure an explicit fork sync request in case external code has detected that there is a + /// stale fork missing. + /// + /// Note that this function should not be used for recent blocks. + /// Sync should be able to download all the recent forks normally. + /// + /// Passing empty `peers` set effectively removes the sync request. + // The implementation is similar to `on_validated_block_announce` with unknown parent hash. + pub fn set_sync_fork_request( + &mut self, + mut peers: Vec, + hash: &B::Hash, + number: NumberFor, + ) { + if peers.is_empty() { + peers = self + .peers + .iter() + // Only request blocks from peers who are ahead or on a par. + .filter(|(_, peer)| peer.best_number >= number) + .map(|(id, _)| *id) + .collect(); + + debug!( + target: LOG_TARGET, + "Explicit sync request for block {hash:?} with no peers specified. \ + Syncing from these peers {peers:?} instead.", + ); + } else { + debug!( + target: LOG_TARGET, + "Explicit sync request for block {hash:?} with {peers:?}", + ); + } + + if self.is_known(hash) { + debug!(target: LOG_TARGET, "Refusing to sync known hash {hash:?}"); + return + } + + trace!(target: LOG_TARGET, "Downloading requested old fork {hash:?}"); + for peer_id in &peers { + if let Some(peer) = self.peers.get_mut(peer_id) { + if let PeerSyncState::AncestorSearch { .. } = peer.state { + continue + } + + if number > peer.best_number { + peer.best_number = number; + peer.best_hash = *hash; + } + self.allowed_requests.add(peer_id); + } + } + + self.fork_targets + .entry(*hash) + .or_insert_with(|| ForkTarget { number, peers: Default::default(), parent_hash: None }) + .peers + .extend(peers); + } + + /// Submit a block response for processing. + #[must_use] + pub fn on_block_data( + &mut self, + who: &PeerId, + request: Option>, + response: BlockResponse, + ) -> Result, BadPeer> { + self.downloaded_blocks += response.blocks.len(); + let mut gap = false; + let new_blocks: Vec> = if let Some(peer) = self.peers.get_mut(who) { + let mut blocks = response.blocks; + if request.as_ref().map_or(false, |r| r.direction == Direction::Descending) { + trace!(target: LOG_TARGET, "Reversing incoming block list"); + blocks.reverse() + } + self.allowed_requests.add(who); + if let Some(request) = request { + match &mut peer.state { + PeerSyncState::DownloadingNew(_) => { + self.blocks.clear_peer_download(who); + peer.state = PeerSyncState::Available; + if let Some(start_block) = + validate_blocks::(&blocks, who, Some(request))? + { + self.blocks.insert(start_block, blocks, *who); + } + self.ready_blocks() + }, + PeerSyncState::DownloadingGap(_) => { + peer.state = PeerSyncState::Available; + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_peer_download(who); + if let Some(start_block) = + validate_blocks::(&blocks, who, Some(request))? + { + gap_sync.blocks.insert(start_block, blocks, *who); + } + gap = true; + let blocks: Vec<_> = gap_sync + .blocks + .ready_blocks(gap_sync.best_queued_number + One::one()) + .into_iter() + .map(|block_data| { + let justifications = + block_data.block.justifications.or_else(|| { + legacy_justification_mapping( + block_data.block.justification, + ) + }); + IncomingBlock { + hash: block_data.block.hash, + header: block_data.block.header, + body: block_data.block.body, + indexed_body: block_data.block.indexed_body, + justifications, + origin: block_data.origin, + allow_missing_state: true, + import_existing: self.import_existing, + skip_execution: true, + state: None, + } + }) + .collect(); + debug!( + target: LOG_TARGET, + "Drained {} gap blocks from {}", + blocks.len(), + gap_sync.best_queued_number, + ); + blocks + } else { + debug!(target: LOG_TARGET, "Unexpected gap block response from {who}"); + return Err(BadPeer(*who, rep::NO_BLOCK)) + } + }, + PeerSyncState::DownloadingStale(_) => { + peer.state = PeerSyncState::Available; + if blocks.is_empty() { + debug!(target: LOG_TARGET, "Empty block response from {who}"); + return Err(BadPeer(*who, rep::NO_BLOCK)) + } + validate_blocks::(&blocks, who, Some(request))?; + blocks + .into_iter() + .map(|b| { + let justifications = b + .justifications + .or_else(|| legacy_justification_mapping(b.justification)); + IncomingBlock { + hash: b.hash, + header: b.header, + body: b.body, + indexed_body: None, + justifications, + origin: Some(*who), + allow_missing_state: true, + import_existing: self.import_existing, + skip_execution: self.skip_execution(), + state: None, + } + }) + .collect() + }, + PeerSyncState::AncestorSearch { current, start, state } => { + let matching_hash = match (blocks.get(0), self.client.hash(*current)) { + (Some(block), Ok(maybe_our_block_hash)) => { + trace!( + target: LOG_TARGET, + "Got ancestry block #{} ({}) from peer {}", + current, + block.hash, + who, + ); + maybe_our_block_hash.filter(|x| x == &block.hash) + }, + (None, _) => { + debug!( + target: LOG_TARGET, + "Invalid response when searching for ancestor from {who}", + ); + return Err(BadPeer(*who, rep::UNKNOWN_ANCESTOR)) + }, + (_, Err(e)) => { + info!( + target: LOG_TARGET, + "❌ Error answering legitimate blockchain query: {e}", + ); + return Err(BadPeer(*who, rep::BLOCKCHAIN_READ_ERROR)) + }, + }; + if matching_hash.is_some() { + if *start < self.best_queued_number && + self.best_queued_number <= peer.best_number + { + // We've made progress on this chain since the search was started. + // Opportunistically set common number to updated number + // instead of the one that started the search. + peer.common_number = self.best_queued_number; + } else if peer.common_number < *current { + peer.common_number = *current; + } + } + if matching_hash.is_none() && current.is_zero() { + trace!( + target:LOG_TARGET, + "Ancestry search: genesis mismatch for peer {who}", + ); + return Err(BadPeer(*who, rep::GENESIS_MISMATCH)) + } + if let Some((next_state, next_num)) = + handle_ancestor_search_state(state, *current, matching_hash.is_some()) + { + peer.state = PeerSyncState::AncestorSearch { + current: next_num, + start: *start, + state: next_state, + }; + return Ok(OnBlockData::Request(*who, ancestry_request::(next_num))) + } else { + // Ancestry search is complete. Check if peer is on a stale fork unknown + // to us and add it to sync targets if necessary. + trace!( + target: LOG_TARGET, + "Ancestry search complete. Ours={} ({}), Theirs={} ({}), Common={:?} ({})", + self.best_queued_hash, + self.best_queued_number, + peer.best_hash, + peer.best_number, + matching_hash, + peer.common_number, + ); + if peer.common_number < peer.best_number && + peer.best_number < self.best_queued_number + { + trace!( + target: LOG_TARGET, + "Added fork target {} for {}", + peer.best_hash, + who, + ); + self.fork_targets + .entry(peer.best_hash) + .or_insert_with(|| ForkTarget { + number: peer.best_number, + parent_hash: None, + peers: Default::default(), + }) + .peers + .insert(*who); + } + peer.state = PeerSyncState::Available; + Vec::new() + } + }, + PeerSyncState::DownloadingWarpTargetBlock => { + peer.state = PeerSyncState::Available; + if let Some(warp_sync) = &mut self.warp_sync { + if blocks.len() == 1 { + validate_blocks::(&blocks, who, Some(request))?; + match warp_sync.import_target_block( + blocks.pop().expect("`blocks` len checked above."), + ) { + warp::TargetBlockImportResult::Success => + return Ok(OnBlockData::Continue), + warp::TargetBlockImportResult::BadResponse => + return Err(BadPeer(*who, rep::VERIFICATION_FAIL)), + } + } else if blocks.is_empty() { + debug!(target: LOG_TARGET, "Empty block response from {who}"); + return Err(BadPeer(*who, rep::NO_BLOCK)) + } else { + debug!( + target: LOG_TARGET, + "Too many blocks ({}) in warp target block response from {}", + blocks.len(), + who, + ); + return Err(BadPeer(*who, rep::NOT_REQUESTED)) + } + } else { + debug!( + target: LOG_TARGET, + "Logic error: we think we are downloading warp target block from {}, but no warp sync is happening.", + who, + ); + return Ok(OnBlockData::Continue) + } + }, + PeerSyncState::Available | + PeerSyncState::DownloadingJustification(..) | + PeerSyncState::DownloadingState | + PeerSyncState::DownloadingWarpProof => Vec::new(), + } + } else { + // When request.is_none() this is a block announcement. Just accept blocks. + validate_blocks::(&blocks, who, None)?; + blocks + .into_iter() + .map(|b| { + let justifications = b + .justifications + .or_else(|| legacy_justification_mapping(b.justification)); + IncomingBlock { + hash: b.hash, + header: b.header, + body: b.body, + indexed_body: None, + justifications, + origin: Some(*who), + allow_missing_state: true, + import_existing: false, + skip_execution: true, + state: None, + } + }) + .collect() + } + } else { + // We don't know of this peer, so we also did not request anything from it. + return Err(BadPeer(*who, rep::NOT_REQUESTED)) + }; + + Ok(OnBlockData::Import(self.validate_and_queue_blocks(new_blocks, gap))) + } + + /// Submit a justification response for processing. + #[must_use] + pub fn on_block_justification( + &mut self, + who: PeerId, + response: BlockResponse, + ) -> Result, BadPeer> { + let peer = if let Some(peer) = self.peers.get_mut(&who) { + peer + } else { + error!( + target: LOG_TARGET, + "💔 Called on_block_justification with a peer ID of an unknown peer", + ); + return Ok(OnBlockJustification::Nothing) + }; + + self.allowed_requests.add(&who); + if let PeerSyncState::DownloadingJustification(hash) = peer.state { + peer.state = PeerSyncState::Available; + + // We only request one justification at a time + let justification = if let Some(block) = response.blocks.into_iter().next() { + if hash != block.hash { + warn!( + target: LOG_TARGET, + "💔 Invalid block justification provided by {}: requested: {:?} got: {:?}", + who, + hash, + block.hash, + ); + return Err(BadPeer(who, rep::BAD_JUSTIFICATION)) + } + + block + .justifications + .or_else(|| legacy_justification_mapping(block.justification)) + } else { + // we might have asked the peer for a justification on a block that we assumed it + // had but didn't (regardless of whether it had a justification for it or not). + trace!( + target: LOG_TARGET, + "Peer {who:?} provided empty response for justification request {hash:?}", + ); + + None + }; + + if let Some((peer_id, hash, number, justifications)) = + self.extra_justifications.on_response(who, justification) + { + return Ok(OnBlockJustification::Import { peer_id, hash, number, justifications }) + } + } + + Ok(OnBlockJustification::Nothing) + } + + /// Report a justification import (successful or not). + pub fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { + let finalization_result = if success { Ok((hash, number)) } else { Err(()) }; + self.extra_justifications + .try_finalize_root((hash, number), finalization_result, true); + self.allowed_requests.set_all(); + } + + /// Notify sync that a block has been finalized. + pub fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { + let client = &self.client; + let r = self.extra_justifications.on_block_finalized(hash, number, |base, block| { + is_descendent_of(&**client, base, block) + }); + + if let SyncMode::LightState { skip_proofs, .. } = &self.mode { + if self.state_sync.is_none() && !self.peers.is_empty() && self.queue_blocks.is_empty() { + // Finalized a recent block. + let mut heads: Vec<_> = self.peers.values().map(|peer| peer.best_number).collect(); + heads.sort(); + let median = heads[heads.len() / 2]; + if number + STATE_SYNC_FINALITY_THRESHOLD.saturated_into() >= median { + if let Ok(Some(header)) = self.client.header(*hash) { + log::debug!( + target: LOG_TARGET, + "Starting state sync for #{number} ({hash})", + ); + self.state_sync = Some(StateSync::new( + self.client.clone(), + header, + None, + None, + *skip_proofs, + )); + self.allowed_requests.set_all(); + } + } + } + } + + if let Err(err) = r { + warn!( + target: LOG_TARGET, + "💔 Error cleaning up pending extra justification data requests: {err}", + ); + } + } + + /// Submit a validated block announcement. + pub fn on_validated_block_announce( + &mut self, + is_best: bool, + who: PeerId, + announce: &BlockAnnounce, + ) { + let number = *announce.header.number(); + let hash = announce.header.hash(); + let parent_status = + self.block_status(announce.header.parent_hash()).unwrap_or(BlockStatus::Unknown); + let known_parent = parent_status != BlockStatus::Unknown; + let ancient_parent = parent_status == BlockStatus::InChainPruned; + + let known = self.is_known(&hash); + let peer = if let Some(peer) = self.peers.get_mut(&who) { + peer + } else { + error!(target: LOG_TARGET, "💔 Called `on_validated_block_announce` with a bad peer ID"); + return + }; + + if let PeerSyncState::AncestorSearch { .. } = peer.state { + trace!(target: LOG_TARGET, "Peer {} is in the ancestor search state.", who); + return + } + + if is_best { + // update their best block + peer.best_number = number; + peer.best_hash = hash; + } + + // If the announced block is the best they have and is not ahead of us, our common number + // is either one further ahead or it's the one they just announced, if we know about it. + if is_best { + if known && self.best_queued_number >= number { + self.update_peer_common_number(&who, number); + } else if announce.header.parent_hash() == &self.best_queued_hash || + known_parent && self.best_queued_number >= number + { + self.update_peer_common_number(&who, number.saturating_sub(One::one())); + } + } + self.allowed_requests.add(&who); + + // known block case + if known || self.is_already_downloading(&hash) { + trace!(target: "sync", "Known block announce from {}: {}", who, hash); + if let Some(target) = self.fork_targets.get_mut(&hash) { + target.peers.insert(who); + } + return + } + + if ancient_parent { + trace!( + target: "sync", + "Ignored ancient block announced from {}: {} {:?}", + who, + hash, + announce.header, + ); + return + } + + if self.status().state == SyncState::Idle { + trace!( + target: "sync", + "Added sync target for block announced from {}: {} {:?}", + who, + hash, + announce.summary(), + ); + self.fork_targets + .entry(hash) + .or_insert_with(|| ForkTarget { + number, + parent_hash: Some(*announce.header.parent_hash()), + peers: Default::default(), + }) + .peers + .insert(who); + } + } + + /// Notify that a sync peer has disconnected. + #[must_use] + pub fn peer_disconnected(&mut self, who: &PeerId) -> Option> { + self.blocks.clear_peer_download(who); + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_peer_download(who) + } + self.peers.remove(who); + self.extra_justifications.peer_disconnected(who); + self.allowed_requests.set_all(); + self.fork_targets.retain(|_, target| { + target.peers.remove(who); + !target.peers.is_empty() + }); + + let blocks = self.ready_blocks(); + + (!blocks.is_empty()).then(|| self.validate_and_queue_blocks(blocks, false)) + } + + /// Get prometheus metrics. + pub fn metrics(&self) -> Metrics { + Metrics { + queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), + fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), + justifications: self.extra_justifications.metrics(), + } + } + + /// Returns the median seen block number. + fn median_seen(&self) -> Option> { + let mut best_seens = self.peers.values().map(|p| p.best_number).collect::>(); + + if best_seens.is_empty() { + None + } else { + let middle = best_seens.len() / 2; + + // Not the "perfect median" when we have an even number of peers. + Some(*best_seens.select_nth_unstable(middle).1) + } + } + + fn required_block_attributes(&self) -> BlockAttributes { + match self.mode { + SyncMode::Full => + BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, + SyncMode::LightState { storage_chain_mode: false, .. } | SyncMode::Warp => + BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, + SyncMode::LightState { storage_chain_mode: true, .. } => + BlockAttributes::HEADER | + BlockAttributes::JUSTIFICATION | + BlockAttributes::INDEXED_BODY, + } + } + + fn skip_execution(&self) -> bool { + match self.mode { + SyncMode::Full => false, + SyncMode::LightState { .. } => true, + SyncMode::Warp => true, + } + } + + fn validate_and_queue_blocks( + &mut self, + mut new_blocks: Vec>, + gap: bool, + ) -> ImportBlocksAction { + let orig_len = new_blocks.len(); + new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); + if new_blocks.len() != orig_len { + debug!( + target: LOG_TARGET, + "Ignoring {} blocks that are already queued", + orig_len - new_blocks.len(), + ); + } + + let origin = if !gap && !self.status().state.is_major_syncing() { + BlockOrigin::NetworkBroadcast + } else { + BlockOrigin::NetworkInitialSync + }; + + if let Some((h, n)) = new_blocks + .last() + .and_then(|b| b.header.as_ref().map(|h| (&b.hash, *h.number()))) + { + trace!( + target:LOG_TARGET, + "Accepted {} blocks ({:?}) with origin {:?}", + new_blocks.len(), + h, + origin, + ); + self.on_block_queued(h, n) + } + self.queue_blocks.extend(new_blocks.iter().map(|b| b.hash)); + + ImportBlocksAction { origin, blocks: new_blocks } + } + + fn update_peer_common_number(&mut self, peer_id: &PeerId, new_common: NumberFor) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.update_common_number(new_common); + } + } + + /// Called when a block has been queued for import. + /// + /// Updates our internal state for best queued block and then goes + /// through all peers to update our view of their state as well. + fn on_block_queued(&mut self, hash: &B::Hash, number: NumberFor) { + if self.fork_targets.remove(hash).is_some() { + trace!(target: LOG_TARGET, "Completed fork sync {hash:?}"); + } + if let Some(gap_sync) = &mut self.gap_sync { + if number > gap_sync.best_queued_number && number <= gap_sync.target { + gap_sync.best_queued_number = number; + } + } + if number > self.best_queued_number { + self.best_queued_number = number; + self.best_queued_hash = *hash; + // Update common blocks + for (n, peer) in self.peers.iter_mut() { + if let PeerSyncState::AncestorSearch { .. } = peer.state { + // Wait for ancestry search to complete first. + continue + } + let new_common_number = + if peer.best_number >= number { number } else { peer.best_number }; + trace!( + target: LOG_TARGET, + "Updating peer {} info, ours={}, common={}->{}, their best={}", + n, + number, + peer.common_number, + new_common_number, + peer.best_number, + ); + peer.common_number = new_common_number; + } + } + self.allowed_requests.set_all(); + } + + /// Restart the sync process. This will reset all pending block requests and return an iterator + /// of new block requests to make to peers. Peers that were downloading finality data (i.e. + /// their state was `DownloadingJustification`) are unaffected and will stay in the same state. + fn restart(&mut self) -> impl Iterator, BadPeer>> + '_ { + self.blocks.clear(); + if let Err(e) = self.reset_sync_start_point() { + warn!(target: LOG_TARGET, "💔 Unable to restart sync: {e}"); + } + self.allowed_requests.set_all(); + debug!( + target: LOG_TARGET, + "Restarted with {} ({})", + self.best_queued_number, + self.best_queued_hash, + ); + let old_peers = std::mem::take(&mut self.peers); + + old_peers.into_iter().filter_map(move |(peer_id, mut p)| { + // peers that were downloading justifications + // should be kept in that state. + if let PeerSyncState::DownloadingJustification(_) = p.state { + // We make sure our commmon number is at least something we have. + p.common_number = self.best_queued_number; + self.peers.insert(peer_id, p); + return None + } + + // handle peers that were in other states. + match self.new_peer(peer_id, p.best_hash, p.best_number) { + // since the request is not a justification, remove it from pending responses + Ok(None) => Some(Ok(BlockRequestAction::RemoveStale { peer_id })), + // update the request if the new one is available + Ok(Some(request)) => Some(Ok(BlockRequestAction::SendRequest { peer_id, request })), + // this implies that we need to drop pending response from the peer + Err(e) => Some(Err(e)), + } + }) + } + + /// Find a block to start sync from. If we sync with state, that's the latest block we have + /// state for. + fn reset_sync_start_point(&mut self) -> Result<(), ClientError> { + let info = self.client.info(); + if matches!(self.mode, SyncMode::LightState { .. }) && info.finalized_state.is_some() { + warn!( + target: LOG_TARGET, + "Can't use fast sync mode with a partially synced database. Reverting to full sync mode." + ); + self.mode = SyncMode::Full; + } + if matches!(self.mode, SyncMode::Warp) && info.finalized_state.is_some() { + warn!( + target: LOG_TARGET, + "Can't use warp sync mode with a partially synced database. Reverting to full sync mode." + ); + self.mode = SyncMode::Full; + } + self.import_existing = false; + self.best_queued_hash = info.best_hash; + self.best_queued_number = info.best_number; + + if self.mode == SyncMode::Full && + self.client.block_status(info.best_hash)? != BlockStatus::InChainWithState + { + self.import_existing = true; + // Latest state is missing, start with the last finalized state or genesis instead. + if let Some((hash, number)) = info.finalized_state { + debug!(target: LOG_TARGET, "Starting from finalized state #{number}"); + self.best_queued_hash = hash; + self.best_queued_number = number; + } else { + debug!(target: LOG_TARGET, "Restarting from genesis"); + self.best_queued_hash = Default::default(); + self.best_queued_number = Zero::zero(); + } + } + + if let Some((start, end)) = info.block_gap { + debug!(target: LOG_TARGET, "Starting gap sync #{start} - #{end}"); + self.gap_sync = Some(GapSync { + best_queued_number: start - One::one(), + target: end, + blocks: BlockCollection::new(), + }); + } + trace!( + target: LOG_TARGET, + "Restarted sync at #{} ({:?})", + self.best_queued_number, + self.best_queued_hash, + ); + Ok(()) + } + + /// What is the status of the block corresponding to the given hash? + fn block_status(&self, hash: &B::Hash) -> Result { + if self.queue_blocks.contains(hash) { + return Ok(BlockStatus::Queued) + } + self.client.block_status(*hash) + } + + /// Is the block corresponding to the given hash known? + fn is_known(&self, hash: &B::Hash) -> bool { + self.block_status(hash).ok().map_or(false, |s| s != BlockStatus::Unknown) + } + + /// Is any peer downloading the given hash? + fn is_already_downloading(&self, hash: &B::Hash) -> bool { + self.peers + .iter() + .any(|(_, p)| p.state == PeerSyncState::DownloadingStale(*hash)) + } + + /// Check if the peer is known to the sync state machine. Used for sanity checks. + pub fn is_peer_known(&self, peer_id: &PeerId) -> bool { + self.peers.contains_key(peer_id) + } + + /// Get the set of downloaded blocks that are ready to be queued for import. + fn ready_blocks(&mut self) -> Vec> { + self.blocks + .ready_blocks(self.best_queued_number + One::one()) + .into_iter() + .map(|block_data| { + let justifications = block_data + .block + .justifications + .or_else(|| legacy_justification_mapping(block_data.block.justification)); + IncomingBlock { + hash: block_data.block.hash, + header: block_data.block.header, + body: block_data.block.body, + indexed_body: block_data.block.indexed_body, + justifications, + origin: block_data.origin, + allow_missing_state: true, + import_existing: self.import_existing, + skip_execution: self.skip_execution(), + state: None, + } + }) + .collect() + } + + /// Set the warp sync target block externally in case we skip warp proofs downloading. + pub fn set_warp_sync_target_block(&mut self, header: B::Header) { + if let Some(ref mut warp_sync) = self.warp_sync { + warp_sync.set_target_block(header); + } else { + self.warp_sync_target_block_header = Some(header); + } + } + + /// Generate block request for downloading of the target block body during warp sync. + fn warp_target_block_request(&mut self) -> Option<(PeerId, BlockRequest)> { + let sync = &self.warp_sync.as_ref()?; + + if self.allowed_requests.is_empty() || + sync.is_complete() || + self.peers + .iter() + .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpTargetBlock) + { + // Only one pending warp target block request is allowed. + return None + } + + if let Some((target_number, request)) = sync.next_target_block_request() { + // Find a random peer that has a block with the target number. + for (id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= target_number { + trace!(target: LOG_TARGET, "New warp target block request for {id}"); + peer.state = PeerSyncState::DownloadingWarpTargetBlock; + self.allowed_requests.clear(); + return Some((*id, request)) + } + } + } + + None + } + + /// Submit blocks received in a response. + #[must_use] + pub fn on_block_response( + &mut self, + peer_id: PeerId, + request: BlockRequest, + blocks: Vec>, + ) -> OnBlockResponse { + let block_response = BlockResponse:: { id: request.id, blocks }; + + let blocks_range = || match ( + block_response + .blocks + .first() + .and_then(|b| b.header.as_ref().map(|h| h.number())), + block_response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), + ) { + (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), + (Some(first), Some(_)) => format!(" ({})", first), + _ => Default::default(), + }; + trace!( + target: LOG_TARGET, + "BlockResponse {} from {} with {} blocks {}", + block_response.id, + peer_id, + block_response.blocks.len(), + blocks_range(), + ); + + if request.fields == BlockAttributes::JUSTIFICATION { + match self.on_block_justification(peer_id, block_response) { + Ok(OnBlockJustification::Nothing) => OnBlockResponse::Nothing, + Ok(OnBlockJustification::Import { peer_id, hash, number, justifications }) => + OnBlockResponse::ImportJustifications(ImportJustificationsAction { + peer_id, + hash, + number, + justifications, + }), + Err(BadPeer(id, repu)) => { + self.network_service + .disconnect_peer(id, self.block_announce_protocol_name.clone()); + self.network_service.report_peer(id, repu); + OnBlockResponse::Nothing + }, + } + } else { + match self.on_block_data(&peer_id, Some(request), block_response) { + Ok(OnBlockData::Import(action)) => OnBlockResponse::ImportBlocks(action), + Ok(OnBlockData::Request(peer_id, request)) => + OnBlockResponse::SendBlockRequest { peer_id, request }, + Ok(OnBlockData::Continue) => OnBlockResponse::Nothing, + Err(BadPeer(id, repu)) => { + self.network_service + .disconnect_peer(id, self.block_announce_protocol_name.clone()); + self.network_service.report_peer(id, repu); + OnBlockResponse::Nothing + }, + } + } + } + + /// Submit a state received in a response. + #[must_use] + pub fn on_state_response( + &mut self, + peer_id: PeerId, + response: OpaqueStateResponse, + ) -> Option> { + match self.on_state_data(&peer_id, response) { + Ok(OnStateData::Import(origin, block)) => + Some(ImportBlocksAction { origin, blocks: vec![block] }), + Ok(OnStateData::Continue) => None, + Err(BadPeer(id, repu)) => { + self.network_service + .disconnect_peer(id, self.block_announce_protocol_name.clone()); + self.network_service.report_peer(id, repu); + None + }, + } + } + + /// Submit a warp proof response received. + pub fn on_warp_sync_response(&mut self, peer_id: PeerId, response: EncodedProof) { + if let Err(BadPeer(id, repu)) = self.on_warp_sync_data(&peer_id, response) { + self.network_service + .disconnect_peer(id, self.block_announce_protocol_name.clone()); + self.network_service.report_peer(id, repu); + } + } + + /// Get justification requests scheduled by sync to be sent out. + pub fn justification_requests(&mut self) -> Vec<(PeerId, BlockRequest)> { + let peers = &mut self.peers; + let mut matcher = self.extra_justifications.matcher(); + std::iter::from_fn(move || { + if let Some((peer, request)) = matcher.next(peers) { + peers + .get_mut(&peer) + .expect( + "`Matcher::next` guarantees the `PeerId` comes from the given peers; qed", + ) + .state = PeerSyncState::DownloadingJustification(request.0); + let req = BlockRequest:: { + id: 0, + fields: BlockAttributes::JUSTIFICATION, + from: FromBlock::Hash(request.0), + direction: Direction::Ascending, + max: Some(1), + }; + Some((peer, req)) + } else { + None + } + }) + .collect() + } + + /// Get block requests scheduled by sync to be sent out. + pub fn block_requests(&mut self) -> Vec<(PeerId, BlockRequest)> { + if self.mode == SyncMode::Warp { + return self + .warp_target_block_request() + .map_or_else(|| Vec::new(), |req| Vec::from([req])) + } + + if self.allowed_requests.is_empty() || self.state_sync.is_some() { + return Vec::new() + } + + if self.queue_blocks.len() > MAX_IMPORTING_BLOCKS { + trace!(target: LOG_TARGET, "Too many blocks in the queue."); + return Vec::new() + } + let is_major_syncing = self.status().state.is_major_syncing(); + let attrs = self.required_block_attributes(); + let blocks = &mut self.blocks; + let fork_targets = &mut self.fork_targets; + let last_finalized = + std::cmp::min(self.best_queued_number, self.client.info().finalized_number); + let best_queued = self.best_queued_number; + let client = &self.client; + let queue = &self.queue_blocks; + let allowed_requests = self.allowed_requests.take(); + let max_parallel = if is_major_syncing { 1 } else { self.max_parallel_downloads }; + let max_blocks_per_request = self.max_blocks_per_request; + let gap_sync = &mut self.gap_sync; + self.peers + .iter_mut() + .filter_map(move |(&id, peer)| { + if !peer.state.is_available() || !allowed_requests.contains(&id) { + return None + } + + // If our best queued is more than `MAX_BLOCKS_TO_LOOK_BACKWARDS` blocks away from + // the common number, the peer best number is higher than our best queued and the + // common number is smaller than the last finalized block number, we should do an + // ancestor search to find a better common block. If the queue is full we wait till + // all blocks are imported though. + if best_queued.saturating_sub(peer.common_number) > + MAX_BLOCKS_TO_LOOK_BACKWARDS.into() && + best_queued < peer.best_number && + peer.common_number < last_finalized && + queue.len() <= MAJOR_SYNC_BLOCKS.into() + { + trace!( + target: LOG_TARGET, + "Peer {:?} common block {} too far behind of our best {}. Starting ancestry search.", + id, + peer.common_number, + best_queued, + ); + let current = std::cmp::min(peer.best_number, best_queued); + peer.state = PeerSyncState::AncestorSearch { + current, + start: best_queued, + state: AncestorSearchState::ExponentialBackoff(One::one()), + }; + Some((id, ancestry_request::(current))) + } else if let Some((range, req)) = peer_block_request( + &id, + peer, + blocks, + attrs, + max_parallel, + max_blocks_per_request, + last_finalized, + best_queued, + ) { + peer.state = PeerSyncState::DownloadingNew(range.start); + trace!( + target: LOG_TARGET, + "New block request for {}, (best:{}, common:{}) {:?}", + id, + peer.best_number, + peer.common_number, + req, + ); + Some((id, req)) + } else if let Some((hash, req)) = fork_sync_request( + &id, + fork_targets, + best_queued, + last_finalized, + attrs, + |hash| { + if queue.contains(hash) { + BlockStatus::Queued + } else { + client.block_status(*hash).unwrap_or(BlockStatus::Unknown) + } + }, + max_blocks_per_request, + ) { + trace!(target: LOG_TARGET, "Downloading fork {hash:?} from {id}"); + peer.state = PeerSyncState::DownloadingStale(hash); + Some((id, req)) + } else if let Some((range, req)) = gap_sync.as_mut().and_then(|sync| { + peer_gap_block_request( + &id, + peer, + &mut sync.blocks, + attrs, + sync.target, + sync.best_queued_number, + max_blocks_per_request, + ) + }) { + peer.state = PeerSyncState::DownloadingGap(range.start); + trace!( + target: LOG_TARGET, + "New gap block request for {}, (best:{}, common:{}) {:?}", + id, + peer.best_number, + peer.common_number, + req, + ); + Some((id, req)) + } else { + None + } + }) + .collect() + } + + /// Get a state request scheduled by sync to be sent out (if any). + pub fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { + if self.allowed_requests.is_empty() { + return None + } + if (self.state_sync.is_some() || self.warp_sync.is_some()) && + self.peers.iter().any(|(_, peer)| peer.state == PeerSyncState::DownloadingState) + { + // Only one pending state request is allowed. + return None + } + if let Some(sync) = &self.state_sync { + if sync.is_complete() { + return None + } + + for (id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.common_number >= sync.target_block_num() { + peer.state = PeerSyncState::DownloadingState; + let request = sync.next_request(); + trace!(target: LOG_TARGET, "New StateRequest for {}: {:?}", id, request); + self.allowed_requests.clear(); + return Some((*id, OpaqueStateRequest(Box::new(request)))) + } + } + } + if let Some(sync) = &self.warp_sync { + if sync.is_complete() { + return None + } + if let (Some(request), Some(target)) = + (sync.next_state_request(), sync.target_block_number()) + { + for (id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= target { + trace!(target: LOG_TARGET, "New StateRequest for {id}: {request:?}"); + peer.state = PeerSyncState::DownloadingState; + self.allowed_requests.clear(); + return Some((*id, OpaqueStateRequest(Box::new(request)))) + } + } + } + } + None + } + + /// Get a warp proof request scheduled by sync to be sent out (if any). + pub fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { + if let Some(sync) = &self.warp_sync { + if self.allowed_requests.is_empty() || + sync.is_complete() || + self.peers + .iter() + .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpProof) + { + // Only one pending state request is allowed. + return None + } + if let Some(request) = sync.next_warp_proof_request() { + let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); + if !targets.is_empty() { + targets.sort(); + let median = targets[targets.len() / 2]; + // Find a random peer that is synced as much as peer majority. + for (id, peer) in self.peers.iter_mut() { + if peer.state.is_available() && peer.best_number >= median { + trace!(target: LOG_TARGET, "New WarpProofRequest for {id}"); + peer.state = PeerSyncState::DownloadingWarpProof; + self.allowed_requests.clear(); + return Some((*id, request)) + } + } + } + } + } + None + } + + fn on_state_data( + &mut self, + who: &PeerId, + response: OpaqueStateResponse, + ) -> Result, BadPeer> { + let response: Box = response.0.downcast().map_err(|_error| { + error!( + target: LOG_TARGET, + "Failed to downcast opaque state response, this is an implementation bug." + ); + + BadPeer(*who, rep::BAD_RESPONSE) + })?; + + if let Some(peer) = self.peers.get_mut(who) { + if let PeerSyncState::DownloadingState = peer.state { + peer.state = PeerSyncState::Available; + self.allowed_requests.set_all(); + } + } + let import_result = if let Some(sync) = &mut self.state_sync { + debug!( + target: LOG_TARGET, + "Importing state data from {} with {} keys, {} proof nodes.", + who, + response.entries.len(), + response.proof.len(), + ); + sync.import(*response) + } else if let Some(sync) = &mut self.warp_sync { + debug!( + target: LOG_TARGET, + "Importing state data from {} with {} keys, {} proof nodes.", + who, + response.entries.len(), + response.proof.len(), + ); + sync.import_state(*response) + } else { + debug!(target: LOG_TARGET, "Ignored obsolete state response from {who}"); + return Err(BadPeer(*who, rep::NOT_REQUESTED)) + }; + + match import_result { + ImportResult::Import(hash, header, state, body, justifications) => { + let origin = BlockOrigin::NetworkInitialSync; + let block = IncomingBlock { + hash, + header: Some(header), + body, + indexed_body: None, + justifications, + origin: None, + allow_missing_state: true, + import_existing: true, + skip_execution: self.skip_execution(), + state: Some(state), + }; + debug!(target: LOG_TARGET, "State download is complete. Import is queued"); + Ok(OnStateData::Import(origin, block)) + }, + ImportResult::Continue => Ok(OnStateData::Continue), + ImportResult::BadResponse => { + debug!(target: LOG_TARGET, "Bad state data received from {who}"); + Err(BadPeer(*who, rep::BAD_BLOCK)) + }, + } + } + + fn on_warp_sync_data(&mut self, who: &PeerId, response: EncodedProof) -> Result<(), BadPeer> { + if let Some(peer) = self.peers.get_mut(who) { + if let PeerSyncState::DownloadingWarpProof = peer.state { + peer.state = PeerSyncState::Available; + self.allowed_requests.set_all(); + } + } + let import_result = if let Some(sync) = &mut self.warp_sync { + debug!( + target: LOG_TARGET, + "Importing warp proof data from {}, {} bytes.", + who, + response.0.len(), + ); + sync.import_warp_proof(response) + } else { + debug!(target: LOG_TARGET, "Ignored obsolete warp sync response from {who}"); + return Err(BadPeer(*who, rep::NOT_REQUESTED)) + }; + + match import_result { + WarpProofImportResult::Success => Ok(()), + WarpProofImportResult::BadResponse => { + debug!(target: LOG_TARGET, "Bad proof data received from {who}"); + Err(BadPeer(*who, rep::BAD_BLOCK)) + }, + } + } + + /// A batch of blocks have been processed, with or without errors. + /// + /// Call this when a batch of blocks have been processed by the import + /// queue, with or without errors. If an error is returned, the pending response + /// from the peer must be dropped. + #[must_use] + pub fn on_blocks_processed( + &mut self, + imported: usize, + count: usize, + results: Vec<(Result>, BlockImportError>, B::Hash)>, + ) -> Box, BadPeer>>> { + trace!(target: LOG_TARGET, "Imported {imported} of {count}"); + + let mut output = Vec::new(); + + let mut has_error = false; + for (_, hash) in &results { + self.queue_blocks.remove(hash); + self.blocks.clear_queued(hash); + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_queued(hash); + } + } + for (result, hash) in results { + if has_error { + break + } + + has_error |= result.is_err(); + + match result { + Ok(BlockImportStatus::ImportedKnown(number, who)) => + if let Some(peer) = who { + self.update_peer_common_number(&peer, number); + }, + Ok(BlockImportStatus::ImportedUnknown(number, aux, who)) => { + if aux.clear_justification_requests { + trace!( + target: LOG_TARGET, + "Block imported clears all pending justification requests {number}: {hash:?}", + ); + self.clear_justification_requests(); + } + + if aux.needs_justification { + trace!( + target: LOG_TARGET, + "Block imported but requires justification {number}: {hash:?}", + ); + self.request_justification(&hash, number); + } + + if aux.bad_justification { + if let Some(ref peer) = who { + warn!("💔 Sent block with bad justification to import"); + output.push(Err(BadPeer(*peer, rep::BAD_JUSTIFICATION))); + } + } + + if let Some(peer) = who { + self.update_peer_common_number(&peer, number); + } + let state_sync_complete = + self.state_sync.as_ref().map_or(false, |s| s.target() == hash); + if state_sync_complete { + info!( + target: LOG_TARGET, + "State sync is complete ({} MiB), restarting block sync.", + self.state_sync.as_ref().map_or(0, |s| s.progress().size / (1024 * 1024)), + ); + self.state_sync = None; + self.mode = SyncMode::Full; + output.extend(self.restart()); + } + let warp_sync_complete = self + .warp_sync + .as_ref() + .map_or(false, |s| s.target_block_hash() == Some(hash)); + if warp_sync_complete { + info!( + target: LOG_TARGET, + "Warp sync is complete ({} MiB), restarting block sync.", + self.warp_sync.as_ref().map_or(0, |s| s.progress().total_bytes / (1024 * 1024)), + ); + self.warp_sync = None; + self.mode = SyncMode::Full; + output.extend(self.restart()); + } + let gap_sync_complete = + self.gap_sync.as_ref().map_or(false, |s| s.target == number); + if gap_sync_complete { + info!( + target: LOG_TARGET, + "Block history download is complete." + ); + self.gap_sync = None; + } + }, + Err(BlockImportError::IncompleteHeader(who)) => + if let Some(peer) = who { + warn!( + target: LOG_TARGET, + "💔 Peer sent block with incomplete header to import", + ); + output.push(Err(BadPeer(peer, rep::INCOMPLETE_HEADER))); + output.extend(self.restart()); + }, + Err(BlockImportError::VerificationFailed(who, e)) => { + let extra_message = + who.map_or_else(|| "".into(), |peer| format!(" received from ({peer})")); + + warn!( + target: LOG_TARGET, + "💔 Verification failed for block {hash:?}{extra_message}: {e:?}", + ); + + if let Some(peer) = who { + output.push(Err(BadPeer(peer, rep::VERIFICATION_FAIL))); + } + + output.extend(self.restart()); + }, + Err(BlockImportError::BadBlock(who)) => + if let Some(peer) = who { + warn!( + target: LOG_TARGET, + "💔 Block {hash:?} received from peer {peer} has been blacklisted", + ); + output.push(Err(BadPeer(peer, rep::BAD_BLOCK))); + }, + Err(BlockImportError::MissingState) => { + // This may happen if the chain we were requesting upon has been discarded + // in the meantime because other chain has been finalized. + // Don't mark it as bad as it still may be synced if explicitly requested. + trace!(target: LOG_TARGET, "Obsolete block {hash:?}"); + }, + e @ Err(BlockImportError::UnknownParent) | e @ Err(BlockImportError::Other(_)) => { + warn!(target: LOG_TARGET, "💔 Error importing block {hash:?}: {}", e.unwrap_err()); + self.state_sync = None; + self.warp_sync = None; + output.extend(self.restart()); + }, + Err(BlockImportError::Cancelled) => {}, + }; + } + + self.allowed_requests.set_all(); + Box::new(output.into_iter()) + } +} + +// This is purely during a backwards compatible transitionary period and should be removed +// once we can assume all nodes can send and receive multiple Justifications +// The ID tag is hardcoded here to avoid depending on the GRANDPA crate. +// See: https://github.com/paritytech/substrate/issues/8172 +fn legacy_justification_mapping( + justification: Option, +) -> Option { + justification.map(|just| (*b"FRNK", just).into()) +} + +/// Request the ancestry for a block. Sends a request for header and justification for the given +/// block number. Used during ancestry search. +fn ancestry_request(block: NumberFor) -> BlockRequest { + BlockRequest:: { + id: 0, + fields: BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION, + from: FromBlock::Number(block), + direction: Direction::Ascending, + max: Some(1), + } +} + +/// The ancestor search state expresses which algorithm, and its stateful parameters, we are using +/// to try to find an ancestor block +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub(crate) enum AncestorSearchState { + /// Use exponential backoff to find an ancestor, then switch to binary search. + /// We keep track of the exponent. + ExponentialBackoff(NumberFor), + /// Using binary search to find the best ancestor. + /// We keep track of left and right bounds. + BinarySearch(NumberFor, NumberFor), +} + +/// This function handles the ancestor search strategy used. The goal is to find a common point +/// that both our chains agree on that is as close to the tip as possible. +/// The way this works is we first have an exponential backoff strategy, where we try to step +/// forward until we find a block hash mismatch. The size of the step doubles each step we take. +/// +/// When we've found a block hash mismatch we then fall back to a binary search between the two +/// last known points to find the common block closest to the tip. +fn handle_ancestor_search_state( + state: &AncestorSearchState, + curr_block_num: NumberFor, + block_hash_match: bool, +) -> Option<(AncestorSearchState, NumberFor)> { + let two = >::one() + >::one(); + match state { + AncestorSearchState::ExponentialBackoff(next_distance_to_tip) => { + let next_distance_to_tip = *next_distance_to_tip; + if block_hash_match && next_distance_to_tip == One::one() { + // We found the ancestor in the first step so there is no need to execute binary + // search. + return None + } + if block_hash_match { + let left = curr_block_num; + let right = left + next_distance_to_tip / two; + let middle = left + (right - left) / two; + Some((AncestorSearchState::BinarySearch(left, right), middle)) + } else { + let next_block_num = + curr_block_num.checked_sub(&next_distance_to_tip).unwrap_or_else(Zero::zero); + let next_distance_to_tip = next_distance_to_tip * two; + Some(( + AncestorSearchState::ExponentialBackoff(next_distance_to_tip), + next_block_num, + )) + } + }, + AncestorSearchState::BinarySearch(mut left, mut right) => { + if left >= curr_block_num { + return None + } + if block_hash_match { + left = curr_block_num; + } else { + right = curr_block_num; + } + assert!(right >= left); + let middle = left + (right - left) / two; + if middle == curr_block_num { + None + } else { + Some((AncestorSearchState::BinarySearch(left, right), middle)) + } + }, + } +} + +/// Get a new block request for the peer if any. +fn peer_block_request( + id: &PeerId, + peer: &PeerSync, + blocks: &mut BlockCollection, + attrs: BlockAttributes, + max_parallel_downloads: u32, + max_blocks_per_request: u32, + finalized: NumberFor, + best_num: NumberFor, +) -> Option<(Range>, BlockRequest)> { + if best_num >= peer.best_number { + // Will be downloaded as alternative fork instead. + return None + } else if peer.common_number < finalized { + trace!( + target: LOG_TARGET, + "Requesting pre-finalized chain from {:?}, common={}, finalized={}, peer best={}, our best={}", + id, peer.common_number, finalized, peer.best_number, best_num, + ); + } + let range = blocks.needed_blocks( + *id, + max_blocks_per_request, + peer.best_number, + peer.common_number, + max_parallel_downloads, + MAX_DOWNLOAD_AHEAD, + )?; + + // The end is not part of the range. + let last = range.end.saturating_sub(One::one()); + + let from = if peer.best_number == last { + FromBlock::Hash(peer.best_hash) + } else { + FromBlock::Number(last) + }; + + let request = BlockRequest:: { + id: 0, + fields: attrs, + from, + direction: Direction::Descending, + max: Some((range.end - range.start).saturated_into::()), + }; + + Some((range, request)) +} + +/// Get a new block request for the peer if any. +fn peer_gap_block_request( + id: &PeerId, + peer: &PeerSync, + blocks: &mut BlockCollection, + attrs: BlockAttributes, + target: NumberFor, + common_number: NumberFor, + max_blocks_per_request: u32, +) -> Option<(Range>, BlockRequest)> { + let range = blocks.needed_blocks( + *id, + max_blocks_per_request, + std::cmp::min(peer.best_number, target), + common_number, + 1, + MAX_DOWNLOAD_AHEAD, + )?; + + // The end is not part of the range. + let last = range.end.saturating_sub(One::one()); + let from = FromBlock::Number(last); + + let request = BlockRequest:: { + id: 0, + fields: attrs, + from, + direction: Direction::Descending, + max: Some((range.end - range.start).saturated_into::()), + }; + Some((range, request)) +} + +/// Get pending fork sync targets for a peer. +fn fork_sync_request( + id: &PeerId, + targets: &mut HashMap>, + best_num: NumberFor, + finalized: NumberFor, + attributes: BlockAttributes, + check_block: impl Fn(&B::Hash) -> BlockStatus, + max_blocks_per_request: u32, +) -> Option<(B::Hash, BlockRequest)> { + targets.retain(|hash, r| { + if r.number <= finalized { + trace!( + target: LOG_TARGET, + "Removed expired fork sync request {:?} (#{})", + hash, + r.number, + ); + return false + } + if check_block(hash) != BlockStatus::Unknown { + trace!( + target: LOG_TARGET, + "Removed obsolete fork sync request {:?} (#{})", + hash, + r.number, + ); + return false + } + true + }); + for (hash, r) in targets { + if !r.peers.contains(&id) { + continue + } + // Download the fork only if it is behind or not too far ahead our tip of the chain + // Otherwise it should be downloaded in full sync mode. + if r.number <= best_num || + (r.number - best_num).saturated_into::() < max_blocks_per_request as u32 + { + let parent_status = r.parent_hash.as_ref().map_or(BlockStatus::Unknown, check_block); + let count = if parent_status == BlockStatus::Unknown { + (r.number - finalized).saturated_into::() // up to the last finalized block + } else { + // request only single block + 1 + }; + trace!( + target: LOG_TARGET, + "Downloading requested fork {hash:?} from {id}, {count} blocks", + ); + return Some(( + *hash, + BlockRequest:: { + id: 0, + fields: attributes, + from: FromBlock::Hash(*hash), + direction: Direction::Descending, + max: Some(count), + }, + )) + } else { + trace!(target: LOG_TARGET, "Fork too far in the future: {:?} (#{})", hash, r.number); + } + } + None +} + +/// Returns `true` if the given `block` is a descendent of `base`. +fn is_descendent_of( + client: &T, + base: &Block::Hash, + block: &Block::Hash, +) -> sp_blockchain::Result +where + Block: BlockT, + T: HeaderMetadata + ?Sized, +{ + if base == block { + return Ok(false) + } + + let ancestor = sp_blockchain::lowest_common_ancestor(client, *block, *base)?; + + Ok(ancestor.hash == *base) +} + +/// Validate that the given `blocks` are correct. +/// Returns the number of the first block in the sequence. +/// +/// It is expected that `blocks` are in ascending order. +fn validate_blocks( + blocks: &Vec>, + who: &PeerId, + request: Option>, +) -> Result>, BadPeer> { + if let Some(request) = request { + if Some(blocks.len() as _) > request.max { + debug!( + target: LOG_TARGET, + "Received more blocks than requested from {}. Expected in maximum {:?}, got {}.", + who, + request.max, + blocks.len(), + ); + + return Err(BadPeer(*who, rep::NOT_REQUESTED)) + } + + let block_header = + if request.direction == Direction::Descending { blocks.last() } else { blocks.first() } + .and_then(|b| b.header.as_ref()); + + let expected_block = block_header.as_ref().map_or(false, |h| match request.from { + FromBlock::Hash(hash) => h.hash() == hash, + FromBlock::Number(n) => h.number() == &n, + }); + + if !expected_block { + debug!( + target: LOG_TARGET, + "Received block that was not requested. Requested {:?}, got {:?}.", + request.from, + block_header, + ); + + return Err(BadPeer(*who, rep::NOT_REQUESTED)) + } + + if request.fields.contains(BlockAttributes::HEADER) && + blocks.iter().any(|b| b.header.is_none()) + { + trace!( + target: LOG_TARGET, + "Missing requested header for a block in response from {who}.", + ); + + return Err(BadPeer(*who, rep::BAD_RESPONSE)) + } + + if request.fields.contains(BlockAttributes::BODY) && blocks.iter().any(|b| b.body.is_none()) + { + trace!( + target: LOG_TARGET, + "Missing requested body for a block in response from {who}.", + ); + + return Err(BadPeer(*who, rep::BAD_RESPONSE)) + } + } + + for b in blocks { + if let Some(header) = &b.header { + let hash = header.hash(); + if hash != b.hash { + debug!( + target:LOG_TARGET, + "Bad header received from {}. Expected hash {:?}, got {:?}", + who, + b.hash, + hash, + ); + return Err(BadPeer(*who, rep::BAD_BLOCK)) + } + } + if let (Some(header), Some(body)) = (&b.header, &b.body) { + let expected = *header.extrinsics_root(); + let got = HashingFor::::ordered_trie_root( + body.iter().map(Encode::encode).collect(), + sp_runtime::StateVersion::V0, + ); + if expected != got { + debug!( + target:LOG_TARGET, + "Bad extrinsic root for a block {} received from {}. Expected {:?}, got {:?}", + b.hash, + who, + expected, + got, + ); + return Err(BadPeer(*who, rep::BAD_BLOCK)) + } + } + } + + Ok(blocks.first().and_then(|b| b.header.as_ref()).map(|h| *h.number())) +} diff --git a/substrate/client/network/sync/src/chain_sync/test.rs b/substrate/client/network/sync/src/chain_sync/test.rs new file mode 100644 index 000000000000..6f9fea1b161b --- /dev/null +++ b/substrate/client/network/sync/src/chain_sync/test.rs @@ -0,0 +1,1085 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Tests of [`ChainSync`]. + +use super::*; +use crate::service::network::NetworkServiceProvider; +use futures::executor::block_on; +use sc_block_builder::BlockBuilderProvider; +use sc_network_common::sync::message::{BlockAnnounce, BlockData, BlockState, FromBlock}; +use sp_blockchain::HeaderBackend; +use substrate_test_runtime_client::{ + runtime::{Block, Hash, Header}, + BlockBuilderExt, ClientBlockImportExt, ClientExt, DefaultTestClientBuilderExt, TestClient, + TestClientBuilder, TestClientBuilderExt, +}; + +#[test] +fn processes_empty_response_on_justification_request_for_unknown_block() { + // if we ask for a justification for a given block to a peer that doesn't know that block + // (different from not having a justification), the peer will reply with an empty response. + // internally we should process the response as the justification not being available. + + let client = Arc::new(TestClientBuilder::new().build()); + let peer_id = PeerId::random(); + + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 1, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let (a1_hash, a1_number) = { + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + (a1.hash(), *a1.header.number()) + }; + + // add a new peer with the same best block + sync.new_peer(peer_id, a1_hash, a1_number).unwrap(); + + // and request a justification for the block + sync.request_justification(&a1_hash, a1_number); + + // the justification request should be scheduled to that peer + assert!(sync + .justification_requests() + .iter() + .any(|(who, request)| { *who == peer_id && request.from == FromBlock::Hash(a1_hash) })); + + // there are no extra pending requests + assert_eq!(sync.extra_justifications.pending_requests().count(), 0); + + // there's one in-flight extra request to the expected peer + assert!(sync.extra_justifications.active_requests().any(|(who, (hash, number))| { + *who == peer_id && *hash == a1_hash && *number == a1_number + })); + + // if the peer replies with an empty response (i.e. it doesn't know the block), + // the active request should be cleared. + assert_eq!( + sync.on_block_justification(peer_id, BlockResponse:: { id: 0, blocks: vec![] }), + Ok(OnBlockJustification::Nothing), + ); + + // there should be no in-flight requests + assert_eq!(sync.extra_justifications.active_requests().count(), 0); + + // and the request should now be pending again, waiting for reschedule + assert!(sync + .extra_justifications + .pending_requests() + .any(|(hash, number)| { *hash == a1_hash && *number == a1_number })); +} + +#[test] +fn restart_doesnt_affect_peers_downloading_finality_data() { + let mut client = Arc::new(TestClientBuilder::new().build()); + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 1, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let peer_id1 = PeerId::random(); + let peer_id2 = PeerId::random(); + let peer_id3 = PeerId::random(); + + let mut new_blocks = |n| { + for _ in 0..n { + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + block_on(client.import(BlockOrigin::Own, block.clone())).unwrap(); + } + + let info = client.info(); + (info.best_hash, info.best_number) + }; + + let (b1_hash, b1_number) = new_blocks(50); + + // add 2 peers at blocks that we don't have locally + sync.new_peer(peer_id1, Hash::random(), 42).unwrap(); + sync.new_peer(peer_id2, Hash::random(), 10).unwrap(); + + // we wil send block requests to these peers + // for these blocks we don't know about + assert!(sync + .block_requests() + .into_iter() + .all(|(p, _)| { p == peer_id1 || p == peer_id2 })); + + // add a new peer at a known block + sync.new_peer(peer_id3, b1_hash, b1_number).unwrap(); + + // we request a justification for a block we have locally + sync.request_justification(&b1_hash, b1_number); + + // the justification request should be scheduled to the + // new peer which is at the given block + assert!(sync.justification_requests().iter().any(|(p, r)| { + *p == peer_id3 && + r.fields == BlockAttributes::JUSTIFICATION && + r.from == FromBlock::Hash(b1_hash) + })); + + assert_eq!( + sync.peers.get(&peer_id3).unwrap().state, + PeerSyncState::DownloadingJustification(b1_hash), + ); + + // we restart the sync state + let block_requests = sync.restart(); + + // which should make us send out block requests to the first two peers + assert!(block_requests.map(|r| r.unwrap()).all(|event| match event { + BlockRequestAction::SendRequest { peer_id, .. } => + peer_id == peer_id1 || peer_id == peer_id2, + BlockRequestAction::RemoveStale { .. } => false, + })); + + // peer 3 should be unaffected it was downloading finality data + assert_eq!( + sync.peers.get(&peer_id3).unwrap().state, + PeerSyncState::DownloadingJustification(b1_hash), + ); + + // Set common block to something that we don't have (e.g. failed import) + sync.peers.get_mut(&peer_id3).unwrap().common_number = 100; + let _ = sync.restart().count(); + assert_eq!(sync.peers.get(&peer_id3).unwrap().common_number, 50); +} + +/// Send a block annoucnement for the given `header`. +fn send_block_announce(header: Header, peer_id: PeerId, sync: &mut ChainSync) { + let announce = BlockAnnounce { + header: header.clone(), + state: Some(BlockState::Best), + data: Some(Vec::new()), + }; + + sync.on_validated_block_announce(true, peer_id, &announce); +} + +/// Create a block response from the given `blocks`. +fn create_block_response(blocks: Vec) -> BlockResponse { + BlockResponse:: { + id: 0, + blocks: blocks + .into_iter() + .map(|b| BlockData:: { + hash: b.hash(), + header: Some(b.header().clone()), + body: Some(b.deconstruct().1), + indexed_body: None, + receipt: None, + message_queue: None, + justification: None, + justifications: None, + }) + .collect(), + } +} + +/// Get a block request from `sync` and check that is matches the expected request. +fn get_block_request( + sync: &mut ChainSync, + from: FromBlock, + max: u32, + peer: &PeerId, +) -> BlockRequest { + let requests = sync.block_requests(); + + log::trace!(target: LOG_TARGET, "Requests: {requests:?}"); + + assert_eq!(1, requests.len()); + assert_eq!(*peer, requests[0].0); + + let request = requests[0].1.clone(); + + assert_eq!(from, request.from); + assert_eq!(Some(max), request.max); + request +} + +/// Build and import a new best block. +fn build_block(client: &mut Arc, at: Option, fork: bool) -> Block { + let at = at.unwrap_or_else(|| client.info().best_hash); + + let mut block_builder = client.new_block_at(at, Default::default(), false).unwrap(); + + if fork { + block_builder.push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])).unwrap(); + } + + let block = block_builder.build().unwrap().block; + + block_on(client.import(BlockOrigin::Own, block.clone())).unwrap(); + block +} + +/// This test is a regression test as observed on a real network. +/// +/// The node is connected to multiple peers. Both of these peers are having a best block (1) +/// that is below our best block (3). Now peer 2 announces a fork of block 3 that we will +/// request from peer 2. After importing the fork, peer 2 and then peer 1 will announce block 4. +/// But as peer 1 in our view is still at block 1, we will request block 2 (which we already +/// have) from it. In the meanwhile peer 2 sends us block 4 and 3 and we send another request +/// for block 2 to peer 2. Peer 1 answers with block 2 and then peer 2. This will need to +/// succeed, as we have requested block 2 from both peers. +#[test] +fn do_not_report_peer_on_block_response_for_block_request() { + sp_tracing::try_init_simple(); + + let mut client = Arc::new(TestClientBuilder::new().build()); + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 5, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let peer_id1 = PeerId::random(); + let peer_id2 = PeerId::random(); + + let mut client2 = client.clone(); + let mut build_block_at = |at, import| { + let mut block_builder = client2.new_block_at(at, Default::default(), false).unwrap(); + // Make sure we generate a different block as fork + block_builder.push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])).unwrap(); + + let block = block_builder.build().unwrap().block; + + if import { + block_on(client2.import(BlockOrigin::Own, block.clone())).unwrap(); + } + + block + }; + + let block1 = build_block(&mut client, None, false); + let block2 = build_block(&mut client, None, false); + let block3 = build_block(&mut client, None, false); + let block3_fork = build_block_at(block2.hash(), false); + + // Add two peers which are on block 1. + sync.new_peer(peer_id1, block1.hash(), 1).unwrap(); + sync.new_peer(peer_id2, block1.hash(), 1).unwrap(); + + // Tell sync that our best block is 3. + sync.update_chain_info(&block3.hash(), 3); + + // There should be no requests. + assert!(sync.block_requests().is_empty()); + + // Let peer2 announce a fork of block 3 + send_block_announce(block3_fork.header().clone(), peer_id2, &mut sync); + + // Import and tell sync that we now have the fork. + block_on(client.import(BlockOrigin::Own, block3_fork.clone())).unwrap(); + sync.update_chain_info(&block3_fork.hash(), 3); + + let block4 = build_block_at(block3_fork.hash(), false); + + // Let peer2 announce block 4 and check that sync wants to get the block. + send_block_announce(block4.header().clone(), peer_id2, &mut sync); + + let request = get_block_request(&mut sync, FromBlock::Hash(block4.hash()), 2, &peer_id2); + + // Peer1 announces the same block, but as the common block is still `1`, sync will request + // block 2 again. + send_block_announce(block4.header().clone(), peer_id1, &mut sync); + + let request2 = get_block_request(&mut sync, FromBlock::Number(2), 1, &peer_id1); + + let response = create_block_response(vec![block4.clone(), block3_fork.clone()]); + let res = sync.on_block_data(&peer_id2, Some(request), response).unwrap(); + + // We should not yet import the blocks, because there is still an open request for fetching + // block `2` which blocks the import. + assert!( + matches!(res, OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.is_empty()) + ); + + let request3 = get_block_request(&mut sync, FromBlock::Number(2), 1, &peer_id2); + + let response = create_block_response(vec![block2.clone()]); + let res = sync.on_block_data(&peer_id1, Some(request2), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) + if blocks.iter().all(|b| [2, 3, 4].contains(b.header.as_ref().unwrap().number())) + )); + + let response = create_block_response(vec![block2.clone()]); + let res = sync.on_block_data(&peer_id2, Some(request3), response).unwrap(); + // Nothing to import + assert!( + matches!(res, OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.is_empty()) + ); +} + +fn unwrap_from_block_number(from: FromBlock) -> u64 { + if let FromBlock::Number(from) = from { + from + } else { + panic!("Expected a number!"); + } +} + +/// A regression test for a behavior we have seen on a live network. +/// +/// The scenario is that the node is doing a full resync and is connected to some node that is +/// doing a major sync as well. This other node that is doing a major sync will finish before +/// our node and send a block announcement message, but we don't have seen any block +/// announcement from this node in its sync process. Meaning our common number didn't change. It +/// is now expected that we start an ancestor search to find the common number. +#[test] +fn do_ancestor_search_when_common_block_to_best_qeued_gap_is_to_big() { + sp_tracing::try_init_simple(); + + let blocks = { + let mut client = Arc::new(TestClientBuilder::new().build()); + (0..MAX_DOWNLOAD_AHEAD * 2) + .map(|_| build_block(&mut client, None, false)) + .collect::>() + }; + + let mut client = Arc::new(TestClientBuilder::new().build()); + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let info = client.info(); + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 5, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let peer_id1 = PeerId::random(); + let peer_id2 = PeerId::random(); + + let best_block = blocks.last().unwrap().clone(); + let max_blocks_to_request = sync.max_blocks_per_request; + // Connect the node we will sync from + sync.new_peer(peer_id1, best_block.hash(), *best_block.header().number()) + .unwrap(); + sync.new_peer(peer_id2, info.best_hash, 0).unwrap(); + + let mut best_block_num = 0; + while best_block_num < MAX_DOWNLOAD_AHEAD { + let request = get_block_request( + &mut sync, + FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), + max_blocks_to_request as u32, + &peer_id1, + ); + + let from = unwrap_from_block_number(request.from.clone()); + + let mut resp_blocks = blocks[best_block_num as usize..from as usize].to_vec(); + resp_blocks.reverse(); + + let response = create_block_response(resp_blocks.clone()); + + let res = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == max_blocks_to_request as usize + ),); + + best_block_num += max_blocks_to_request as u32; + + let _ = sync.on_blocks_processed( + max_blocks_to_request as usize, + max_blocks_to_request as usize, + resp_blocks + .iter() + .rev() + .map(|b| { + ( + Ok(BlockImportStatus::ImportedUnknown( + *b.header().number(), + Default::default(), + Some(peer_id1), + )), + b.hash(), + ) + }) + .collect(), + ); + + resp_blocks + .into_iter() + .rev() + .for_each(|b| block_on(client.import_as_final(BlockOrigin::Own, b)).unwrap()); + } + + // "Wait" for the queue to clear + sync.queue_blocks.clear(); + + // Let peer2 announce that it finished syncing + send_block_announce(best_block.header().clone(), peer_id2, &mut sync); + + let (peer1_req, peer2_req) = + sync.block_requests().into_iter().fold((None, None), |res, req| { + if req.0 == peer_id1 { + (Some(req.1), res.1) + } else if req.0 == peer_id2 { + (res.0, Some(req.1)) + } else { + panic!("Unexpected req: {:?}", req) + } + }); + + // We should now do an ancestor search to find the correct common block. + let peer2_req = peer2_req.unwrap(); + assert_eq!(Some(1), peer2_req.max); + assert_eq!(FromBlock::Number(best_block_num as u64), peer2_req.from); + + let response = create_block_response(vec![blocks[(best_block_num - 1) as usize].clone()]); + let res = sync.on_block_data(&peer_id2, Some(peer2_req), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.is_empty() + ),); + + let peer1_from = unwrap_from_block_number(peer1_req.unwrap().from); + + // As we are on the same chain, we should directly continue with requesting blocks from + // peer 2 as well. + get_block_request( + &mut sync, + FromBlock::Number(peer1_from + max_blocks_to_request as u64), + max_blocks_to_request as u32, + &peer_id2, + ); +} + +/// A test that ensures that we can sync a huge fork. +/// +/// The following scenario: +/// A peer connects to us and we both have the common block 512. The last finalized is 2048. +/// Our best block is 4096. The peer send us a block announcement with 4097 from a fork. +/// +/// We will first do an ancestor search to find the common block. After that we start to sync +/// the fork and finish it ;) +#[test] +fn can_sync_huge_fork() { + sp_tracing::try_init_simple(); + + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut client = Arc::new(TestClientBuilder::new().build()); + let blocks = (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 4) + .map(|_| build_block(&mut client, None, false)) + .collect::>(); + + let fork_blocks = { + let mut client = Arc::new(TestClientBuilder::new().build()); + let fork_blocks = blocks[..MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2] + .into_iter() + .inspect(|b| block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap()) + .cloned() + .collect::>(); + + fork_blocks + .into_iter() + .chain( + (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 2 + 1) + .map(|_| build_block(&mut client, None, true)), + ) + .collect::>() + }; + + let info = client.info(); + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 5, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); + let just = (*b"TEST", Vec::new()); + client.finalize_block(finalized_block.hash(), Some(just)).unwrap(); + sync.update_chain_info(&info.best_hash, info.best_number); + + let peer_id1 = PeerId::random(); + + let common_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize / 2].clone(); + // Connect the node we will sync from + sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) + .unwrap(); + + send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); + + let mut request = + get_block_request(&mut sync, FromBlock::Number(info.best_number), 1, &peer_id1); + + // Do the ancestor search + loop { + let block = &fork_blocks[unwrap_from_block_number(request.from.clone()) as usize - 1]; + let response = create_block_response(vec![block.clone()]); + + let on_block_data = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + request = if let OnBlockData::Request(_peer, request) = on_block_data { + request + } else { + // We found the ancenstor + break + }; + + log::trace!(target: LOG_TARGET, "Request: {request:?}"); + } + + // Now request and import the fork. + let mut best_block_num = *finalized_block.header().number() as u32; + let max_blocks_to_request = sync.max_blocks_per_request; + while best_block_num < *fork_blocks.last().unwrap().header().number() as u32 - 1 { + let request = get_block_request( + &mut sync, + FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), + max_blocks_to_request as u32, + &peer_id1, + ); + + let from = unwrap_from_block_number(request.from.clone()); + + let mut resp_blocks = fork_blocks[best_block_num as usize..from as usize].to_vec(); + resp_blocks.reverse(); + + let response = create_block_response(resp_blocks.clone()); + + let res = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == sync.max_blocks_per_request as usize + ),); + + best_block_num += sync.max_blocks_per_request as u32; + + let _ = sync.on_blocks_processed( + max_blocks_to_request as usize, + max_blocks_to_request as usize, + resp_blocks + .iter() + .rev() + .map(|b| { + ( + Ok(BlockImportStatus::ImportedUnknown( + *b.header().number(), + Default::default(), + Some(peer_id1), + )), + b.hash(), + ) + }) + .collect(), + ); + + resp_blocks + .into_iter() + .rev() + .for_each(|b| block_on(client.import(BlockOrigin::Own, b)).unwrap()); + } + + // Request the tip + get_block_request(&mut sync, FromBlock::Hash(fork_blocks.last().unwrap().hash()), 1, &peer_id1); +} + +#[test] +fn syncs_fork_without_duplicate_requests() { + sp_tracing::try_init_simple(); + + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut client = Arc::new(TestClientBuilder::new().build()); + let blocks = (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 4) + .map(|_| build_block(&mut client, None, false)) + .collect::>(); + + let fork_blocks = { + let mut client = Arc::new(TestClientBuilder::new().build()); + let fork_blocks = blocks[..MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2] + .into_iter() + .inspect(|b| block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap()) + .cloned() + .collect::>(); + + fork_blocks + .into_iter() + .chain( + (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 2 + 1) + .map(|_| build_block(&mut client, None, true)), + ) + .collect::>() + }; + + let info = client.info(); + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 5, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); + let just = (*b"TEST", Vec::new()); + client.finalize_block(finalized_block.hash(), Some(just)).unwrap(); + sync.update_chain_info(&info.best_hash, info.best_number); + + let peer_id1 = PeerId::random(); + + let common_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize / 2].clone(); + // Connect the node we will sync from + sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) + .unwrap(); + + send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); + + let mut request = + get_block_request(&mut sync, FromBlock::Number(info.best_number), 1, &peer_id1); + + // Do the ancestor search + loop { + let block = &fork_blocks[unwrap_from_block_number(request.from.clone()) as usize - 1]; + let response = create_block_response(vec![block.clone()]); + + let on_block_data = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + request = if let OnBlockData::Request(_peer, request) = on_block_data { + request + } else { + // We found the ancenstor + break + }; + + log::trace!(target: LOG_TARGET, "Request: {request:?}"); + } + + // Now request and import the fork. + let mut best_block_num = *finalized_block.header().number() as u32; + let max_blocks_to_request = sync.max_blocks_per_request; + + let mut request = get_block_request( + &mut sync, + FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), + max_blocks_to_request as u32, + &peer_id1, + ); + let last_block_num = *fork_blocks.last().unwrap().header().number() as u32 - 1; + while best_block_num < last_block_num { + let from = unwrap_from_block_number(request.from.clone()); + + let mut resp_blocks = fork_blocks[best_block_num as usize..from as usize].to_vec(); + resp_blocks.reverse(); + + let response = create_block_response(resp_blocks.clone()); + + let res = sync.on_block_data(&peer_id1, Some(request.clone()), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == max_blocks_to_request as usize + ),); + + best_block_num += max_blocks_to_request as u32; + + if best_block_num < last_block_num { + // make sure we're not getting a duplicate request in the time before the blocks are + // processed + request = get_block_request( + &mut sync, + FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), + max_blocks_to_request as u32, + &peer_id1, + ); + } + + let mut notify_imported: Vec<_> = resp_blocks + .iter() + .rev() + .map(|b| { + ( + Ok(BlockImportStatus::ImportedUnknown( + *b.header().number(), + Default::default(), + Some(peer_id1), + )), + b.hash(), + ) + }) + .collect(); + + // The import queue may send notifications in batches of varying size. So we simulate + // this here by splitting the batch into 2 notifications. + let max_blocks_to_request = sync.max_blocks_per_request; + let second_batch = notify_imported.split_off(notify_imported.len() / 2); + let _ = sync.on_blocks_processed( + max_blocks_to_request as usize, + max_blocks_to_request as usize, + notify_imported, + ); + + let _ = sync.on_blocks_processed( + max_blocks_to_request as usize, + max_blocks_to_request as usize, + second_batch, + ); + + resp_blocks + .into_iter() + .rev() + .for_each(|b| block_on(client.import(BlockOrigin::Own, b)).unwrap()); + } + + // Request the tip + get_block_request(&mut sync, FromBlock::Hash(fork_blocks.last().unwrap().hash()), 1, &peer_id1); +} + +#[test] +fn removes_target_fork_on_disconnect() { + sp_tracing::try_init_simple(); + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut client = Arc::new(TestClientBuilder::new().build()); + let blocks = (0..3).map(|_| build_block(&mut client, None, false)).collect::>(); + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 1, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let peer_id1 = PeerId::random(); + let common_block = blocks[1].clone(); + // Connect the node we will sync from + sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) + .unwrap(); + + // Create a "new" header and announce it + let mut header = blocks[0].header().clone(); + header.number = 4; + send_block_announce(header, peer_id1, &mut sync); + assert!(sync.fork_targets.len() == 1); + + let _ = sync.peer_disconnected(&peer_id1); + assert!(sync.fork_targets.len() == 0); +} + +#[test] +fn can_import_response_with_missing_blocks() { + sp_tracing::try_init_simple(); + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut client2 = Arc::new(TestClientBuilder::new().build()); + let blocks = (0..4).map(|_| build_block(&mut client2, None, false)).collect::>(); + + let empty_client = Arc::new(TestClientBuilder::new().build()); + + let mut sync = ChainSync::new( + SyncMode::Full, + empty_client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 1, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let peer_id1 = PeerId::random(); + let best_block = blocks[3].clone(); + sync.new_peer(peer_id1, best_block.hash(), *best_block.header().number()) + .unwrap(); + + sync.peers.get_mut(&peer_id1).unwrap().state = PeerSyncState::Available; + sync.peers.get_mut(&peer_id1).unwrap().common_number = 0; + + // Request all missing blocks and respond only with some. + let request = get_block_request(&mut sync, FromBlock::Hash(best_block.hash()), 4, &peer_id1); + let response = + create_block_response(vec![blocks[3].clone(), blocks[2].clone(), blocks[1].clone()]); + sync.on_block_data(&peer_id1, Some(request.clone()), response).unwrap(); + assert_eq!(sync.best_queued_number, 0); + + // Request should only contain the missing block. + let request = get_block_request(&mut sync, FromBlock::Number(1), 1, &peer_id1); + let response = create_block_response(vec![blocks[0].clone()]); + sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + assert_eq!(sync.best_queued_number, 4); +} +#[test] +fn ancestor_search_repeat() { + let state = AncestorSearchState::::BinarySearch(1, 3); + assert!(handle_ancestor_search_state(&state, 2, true).is_none()); +} + +#[test] +fn sync_restart_removes_block_but_not_justification_requests() { + let mut client = Arc::new(TestClientBuilder::new().build()); + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 1, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + let peers = vec![PeerId::random(), PeerId::random()]; + + let mut new_blocks = |n| { + for _ in 0..n { + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + block_on(client.import(BlockOrigin::Own, block.clone())).unwrap(); + } + + let info = client.info(); + (info.best_hash, info.best_number) + }; + + let (b1_hash, b1_number) = new_blocks(50); + + // add new peer and request blocks from them + sync.new_peer(peers[0], Hash::random(), 42).unwrap(); + + // we don't actually perform any requests, just keep track of peers waiting for a response + let mut pending_responses = HashSet::new(); + + // we wil send block requests to these peers + // for these blocks we don't know about + for (peer, _request) in sync.block_requests() { + // "send" request + pending_responses.insert(peer); + } + + // add a new peer at a known block + sync.new_peer(peers[1], b1_hash, b1_number).unwrap(); + + // we request a justification for a block we have locally + sync.request_justification(&b1_hash, b1_number); + + // the justification request should be scheduled to the + // new peer which is at the given block + let mut requests = sync.justification_requests(); + assert_eq!(requests.len(), 1); + let (peer, _request) = requests.remove(0); + // "send" request + assert!(pending_responses.insert(peer)); + + assert!(!std::matches!( + sync.peers.get(&peers[0]).unwrap().state, + PeerSyncState::DownloadingJustification(_), + )); + assert_eq!( + sync.peers.get(&peers[1]).unwrap().state, + PeerSyncState::DownloadingJustification(b1_hash), + ); + assert_eq!(pending_responses.len(), 2); + + // restart sync + let request_events = sync.restart().collect::>(); + for event in request_events.iter() { + match event.as_ref().unwrap() { + BlockRequestAction::RemoveStale { peer_id } => { + pending_responses.remove(&peer_id); + }, + BlockRequestAction::SendRequest { peer_id, .. } => { + // we drop obsolete response, but don't register a new request, it's checked in + // the `assert!` below + pending_responses.remove(&peer_id); + }, + } + } + assert!(request_events.iter().any(|event| { + match event.as_ref().unwrap() { + BlockRequestAction::RemoveStale { .. } => false, + BlockRequestAction::SendRequest { peer_id, .. } => peer_id == &peers[0], + } + })); + + assert_eq!(pending_responses.len(), 1); + assert!(pending_responses.contains(&peers[1])); + assert_eq!( + sync.peers.get(&peers[1]).unwrap().state, + PeerSyncState::DownloadingJustification(b1_hash), + ); + let _ = sync.peer_disconnected(&peers[1]); + pending_responses.remove(&peers[1]); + assert_eq!(pending_responses.len(), 0); +} + +/// The test demonstrates https://github.com/paritytech/polkadot-sdk/issues/2094. +/// TODO: convert it into desired behavior test once the issue is fixed (see inline comments). +/// The issue: we currently rely on block numbers instead of block hash +/// to download blocks from peers. As a result, we can end up with blocks +/// from different forks as shown by the test. +#[test] +#[should_panic] +fn request_across_forks() { + sp_tracing::try_init_simple(); + + let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let mut client = Arc::new(TestClientBuilder::new().build()); + let blocks = (0..100).map(|_| build_block(&mut client, None, false)).collect::>(); + + let fork_a_blocks = { + let mut client = Arc::new(TestClientBuilder::new().build()); + let mut fork_blocks = blocks[..] + .into_iter() + .inspect(|b| { + assert!(matches!(client.block(*b.header.parent_hash()), Ok(Some(_)))); + block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap() + }) + .cloned() + .collect::>(); + for _ in 0..10 { + fork_blocks.push(build_block(&mut client, None, false)); + } + fork_blocks + }; + + let fork_b_blocks = { + let mut client = Arc::new(TestClientBuilder::new().build()); + let mut fork_blocks = blocks[..] + .into_iter() + .inspect(|b| { + assert!(matches!(client.block(*b.header.parent_hash()), Ok(Some(_)))); + block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap() + }) + .cloned() + .collect::>(); + for _ in 0..10 { + fork_blocks.push(build_block(&mut client, None, true)); + } + fork_blocks + }; + + let mut sync = ChainSync::new( + SyncMode::Full, + client.clone(), + ProtocolName::from("test-block-announce-protocol"), + 5, + 64, + None, + chain_sync_network_handle, + ) + .unwrap(); + + // Add the peers, all at the common ancestor 100. + let common_block = blocks.last().unwrap(); + let peer_id1 = PeerId::random(); + sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) + .unwrap(); + let peer_id2 = PeerId::random(); + sync.new_peer(peer_id2, common_block.hash(), *common_block.header().number()) + .unwrap(); + + // Peer 1 announces 107 from fork 1, 100-107 get downloaded. + { + let block = (&fork_a_blocks[106]).clone(); + let peer = peer_id1; + log::trace!(target: LOG_TARGET, "<1> {peer} announces from fork 1"); + send_block_announce(block.header().clone(), peer, &mut sync); + let request = get_block_request(&mut sync, FromBlock::Hash(block.hash()), 7, &peer); + let mut resp_blocks = fork_a_blocks[100_usize..107_usize].to_vec(); + resp_blocks.reverse(); + let response = create_block_response(resp_blocks.clone()); + let res = sync.on_block_data(&peer, Some(request), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == 7_usize + ),); + assert_eq!(sync.best_queued_number, 107); + assert_eq!(sync.best_queued_hash, block.hash()); + assert!(sync.is_known(&block.header.parent_hash())); + } + + // Peer 2 also announces 107 from fork 1. + { + let prev_best_number = sync.best_queued_number; + let prev_best_hash = sync.best_queued_hash; + let peer = peer_id2; + log::trace!(target: LOG_TARGET, "<2> {peer} announces from fork 1"); + for i in 100..107 { + let block = (&fork_a_blocks[i]).clone(); + send_block_announce(block.header().clone(), peer, &mut sync); + assert!(sync.block_requests().is_empty()); + } + assert_eq!(sync.best_queued_number, prev_best_number); + assert_eq!(sync.best_queued_hash, prev_best_hash); + } + + // Peer 2 undergoes reorg, announces 108 from fork 2, gets downloaded even though we + // don't have the parent from fork 2. + { + let block = (&fork_b_blocks[107]).clone(); + let peer = peer_id2; + log::trace!(target: LOG_TARGET, "<3> {peer} announces from fork 2"); + send_block_announce(block.header().clone(), peer, &mut sync); + // TODO: when the issue is fixed, this test can be changed to test the + // expected behavior instead. The needed changes would be: + // 1. Remove the `#[should_panic]` directive + // 2. These should be changed to check that sync.block_requests().is_empty(), after the + // block is announced. + let request = get_block_request(&mut sync, FromBlock::Hash(block.hash()), 1, &peer); + let response = create_block_response(vec![block.clone()]); + let res = sync.on_block_data(&peer, Some(request), response).unwrap(); + assert!(matches!( + res, + OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == 1_usize + ),); + assert!(sync.is_known(&block.header.parent_hash())); + } +} diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 1a383cdde479..0f689742bc58 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -24,12 +24,21 @@ use crate::{ BlockAnnounceValidationResult, BlockAnnounceValidator as BlockAnnounceValidatorStream, }, block_relay_protocol::{BlockDownloader, BlockResponseError}, + block_request_handler::MAX_BLOCKS_IN_RESPONSE, + chain_sync::{ + BlockRequestAction, ChainSync, ImportBlocksAction, ImportJustificationsAction, + OnBlockResponse, + }, pending_responses::{PendingResponses, ResponseEvent}, schema::v1::{StateRequest, StateResponse}, - service::{self, chain_sync::ToServiceCommand}, - warp::WarpSyncParams, - BlockRequestAction, ChainSync, ClientError, ImportBlocksAction, ImportJustificationsAction, - OnBlockResponse, SyncingService, + service::{ + self, + chain_sync::{SyncingService, ToServiceCommand}, + }, + types::{ + BadPeer, ExtendedPeerInfo, OpaqueStateRequest, OpaqueStateResponse, PeerRequest, SyncEvent, + }, + warp::{EncodedProof, WarpProofRequest, WarpSyncParams}, }; use codec::{Decode, Encode}; @@ -61,15 +70,10 @@ use sc_network::{ }; use sc_network_common::{ role::Roles, - sync::{ - message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, - warp::{EncodedProof, WarpProofRequest}, - BadPeer, ChainSync as ChainSyncT, ExtendedPeerInfo, OpaqueStateRequest, - OpaqueStateResponse, PeerRequest, SyncEvent, - }, + sync::message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState}, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; -use sp_blockchain::HeaderMetadata; +use sp_blockchain::{Error as ClientError, HeaderMetadata}; use sp_consensus::block_validation::BlockAnnounceValidator; use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; @@ -363,18 +367,17 @@ where ) -> Result<(Self, SyncingService, NonDefaultSetConfig), ClientError> { let mode = net_config.network_config.sync_mode; let max_parallel_downloads = net_config.network_config.max_parallel_downloads; - let max_blocks_per_request = if net_config.network_config.max_blocks_per_request > - crate::MAX_BLOCKS_IN_RESPONSE as u32 - { - log::info!( - target: LOG_TARGET, - "clamping maximum blocks per request to {}", - crate::MAX_BLOCKS_IN_RESPONSE, - ); - crate::MAX_BLOCKS_IN_RESPONSE as u32 - } else { - net_config.network_config.max_blocks_per_request - }; + let max_blocks_per_request = + if net_config.network_config.max_blocks_per_request > MAX_BLOCKS_IN_RESPONSE as u32 { + log::info!( + target: LOG_TARGET, + "clamping maximum blocks per request to {}", + MAX_BLOCKS_IN_RESPONSE, + ); + MAX_BLOCKS_IN_RESPONSE as u32 + } else { + net_config.network_config.max_blocks_per_request + }; let cache_capacity = (net_config.network_config.default_peers_set.in_peers + net_config.network_config.default_peers_set.out_peers) .max(1); diff --git a/substrate/client/network/sync/src/extra_requests.rs b/substrate/client/network/sync/src/extra_requests.rs index 09e6bdb57399..8edd1a772e26 100644 --- a/substrate/client/network/sync/src/extra_requests.rs +++ b/substrate/client/network/sync/src/extra_requests.rs @@ -16,11 +16,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{PeerSync, PeerSyncState}; +use crate::{ + chain_sync::{PeerSync, PeerSyncState}, + request_metrics::Metrics, +}; use fork_tree::ForkTree; use libp2p::PeerId; use log::{debug, trace, warn}; -use sc_network_common::sync::metrics::Metrics; use sp_blockchain::Error as ClientError; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; use std::{ @@ -343,7 +345,7 @@ impl<'a, B: BlockT> Matcher<'a, B> { #[cfg(test)] mod tests { use super::*; - use crate::PeerSync; + use crate::chain_sync::PeerSync; use quickcheck::{Arbitrary, Gen, QuickCheck}; use sp_blockchain::Error as ClientError; use sp_test_primitives::{Block, BlockNumber, Hash}; diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index 0c8e8a104e2a..c42b0601e659 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -16,67 +16,19 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains the state of the chain synchronization process -//! -//! At any given point in time, a running node tries as much as possible to be at the head of the -//! chain. This module handles the logic of which blocks to request from remotes, and processing -//! responses. It yields blocks to check and potentially move to the database. -//! -//! # Usage -//! -//! The `ChainSync` struct maintains the state of the block requests. Whenever something happens on -//! the network, or whenever a block has been successfully verified, call the appropriate method in -//! order to update it. - -use crate::{ - blocks::BlockCollection, - schema::v1::StateResponse, - state::StateSync, - warp::{WarpProofImportResult, WarpSync, WarpSyncConfig}, -}; - -use codec::Encode; -use extra_requests::ExtraRequests; -use libp2p::PeerId; -use log::{debug, error, info, trace, warn}; - -use sc_client_api::{BlockBackend, ProofProvider}; -use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock}; -use sc_network::types::ProtocolName; -use sc_network_common::sync::{ - message::{ - BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, Direction, - FromBlock, - }, - warp::{EncodedProof, WarpProofRequest, WarpSyncPhase, WarpSyncProgress}, - BadPeer, ChainSync as ChainSyncT, ImportBlocksAction, Metrics, OnBlockData, - OnBlockJustification, OnStateData, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, SyncMode, - SyncState, SyncStatus, -}; -use sp_arithmetic::traits::Saturating; -use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; -use sp_consensus::{BlockOrigin, BlockStatus}; -use sp_runtime::{ - traits::{ - Block as BlockT, CheckedSub, Hash, HashingFor, Header as HeaderT, NumberFor, One, - SaturatedConversion, Zero, - }, - EncodedJustification, Justifications, -}; - -use std::{ - collections::{HashMap, HashSet}, - ops::Range, - sync::Arc, -}; +//! Blockchain syncing implementation in Substrate. pub use service::chain_sync::SyncingService; +pub use types::{SyncEvent, SyncEventStream, SyncState, SyncStatus, SyncStatusProvider}; mod block_announce_validator; +mod chain_sync; mod extra_requests; mod futures_stream; mod pending_responses; +mod request_metrics; mod schema; +mod types; pub mod block_relay_protocol; pub mod block_request_handler; @@ -88,3405 +40,3 @@ pub mod state; pub mod state_request_handler; pub mod warp; pub mod warp_request_handler; - -/// Log target for this file. -const LOG_TARGET: &'static str = "sync"; - -/// Maximum blocks to store in the import queue. -const MAX_IMPORTING_BLOCKS: usize = 2048; - -/// Maximum blocks to download ahead of any gap. -const MAX_DOWNLOAD_AHEAD: u32 = 2048; - -/// Maximum blocks to look backwards. The gap is the difference between the highest block and the -/// common block of a node. -const MAX_BLOCKS_TO_LOOK_BACKWARDS: u32 = MAX_DOWNLOAD_AHEAD / 2; - -/// Pick the state to sync as the latest finalized number minus this. -const STATE_SYNC_FINALITY_THRESHOLD: u32 = 8; - -/// We use a heuristic that with a high likelihood, by the time -/// `MAJOR_SYNC_BLOCKS` have been imported we'll be on the same -/// chain as (or at least closer to) the peer so we want to delay -/// the ancestor search to not waste time doing that when we are -/// so far behind. -const MAJOR_SYNC_BLOCKS: u8 = 5; - -/// Number of peers that need to be connected before warp sync is started. -const MIN_PEERS_TO_START_WARP_SYNC: usize = 3; - -/// Maximum blocks per response. -pub(crate) const MAX_BLOCKS_IN_RESPONSE: usize = 128; - -mod rep { - use sc_network::ReputationChange as Rep; - /// Reputation change when a peer sent us a message that led to a - /// database read error. - pub const BLOCKCHAIN_READ_ERROR: Rep = Rep::new(-(1 << 16), "DB Error"); - - /// Reputation change when a peer sent us a status message with a different - /// genesis than us. - pub const GENESIS_MISMATCH: Rep = Rep::new(i32::MIN, "Genesis mismatch"); - - /// Reputation change for peers which send us a block with an incomplete header. - pub const INCOMPLETE_HEADER: Rep = Rep::new(-(1 << 20), "Incomplete header"); - - /// Reputation change for peers which send us a block which we fail to verify. - pub const VERIFICATION_FAIL: Rep = Rep::new(-(1 << 29), "Block verification failed"); - - /// Reputation change for peers which send us a known bad block. - pub const BAD_BLOCK: Rep = Rep::new(-(1 << 29), "Bad block"); - - /// Peer did not provide us with advertised block data. - pub const NO_BLOCK: Rep = Rep::new(-(1 << 29), "No requested block data"); - - /// Reputation change for peers which send us non-requested block data. - pub const NOT_REQUESTED: Rep = Rep::new(-(1 << 29), "Not requested block data"); - - /// Reputation change for peers which send us a block with bad justifications. - pub const BAD_JUSTIFICATION: Rep = Rep::new(-(1 << 16), "Bad justification"); - - /// Reputation change when a peer sent us invlid ancestry result. - pub const UNKNOWN_ANCESTOR: Rep = Rep::new(-(1 << 16), "DB Error"); - - /// Peer response data does not have requested bits. - pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); -} - -enum AllowedRequests { - Some(HashSet), - All, -} - -impl AllowedRequests { - fn add(&mut self, id: &PeerId) { - if let Self::Some(ref mut set) = self { - set.insert(*id); - } - } - - fn take(&mut self) -> Self { - std::mem::take(self) - } - - fn set_all(&mut self) { - *self = Self::All; - } - - fn contains(&self, id: &PeerId) -> bool { - match self { - Self::Some(set) => set.contains(id), - Self::All => true, - } - } - - fn is_empty(&self) -> bool { - match self { - Self::Some(set) => set.is_empty(), - Self::All => false, - } - } - - fn clear(&mut self) { - std::mem::take(self); - } -} - -impl Default for AllowedRequests { - fn default() -> Self { - Self::Some(HashSet::default()) - } -} - -struct GapSync { - blocks: BlockCollection, - best_queued_number: NumberFor, - target: NumberFor, -} - -/// Action that [`engine::SyncingEngine`] should perform after reporting imported blocks with -/// [`ChainSync::on_blocks_processed`]. -enum BlockRequestAction { - /// Send block request to peer. Always implies dropping a stale block request to the same peer. - SendRequest { peer_id: PeerId, request: BlockRequest }, - /// Drop stale block request. - RemoveStale { peer_id: PeerId }, -} - -/// Action that [`engine::SyncingEngine`] should perform if we want to import justifications. -struct ImportJustificationsAction { - peer_id: PeerId, - hash: B::Hash, - number: NumberFor, - justifications: Justifications, -} - -/// Action that [`engine::SyncingEngine`] should perform on behalf of [`ChainSync`] -/// after reporting block response with [`ChainSync::on_block_response`]. -enum OnBlockResponse { - /// Nothing to do - Nothing, - /// Perform block request. - SendBlockRequest { peer_id: PeerId, request: BlockRequest }, - /// Import blocks. - ImportBlocks(ImportBlocksAction), - /// Import justifications. - ImportJustifications(ImportJustificationsAction), -} - -/// The main data structure which contains all the state for a chains -/// active syncing strategy. -pub struct ChainSync { - /// Chain client. - client: Arc, - /// The active peers that we are using to sync and their PeerSync status - peers: HashMap>, - /// A `BlockCollection` of blocks that are being downloaded from peers - blocks: BlockCollection, - /// The best block number in our queue of blocks to import - best_queued_number: NumberFor, - /// The best block hash in our queue of blocks to import - best_queued_hash: B::Hash, - /// Current mode (full/light) - mode: SyncMode, - /// Any extra justification requests. - extra_justifications: ExtraRequests, - /// A set of hashes of blocks that are being downloaded or have been - /// downloaded and are queued for import. - queue_blocks: HashSet, - /// Fork sync targets. - fork_targets: HashMap>, - /// A set of peers for which there might be potential block requests - allowed_requests: AllowedRequests, - /// Maximum number of peers to ask the same blocks in parallel. - max_parallel_downloads: u32, - /// Maximum blocks per request. - max_blocks_per_request: u32, - /// Total number of downloaded blocks. - downloaded_blocks: usize, - /// State sync in progress, if any. - state_sync: Option>, - /// Warp sync in progress, if any. - warp_sync: Option>, - /// Warp sync configuration. - /// - /// Will be `None` after `self.warp_sync` is `Some(_)`. - warp_sync_config: Option>, - /// A temporary storage for warp sync target block until warp sync is initialized. - warp_sync_target_block_header: Option, - /// Enable importing existing blocks. This is used used after the state download to - /// catch up to the latest state while re-importing blocks. - import_existing: bool, - /// Gap download process. - gap_sync: Option>, - /// Handle for communicating with `NetworkService` - network_service: service::network::NetworkServiceHandle, - /// Protocol name used for block announcements - block_announce_protocol_name: ProtocolName, -} - -/// All the data we have about a Peer that we are trying to sync with -#[derive(Debug, Clone)] -pub struct PeerSync { - /// Peer id of this peer. - pub peer_id: PeerId, - /// The common number is the block number that is a common point of - /// ancestry for both our chains (as far as we know). - pub common_number: NumberFor, - /// The hash of the best block that we've seen for this peer. - pub best_hash: B::Hash, - /// The number of the best block that we've seen for this peer. - pub best_number: NumberFor, - /// The state of syncing this peer is in for us, generally categories - /// into `Available` or "busy" with something as defined by `PeerSyncState`. - pub state: PeerSyncState, -} - -impl PeerSync { - /// Update the `common_number` iff `new_common > common_number`. - fn update_common_number(&mut self, new_common: NumberFor) { - if self.common_number < new_common { - trace!( - target: LOG_TARGET, - "Updating peer {} common number from={} => to={}.", - self.peer_id, - self.common_number, - new_common, - ); - self.common_number = new_common; - } - } -} - -struct ForkTarget { - number: NumberFor, - parent_hash: Option, - peers: HashSet, -} - -/// The state of syncing between a Peer and ourselves. -/// -/// Generally two categories, "busy" or `Available`. If busy, the enum -/// defines what we are busy with. -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum PeerSyncState { - /// Available for sync requests. - Available, - /// Searching for ancestors the Peer has in common with us. - AncestorSearch { start: NumberFor, current: NumberFor, state: AncestorSearchState }, - /// Actively downloading new blocks, starting from the given Number. - DownloadingNew(NumberFor), - /// Downloading a stale block with given Hash. Stale means that it is a - /// block with a number that is lower than our best number. It might be - /// from a fork and not necessarily already imported. - DownloadingStale(B::Hash), - /// Downloading justification for given block hash. - DownloadingJustification(B::Hash), - /// Downloading state. - DownloadingState, - /// Downloading warp proof. - DownloadingWarpProof, - /// Downloading warp sync target block. - DownloadingWarpTargetBlock, - /// Actively downloading block history after warp sync. - DownloadingGap(NumberFor), -} - -impl PeerSyncState { - pub fn is_available(&self) -> bool { - matches!(self, Self::Available) - } -} - -impl ChainSyncT for ChainSync -where - B: BlockT, - Client: HeaderBackend - + BlockBackend - + HeaderMetadata - + ProofProvider - + Send - + Sync - + 'static, -{ - fn peer_info(&self, who: &PeerId) -> Option> { - self.peers - .get(who) - .map(|p| PeerInfo { best_hash: p.best_hash, best_number: p.best_number }) - } - - /// Returns the current sync status. - fn status(&self) -> SyncStatus { - let median_seen = self.median_seen(); - let best_seen_block = - median_seen.and_then(|median| (median > self.best_queued_number).then_some(median)); - let sync_state = if let Some(target) = median_seen { - // A chain is classified as downloading if the provided best block is - // more than `MAJOR_SYNC_BLOCKS` behind the best block or as importing - // if the same can be said about queued blocks. - let best_block = self.client.info().best_number; - if target > best_block && target - best_block > MAJOR_SYNC_BLOCKS.into() { - // If target is not queued, we're downloading, otherwise importing. - if target > self.best_queued_number { - SyncState::Downloading { target } - } else { - SyncState::Importing { target } - } - } else { - SyncState::Idle - } - } else { - SyncState::Idle - }; - - let warp_sync_progress = match (&self.warp_sync, &self.mode, &self.gap_sync) { - (_, _, Some(gap_sync)) => Some(WarpSyncProgress { - phase: WarpSyncPhase::DownloadingBlocks(gap_sync.best_queued_number), - total_bytes: 0, - }), - (None, SyncMode::Warp, _) => Some(WarpSyncProgress { - phase: WarpSyncPhase::AwaitingPeers { - required_peers: MIN_PEERS_TO_START_WARP_SYNC, - }, - total_bytes: 0, - }), - (Some(sync), _, _) => Some(sync.progress()), - _ => None, - }; - - SyncStatus { - state: sync_state, - best_seen_block, - num_peers: self.peers.len() as u32, - num_connected_peers: 0u32, - queued_blocks: self.queue_blocks.len() as u32, - state_sync: self.state_sync.as_ref().map(|s| s.progress()), - warp_sync: warp_sync_progress, - } - } - - fn num_sync_requests(&self) -> usize { - self.fork_targets - .values() - .filter(|f| f.number <= self.best_queued_number) - .count() - } - - fn num_downloaded_blocks(&self) -> usize { - self.downloaded_blocks - } - - fn num_peers(&self) -> usize { - self.peers.len() - } - - #[must_use] - fn new_peer( - &mut self, - who: PeerId, - best_hash: B::Hash, - best_number: NumberFor, - ) -> Result>, BadPeer> { - // There is nothing sync can get from the node that has no blockchain data. - match self.block_status(&best_hash) { - Err(e) => { - debug!(target:LOG_TARGET, "Error reading blockchain: {e}"); - Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) - }, - Ok(BlockStatus::KnownBad) => { - info!("💔 New peer with known bad best block {} ({}).", best_hash, best_number); - Err(BadPeer(who, rep::BAD_BLOCK)) - }, - Ok(BlockStatus::Unknown) => { - if best_number.is_zero() { - info!("💔 New peer with unknown genesis hash {} ({}).", best_hash, best_number); - return Err(BadPeer(who, rep::GENESIS_MISMATCH)) - } - - // If there are more than `MAJOR_SYNC_BLOCKS` in the import queue then we have - // enough to do in the import queue that it's not worth kicking off - // an ancestor search, which is what we do in the next match case below. - if self.queue_blocks.len() > MAJOR_SYNC_BLOCKS.into() { - debug!( - target:LOG_TARGET, - "New peer with unknown best hash {} ({}), assuming common block.", - self.best_queued_hash, - self.best_queued_number - ); - self.peers.insert( - who, - PeerSync { - peer_id: who, - common_number: self.best_queued_number, - best_hash, - best_number, - state: PeerSyncState::Available, - }, - ); - return Ok(None) - } - - // If we are at genesis, just start downloading. - let (state, req) = if self.best_queued_number.is_zero() { - debug!( - target:LOG_TARGET, - "New peer with best hash {best_hash} ({best_number}).", - ); - - (PeerSyncState::Available, None) - } else { - let common_best = std::cmp::min(self.best_queued_number, best_number); - - debug!( - target:LOG_TARGET, - "New peer with unknown best hash {} ({}), searching for common ancestor.", - best_hash, - best_number - ); - - ( - PeerSyncState::AncestorSearch { - current: common_best, - start: self.best_queued_number, - state: AncestorSearchState::ExponentialBackoff(One::one()), - }, - Some(ancestry_request::(common_best)), - ) - }; - - self.allowed_requests.add(&who); - self.peers.insert( - who, - PeerSync { - peer_id: who, - common_number: Zero::zero(), - best_hash, - best_number, - state, - }, - ); - - if let SyncMode::Warp = self.mode { - if self.peers.len() >= MIN_PEERS_TO_START_WARP_SYNC && self.warp_sync.is_none() - { - log::debug!(target: LOG_TARGET, "Starting warp state sync."); - - if let Some(config) = self.warp_sync_config.take() { - let mut warp_sync = WarpSync::new(self.client.clone(), config); - if let Some(header) = self.warp_sync_target_block_header.take() { - warp_sync.set_target_block(header); - } - self.warp_sync = Some(warp_sync); - } - } - } - Ok(req) - }, - Ok(BlockStatus::Queued) | - Ok(BlockStatus::InChainWithState) | - Ok(BlockStatus::InChainPruned) => { - debug!( - target: LOG_TARGET, - "New peer with known best hash {best_hash} ({best_number}).", - ); - self.peers.insert( - who, - PeerSync { - peer_id: who, - common_number: std::cmp::min(self.best_queued_number, best_number), - best_hash, - best_number, - state: PeerSyncState::Available, - }, - ); - self.allowed_requests.add(&who); - Ok(None) - }, - } - } - - fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { - self.on_block_queued(best_hash, best_number); - } - - fn request_justification(&mut self, hash: &B::Hash, number: NumberFor) { - let client = &self.client; - self.extra_justifications - .schedule((*hash, number), |base, block| is_descendent_of(&**client, base, block)) - } - - fn clear_justification_requests(&mut self) { - self.extra_justifications.reset(); - } - - // The implementation is similar to `on_validated_block_announce` with unknown parent hash. - fn set_sync_fork_request( - &mut self, - mut peers: Vec, - hash: &B::Hash, - number: NumberFor, - ) { - if peers.is_empty() { - peers = self - .peers - .iter() - // Only request blocks from peers who are ahead or on a par. - .filter(|(_, peer)| peer.best_number >= number) - .map(|(id, _)| *id) - .collect(); - - debug!( - target: LOG_TARGET, - "Explicit sync request for block {hash:?} with no peers specified. \ - Syncing from these peers {peers:?} instead.", - ); - } else { - debug!( - target: LOG_TARGET, - "Explicit sync request for block {hash:?} with {peers:?}", - ); - } - - if self.is_known(hash) { - debug!(target: LOG_TARGET, "Refusing to sync known hash {hash:?}"); - return - } - - trace!(target: LOG_TARGET, "Downloading requested old fork {hash:?}"); - for peer_id in &peers { - if let Some(peer) = self.peers.get_mut(peer_id) { - if let PeerSyncState::AncestorSearch { .. } = peer.state { - continue - } - - if number > peer.best_number { - peer.best_number = number; - peer.best_hash = *hash; - } - self.allowed_requests.add(peer_id); - } - } - - self.fork_targets - .entry(*hash) - .or_insert_with(|| ForkTarget { number, peers: Default::default(), parent_hash: None }) - .peers - .extend(peers); - } - - #[must_use] - fn on_block_data( - &mut self, - who: &PeerId, - request: Option>, - response: BlockResponse, - ) -> Result, BadPeer> { - self.downloaded_blocks += response.blocks.len(); - let mut gap = false; - let new_blocks: Vec> = if let Some(peer) = self.peers.get_mut(who) { - let mut blocks = response.blocks; - if request.as_ref().map_or(false, |r| r.direction == Direction::Descending) { - trace!(target: LOG_TARGET, "Reversing incoming block list"); - blocks.reverse() - } - self.allowed_requests.add(who); - if let Some(request) = request { - match &mut peer.state { - PeerSyncState::DownloadingNew(_) => { - self.blocks.clear_peer_download(who); - peer.state = PeerSyncState::Available; - if let Some(start_block) = - validate_blocks::(&blocks, who, Some(request))? - { - self.blocks.insert(start_block, blocks, *who); - } - self.ready_blocks() - }, - PeerSyncState::DownloadingGap(_) => { - peer.state = PeerSyncState::Available; - if let Some(gap_sync) = &mut self.gap_sync { - gap_sync.blocks.clear_peer_download(who); - if let Some(start_block) = - validate_blocks::(&blocks, who, Some(request))? - { - gap_sync.blocks.insert(start_block, blocks, *who); - } - gap = true; - let blocks: Vec<_> = gap_sync - .blocks - .ready_blocks(gap_sync.best_queued_number + One::one()) - .into_iter() - .map(|block_data| { - let justifications = - block_data.block.justifications.or_else(|| { - legacy_justification_mapping( - block_data.block.justification, - ) - }); - IncomingBlock { - hash: block_data.block.hash, - header: block_data.block.header, - body: block_data.block.body, - indexed_body: block_data.block.indexed_body, - justifications, - origin: block_data.origin, - allow_missing_state: true, - import_existing: self.import_existing, - skip_execution: true, - state: None, - } - }) - .collect(); - debug!( - target: LOG_TARGET, - "Drained {} gap blocks from {}", - blocks.len(), - gap_sync.best_queued_number, - ); - blocks - } else { - debug!(target: LOG_TARGET, "Unexpected gap block response from {who}"); - return Err(BadPeer(*who, rep::NO_BLOCK)) - } - }, - PeerSyncState::DownloadingStale(_) => { - peer.state = PeerSyncState::Available; - if blocks.is_empty() { - debug!(target: LOG_TARGET, "Empty block response from {who}"); - return Err(BadPeer(*who, rep::NO_BLOCK)) - } - validate_blocks::(&blocks, who, Some(request))?; - blocks - .into_iter() - .map(|b| { - let justifications = b - .justifications - .or_else(|| legacy_justification_mapping(b.justification)); - IncomingBlock { - hash: b.hash, - header: b.header, - body: b.body, - indexed_body: None, - justifications, - origin: Some(*who), - allow_missing_state: true, - import_existing: self.import_existing, - skip_execution: self.skip_execution(), - state: None, - } - }) - .collect() - }, - PeerSyncState::AncestorSearch { current, start, state } => { - let matching_hash = match (blocks.get(0), self.client.hash(*current)) { - (Some(block), Ok(maybe_our_block_hash)) => { - trace!( - target: LOG_TARGET, - "Got ancestry block #{} ({}) from peer {}", - current, - block.hash, - who, - ); - maybe_our_block_hash.filter(|x| x == &block.hash) - }, - (None, _) => { - debug!( - target: LOG_TARGET, - "Invalid response when searching for ancestor from {who}", - ); - return Err(BadPeer(*who, rep::UNKNOWN_ANCESTOR)) - }, - (_, Err(e)) => { - info!( - target: LOG_TARGET, - "❌ Error answering legitimate blockchain query: {e}", - ); - return Err(BadPeer(*who, rep::BLOCKCHAIN_READ_ERROR)) - }, - }; - if matching_hash.is_some() { - if *start < self.best_queued_number && - self.best_queued_number <= peer.best_number - { - // We've made progress on this chain since the search was started. - // Opportunistically set common number to updated number - // instead of the one that started the search. - peer.common_number = self.best_queued_number; - } else if peer.common_number < *current { - peer.common_number = *current; - } - } - if matching_hash.is_none() && current.is_zero() { - trace!( - target:LOG_TARGET, - "Ancestry search: genesis mismatch for peer {who}", - ); - return Err(BadPeer(*who, rep::GENESIS_MISMATCH)) - } - if let Some((next_state, next_num)) = - handle_ancestor_search_state(state, *current, matching_hash.is_some()) - { - peer.state = PeerSyncState::AncestorSearch { - current: next_num, - start: *start, - state: next_state, - }; - return Ok(OnBlockData::Request(*who, ancestry_request::(next_num))) - } else { - // Ancestry search is complete. Check if peer is on a stale fork unknown - // to us and add it to sync targets if necessary. - trace!( - target: LOG_TARGET, - "Ancestry search complete. Ours={} ({}), Theirs={} ({}), Common={:?} ({})", - self.best_queued_hash, - self.best_queued_number, - peer.best_hash, - peer.best_number, - matching_hash, - peer.common_number, - ); - if peer.common_number < peer.best_number && - peer.best_number < self.best_queued_number - { - trace!( - target: LOG_TARGET, - "Added fork target {} for {}", - peer.best_hash, - who, - ); - self.fork_targets - .entry(peer.best_hash) - .or_insert_with(|| ForkTarget { - number: peer.best_number, - parent_hash: None, - peers: Default::default(), - }) - .peers - .insert(*who); - } - peer.state = PeerSyncState::Available; - Vec::new() - } - }, - PeerSyncState::DownloadingWarpTargetBlock => { - peer.state = PeerSyncState::Available; - if let Some(warp_sync) = &mut self.warp_sync { - if blocks.len() == 1 { - validate_blocks::(&blocks, who, Some(request))?; - match warp_sync.import_target_block( - blocks.pop().expect("`blocks` len checked above."), - ) { - warp::TargetBlockImportResult::Success => - return Ok(OnBlockData::Continue), - warp::TargetBlockImportResult::BadResponse => - return Err(BadPeer(*who, rep::VERIFICATION_FAIL)), - } - } else if blocks.is_empty() { - debug!(target: LOG_TARGET, "Empty block response from {who}"); - return Err(BadPeer(*who, rep::NO_BLOCK)) - } else { - debug!( - target: LOG_TARGET, - "Too many blocks ({}) in warp target block response from {}", - blocks.len(), - who, - ); - return Err(BadPeer(*who, rep::NOT_REQUESTED)) - } - } else { - debug!( - target: LOG_TARGET, - "Logic error: we think we are downloading warp target block from {}, but no warp sync is happening.", - who, - ); - return Ok(OnBlockData::Continue) - } - }, - PeerSyncState::Available | - PeerSyncState::DownloadingJustification(..) | - PeerSyncState::DownloadingState | - PeerSyncState::DownloadingWarpProof => Vec::new(), - } - } else { - // When request.is_none() this is a block announcement. Just accept blocks. - validate_blocks::(&blocks, who, None)?; - blocks - .into_iter() - .map(|b| { - let justifications = b - .justifications - .or_else(|| legacy_justification_mapping(b.justification)); - IncomingBlock { - hash: b.hash, - header: b.header, - body: b.body, - indexed_body: None, - justifications, - origin: Some(*who), - allow_missing_state: true, - import_existing: false, - skip_execution: true, - state: None, - } - }) - .collect() - } - } else { - // We don't know of this peer, so we also did not request anything from it. - return Err(BadPeer(*who, rep::NOT_REQUESTED)) - }; - - Ok(OnBlockData::Import(self.validate_and_queue_blocks(new_blocks, gap))) - } - - #[must_use] - fn on_block_justification( - &mut self, - who: PeerId, - response: BlockResponse, - ) -> Result, BadPeer> { - let peer = if let Some(peer) = self.peers.get_mut(&who) { - peer - } else { - error!( - target: LOG_TARGET, - "💔 Called on_block_justification with a peer ID of an unknown peer", - ); - return Ok(OnBlockJustification::Nothing) - }; - - self.allowed_requests.add(&who); - if let PeerSyncState::DownloadingJustification(hash) = peer.state { - peer.state = PeerSyncState::Available; - - // We only request one justification at a time - let justification = if let Some(block) = response.blocks.into_iter().next() { - if hash != block.hash { - warn!( - target: LOG_TARGET, - "💔 Invalid block justification provided by {}: requested: {:?} got: {:?}", - who, - hash, - block.hash, - ); - return Err(BadPeer(who, rep::BAD_JUSTIFICATION)) - } - - block - .justifications - .or_else(|| legacy_justification_mapping(block.justification)) - } else { - // we might have asked the peer for a justification on a block that we assumed it - // had but didn't (regardless of whether it had a justification for it or not). - trace!( - target: LOG_TARGET, - "Peer {who:?} provided empty response for justification request {hash:?}", - ); - - None - }; - - if let Some((peer_id, hash, number, justifications)) = - self.extra_justifications.on_response(who, justification) - { - return Ok(OnBlockJustification::Import { peer_id, hash, number, justifications }) - } - } - - Ok(OnBlockJustification::Nothing) - } - - fn on_justification_import(&mut self, hash: B::Hash, number: NumberFor, success: bool) { - let finalization_result = if success { Ok((hash, number)) } else { Err(()) }; - self.extra_justifications - .try_finalize_root((hash, number), finalization_result, true); - self.allowed_requests.set_all(); - } - - fn on_block_finalized(&mut self, hash: &B::Hash, number: NumberFor) { - let client = &self.client; - let r = self.extra_justifications.on_block_finalized(hash, number, |base, block| { - is_descendent_of(&**client, base, block) - }); - - if let SyncMode::LightState { skip_proofs, .. } = &self.mode { - if self.state_sync.is_none() && !self.peers.is_empty() && self.queue_blocks.is_empty() { - // Finalized a recent block. - let mut heads: Vec<_> = self.peers.values().map(|peer| peer.best_number).collect(); - heads.sort(); - let median = heads[heads.len() / 2]; - if number + STATE_SYNC_FINALITY_THRESHOLD.saturated_into() >= median { - if let Ok(Some(header)) = self.client.header(*hash) { - log::debug!( - target: LOG_TARGET, - "Starting state sync for #{number} ({hash})", - ); - self.state_sync = Some(StateSync::new( - self.client.clone(), - header, - None, - None, - *skip_proofs, - )); - self.allowed_requests.set_all(); - } - } - } - } - - if let Err(err) = r { - warn!( - target: LOG_TARGET, - "💔 Error cleaning up pending extra justification data requests: {err}", - ); - } - } - - fn on_validated_block_announce( - &mut self, - is_best: bool, - who: PeerId, - announce: &BlockAnnounce, - ) { - let number = *announce.header.number(); - let hash = announce.header.hash(); - let parent_status = - self.block_status(announce.header.parent_hash()).unwrap_or(BlockStatus::Unknown); - let known_parent = parent_status != BlockStatus::Unknown; - let ancient_parent = parent_status == BlockStatus::InChainPruned; - - let known = self.is_known(&hash); - let peer = if let Some(peer) = self.peers.get_mut(&who) { - peer - } else { - error!(target: LOG_TARGET, "💔 Called `on_validated_block_announce` with a bad peer ID"); - return - }; - - if let PeerSyncState::AncestorSearch { .. } = peer.state { - trace!(target: LOG_TARGET, "Peer {} is in the ancestor search state.", who); - return - } - - if is_best { - // update their best block - peer.best_number = number; - peer.best_hash = hash; - } - - // If the announced block is the best they have and is not ahead of us, our common number - // is either one further ahead or it's the one they just announced, if we know about it. - if is_best { - if known && self.best_queued_number >= number { - self.update_peer_common_number(&who, number); - } else if announce.header.parent_hash() == &self.best_queued_hash || - known_parent && self.best_queued_number >= number - { - self.update_peer_common_number(&who, number.saturating_sub(One::one())); - } - } - self.allowed_requests.add(&who); - - // known block case - if known || self.is_already_downloading(&hash) { - trace!(target: "sync", "Known block announce from {}: {}", who, hash); - if let Some(target) = self.fork_targets.get_mut(&hash) { - target.peers.insert(who); - } - return - } - - if ancient_parent { - trace!( - target: "sync", - "Ignored ancient block announced from {}: {} {:?}", - who, - hash, - announce.header, - ); - return - } - - if self.status().state == SyncState::Idle { - trace!( - target: "sync", - "Added sync target for block announced from {}: {} {:?}", - who, - hash, - announce.summary(), - ); - self.fork_targets - .entry(hash) - .or_insert_with(|| ForkTarget { - number, - parent_hash: Some(*announce.header.parent_hash()), - peers: Default::default(), - }) - .peers - .insert(who); - } - } - - #[must_use] - fn peer_disconnected(&mut self, who: &PeerId) -> Option> { - self.blocks.clear_peer_download(who); - if let Some(gap_sync) = &mut self.gap_sync { - gap_sync.blocks.clear_peer_download(who) - } - self.peers.remove(who); - self.extra_justifications.peer_disconnected(who); - self.allowed_requests.set_all(); - self.fork_targets.retain(|_, target| { - target.peers.remove(who); - !target.peers.is_empty() - }); - - let blocks = self.ready_blocks(); - - (!blocks.is_empty()).then(|| self.validate_and_queue_blocks(blocks, false)) - } - - fn metrics(&self) -> Metrics { - Metrics { - queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), - fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), - justifications: self.extra_justifications.metrics(), - } - } -} - -impl ChainSync -where - Self: ChainSyncT, - B: BlockT, - Client: HeaderBackend - + BlockBackend - + HeaderMetadata - + ProofProvider - + Send - + Sync - + 'static, -{ - /// Create a new instance. - pub fn new( - mode: SyncMode, - client: Arc, - block_announce_protocol_name: ProtocolName, - max_parallel_downloads: u32, - max_blocks_per_request: u32, - warp_sync_config: Option>, - network_service: service::network::NetworkServiceHandle, - ) -> Result { - let mut sync = Self { - client, - peers: HashMap::new(), - blocks: BlockCollection::new(), - best_queued_hash: Default::default(), - best_queued_number: Zero::zero(), - extra_justifications: ExtraRequests::new("justification"), - mode, - queue_blocks: Default::default(), - fork_targets: Default::default(), - allowed_requests: Default::default(), - max_parallel_downloads, - max_blocks_per_request, - downloaded_blocks: 0, - state_sync: None, - warp_sync: None, - import_existing: false, - gap_sync: None, - network_service, - warp_sync_config, - warp_sync_target_block_header: None, - block_announce_protocol_name, - }; - - sync.reset_sync_start_point()?; - Ok(sync) - } - - /// Returns the median seen block number. - fn median_seen(&self) -> Option> { - let mut best_seens = self.peers.values().map(|p| p.best_number).collect::>(); - - if best_seens.is_empty() { - None - } else { - let middle = best_seens.len() / 2; - - // Not the "perfect median" when we have an even number of peers. - Some(*best_seens.select_nth_unstable(middle).1) - } - } - - fn required_block_attributes(&self) -> BlockAttributes { - match self.mode { - SyncMode::Full => - BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, - SyncMode::LightState { storage_chain_mode: false, .. } | SyncMode::Warp => - BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION | BlockAttributes::BODY, - SyncMode::LightState { storage_chain_mode: true, .. } => - BlockAttributes::HEADER | - BlockAttributes::JUSTIFICATION | - BlockAttributes::INDEXED_BODY, - } - } - - fn skip_execution(&self) -> bool { - match self.mode { - SyncMode::Full => false, - SyncMode::LightState { .. } => true, - SyncMode::Warp => true, - } - } - - fn validate_and_queue_blocks( - &mut self, - mut new_blocks: Vec>, - gap: bool, - ) -> ImportBlocksAction { - let orig_len = new_blocks.len(); - new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); - if new_blocks.len() != orig_len { - debug!( - target: LOG_TARGET, - "Ignoring {} blocks that are already queued", - orig_len - new_blocks.len(), - ); - } - - let origin = if !gap && !self.status().state.is_major_syncing() { - BlockOrigin::NetworkBroadcast - } else { - BlockOrigin::NetworkInitialSync - }; - - if let Some((h, n)) = new_blocks - .last() - .and_then(|b| b.header.as_ref().map(|h| (&b.hash, *h.number()))) - { - trace!( - target:LOG_TARGET, - "Accepted {} blocks ({:?}) with origin {:?}", - new_blocks.len(), - h, - origin, - ); - self.on_block_queued(h, n) - } - self.queue_blocks.extend(new_blocks.iter().map(|b| b.hash)); - - ImportBlocksAction { origin, blocks: new_blocks } - } - - fn update_peer_common_number(&mut self, peer_id: &PeerId, new_common: NumberFor) { - if let Some(peer) = self.peers.get_mut(peer_id) { - peer.update_common_number(new_common); - } - } - - /// Called when a block has been queued for import. - /// - /// Updates our internal state for best queued block and then goes - /// through all peers to update our view of their state as well. - fn on_block_queued(&mut self, hash: &B::Hash, number: NumberFor) { - if self.fork_targets.remove(hash).is_some() { - trace!(target: LOG_TARGET, "Completed fork sync {hash:?}"); - } - if let Some(gap_sync) = &mut self.gap_sync { - if number > gap_sync.best_queued_number && number <= gap_sync.target { - gap_sync.best_queued_number = number; - } - } - if number > self.best_queued_number { - self.best_queued_number = number; - self.best_queued_hash = *hash; - // Update common blocks - for (n, peer) in self.peers.iter_mut() { - if let PeerSyncState::AncestorSearch { .. } = peer.state { - // Wait for ancestry search to complete first. - continue - } - let new_common_number = - if peer.best_number >= number { number } else { peer.best_number }; - trace!( - target: LOG_TARGET, - "Updating peer {} info, ours={}, common={}->{}, their best={}", - n, - number, - peer.common_number, - new_common_number, - peer.best_number, - ); - peer.common_number = new_common_number; - } - } - self.allowed_requests.set_all(); - } - - /// Restart the sync process. This will reset all pending block requests and return an iterator - /// of new block requests to make to peers. Peers that were downloading finality data (i.e. - /// their state was `DownloadingJustification`) are unaffected and will stay in the same state. - fn restart(&mut self) -> impl Iterator, BadPeer>> + '_ { - self.blocks.clear(); - if let Err(e) = self.reset_sync_start_point() { - warn!(target: LOG_TARGET, "💔 Unable to restart sync: {e}"); - } - self.allowed_requests.set_all(); - debug!( - target: LOG_TARGET, - "Restarted with {} ({})", - self.best_queued_number, - self.best_queued_hash, - ); - let old_peers = std::mem::take(&mut self.peers); - - old_peers.into_iter().filter_map(move |(peer_id, mut p)| { - // peers that were downloading justifications - // should be kept in that state. - if let PeerSyncState::DownloadingJustification(_) = p.state { - // We make sure our commmon number is at least something we have. - p.common_number = self.best_queued_number; - self.peers.insert(peer_id, p); - return None - } - - // handle peers that were in other states. - match self.new_peer(peer_id, p.best_hash, p.best_number) { - // since the request is not a justification, remove it from pending responses - Ok(None) => Some(Ok(BlockRequestAction::RemoveStale { peer_id })), - // update the request if the new one is available - Ok(Some(request)) => Some(Ok(BlockRequestAction::SendRequest { peer_id, request })), - // this implies that we need to drop pending response from the peer - Err(e) => Some(Err(e)), - } - }) - } - - /// Find a block to start sync from. If we sync with state, that's the latest block we have - /// state for. - fn reset_sync_start_point(&mut self) -> Result<(), ClientError> { - let info = self.client.info(); - if matches!(self.mode, SyncMode::LightState { .. }) && info.finalized_state.is_some() { - warn!( - target: LOG_TARGET, - "Can't use fast sync mode with a partially synced database. Reverting to full sync mode." - ); - self.mode = SyncMode::Full; - } - if matches!(self.mode, SyncMode::Warp) && info.finalized_state.is_some() { - warn!( - target: LOG_TARGET, - "Can't use warp sync mode with a partially synced database. Reverting to full sync mode." - ); - self.mode = SyncMode::Full; - } - self.import_existing = false; - self.best_queued_hash = info.best_hash; - self.best_queued_number = info.best_number; - - if self.mode == SyncMode::Full && - self.client.block_status(info.best_hash)? != BlockStatus::InChainWithState - { - self.import_existing = true; - // Latest state is missing, start with the last finalized state or genesis instead. - if let Some((hash, number)) = info.finalized_state { - debug!(target: LOG_TARGET, "Starting from finalized state #{number}"); - self.best_queued_hash = hash; - self.best_queued_number = number; - } else { - debug!(target: LOG_TARGET, "Restarting from genesis"); - self.best_queued_hash = Default::default(); - self.best_queued_number = Zero::zero(); - } - } - - if let Some((start, end)) = info.block_gap { - debug!(target: LOG_TARGET, "Starting gap sync #{start} - #{end}"); - self.gap_sync = Some(GapSync { - best_queued_number: start - One::one(), - target: end, - blocks: BlockCollection::new(), - }); - } - trace!( - target: LOG_TARGET, - "Restarted sync at #{} ({:?})", - self.best_queued_number, - self.best_queued_hash, - ); - Ok(()) - } - - /// What is the status of the block corresponding to the given hash? - fn block_status(&self, hash: &B::Hash) -> Result { - if self.queue_blocks.contains(hash) { - return Ok(BlockStatus::Queued) - } - self.client.block_status(*hash) - } - - /// Is the block corresponding to the given hash known? - fn is_known(&self, hash: &B::Hash) -> bool { - self.block_status(hash).ok().map_or(false, |s| s != BlockStatus::Unknown) - } - - /// Is any peer downloading the given hash? - fn is_already_downloading(&self, hash: &B::Hash) -> bool { - self.peers - .iter() - .any(|(_, p)| p.state == PeerSyncState::DownloadingStale(*hash)) - } - - /// Is the peer know to the sync state machine? - pub fn is_peer_known(&self, peer_id: &PeerId) -> bool { - self.peers.contains_key(peer_id) - } - - /// Get the set of downloaded blocks that are ready to be queued for import. - fn ready_blocks(&mut self) -> Vec> { - self.blocks - .ready_blocks(self.best_queued_number + One::one()) - .into_iter() - .map(|block_data| { - let justifications = block_data - .block - .justifications - .or_else(|| legacy_justification_mapping(block_data.block.justification)); - IncomingBlock { - hash: block_data.block.hash, - header: block_data.block.header, - body: block_data.block.body, - indexed_body: block_data.block.indexed_body, - justifications, - origin: block_data.origin, - allow_missing_state: true, - import_existing: self.import_existing, - skip_execution: self.skip_execution(), - state: None, - } - }) - .collect() - } - - /// Set warp sync target block externally in case we skip warp proof downloading. - pub fn set_warp_sync_target_block(&mut self, header: B::Header) { - if let Some(ref mut warp_sync) = self.warp_sync { - warp_sync.set_target_block(header); - } else { - self.warp_sync_target_block_header = Some(header); - } - } - - /// Generate block request for downloading of the target block body during warp sync. - fn warp_target_block_request(&mut self) -> Option<(PeerId, BlockRequest)> { - let sync = &self.warp_sync.as_ref()?; - - if self.allowed_requests.is_empty() || - sync.is_complete() || - self.peers - .iter() - .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpTargetBlock) - { - // Only one pending warp target block request is allowed. - return None - } - - if let Some((target_number, request)) = sync.next_target_block_request() { - // Find a random peer that has a block with the target number. - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= target_number { - trace!(target: LOG_TARGET, "New warp target block request for {id}"); - peer.state = PeerSyncState::DownloadingWarpTargetBlock; - self.allowed_requests.clear(); - return Some((*id, request)) - } - } - } - - None - } - - /// Process blocks received in a response. - #[must_use] - pub(crate) fn on_block_response( - &mut self, - peer_id: PeerId, - request: BlockRequest, - blocks: Vec>, - ) -> OnBlockResponse { - let block_response = BlockResponse:: { id: request.id, blocks }; - - let blocks_range = || match ( - block_response - .blocks - .first() - .and_then(|b| b.header.as_ref().map(|h| h.number())), - block_response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), - ) { - (Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last), - (Some(first), Some(_)) => format!(" ({})", first), - _ => Default::default(), - }; - trace!( - target: LOG_TARGET, - "BlockResponse {} from {} with {} blocks {}", - block_response.id, - peer_id, - block_response.blocks.len(), - blocks_range(), - ); - - if request.fields == BlockAttributes::JUSTIFICATION { - match self.on_block_justification(peer_id, block_response) { - Ok(OnBlockJustification::Nothing) => OnBlockResponse::Nothing, - Ok(OnBlockJustification::Import { peer_id, hash, number, justifications }) => - OnBlockResponse::ImportJustifications(ImportJustificationsAction { - peer_id, - hash, - number, - justifications, - }), - Err(BadPeer(id, repu)) => { - self.network_service - .disconnect_peer(id, self.block_announce_protocol_name.clone()); - self.network_service.report_peer(id, repu); - OnBlockResponse::Nothing - }, - } - } else { - match self.on_block_data(&peer_id, Some(request), block_response) { - Ok(OnBlockData::Import(action)) => OnBlockResponse::ImportBlocks(action), - Ok(OnBlockData::Request(peer_id, request)) => - OnBlockResponse::SendBlockRequest { peer_id, request }, - Ok(OnBlockData::Continue) => OnBlockResponse::Nothing, - Err(BadPeer(id, repu)) => { - self.network_service - .disconnect_peer(id, self.block_announce_protocol_name.clone()); - self.network_service.report_peer(id, repu); - OnBlockResponse::Nothing - }, - } - } - } - - /// Process state received in a response. - #[must_use] - pub fn on_state_response( - &mut self, - peer_id: PeerId, - response: OpaqueStateResponse, - ) -> Option> { - match self.on_state_data(&peer_id, response) { - Ok(OnStateData::Import(origin, block)) => - Some(ImportBlocksAction { origin, blocks: vec![block] }), - Ok(OnStateData::Continue) => None, - Err(BadPeer(id, repu)) => { - self.network_service - .disconnect_peer(id, self.block_announce_protocol_name.clone()); - self.network_service.report_peer(id, repu); - None - }, - } - } - - pub fn on_warp_sync_response(&mut self, peer_id: PeerId, response: EncodedProof) { - if let Err(BadPeer(id, repu)) = self.on_warp_sync_data(&peer_id, response) { - self.network_service - .disconnect_peer(id, self.block_announce_protocol_name.clone()); - self.network_service.report_peer(id, repu); - } - } - - fn justification_requests(&mut self) -> Vec<(PeerId, BlockRequest)> { - let peers = &mut self.peers; - let mut matcher = self.extra_justifications.matcher(); - std::iter::from_fn(move || { - if let Some((peer, request)) = matcher.next(peers) { - peers - .get_mut(&peer) - .expect( - "`Matcher::next` guarantees the `PeerId` comes from the given peers; qed", - ) - .state = PeerSyncState::DownloadingJustification(request.0); - let req = BlockRequest:: { - id: 0, - fields: BlockAttributes::JUSTIFICATION, - from: FromBlock::Hash(request.0), - direction: Direction::Ascending, - max: Some(1), - }; - Some((peer, req)) - } else { - None - } - }) - .collect() - } - - fn block_requests(&mut self) -> Vec<(PeerId, BlockRequest)> { - if self.mode == SyncMode::Warp { - return self - .warp_target_block_request() - .map_or_else(|| Vec::new(), |req| Vec::from([req])) - } - - if self.allowed_requests.is_empty() || self.state_sync.is_some() { - return Vec::new() - } - - if self.queue_blocks.len() > MAX_IMPORTING_BLOCKS { - trace!(target: LOG_TARGET, "Too many blocks in the queue."); - return Vec::new() - } - let is_major_syncing = self.status().state.is_major_syncing(); - let attrs = self.required_block_attributes(); - let blocks = &mut self.blocks; - let fork_targets = &mut self.fork_targets; - let last_finalized = - std::cmp::min(self.best_queued_number, self.client.info().finalized_number); - let best_queued = self.best_queued_number; - let client = &self.client; - let queue = &self.queue_blocks; - let allowed_requests = self.allowed_requests.take(); - let max_parallel = if is_major_syncing { 1 } else { self.max_parallel_downloads }; - let max_blocks_per_request = self.max_blocks_per_request; - let gap_sync = &mut self.gap_sync; - self.peers - .iter_mut() - .filter_map(move |(&id, peer)| { - if !peer.state.is_available() || !allowed_requests.contains(&id) { - return None - } - - // If our best queued is more than `MAX_BLOCKS_TO_LOOK_BACKWARDS` blocks away from - // the common number, the peer best number is higher than our best queued and the - // common number is smaller than the last finalized block number, we should do an - // ancestor search to find a better common block. If the queue is full we wait till - // all blocks are imported though. - if best_queued.saturating_sub(peer.common_number) > - MAX_BLOCKS_TO_LOOK_BACKWARDS.into() && - best_queued < peer.best_number && - peer.common_number < last_finalized && - queue.len() <= MAJOR_SYNC_BLOCKS.into() - { - trace!( - target: LOG_TARGET, - "Peer {:?} common block {} too far behind of our best {}. Starting ancestry search.", - id, - peer.common_number, - best_queued, - ); - let current = std::cmp::min(peer.best_number, best_queued); - peer.state = PeerSyncState::AncestorSearch { - current, - start: best_queued, - state: AncestorSearchState::ExponentialBackoff(One::one()), - }; - Some((id, ancestry_request::(current))) - } else if let Some((range, req)) = peer_block_request( - &id, - peer, - blocks, - attrs, - max_parallel, - max_blocks_per_request, - last_finalized, - best_queued, - ) { - peer.state = PeerSyncState::DownloadingNew(range.start); - trace!( - target: LOG_TARGET, - "New block request for {}, (best:{}, common:{}) {:?}", - id, - peer.best_number, - peer.common_number, - req, - ); - Some((id, req)) - } else if let Some((hash, req)) = fork_sync_request( - &id, - fork_targets, - best_queued, - last_finalized, - attrs, - |hash| { - if queue.contains(hash) { - BlockStatus::Queued - } else { - client.block_status(*hash).unwrap_or(BlockStatus::Unknown) - } - }, - max_blocks_per_request, - ) { - trace!(target: LOG_TARGET, "Downloading fork {hash:?} from {id}"); - peer.state = PeerSyncState::DownloadingStale(hash); - Some((id, req)) - } else if let Some((range, req)) = gap_sync.as_mut().and_then(|sync| { - peer_gap_block_request( - &id, - peer, - &mut sync.blocks, - attrs, - sync.target, - sync.best_queued_number, - max_blocks_per_request, - ) - }) { - peer.state = PeerSyncState::DownloadingGap(range.start); - trace!( - target: LOG_TARGET, - "New gap block request for {}, (best:{}, common:{}) {:?}", - id, - peer.best_number, - peer.common_number, - req, - ); - Some((id, req)) - } else { - None - } - }) - .collect() - } - - fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)> { - if self.allowed_requests.is_empty() { - return None - } - if (self.state_sync.is_some() || self.warp_sync.is_some()) && - self.peers.iter().any(|(_, peer)| peer.state == PeerSyncState::DownloadingState) - { - // Only one pending state request is allowed. - return None - } - if let Some(sync) = &self.state_sync { - if sync.is_complete() { - return None - } - - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.common_number >= sync.target_block_num() { - peer.state = PeerSyncState::DownloadingState; - let request = sync.next_request(); - trace!(target: LOG_TARGET, "New StateRequest for {}: {:?}", id, request); - self.allowed_requests.clear(); - return Some((*id, OpaqueStateRequest(Box::new(request)))) - } - } - } - if let Some(sync) = &self.warp_sync { - if sync.is_complete() { - return None - } - if let (Some(request), Some(target)) = - (sync.next_state_request(), sync.target_block_number()) - { - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= target { - trace!(target: LOG_TARGET, "New StateRequest for {id}: {request:?}"); - peer.state = PeerSyncState::DownloadingState; - self.allowed_requests.clear(); - return Some((*id, OpaqueStateRequest(Box::new(request)))) - } - } - } - } - None - } - - fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { - if let Some(sync) = &self.warp_sync { - if self.allowed_requests.is_empty() || - sync.is_complete() || - self.peers - .iter() - .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpProof) - { - // Only one pending state request is allowed. - return None - } - if let Some(request) = sync.next_warp_proof_request() { - let mut targets: Vec<_> = self.peers.values().map(|p| p.best_number).collect(); - if !targets.is_empty() { - targets.sort(); - let median = targets[targets.len() / 2]; - // Find a random peer that is synced as much as peer majority. - for (id, peer) in self.peers.iter_mut() { - if peer.state.is_available() && peer.best_number >= median { - trace!(target: LOG_TARGET, "New WarpProofRequest for {id}"); - peer.state = PeerSyncState::DownloadingWarpProof; - self.allowed_requests.clear(); - return Some((*id, request)) - } - } - } - } - } - None - } - - fn on_state_data( - &mut self, - who: &PeerId, - response: OpaqueStateResponse, - ) -> Result, BadPeer> { - let response: Box = response.0.downcast().map_err(|_error| { - error!( - target: LOG_TARGET, - "Failed to downcast opaque state response, this is an implementation bug." - ); - - BadPeer(*who, rep::BAD_RESPONSE) - })?; - - if let Some(peer) = self.peers.get_mut(who) { - if let PeerSyncState::DownloadingState = peer.state { - peer.state = PeerSyncState::Available; - self.allowed_requests.set_all(); - } - } - let import_result = if let Some(sync) = &mut self.state_sync { - debug!( - target: LOG_TARGET, - "Importing state data from {} with {} keys, {} proof nodes.", - who, - response.entries.len(), - response.proof.len(), - ); - sync.import(*response) - } else if let Some(sync) = &mut self.warp_sync { - debug!( - target: LOG_TARGET, - "Importing state data from {} with {} keys, {} proof nodes.", - who, - response.entries.len(), - response.proof.len(), - ); - sync.import_state(*response) - } else { - debug!(target: LOG_TARGET, "Ignored obsolete state response from {who}"); - return Err(BadPeer(*who, rep::NOT_REQUESTED)) - }; - - match import_result { - state::ImportResult::Import(hash, header, state, body, justifications) => { - let origin = BlockOrigin::NetworkInitialSync; - let block = IncomingBlock { - hash, - header: Some(header), - body, - indexed_body: None, - justifications, - origin: None, - allow_missing_state: true, - import_existing: true, - skip_execution: self.skip_execution(), - state: Some(state), - }; - debug!(target: LOG_TARGET, "State download is complete. Import is queued"); - Ok(OnStateData::Import(origin, block)) - }, - state::ImportResult::Continue => Ok(OnStateData::Continue), - state::ImportResult::BadResponse => { - debug!(target: LOG_TARGET, "Bad state data received from {who}"); - Err(BadPeer(*who, rep::BAD_BLOCK)) - }, - } - } - - fn on_warp_sync_data(&mut self, who: &PeerId, response: EncodedProof) -> Result<(), BadPeer> { - if let Some(peer) = self.peers.get_mut(who) { - if let PeerSyncState::DownloadingWarpProof = peer.state { - peer.state = PeerSyncState::Available; - self.allowed_requests.set_all(); - } - } - let import_result = if let Some(sync) = &mut self.warp_sync { - debug!( - target: LOG_TARGET, - "Importing warp proof data from {}, {} bytes.", - who, - response.0.len(), - ); - sync.import_warp_proof(response) - } else { - debug!(target: LOG_TARGET, "Ignored obsolete warp sync response from {who}"); - return Err(BadPeer(*who, rep::NOT_REQUESTED)) - }; - - match import_result { - WarpProofImportResult::Success => Ok(()), - WarpProofImportResult::BadResponse => { - debug!(target: LOG_TARGET, "Bad proof data received from {who}"); - Err(BadPeer(*who, rep::BAD_BLOCK)) - }, - } - } - - /// A batch of blocks have been processed, with or without errors. - /// - /// Call this when a batch of blocks have been processed by the import - /// queue, with or without errors. If an error is returned, the pending response - /// from the peer must be dropped. - #[must_use] - fn on_blocks_processed( - &mut self, - imported: usize, - count: usize, - results: Vec<(Result>, BlockImportError>, B::Hash)>, - ) -> Box, BadPeer>>> { - trace!(target: LOG_TARGET, "Imported {imported} of {count}"); - - let mut output = Vec::new(); - - let mut has_error = false; - for (_, hash) in &results { - self.queue_blocks.remove(hash); - self.blocks.clear_queued(hash); - if let Some(gap_sync) = &mut self.gap_sync { - gap_sync.blocks.clear_queued(hash); - } - } - for (result, hash) in results { - if has_error { - break - } - - has_error |= result.is_err(); - - match result { - Ok(BlockImportStatus::ImportedKnown(number, who)) => - if let Some(peer) = who { - self.update_peer_common_number(&peer, number); - }, - Ok(BlockImportStatus::ImportedUnknown(number, aux, who)) => { - if aux.clear_justification_requests { - trace!( - target: LOG_TARGET, - "Block imported clears all pending justification requests {number}: {hash:?}", - ); - self.clear_justification_requests(); - } - - if aux.needs_justification { - trace!( - target: LOG_TARGET, - "Block imported but requires justification {number}: {hash:?}", - ); - self.request_justification(&hash, number); - } - - if aux.bad_justification { - if let Some(ref peer) = who { - warn!("💔 Sent block with bad justification to import"); - output.push(Err(BadPeer(*peer, rep::BAD_JUSTIFICATION))); - } - } - - if let Some(peer) = who { - self.update_peer_common_number(&peer, number); - } - let state_sync_complete = - self.state_sync.as_ref().map_or(false, |s| s.target() == hash); - if state_sync_complete { - info!( - target: LOG_TARGET, - "State sync is complete ({} MiB), restarting block sync.", - self.state_sync.as_ref().map_or(0, |s| s.progress().size / (1024 * 1024)), - ); - self.state_sync = None; - self.mode = SyncMode::Full; - output.extend(self.restart()); - } - let warp_sync_complete = self - .warp_sync - .as_ref() - .map_or(false, |s| s.target_block_hash() == Some(hash)); - if warp_sync_complete { - info!( - target: LOG_TARGET, - "Warp sync is complete ({} MiB), restarting block sync.", - self.warp_sync.as_ref().map_or(0, |s| s.progress().total_bytes / (1024 * 1024)), - ); - self.warp_sync = None; - self.mode = SyncMode::Full; - output.extend(self.restart()); - } - let gap_sync_complete = - self.gap_sync.as_ref().map_or(false, |s| s.target == number); - if gap_sync_complete { - info!( - target: LOG_TARGET, - "Block history download is complete." - ); - self.gap_sync = None; - } - }, - Err(BlockImportError::IncompleteHeader(who)) => - if let Some(peer) = who { - warn!( - target: LOG_TARGET, - "💔 Peer sent block with incomplete header to import", - ); - output.push(Err(BadPeer(peer, rep::INCOMPLETE_HEADER))); - output.extend(self.restart()); - }, - Err(BlockImportError::VerificationFailed(who, e)) => { - let extra_message = - who.map_or_else(|| "".into(), |peer| format!(" received from ({peer})")); - - warn!( - target: LOG_TARGET, - "💔 Verification failed for block {hash:?}{extra_message}: {e:?}", - ); - - if let Some(peer) = who { - output.push(Err(BadPeer(peer, rep::VERIFICATION_FAIL))); - } - - output.extend(self.restart()); - }, - Err(BlockImportError::BadBlock(who)) => - if let Some(peer) = who { - warn!( - target: LOG_TARGET, - "💔 Block {hash:?} received from peer {peer} has been blacklisted", - ); - output.push(Err(BadPeer(peer, rep::BAD_BLOCK))); - }, - Err(BlockImportError::MissingState) => { - // This may happen if the chain we were requesting upon has been discarded - // in the meantime because other chain has been finalized. - // Don't mark it as bad as it still may be synced if explicitly requested. - trace!(target: LOG_TARGET, "Obsolete block {hash:?}"); - }, - e @ Err(BlockImportError::UnknownParent) | e @ Err(BlockImportError::Other(_)) => { - warn!(target: LOG_TARGET, "💔 Error importing block {hash:?}: {}", e.unwrap_err()); - self.state_sync = None; - self.warp_sync = None; - output.extend(self.restart()); - }, - Err(BlockImportError::Cancelled) => {}, - }; - } - - self.allowed_requests.set_all(); - Box::new(output.into_iter()) - } -} - -// This is purely during a backwards compatible transitionary period and should be removed -// once we can assume all nodes can send and receive multiple Justifications -// The ID tag is hardcoded here to avoid depending on the GRANDPA crate. -// See: https://github.com/paritytech/substrate/issues/8172 -fn legacy_justification_mapping( - justification: Option, -) -> Option { - justification.map(|just| (*b"FRNK", just).into()) -} - -/// Request the ancestry for a block. Sends a request for header and justification for the given -/// block number. Used during ancestry search. -fn ancestry_request(block: NumberFor) -> BlockRequest { - BlockRequest:: { - id: 0, - fields: BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION, - from: FromBlock::Number(block), - direction: Direction::Ascending, - max: Some(1), - } -} - -/// The ancestor search state expresses which algorithm, and its stateful parameters, we are using -/// to try to find an ancestor block -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum AncestorSearchState { - /// Use exponential backoff to find an ancestor, then switch to binary search. - /// We keep track of the exponent. - ExponentialBackoff(NumberFor), - /// Using binary search to find the best ancestor. - /// We keep track of left and right bounds. - BinarySearch(NumberFor, NumberFor), -} - -/// This function handles the ancestor search strategy used. The goal is to find a common point -/// that both our chains agree on that is as close to the tip as possible. -/// The way this works is we first have an exponential backoff strategy, where we try to step -/// forward until we find a block hash mismatch. The size of the step doubles each step we take. -/// -/// When we've found a block hash mismatch we then fall back to a binary search between the two -/// last known points to find the common block closest to the tip. -fn handle_ancestor_search_state( - state: &AncestorSearchState, - curr_block_num: NumberFor, - block_hash_match: bool, -) -> Option<(AncestorSearchState, NumberFor)> { - let two = >::one() + >::one(); - match state { - AncestorSearchState::ExponentialBackoff(next_distance_to_tip) => { - let next_distance_to_tip = *next_distance_to_tip; - if block_hash_match && next_distance_to_tip == One::one() { - // We found the ancestor in the first step so there is no need to execute binary - // search. - return None - } - if block_hash_match { - let left = curr_block_num; - let right = left + next_distance_to_tip / two; - let middle = left + (right - left) / two; - Some((AncestorSearchState::BinarySearch(left, right), middle)) - } else { - let next_block_num = - curr_block_num.checked_sub(&next_distance_to_tip).unwrap_or_else(Zero::zero); - let next_distance_to_tip = next_distance_to_tip * two; - Some(( - AncestorSearchState::ExponentialBackoff(next_distance_to_tip), - next_block_num, - )) - } - }, - AncestorSearchState::BinarySearch(mut left, mut right) => { - if left >= curr_block_num { - return None - } - if block_hash_match { - left = curr_block_num; - } else { - right = curr_block_num; - } - assert!(right >= left); - let middle = left + (right - left) / two; - if middle == curr_block_num { - None - } else { - Some((AncestorSearchState::BinarySearch(left, right), middle)) - } - }, - } -} - -/// Get a new block request for the peer if any. -fn peer_block_request( - id: &PeerId, - peer: &PeerSync, - blocks: &mut BlockCollection, - attrs: BlockAttributes, - max_parallel_downloads: u32, - max_blocks_per_request: u32, - finalized: NumberFor, - best_num: NumberFor, -) -> Option<(Range>, BlockRequest)> { - if best_num >= peer.best_number { - // Will be downloaded as alternative fork instead. - return None - } else if peer.common_number < finalized { - trace!( - target: LOG_TARGET, - "Requesting pre-finalized chain from {:?}, common={}, finalized={}, peer best={}, our best={}", - id, peer.common_number, finalized, peer.best_number, best_num, - ); - } - let range = blocks.needed_blocks( - *id, - max_blocks_per_request, - peer.best_number, - peer.common_number, - max_parallel_downloads, - MAX_DOWNLOAD_AHEAD, - )?; - - // The end is not part of the range. - let last = range.end.saturating_sub(One::one()); - - let from = if peer.best_number == last { - FromBlock::Hash(peer.best_hash) - } else { - FromBlock::Number(last) - }; - - let request = BlockRequest:: { - id: 0, - fields: attrs, - from, - direction: Direction::Descending, - max: Some((range.end - range.start).saturated_into::()), - }; - - Some((range, request)) -} - -/// Get a new block request for the peer if any. -fn peer_gap_block_request( - id: &PeerId, - peer: &PeerSync, - blocks: &mut BlockCollection, - attrs: BlockAttributes, - target: NumberFor, - common_number: NumberFor, - max_blocks_per_request: u32, -) -> Option<(Range>, BlockRequest)> { - let range = blocks.needed_blocks( - *id, - max_blocks_per_request, - std::cmp::min(peer.best_number, target), - common_number, - 1, - MAX_DOWNLOAD_AHEAD, - )?; - - // The end is not part of the range. - let last = range.end.saturating_sub(One::one()); - let from = FromBlock::Number(last); - - let request = BlockRequest:: { - id: 0, - fields: attrs, - from, - direction: Direction::Descending, - max: Some((range.end - range.start).saturated_into::()), - }; - Some((range, request)) -} - -/// Get pending fork sync targets for a peer. -fn fork_sync_request( - id: &PeerId, - targets: &mut HashMap>, - best_num: NumberFor, - finalized: NumberFor, - attributes: BlockAttributes, - check_block: impl Fn(&B::Hash) -> BlockStatus, - max_blocks_per_request: u32, -) -> Option<(B::Hash, BlockRequest)> { - targets.retain(|hash, r| { - if r.number <= finalized { - trace!( - target: LOG_TARGET, - "Removed expired fork sync request {:?} (#{})", - hash, - r.number, - ); - return false - } - if check_block(hash) != BlockStatus::Unknown { - trace!( - target: LOG_TARGET, - "Removed obsolete fork sync request {:?} (#{})", - hash, - r.number, - ); - return false - } - true - }); - for (hash, r) in targets { - if !r.peers.contains(&id) { - continue - } - // Download the fork only if it is behind or not too far ahead our tip of the chain - // Otherwise it should be downloaded in full sync mode. - if r.number <= best_num || - (r.number - best_num).saturated_into::() < max_blocks_per_request as u32 - { - let parent_status = r.parent_hash.as_ref().map_or(BlockStatus::Unknown, check_block); - let count = if parent_status == BlockStatus::Unknown { - (r.number - finalized).saturated_into::() // up to the last finalized block - } else { - // request only single block - 1 - }; - trace!( - target: LOG_TARGET, - "Downloading requested fork {hash:?} from {id}, {count} blocks", - ); - return Some(( - *hash, - BlockRequest:: { - id: 0, - fields: attributes, - from: FromBlock::Hash(*hash), - direction: Direction::Descending, - max: Some(count), - }, - )) - } else { - trace!(target: LOG_TARGET, "Fork too far in the future: {:?} (#{})", hash, r.number); - } - } - None -} - -/// Returns `true` if the given `block` is a descendent of `base`. -fn is_descendent_of( - client: &T, - base: &Block::Hash, - block: &Block::Hash, -) -> sp_blockchain::Result -where - Block: BlockT, - T: HeaderMetadata + ?Sized, -{ - if base == block { - return Ok(false) - } - - let ancestor = sp_blockchain::lowest_common_ancestor(client, *block, *base)?; - - Ok(ancestor.hash == *base) -} - -/// Validate that the given `blocks` are correct. -/// Returns the number of the first block in the sequence. -/// -/// It is expected that `blocks` are in ascending order. -fn validate_blocks( - blocks: &Vec>, - who: &PeerId, - request: Option>, -) -> Result>, BadPeer> { - if let Some(request) = request { - if Some(blocks.len() as _) > request.max { - debug!( - target: LOG_TARGET, - "Received more blocks than requested from {}. Expected in maximum {:?}, got {}.", - who, - request.max, - blocks.len(), - ); - - return Err(BadPeer(*who, rep::NOT_REQUESTED)) - } - - let block_header = - if request.direction == Direction::Descending { blocks.last() } else { blocks.first() } - .and_then(|b| b.header.as_ref()); - - let expected_block = block_header.as_ref().map_or(false, |h| match request.from { - FromBlock::Hash(hash) => h.hash() == hash, - FromBlock::Number(n) => h.number() == &n, - }); - - if !expected_block { - debug!( - target: LOG_TARGET, - "Received block that was not requested. Requested {:?}, got {:?}.", - request.from, - block_header, - ); - - return Err(BadPeer(*who, rep::NOT_REQUESTED)) - } - - if request.fields.contains(BlockAttributes::HEADER) && - blocks.iter().any(|b| b.header.is_none()) - { - trace!( - target: LOG_TARGET, - "Missing requested header for a block in response from {who}.", - ); - - return Err(BadPeer(*who, rep::BAD_RESPONSE)) - } - - if request.fields.contains(BlockAttributes::BODY) && blocks.iter().any(|b| b.body.is_none()) - { - trace!( - target: LOG_TARGET, - "Missing requested body for a block in response from {who}.", - ); - - return Err(BadPeer(*who, rep::BAD_RESPONSE)) - } - } - - for b in blocks { - if let Some(header) = &b.header { - let hash = header.hash(); - if hash != b.hash { - debug!( - target:LOG_TARGET, - "Bad header received from {}. Expected hash {:?}, got {:?}", - who, - b.hash, - hash, - ); - return Err(BadPeer(*who, rep::BAD_BLOCK)) - } - } - if let (Some(header), Some(body)) = (&b.header, &b.body) { - let expected = *header.extrinsics_root(); - let got = HashingFor::::ordered_trie_root( - body.iter().map(Encode::encode).collect(), - sp_runtime::StateVersion::V0, - ); - if expected != got { - debug!( - target:LOG_TARGET, - "Bad extrinsic root for a block {} received from {}. Expected {:?}, got {:?}", - b.hash, - who, - expected, - got, - ); - return Err(BadPeer(*who, rep::BAD_BLOCK)) - } - } - } - - Ok(blocks.first().and_then(|b| b.header.as_ref()).map(|h| *h.number())) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::service::network::NetworkServiceProvider; - use futures::executor::block_on; - use sc_block_builder::BlockBuilderProvider; - use sc_network_common::sync::message::{BlockAnnounce, BlockData, BlockState, FromBlock}; - use sp_blockchain::HeaderBackend; - use substrate_test_runtime_client::{ - runtime::{Block, Hash, Header}, - BlockBuilderExt, ClientBlockImportExt, ClientExt, DefaultTestClientBuilderExt, TestClient, - TestClientBuilder, TestClientBuilderExt, - }; - - #[test] - fn processes_empty_response_on_justification_request_for_unknown_block() { - // if we ask for a justification for a given block to a peer that doesn't know that block - // (different from not having a justification), the peer will reply with an empty response. - // internally we should process the response as the justification not being available. - - let client = Arc::new(TestClientBuilder::new().build()); - let peer_id = PeerId::random(); - - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 1, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let (a1_hash, a1_number) = { - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - (a1.hash(), *a1.header.number()) - }; - - // add a new peer with the same best block - sync.new_peer(peer_id, a1_hash, a1_number).unwrap(); - - // and request a justification for the block - sync.request_justification(&a1_hash, a1_number); - - // the justification request should be scheduled to that peer - assert!(sync - .justification_requests() - .iter() - .any(|(who, request)| { *who == peer_id && request.from == FromBlock::Hash(a1_hash) })); - - // there are no extra pending requests - assert_eq!(sync.extra_justifications.pending_requests().count(), 0); - - // there's one in-flight extra request to the expected peer - assert!(sync.extra_justifications.active_requests().any(|(who, (hash, number))| { - *who == peer_id && *hash == a1_hash && *number == a1_number - })); - - // if the peer replies with an empty response (i.e. it doesn't know the block), - // the active request should be cleared. - assert_eq!( - sync.on_block_justification(peer_id, BlockResponse:: { id: 0, blocks: vec![] }), - Ok(OnBlockJustification::Nothing), - ); - - // there should be no in-flight requests - assert_eq!(sync.extra_justifications.active_requests().count(), 0); - - // and the request should now be pending again, waiting for reschedule - assert!(sync - .extra_justifications - .pending_requests() - .any(|(hash, number)| { *hash == a1_hash && *number == a1_number })); - } - - #[test] - fn restart_doesnt_affect_peers_downloading_finality_data() { - let mut client = Arc::new(TestClientBuilder::new().build()); - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 1, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let peer_id1 = PeerId::random(); - let peer_id2 = PeerId::random(); - let peer_id3 = PeerId::random(); - - let mut new_blocks = |n| { - for _ in 0..n { - let block = client.new_block(Default::default()).unwrap().build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block.clone())).unwrap(); - } - - let info = client.info(); - (info.best_hash, info.best_number) - }; - - let (b1_hash, b1_number) = new_blocks(50); - - // add 2 peers at blocks that we don't have locally - sync.new_peer(peer_id1, Hash::random(), 42).unwrap(); - sync.new_peer(peer_id2, Hash::random(), 10).unwrap(); - - // we wil send block requests to these peers - // for these blocks we don't know about - assert!(sync - .block_requests() - .into_iter() - .all(|(p, _)| { p == peer_id1 || p == peer_id2 })); - - // add a new peer at a known block - sync.new_peer(peer_id3, b1_hash, b1_number).unwrap(); - - // we request a justification for a block we have locally - sync.request_justification(&b1_hash, b1_number); - - // the justification request should be scheduled to the - // new peer which is at the given block - assert!(sync.justification_requests().iter().any(|(p, r)| { - *p == peer_id3 && - r.fields == BlockAttributes::JUSTIFICATION && - r.from == FromBlock::Hash(b1_hash) - })); - - assert_eq!( - sync.peers.get(&peer_id3).unwrap().state, - PeerSyncState::DownloadingJustification(b1_hash), - ); - - // we restart the sync state - let block_requests = sync.restart(); - - // which should make us send out block requests to the first two peers - assert!(block_requests.map(|r| r.unwrap()).all(|event| match event { - BlockRequestAction::SendRequest { peer_id, .. } => - peer_id == peer_id1 || peer_id == peer_id2, - BlockRequestAction::RemoveStale { .. } => false, - })); - - // peer 3 should be unaffected it was downloading finality data - assert_eq!( - sync.peers.get(&peer_id3).unwrap().state, - PeerSyncState::DownloadingJustification(b1_hash), - ); - - // Set common block to something that we don't have (e.g. failed import) - sync.peers.get_mut(&peer_id3).unwrap().common_number = 100; - let _ = sync.restart().count(); - assert_eq!(sync.peers.get(&peer_id3).unwrap().common_number, 50); - } - - /// Send a block annoucnement for the given `header`. - fn send_block_announce( - header: Header, - peer_id: PeerId, - sync: &mut ChainSync, - ) { - let announce = BlockAnnounce { - header: header.clone(), - state: Some(BlockState::Best), - data: Some(Vec::new()), - }; - - sync.on_validated_block_announce(true, peer_id, &announce); - } - - /// Create a block response from the given `blocks`. - fn create_block_response(blocks: Vec) -> BlockResponse { - BlockResponse:: { - id: 0, - blocks: blocks - .into_iter() - .map(|b| BlockData:: { - hash: b.hash(), - header: Some(b.header().clone()), - body: Some(b.deconstruct().1), - indexed_body: None, - receipt: None, - message_queue: None, - justification: None, - justifications: None, - }) - .collect(), - } - } - - /// Get a block request from `sync` and check that is matches the expected request. - fn get_block_request( - sync: &mut ChainSync, - from: FromBlock, - max: u32, - peer: &PeerId, - ) -> BlockRequest { - let requests = sync.block_requests(); - - log::trace!(target: LOG_TARGET, "Requests: {requests:?}"); - - assert_eq!(1, requests.len()); - assert_eq!(*peer, requests[0].0); - - let request = requests[0].1.clone(); - - assert_eq!(from, request.from); - assert_eq!(Some(max), request.max); - request - } - - /// Build and import a new best block. - fn build_block(client: &mut Arc, at: Option, fork: bool) -> Block { - let at = at.unwrap_or_else(|| client.info().best_hash); - - let mut block_builder = client.new_block_at(at, Default::default(), false).unwrap(); - - if fork { - block_builder.push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])).unwrap(); - } - - let block = block_builder.build().unwrap().block; - - block_on(client.import(BlockOrigin::Own, block.clone())).unwrap(); - block - } - - /// This test is a regression test as observed on a real network. - /// - /// The node is connected to multiple peers. Both of these peers are having a best block (1) - /// that is below our best block (3). Now peer 2 announces a fork of block 3 that we will - /// request from peer 2. After importing the fork, peer 2 and then peer 1 will announce block 4. - /// But as peer 1 in our view is still at block 1, we will request block 2 (which we already - /// have) from it. In the meanwhile peer 2 sends us block 4 and 3 and we send another request - /// for block 2 to peer 2. Peer 1 answers with block 2 and then peer 2. This will need to - /// succeed, as we have requested block 2 from both peers. - #[test] - fn do_not_report_peer_on_block_response_for_block_request() { - sp_tracing::try_init_simple(); - - let mut client = Arc::new(TestClientBuilder::new().build()); - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 5, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let peer_id1 = PeerId::random(); - let peer_id2 = PeerId::random(); - - let mut client2 = client.clone(); - let mut build_block_at = |at, import| { - let mut block_builder = client2.new_block_at(at, Default::default(), false).unwrap(); - // Make sure we generate a different block as fork - block_builder.push_storage_change(vec![1, 2, 3], Some(vec![4, 5, 6])).unwrap(); - - let block = block_builder.build().unwrap().block; - - if import { - block_on(client2.import(BlockOrigin::Own, block.clone())).unwrap(); - } - - block - }; - - let block1 = build_block(&mut client, None, false); - let block2 = build_block(&mut client, None, false); - let block3 = build_block(&mut client, None, false); - let block3_fork = build_block_at(block2.hash(), false); - - // Add two peers which are on block 1. - sync.new_peer(peer_id1, block1.hash(), 1).unwrap(); - sync.new_peer(peer_id2, block1.hash(), 1).unwrap(); - - // Tell sync that our best block is 3. - sync.update_chain_info(&block3.hash(), 3); - - // There should be no requests. - assert!(sync.block_requests().is_empty()); - - // Let peer2 announce a fork of block 3 - send_block_announce(block3_fork.header().clone(), peer_id2, &mut sync); - - // Import and tell sync that we now have the fork. - block_on(client.import(BlockOrigin::Own, block3_fork.clone())).unwrap(); - sync.update_chain_info(&block3_fork.hash(), 3); - - let block4 = build_block_at(block3_fork.hash(), false); - - // Let peer2 announce block 4 and check that sync wants to get the block. - send_block_announce(block4.header().clone(), peer_id2, &mut sync); - - let request = get_block_request(&mut sync, FromBlock::Hash(block4.hash()), 2, &peer_id2); - - // Peer1 announces the same block, but as the common block is still `1`, sync will request - // block 2 again. - send_block_announce(block4.header().clone(), peer_id1, &mut sync); - - let request2 = get_block_request(&mut sync, FromBlock::Number(2), 1, &peer_id1); - - let response = create_block_response(vec![block4.clone(), block3_fork.clone()]); - let res = sync.on_block_data(&peer_id2, Some(request), response).unwrap(); - - // We should not yet import the blocks, because there is still an open request for fetching - // block `2` which blocks the import. - assert!( - matches!(res, OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.is_empty()) - ); - - let request3 = get_block_request(&mut sync, FromBlock::Number(2), 1, &peer_id2); - - let response = create_block_response(vec![block2.clone()]); - let res = sync.on_block_data(&peer_id1, Some(request2), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) - if blocks.iter().all(|b| [2, 3, 4].contains(b.header.as_ref().unwrap().number())) - )); - - let response = create_block_response(vec![block2.clone()]); - let res = sync.on_block_data(&peer_id2, Some(request3), response).unwrap(); - // Nothing to import - assert!( - matches!(res, OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.is_empty()) - ); - } - - fn unwrap_from_block_number(from: FromBlock) -> u64 { - if let FromBlock::Number(from) = from { - from - } else { - panic!("Expected a number!"); - } - } - - /// A regression test for a behavior we have seen on a live network. - /// - /// The scenario is that the node is doing a full resync and is connected to some node that is - /// doing a major sync as well. This other node that is doing a major sync will finish before - /// our node and send a block announcement message, but we don't have seen any block - /// announcement from this node in its sync process. Meaning our common number didn't change. It - /// is now expected that we start an ancestor search to find the common number. - #[test] - fn do_ancestor_search_when_common_block_to_best_qeued_gap_is_to_big() { - sp_tracing::try_init_simple(); - - let blocks = { - let mut client = Arc::new(TestClientBuilder::new().build()); - (0..MAX_DOWNLOAD_AHEAD * 2) - .map(|_| build_block(&mut client, None, false)) - .collect::>() - }; - - let mut client = Arc::new(TestClientBuilder::new().build()); - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let info = client.info(); - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 5, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let peer_id1 = PeerId::random(); - let peer_id2 = PeerId::random(); - - let best_block = blocks.last().unwrap().clone(); - let max_blocks_to_request = sync.max_blocks_per_request; - // Connect the node we will sync from - sync.new_peer(peer_id1, best_block.hash(), *best_block.header().number()) - .unwrap(); - sync.new_peer(peer_id2, info.best_hash, 0).unwrap(); - - let mut best_block_num = 0; - while best_block_num < MAX_DOWNLOAD_AHEAD { - let request = get_block_request( - &mut sync, - FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), - max_blocks_to_request as u32, - &peer_id1, - ); - - let from = unwrap_from_block_number(request.from.clone()); - - let mut resp_blocks = blocks[best_block_num as usize..from as usize].to_vec(); - resp_blocks.reverse(); - - let response = create_block_response(resp_blocks.clone()); - - let res = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == max_blocks_to_request as usize - ),); - - best_block_num += max_blocks_to_request as u32; - - let _ = sync.on_blocks_processed( - max_blocks_to_request as usize, - max_blocks_to_request as usize, - resp_blocks - .iter() - .rev() - .map(|b| { - ( - Ok(BlockImportStatus::ImportedUnknown( - *b.header().number(), - Default::default(), - Some(peer_id1), - )), - b.hash(), - ) - }) - .collect(), - ); - - resp_blocks - .into_iter() - .rev() - .for_each(|b| block_on(client.import_as_final(BlockOrigin::Own, b)).unwrap()); - } - - // "Wait" for the queue to clear - sync.queue_blocks.clear(); - - // Let peer2 announce that it finished syncing - send_block_announce(best_block.header().clone(), peer_id2, &mut sync); - - let (peer1_req, peer2_req) = - sync.block_requests().into_iter().fold((None, None), |res, req| { - if req.0 == peer_id1 { - (Some(req.1), res.1) - } else if req.0 == peer_id2 { - (res.0, Some(req.1)) - } else { - panic!("Unexpected req: {:?}", req) - } - }); - - // We should now do an ancestor search to find the correct common block. - let peer2_req = peer2_req.unwrap(); - assert_eq!(Some(1), peer2_req.max); - assert_eq!(FromBlock::Number(best_block_num as u64), peer2_req.from); - - let response = create_block_response(vec![blocks[(best_block_num - 1) as usize].clone()]); - let res = sync.on_block_data(&peer_id2, Some(peer2_req), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.is_empty() - ),); - - let peer1_from = unwrap_from_block_number(peer1_req.unwrap().from); - - // As we are on the same chain, we should directly continue with requesting blocks from - // peer 2 as well. - get_block_request( - &mut sync, - FromBlock::Number(peer1_from + max_blocks_to_request as u64), - max_blocks_to_request as u32, - &peer_id2, - ); - } - - /// A test that ensures that we can sync a huge fork. - /// - /// The following scenario: - /// A peer connects to us and we both have the common block 512. The last finalized is 2048. - /// Our best block is 4096. The peer send us a block announcement with 4097 from a fork. - /// - /// We will first do an ancestor search to find the common block. After that we start to sync - /// the fork and finish it ;) - #[test] - fn can_sync_huge_fork() { - sp_tracing::try_init_simple(); - - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut client = Arc::new(TestClientBuilder::new().build()); - let blocks = (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 4) - .map(|_| build_block(&mut client, None, false)) - .collect::>(); - - let fork_blocks = { - let mut client = Arc::new(TestClientBuilder::new().build()); - let fork_blocks = blocks[..MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2] - .into_iter() - .inspect(|b| block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap()) - .cloned() - .collect::>(); - - fork_blocks - .into_iter() - .chain( - (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 2 + 1) - .map(|_| build_block(&mut client, None, true)), - ) - .collect::>() - }; - - let info = client.info(); - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 5, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); - let just = (*b"TEST", Vec::new()); - client.finalize_block(finalized_block.hash(), Some(just)).unwrap(); - sync.update_chain_info(&info.best_hash, info.best_number); - - let peer_id1 = PeerId::random(); - - let common_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize / 2].clone(); - // Connect the node we will sync from - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) - .unwrap(); - - send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); - - let mut request = - get_block_request(&mut sync, FromBlock::Number(info.best_number), 1, &peer_id1); - - // Do the ancestor search - loop { - let block = &fork_blocks[unwrap_from_block_number(request.from.clone()) as usize - 1]; - let response = create_block_response(vec![block.clone()]); - - let on_block_data = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - request = if let OnBlockData::Request(_peer, request) = on_block_data { - request - } else { - // We found the ancenstor - break - }; - - log::trace!(target: LOG_TARGET, "Request: {request:?}"); - } - - // Now request and import the fork. - let mut best_block_num = *finalized_block.header().number() as u32; - let max_blocks_to_request = sync.max_blocks_per_request; - while best_block_num < *fork_blocks.last().unwrap().header().number() as u32 - 1 { - let request = get_block_request( - &mut sync, - FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), - max_blocks_to_request as u32, - &peer_id1, - ); - - let from = unwrap_from_block_number(request.from.clone()); - - let mut resp_blocks = fork_blocks[best_block_num as usize..from as usize].to_vec(); - resp_blocks.reverse(); - - let response = create_block_response(resp_blocks.clone()); - - let res = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == sync.max_blocks_per_request as usize - ),); - - best_block_num += sync.max_blocks_per_request as u32; - - let _ = sync.on_blocks_processed( - max_blocks_to_request as usize, - max_blocks_to_request as usize, - resp_blocks - .iter() - .rev() - .map(|b| { - ( - Ok(BlockImportStatus::ImportedUnknown( - *b.header().number(), - Default::default(), - Some(peer_id1), - )), - b.hash(), - ) - }) - .collect(), - ); - - resp_blocks - .into_iter() - .rev() - .for_each(|b| block_on(client.import(BlockOrigin::Own, b)).unwrap()); - } - - // Request the tip - get_block_request( - &mut sync, - FromBlock::Hash(fork_blocks.last().unwrap().hash()), - 1, - &peer_id1, - ); - } - - #[test] - fn syncs_fork_without_duplicate_requests() { - sp_tracing::try_init_simple(); - - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut client = Arc::new(TestClientBuilder::new().build()); - let blocks = (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 4) - .map(|_| build_block(&mut client, None, false)) - .collect::>(); - - let fork_blocks = { - let mut client = Arc::new(TestClientBuilder::new().build()); - let fork_blocks = blocks[..MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2] - .into_iter() - .inspect(|b| block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap()) - .cloned() - .collect::>(); - - fork_blocks - .into_iter() - .chain( - (0..MAX_BLOCKS_TO_LOOK_BACKWARDS * 2 + 1) - .map(|_| build_block(&mut client, None, true)), - ) - .collect::>() - }; - - let info = client.info(); - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 5, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone(); - let just = (*b"TEST", Vec::new()); - client.finalize_block(finalized_block.hash(), Some(just)).unwrap(); - sync.update_chain_info(&info.best_hash, info.best_number); - - let peer_id1 = PeerId::random(); - - let common_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize / 2].clone(); - // Connect the node we will sync from - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) - .unwrap(); - - send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); - - let mut request = - get_block_request(&mut sync, FromBlock::Number(info.best_number), 1, &peer_id1); - - // Do the ancestor search - loop { - let block = &fork_blocks[unwrap_from_block_number(request.from.clone()) as usize - 1]; - let response = create_block_response(vec![block.clone()]); - - let on_block_data = sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - request = if let OnBlockData::Request(_peer, request) = on_block_data { - request - } else { - // We found the ancenstor - break - }; - - log::trace!(target: LOG_TARGET, "Request: {request:?}"); - } - - // Now request and import the fork. - let mut best_block_num = *finalized_block.header().number() as u32; - let max_blocks_to_request = sync.max_blocks_per_request; - - let mut request = get_block_request( - &mut sync, - FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), - max_blocks_to_request as u32, - &peer_id1, - ); - let last_block_num = *fork_blocks.last().unwrap().header().number() as u32 - 1; - while best_block_num < last_block_num { - let from = unwrap_from_block_number(request.from.clone()); - - let mut resp_blocks = fork_blocks[best_block_num as usize..from as usize].to_vec(); - resp_blocks.reverse(); - - let response = create_block_response(resp_blocks.clone()); - - let res = sync.on_block_data(&peer_id1, Some(request.clone()), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == max_blocks_to_request as usize - ),); - - best_block_num += max_blocks_to_request as u32; - - if best_block_num < last_block_num { - // make sure we're not getting a duplicate request in the time before the blocks are - // processed - request = get_block_request( - &mut sync, - FromBlock::Number(max_blocks_to_request as u64 + best_block_num as u64), - max_blocks_to_request as u32, - &peer_id1, - ); - } - - let mut notify_imported: Vec<_> = resp_blocks - .iter() - .rev() - .map(|b| { - ( - Ok(BlockImportStatus::ImportedUnknown( - *b.header().number(), - Default::default(), - Some(peer_id1), - )), - b.hash(), - ) - }) - .collect(); - - // The import queue may send notifications in batches of varying size. So we simulate - // this here by splitting the batch into 2 notifications. - let max_blocks_to_request = sync.max_blocks_per_request; - let second_batch = notify_imported.split_off(notify_imported.len() / 2); - let _ = sync.on_blocks_processed( - max_blocks_to_request as usize, - max_blocks_to_request as usize, - notify_imported, - ); - - let _ = sync.on_blocks_processed( - max_blocks_to_request as usize, - max_blocks_to_request as usize, - second_batch, - ); - - resp_blocks - .into_iter() - .rev() - .for_each(|b| block_on(client.import(BlockOrigin::Own, b)).unwrap()); - } - - // Request the tip - get_block_request( - &mut sync, - FromBlock::Hash(fork_blocks.last().unwrap().hash()), - 1, - &peer_id1, - ); - } - - #[test] - fn removes_target_fork_on_disconnect() { - sp_tracing::try_init_simple(); - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut client = Arc::new(TestClientBuilder::new().build()); - let blocks = (0..3).map(|_| build_block(&mut client, None, false)).collect::>(); - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 1, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let peer_id1 = PeerId::random(); - let common_block = blocks[1].clone(); - // Connect the node we will sync from - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) - .unwrap(); - - // Create a "new" header and announce it - let mut header = blocks[0].header().clone(); - header.number = 4; - send_block_announce(header, peer_id1, &mut sync); - assert!(sync.fork_targets.len() == 1); - - let _ = sync.peer_disconnected(&peer_id1); - assert!(sync.fork_targets.len() == 0); - } - - #[test] - fn can_import_response_with_missing_blocks() { - sp_tracing::try_init_simple(); - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut client2 = Arc::new(TestClientBuilder::new().build()); - let blocks = (0..4).map(|_| build_block(&mut client2, None, false)).collect::>(); - - let empty_client = Arc::new(TestClientBuilder::new().build()); - - let mut sync = ChainSync::new( - SyncMode::Full, - empty_client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 1, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let peer_id1 = PeerId::random(); - let best_block = blocks[3].clone(); - sync.new_peer(peer_id1, best_block.hash(), *best_block.header().number()) - .unwrap(); - - sync.peers.get_mut(&peer_id1).unwrap().state = PeerSyncState::Available; - sync.peers.get_mut(&peer_id1).unwrap().common_number = 0; - - // Request all missing blocks and respond only with some. - let request = - get_block_request(&mut sync, FromBlock::Hash(best_block.hash()), 4, &peer_id1); - let response = - create_block_response(vec![blocks[3].clone(), blocks[2].clone(), blocks[1].clone()]); - sync.on_block_data(&peer_id1, Some(request.clone()), response).unwrap(); - assert_eq!(sync.best_queued_number, 0); - - // Request should only contain the missing block. - let request = get_block_request(&mut sync, FromBlock::Number(1), 1, &peer_id1); - let response = create_block_response(vec![blocks[0].clone()]); - sync.on_block_data(&peer_id1, Some(request), response).unwrap(); - assert_eq!(sync.best_queued_number, 4); - } - #[test] - fn ancestor_search_repeat() { - let state = AncestorSearchState::::BinarySearch(1, 3); - assert!(handle_ancestor_search_state(&state, 2, true).is_none()); - } - - #[test] - fn sync_restart_removes_block_but_not_justification_requests() { - let mut client = Arc::new(TestClientBuilder::new().build()); - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 1, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - let peers = vec![PeerId::random(), PeerId::random()]; - - let mut new_blocks = |n| { - for _ in 0..n { - let block = client.new_block(Default::default()).unwrap().build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block.clone())).unwrap(); - } - - let info = client.info(); - (info.best_hash, info.best_number) - }; - - let (b1_hash, b1_number) = new_blocks(50); - - // add new peer and request blocks from them - sync.new_peer(peers[0], Hash::random(), 42).unwrap(); - - // we don't actually perform any requests, just keep track of peers waiting for a response - let mut pending_responses = HashSet::new(); - - // we wil send block requests to these peers - // for these blocks we don't know about - for (peer, _request) in sync.block_requests() { - // "send" request - pending_responses.insert(peer); - } - - // add a new peer at a known block - sync.new_peer(peers[1], b1_hash, b1_number).unwrap(); - - // we request a justification for a block we have locally - sync.request_justification(&b1_hash, b1_number); - - // the justification request should be scheduled to the - // new peer which is at the given block - let mut requests = sync.justification_requests(); - assert_eq!(requests.len(), 1); - let (peer, _request) = requests.remove(0); - // "send" request - assert!(pending_responses.insert(peer)); - - assert!(!std::matches!( - sync.peers.get(&peers[0]).unwrap().state, - PeerSyncState::DownloadingJustification(_), - )); - assert_eq!( - sync.peers.get(&peers[1]).unwrap().state, - PeerSyncState::DownloadingJustification(b1_hash), - ); - assert_eq!(pending_responses.len(), 2); - - // restart sync - let request_events = sync.restart().collect::>(); - for event in request_events.iter() { - match event.as_ref().unwrap() { - BlockRequestAction::RemoveStale { peer_id } => { - pending_responses.remove(&peer_id); - }, - BlockRequestAction::SendRequest { peer_id, .. } => { - // we drop obsolete response, but don't register a new request, it's checked in - // the `assert!` below - pending_responses.remove(&peer_id); - }, - } - } - assert!(request_events.iter().any(|event| { - match event.as_ref().unwrap() { - BlockRequestAction::RemoveStale { .. } => false, - BlockRequestAction::SendRequest { peer_id, .. } => peer_id == &peers[0], - } - })); - - assert_eq!(pending_responses.len(), 1); - assert!(pending_responses.contains(&peers[1])); - assert_eq!( - sync.peers.get(&peers[1]).unwrap().state, - PeerSyncState::DownloadingJustification(b1_hash), - ); - let _ = sync.peer_disconnected(&peers[1]); - pending_responses.remove(&peers[1]); - assert_eq!(pending_responses.len(), 0); - } - - /// The test demonstrates https://github.com/paritytech/polkadot-sdk/issues/2094. - /// TODO: convert it into desired behavior test once the issue is fixed (see inline comments). - /// The issue: we currently rely on block numbers instead of block hash - /// to download blocks from peers. As a result, we can end up with blocks - /// from different forks as shown by the test. - #[test] - #[should_panic] - fn request_across_forks() { - sp_tracing::try_init_simple(); - - let (_chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); - let mut client = Arc::new(TestClientBuilder::new().build()); - let blocks = (0..100).map(|_| build_block(&mut client, None, false)).collect::>(); - - let fork_a_blocks = { - let mut client = Arc::new(TestClientBuilder::new().build()); - let mut fork_blocks = blocks[..] - .into_iter() - .inspect(|b| { - assert!(matches!(client.block(*b.header.parent_hash()), Ok(Some(_)))); - block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap() - }) - .cloned() - .collect::>(); - for _ in 0..10 { - fork_blocks.push(build_block(&mut client, None, false)); - } - fork_blocks - }; - - let fork_b_blocks = { - let mut client = Arc::new(TestClientBuilder::new().build()); - let mut fork_blocks = blocks[..] - .into_iter() - .inspect(|b| { - assert!(matches!(client.block(*b.header.parent_hash()), Ok(Some(_)))); - block_on(client.import(BlockOrigin::Own, (*b).clone())).unwrap() - }) - .cloned() - .collect::>(); - for _ in 0..10 { - fork_blocks.push(build_block(&mut client, None, true)); - } - fork_blocks - }; - - let mut sync = ChainSync::new( - SyncMode::Full, - client.clone(), - ProtocolName::from("test-block-announce-protocol"), - 5, - 64, - None, - chain_sync_network_handle, - ) - .unwrap(); - - // Add the peers, all at the common ancestor 100. - let common_block = blocks.last().unwrap(); - let peer_id1 = PeerId::random(); - sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) - .unwrap(); - let peer_id2 = PeerId::random(); - sync.new_peer(peer_id2, common_block.hash(), *common_block.header().number()) - .unwrap(); - - // Peer 1 announces 107 from fork 1, 100-107 get downloaded. - { - let block = (&fork_a_blocks[106]).clone(); - let peer = peer_id1; - log::trace!(target: LOG_TARGET, "<1> {peer} announces from fork 1"); - send_block_announce(block.header().clone(), peer, &mut sync); - let request = get_block_request(&mut sync, FromBlock::Hash(block.hash()), 7, &peer); - let mut resp_blocks = fork_a_blocks[100_usize..107_usize].to_vec(); - resp_blocks.reverse(); - let response = create_block_response(resp_blocks.clone()); - let res = sync.on_block_data(&peer, Some(request), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == 7_usize - ),); - assert_eq!(sync.best_queued_number, 107); - assert_eq!(sync.best_queued_hash, block.hash()); - assert!(sync.is_known(&block.header.parent_hash())); - } - - // Peer 2 also announces 107 from fork 1. - { - let prev_best_number = sync.best_queued_number; - let prev_best_hash = sync.best_queued_hash; - let peer = peer_id2; - log::trace!(target: LOG_TARGET, "<2> {peer} announces from fork 1"); - for i in 100..107 { - let block = (&fork_a_blocks[i]).clone(); - send_block_announce(block.header().clone(), peer, &mut sync); - assert!(sync.block_requests().is_empty()); - } - assert_eq!(sync.best_queued_number, prev_best_number); - assert_eq!(sync.best_queued_hash, prev_best_hash); - } - - // Peer 2 undergoes reorg, announces 108 from fork 2, gets downloaded even though we - // don't have the parent from fork 2. - { - let block = (&fork_b_blocks[107]).clone(); - let peer = peer_id2; - log::trace!(target: LOG_TARGET, "<3> {peer} announces from fork 2"); - send_block_announce(block.header().clone(), peer, &mut sync); - // TODO: when the issue is fixed, this test can be changed to test the - // expected behavior instead. The needed changes would be: - // 1. Remove the `#[should_panic]` directive - // 2. These should be changed to check that sync.block_requests().is_empty(), after the - // block is announced. - let request = get_block_request(&mut sync, FromBlock::Hash(block.hash()), 1, &peer); - let response = create_block_response(vec![block.clone()]); - let res = sync.on_block_data(&peer, Some(request), response).unwrap(); - assert!(matches!( - res, - OnBlockData::Import(ImportBlocksAction{ origin: _, blocks }) if blocks.len() == 1_usize - ),); - assert!(sync.is_known(&block.header.parent_hash())); - } - } -} diff --git a/substrate/client/network/sync/src/mock.rs b/substrate/client/network/sync/src/mock.rs index ed7c647c7977..42220096e069 100644 --- a/substrate/client/network/sync/src/mock.rs +++ b/substrate/client/network/sync/src/mock.rs @@ -23,65 +23,8 @@ use crate::block_relay_protocol::{BlockDownloader as BlockDownloaderT, BlockResp use futures::channel::oneshot; use libp2p::PeerId; use sc_network::RequestFailure; -use sc_network_common::sync::{ - message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse}, - BadPeer, ChainSync as ChainSyncT, ImportBlocksAction, Metrics, OnBlockData, - OnBlockJustification, PeerInfo, SyncStatus, -}; -use sp_runtime::traits::{Block as BlockT, NumberFor}; - -mockall::mock! { - pub ChainSync {} - - impl ChainSyncT for ChainSync { - fn peer_info(&self, who: &PeerId) -> Option>; - fn status(&self) -> SyncStatus; - fn num_sync_requests(&self) -> usize; - fn num_downloaded_blocks(&self) -> usize; - fn num_peers(&self) -> usize; - fn new_peer( - &mut self, - who: PeerId, - best_hash: Block::Hash, - best_number: NumberFor, - ) -> Result>, BadPeer>; - fn update_chain_info(&mut self, best_hash: &Block::Hash, best_number: NumberFor); - fn request_justification(&mut self, hash: &Block::Hash, number: NumberFor); - fn clear_justification_requests(&mut self); - fn set_sync_fork_request( - &mut self, - peers: Vec, - hash: &Block::Hash, - number: NumberFor, - ); - fn on_block_data( - &mut self, - who: &PeerId, - request: Option>, - response: BlockResponse, - ) -> Result, BadPeer>; - fn on_block_justification( - &mut self, - who: PeerId, - response: BlockResponse, - ) -> Result, BadPeer>; - fn on_justification_import( - &mut self, - hash: Block::Hash, - number: NumberFor, - success: bool, - ); - fn on_block_finalized(&mut self, hash: &Block::Hash, number: NumberFor); - fn on_validated_block_announce( - &mut self, - is_best: bool, - who: PeerId, - announce: &BlockAnnounce, - ); - fn peer_disconnected(&mut self, who: &PeerId) -> Option>; - fn metrics(&self) -> Metrics; - } -} +use sc_network_common::sync::message::{BlockData, BlockRequest}; +use sp_runtime::traits::Block as BlockT; mockall::mock! { pub BlockDownloader {} diff --git a/substrate/client/network/sync/src/pending_responses.rs b/substrate/client/network/sync/src/pending_responses.rs index c863267e7808..9e2fd5cfd674 100644 --- a/substrate/client/network/sync/src/pending_responses.rs +++ b/substrate/client/network/sync/src/pending_responses.rs @@ -19,6 +19,7 @@ //! [`PendingResponses`] is responsible for keeping track of pending responses and //! polling them. +use crate::types::PeerRequest; use futures::{ channel::oneshot, future::BoxFuture, @@ -28,11 +29,13 @@ use futures::{ use libp2p::PeerId; use log::error; use sc_network::request_responses::RequestFailure; -use sc_network_common::sync::PeerRequest; use sp_runtime::traits::Block as BlockT; use std::task::{Context, Poll}; use tokio_stream::StreamMap; +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + /// Response result. type ResponseResult = Result, RequestFailure>, oneshot::Canceled>; @@ -74,7 +77,7 @@ impl PendingResponses { .is_some() { error!( - target: crate::LOG_TARGET, + target: LOG_TARGET, "Discarded pending response from peer {peer_id}, request type: {request_type:?}.", ); debug_assert!(false); diff --git a/substrate/client/network/common/src/sync/metrics.rs b/substrate/client/network/sync/src/request_metrics.rs similarity index 100% rename from substrate/client/network/common/src/sync/metrics.rs rename to substrate/client/network/sync/src/request_metrics.rs diff --git a/substrate/client/network/sync/src/service/chain_sync.rs b/substrate/client/network/sync/src/service/chain_sync.rs index f9e0e401fdf8..3d11880c511c 100644 --- a/substrate/client/network/sync/src/service/chain_sync.rs +++ b/substrate/client/network/sync/src/service/chain_sync.rs @@ -16,14 +16,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use crate::types::{ExtendedPeerInfo, SyncEvent, SyncEventStream, SyncStatus, SyncStatusProvider}; + use futures::{channel::oneshot, Stream}; use libp2p::PeerId; use sc_consensus::{BlockImportError, BlockImportStatus, JustificationSyncLink, Link}; use sc_network::{NetworkBlock, NetworkSyncForkRequest}; -use sc_network_common::sync::{ - ExtendedPeerInfo, SyncEvent, SyncEventStream, SyncStatus, SyncStatusProvider, -}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_runtime::traits::{Block as BlockT, NumberFor}; diff --git a/substrate/client/network/sync/src/state.rs b/substrate/client/network/sync/src/state.rs index 305f0ee6838a..5d34613d1c5e 100644 --- a/substrate/client/network/sync/src/state.rs +++ b/substrate/client/network/sync/src/state.rs @@ -18,12 +18,14 @@ //! State sync support. -use crate::schema::v1::{StateEntry, StateRequest, StateResponse}; +use crate::{ + schema::v1::{StateEntry, StateRequest, StateResponse}, + types::StateDownloadProgress, +}; use codec::{Decode, Encode}; use log::debug; use sc_client_api::{CompactProof, ProofProvider}; use sc_consensus::ImportedState; -use sc_network_common::sync::StateDownloadProgress; use smallvec::SmallVec; use sp_core::storage::well_known_keys; use sp_runtime::{ diff --git a/substrate/client/network/sync/src/types.rs b/substrate/client/network/sync/src/types.rs new file mode 100644 index 000000000000..5931cf47b28a --- /dev/null +++ b/substrate/client/network/sync/src/types.rs @@ -0,0 +1,206 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Common syncing types. + +use futures::Stream; +use sc_network_common::{role::Roles, types::ReputationChange}; + +use libp2p::PeerId; + +use crate::warp::WarpSyncProgress; +use sc_network_common::sync::message::BlockRequest; +use sp_runtime::traits::{Block as BlockT, NumberFor}; + +use std::{any::Any, fmt, fmt::Formatter, pin::Pin, sync::Arc}; + +pub use sc_network_common::sync::SyncMode; + +/// The sync status of a peer we are trying to sync with +#[derive(Debug)] +pub struct PeerInfo { + /// Their best block hash. + pub best_hash: Block::Hash, + /// Their best block number. + pub best_number: NumberFor, +} + +/// Info about a peer's known state (both full and light). +#[derive(Clone, Debug)] +pub struct ExtendedPeerInfo { + /// Roles + pub roles: Roles, + /// Peer best block hash + pub best_hash: B::Hash, + /// Peer best block number + pub best_number: NumberFor, +} + +/// Reported sync state. +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum SyncState { + /// Initial sync is complete, keep-up sync is active. + Idle, + /// Actively catching up with the chain. + Downloading { target: BlockNumber }, + /// All blocks are downloaded and are being imported. + Importing { target: BlockNumber }, +} + +impl SyncState { + /// Are we actively catching up with the chain? + pub fn is_major_syncing(&self) -> bool { + !matches!(self, SyncState::Idle) + } +} + +/// Reported state download progress. +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct StateDownloadProgress { + /// Estimated download percentage. + pub percentage: u32, + /// Total state size in bytes downloaded so far. + pub size: u64, +} + +/// Syncing status and statistics. +#[derive(Debug, Clone)] +pub struct SyncStatus { + /// Current global sync state. + pub state: SyncState>, + /// Target sync block number. + pub best_seen_block: Option>, + /// Number of peers participating in syncing. + pub num_peers: u32, + /// Number of peers known to `SyncingEngine` (both full and light). + pub num_connected_peers: u32, + /// Number of blocks queued for import + pub queued_blocks: u32, + /// State sync status in progress, if any. + pub state_sync: Option, + /// Warp sync in progress, if any. + pub warp_sync: Option>, +} + +/// A peer did not behave as expected and should be reported. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BadPeer(pub PeerId, pub ReputationChange); + +impl fmt::Display for BadPeer { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Bad peer {}; Reputation change: {:?}", self.0, self.1) + } +} + +impl std::error::Error for BadPeer {} + +#[derive(Debug)] +pub struct Metrics { + pub queued_blocks: u32, + pub fork_targets: u32, + pub justifications: crate::request_metrics::Metrics, +} + +#[derive(Debug)] +pub enum PeerRequest { + Block(BlockRequest), + State, + WarpProof, +} + +#[derive(Debug)] +pub enum PeerRequestType { + Block, + State, + WarpProof, +} + +impl PeerRequest { + pub fn get_type(&self) -> PeerRequestType { + match self { + PeerRequest::Block(_) => PeerRequestType::Block, + PeerRequest::State => PeerRequestType::State, + PeerRequest::WarpProof => PeerRequestType::WarpProof, + } + } +} + +/// Wrapper for implementation-specific state request. +/// +/// NOTE: Implementation must be able to encode and decode it for network purposes. +pub struct OpaqueStateRequest(pub Box); + +impl fmt::Debug for OpaqueStateRequest { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpaqueStateRequest").finish() + } +} + +/// Wrapper for implementation-specific state response. +/// +/// NOTE: Implementation must be able to encode and decode it for network purposes. +pub struct OpaqueStateResponse(pub Box); + +impl fmt::Debug for OpaqueStateResponse { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpaqueStateResponse").finish() + } +} + +/// Provides high-level status of syncing. +#[async_trait::async_trait] +pub trait SyncStatusProvider: Send + Sync { + /// Get high-level view of the syncing status. + async fn status(&self) -> Result, ()>; +} + +#[async_trait::async_trait] +impl SyncStatusProvider for Arc +where + T: ?Sized, + T: SyncStatusProvider, + Block: BlockT, +{ + async fn status(&self) -> Result, ()> { + T::status(self).await + } +} + +/// Syncing-related events that other protocols can subscribe to. +pub enum SyncEvent { + /// Peer that the syncing implementation is tracking connected. + PeerConnected(PeerId), + + /// Peer that the syncing implementation was tracking disconnected. + PeerDisconnected(PeerId), +} + +pub trait SyncEventStream: Send + Sync { + /// Subscribe to syncing-related events. + fn event_stream(&self, name: &'static str) -> Pin + Send>>; +} + +impl SyncEventStream for Arc +where + T: ?Sized, + T: SyncEventStream, +{ + fn event_stream(&self, name: &'static str) -> Pin + Send>> { + T::event_stream(self, name) + } +} diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 74835a6e015e..2c0adc856c12 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -18,28 +18,107 @@ //! Warp sync support. +pub use sp_consensus_grandpa::{AuthorityList, SetId}; + use crate::{ schema::v1::{StateRequest, StateResponse}, state::{ImportResult, StateSync}, }; +use codec::{Decode, Encode}; use futures::channel::oneshot; use log::error; use sc_client_api::ProofProvider; -use sc_network_common::sync::{ - message::{BlockAttributes, BlockData, BlockRequest, Direction, FromBlock}, - warp::{ - EncodedProof, VerificationResult, WarpProofRequest, WarpSyncPhase, WarpSyncProgress, - WarpSyncProvider, - }, +use sc_network_common::sync::message::{ + BlockAttributes, BlockData, BlockRequest, Direction, FromBlock, }; use sp_blockchain::HeaderBackend; -use sp_consensus_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; -use std::sync::Arc; +use std::{fmt, sync::Arc}; /// Log target for this file. const LOG_TARGET: &'static str = "sync"; +/// Scale-encoded warp sync proof response. +pub struct EncodedProof(pub Vec); + +/// Warp sync request +#[derive(Encode, Decode, Debug)] +pub struct WarpProofRequest { + /// Start collecting proofs from this block. + pub begin: B::Hash, +} + +/// Proof verification result. +pub enum VerificationResult { + /// Proof is valid, but the target was not reached. + Partial(SetId, AuthorityList, Block::Hash), + /// Target finality is proved. + Complete(SetId, AuthorityList, Block::Header), +} + +/// Warp sync backend. Handles retrieving and verifying warp sync proofs. +pub trait WarpSyncProvider: Send + Sync { + /// Generate proof starting at given block hash. The proof is accumulated until maximum proof + /// size is reached. + fn generate( + &self, + start: Block::Hash, + ) -> Result>; + /// Verify warp proof against current set of authorities. + fn verify( + &self, + proof: &EncodedProof, + set_id: SetId, + authorities: AuthorityList, + ) -> Result, Box>; + /// Get current list of authorities. This is supposed to be genesis authorities when starting + /// sync. + fn current_authorities(&self) -> AuthorityList; +} + +/// Reported warp sync phase. +#[derive(Clone, Eq, PartialEq, Debug)] +pub enum WarpSyncPhase { + /// Waiting for peers to connect. + AwaitingPeers { required_peers: usize }, + /// Waiting for target block to be received. + AwaitingTargetBlock, + /// Downloading and verifying grandpa warp proofs. + DownloadingWarpProofs, + /// Downloading target block. + DownloadingTargetBlock, + /// Downloading state data. + DownloadingState, + /// Importing state. + ImportingState, + /// Downloading block history. + DownloadingBlocks(NumberFor), +} + +impl fmt::Display for WarpSyncPhase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::AwaitingPeers { required_peers } => + write!(f, "Waiting for {required_peers} peers to be connected"), + Self::AwaitingTargetBlock => write!(f, "Waiting for target block to be received"), + Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"), + Self::DownloadingTargetBlock => write!(f, "Downloading target block"), + Self::DownloadingState => write!(f, "Downloading state"), + Self::ImportingState => write!(f, "Importing state"), + Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n), + } + } +} + +/// Reported warp sync progress. +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct WarpSyncProgress { + /// Estimated download percentage. + pub phase: WarpSyncPhase, + /// Total bytes downloaded so far. + pub total_bytes: u64, +} + /// The different types of warp syncing, passed to `build_network`. pub enum WarpSyncParams { /// Standard warp sync for the chain. diff --git a/substrate/client/network/sync/src/warp_request_handler.rs b/substrate/client/network/sync/src/warp_request_handler.rs index 0e502a6dba59..b23f30c50dd2 100644 --- a/substrate/client/network/sync/src/warp_request_handler.rs +++ b/substrate/client/network/sync/src/warp_request_handler.rs @@ -20,13 +20,13 @@ use codec::Decode; use futures::{channel::oneshot, stream::StreamExt}; use log::debug; +use crate::warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}; use sc_network::{ config::ProtocolId, request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, }; -use sc_network_common::sync::warp::{EncodedProof, WarpProofRequest, WarpSyncProvider}; use sp_runtime::traits::Block as BlockT; use std::{sync::Arc, time::Duration}; diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index 11505903e35d..ad4201f94a20 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -60,16 +60,15 @@ use sc_network::{ Multiaddr, NetworkBlock, NetworkService, NetworkStateInfo, NetworkSyncForkRequest, NetworkWorker, }; -use sc_network_common::{ - role::Roles, - sync::warp::{AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncProvider}, -}; +use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ block_request_handler::BlockRequestHandler, service::{chain_sync::SyncingService, network::NetworkServiceProvider}, state_request_handler::StateRequestHandler, - warp::WarpSyncParams, + warp::{ + AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncParams, WarpSyncProvider, + }, warp_request_handler, }; use sc_service::client::Client; diff --git a/substrate/client/network/transactions/Cargo.toml b/substrate/client/network/transactions/Cargo.toml index 5e42465974bb..2a6aa4b3a40a 100644 --- a/substrate/client/network/transactions/Cargo.toml +++ b/substrate/client/network/transactions/Cargo.toml @@ -21,6 +21,7 @@ log = "0.4.17" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus" } sc-network = { path = ".." } sc-network-common = { path = "../common" } +sc-network-sync = { path = "../sync" } sc-utils = { path = "../../utils" } sp-runtime = { path = "../../../primitives/runtime" } sp-consensus = { path = "../../../primitives/consensus/common" } diff --git a/substrate/client/network/transactions/src/lib.rs b/substrate/client/network/transactions/src/lib.rs index b46733d42723..1b97d4b96c97 100644 --- a/substrate/client/network/transactions/src/lib.rs +++ b/substrate/client/network/transactions/src/lib.rs @@ -42,11 +42,8 @@ use sc_network::{ utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, }; -use sc_network_common::{ - role::ObservedRole, - sync::{SyncEvent, SyncEventStream}, - ExHashT, -}; +use sc_network_common::{role::ObservedRole, ExHashT}; +use sc_network_sync::{SyncEvent, SyncEventStream}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::Block as BlockT; diff --git a/substrate/client/service/src/metrics.rs b/substrate/client/service/src/metrics.rs index ece5758be771..a411a83a784e 100644 --- a/substrate/client/service/src/metrics.rs +++ b/substrate/client/service/src/metrics.rs @@ -23,7 +23,7 @@ use futures_timer::Delay; use prometheus_endpoint::{register, Gauge, GaugeVec, Opts, PrometheusError, Registry, U64}; use sc_client_api::{ClientInfo, UsageProvider}; use sc_network::{config::Role, NetworkStatus, NetworkStatusProvider}; -use sc_network_common::sync::{SyncStatus, SyncStatusProvider}; +use sc_network_sync::{SyncStatus, SyncStatusProvider}; use sc_telemetry::{telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sc_transaction_pool_api::{MaintainedTransactionPool, PoolStatus}; use sc_utils::metrics::register_globals;