diff --git a/Cargo.lock b/Cargo.lock index 1197b7541..7167af1ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,13 +1789,13 @@ dependencies = [ [[package]] name = "ethy-gadget" -version = "0.1.0" +version = "0.1.2" dependencies = [ "ethabi", "futures", "futures-timer", "hex", - "hex-literal 0.4.1", + "hex-literal", "libsecp256k1 0.6.0", "log", "parity-scale-codec", @@ -2963,12 +2963,6 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" -[[package]] -name = "hex-literal" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" - [[package]] name = "hex_fmt" version = "0.3.0" @@ -5170,7 +5164,7 @@ dependencies = [ "frame-executive", "frame-support", "frame-system", - "hex-literal 0.3.4", + "hex-literal", "pallet-assets", "pallet-assets-ext", "pallet-balances", @@ -5256,7 +5250,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "hex-literal 0.3.4", + "hex-literal", "log", "pallet-assets", "pallet-assets-ext", @@ -5309,7 +5303,7 @@ dependencies = [ "frame-support", "frame-system", "hex", - "hex-literal 0.3.4", + "hex-literal", "pallet-assets", "pallet-assets-ext", "pallet-balances", @@ -5643,7 +5637,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "hex-literal 0.3.4", + "hex-literal", "pallet-assets", "pallet-assets-ext", "pallet-balances", @@ -5673,7 +5667,7 @@ dependencies = [ "frame-support", "frame-system", "hex", - "hex-literal 0.3.4", + "hex-literal", "log", "pallet-assets", "pallet-assets-ext", @@ -5845,7 +5839,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "hex-literal 0.3.4", + "hex-literal", "log", "pallet-assets", "pallet-assets-ext", @@ -6157,7 +6151,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "hex-literal 0.3.4", + "hex-literal", "pallet-assets", "pallet-assets-ext", "pallet-balances", @@ -6339,7 +6333,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "hex-literal 0.3.4", + "hex-literal", "log", "pallet-assets", "pallet-assets-ext", @@ -6746,7 +6740,7 @@ dependencies = [ "frame-support", "frame-system", "hex", - "hex-literal 0.3.4", + "hex-literal", "impl-trait-for-tuples", "log", "num_enum", @@ -8759,7 +8753,7 @@ dependencies = [ [[package]] name = "seed-client" -version = "5.0.0" +version = "7.0.0" dependencies = [ "clap", "ethy-gadget", @@ -8778,7 +8772,7 @@ dependencies = [ "frame-try-runtime", "futures", "hex", - "hex-literal 0.3.4", + "hex-literal", "jsonrpsee", "libsecp256k1 0.6.0", "log", @@ -8852,7 +8846,7 @@ dependencies = [ "frame-support", "frame-system", "hex", - "hex-literal 0.3.4", + "hex-literal", "impl-serde 0.4.0", "libsecp256k1 0.7.1", "log", @@ -8893,7 +8887,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "frame-try-runtime", "hex", - "hex-literal 0.3.4", + "hex-literal", "log", "pallet-assets", "pallet-assets-ext", diff --git a/client/Cargo.toml b/client/Cargo.toml index d9fe66ff1..331e0046c 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "seed-client" -version = "5.0.0" +version = "7.0.0" authors = ["The Root Network Team"] edition = "2021" publish = false diff --git a/ethy-gadget/Cargo.toml b/ethy-gadget/Cargo.toml index 4f2ee855d..3ba792c7a 100644 --- a/ethy-gadget/Cargo.toml +++ b/ethy-gadget/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ethy-gadget" -version = "0.1.0" +version = "0.1.2" authors = ["Parity Technologies ", "The Root Network Team"] edition = "2021" license = "Apache-2.0" diff --git a/ethy-gadget/src/gossip.rs b/ethy-gadget/src/gossip.rs index 066c7b07e..addce6bd7 100644 --- a/ethy-gadget/src/gossip.rs +++ b/ethy-gadget/src/gossip.rs @@ -14,13 +14,17 @@ // You may obtain a copy of the License at the root of this project source code use codec::Decode; -use log::{error, info, trace, warn}; +use log::{debug, error, info, trace, warn}; use parking_lot::{Mutex, RwLock}; +use sc_client_api::Backend; use sc_network::PeerId; use sc_network_gossip::{MessageIntent, ValidationResult, Validator, ValidatorContext}; +use sp_api::{BlockT, HeaderT}; +use sp_blockchain::HeaderBackend; use sp_runtime::traits::{Block, Hash, Header}; use std::{ collections::{BTreeMap, VecDeque}, + sync::Arc, time::{Duration, Instant}, }; @@ -37,19 +41,31 @@ where } /// Number of recent complete events to keep in memory -const MAX_COMPLETE_EVENT_CACHE: usize = 30; +// Theoretically This buffer should hold completed events until they go out of live window. +// rough theoretical value of 2520 is suitable at the expense of increased search time. Should not +// be problematic. can change as the network grows if required. +const MAX_COMPLETE_EVENT_CACHE: usize = 500; // Timeout for rebroadcasting messages. -const REBROADCAST_AFTER: Duration = Duration::from_secs(60 * 5); +const REBROADCAST_AFTER: Duration = Duration::from_secs(60 * 3); + +// Window size in blocks within which we expect the request to reach terminal state. +// We take the WINDOW_SIZE approximately as 6 mins. This gives at-least another rebroadcast before +// going out of live window. +#[cfg(not(test))] +const WINDOW_SIZE: u64 = 90; +#[cfg(test)] +const WINDOW_SIZE: u64 = 5; /// ETHY gossip validator /// /// Validate ETHY gossip messages /// /// All messaging is handled in a single ETHY global topic. -pub(crate) struct GossipValidator +pub(crate) struct GossipValidator where B: Block, + BE: Backend, { topic: B::Hash, known_votes: RwLock>>, @@ -59,19 +75,23 @@ where active_validators: RwLock>, /// Scheduled time for re-broadcasting event witnesses next_rebroadcast: Mutex, + /// client backend + backend: Arc, } -impl GossipValidator +impl GossipValidator where B: Block, + BE: Backend, { - pub fn new(active_validators: Vec) -> GossipValidator { + pub fn new(active_validators: Vec, backend: Arc) -> GossipValidator { GossipValidator { topic: topic::(), known_votes: RwLock::new(BTreeMap::new()), active_validators: RwLock::new(active_validators), complete_events: RwLock::new(Default::default()), next_rebroadcast: Mutex::new(Instant::now() + REBROADCAST_AFTER), + backend, } } @@ -106,9 +126,11 @@ where } } -impl Validator for GossipValidator +impl Validator for GossipValidator where B: Block, + BE: Backend, + <::Header as HeaderT>::Number: Into, { fn validate( &self, @@ -116,8 +138,15 @@ where sender: &PeerId, mut data: &[u8], ) -> ValidationResult { - if let Ok(Witness { authority_id, event_id, validator_set_id, digest, signature, .. }) = - Witness::decode(&mut data) + if let Ok(Witness { + authority_id, + event_id, + validator_set_id, + digest, + signature, + block_number, + .. + }) = Witness::decode(&mut data) { trace!(target: "ethy", "💎 witness from: {:?}, validator set: {:?}, event: {:?}", authority_id, validator_set_id, event_id); @@ -155,6 +184,12 @@ where } trace!(target: "ethy", "💎 valid witness: {:?}, event: {:?}", &authority_id, event_id); + let finalized_number = self.backend.blockchain().info().finalized_number; + if block_number < finalized_number.into().saturating_sub(WINDOW_SIZE) { + info!(target: "ethy", "💎 witness: {:?}, event: {:?} sender: {:?} out of live window. mark as discard.", &authority_id, event_id, sender); + return ValidationResult::Discard + } + return ValidationResult::ProcessAndKeep(self.topic) } else { // TODO: decrease peer reputation @@ -174,7 +209,13 @@ where Err(_) => return true, }; - let expired = complete_events.binary_search(&witness.event_id).is_ok(); + let finalized_number = self.backend.blockchain().info().finalized_number; + if witness.block_number < finalized_number.into().saturating_sub(WINDOW_SIZE) { + debug!(target: "ethy", "💎 Message for event #{} is out of live window. marked as expired: {}", witness.event_id, true); + return true + } + + let expired = complete_events.binary_search(&witness.event_id).is_ok(); // spk trace!(target: "ethy", "💎 Message for event #{} expired: {}", witness.event_id, expired); expired @@ -204,9 +245,14 @@ where let witness = match Witness::decode(&mut data) { Ok(w) => w, - Err(_) => return true, + Err(_) => return false, }; + let finalized_number = self.backend.blockchain().info().finalized_number; + if witness.block_number < finalized_number.into().saturating_sub(WINDOW_SIZE) { + debug!(target: "ethy", "💎 Message for event #{} is out of live window. marked as allowed: {}", witness.event_id, false); + return false + } // Check if message is incomplete let allowed = complete_events.binary_search(&witness.event_id).is_err(); @@ -221,14 +267,15 @@ where mod tests { use codec::Encode; use sc_network::PeerId; - use sc_network_gossip::{ValidationResult, Validator, ValidatorContext}; - use sc_network_test::{Block, Hash}; + use sc_network_gossip::{MessageIntent, ValidationResult, Validator, ValidatorContext}; + use sc_network_test::{Block, Hash, TestNetFactory}; use sp_core::keccak_256; + use sp_runtime::generic::BlockId; use seed_primitives::ethy::{EthyChainId, Witness}; use super::{GossipValidator, MAX_COMPLETE_EVENT_CACHE}; - use crate::{assert_validation_result, testing::Keyring}; + use crate::{assert_validation_result, gossip::topic, testing::Keyring, tests::EthyTestNet}; #[macro_export] /// sc_network_gossip::ValidationResult is missing Eq impl @@ -260,7 +307,9 @@ mod tests { let alice = &validators[0]; let mut context = NoopContext {}; let sender_peer_id = PeerId::random(); - let gv = GossipValidator::::new(vec![]); + let mut net = EthyTestNet::new(1, 0); + let backend = net.peer(0).client().as_backend(); + let gv = GossipValidator::new(vec![], backend); let event_id = 5; let message = b"hello world"; @@ -273,6 +322,7 @@ mod tests { // fn sign(&self, message: &[u8]) -> Signature { // self.sign_prehashed(&blake2_256(message)) signature: alice.sign(message), + block_number: 0, } .encode(); @@ -296,8 +346,10 @@ mod tests { let validators = mock_signers(); let alice = &validators[0]; let bob = &validators[1]; + let mut net = EthyTestNet::new(1, 0); + let backend = net.peer(0).client().as_backend(); let gv = - GossipValidator::::new(validators.iter().map(|x| x.public().clone()).collect()); + GossipValidator::new(validators.iter().map(|x| x.public().clone()).collect(), backend); let event_id = 5; let message = b"hello world"; @@ -308,6 +360,7 @@ mod tests { validator_set_id: 123, authority_id: alice.public(), signature: bob.sign(message), // signed by bob + block_number: 0, } .encode(); @@ -319,7 +372,9 @@ mod tests { #[test] fn keeps_most_recent_events() { - let gv = GossipValidator::::new(vec![]); + let mut net = EthyTestNet::new(1, 0); + let backend = net.peer(0).client().as_backend(); + let gv = GossipValidator::new(vec![], backend); for event_id in 1..=MAX_COMPLETE_EVENT_CACHE { gv.mark_complete(event_id as u64); } @@ -330,4 +385,164 @@ mod tests { assert_eq!(gv.complete_events.read().len(), MAX_COMPLETE_EVENT_CACHE); } + + #[test] + fn witness_validate_events_outside_live_window_discarded() { + let validators = mock_signers(); + let alice = &validators[0]; + let mut context = NoopContext {}; + let sender_peer_id = PeerId::random(); + let mut net = EthyTestNet::new(1, 0); + let backend = net.peer(0).client().as_backend(); + let gv = GossipValidator::new(vec![], backend); + + let event_id = 5; + let message = b"hello world"; + let mut witness = Witness { + digest: sp_core::keccak_256(message), + chain_id: EthyChainId::Ethereum, + event_id, + validator_set_id: 123, + authority_id: alice.public(), + signature: alice.sign(message), + block_number: 0, + }; + + // set validtors + gv.set_active_validators(validators.into_iter().map(|x| x.public()).collect()); + + // finalized number is 0 atm. validate now, should pass + let result = gv.validate(&mut context, &sender_peer_id, witness.clone().encode().as_ref()); + assert_validation_result!(ValidationResult::ProcessAndKeep(_), result); + assert!(gv.is_tracking_event(&event_id)); + + // set the finalized block number to 6. try to validate now. should fail since out of live + // window. i.e. WINDOW_SIZE = 5 + net.peer(0).push_blocks(6, false); + net.block_until_sync(); + assert_eq!(net.peer(0).client().justifications(&BlockId::Number(10)).unwrap(), None); + let just = (*b"FRNK", Vec::new()); + net.peer(0) + .client() + .finalize_block(BlockId::Number(6), Some(just.clone()), true) + .unwrap(); + assert_eq!( + net.peer(0).client().info().finalized_number, + 6, + "Peer #{} finalized block number is not 6", + 0 + ); + + // modify the event_id to avoid duplicate check + witness.event_id += 1; + // now validate, should fail since out of live window. + let result = gv.validate(&mut context, &sender_peer_id, witness.clone().encode().as_ref()); + assert_validation_result!(ValidationResult::Discard, result); + } + + #[test] + fn witness_expired_events_outside_live_window_discarded() { + let validators = mock_signers(); + let alice = &validators[0]; + let mut net = EthyTestNet::new(1, 0); + let backend = net.peer(0).client().as_backend(); + let gv = GossipValidator::new(vec![], backend); + + let event_id = 5; + let message = b"hello world"; + let mut witness = Witness { + digest: sp_core::keccak_256(message), + chain_id: EthyChainId::Ethereum, + event_id, + validator_set_id: 123, + authority_id: alice.public(), + signature: alice.sign(message), + block_number: 0, + }; + + // finalized number is 0 atm. check now, should give false + let result = gv.message_expired()(topic::(), witness.clone().encode().as_ref()); + assert_eq!(result, false); + + // set the finalized block number to 6. try to validate now. should fail since out of live + // window. i.e. WINDOW_SIZE = 5 + net.peer(0).push_blocks(6, false); + net.block_until_sync(); + assert_eq!(net.peer(0).client().justifications(&BlockId::Number(10)).unwrap(), None); + let just = (*b"FRNK", Vec::new()); + net.peer(0) + .client() + .finalize_block(BlockId::Number(6), Some(just.clone()), true) + .unwrap(); + assert_eq!( + net.peer(0).client().info().finalized_number, + 6, + "Peer #{} finalized block number is not 6", + 0 + ); + + // modify the event_id to avoid duplicate check + witness.event_id += 1; + // check now, should give true since out of live window. + let result = gv.message_expired()(topic::(), witness.clone().encode().as_ref()); + assert_eq!(result, true); + } + + #[test] + fn witness_allowed_events_outside_live_window_discarded() { + let validators = mock_signers(); + let alice = &validators[0]; + let mut net = EthyTestNet::new(1, 0); + let backend = net.peer(0).client().as_backend(); + let gv = GossipValidator::new(vec![], backend); + + let event_id = 5; + let message = b"hello world"; + let mut witness = Witness { + digest: sp_core::keccak_256(message), + chain_id: EthyChainId::Ethereum, + event_id, + validator_set_id: 123, + authority_id: alice.public(), + signature: alice.sign(message), + block_number: 0, + }; + + // finalized number is 0 atm. check now, should give true + let result = gv.message_allowed()( + &PeerId::random(), + MessageIntent::Broadcast, + &topic::(), + witness.clone().encode().as_ref(), + ); + assert_eq!(result, true); + + // set the finalized block number to 6. try to validate now. should fail since out of live + // window. i.e. WINDOW_SIZE = 5 + net.peer(0).push_blocks(6, false); + net.block_until_sync(); + assert_eq!(net.peer(0).client().justifications(&BlockId::Number(10)).unwrap(), None); + let just = (*b"FRNK", Vec::new()); + net.peer(0) + .client() + .finalize_block(BlockId::Number(6), Some(just.clone()), true) + .unwrap(); + assert_eq!( + net.peer(0).client().info().finalized_number, + 6, + "Peer #{} finalized block number is not 6", + 0 + ); + + // modify the event_id to avoid duplicate check + witness.event_id += 1; + // check now, should give false since out of live window. + let result = gv.message_allowed()( + &PeerId::random(), + MessageIntent::Broadcast, + &topic::(), + witness.clone().encode().as_ref(), + ); + assert_eq!(result, false); + } } diff --git a/ethy-gadget/src/lib.rs b/ethy-gadget/src/lib.rs index ba888cfbb..cdd698367 100644 --- a/ethy-gadget/src/lib.rs +++ b/ethy-gadget/src/lib.rs @@ -33,7 +33,7 @@ use sc_client_api::{Backend, BlockchainEvents, Finalizer}; use sc_network::ProtocolName; use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; use seed_primitives::ethy::EthyApi; -use sp_api::ProvideRuntimeApi; +use sp_api::{BlockT, HeaderT, ProvideRuntimeApi}; use sp_blockchain::HeaderBackend; use sp_consensus::SyncOracle; use sp_keystore::SyncCryptoStorePtr; @@ -149,11 +149,12 @@ where pub async fn start_ethy_gadget(ethy_params: EthyParams) where B: Block, - BE: Backend, + BE: Backend + 'static, C: Client, R: ProvideRuntimeApi, R::Api: EthyApi, N: GossipNetwork + Clone + SyncOracle + Sync + Send + 'static, + <::Header as HeaderT>::Number: Into, { let EthyParams { client, @@ -168,7 +169,8 @@ where } = ethy_params; let sync_oracle = network.clone(); - let gossip_validator = Arc::new(gossip::GossipValidator::new(Default::default())); + let gossip_validator = + Arc::new(gossip::GossipValidator::new(Default::default(), backend.clone())); let gossip_engine = GossipEngine::new(network, protocol_name, gossip_validator.clone(), None); let metrics = diff --git a/ethy-gadget/src/witness_record.rs b/ethy-gadget/src/witness_record.rs index 403a63440..c7788562f 100644 --- a/ethy-gadget/src/witness_record.rs +++ b/ethy-gadget/src/witness_record.rs @@ -129,7 +129,7 @@ impl WitnessRecord { } .unwrap_or(0_usize); - trace!(target: "ethy", "💎 event {:?}, has # support: {:?}", event_id, witness_count); + trace!(target: "ethy", "💎 event {:?}, has # support: {:?}, proof threshold: {:?}", event_id, witness_count, proof_threshold); witness_count >= proof_threshold } @@ -291,7 +291,7 @@ fn compact_sequence(completed_events: &mut [EventProofId]) -> &[EventProofId] { watermark_idx = i + 1; continue } else { - break + break // Note - fix the algo } } @@ -337,6 +337,7 @@ pub(crate) mod test { validator_set_id: 5_u64, authority_id: validator.public(), signature: keystore.sign_prehashed(&validator.public(), &digest).unwrap(), + block_number: 0, } } diff --git a/ethy-gadget/src/worker.rs b/ethy-gadget/src/worker.rs index 5b1505474..83f3c4941 100644 --- a/ethy-gadget/src/worker.rs +++ b/ethy-gadget/src/worker.rs @@ -46,6 +46,7 @@ use crate::{ pub(crate) struct WorkerParams where B: Block, + BE: Backend, { pub client: Arc, pub backend: Arc, @@ -53,7 +54,7 @@ where pub key_store: EthyKeystore, pub event_proof_sender: notification::EthyEventProofSender, pub gossip_engine: GossipEngine, - pub gossip_validator: Arc>, + pub gossip_validator: Arc>, pub metrics: Option, pub sync_oracle: SO, } @@ -73,7 +74,7 @@ where key_store: EthyKeystore, event_proof_sender: notification::EthyEventProofSender, gossip_engine: GossipEngine, - gossip_validator: Arc>, + gossip_validator: Arc>, metrics: Option, /// Tracks on-going witnesses witness_record: WitnessRecord, @@ -224,19 +225,15 @@ where event_id, authority_id: authority_id.clone(), signature, + block_number: (*notification.header.number()).try_into().unwrap_or_default(), }; - let broadcast_witness = witness.encode(); metric_inc!(self, ethy_witness_sent); - debug!(target: "ethy", "💎 Sent witness: {:?}", witness); // process the witness self.witness_record.note_event_metadata(event_id, data, block, chain_id); self.handle_witness(witness.clone()); - - // broadcast the witness - self.gossip_engine.gossip_message(topic::(), broadcast_witness, false); - debug!(target: "ethy", "💎 gossiped witness for event: {:?}", witness.event_id); + debug!(target: "ethy", "💎 Sent witness: {:?}", witness); } } @@ -329,9 +326,14 @@ where return } + // gossip the witness. will gossip even if we don't have event metadata yet. This would + // increase the network activity, but gives more room for validator Witnesses to spread + // across the network. + trace!(target: "ethy", "💎 gossiping witness: {:?}", witness.event_id); self.gossip_engine.gossip_message(topic::(), witness.encode(), false); - // after processing `witness` there may now be enough info to make a proof - self.try_make_proof(witness.event_id); + + // Try to make proof + self.try_make_proof(witness.event_id.clone()); } /// Try to make an event proof @@ -344,12 +346,10 @@ where /// 2) Store proof in DB /// 3) Notify listeners of the new proof fn try_make_proof(&mut self, event_id: EventProofId) { - { - let event_metadata = self.witness_record.event_metadata(event_id); - if event_metadata.is_none() { - debug!(target: "ethy", "💎 missing event metadata: {:?}, can't make proof yet", event_id); - return - } + let event_metadata = self.witness_record.event_metadata(event_id); + if event_metadata.is_none() { + debug!(target: "ethy", "💎 missing event metadata: {:?}, can't make proof yet", event_id); + return } // process any unverified witnesses, received before event metadata was known @@ -558,7 +558,8 @@ pub(crate) mod test { let api = Arc::new(TestApi {}); let network = peer.network_service().clone(); let sync_oracle = network.clone(); - let gossip_validator = Arc::new(crate::gossip::GossipValidator::new(validators)); + let gossip_validator = + Arc::new(crate::gossip::GossipValidator::new(validators, peer.client().as_backend())); let gossip_engine = GossipEngine::new(network, ETHY_PROTOCOL_NAME, gossip_validator.clone(), None); let (sender, _receiver) = NotificationStream::<_, EthyEventProofTracingKey>::channel(); diff --git a/primitives/src/ethy.rs b/primitives/src/ethy.rs index 7a2892b4c..6a2a1dfbc 100644 --- a/primitives/src/ethy.rs +++ b/primitives/src/ethy.rs @@ -159,6 +159,8 @@ pub struct Witness { pub authority_id: AuthorityId, /// ECDSA signature over `digest` pub signature: AuthoritySignature, + /// proof requested block number + pub block_number: u64, } /// An Ethy event proof with validator signatures.