Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Commit

Permalink
Fix broadcast metrics (#9461)
Browse files Browse the repository at this point in the history
* Rework broadcast metrics to support multiple threads

* Update dashboards

Co-authored-by: Carl <carl@solana.com>
  • Loading branch information
2 people authored and mvines committed Apr 16, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 420ea2f commit aaec7de
Showing 7 changed files with 1,451 additions and 682 deletions.
37 changes: 20 additions & 17 deletions core/src/broadcast_stage.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! A stage to broadcast data from a leader node to validators
use self::{
broadcast_fake_shreds_run::BroadcastFakeShredsRun,
broadcast_fake_shreds_run::BroadcastFakeShredsRun, broadcast_metrics::*,
fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
standard_broadcast_run::StandardBroadcastRun,
};
@@ -35,13 +35,16 @@ use std::{
};

mod broadcast_fake_shreds_run;
pub(crate) mod broadcast_metrics;
pub(crate) mod broadcast_utils;
mod fail_entry_verification_broadcast_run;
mod standard_broadcast_run;

pub const NUM_INSERT_THREADS: usize = 2;
pub type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
pub type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
pub(crate) const NUM_INSERT_THREADS: usize = 2;
pub(crate) type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
pub(crate) type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
pub(crate) type RecordReceiver = Receiver<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>;
pub(crate) type TransmitReceiver = Receiver<(TransmitShreds, Option<BroadcastShredBatchInfo>)>;

#[derive(Debug, PartialEq, Eq, Clone)]
pub enum BroadcastStageReturnType {
@@ -107,18 +110,18 @@ trait BroadcastRun {
&mut self,
blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<TransmitShreds>,
blockstore_sender: &Sender<Arc<Vec<Shred>>>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
) -> Result<()>;
fn transmit(
&mut self,
receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
receiver: &Arc<Mutex<TransmitReceiver>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sock: &UdpSocket,
) -> Result<()>;
fn record(
&self,
receiver: &Arc<Mutex<Receiver<Arc<Vec<Shred>>>>>,
&mut self,
receiver: &Arc<Mutex<RecordReceiver>>,
blockstore: &Arc<Blockstore>,
) -> Result<()>;
}
@@ -150,8 +153,8 @@ impl BroadcastStage {
fn run(
blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<TransmitShreds>,
blockstore_sender: &Sender<Arc<Vec<Shred>>>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
mut broadcast_stage_run: impl BroadcastRun,
) -> BroadcastStageReturnType {
loop {
@@ -250,7 +253,7 @@ impl BroadcastStage {
let blockstore_receiver = Arc::new(Mutex::new(blockstore_receiver));
for _ in 0..NUM_INSERT_THREADS {
let blockstore_receiver = blockstore_receiver.clone();
let bs_record = broadcast_stage_run.clone();
let mut bs_record = broadcast_stage_run.clone();
let btree = blockstore.clone();
let t = Builder::new()
.name("solana-broadcaster-record".to_string())
@@ -289,7 +292,7 @@ impl BroadcastStage {
fn check_retransmit_signals(
blockstore: &Blockstore,
retransmit_slots_receiver: &RetransmitSlotsReceiver,
socket_sender: &Sender<TransmitShreds>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
) -> Result<()> {
let timer = Duration::from_millis(100);

@@ -310,7 +313,7 @@ impl BroadcastStage {
);

if !data_shreds.is_empty() {
socket_sender.send((stakes.clone(), data_shreds))?;
socket_sender.send(((stakes.clone(), data_shreds), None))?;
}

let coding_shreds = Arc::new(
@@ -320,7 +323,7 @@ impl BroadcastStage {
);

if !coding_shreds.is_empty() {
socket_sender.send((stakes.clone(), coding_shreds))?;
socket_sender.send(((stakes.clone(), coding_shreds), None))?;
}
}

@@ -478,13 +481,13 @@ pub mod test {
}

fn check_all_shreds_received(
transmit_receiver: &Receiver<TransmitShreds>,
transmit_receiver: &TransmitReceiver,
mut data_index: u64,
mut coding_index: u64,
num_expected_data_shreds: u64,
num_expected_coding_shreds: u64,
) {
while let Ok(new_retransmit_slots) = transmit_receiver.try_recv() {
while let Ok((new_retransmit_slots, _)) = transmit_receiver.try_recv() {
if new_retransmit_slots.1[0].is_data() {
for data_shred in new_retransmit_slots.1.iter() {
assert_eq!(data_shred.index() as u64, data_index);
30 changes: 18 additions & 12 deletions core/src/broadcast_stage/broadcast_fake_shreds_run.rs
Original file line number Diff line number Diff line change
@@ -28,8 +28,8 @@ impl BroadcastRun for BroadcastFakeShredsRun {
&mut self,
blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<TransmitShreds>,
blockstore_sender: &Sender<Arc<Vec<Shred>>>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
) -> Result<()> {
// 1) Pull entries from banking stage
let receive_results = broadcast_utils::recv_slot_entries(receiver)?;
@@ -83,25 +83,31 @@ impl BroadcastRun for BroadcastFakeShredsRun {
}

let data_shreds = Arc::new(data_shreds);
blockstore_sender.send(data_shreds.clone())?;
blockstore_sender.send((data_shreds.clone(), None))?;

// 3) Start broadcast step
//some indicates fake shreds
socket_sender.send((Some(Arc::new(HashMap::new())), Arc::new(fake_data_shreds)))?;
socket_sender.send((Some(Arc::new(HashMap::new())), Arc::new(fake_coding_shreds)))?;
socket_sender.send((
(Some(Arc::new(HashMap::new())), Arc::new(fake_data_shreds)),
None,
))?;
socket_sender.send((
(Some(Arc::new(HashMap::new())), Arc::new(fake_coding_shreds)),
None,
))?;
//none indicates real shreds
socket_sender.send((None, data_shreds))?;
socket_sender.send((None, Arc::new(coding_shreds)))?;
socket_sender.send(((None, data_shreds), None))?;
socket_sender.send(((None, Arc::new(coding_shreds)), None))?;

Ok(())
}
fn transmit(
&mut self,
receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
receiver: &Arc<Mutex<TransmitReceiver>>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
sock: &UdpSocket,
) -> Result<()> {
for (stakes, data_shreds) in receiver.lock().unwrap().iter() {
for ((stakes, data_shreds), _) in receiver.lock().unwrap().iter() {
let peers = cluster_info.read().unwrap().tvu_peers();
peers.iter().enumerate().for_each(|(i, peer)| {
if i <= self.partition && stakes.is_some() {
@@ -119,11 +125,11 @@ impl BroadcastRun for BroadcastFakeShredsRun {
Ok(())
}
fn record(
&self,
receiver: &Arc<Mutex<Receiver<Arc<Vec<Shred>>>>>,
&mut self,
receiver: &Arc<Mutex<RecordReceiver>>,
blockstore: &Arc<Blockstore>,
) -> Result<()> {
for data_shreds in receiver.lock().unwrap().iter() {
for (data_shreds, _) in receiver.lock().unwrap().iter() {
blockstore.insert_shreds(data_shreds.to_vec(), None, true)?;
}
Ok(())
Loading

0 comments on commit aaec7de

Please sign in to comment.