From b34ef05e5aa28ecc7bb46dabd19c3cba0a16a4c6 Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Tue, 24 Oct 2023 20:15:54 +0000 Subject: [PATCH 1/4] blockstore: add merkle root to ErasureMeta column family --- gossip/src/duplicate_shred.rs | 4 +- ledger/src/blockstore.rs | 72 +++++++++++++++++++++++++++++------ ledger/src/blockstore_db.rs | 63 ++++++++++++++++++++++++++++-- ledger/src/blockstore_meta.rs | 64 +++++++++++++++++++++++++------ ledger/src/lib.rs | 1 + 5 files changed, 176 insertions(+), 28 deletions(-) diff --git a/gossip/src/duplicate_shred.rs b/gossip/src/duplicate_shred.rs index b1ceab79b26949..176c93b2814f7b 100644 --- a/gossip/src/duplicate_shred.rs +++ b/gossip/src/duplicate_shred.rs @@ -3,7 +3,7 @@ use { itertools::Itertools, solana_ledger::{ blockstore::BlockstoreError, - blockstore_meta::{DuplicateSlotProof, ErasureMeta}, + blockstore_meta::{DuplicateSlotProof, MerkleErasureMeta}, shred::{self, Shred, ShredType}, }, solana_sdk::{ @@ -141,7 +141,7 @@ where // a part of the same fec set. Further work to enhance detection is planned in // https://github.com/solana-labs/solana/issues/33037 if shred1.fec_set_index() == shred2.fec_set_index() - && !ErasureMeta::check_erasure_consistency(shred1, shred2) + && !MerkleErasureMeta::check_erasure_consistency(shred1, shred2) { return Ok(()); } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index ce9336e1132192..26ab0a8edf8e78 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -457,8 +457,8 @@ impl Blockstore { false } - fn erasure_meta(&self, erasure_set: ErasureSetId) -> Result> { - self.erasure_meta_cf.get(erasure_set.store_key()) + fn erasure_meta(&self, erasure_set: ErasureSetId) -> Result> { + self.erasure_meta_cf.get_new_or_old(erasure_set.store_key()) } /// Check whether the specified slot is an orphan slot which does not @@ -594,7 +594,7 @@ impl Blockstore { fn get_recovery_data_shreds<'a>( index: &'a Index, slot: Slot, - erasure_meta: &'a ErasureMeta, + erasure_meta: &'a MerkleErasureMeta, prev_inserted_shreds: &'a HashMap, data_cf: &'a LedgerColumn, ) -> impl Iterator + 'a { @@ -619,7 +619,7 @@ impl Blockstore { fn get_recovery_coding_shreds<'a>( index: &'a Index, slot: Slot, - erasure_meta: &'a ErasureMeta, + erasure_meta: &'a MerkleErasureMeta, prev_inserted_shreds: &'a HashMap, code_cf: &'a LedgerColumn, ) -> impl Iterator + 'a { @@ -643,7 +643,7 @@ impl Blockstore { fn recover_shreds( index: &Index, - erasure_meta: &ErasureMeta, + erasure_meta: &MerkleErasureMeta, prev_inserted_shreds: &HashMap, recovered_shreds: &mut Vec, data_cf: &LedgerColumn, @@ -677,7 +677,7 @@ impl Blockstore { fn submit_metrics( slot: Slot, - erasure_meta: &ErasureMeta, + erasure_meta: &MerkleErasureMeta, attempted: bool, status: String, recovered: usize, @@ -725,7 +725,7 @@ impl Blockstore { fn try_shred_recovery( &self, - erasure_metas: &HashMap, + erasure_metas: &HashMap, index_working_set: &mut HashMap, prev_inserted_shreds: &HashMap, reed_solomon_cache: &ReedSolomonCache, @@ -1164,7 +1164,7 @@ impl Blockstore { fn check_insert_coding_shred( &self, shred: Shred, - erasure_metas: &mut HashMap, + erasure_metas: &mut HashMap, index_working_set: &mut HashMap, write_batch: &mut WriteBatch, just_received_shreds: &mut HashMap, @@ -1202,7 +1202,7 @@ impl Blockstore { let erasure_meta = erasure_metas.entry(erasure_set).or_insert_with(|| { self.erasure_meta(erasure_set) .expect("Expect database get to succeed") - .unwrap_or_else(|| ErasureMeta::from_coding_shred(&shred).unwrap()) + .unwrap_or_else(|| MerkleErasureMeta::from_coding_shred(&shred).unwrap()) }); if !erasure_meta.check_coding_shred(&shred) { @@ -1275,7 +1275,7 @@ impl Blockstore { &self, shred: &Shred, slot: Slot, - erasure_meta: &ErasureMeta, + erasure_meta: &MerkleErasureMeta, just_received_shreds: &HashMap, ) -> Option> { // Search for the shred which set the initial erasure config, either inserted, @@ -1340,7 +1340,7 @@ impl Blockstore { fn check_insert_data_shred( &self, shred: Shred, - erasure_metas: &mut HashMap, + erasure_metas: &mut HashMap, index_working_set: &mut HashMap, slot_meta_working_set: &mut HashMap, write_batch: &mut WriteBatch, @@ -7504,6 +7504,56 @@ pub mod tests { assert_eq!(meta.fee, index1_slot * 1000); } + #[test] + #[allow(deprecated)] + fn test_erasure_meta_migration() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let erasure_meta_cf = &blockstore.erasure_meta_cf; + + let config = ErasureConfig { + num_data: 1, + num_coding: 17, + }; + let erasure_meta_old = ErasureMeta { + set_index: 5, + first_coding_index: 8, + config, + __unused_size: 0, + }; + + erasure_meta_cf + .put_old_type((100, 5), &erasure_meta_old) + .unwrap(); + + let erasure_meta = erasure_meta_cf.get_new_or_old((100, 5)).unwrap().unwrap(); + assert_eq!(erasure_meta.set_index(), erasure_meta_old.set_index); + assert_eq!( + erasure_meta.first_coding_index(), + erasure_meta_old.first_coding_index + ); + assert_eq!(erasure_meta.config(), erasure_meta_old.config); + assert_eq!(erasure_meta.merkle_root(), Hash::default()); + + let erasure_meta_new = ErasureMeta { + set_index: 3, + first_coding_index: 2, + config, + __unused_size: 0, + } + .into(); + erasure_meta_cf.put((101, 3), &erasure_meta_new).unwrap(); + + let erasure_meta = erasure_meta_cf.get_new_or_old((101, 3)).unwrap().unwrap(); + assert_eq!(erasure_meta.set_index(), erasure_meta_new.set_index()); + assert_eq!( + erasure_meta.first_coding_index(), + erasure_meta_new.first_coding_index() + ); + assert_eq!(erasure_meta.config(), erasure_meta_new.config()); + assert_eq!(erasure_meta.merkle_root(), Hash::default()); + } + #[test] fn test_get_transaction_status() { let ledger_path = get_tmp_ledger_path_auto_delete!(); diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index b65df82ee00c9e..0ececb8f3eb401 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -214,13 +214,13 @@ pub mod columns { /// /// This column family stores ErasureMeta which includes metadata about /// dropped network packets (or erasures) that can be used to recover - /// missing data shreds. + /// missing data shreds. For merkle shreds, it also stores the merkle root. /// /// Its index type is `crate::shred::ErasureSetId`, which consists of a Slot ID /// and a FEC (Forward Error Correction) set index. /// /// * index type: `crate::shred::ErasureSetId` `(Slot, fec_set_index: u64)` - /// * value type: [`blockstore_meta::ErasureMeta`] + /// * value type: [`blockstore_meta::MerkleErasureMeta`] pub struct ErasureMeta; #[derive(Debug)] @@ -802,6 +802,12 @@ pub trait ColumnIndexDeprecation: Column { } } +/// Helper trait to transition a column between bincode formats +pub trait BincodeTypeTransition: Column + TypedColumn { + /// This should have a different serialized size than the TypedColumn::Type + type OldType: Serialize + DeserializeOwned + Into; +} + impl Column for columns::TransactionStatus { type Index = (Signature, Slot); @@ -1216,7 +1222,11 @@ impl ColumnName for columns::ErasureMeta { const NAME: &'static str = ERASURE_META_CF; } impl TypedColumn for columns::ErasureMeta { - type Type = blockstore_meta::ErasureMeta; + type Type = blockstore_meta::MerkleErasureMeta; +} +impl BincodeTypeTransition for columns::ErasureMeta { + #[allow(deprecated)] + type OldType = blockstore_meta::ErasureMeta; } impl SlotColumn for columns::OptimisticSlots {} @@ -1796,6 +1806,42 @@ where } } +impl LedgerColumn +where + C: BincodeTypeTransition + ColumnName, +{ + /// Read the column in either the `C::Type` or `C::OldType` format. + pub fn get_new_or_old(&self, key: C::Index) -> Result> { + self.get_new_or_old_raw(&C::key(key)) + } + + pub fn get_new_or_old_raw(&self, key: &[u8]) -> Result> { + let mut result = Ok(None); + let is_perf_enabled = maybe_enable_rocksdb_perf( + self.column_options.rocks_perf_sample_interval, + &self.read_perf_status, + ); + if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), key)? { + let value = if let Ok(value) = deserialize::(pinnable_slice.as_ref()) { + value + } else { + deserialize::(pinnable_slice.as_ref())?.into() + }; + result = Ok(Some(value)) + } + + if let Some(op_start_instant) = is_perf_enabled { + report_rocksdb_read_perf( + C::NAME, + PERF_METRIC_OP_NAME_GET, + &op_start_instant.elapsed(), + &self.column_options, + ); + } + result + } +} + impl<'a> WriteBatch<'a> { pub fn put_bytes(&mut self, key: C::Index, bytes: &[u8]) -> Result<()> { self.write_batch @@ -2236,6 +2282,17 @@ pub mod tests { } } + impl LedgerColumn + where + C: BincodeTypeTransition + ColumnName, + { + pub fn put_old_type(&self, key: C::Index, value: &C::OldType) -> Result<()> { + let serialized_value = serialize(value)?; + self.backend + .put_cf(self.handle(), &C::key(key), &serialized_value) + } + } + impl LedgerColumn where C: ColumnIndexDeprecation + ColumnName, diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 79954ee96b6d04..44d1140bebfd8a 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -118,24 +118,38 @@ pub struct ShredIndex { index: BTreeSet, } +#[deprecated = "Use MerkleErasureMeta"] #[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] /// Erasure coding information pub struct ErasureMeta { /// Which erasure set in the slot this is - set_index: u64, + pub(crate) set_index: u64, /// First coding index in the FEC set - first_coding_index: u64, + pub(crate) first_coding_index: u64, /// Size of shards in this erasure set #[serde(rename = "size")] - __unused_size: usize, + pub(crate) __unused_size: usize, + /// Erasure configuration for this erasure set + pub(crate) config: ErasureConfig, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] +/// Erasure coding information for merkle shreds +pub struct MerkleErasureMeta { + /// Which erasure set in the slot this is + set_index: u64, + /// First coding index in the FEC set + first_coding_index: u64, /// Erasure configuration for this erasure set config: ErasureConfig, + /// Merkle root for this FEC set + merkle_root: Hash, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct ErasureConfig { - num_data: usize, - num_coding: usize, + pub(crate) num_data: usize, + pub(crate) num_coding: usize, } #[derive(Deserialize, Serialize)] @@ -321,7 +335,7 @@ impl SlotMeta { } } -impl ErasureMeta { +impl MerkleErasureMeta { pub(crate) fn from_coding_shred(shred: &Shred) -> Option { match shred.shred_type() { ShredType::Data => None, @@ -331,11 +345,12 @@ impl ErasureMeta { num_coding: usize::from(shred.num_coding_shreds().ok()?), }; let first_coding_index = u64::from(shred.first_coding_index()?); - let erasure_meta = ErasureMeta { + let merkle_root = Hash::default(); + let erasure_meta = MerkleErasureMeta { set_index: u64::from(shred.fec_set_index()), config, first_coding_index, - __unused_size: 0, + merkle_root, }; Some(erasure_meta) } @@ -345,10 +360,9 @@ impl ErasureMeta { // Returns true if the erasure fields on the shred // are consistent with the erasure-meta. pub(crate) fn check_coding_shred(&self, shred: &Shred) -> bool { - let Some(mut other) = Self::from_coding_shred(shred) else { + let Some(other) = Self::from_coding_shred(shred) else { return false; }; - other.__unused_size = self.__unused_size; self == &other } @@ -394,6 +408,32 @@ impl ErasureMeta { StillNeed(num_needed) } } + + #[cfg(test)] + pub(crate) fn set_index(&self) -> u64 { + self.set_index + } + + #[cfg(test)] + pub(crate) fn first_coding_index(&self) -> u64 { + self.first_coding_index + } + + #[cfg(test)] + pub(crate) fn merkle_root(&self) -> Hash { + self.merkle_root + } +} + +impl From for MerkleErasureMeta { + fn from(erasure_meta: ErasureMeta) -> MerkleErasureMeta { + MerkleErasureMeta { + set_index: erasure_meta.set_index, + first_coding_index: erasure_meta.first_coding_index, + config: erasure_meta.config, + merkle_root: Hash::default(), + } + } } impl DuplicateSlotProof { @@ -511,11 +551,11 @@ mod test { num_data: 8, num_coding: 16, }; - let e_meta = ErasureMeta { + let e_meta = MerkleErasureMeta { set_index, first_coding_index: set_index, config: erasure_config, - __unused_size: 0, + merkle_root: Hash::default(), }; let mut rng = thread_rng(); let mut index = Index::new(0); diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index 0f311ca1216ec4..994a21f5f9360b 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -10,6 +10,7 @@ pub mod block_error; pub mod blockstore; pub mod ancestor_iterator; pub mod blockstore_db; +#[allow(deprecated)] pub mod blockstore_meta; pub mod blockstore_metrics; pub mod blockstore_options; From 2e357435087d26629af69b0388dbd04c732947a6 Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Wed, 25 Oct 2023 18:17:32 +0000 Subject: [PATCH 2/4] pr feedback: use ErasureMetaLegacy naming scheme --- gossip/src/duplicate_shred.rs | 4 ++-- ledger/src/blockstore.rs | 24 ++++++++++++------------ ledger/src/blockstore_db.rs | 6 +++--- ledger/src/blockstore_meta.rs | 18 +++++++++--------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/gossip/src/duplicate_shred.rs b/gossip/src/duplicate_shred.rs index 176c93b2814f7b..b1ceab79b26949 100644 --- a/gossip/src/duplicate_shred.rs +++ b/gossip/src/duplicate_shred.rs @@ -3,7 +3,7 @@ use { itertools::Itertools, solana_ledger::{ blockstore::BlockstoreError, - blockstore_meta::{DuplicateSlotProof, MerkleErasureMeta}, + blockstore_meta::{DuplicateSlotProof, ErasureMeta}, shred::{self, Shred, ShredType}, }, solana_sdk::{ @@ -141,7 +141,7 @@ where // a part of the same fec set. Further work to enhance detection is planned in // https://github.com/solana-labs/solana/issues/33037 if shred1.fec_set_index() == shred2.fec_set_index() - && !MerkleErasureMeta::check_erasure_consistency(shred1, shred2) + && !ErasureMeta::check_erasure_consistency(shred1, shred2) { return Ok(()); } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 26ab0a8edf8e78..3cfef5899f08bd 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -457,7 +457,7 @@ impl Blockstore { false } - fn erasure_meta(&self, erasure_set: ErasureSetId) -> Result> { + fn erasure_meta(&self, erasure_set: ErasureSetId) -> Result> { self.erasure_meta_cf.get_new_or_old(erasure_set.store_key()) } @@ -594,7 +594,7 @@ impl Blockstore { fn get_recovery_data_shreds<'a>( index: &'a Index, slot: Slot, - erasure_meta: &'a MerkleErasureMeta, + erasure_meta: &'a ErasureMeta, prev_inserted_shreds: &'a HashMap, data_cf: &'a LedgerColumn, ) -> impl Iterator + 'a { @@ -619,7 +619,7 @@ impl Blockstore { fn get_recovery_coding_shreds<'a>( index: &'a Index, slot: Slot, - erasure_meta: &'a MerkleErasureMeta, + erasure_meta: &'a ErasureMeta, prev_inserted_shreds: &'a HashMap, code_cf: &'a LedgerColumn, ) -> impl Iterator + 'a { @@ -643,7 +643,7 @@ impl Blockstore { fn recover_shreds( index: &Index, - erasure_meta: &MerkleErasureMeta, + erasure_meta: &ErasureMeta, prev_inserted_shreds: &HashMap, recovered_shreds: &mut Vec, data_cf: &LedgerColumn, @@ -677,7 +677,7 @@ impl Blockstore { fn submit_metrics( slot: Slot, - erasure_meta: &MerkleErasureMeta, + erasure_meta: &ErasureMeta, attempted: bool, status: String, recovered: usize, @@ -725,7 +725,7 @@ impl Blockstore { fn try_shred_recovery( &self, - erasure_metas: &HashMap, + erasure_metas: &HashMap, index_working_set: &mut HashMap, prev_inserted_shreds: &HashMap, reed_solomon_cache: &ReedSolomonCache, @@ -1164,7 +1164,7 @@ impl Blockstore { fn check_insert_coding_shred( &self, shred: Shred, - erasure_metas: &mut HashMap, + erasure_metas: &mut HashMap, index_working_set: &mut HashMap, write_batch: &mut WriteBatch, just_received_shreds: &mut HashMap, @@ -1202,7 +1202,7 @@ impl Blockstore { let erasure_meta = erasure_metas.entry(erasure_set).or_insert_with(|| { self.erasure_meta(erasure_set) .expect("Expect database get to succeed") - .unwrap_or_else(|| MerkleErasureMeta::from_coding_shred(&shred).unwrap()) + .unwrap_or_else(|| ErasureMeta::from_coding_shred(&shred).unwrap()) }); if !erasure_meta.check_coding_shred(&shred) { @@ -1275,7 +1275,7 @@ impl Blockstore { &self, shred: &Shred, slot: Slot, - erasure_meta: &MerkleErasureMeta, + erasure_meta: &ErasureMeta, just_received_shreds: &HashMap, ) -> Option> { // Search for the shred which set the initial erasure config, either inserted, @@ -1340,7 +1340,7 @@ impl Blockstore { fn check_insert_data_shred( &self, shred: Shred, - erasure_metas: &mut HashMap, + erasure_metas: &mut HashMap, index_working_set: &mut HashMap, slot_meta_working_set: &mut HashMap, write_batch: &mut WriteBatch, @@ -7515,7 +7515,7 @@ pub mod tests { num_data: 1, num_coding: 17, }; - let erasure_meta_old = ErasureMeta { + let erasure_meta_old = ErasureMetaLegacy { set_index: 5, first_coding_index: 8, config, @@ -7535,7 +7535,7 @@ pub mod tests { assert_eq!(erasure_meta.config(), erasure_meta_old.config); assert_eq!(erasure_meta.merkle_root(), Hash::default()); - let erasure_meta_new = ErasureMeta { + let erasure_meta_new = ErasureMetaLegacy { set_index: 3, first_coding_index: 2, config, diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 0ececb8f3eb401..cf4ce8e5fa9906 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -220,7 +220,7 @@ pub mod columns { /// and a FEC (Forward Error Correction) set index. /// /// * index type: `crate::shred::ErasureSetId` `(Slot, fec_set_index: u64)` - /// * value type: [`blockstore_meta::MerkleErasureMeta`] + /// * value type: [`blockstore_meta::ErasureMeta`] pub struct ErasureMeta; #[derive(Debug)] @@ -1222,11 +1222,11 @@ impl ColumnName for columns::ErasureMeta { const NAME: &'static str = ERASURE_META_CF; } impl TypedColumn for columns::ErasureMeta { - type Type = blockstore_meta::MerkleErasureMeta; + type Type = blockstore_meta::ErasureMeta; } impl BincodeTypeTransition for columns::ErasureMeta { #[allow(deprecated)] - type OldType = blockstore_meta::ErasureMeta; + type OldType = blockstore_meta::ErasureMetaLegacy; } impl SlotColumn for columns::OptimisticSlots {} diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 44d1140bebfd8a..4822814636f75e 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -118,10 +118,10 @@ pub struct ShredIndex { index: BTreeSet, } -#[deprecated = "Use MerkleErasureMeta"] +#[deprecated = "Use ErasureMeta"] #[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] /// Erasure coding information -pub struct ErasureMeta { +pub struct ErasureMetaLegacy { /// Which erasure set in the slot this is pub(crate) set_index: u64, /// First coding index in the FEC set @@ -135,7 +135,7 @@ pub struct ErasureMeta { #[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] /// Erasure coding information for merkle shreds -pub struct MerkleErasureMeta { +pub struct ErasureMeta { /// Which erasure set in the slot this is set_index: u64, /// First coding index in the FEC set @@ -335,7 +335,7 @@ impl SlotMeta { } } -impl MerkleErasureMeta { +impl ErasureMeta { pub(crate) fn from_coding_shred(shred: &Shred) -> Option { match shred.shred_type() { ShredType::Data => None, @@ -346,7 +346,7 @@ impl MerkleErasureMeta { }; let first_coding_index = u64::from(shred.first_coding_index()?); let merkle_root = Hash::default(); - let erasure_meta = MerkleErasureMeta { + let erasure_meta = ErasureMeta { set_index: u64::from(shred.fec_set_index()), config, first_coding_index, @@ -425,9 +425,9 @@ impl MerkleErasureMeta { } } -impl From for MerkleErasureMeta { - fn from(erasure_meta: ErasureMeta) -> MerkleErasureMeta { - MerkleErasureMeta { +impl From for ErasureMeta { + fn from(erasure_meta: ErasureMetaLegacy) -> ErasureMeta { + ErasureMeta { set_index: erasure_meta.set_index, first_coding_index: erasure_meta.first_coding_index, config: erasure_meta.config, @@ -551,7 +551,7 @@ mod test { num_data: 8, num_coding: 16, }; - let e_meta = MerkleErasureMeta { + let e_meta = ErasureMeta { set_index, first_coding_index: set_index, config: erasure_config, From 3196f268bda9472b22b3b6006ce21ff83a4a9171 Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Wed, 25 Oct 2023 18:21:51 +0000 Subject: [PATCH 3/4] pr feedback: store received coding index in erasure meta for duplicate proof gen --- ledger/src/blockstore.rs | 8 ++++++++ ledger/src/blockstore_meta.rs | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 3cfef5899f08bd..28d77c0fd72b58 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7533,6 +7533,10 @@ pub mod tests { erasure_meta_old.first_coding_index ); assert_eq!(erasure_meta.config(), erasure_meta_old.config); + assert_eq!( + erasure_meta.first_received_coding_index(), + erasure_meta_old.first_coding_index + ); assert_eq!(erasure_meta.merkle_root(), Hash::default()); let erasure_meta_new = ErasureMetaLegacy { @@ -7551,6 +7555,10 @@ pub mod tests { erasure_meta_new.first_coding_index() ); assert_eq!(erasure_meta.config(), erasure_meta_new.config()); + assert_eq!( + erasure_meta.first_received_coding_index(), + erasure_meta_new.first_coding_index() + ); assert_eq!(erasure_meta.merkle_root(), Hash::default()); } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 4822814636f75e..b3848519d9b4ef 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -140,6 +140,8 @@ pub struct ErasureMeta { set_index: u64, /// First coding index in the FEC set first_coding_index: u64, + /// First coding shred received, from which we populated this ErasureMeta + first_received_coding_index: u64, /// Erasure configuration for this erasure set config: ErasureConfig, /// Merkle root for this FEC set @@ -345,11 +347,13 @@ impl ErasureMeta { num_coding: usize::from(shred.num_coding_shreds().ok()?), }; let first_coding_index = u64::from(shred.first_coding_index()?); + let first_received_coding_index = u64::from(shred.index()); let merkle_root = Hash::default(); let erasure_meta = ErasureMeta { set_index: u64::from(shred.fec_set_index()), config, first_coding_index, + first_received_coding_index, merkle_root, }; Some(erasure_meta) @@ -360,9 +364,11 @@ impl ErasureMeta { // Returns true if the erasure fields on the shred // are consistent with the erasure-meta. pub(crate) fn check_coding_shred(&self, shred: &Shred) -> bool { - let Some(other) = Self::from_coding_shred(shred) else { + let Some(mut other) = Self::from_coding_shred(shred) else { return false; }; + // The order of received shreds should not impact consistency + other.first_received_coding_index = self.first_received_coding_index; self == &other } @@ -419,6 +425,11 @@ impl ErasureMeta { self.first_coding_index } + #[cfg(test)] + pub(crate) fn first_received_coding_index(&self) -> u64 { + self.first_received_coding_index + } + #[cfg(test)] pub(crate) fn merkle_root(&self) -> Hash { self.merkle_root @@ -430,6 +441,10 @@ impl From for ErasureMeta { ErasureMeta { set_index: erasure_meta.set_index, first_coding_index: erasure_meta.first_coding_index, + // We only use this in the context of merkle_root duplicate shred + // proofs, so it's fine to not have the correct value here while + // merkle_root is Hash::default() + first_received_coding_index: erasure_meta.first_coding_index, config: erasure_meta.config, merkle_root: Hash::default(), } @@ -554,6 +569,7 @@ mod test { let e_meta = ErasureMeta { set_index, first_coding_index: set_index, + first_received_coding_index: 0, config: erasure_config, merkle_root: Hash::default(), }; From 9d9252a63856c4604a820cf6ce606fab25dbbd8a Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Wed, 25 Oct 2023 18:26:00 +0000 Subject: [PATCH 4/4] pr feedback: add test impl for ErasureConfig --- ledger/src/blockstore.rs | 5 +---- ledger/src/blockstore_meta.rs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 28d77c0fd72b58..0d185dd499128f 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7511,10 +7511,7 @@ pub mod tests { let blockstore = Blockstore::open(ledger_path.path()).unwrap(); let erasure_meta_cf = &blockstore.erasure_meta_cf; - let config = ErasureConfig { - num_data: 1, - num_coding: 17, - }; + let config = ErasureConfig::new(1, 17); let erasure_meta_old = ErasureMetaLegacy { set_index: 5, first_coding_index: 8, diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index b3848519d9b4ef..8dcbfe0c2c8625 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -150,8 +150,18 @@ pub struct ErasureMeta { #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct ErasureConfig { - pub(crate) num_data: usize, - pub(crate) num_coding: usize, + num_data: usize, + num_coding: usize, +} + +#[cfg(test)] +impl ErasureConfig { + pub(crate) fn new(num_data: usize, num_coding: usize) -> Self { + ErasureConfig { + num_data, + num_coding, + } + } } #[derive(Deserialize, Serialize)]