Merge branch 'das' into fork/reconstruct-without-blocking-import

beacon_node/lighthouse_network/src/config.rs

@@ -42,7 +42,7 @@ pub struct Config {
     pub network_dir: PathBuf,
 
     /// IP addresses to listen on.
-    listen_addresses: ListenAddress,
+    pub(crate) listen_addresses: ListenAddress,
 
     /// The address to broadcast to peers about which address we are listening on. None indicates
     /// that no discovery address has been set in the CLI args.

beacon_node/lighthouse_network/src/discovery/enr.rs

@@ -25,7 +25,7 @@ pub const ATTESTATION_BITFIELD_ENR_KEY: &str = "attnets";
 /// The ENR field specifying the sync committee subnet bitfield.
 pub const SYNC_COMMITTEE_BITFIELD_ENR_KEY: &str = "syncnets";
 /// The ENR field specifying the peerdas custody subnet count.
-pub const PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY: &str = "custody_subnet_count";
+pub const PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY: &str = "csc";
 
 /// Extension trait for ENR's within Eth2.
 pub trait Eth2Enr {
@@ -67,8 +67,10 @@ impl Eth2Enr for Enr {
     /// if the custody value is non-existent in the ENR, then we assume the minimum custody value
     /// defined in the spec.
     fn custody_subnet_count<E: EthSpec>(&self, spec: &ChainSpec) -> u64 {
-        self.get(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY)
-            .and_then(|custody_bytes| u64::from_ssz_bytes(custody_bytes).ok())
+        self.get_decodable::<u64>(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY)
+            .and_then(|r| r.ok())
+            // If value supplied in ENR is invalid, fallback to `custody_requirement`
+            .filter(|csc| csc <= &spec.data_column_sidecar_subnet_count)
             .unwrap_or(spec.custody_requirement)
     }
 
@@ -243,10 +245,7 @@ pub fn build_enr<E: EthSpec>(
         spec.custody_requirement
     };
 
-    builder.add_value(
-        PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY,
-        &custody_subnet_count.as_ssz_bytes(),
-    );
+    builder.add_value(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY, &custody_subnet_count);
 
     builder
         .build(enr_key)
@@ -309,3 +308,70 @@ pub fn save_enr_to_disk(dir: &Path, enr: &Enr, log: &slog::Logger) {
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::config::Config as NetworkConfig;
+    use types::MainnetEthSpec;
+
+    type E = MainnetEthSpec;
+
+    #[test]
+    fn custody_subnet_count_default() {
+        let config = NetworkConfig {
+            subscribe_all_data_column_subnets: false,
+            ..NetworkConfig::default()
+        };
+        let spec = E::default_spec();
+        let enr = build_enr_with_config(config, &spec).0;
+
+        assert_eq!(
+            enr.custody_subnet_count::<E>(&spec),
+            spec.custody_requirement,
+        );
+    }
+
+    #[test]
+    fn custody_subnet_count_all() {
+        let config = NetworkConfig {
+            subscribe_all_data_column_subnets: true,
+            ..NetworkConfig::default()
+        };
+        let spec = E::default_spec();
+        let enr = build_enr_with_config(config, &spec).0;
+
+        assert_eq!(
+            enr.custody_subnet_count::<E>(&spec),
+            spec.data_column_sidecar_subnet_count,
+        );
+    }
+
+    #[test]
+    fn custody_subnet_count_fallback_default() {
+        let config = NetworkConfig::default();
+        let spec = E::default_spec();
+        let (mut enr, enr_key) = build_enr_with_config(config, &spec);
+        let invalid_subnet_count = 99u64;
+
+        enr.insert(
+            PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY,
+            &invalid_subnet_count,
+            &enr_key,
+        )
+        .unwrap();
+
+        assert_eq!(
+            enr.custody_subnet_count::<E>(&spec),
+            spec.custody_requirement,
+        );
+    }
+
+    fn build_enr_with_config(config: NetworkConfig, spec: &ChainSpec) -> (Enr, CombinedKey) {
+        let keypair = libp2p::identity::secp256k1::Keypair::generate();
+        let enr_key = CombinedKey::from_secp256k1(&keypair);
+        let enr_fork_id = EnrForkId::default();
+        let enr = build_enr::<E>(&enr_key, &config, &enr_fork_id, spec).unwrap();
+        (enr, enr_key)
+    }
+}

beacon_node/lighthouse_network/src/peer_manager/peerdb.rs

@@ -5,7 +5,6 @@ use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerInfo};
 use rand::seq::SliceRandom;
 use score::{PeerAction, ReportSource, Score, ScoreState};
 use slog::{crit, debug, error, trace, warn};
-use ssz::Encode;
 use std::net::IpAddr;
 use std::time::Instant;
 use std::{cmp::Ordering, fmt::Display};
@@ -687,7 +686,7 @@ impl<E: EthSpec> PeerDB<E> {
         if supernode {
             enr.insert(
                 PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY,
-                &spec.data_column_sidecar_subnet_count.as_ssz_bytes(),
+                &spec.data_column_sidecar_subnet_count,
                 &enr_key,
             )
             .expect("u64 can be encoded");

beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs

@@ -801,7 +801,8 @@ mod tests {
     use crate::types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield};
     use types::{
         blob_sidecar::BlobIdentifier, BeaconBlock, BeaconBlockAltair, BeaconBlockBase,
-        BeaconBlockBellatrix, EmptyBlock, Epoch, FullPayload, Signature, Slot,
+        BeaconBlockBellatrix, DataColumnIdentifier, EmptyBlock, Epoch, FullPayload, Signature,
+        Slot,
     };
 
     type Spec = types::MainnetEthSpec;
@@ -848,6 +849,10 @@ mod tests {
         Arc::new(BlobSidecar::empty())
     }
 
+    fn empty_data_column_sidecar() -> Arc<DataColumnSidecar<Spec>> {
+        Arc::new(DataColumnSidecar::empty())
+    }
+
     /// Bellatrix block with length < max_rpc_size.
     fn bellatrix_block_small(
         fork_context: &ForkContext,
@@ -909,6 +914,27 @@ mod tests {
         }
     }
 
+    fn dcbrange_request() -> DataColumnsByRangeRequest {
+        DataColumnsByRangeRequest {
+            start_slot: 0,
+            count: 10,
+            columns: vec![1, 2, 3],
+        }
+    }
+
+    fn dcbroot_request(spec: &ChainSpec) -> DataColumnsByRootRequest {
+        DataColumnsByRootRequest {
+            data_column_ids: RuntimeVariableList::new(
+                vec![DataColumnIdentifier {
+                    block_root: Hash256::zero(),
+                    index: 0,
+                }],
+                spec.max_request_data_column_sidecars as usize,
+            )
+            .unwrap(),
+        }
+    }
+
     fn bbroot_request_v1(spec: &ChainSpec) -> BlocksByRootRequest {
         BlocksByRootRequest::new_v1(vec![Hash256::zero()], spec)
     }
@@ -1198,6 +1224,34 @@ mod tests {
             ),
             Ok(Some(RPCResponse::BlobsByRoot(empty_blob_sidecar()))),
         );
+
+        assert_eq!(
+            encode_then_decode_response(
+                SupportedProtocol::DataColumnsByRangeV1,
+                RPCCodedResponse::Success(RPCResponse::DataColumnsByRange(
+                    empty_data_column_sidecar()
+                )),
+                ForkName::Deneb,
+                &chain_spec
+            ),
+            Ok(Some(RPCResponse::DataColumnsByRange(
+                empty_data_column_sidecar()
+            ))),
+        );
+
+        assert_eq!(
+            encode_then_decode_response(
+                SupportedProtocol::DataColumnsByRootV1,
+                RPCCodedResponse::Success(RPCResponse::DataColumnsByRoot(
+                    empty_data_column_sidecar()
+                )),
+                ForkName::Deneb,
+                &chain_spec
+            ),
+            Ok(Some(RPCResponse::DataColumnsByRoot(
+                empty_data_column_sidecar()
+            ))),
+        );
     }
 
     // Test RPCResponse encoding/decoding for V1 messages
@@ -1551,6 +1605,8 @@ mod tests {
             OutboundRequest::MetaData(MetadataRequest::new_v1()),
             OutboundRequest::BlobsByRange(blbrange_request()),
             OutboundRequest::BlobsByRoot(blbroot_request(&chain_spec)),
+            OutboundRequest::DataColumnsByRange(dcbrange_request()),
+            OutboundRequest::DataColumnsByRoot(dcbroot_request(&chain_spec)),
             OutboundRequest::MetaData(MetadataRequest::new_v2()),
         ];
 

beacon_node/lighthouse_network/src/rpc/methods.rs

@@ -312,6 +312,26 @@ impl DataColumnsByRangeRequest {
             .saturating_mul(E::max_blobs_per_block() as u64)
             .saturating_mul(self.columns.len() as u64)
     }
+
+    pub fn ssz_min_len() -> usize {
+        DataColumnsByRangeRequest {
+            start_slot: 0,
+            count: 0,
+            columns: vec![0],
+        }
+        .as_ssz_bytes()
+        .len()
+    }
+
+    pub fn ssz_max_len(spec: &ChainSpec) -> usize {
+        DataColumnsByRangeRequest {
+            start_slot: 0,
+            count: 0,
+            columns: vec![0; spec.number_of_columns],
+        }
+        .as_ssz_bytes()
+        .len()
+    }
 }
 
 /// Request a number of beacon block roots from a peer.

beacon_node/lighthouse_network/src/rpc/protocol.rs

@@ -496,8 +496,8 @@ impl ProtocolId {
             Protocol::BlobsByRoot => RpcLimits::new(0, spec.max_blobs_by_root_request),
             Protocol::DataColumnsByRoot => RpcLimits::new(0, spec.max_data_columns_by_root_request),
             Protocol::DataColumnsByRange => RpcLimits::new(
-                <DataColumnsByRangeRequest as Encode>::ssz_fixed_len(),
-                <DataColumnsByRangeRequest as Encode>::ssz_fixed_len(),
+                DataColumnsByRangeRequest::ssz_min_len(),
+                DataColumnsByRangeRequest::ssz_max_len(spec),
             ),
             Protocol::Ping => RpcLimits::new(
                 <Ping as Encode>::ssz_fixed_len(),

consensus/types/src/data_column_subnet_id.rs

@@ -107,15 +107,15 @@ impl From<u64> for DataColumnSubnetId {
     }
 }
 
-impl Into<u64> for DataColumnSubnetId {
-    fn into(self) -> u64 {
-        self.0
+impl From<DataColumnSubnetId> for u64 {
+    fn from(val: DataColumnSubnetId) -> Self {
+        val.0
     }
 }
 
-impl Into<u64> for &DataColumnSubnetId {
-    fn into(self) -> u64 {
-        self.0
+impl From<&DataColumnSubnetId> for u64 {
+    fn from(val: &DataColumnSubnetId) -> Self {
+        val.0
     }
 }
 

testing/ef_tests/Makefile

@@ -1,4 +1,4 @@
-TESTS_TAG := v1.5.0-alpha.2
+TESTS_TAG := v1.5.0-alpha.3
 TESTS = general minimal mainnet
 TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
 

testing/ef_tests/check_all_files_accessed.py

@@ -35,12 +35,12 @@
     "tests/.*/.*/ssz_static/LightClientStore",
     # LightClientSnapshot
     "tests/.*/.*/ssz_static/LightClientSnapshot",
+    # Unused container for das
+    "tests/.*/.*/ssz_static/MatrixEntry",
     # Unused kzg methods
-    "tests/.*/.*/kzg/compute_cells",
-    "tests/.*/.*/kzg/recover_all_cells",
     "tests/.*/.*/kzg/verify_cell_kzg_proof",
     # One of the EF researchers likes to pack the tarballs on a Mac
-    ".*\.DS_Store.*",
+    ".*/.DS_Store.*",
     # More Mac weirdness.
     "tests/mainnet/bellatrix/operations/deposit/pyspec_tests/deposit_with_previous_fork_version__valid_ineffective/._meta.yaml",
     "tests/mainnet/eip7594/networking/get_custody_columns/pyspec_tests/get_custody_columns__short_node_id/._meta.yaml",

testing/ef_tests/src/cases.rs

@@ -23,6 +23,7 @@ mod kzg_blob_to_kzg_commitment;
 mod kzg_compute_blob_kzg_proof;
 mod kzg_compute_cells_and_kzg_proofs;
 mod kzg_compute_kzg_proof;
+mod kzg_recover_cells_and_kzg_proofs;
 mod kzg_verify_blob_kzg_proof;
 mod kzg_verify_blob_kzg_proof_batch;
 mod kzg_verify_cell_kzg_proof_batch;
@@ -56,6 +57,7 @@ pub use kzg_blob_to_kzg_commitment::*;
 pub use kzg_compute_blob_kzg_proof::*;
 pub use kzg_compute_cells_and_kzg_proofs::*;
 pub use kzg_compute_kzg_proof::*;
+pub use kzg_recover_cells_and_kzg_proofs::*;
 pub use kzg_verify_blob_kzg_proof::*;
 pub use kzg_verify_blob_kzg_proof_batch::*;
 pub use kzg_verify_cell_kzg_proof_batch::*;