Associate block with metadata and fix transaction encoding

crates/hotshot/examples/infra/mod.rs

@@ -193,8 +193,7 @@ pub trait RunDA<
     /// get the anchored view
     /// Note: sequencing leaf does not have state, so does not return state
     async fn initialize_state_and_hotshot(&self) -> SystemContextHandle<TYPES, NODE> {
-        let genesis_block = TYPES::BlockPayload::genesis();
-        let initializer = hotshot::HotShotInitializer::<TYPES>::from_genesis(genesis_block)
+        let initializer = hotshot::HotShotInitializer::<TYPES>::from_genesis()
             .expect("Couldn't generate genesis block");
 
         let config = self.get_config();

crates/hotshot/src/demo.rs

@@ -13,9 +13,10 @@ use derivative::Derivative;
 
 use hotshot_signature_key::bn254::BLSPubKey;
 use hotshot_types::{
-    block_impl::{BlockPayloadError, VIDBlockHeader, VIDBlockPayload, VIDTransaction},
+    block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction},
     data::{fake_commitment, ViewNumber},
     traits::{
+        block_contents::BlockError,
         election::Membership,
         node_implementation::NodeType,
         state::{ConsensusTime, TestableState},
@@ -62,7 +63,7 @@ impl Default for DemoState {
 }
 
 impl State for DemoState {
-    type Error = BlockPayloadError;
+    type Error = BlockError;
 
     type BlockHeader = VIDBlockHeader;
 
@@ -90,7 +91,7 @@ impl State for DemoState {
         view_number: &Self::Time,
     ) -> Result<Self, Self::Error> {
         if !self.validate_block(block_header, view_number) {
-            return Err(BlockPayloadError::InvalidBlock);
+            return Err(BlockError::InvalidBlockHeader);
         }
 
         Ok(DemoState {

crates/hotshot/src/lib.rs

@@ -56,7 +56,6 @@ use hotshot_types::{
 };
 
 use hotshot_types::{
-    block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction},
     consensus::{BlockPayloadStore, Consensus, ConsensusMetricsValue, View, ViewInner, ViewQueue},
     data::Leaf,
     error::StorageSnafu,
@@ -612,11 +611,7 @@ pub trait HotShotType<TYPES: NodeType, I: NodeImplementation<TYPES>> {
 
 #[async_trait]
 impl<
-        TYPES: NodeType<
-            BlockHeader = VIDBlockHeader,
-            BlockPayload = VIDBlockPayload,
-            Transaction = VIDTransaction,
-        >,
+        TYPES: NodeType,
         I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
         MEMBERSHIP: Membership<TYPES>,
     > HotShotType<TYPES, I> for SystemContext<TYPES, I>
@@ -979,12 +974,9 @@ impl<TYPES: NodeType> HotShotInitializer<TYPES> {
     /// initialize from genesis
     /// # Errors
     /// If we are unable to apply the genesis block to the default state
-    pub fn from_genesis(genesis_payload: TYPES::BlockPayload) -> Result<Self, HotShotError<TYPES>> {
-        let time = TYPES::Time::genesis();
-        let justify_qc = QuorumCertificate2::<TYPES>::genesis();
-
+    pub fn from_genesis() -> Result<Self, HotShotError<TYPES>> {
         Ok(Self {
-            inner: Leaf::new(time, justify_qc, genesis_payload),
+            inner: Leaf::genesis(),
         })
     }
 

crates/hotshot/src/tasks/mod.rs

@@ -25,7 +25,6 @@ use hotshot_task_impls::{
     view_sync::{ViewSyncTaskState, ViewSyncTaskStateTypes},
 };
 use hotshot_types::{
-    block_impl::{VIDBlockPayload, VIDTransaction},
     data::Leaf,
     event::Event,
     message::{Message, Messages, SequencingMessage},
@@ -37,6 +36,7 @@ use hotshot_types::{
             ViewSyncEx,
         },
         state::ConsensusTime,
+        BlockPayload,
     },
 };
 use std::{
@@ -221,7 +221,7 @@ where
 /// # Panics
 /// Is unable to panic. This section here is just to satisfy clippy
 pub async fn add_consensus_task<
-    TYPES: NodeType<BlockPayload = VIDBlockPayload, Transaction = VIDTransaction>,
+    TYPES: NodeType,
     I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
 >(
     task_runner: TaskRunner,
@@ -242,13 +242,14 @@ where
         inner: handle.hotshot.inner.clone(),
     };
     let registry = task_runner.registry.clone();
+    let (payload, metadata) = <TYPES::BlockPayload as BlockPayload>::genesis();
     // build the consensus task
     let consensus_state = ConsensusTaskState {
         registry: registry.clone(),
         consensus,
         timeout: handle.hotshot.inner.config.next_view_timeout,
         cur_view: TYPES::Time::new(0),
-        payload_commitment: Some(VIDBlockPayload::genesis().commit()),
+        payload_commitment_and_metadata: Some((payload.commit(), metadata)),
         quorum_exchange: c_api.inner.exchanges.quorum_exchange().clone().into(),
         timeout_exchange: c_api.inner.exchanges.timeout_exchange().clone().into(),
         api: c_api.clone(),
@@ -436,7 +437,7 @@ where
 /// # Panics
 /// Is unable to panic. This section here is just to satisfy clippy
 pub async fn add_transaction_task<
-    TYPES: NodeType<Transaction = VIDTransaction, BlockPayload = VIDBlockPayload>,
+    TYPES: NodeType,
     I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
 >(
     task_runner: TaskRunner,

crates/task-impls/src/consensus.rs

@@ -16,7 +16,6 @@ use hotshot_task::{
     task_impls::{HSTWithEvent, TaskBuilder},
 };
 use hotshot_types::{
-    block_impl::{VIDBlockPayload, VIDTransaction},
     consensus::{Consensus, View},
     data::{Leaf, QuorumProposal},
     event::{Event, EventType},
@@ -57,10 +56,13 @@ use tracing::{debug, error, info, instrument};
 #[derive(Snafu, Debug)]
 pub struct ConsensusTaskError {}
 
+/// Alias for the block payload commitment and the associated metadata.
+type CommitmentAndMetadata<PAYLOAD> = (Commitment<PAYLOAD>, <PAYLOAD as BlockPayload>::Metadata);
+
 /// The state for the consensus task.  Contains all of the information for the implementation
 /// of consensus
 pub struct ConsensusTaskState<
-    TYPES: NodeType<Transaction = VIDTransaction>,
+    TYPES: NodeType,
     I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
     A: ConsensusApi<TYPES, I> + 'static,
 > where
@@ -80,8 +82,8 @@ pub struct ConsensusTaskState<
     /// View number this view is executing in.
     pub cur_view: TYPES::Time,
 
-    /// The commitment to the current block payload submitted to DA
-    pub payload_commitment: Option<Commitment<TYPES::BlockPayload>>,
+    /// The commitment to the current block payload and its metadata submitted to DA.
+    pub payload_commitment_and_metadata: Option<CommitmentAndMetadata<TYPES::BlockPayload>>,
 
     /// the quorum exchange
     pub quorum_exchange: Arc<QuorumEx<TYPES, I>>,
@@ -296,12 +298,11 @@ where
 }
 
 impl<
-        TYPES: NodeType<BlockPayload = VIDBlockPayload, Transaction = VIDTransaction>,
+        TYPES: NodeType,
         I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
         A: ConsensusApi<TYPES, I> + 'static,
     > ConsensusTaskState<TYPES, I, A>
 where
-    TYPES::BlockHeader: BlockHeader<Payload = VIDBlockPayload>,
     QuorumEx<TYPES, I>:
         ConsensusExchange<TYPES, Message<TYPES, I>, Commitment = Commitment<Leaf<TYPES>>>,
     CommitteeEx<TYPES, I>:
@@ -1207,8 +1208,8 @@ where
                 let consensus = self.consensus.read().await;
                 consensus.metrics.number_of_timeouts.add(1);
             }
-            HotShotEvent::SendPayloadCommitment(payload_commitment) => {
-                self.payload_commitment = Some(payload_commitment);
+            HotShotEvent::SendPayloadCommitmentAndMetadata(payload_commitment, metadata) => {
+                self.payload_commitment_and_metadata = Some((payload_commitment, metadata));
             }
             _ => {}
         }
@@ -1284,12 +1285,16 @@ where
             // TODO do some sort of sanity check on the view number that it matches decided
         }
 
-        if let Some(payload_commitment) = &self.payload_commitment {
+        if let Some((payload_commitment, metadata)) = &self.payload_commitment_and_metadata {
             let leaf = Leaf {
                 view_number: view,
                 justify_qc: consensus.high_qc.clone(),
                 parent_commitment: parent_leaf.commit(),
-                block_header: TYPES::BlockHeader::new(*payload_commitment, &parent_header),
+                block_header: TYPES::BlockHeader::new(
+                    *payload_commitment,
+                    metadata.clone(),
+                    &parent_header,
+                ),
                 block_payload: None,
                 rejected: vec![],
                 timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(),
@@ -1301,7 +1306,7 @@ where
                 .sign_validating_or_commitment_proposal::<I>(&leaf.commit());
             // TODO: DA cert is sent as part of the proposal here, we should split this out so we don't have to wait for it.
             let proposal = QuorumProposal {
-                block_header: TYPES::BlockHeader::new(*payload_commitment, &parent_header),
+                block_header: leaf.block_header.clone(),
                 view_number: leaf.view_number,
                 justify_qc: consensus.high_qc.clone(),
                 timeout_certificate: timeout_certificate.or_else(|| None),
@@ -1323,7 +1328,7 @@ where
                     self.quorum_exchange.public_key().clone(),
                 ))
                 .await;
-            self.payload_commitment = None;
+            self.payload_commitment_and_metadata = None;
             return true;
         }
         debug!("Self block was None");
@@ -1332,7 +1337,7 @@ where
 }
 
 impl<
-        TYPES: NodeType<Transaction = VIDTransaction>,
+        TYPES: NodeType,
         I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
         A: ConsensusApi<TYPES, I>,
     > TS for ConsensusTaskState<TYPES, I, A>
@@ -1364,7 +1369,7 @@ pub type ConsensusTaskTypes<TYPES, I, A> = HSTWithEvent<
 
 /// Event handle for consensus
 pub async fn sequencing_consensus_handle<
-    TYPES: NodeType<BlockPayload = VIDBlockPayload, Transaction = VIDTransaction>,
+    TYPES: NodeType,
     I: NodeImplementation<TYPES, ConsensusMessage = SequencingMessage<TYPES, I>>,
     A: ConsensusApi<TYPES, I> + 'static,
 >(
@@ -1375,7 +1380,6 @@ pub async fn sequencing_consensus_handle<
     ConsensusTaskState<TYPES, I, A>,
 )
 where
-    TYPES::BlockHeader: BlockHeader<Payload = VIDBlockPayload>,
     QuorumEx<TYPES, I>:
         ConsensusExchange<TYPES, Message<TYPES, I>, Commitment = Commitment<Leaf<TYPES>>>,
     CommitteeEx<TYPES, I>:
@@ -1402,7 +1406,7 @@ pub fn consensus_event_filter<TYPES: NodeType, I: NodeImplementation<TYPES>>(
             | HotShotEvent::QCFormed(_)
             | HotShotEvent::DACRecv(_)
             | HotShotEvent::ViewChange(_)
-            | HotShotEvent::SendPayloadCommitment(_)
+            | HotShotEvent::SendPayloadCommitmentAndMetadata(_, _)
             | HotShotEvent::Timeout(_)
             | HotShotEvent::TimeoutVoteRecv(_)
             | HotShotEvent::Shutdown,

crates/task-impls/src/da.rs

@@ -394,18 +394,20 @@ where
 
                 return None;
             }
-            HotShotEvent::BlockReady(block, view) => {
+            HotShotEvent::BlockReady(payload, metadata, view) => {
                 self.committee_exchange
                     .network()
                     .inject_consensus_info(ConsensusIntentEvent::CancelPollForTransactions(*view))
                     .await;
 
-                let payload_commitment = block.commit();
+                let payload_commitment = payload.commit();
                 let signature = self
                     .committee_exchange
                     .sign_da_proposal(&payload_commitment);
+                // TODO (Keyao) Fix the payload sending and receiving for the DA proposal.
+                // <https://github.com/EspressoSystems/HotShot/issues/2026>
                 let data: DAProposal<TYPES> = DAProposal {
-                    block_payload: block.clone(),
+                    block_payload: payload.clone(),
                     // Upon entering a new view we want to send a DA Proposal for the next view -> Is it always the case that this is cur_view + 1?
                     view_number: view,
                 };
@@ -418,7 +420,10 @@ where
                 };
 
                 self.event_stream
-                    .publish(HotShotEvent::SendPayloadCommitment(payload_commitment))
+                    .publish(HotShotEvent::SendPayloadCommitmentAndMetadata(
+                        payload_commitment,
+                        metadata,
+                    ))
                     .await;
                 self.event_stream
                     .publish(HotShotEvent::DAProposalSend(
@@ -453,7 +458,7 @@ where
             HotShotEvent::DAProposalRecv(_, _)
                 | HotShotEvent::DAVoteRecv(_)
                 | HotShotEvent::Shutdown
-                | HotShotEvent::BlockReady(_, _)
+                | HotShotEvent::BlockReady(_, _, _)
                 | HotShotEvent::Timeout(_)
                 | HotShotEvent::ViewChange(_)
         )

crates/task-impls/src/events.rs

@@ -13,9 +13,12 @@ use hotshot_types::{
         DAVote2, QuorumVote, TimeoutVote2, VIDVote2, ViewSyncCommitVote, ViewSyncFinalizeVote,
         ViewSyncPreCommitVote,
     },
-    traits::node_implementation::{
-        CommitteeMembership, NodeImplementation, NodeType, QuorumMembership, VIDMembership,
-        ViewSyncMembership,
+    traits::{
+        node_implementation::{
+            CommitteeMembership, NodeImplementation, NodeType, QuorumMembership, VIDMembership,
+            ViewSyncMembership,
+        },
+        BlockPayload,
     },
 };
 
@@ -91,10 +94,17 @@ pub enum HotShotEvent<TYPES: NodeType, I: NodeImplementation<TYPES>> {
     TransactionsRecv(Vec<TYPES::Transaction>),
     /// Send transactions to the network
     TransactionSend(TYPES::Transaction, TYPES::SignatureKey),
-    /// Event to send block payload commitment from DA leader to the quorum; internal event only
-    SendPayloadCommitment(Commitment<TYPES::BlockPayload>),
+    /// Event to send block payload commitment and metadata from DA leader to the quorum; internal event only
+    SendPayloadCommitmentAndMetadata(
+        Commitment<TYPES::BlockPayload>,
+        <TYPES::BlockPayload as BlockPayload>::Metadata,
+    ),
     /// Event when the transactions task has a block formed
-    BlockReady(TYPES::BlockPayload, TYPES::Time),
+    BlockReady(
+        TYPES::BlockPayload,
+        <TYPES::BlockPayload as BlockPayload>::Metadata,
+        TYPES::Time,
+    ),
     /// Event when consensus decided on a leaf
     LeafDecided(Vec<Leaf<TYPES>>),
     /// Send VID shares to VID storage nodes; emitted by the DA leader