[WEEKLY RELEASE] HotShot - rc-0.5.77 (#2109)

Weekly Release
<!-- These comments should help create a useful PR message, please
delete any remaining comments before opening the PR. -->
<!-- If there is no issue number make sure to describe clearly *why*
this PR is necessary. -->
<!-- Mention open questions, remaining TODOs, if any -->

### This PR:
<!-- Describe what this PR adds to this repo and why -->
<!-- E.g. -->
<!-- * Implements feature 1 -->
<!-- * Fixes bug 3 -->
Weekly merge include:
1.) File changes for the Vote Commitment for UTs to pass (see
#2111)
2.) Libp2p refactoring
3.) Adding table to store upgrade certificate that hotshot is able to
load after a restart. Add a test to ensure we can store/load the upgrade
certificate.
### This PR does not:
<!-- Describe what is out of scope for this PR, if applicable. Leave
this section blank if it's not applicable -->
<!-- This section helps avoid the reviewer having to needlessly point
out missing parts -->
<!-- * Implement feature 3 because that feature is blocked by Issue 4
-->
<!-- * Implement xyz because that is tracked in issue #123. -->
<!-- * Address xzy for which I opened issue #456 -->

### Key places to review:
<!-- Describe key places for reviewers to pay close attention to -->
<!-- * file.rs, `add_integers` function -->
<!-- Or directly comment on those files/lines to make it easier for the
reviewers -->

<!-- ### How to test this PR:  -->
<!-- Optional, uncomment the above line if this is relevant to your PR
-->
<!-- If your PR is fully tested through CI there is no need to add this
section -->
<!-- * E.g. `just test` -->

<!-- ### Things tested -->
<!-- Anything that was manually tested (that is not tested in CI). -->
<!-- E.g. building/running of docker containers. Changes to docker demo,
... -->
<!-- Especially mention anything untested, with reasoning and link an
issue to resolve this. -->

<!-- Complete the following items before creating this PR -->
<!-- [ ] Issue linked or PR description mentions why this change is
necessary. -->
<!-- [ ] PR description is clear enough for reviewers. -->
<!-- [ ] Documentation for changes (additions) has been updated (added).
-->
<!-- [ ] If this is a draft it is marked as "draft".  -->

<!-- To make changes to this template edit
https://github.com/EspressoSystems/.github/blob/main/PULL_REQUEST_TEMPLATE.md
-->

Cargo.toml

@@ -57,23 +57,23 @@ dyn-clone = "1.0"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76", features = ["dependency-tasks"] }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77", features = ["dependency-tasks"] }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.76" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.77" }
 hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.51" }
-marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.50" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.48" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.61" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" }
+marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.51" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.49" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.62" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76", features = ["dependency-tasks"] }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" }
-libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77", features = ["dependency-tasks"] }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" }
+libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" }
 hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 # Temporary, used to pull in the mock auction results provider
-hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" }
+hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [
@@ -144,15 +144,3 @@ paste = "1.0"
 rand = "0.8.5"
 time = "0.3"
 trait-set = "0.3.0"
-
-[patch."https://github.com/EspressoSystems/HotShot.git"]
-hotshot = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-builder-api = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-orchestrator = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-stake-table = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-task ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-task-impls ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-testing ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-hotshot-types = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }
-libp2p-networking = { git = "https://github.com/EspressoSystems//HotShot.git", tag = "0.5.75-patch1" }
-hotshot-example-types = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" }

builder/src/bin/permissioned-builder.rs

@@ -1,6 +1,4 @@
-use std::{
-    collections::HashMap, net::ToSocketAddrs, num::NonZeroUsize, path::PathBuf, time::Duration,
-};
+use std::{collections::HashMap, num::NonZeroUsize, path::PathBuf, time::Duration};
 
 use anyhow::{bail, Context};
 use builder::permissioned::init_node;
@@ -235,26 +233,10 @@ async fn run<V: Versions>(
 
     let builder_key_pair = EthKeyPair::from_mnemonic(&opt.eth_mnemonic, opt.eth_account_index)?;
 
-    // Parse supplied Libp2p addresses to their socket form
-    // We expect all nodes to be reachable via IPv4, so we filter out any IPv6 addresses.
-    // Downstream in HotShot we pin the IP address to v4, but this can be fixed in the future.
-    let libp2p_advertise_address = opt
-        .libp2p_advertise_address
-        .to_socket_addrs()?
-        .find(|x| x.is_ipv4())
-        .ok_or(anyhow::anyhow!(
-            "Failed to resolve Libp2p advertise address"
-        ))?;
-    let libp2p_bind_address = opt
-        .libp2p_bind_address
-        .to_socket_addrs()?
-        .find(|x| x.is_ipv4())
-        .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?;
-
     let network_params = NetworkParams {
         cdn_endpoint: opt.cdn_endpoint,
-        libp2p_advertise_address,
-        libp2p_bind_address,
+        libp2p_advertise_address: opt.libp2p_advertise_address,
+        libp2p_bind_address: opt.libp2p_bind_address,
         libp2p_bootstrap_nodes: opt.libp2p_bootstrap_nodes,
         orchestrator_url: opt.orchestrator_url,
         state_relay_server_url: opt.state_relay_server_url,

builder/src/lib.rs

@@ -28,7 +28,7 @@ use futures::{
     stream::{Stream, StreamExt},
 };
 use hotshot::{
-    traits::election::static_committee::GeneralStaticCommittee,
+    traits::election::static_committee::StaticCommittee,
     types::{SignatureKey, SystemContextHandle},
     HotShotInitializer, Memberships, SystemContext,
 };

builder/src/non_permissioned.rs

@@ -47,7 +47,7 @@ use sequencer::{catchup::StatePeers, L1Params, NetworkParams, SequencerApiVersio
 use surf::http::headers::ACCEPT;
 use surf_disco::Client;
 use tide_disco::{app, method::ReadState, App, Url};
-use vbs::version::{StaticVersionType, Version};
+use vbs::version::{StaticVersion, StaticVersionType, Version};
 
 use crate::run_builder_api_service;
 

builder/src/permissioned.rs

@@ -41,10 +41,11 @@ use futures::{
 };
 use hotshot::{
     traits::{
-        election::static_committee::GeneralStaticCommittee,
+        election::static_committee::StaticCommittee,
         implementations::{
-            derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, CombinedNetworks, GossipConfig,
-            KeyPair, Libp2pNetwork, PushCdnNetwork, WrappedSignatureKey,
+            derive_libp2p_multiaddr, derive_libp2p_peer_id, CdnMetricsValue, CdnTopic,
+            CombinedNetworks, GossipConfig, KeyPair, Libp2pNetwork, PushCdnNetwork,
+            WrappedSignatureKey,
         },
         BlockPayload,
     },
@@ -105,6 +106,7 @@ use sequencer::{
 };
 use surf_disco::Client;
 use tide_disco::{app, method::ReadState, App, Url};
+use tracing::info;
 use vbs::version::StaticVersionType;
 
 use crate::run_builder_api_service;
@@ -148,13 +150,7 @@ pub async fn init_node<P: SequencerPersistence, V: Versions>(
     maximize_txns_count_timeout_duration: Duration,
 ) -> anyhow::Result<BuilderContext<network::Production, P, V>> {
     // Orchestrator client
-    let validator_args = ValidatorArgs {
-        url: network_params.orchestrator_url,
-        advertise_address: Some(network_params.libp2p_advertise_address),
-        builder_address: None,
-        network_config_file: None,
-    };
-    let orchestrator_client = OrchestratorClient::new(validator_args);
+    let orchestrator_client = OrchestratorClient::new(network_params.orchestrator_url);
     let state_key_pair = StateKeyPair::from_sign_key(network_params.private_state_key);
     let my_config = ValidatorConfig {
         public_key: BLSPubKey::from_private(&network_params.private_staking_key),
@@ -164,6 +160,25 @@ pub async fn init_node<P: SequencerPersistence, V: Versions>(
         is_da,
     };
 
+    // Parse the Libp2p bind and advertise addresses to multiaddresses
+    let libp2p_bind_address = derive_libp2p_multiaddr(&network_params.libp2p_bind_address)
+        .with_context(|| {
+            format!(
+                "Failed to derive Libp2p bind address of {}",
+                &network_params.libp2p_bind_address
+            )
+        })?;
+    let libp2p_advertise_address =
+        derive_libp2p_multiaddr(&network_params.libp2p_advertise_address).with_context(|| {
+            format!(
+                "Failed to derive Libp2p advertise address of {}",
+                &network_params.libp2p_advertise_address
+            )
+        })?;
+
+    info!("Libp2p bind address: {}", libp2p_bind_address);
+    info!("Libp2p advertise address: {}", libp2p_advertise_address);
+
     // Derive our Libp2p public key from our private key
     let libp2p_public_key =
         derive_libp2p_peer_id::<<SeqTypes as NodeType>::SignatureKey>(&my_config.private_key)
@@ -174,7 +189,7 @@ pub async fn init_node<P: SequencerPersistence, V: Versions>(
         my_config.clone(),
         // Register in our Libp2p advertise address and public key so other nodes
         // can contact us on startup
-        Some(network_params.libp2p_advertise_address),
+        Some(libp2p_advertise_address),
         Some(libp2p_public_key),
     )
     .await?
@@ -224,7 +239,7 @@ pub async fn init_node<P: SequencerPersistence, V: Versions>(
     let p2p_network = Libp2pNetwork::from_config::<SeqTypes>(
         config.clone(),
         GossipConfig::default(),
-        network_params.libp2p_bind_address,
+        libp2p_bind_address,
         &my_config.public_key,
         // We need the private key so we can derive our Libp2p keypair
         // (using https://docs.rs/blake3/latest/blake3/fn.derive_key.html)
@@ -342,12 +357,12 @@ pub async fn init_hotshot<N: ConnectedNetwork<PubKey>, P: SequencerPersistence,
         None => config.known_nodes_with_stake.clone(),
     };
 
-    let quorum_membership = GeneralStaticCommittee::new(
+    let quorum_membership = StaticCommittee::new(
         combined_known_nodes_with_stake.clone(),
         combined_known_nodes_with_stake.clone(),
         Topic::Global,
     );
-    let da_membership = GeneralStaticCommittee::new(
+    let da_membership = StaticCommittee::new(
         combined_known_nodes_with_stake.clone(),
         combined_known_nodes_with_stake,
         Topic::Da,

data/v1/messages.json

@@ -42,7 +42,7 @@
                 },
                 "signatures": null,
                 "view_number": 0,
-                "vote_commitment": "COMMIT~tzMwZg2x5QJTx7Di9eEvNC0P8CZ_iNX7LJ01DHA9XEdO"
+                "vote_commitment": "COMMIT~f11eqw2KFTy-i-EaXUfEC1v0CPo8gPcd2My10Y5zSNok"
               },
               "proposal_certificate": {
                 "Timeout": {

data/v2/messages.json

@@ -50,7 +50,7 @@
                 },
                 "signatures": null,
                 "view_number": 0,
-                "vote_commitment": "COMMIT~tzMwZg2x5QJTx7Di9eEvNC0P8CZ_iNX7LJ01DHA9XEdO"
+                "vote_commitment": "COMMIT~f11eqw2KFTy-i-EaXUfEC1v0CPo8gPcd2My10Y5zSNok"
               },
               "proposal_certificate": {
                 "Timeout": {

data/v3/messages.json

@@ -58,7 +58,7 @@
                 },
                 "signatures": null,
                 "view_number": 0,
-                "vote_commitment": "COMMIT~tzMwZg2x5QJTx7Di9eEvNC0P8CZ_iNX7LJ01DHA9XEdO"
+                "vote_commitment": "COMMIT~f11eqw2KFTy-i-EaXUfEC1v0CPo8gPcd2My10Y5zSNok"
               },
               "proposal_certificate": {
                 "Timeout": {

marketplace-builder/src/lib.rs

@@ -31,7 +31,7 @@ use futures::{
     stream::{Stream, StreamExt},
 };
 use hotshot::{
-    traits::election::static_committee::GeneralStaticCommittee,
+    traits::election::static_committee::StaticCommittee,
     types::{SignatureKey, SystemContextHandle},
     HotShotInitializer, Memberships, SystemContext,
 };

node-metrics/src/service/client_state/mod.rs

@@ -406,14 +406,14 @@ where
         block_size: data_state_read_lock_guard
             .latest_blocks()
             .skip(1)
-            .map(|block| block.size)
+            .map(|block| Some(block.size))
             .collect(),
         block_time: data_state_read_lock_guard
             .latest_blocks()
             .skip(1)
             .zip(data_state_read_lock_guard.latest_blocks())
             .map(|(block_i, block_i_sub_1)| {
-                (block_i.time.0 - block_i_sub_1.time.0).whole_seconds() as u64
+                Some((block_i.time.0 - block_i_sub_1.time.0).whole_seconds() as u64)
             })
             .collect(),
         block_transactions: data_state_read_lock_guard

sequencer/api/migrations/V38__upgrade_certificate.sql

@@ -0,0 +1,5 @@
+CREATE TABLE upgrade_certificate (
+    id bool PRIMARY KEY DEFAULT true,
+    data BYTEA
+);
+REVOKE DELETE, TRUNCATE ON upgrade_certificate FROM public;

sequencer/src/bin/cdn-whitelist.rs

@@ -8,10 +8,7 @@ use anyhow::{Context, Result};
 use cdn_broker::reexports::discovery::{DiscoveryClient, Embedded, Redis};
 use clap::Parser;
 use espresso_types::SeqTypes;
-use hotshot_orchestrator::{
-    client::{OrchestratorClient, ValidatorArgs},
-    config::NetworkConfig,
-};
+use hotshot_orchestrator::{client::OrchestratorClient, config::NetworkConfig};
 use hotshot_types::traits::{node_implementation::NodeType, signature_key::SignatureKey};
 use surf_disco::Url;
 
@@ -44,12 +41,9 @@ async fn main() -> Result<()> {
     tracing_subscriber::fmt::init();
 
     // Create a new `OrchestratorClient` from the supplied URL
-    let orchestrator_client = OrchestratorClient::new(ValidatorArgs {
-        url: Url::from_str(&args.orchestrator_url).with_context(|| "Invalid URL")?,
-        advertise_address: None,
-        builder_address: None,
-        network_config_file: None,
-    });
+    let orchestrator_client = OrchestratorClient::new(
+        Url::from_str(&args.orchestrator_url).with_context(|| "Invalid URL")?,
+    );
 
     tracing::info!(
         "Waiting for config from orchestrator on {}",

sequencer/src/context.rs

@@ -15,7 +15,7 @@ use futures::{
     stream::{Stream, StreamExt},
 };
 use hotshot::{
-    traits::election::static_committee::GeneralStaticCommittee,
+    traits::election::static_committee::StaticCommittee,
     types::{Event, EventType, SystemContextHandle},
     MarketplaceConfig, Memberships, SystemContext,
 };
@@ -103,13 +103,13 @@ impl<N: ConnectedNetwork<PubKey>, P: SequencerPersistence, V: Versions> Sequence
             .load_consensus_state::<V>(instance_state.clone())
             .await?;
 
-        let committee_membership = GeneralStaticCommittee::new(
+        let committee_membership = StaticCommittee::new(
             config.known_nodes_with_stake.clone(),
             config.known_nodes_with_stake.clone(),
             Topic::Global,
         );
 
-        let da_membership = GeneralStaticCommittee::new(
+        let da_membership = StaticCommittee::new(
             config.known_nodes_with_stake.clone(),
             config.known_da_nodes.clone(),
             Topic::Da,