add endpoint for hotshot config and env variables (#1481)

Closes #1413
<!-- If there is no issue number make sure to describe clearly *why*
this PR is necessary. -->
<!-- Mention open questions, remaining TODOs, if any -->

### This PR:
<!-- Describe what this PR adds to this repo and why -->
<!-- E.g. -->
<!-- * Implements feature 1 -->
<!-- * Fixes bug 3 -->

adds endpoints for
- hotshot config
- env vars

### This PR does not:
<!-- Describe what is out of scope for this PR, if applicable. Leave
this section blank if it's not applicable -->
<!-- This section helps avoid the reviewer having to needlessly point
out missing parts -->
<!-- * Implement feature 3 because that feature is blocked by Issue 4
-->
<!-- * Implement xyz because that is tracked in issue #123. -->
<!-- * Address xzy for which I opened issue #456 -->

### Key places to review:
<!-- Describe key places for reviewers to pay close attention to -->
<!-- * file.rs, `add_integers` function -->
<!-- Or directly comment on those files/lines to make it easier for the
reviewers -->

<!-- ### How to test this PR:  -->
<!-- Optional, uncomment the above line if this is relevant to your PR
-->
<!-- If your PR is fully tested through CI there is no need to add this
section -->
<!-- * E.g. `just test` -->

<!-- ### Things tested -->
<!-- Anything that was manually tested (that it is not tested in CI).
-->
<!-- E.g. building/running of docker containers. Changes to docker demo,
... -->
<!-- Especially mention anything untested, with reasoning and link an
issue to resolve this. -->

<!-- Complete the following items before creating this PR -->
<!-- [ ] Issue linked or PR description mentions why this change is
necessary. -->
<!-- [ ] PR description is clear enough for reviewers. -->
<!-- [ ] If this is a draft it is marked as "draft".  -->

<!-- To make changes to this template edit
https://github.com/EspressoSystems/.github/blob/main/PULL_REQUEST_TEMPLATE.md
-->

docker-compose.yaml

@@ -212,7 +212,7 @@ services:
       - "$ESPRESSO_SEQUENCER_API_PORT:$ESPRESSO_SEQUENCER_API_PORT"
       - "$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT"
     # Run the full API server with all modules, Postgres storage
-    command: sequencer -- storage-sql -- http -- query -- catchup -- status -- submit -- hotshot-events
+    command: sequencer -- storage-sql -- http -- query -- catchup -- status -- submit -- hotshot-events -- config
     environment:
       - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL
       - ESPRESSO_SEQUENCER_CDN_ENDPOINT

process-compose.yaml

@@ -104,7 +104,7 @@ processes:
         condition: process_completed
 
   sequencer0:
-    command: sequencer -- storage-sql -- http -- query -- catchup -- status -- submit -- hotshot-events
+    command: sequencer -- storage-sql -- http -- query -- catchup -- status -- submit -- hotshot-events -- config
     environment:
       - ESPRESSO_SEQUENCER_API_PORT=$ESPRESSO_SEQUENCER_API_PORT
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0

sequencer/api/config.toml

@@ -0,0 +1,9 @@
+[route.hotshot]
+PATH = ["/hotshot"]
+METHOD = "GET"
+DOC = "Get the Hotshot configuration for the current node."
+
+[route.env]
+PATH = ["/env"]
+METHOD = "GET"
+DOC = "Get all ESPRESSO environment variables set for the current node."

sequencer/api/public-env-vars.toml

@@ -0,0 +1,105 @@
+# TOML file for the `/env` route of Config endpoint
+# This file is used when the Config module is enabled.
+# It specifies a list of environment variable names (`variables`) 
+# that will be exposed by the `/env` route.
+
+
+variables = [
+    "ESPRESSO_SEQUENCER_ORCHESTRATOR_URL",
+    "ESPRESSO_SEQUENCER_CDN_ENDPOINT",
+    "ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS",
+    "ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS",
+    "ESPRESSO_STATE_RELAY_SERVER_URL",
+    "ESPRESSO_SEQUENCER_PREFUNDED_BUILDER_ACCOUNTS",
+    "ESPRESSO_SEQUENCER_L1_GENESIS",
+    "ESPRESSO_SEQUENCER_L1_EVENTS_MAX_BLOCK_RANGE",
+    "ESPRESSO_SEQUENCER_IS_DA",
+    "ESPRESSO_SEQUENCER_STATE_PEERS",
+    "ESPRESSO_SEQUENCER_STAKE_TABLE_CAPACITY",
+    "ESPRESSO_SEQUENCER_API_PEERS",
+    "ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT",
+    "ESPRESSO_SEQUENCER_POSTGRES_HOST",
+    "ESPRESSO_SEQUENCER_POSTGRES_PORT",
+    "ESPRESSO_SEQUENCER_POSTGRES_DATABASE",
+    "ESPRESSO_SEQUENCER_POSTGRES_USER",
+    "ESPRESSO_SEQUENCER_POSTGRES_USE_TLS",
+    "ESPRESSO_SEQUENCER_POSTGRES_PRUNE",
+    "ESPRESSO_SEQUENCER_PRUNER_PRUNING_THRESHOLD",
+    "ESPRESSO_SEQUENCER_PRUNER_MINIMUM_RETENTION",
+    "ESPRESSO_SEQUENCER_PRUNER_TARGET_RETENTION",
+    "ESPRESSO_SEQUENCER_PRUNER_BATCH_SIZE",
+    "ESPRESSO_SEQUENCER_PRUNER_MAX_USAGE",
+    "ESPRESSO_SEQUENCER_PRUNER_INTERVAL",
+    "ESPRESSO_SEQUENCER_STORE_UNDECIDED_STATE",
+    "ESPRESSO_SEQUENCER_API_PORT",
+    "ESPRESSO_SEQUENCER_STORAGE_PATH",
+    "ESPRESSO_SEQUENCER_CHAIN_ID",
+    "ESPRESSO_SEQUENCER_MAX_BLOCK_SIZE",
+    "ESPRESSO_SEQUENCER_BASE_FEE",
+    "ESPRESSO_SEQUENCER_FEE_CONTRACT_PROXY_ADDRESS",
+    "ESPRESSO_SEQUENCER_FEE_RECIPIENT",
+    "ESPRESSO_CDN_BROKER_DISCOVERY_ENDPOINT",
+    "ESPRESSO_CDN_BROKER_PUBLIC_BIND_ENDPOINT",
+    "ESPRESSO_CDN_BROKER_PUBLIC_ADVERTISE_ENDPOINT",
+    "ESPRESSO_CDN_BROKER_METRICS_BIND_ENDPOINT",
+    "ESPRESSO_CDN_BROKER_CA_CERT_PATH",
+    "ESPRESSO_CDN_MARSHAL_DISCOVERY_ENDPOINT",
+    "ESPRESSO_CDN_MARSHAL_BIND_PORT",
+    "ESPRESSO_CDN_MARSHAL_METRICS_BIND_ENDPOINT",
+    "ESPRESSO_CDN_MARSHAL_CA_CERT_PATH",
+    "ESPRESSO_CDN_WHITELIST_DISCOVERY_ENDPOINT",
+    "ESPRESSO_SEQUENCER_URL",
+    "ESPRESSO_SEQUENCER_HOTSHOT_ADDRESS",
+    "ESPRESSO_COMMITMENT_TASK_PORT",
+    "ESPRESSO_COMMITMENT_TASK_REQUEST_TIMEOUT",
+    "ESPRESSO_COMMITMENT_TASK_DELAY",
+    "ESPRESSO_DEPLOYER_OUT_PATH",
+    "ESPRESSO_SEQUENCER_STAKE_TABLE_CAPACITY",
+    "ESPRESS_NASTY_CLIENT_HTTP_TIMEOUT",
+    "ESPRESSO_NASTY_CLIENT_PORT",
+    "ESPRESSO_NASTY_CLIENT_MAX_OPEN_STREAMS",
+    "ESPRESSO_NASTY_CLIENT_MAX_BLOCKING_POLLS",
+    "ESPRESSO_NASTY_CLIENT_MAX_RETRIES",
+    "ESPRESSO_NASTY_CLIENT_RETRY_DELAY",
+    "ESPRESSO_NASTY_CLIENT_MIN_RETRIES",
+    "ESPRESSO_NASTY_CLIENT_WEB_SOCKET_TIMEOUT",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_OPEN_STREAM",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_CLOSE_STREAM",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_POLL_STREAM",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY_WINDOW",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY_NAMESPACE",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY_BLOCK_STATE",
+    "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY_FEE_STATE",
+    "ESPRESSO_ORCHESTRATOR_PORT",
+    "ESPRESSO_ORCHESTRATOR_NUM_NODES",
+    "ESPRESSO_ORCHESTRATOR_START_DELAY",
+    "ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT",
+    "ESPRESSO_ORCHESTRATOR_TIMEOUT_RATIO",
+    "ESPRESSO_ORCHESTRATOR_START_THRESHOLD",
+    "ESPRESSO_ORCHESTRATOR_ROUND_START_DELAY",
+    "ESPRESSO_ORCHESTRATOR_LIBP2P_MESH_N",
+    "ESPRESSO_ORCHESTRATOR_BUILDER_URL",
+    "ESPRESSO_ORCHESTRATOR_BUILDER_TIMEOUT",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_MIN_SIZE",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_MAX_SIZE",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_MIN_BATCH_SIZE",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_MAX_BATCH_SIZE",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_MIN_NAMESPACE",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_MAX_NAMESPACE",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_DELAY",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_CHANNEL_BOUND",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_JOBS",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_PENDING_TRANSACTIONS_WARNING_THRESHOLD",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_SLOW_TRANSACTION_WARNING_THRESHOLD",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_PORT",
+    "ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL",
+    "ESPRESSO_PROVIDER",
+    "CONTRACT_ADDRESS",
+    "AMOUNT",
+    "CONFIRMATIONS",
+    "ADDRESS",
+    "BLOCK",
+    "FROM",
+    "TO"
+]

sequencer/src/api.rs

@@ -1,10 +1,10 @@
-use self::data_source::StateSignatureDataSource;
+use self::data_source::{HotShotConfigDataSource, PublicHotShotConfig, StateSignatureDataSource};
 use crate::{
     network,
     persistence::SequencerPersistence,
     state::{BlockMerkleTree, FeeAccountProof},
     state_signature::StateSigner,
-    Node, NodeState, SeqTypes, SequencerContext, Transaction,
+    Node, NodeState, PubKey, SeqTypes, SequencerContext, Transaction,
 };
 use anyhow::Context;
 use async_once_cell::Lazy;
@@ -20,7 +20,7 @@ use futures::{
 use hotshot::types::{Event, SystemContextHandle};
 use hotshot_events_service::events_source::{BuilderEvent, EventsSource, EventsStreamer};
 use hotshot_query_service::data_source::ExtensibleDataSource;
-use hotshot_types::{data::ViewNumber, light_client::StateSignatureRequestBody};
+use hotshot_types::{data::ViewNumber, light_client::StateSignatureRequestBody, HotShotConfig};
 use jf_merkle_tree::MerkleTreeScheme;
 use serde::{Deserialize, Serialize};
 use std::pin::Pin;
@@ -117,6 +117,18 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
     async fn node_state(&self) -> &NodeState {
         &self.consensus.as_ref().get().await.get_ref().node_state
     }
+
+    async fn hotshot_config(&self) -> HotShotConfig<PubKey> {
+        self.consensus
+            .as_ref()
+            .get()
+            .await
+            .get_ref()
+            .handle
+            .hotshot
+            .config
+            .clone()
+    }
 }
 
 type StorageState<N, P, D, Ver> = ExtensibleDataSource<D, ApiState<N, P, Ver>>;
@@ -229,6 +241,22 @@ impl<N: network::Type, Ver: StaticVersionType + 'static, P: SequencerPersistence
     }
 }
 
+impl<N: network::Type, D: Sync, Ver: StaticVersionType + 'static, P: SequencerPersistence>
+    HotShotConfigDataSource for StorageState<N, P, D, Ver>
+{
+    async fn get_config(&self) -> PublicHotShotConfig {
+        self.as_ref().hotshot_config().await.into()
+    }
+}
+
+impl<N: network::Type, Ver: StaticVersionType + 'static, P: SequencerPersistence>
+    HotShotConfigDataSource for ApiState<N, P, Ver>
+{
+    async fn get_config(&self) -> PublicHotShotConfig {
+        self.hotshot_config().await.into()
+    }
+}
+
 #[async_trait]
 impl<N: network::Type, D: Sync, Ver: StaticVersionType + 'static, P: SequencerPersistence>
     StateSignatureDataSource<N> for StorageState<N, P, D, Ver>

sequencer/src/api/data_source.rs

@@ -1,3 +1,5 @@
+use std::{num::NonZeroUsize, time::Duration};
+
 use super::{
     fs,
     options::{Options, Query},
@@ -6,7 +8,7 @@ use super::{
 use crate::{
     network,
     persistence::{self, SequencerPersistence},
-    SeqTypes, Transaction,
+    PubKey, SeqTypes, Transaction,
 };
 use anyhow::bail;
 use async_trait::async_trait;
@@ -19,7 +21,12 @@ use hotshot_query_service::{
     node::NodeDataSource,
     status::StatusDataSource,
 };
-use hotshot_types::{data::ViewNumber, light_client::StateSignatureRequestBody};
+use hotshot_types::{
+    data::ViewNumber, light_client::StateSignatureRequestBody, ExecutionType, HotShotConfig,
+    PeerConfig, ValidatorConfig,
+};
+
+use serde::Serialize;
 use tide_disco::Url;
 use vbs::version::StaticVersionType;
 
@@ -84,6 +91,10 @@ pub(crate) trait SubmitDataSource<N: network::Type, P: SequencerPersistence> {
     fn submit(&self, tx: Transaction) -> impl Send + Future<Output = anyhow::Result<()>>;
 }
 
+pub(crate) trait HotShotConfigDataSource {
+    fn get_config(&self) -> impl Send + Future<Output = PublicHotShotConfig>;
+}
+
 #[async_trait]
 pub(crate) trait StateSignatureDataSource<N: network::Type> {
     async fn get_state_signature(&self, height: u64) -> Option<StateSignatureRequestBody>;
@@ -134,6 +145,122 @@ pub(crate) trait CatchupDataSource {
 
 impl CatchupDataSource for MetricsDataSource {}
 
+/// This struct defines the public Hotshot validator configuration.
+/// Private key and state key pairs are excluded for security reasons.
+
+#[derive(Debug, Serialize)]
+pub struct PublicValidatorConfig {
+    pub public_key: PubKey,
+    pub stake_value: u64,
+    pub is_da: bool,
+    pub private_key: &'static str,
+    pub state_public_key: String,
+    pub state_key_pair: &'static str,
+}
+
+impl From<ValidatorConfig<PubKey>> for PublicValidatorConfig {
+    fn from(v: ValidatorConfig<PubKey>) -> Self {
+        let ValidatorConfig::<PubKey> {
+            public_key,
+            private_key: _,
+            stake_value,
+            state_key_pair,
+            is_da,
+        } = v;
+
+        let state_public_key = state_key_pair.ver_key();
+
+        Self {
+            public_key,
+            stake_value,
+            is_da,
+            state_public_key: state_public_key.to_string(),
+            private_key: "*****",
+            state_key_pair: "*****",
+        }
+    }
+}
+
+/// This struct defines the public Hotshot configuration parameters.
+/// Our config module features a GET endpoint accessible via the route `/hotshot` to display the hotshot config parameters.
+/// Hotshot config has sensitive information like private keys and such fields are excluded from this struct.
+#[derive(Debug, Serialize)]
+pub struct PublicHotShotConfig {
+    pub execution_type: ExecutionType,
+    pub start_threshold: (u64, u64),
+    pub num_nodes_with_stake: NonZeroUsize,
+    pub num_nodes_without_stake: usize,
+    pub known_nodes_with_stake: Vec<PeerConfig<PubKey>>,
+    pub known_da_nodes: Vec<PeerConfig<PubKey>>,
+    pub known_nodes_without_stake: Vec<PubKey>,
+    pub my_own_validator_config: PublicValidatorConfig,
+    pub da_staked_committee_size: usize,
+    pub da_non_staked_committee_size: usize,
+    pub fixed_leader_for_gpuvid: usize,
+    pub next_view_timeout: u64,
+    pub view_sync_timeout: Duration,
+    pub timeout_ratio: (u64, u64),
+    pub round_start_delay: u64,
+    pub start_delay: u64,
+    pub num_bootstrap: usize,
+    pub builder_timeout: Duration,
+    pub data_request_delay: Duration,
+    pub builder_url: Url,
+}
+
+impl From<HotShotConfig<PubKey>> for PublicHotShotConfig {
+    fn from(v: HotShotConfig<PubKey>) -> Self {
+        // Destructure all fields from HotShotConfig to return an error
+        // if new fields are added to HotShotConfig. This makes sure that we handle
+        // all fields appropriately and do not miss any updates.
+        let HotShotConfig::<PubKey> {
+            execution_type,
+            start_threshold,
+            num_nodes_with_stake,
+            num_nodes_without_stake,
+            known_nodes_with_stake,
+            known_da_nodes,
+            known_nodes_without_stake,
+            my_own_validator_config,
+            da_staked_committee_size,
+            da_non_staked_committee_size,
+            fixed_leader_for_gpuvid,
+            next_view_timeout,
+            view_sync_timeout,
+            timeout_ratio,
+            round_start_delay,
+            start_delay,
+            num_bootstrap,
+            builder_timeout,
+            data_request_delay,
+            builder_url,
+        } = v;
+
+        Self {
+            execution_type,
+            start_threshold,
+            num_nodes_with_stake,
+            num_nodes_without_stake,
+            known_nodes_with_stake,
+            known_da_nodes,
+            known_nodes_without_stake,
+            my_own_validator_config: my_own_validator_config.into(),
+            da_staked_committee_size,
+            da_non_staked_committee_size,
+            fixed_leader_for_gpuvid,
+            next_view_timeout,
+            view_sync_timeout,
+            timeout_ratio,
+            round_start_delay,
+            start_delay,
+            num_bootstrap,
+            builder_timeout,
+            data_request_delay,
+            builder_url,
+        }
+    }
+}
+
 #[cfg(test)]
 pub(crate) mod testing {
     use super::super::Options;