From 5995b796c00463f218dcb9426e2db26b61eb64bd Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Tue, 10 Sep 2024 08:35:55 -0700 Subject: [PATCH 1/4] Introduce download config type --- chain/indexer/src/lib.rs | 6 ++- nearcore/src/config.rs | 48 ++++++++++++++++---- neard/src/cli.rs | 10 ++-- runtime/runtime-params-estimator/src/main.rs | 2 +- tools/chainsync-loadtest/src/main.rs | 3 +- tools/indexer/example/src/configs.rs | 9 +++- 6 files changed, 61 insertions(+), 17 deletions(-) diff --git a/chain/indexer/src/lib.rs b/chain/indexer/src/lib.rs index 75cb18e13c8..cd3275fef0f 100644 --- a/chain/indexer/src/lib.rs +++ b/chain/indexer/src/lib.rs @@ -1,6 +1,8 @@ #![doc = include_str!("../README.md")] use anyhow::Context; +use nearcore::config::DownloadConfigType; +use std::str::FromStr; use tokio::sync::mpsc; use near_chain_configs::GenesisValidationMode; @@ -43,7 +45,7 @@ pub struct InitConfigArgs { /// Specify a custom download URL for the records file. pub download_records_url: Option, /// Download the verified NEAR config file automatically. - pub download_config: bool, + pub download_config: Option, /// Specify a custom download URL for the config file. pub download_config_url: Option, /// Specify the boot nodes to bootstrap the network @@ -168,7 +170,7 @@ pub fn indexer_init_configs( params.download_genesis, params.download_genesis_url.as_deref(), params.download_records_url.as_deref(), - params.download_config, + params.download_config.map(|c| DownloadConfigType::from_str(c.as_str()).unwrap()), params.download_config_url.as_deref(), params.boot_nodes.as_deref(), params.max_gas_burnt_view, diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index 466ed046af8..d3a57a28bf6 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -792,6 +792,36 @@ fn set_block_production_delay(chain_id: &str, fast: bool, config: &mut Config) { } } +/// Type of the configuration to download. +pub enum DownloadConfigType { + Validator, + Rpc, + Archival, +} + +impl ToString for DownloadConfigType { + fn to_string(&self) -> String { + match self { + DownloadConfigType::Validator => "validator".to_string(), + DownloadConfigType::Rpc => "rpc".to_string(), + DownloadConfigType::Archival => "archival".to_string(), + } + } +} + +impl FromStr for DownloadConfigType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s.trim().to_lowercase().as_str() { + "validator" => Ok(DownloadConfigType::Validator), + "rpc" => Ok(DownloadConfigType::Rpc), + "archival" => Ok(DownloadConfigType::Archival), + _ => anyhow::bail!("Please use one of the following values for download config: validator, rpc, archival"), + } + } +} + /// Initializes Genesis, client Config, node and validator keys, and stores in the specified folder. /// /// This method supports the following use cases: @@ -811,7 +841,7 @@ pub fn init_configs( should_download_genesis: bool, download_genesis_url: Option<&str>, download_records_url: Option<&str>, - should_download_config: bool, + download_config_type: Option, download_config_url: Option<&str>, boot_nodes: Option<&str>, max_gas_burnt_view: Option, @@ -853,8 +883,8 @@ pub fn init_configs( download_config(url, &dir.join(CONFIG_FILENAME)) .context(format!("Failed to download the config file from {}", url))?; config = Config::from_file(&dir.join(CONFIG_FILENAME))?; - } else if should_download_config { - let url = get_config_url(&chain_id); + } else if let Some(config_type) = download_config_type { + let url = get_config_url(&chain_id, config_type); download_config(&url, &dir.join(CONFIG_FILENAME)) .context(format!("Failed to download the config file from {}", url))?; config = Config::from_file(&dir.join(CONFIG_FILENAME))?; @@ -1327,10 +1357,10 @@ pub fn get_records_url(chain_id: &str) -> String { ) } -pub fn get_config_url(chain_id: &str) -> String { +pub fn get_config_url(chain_id: &str, config_type: DownloadConfigType) -> String { format!( - "https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/{}/config.json", - chain_id, + "https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/{}/{}/config.json", + chain_id, config_type.to_string() ) } @@ -1543,7 +1573,7 @@ mod tests { false, None, None, - false, + None, None, None, None, @@ -1601,7 +1631,7 @@ mod tests { false, None, None, - false, + None, None, None, None, @@ -1634,7 +1664,7 @@ mod tests { false, None, None, - false, + None, None, None, None, diff --git a/neard/src/cli.rs b/neard/src/cli.rs index 6fdeaf2028b..eb5660c3822 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -27,11 +27,13 @@ use near_state_viewer::StateViewerSubCommand; use near_store::db::RocksDB; use near_store::Mode; use near_undo_block::cli::UndoBlockCommand; +use nearcore::config::DownloadConfigType; use serde_json::Value; use std::fs::File; use std::io::BufReader; use std::net::SocketAddr; use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::broadcast; use tokio::sync::broadcast::Receiver; @@ -259,8 +261,10 @@ pub(super) struct InitCmd { #[clap(long)] download_genesis: bool, /// Download the verified NEAR config file automatically. - #[clap(long)] - download_config: bool, + /// Can be one of "validator", "rpc", "archival". + /// If not specified, defaults to "validator". + #[clap(long, default_missing_value = "validator")] + download_config: Option, /// Makes block production fast (TESTING ONLY). #[clap(long)] fast: bool, @@ -358,7 +362,7 @@ impl InitCmd { self.download_genesis, self.download_genesis_url.as_deref(), self.download_records_url.as_deref(), - self.download_config, + self.download_config.map(|c| DownloadConfigType::from_str(c.as_str()).unwrap()), self.download_config_url.as_deref(), self.boot_nodes.as_deref(), self.max_gas_burnt_view, diff --git a/runtime/runtime-params-estimator/src/main.rs b/runtime/runtime-params-estimator/src/main.rs index 4b2666d953c..00314beaede 100644 --- a/runtime/runtime-params-estimator/src/main.rs +++ b/runtime/runtime-params-estimator/src/main.rs @@ -177,7 +177,7 @@ fn run_estimation(cli_args: CliArgs) -> anyhow::Result> { false, None, None, - false, + None, None, None, None, diff --git a/tools/chainsync-loadtest/src/main.rs b/tools/chainsync-loadtest/src/main.rs index c47e642767a..9a42a86ceda 100644 --- a/tools/chainsync-loadtest/src/main.rs +++ b/tools/chainsync-loadtest/src/main.rs @@ -18,6 +18,7 @@ use near_o11y::tracing::{error, info}; use near_primitives::block::GenesisId; use near_primitives::hash::CryptoHash; use nearcore::config; +use nearcore::config::DownloadConfigType; use nearcore::config::NearConfig; use network::Network; use openssl_probe; @@ -59,7 +60,7 @@ pub fn start_with_config(config: NearConfig, qps_limit: u32) -> anyhow::Result anyhow::Result { // Always fetch the config. std::fs::create_dir_all(dir)?; - let url = config::get_config_url(chain_id); + let url = config::get_config_url(chain_id, DownloadConfigType::Validator); let config_path = &dir.join(config::CONFIG_FILENAME); config::download_config(&url, config_path)?; let config = config::Config::from_file(config_path)?; diff --git a/tools/indexer/example/src/configs.rs b/tools/indexer/example/src/configs.rs index f9f0e1cc24e..3a4220c820c 100644 --- a/tools/indexer/example/src/configs.rs +++ b/tools/indexer/example/src/configs.rs @@ -1,5 +1,8 @@ use near_indexer::near_primitives::types::Gas; +/// Indexer uses the RPC configuration type. +const DOWNLOAD_CONFIG_TYPE: &str = "rpc"; + /// NEAR Indexer Example /// Watches for stream of blocks from the chain #[derive(clap::Parser, Debug)] @@ -76,7 +79,11 @@ impl From for near_indexer::InitConfigArgs { download_genesis: config_args.download_genesis, download_genesis_url: config_args.download_genesis_url, download_records_url: config_args.download_records_url, - download_config: config_args.download_config, + download_config: if config_args.download_config { + Some(DOWNLOAD_CONFIG_TYPE.to_string()) + } else { + None + }, download_config_url: config_args.download_config_url, boot_nodes: config_args.boot_nodes, max_gas_burnt_view: config_args.max_gas_burnt_view, From 0ae1572861315cbf28c2887d86a4a686203bf87b Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Tue, 10 Sep 2024 09:02:52 -0700 Subject: [PATCH 2/4] Allow missing value --- chain/indexer/src/lib.rs | 7 ++++++- nearcore/src/config.rs | 7 ++++++- neard/src/cli.rs | 14 ++++++++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/chain/indexer/src/lib.rs b/chain/indexer/src/lib.rs index cd3275fef0f..a309eb9288c 100644 --- a/chain/indexer/src/lib.rs +++ b/chain/indexer/src/lib.rs @@ -159,6 +159,11 @@ pub fn indexer_init_configs( dir: &std::path::PathBuf, params: InitConfigArgs, ) -> Result<(), anyhow::Error> { + let download_config_type = if let Some(config_type) = params.download_config.as_deref() { + Some(DownloadConfigType::from_str(config_type)?) + } else { + None + }; init_configs( dir, params.chain_id, @@ -170,7 +175,7 @@ pub fn indexer_init_configs( params.download_genesis, params.download_genesis_url.as_deref(), params.download_records_url.as_deref(), - params.download_config.map(|c| DownloadConfigType::from_str(c.as_str()).unwrap()), + download_config_type, params.download_config_url.as_deref(), params.boot_nodes.as_deref(), params.max_gas_burnt_view, diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index d3a57a28bf6..ae3b3590716 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -794,8 +794,11 @@ fn set_block_production_delay(chain_id: &str, fast: bool, config: &mut Config) { /// Type of the configuration to download. pub enum DownloadConfigType { + /// Validator node configuration. Validator, + /// Non-validator RPC node configuration. Rpc, + /// Non-validator archival node configuration. Archival, } @@ -817,7 +820,9 @@ impl FromStr for DownloadConfigType { "validator" => Ok(DownloadConfigType::Validator), "rpc" => Ok(DownloadConfigType::Rpc), "archival" => Ok(DownloadConfigType::Archival), - _ => anyhow::bail!("Please use one of the following values for download config: validator, rpc, archival"), + _ => anyhow::bail!( + "Flag download_config must be one of the following: validator, rpc, archival" + ), } } } diff --git a/neard/src/cli.rs b/neard/src/cli.rs index eb5660c3822..af1bf12acb0 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -261,9 +261,9 @@ pub(super) struct InitCmd { #[clap(long)] download_genesis: bool, /// Download the verified NEAR config file automatically. - /// Can be one of "validator", "rpc", "archival". - /// If not specified, defaults to "validator". - #[clap(long, default_missing_value = "validator")] + /// Can be one of "validator", "rpc", and "archival". + /// If flag is present with no value, defaults to "validator". + #[clap(long, default_missing_value = "validator", num_args(0..=1))] download_config: Option, /// Makes block production fast (TESTING ONLY). #[clap(long)] @@ -351,6 +351,12 @@ impl InitCmd { check_release_build(chain) } + let download_config_type = if let Some(config_type) = self.download_config.as_deref() { + Some(DownloadConfigType::from_str(config_type)?) + } else { + None + }; + nearcore::init_configs( home_dir, self.chain_id, @@ -362,7 +368,7 @@ impl InitCmd { self.download_genesis, self.download_genesis_url.as_deref(), self.download_records_url.as_deref(), - self.download_config.map(|c| DownloadConfigType::from_str(c.as_str()).unwrap()), + download_config_type, self.download_config_url.as_deref(), self.boot_nodes.as_deref(), self.max_gas_burnt_view, From 8bb838e12ef22893ec3b18ff69654a5bbe55fa50 Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Tue, 10 Sep 2024 09:14:37 -0700 Subject: [PATCH 3/4] Use RPC config for chain-sync tool --- tools/chainsync-loadtest/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/chainsync-loadtest/src/main.rs b/tools/chainsync-loadtest/src/main.rs index 9a42a86ceda..8fe10406a2a 100644 --- a/tools/chainsync-loadtest/src/main.rs +++ b/tools/chainsync-loadtest/src/main.rs @@ -60,7 +60,7 @@ pub fn start_with_config(config: NearConfig, qps_limit: u32) -> anyhow::Result anyhow::Result { // Always fetch the config. std::fs::create_dir_all(dir)?; - let url = config::get_config_url(chain_id, DownloadConfigType::Validator); + let url = config::get_config_url(chain_id, DownloadConfigType::Rpc); let config_path = &dir.join(config::CONFIG_FILENAME); config::download_config(&url, config_path)?; let config = config::Config::from_file(config_path)?; From 511de971dff067c8fceb85786c43415a294ca76e Mon Sep 17 00:00:00 2001 From: Tayfun Elmas Date: Wed, 11 Sep 2024 08:56:14 -0700 Subject: [PATCH 4/4] Address comments, move DownloadConfigType to near-config-utils. --- Cargo.lock | 3 +++ chain/indexer/Cargo.toml | 1 + chain/indexer/src/lib.rs | 12 +++------ nearcore/src/config.rs | 37 +------------------------- neard/src/cli.rs | 2 +- tools/chainsync-loadtest/Cargo.toml | 1 + tools/chainsync-loadtest/src/main.rs | 3 +-- tools/indexer/example/Cargo.toml | 1 + tools/indexer/example/src/configs.rs | 5 +--- utils/config/src/lib.rs | 39 +++++++++++++++++++++++++++- 10 files changed, 51 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4a654e6042..7f72d48d5b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1332,6 +1332,7 @@ dependencies = [ "log", "near-async", "near-chain-configs", + "near-config-utils", "near-crypto", "near-network", "near-o11y", @@ -3198,6 +3199,7 @@ dependencies = [ "actix", "anyhow", "clap", + "near-config-utils", "near-indexer", "near-o11y", "openssl-probe", @@ -4360,6 +4362,7 @@ dependencies = [ "futures", "near-chain-configs", "near-client", + "near-config-utils", "near-crypto", "near-dyn-configs", "near-indexer-primitives", diff --git a/chain/indexer/Cargo.toml b/chain/indexer/Cargo.toml index 9d74dc66fef..1215cfa2081 100644 --- a/chain/indexer/Cargo.toml +++ b/chain/indexer/Cargo.toml @@ -24,6 +24,7 @@ tracing.workspace = true nearcore.workspace = true near-client.workspace = true near-chain-configs.workspace = true +near-config-utils.workspace = true near-dyn-configs.workspace = true near-crypto.workspace = true near-indexer-primitives.workspace = true diff --git a/chain/indexer/src/lib.rs b/chain/indexer/src/lib.rs index a309eb9288c..bb6bd87c4b7 100644 --- a/chain/indexer/src/lib.rs +++ b/chain/indexer/src/lib.rs @@ -1,8 +1,7 @@ #![doc = include_str!("../README.md")] use anyhow::Context; -use nearcore::config::DownloadConfigType; -use std::str::FromStr; +use near_config_utils::DownloadConfigType; use tokio::sync::mpsc; use near_chain_configs::GenesisValidationMode; @@ -45,7 +44,7 @@ pub struct InitConfigArgs { /// Specify a custom download URL for the records file. pub download_records_url: Option, /// Download the verified NEAR config file automatically. - pub download_config: Option, + pub download_config: Option, /// Specify a custom download URL for the config file. pub download_config_url: Option, /// Specify the boot nodes to bootstrap the network @@ -159,11 +158,6 @@ pub fn indexer_init_configs( dir: &std::path::PathBuf, params: InitConfigArgs, ) -> Result<(), anyhow::Error> { - let download_config_type = if let Some(config_type) = params.download_config.as_deref() { - Some(DownloadConfigType::from_str(config_type)?) - } else { - None - }; init_configs( dir, params.chain_id, @@ -175,7 +169,7 @@ pub fn indexer_init_configs( params.download_genesis, params.download_genesis_url.as_deref(), params.download_records_url.as_deref(), - download_config_type, + params.download_config, params.download_config_url.as_deref(), params.boot_nodes.as_deref(), params.max_gas_burnt_view, diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index f3b6464fa21..91dfc0a31ca 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -28,7 +28,7 @@ use near_chain_configs::{ NUM_BLOCK_PRODUCER_SEATS, PROTOCOL_REWARD_RATE, PROTOCOL_UPGRADE_STAKE_THRESHOLD, TRANSACTION_VALIDITY_PERIOD, }; -use near_config_utils::{ValidationError, ValidationErrors}; +use near_config_utils::{DownloadConfigType, ValidationError, ValidationErrors}; use near_crypto::{InMemorySigner, KeyFile, KeyType, PublicKey}; use near_epoch_manager::EpochManagerHandle; #[cfg(feature = "json_rpc")] @@ -792,41 +792,6 @@ fn set_block_production_delay(chain_id: &str, fast: bool, config: &mut Config) { } } -/// Type of the configuration to download. -pub enum DownloadConfigType { - /// Validator node configuration. - Validator, - /// Non-validator RPC node configuration. - Rpc, - /// Non-validator archival node configuration. - Archival, -} - -impl ToString for DownloadConfigType { - fn to_string(&self) -> String { - match self { - DownloadConfigType::Validator => "validator".to_string(), - DownloadConfigType::Rpc => "rpc".to_string(), - DownloadConfigType::Archival => "archival".to_string(), - } - } -} - -impl FromStr for DownloadConfigType { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s.trim().to_lowercase().as_str() { - "validator" => Ok(DownloadConfigType::Validator), - "rpc" => Ok(DownloadConfigType::Rpc), - "archival" => Ok(DownloadConfigType::Archival), - _ => anyhow::bail!( - "Flag download_config must be one of the following: validator, rpc, archival" - ), - } - } -} - /// Initializes Genesis, client Config, node and validator keys, and stores in the specified folder. /// /// This method supports the following use cases: diff --git a/neard/src/cli.rs b/neard/src/cli.rs index 66da39749f2..b3436d99ed6 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -4,6 +4,7 @@ use near_amend_genesis::AmendGenesisCommand; use near_chain_configs::GenesisValidationMode; use near_client::ConfigUpdater; use near_cold_store_tool::ColdStoreCommand; +use near_config_utils::DownloadConfigType; use near_database_tool::commands::DatabaseCommand; use near_dyn_configs::{UpdateableConfigLoader, UpdateableConfigLoaderError, UpdateableConfigs}; use near_flat_storage::commands::FlatStorageCommand; @@ -27,7 +28,6 @@ use near_state_viewer::StateViewerSubCommand; use near_store::db::RocksDB; use near_store::Mode; use near_undo_block::cli::UndoBlockCommand; -use nearcore::config::DownloadConfigType; use serde_json::Value; use std::fs::File; use std::io::BufReader; diff --git a/tools/chainsync-loadtest/Cargo.toml b/tools/chainsync-loadtest/Cargo.toml index e192bf14434..53732875197 100644 --- a/tools/chainsync-loadtest/Cargo.toml +++ b/tools/chainsync-loadtest/Cargo.toml @@ -31,6 +31,7 @@ tokio.workspace = true near-async.workspace = true near-chain-configs.workspace = true +near-config-utils.workspace = true near-crypto.workspace = true near-primitives.workspace = true near-store.workspace = true diff --git a/tools/chainsync-loadtest/src/main.rs b/tools/chainsync-loadtest/src/main.rs index 8fe10406a2a..64e274f6e39 100644 --- a/tools/chainsync-loadtest/src/main.rs +++ b/tools/chainsync-loadtest/src/main.rs @@ -18,7 +18,6 @@ use near_o11y::tracing::{error, info}; use near_primitives::block::GenesisId; use near_primitives::hash::CryptoHash; use nearcore::config; -use nearcore::config::DownloadConfigType; use nearcore::config::NearConfig; use network::Network; use openssl_probe; @@ -60,7 +59,7 @@ pub fn start_with_config(config: NearConfig, qps_limit: u32) -> anyhow::Result anyhow::Result { // Always fetch the config. std::fs::create_dir_all(dir)?; - let url = config::get_config_url(chain_id, DownloadConfigType::Rpc); + let url = config::get_config_url(chain_id, near_config_utils::DownloadConfigType::RPC); let config_path = &dir.join(config::CONFIG_FILENAME); config::download_config(&url, config_path)?; let config = config::Config::from_file(config_path)?; diff --git a/tools/indexer/example/Cargo.toml b/tools/indexer/example/Cargo.toml index 72e71654b6d..2801855ae3f 100644 --- a/tools/indexer/example/Cargo.toml +++ b/tools/indexer/example/Cargo.toml @@ -20,5 +20,6 @@ serde_json.workspace = true tokio.workspace = true tracing.workspace = true +near-config-utils.workspace = true near-indexer.workspace = true near-o11y.workspace = true diff --git a/tools/indexer/example/src/configs.rs b/tools/indexer/example/src/configs.rs index 3a4220c820c..f251c06215b 100644 --- a/tools/indexer/example/src/configs.rs +++ b/tools/indexer/example/src/configs.rs @@ -1,8 +1,5 @@ use near_indexer::near_primitives::types::Gas; -/// Indexer uses the RPC configuration type. -const DOWNLOAD_CONFIG_TYPE: &str = "rpc"; - /// NEAR Indexer Example /// Watches for stream of blocks from the chain #[derive(clap::Parser, Debug)] @@ -80,7 +77,7 @@ impl From for near_indexer::InitConfigArgs { download_genesis_url: config_args.download_genesis_url, download_records_url: config_args.download_records_url, download_config: if config_args.download_config { - Some(DOWNLOAD_CONFIG_TYPE.to_string()) + Some(near_config_utils::DownloadConfigType::RPC) } else { None }, diff --git a/utils/config/src/lib.rs b/utils/config/src/lib.rs index a8f15c61c5a..b74949e1e80 100644 --- a/utils/config/src/lib.rs +++ b/utils/config/src/lib.rs @@ -1,4 +1,4 @@ -use std::io::Read; +use std::{io::Read, str::FromStr}; use json_comments::StripComments; @@ -134,3 +134,40 @@ impl ValidationErrors { } } } + +/// Type of the configuration to download. +/// Currently used for downloading the `config.json` file for different node types. +#[derive(Debug, Clone)] +pub enum DownloadConfigType { + /// Validator node configuration. + Validator, + /// Non-validator RPC node configuration. + RPC, + /// Non-validator archival node configuration. + Archival, +} + +impl ToString for DownloadConfigType { + fn to_string(&self) -> String { + match self { + DownloadConfigType::Validator => "validator".to_string(), + DownloadConfigType::RPC => "rpc".to_string(), + DownloadConfigType::Archival => "archival".to_string(), + } + } +} + +impl FromStr for DownloadConfigType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s.trim().to_lowercase().as_str() { + "validator" => Ok(DownloadConfigType::Validator), + "rpc" => Ok(DownloadConfigType::RPC), + "archival" => Ok(DownloadConfigType::Archival), + _ => anyhow::bail!( + "Flag download_config must be one of the following: validator, rpc, archival" + ), + } + } +}