From 2f88282e915904109f2df36ea397835aa40e5a9b Mon Sep 17 00:00:00 2001 From: Jeremy Rubin Date: Fri, 19 Apr 2024 08:45:07 -0400 Subject: [PATCH 1/3] Allow higher rpcworkqueue limit conf --- src/index/updater.rs | 6 +++--- src/options.rs | 3 +++ src/settings.rs | 12 ++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/index/updater.rs b/src/index/updater.rs index 3d446f2eb0..a511f89607 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -250,7 +250,7 @@ impl<'index> Updater<'index> { // Default rpcworkqueue in bitcoind is 16, meaning more than 16 concurrent requests will be rejected. // Since we are already requesting blocks on a separate thread, and we don't want to break if anything // else runs a request, we keep this to 12. - const PARALLEL_REQUESTS: usize = 12; + let parallel_requests: usize = settings.bitcoin_rpc_limit() as usize; thread::spawn(move || { let rt = tokio::runtime::Builder::new_multi_thread() @@ -273,8 +273,8 @@ impl<'index> Updater<'index> { outpoints.push(outpoint); } // Break outpoints into chunks for parallel requests - let chunk_size = (outpoints.len() / PARALLEL_REQUESTS) + 1; - let mut futs = Vec::with_capacity(PARALLEL_REQUESTS); + let chunk_size = (outpoints.len() / parallel_requests) + 1; + let mut futs = Vec::with_capacity(parallel_requests); for chunk in outpoints.chunks(chunk_size) { let txids = chunk.iter().map(|outpoint| outpoint.txid).collect(); let fut = fetcher.get_transactions(txids); diff --git a/src/options.rs b/src/options.rs index b77002d6d6..0f1b7290f9 100644 --- a/src/options.rs +++ b/src/options.rs @@ -21,6 +21,9 @@ pub struct Options { help = "Authenticate to Bitcoin Core RPC as ." )] pub(crate) bitcoin_rpc_username: Option, + + #[arg(long, help = "Max requests in flight. [default: 12]")] + pub(crate) bitcoin_rpc_limit: Option, #[arg(long = "chain", value_enum, help = "Use . [default: mainnet]")] pub(crate) chain_argument: Option, #[arg( diff --git a/src/settings.rs b/src/settings.rs index a0a487002a..1110c256d3 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -7,6 +7,7 @@ pub struct Settings { bitcoin_rpc_password: Option, bitcoin_rpc_url: Option, bitcoin_rpc_username: Option, + bitcoin_rpc_limit: Option, chain: Option, commit_interval: Option, config: Option, @@ -111,6 +112,7 @@ impl Settings { bitcoin_rpc_password: self.bitcoin_rpc_password.or(source.bitcoin_rpc_password), bitcoin_rpc_url: self.bitcoin_rpc_url.or(source.bitcoin_rpc_url), bitcoin_rpc_username: self.bitcoin_rpc_username.or(source.bitcoin_rpc_username), + bitcoin_rpc_limit: self.bitcoin_rpc_limit.or(source.bitcoin_rpc_limit), chain: self.chain.or(source.chain), commit_interval: self.commit_interval.or(source.commit_interval), config: self.config.or(source.config), @@ -150,6 +152,7 @@ impl Settings { bitcoin_rpc_password: options.bitcoin_rpc_password, bitcoin_rpc_url: options.bitcoin_rpc_url, bitcoin_rpc_username: options.bitcoin_rpc_username, + bitcoin_rpc_limit: options.bitcoin_rpc_limit, chain: options .signet .then_some(Chain::Signet) @@ -233,6 +236,7 @@ impl Settings { bitcoin_rpc_password: get_string("BITCOIN_RPC_PASSWORD"), bitcoin_rpc_url: get_string("BITCOIN_RPC_URL"), bitcoin_rpc_username: get_string("BITCOIN_RPC_USERNAME"), + bitcoin_rpc_limit: get_u32("BITCOIN_RPC_LIMIT")?, chain: get_chain("CHAIN")?, commit_interval: get_usize("COMMIT_INTERVAL")?, config: get_path("CONFIG"), @@ -262,6 +266,7 @@ impl Settings { bitcoin_rpc_password: None, bitcoin_rpc_url: Some(rpc_url.into()), bitcoin_rpc_username: None, + bitcoin_rpc_limit: None, chain: Some(Chain::Regtest), commit_interval: None, config: None, @@ -328,6 +333,7 @@ impl Settings { .unwrap_or_else(|| format!("127.0.0.1:{}", chain.default_rpc_port())), ), bitcoin_rpc_username: self.bitcoin_rpc_username, + bitcoin_rpc_limit: Some(self.bitcoin_rpc_limit.unwrap_or(12)), chain: Some(chain), commit_interval: Some(self.commit_interval.unwrap_or(5000)), config: None, @@ -550,6 +556,10 @@ impl Settings { } } + pub(crate) fn bitcoin_rpc_limit(&self) -> u32 { + self.bitcoin_rpc_limit.unwrap() + } + pub(crate) fn server_url(&self) -> Option<&str> { self.server_url.as_deref() } @@ -1013,6 +1023,7 @@ mod tests { bitcoin_rpc_password: Some("bitcoin password".into()), bitcoin_rpc_url: Some("url".into()), bitcoin_rpc_username: Some("bitcoin username".into()), + bitcoin_rpc_limit: Some(12), chain: Some(Chain::Signet), commit_interval: Some(1), config: Some("config".into()), @@ -1084,6 +1095,7 @@ mod tests { bitcoin_rpc_password: Some("bitcoin password".into()), bitcoin_rpc_url: Some("url".into()), bitcoin_rpc_username: Some("bitcoin username".into()), + bitcoin_rpc_limit: Some(12), chain: Some(Chain::Signet), commit_interval: Some(1), config: Some("config".into()), From ec6ef58b8d7776c5d42a0b7c3c1921ba4335fb45 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 6 May 2024 11:53:02 +0200 Subject: [PATCH 2/3] fix up tests --- src/settings.rs | 6 ++++-- tests/settings.rs | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/settings.rs b/src/settings.rs index 1110c256d3..cfd05cedb9 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -4,10 +4,10 @@ use {super::*, bitcoincore_rpc::Auth}; #[serde(default, deny_unknown_fields)] pub struct Settings { bitcoin_data_dir: Option, + bitcoin_rpc_limit: Option, bitcoin_rpc_password: Option, bitcoin_rpc_url: Option, bitcoin_rpc_username: Option, - bitcoin_rpc_limit: Option, chain: Option, commit_interval: Option, config: Option, @@ -989,6 +989,7 @@ mod tests { let env = vec![ ("BITCOIN_DATA_DIR", "/bitcoin/data/dir"), ("BITCOIN_RPC_PASSWORD", "bitcoin password"), + ("BITCOIN_RPC_LIMIT", "12"), ("BITCOIN_RPC_URL", "url"), ("BITCOIN_RPC_USERNAME", "bitcoin username"), ("CHAIN", "signet"), @@ -1066,6 +1067,7 @@ mod tests { Options::try_parse_from([ "ord", "--bitcoin-data-dir=/bitcoin/data/dir", + "--bitcoin-rpc-limit=12", "--bitcoin-rpc-password=bitcoin password", "--bitcoin-rpc-url=url", "--bitcoin-rpc-username=bitcoin username", @@ -1092,10 +1094,10 @@ mod tests { ), Settings { bitcoin_data_dir: Some("/bitcoin/data/dir".into()), + bitcoin_rpc_limit: Some(12), bitcoin_rpc_password: Some("bitcoin password".into()), bitcoin_rpc_url: Some("url".into()), bitcoin_rpc_username: Some("bitcoin username".into()), - bitcoin_rpc_limit: Some(12), chain: Some(Chain::Signet), commit_interval: Some(1), config: Some("config".into()), diff --git a/tests/settings.rs b/tests/settings.rs index bfd271d28b..0244259be4 100644 --- a/tests/settings.rs +++ b/tests/settings.rs @@ -7,6 +7,7 @@ fn default() { .stdout_regex( r#"\{ "bitcoin_data_dir": ".*(Bitcoin|bitcoin)", + "bitcoin_rpc_limit": 12, "bitcoin_rpc_password": null, "bitcoin_rpc_url": "127.0.0.1:8332", "bitcoin_rpc_username": null, From c9b92f8b91a0e70083d11cf538e994dd96950437 Mon Sep 17 00:00:00 2001 From: raphjaph Date: Mon, 6 May 2024 11:58:13 +0200 Subject: [PATCH 3/3] Amend --- src/options.rs | 1 - src/settings.rs | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/options.rs b/src/options.rs index 0f1b7290f9..ba76689845 100644 --- a/src/options.rs +++ b/src/options.rs @@ -21,7 +21,6 @@ pub struct Options { help = "Authenticate to Bitcoin Core RPC as ." )] pub(crate) bitcoin_rpc_username: Option, - #[arg(long, help = "Max requests in flight. [default: 12]")] pub(crate) bitcoin_rpc_limit: Option, #[arg(long = "chain", value_enum, help = "Use . [default: mainnet]")] diff --git a/src/settings.rs b/src/settings.rs index cfd05cedb9..295827200a 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -109,10 +109,10 @@ impl Settings { pub(crate) fn or(self, source: Settings) -> Self { Self { bitcoin_data_dir: self.bitcoin_data_dir.or(source.bitcoin_data_dir), + bitcoin_rpc_limit: self.bitcoin_rpc_limit.or(source.bitcoin_rpc_limit), bitcoin_rpc_password: self.bitcoin_rpc_password.or(source.bitcoin_rpc_password), bitcoin_rpc_url: self.bitcoin_rpc_url.or(source.bitcoin_rpc_url), bitcoin_rpc_username: self.bitcoin_rpc_username.or(source.bitcoin_rpc_username), - bitcoin_rpc_limit: self.bitcoin_rpc_limit.or(source.bitcoin_rpc_limit), chain: self.chain.or(source.chain), commit_interval: self.commit_interval.or(source.commit_interval), config: self.config.or(source.config), @@ -149,10 +149,10 @@ impl Settings { pub(crate) fn from_options(options: Options) -> Self { Self { bitcoin_data_dir: options.bitcoin_data_dir, + bitcoin_rpc_limit: options.bitcoin_rpc_limit, bitcoin_rpc_password: options.bitcoin_rpc_password, bitcoin_rpc_url: options.bitcoin_rpc_url, bitcoin_rpc_username: options.bitcoin_rpc_username, - bitcoin_rpc_limit: options.bitcoin_rpc_limit, chain: options .signet .then_some(Chain::Signet) @@ -233,10 +233,10 @@ impl Settings { Ok(Self { bitcoin_data_dir: get_path("BITCOIN_DATA_DIR"), + bitcoin_rpc_limit: get_u32("BITCOIN_RPC_LIMIT")?, bitcoin_rpc_password: get_string("BITCOIN_RPC_PASSWORD"), bitcoin_rpc_url: get_string("BITCOIN_RPC_URL"), bitcoin_rpc_username: get_string("BITCOIN_RPC_USERNAME"), - bitcoin_rpc_limit: get_u32("BITCOIN_RPC_LIMIT")?, chain: get_chain("CHAIN")?, commit_interval: get_usize("COMMIT_INTERVAL")?, config: get_path("CONFIG"), @@ -325,6 +325,7 @@ impl Settings { Ok(Self { bitcoin_data_dir: Some(bitcoin_data_dir), + bitcoin_rpc_limit: Some(self.bitcoin_rpc_limit.unwrap_or(12)), bitcoin_rpc_password: self.bitcoin_rpc_password, bitcoin_rpc_url: Some( self @@ -333,7 +334,6 @@ impl Settings { .unwrap_or_else(|| format!("127.0.0.1:{}", chain.default_rpc_port())), ), bitcoin_rpc_username: self.bitcoin_rpc_username, - bitcoin_rpc_limit: Some(self.bitcoin_rpc_limit.unwrap_or(12)), chain: Some(chain), commit_interval: Some(self.commit_interval.unwrap_or(5000)), config: None, @@ -988,8 +988,8 @@ mod tests { fn from_env() { let env = vec![ ("BITCOIN_DATA_DIR", "/bitcoin/data/dir"), - ("BITCOIN_RPC_PASSWORD", "bitcoin password"), ("BITCOIN_RPC_LIMIT", "12"), + ("BITCOIN_RPC_PASSWORD", "bitcoin password"), ("BITCOIN_RPC_URL", "url"), ("BITCOIN_RPC_USERNAME", "bitcoin username"), ("CHAIN", "signet"), @@ -1021,10 +1021,10 @@ mod tests { Settings::from_env(env).unwrap(), Settings { bitcoin_data_dir: Some("/bitcoin/data/dir".into()), + bitcoin_rpc_limit: Some(12), bitcoin_rpc_password: Some("bitcoin password".into()), bitcoin_rpc_url: Some("url".into()), bitcoin_rpc_username: Some("bitcoin username".into()), - bitcoin_rpc_limit: Some(12), chain: Some(Chain::Signet), commit_interval: Some(1), config: Some("config".into()),