Skip to content
This repository has been archived by the owner on Nov 6, 2020. It is now read-only.

Ensure jsonrpc threading settings are sane #11267

Merged
merged 11 commits into from
Nov 18, 2019
18 changes: 9 additions & 9 deletions parity/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -506,18 +506,18 @@ usage! {
"--jsonrpc-hosts=[HOSTS]",
"List of allowed Host header values. This option will validate the Host header sent by the browser, it is additional security against some attack vectors. Special options: \"all\", \"none\",.",

ARG arg_jsonrpc_threads: (usize) = 4usize, or |c: &Config| c.rpc.as_ref()?.processing_threads,
ARG arg_jsonrpc_threads: (Option<usize>) = Some(4), or |c: &Config| c.rpc.as_ref()?.processing_threads,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the default is Some, should it be usize then? How a user can set it to None and does it make sense? Maybe we should keep an Option and use unwrap_or(4) instead?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

None here implies "use defaults", i.e. the user didn't set a value. Either way, for this particular line it's moot: the jsonrpc-threads option does nothing and I removed it in dc0d349 – essentially it was once used to set the size of a CpuPool but since #9657 it has not had any effect at all.
The equivalent code today sets up the tokio Executor which is used for almost all async subsystems (sync, price fetch, rpcs and more) so it is proooobably not something users should be fiddling with. If the need to do so arises we should either use separate executors or introduce some other mechanism of configuring and reserving threads for each subsystem.

"--jsonrpc-threads=[THREADS]",
"Turn on additional processing threads for JSON-RPC servers (all transports). Setting this to a non-zero value allows parallel execution of cpu-heavy queries.",

ARG arg_jsonrpc_server_threads: (Option<usize>) = Some(4), or |c: &Config| c.rpc.as_ref()?.server_threads,
dvdplm marked this conversation as resolved.
Show resolved Hide resolved
"--jsonrpc-server-threads=[THREADS]",
dvdplm marked this conversation as resolved.
Show resolved Hide resolved
"Enables multiple threads handling incoming connections for HTTP JSON-RPC server.",

ARG arg_jsonrpc_cors: (String) = "none", or |c: &Config| c.rpc.as_ref()?.cors.as_ref().map(|vec| vec.join(",")),
"--jsonrpc-cors=[URL]",
"Specify CORS header for HTTP JSON-RPC API responses. Special options: \"all\", \"none\".",

ARG arg_jsonrpc_server_threads: (Option<usize>) = None, or |c: &Config| c.rpc.as_ref()?.server_threads,
"--jsonrpc-server-threads=[NUM]",
"Enables multiple threads handling incoming connections for HTTP JSON-RPC server.",

ARG arg_jsonrpc_max_payload: (Option<usize>) = None, or |c: &Config| c.rpc.as_ref()?.max_payload,
"--jsonrpc-max-payload=[MB]",
"Specify maximum size for HTTP JSON-RPC requests in megabytes.",
Expand Down Expand Up @@ -1816,8 +1816,8 @@ mod tests {
arg_jsonrpc_cors: "null".into(),
arg_jsonrpc_apis: "web3,eth,net,parity,traces,rpc,secretstore".into(),
arg_jsonrpc_hosts: "none".into(),
arg_jsonrpc_server_threads: None,
arg_jsonrpc_threads: 4,
arg_jsonrpc_server_threads: Some(4),
arg_jsonrpc_threads: Some(4),
arg_jsonrpc_max_payload: None,
arg_poll_lifetime: 60u32,
flag_jsonrpc_allow_missing_blocks: false,
Expand Down Expand Up @@ -2095,8 +2095,8 @@ mod tests {
cors: None,
apis: None,
hosts: None,
server_threads: None,
processing_threads: None,
server_threads: Some(13),
processing_threads: Some(14),
max_payload: None,
keep_alive: None,
experimental_rpcs: None,
Expand Down
2 changes: 2 additions & 0 deletions parity/cli/tests/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ origins = ["none"]
[rpc]
disable = true
port = 8180
server_threads = 13
processing_threads = 14

[ipc]
apis = ["rpc", "eth"]
Expand Down
108 changes: 66 additions & 42 deletions parity/configuration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -367,49 +367,49 @@ impl Configuration {
let (private_provider_conf, private_enc_conf, private_tx_enabled) = self.private_provider_config()?;

let run_cmd = RunCmd {
cache_config: cache_config,
dirs: dirs,
spec: spec,
pruning: pruning,
pruning_history: pruning_history,
cache_config,
dirs,
spec,
pruning,
pruning_history,
pruning_memory: self.args.arg_pruning_memory,
daemon: daemon,
daemon,
logger_config: logger_config.clone(),
miner_options: self.miner_options()?,
gas_price_percentile: self.args.arg_gas_price_percentile,
poll_lifetime: self.args.arg_poll_lifetime,
ws_conf: ws_conf,
snapshot_conf: snapshot_conf,
http_conf: http_conf,
ipc_conf: ipc_conf,
net_conf: net_conf,
network_id: network_id,
ws_conf,
snapshot_conf,
http_conf,
ipc_conf,
net_conf,
network_id,
acc_conf: self.accounts_config()?,
gas_pricer_conf: self.gas_pricer_config()?,
miner_extras: self.miner_extras()?,
stratum: self.stratum_options()?,
update_policy: update_policy,
update_policy,
allow_missing_blocks: self.args.flag_jsonrpc_allow_missing_blocks,
mode: mode,
tracing: tracing,
fat_db: fat_db,
compaction: compaction,
vm_type: vm_type,
warp_sync: warp_sync,
mode,
tracing,
fat_db,
compaction,
vm_type,
warp_sync,
warp_barrier: self.args.arg_warp_barrier,
geth_compatibility: geth_compatibility,
geth_compatibility,
experimental_rpcs,
net_settings: self.network_settings()?,
ipfs_conf: ipfs_conf,
secretstore_conf: secretstore_conf,
private_provider_conf: private_provider_conf,
ipfs_conf,
secretstore_conf,
private_provider_conf,
private_encryptor_conf: private_enc_conf,
private_tx_enabled,
name: self.args.arg_identity,
custom_bootnodes: self.args.arg_bootnodes.is_some(),
check_seal: !self.args.flag_no_seal_check,
download_old_blocks: !self.args.flag_no_ancient_blocks,
verifier_settings: verifier_settings,
verifier_settings,
serve_light: !self.args.flag_no_serve_light,
light: self.args.flag_light,
no_persistent_txqueue: self.args.flag_no_persistent_txqueue,
Expand Down Expand Up @@ -885,24 +885,23 @@ impl Configuration {
}

fn http_config(&self) -> Result<HttpConfiguration, String> {
let conf = HttpConfiguration {
enabled: self.rpc_enabled(),
interface: self.rpc_interface(),
port: self.args.arg_ports_shift + self.args.arg_rpcport.unwrap_or(self.args.arg_jsonrpc_port),
apis: self.rpc_apis().parse()?,
hosts: self.rpc_hosts(),
cors: self.rpc_cors(),
server_threads: match self.args.arg_jsonrpc_server_threads {
Some(threads) if threads > 0 => threads,
_ => 1,
},
processing_threads: self.args.arg_jsonrpc_threads,
max_payload: match self.args.arg_jsonrpc_max_payload {
Some(max) if max > 0 => max as usize,
_ => 5usize,
},
keep_alive: !self.args.flag_jsonrpc_no_keep_alive,
};
let mut conf = HttpConfiguration::default();
conf.enabled = self.rpc_enabled();
conf.interface = self.rpc_interface();
conf.port = self.args.arg_ports_shift + self.args.arg_rpcport.unwrap_or(self.args.arg_jsonrpc_port);
conf.apis = self.rpc_apis().parse()?;
conf.hosts = self.rpc_hosts();
conf.cors = self.rpc_cors();
if let Some(threads) = self.args.arg_jsonrpc_server_threads {
conf.server_threads = std::cmp::max(1, threads);
}
if let Some(threads) = self.args.arg_jsonrpc_threads {
conf.processing_threads = std::cmp::max(1, threads);
dvdplm marked this conversation as resolved.
Show resolved Hide resolved
}
if let Some(max_payload) = self.args.arg_jsonrpc_max_payload {
conf.max_payload = std::cmp::max(1, max_payload);
}
conf.keep_alive = !self.args.flag_jsonrpc_no_keep_alive;

Ok(conf)
}
Expand Down Expand Up @@ -1626,6 +1625,31 @@ mod tests {
assert_eq!(conf3.rpc_hosts(), Some(vec!["parity.io".into(), "something.io".into()]));
}

#[test]
fn ensures_sane_http_settings() {
// given incorrect settings
let conf = parse(&["parity",
"--jsonrpc-server-threads=0",
"--jsonrpc-threads=0",
"--jsonrpc-max-payload=0",
]);

// then things are adjusted to Just Work.
let http_conf = conf.http_config().unwrap();
assert_eq!(http_conf.server_threads, 1);
assert_eq!(http_conf.processing_threads, 1);
assert_eq!(http_conf.max_payload, 1);
}

#[test]
fn jsonrpc_threading_defaults() {
let conf = parse(&["parity"]);
let http_conf = conf.http_config().unwrap();
assert_eq!(http_conf.server_threads, 4);
assert_eq!(http_conf.processing_threads, 4);
assert_eq!(http_conf.max_payload, 5);
}

#[test]
fn should_parse_ipfs_hosts() {
// given
Expand Down
9 changes: 6 additions & 3 deletions parity/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -193,9 +193,12 @@ pub enum ExecutionAction {
fn execute<Cr, Rr>(
command: Execute,
logger: Arc<RotatingLogger>,
on_client_rq: Cr, on_updater_rq: Rr) -> Result<ExecutionAction, String>
where Cr: Fn(String) + 'static + Send,
Rr: Fn() + 'static + Send
on_client_rq: Cr,
on_updater_rq: Rr
) -> Result<ExecutionAction, String>
where
Cr: Fn(String) + 'static + Send,
Rr: Fn() + 'static + Send
{
#[cfg(feature = "deadlock_detection")]
run_deadlock_detection_thread();
Expand Down
13 changes: 12 additions & 1 deletion parity/rpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,26 @@ pub const DAPPS_DOMAIN: &'static str = "web3.site";

#[derive(Debug, Clone, PartialEq)]
pub struct HttpConfiguration {
/// Is RPC over HTTP enabled (default is true)?
pub enabled: bool,
/// The IP of the network interface used (default is 127.0.0.1).
pub interface: String,
/// The network port (default is 8545).
pub port: u16,
/// The categories of RPC calls enabled.
pub apis: ApiSet,
/// CORS headers
pub cors: Option<Vec<String>>,
/// Specify a list of valid hosts we accept requests from.
pub hosts: Option<Vec<String>>,
/// Number of threads to use to handle incoming requests (default is 4).
dvdplm marked this conversation as resolved.
Show resolved Hide resolved
pub server_threads: usize,
/// Number of threads to use to process HTTP requests (default is 4).
dvdplm marked this conversation as resolved.
Show resolved Hide resolved
pub processing_threads: usize,
/// Sets the maximum size of a request body in megabytes (default is 5 MiB).
pub max_payload: usize,
/// Use keepalive messages on the underlying socket: SO_KEEPALIVE as well as the TCP_KEEPALIVE
/// or TCP_KEEPIDLE options depending on your platform (default is true).
pub keep_alive: bool,
}

Expand All @@ -56,7 +67,7 @@ impl Default for HttpConfiguration {
apis: ApiSet::UnsafeContext,
cors: Some(vec![]),
hosts: Some(vec![]),
server_threads: 1,
server_threads: 4,
processing_threads: 4,
max_payload: 5,
keep_alive: true,
Expand Down
22 changes: 16 additions & 6 deletions parity/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -361,9 +361,14 @@ fn execute_light_impl<Cr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq
})
}

fn execute_impl<Cr, Rr>(cmd: RunCmd, logger: Arc<RotatingLogger>, on_client_rq: Cr,
on_updater_rq: Rr) -> Result<RunningClient, String>
where Cr: Fn(String) + 'static + Send,
fn execute_impl<Cr, Rr>(
cmd: RunCmd,
logger: Arc<RotatingLogger>,
on_client_rq: Cr,
on_updater_rq: Rr
) -> Result<RunningClient, String>
where
Cr: Fn(String) + 'static + Send,
Rr: Fn() + 'static + Send
{
// load spec
Expand Down Expand Up @@ -910,9 +915,14 @@ impl RunningClient {
/// `on_updater_rq` is the action to perform when the updater has a new binary to execute.
///
/// On error, returns what to print on stderr.
pub fn execute<Cr, Rr>(cmd: RunCmd, logger: Arc<RotatingLogger>,
on_client_rq: Cr, on_updater_rq: Rr) -> Result<RunningClient, String>
where Cr: Fn(String) + 'static + Send,
pub fn execute<Cr, Rr>(
cmd: RunCmd,
logger: Arc<RotatingLogger>,
on_client_rq: Cr,
on_updater_rq: Rr
) -> Result<RunningClient, String>
where
Cr: Fn(String) + 'static + Send,
Rr: Fn() + 'static + Send
{
if cmd.light {
Expand Down