From 68b8ffdb638d07937f841f7217edeb25efdb3b5d Mon Sep 17 00:00:00 2001 From: Mark Travis Date: Thu, 8 Jun 2017 10:00:29 -0700 Subject: [PATCH] Improve automatic tuning of thread pool: The job queue can automatically tune the number of threads that it creates based on the number of processors or processor cores that are available. The existing tuning was very conservative, limiting the maximum number of threads to only 6. Adjust the new algorithm to allow a larger number of threads and allow server administrators to override the value in the config file. --- doc/rippled-example.cfg | 8 ++++++++ src/ripple/app/main/Application.cpp | 4 +++- src/ripple/core/Config.h | 3 +++ src/ripple/core/ConfigSections.h | 1 + src/ripple/core/JobQueue.h | 3 ++- src/ripple/core/impl/Config.cpp | 3 +++ src/ripple/core/impl/JobQueue.cpp | 19 ++++++++++++++----- 7 files changed, 34 insertions(+), 7 deletions(-) diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg index b9cb9898cdc..c645802343f 100644 --- a/doc/rippled-example.cfg +++ b/doc/rippled-example.cfg @@ -690,6 +690,14 @@ # # # +# [workers] +# +# Configures the number of threads for processing work submitted by peers +# and clients. If not specified, then the value is automatically determined +# by factors including the number of system processors and whether this +# node is a validator. +# +# #------------------------------------------------------------------------------- # # 4. HTTPS Client diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 89f475f229f..7f3675a7be8 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -966,7 +966,9 @@ class ApplicationImp bool ApplicationImp::setup() { // VFALCO NOTE: 0 means use heuristics to determine the thread count. - m_jobQueue->setThreadCount (0, config_->standalone()); + m_jobQueue->setThreadCount (config_->WORKERS, config_->standalone(), + config_->exists (SECTION_VALIDATOR_TOKEN) || + config_->exists (SECTION_VALIDATION_SEED)); // We want to intercept and wait for CTRL-C to terminate the process m_signals.add (SIGINT); diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index fb10daab382..8be4e41242a 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -166,6 +166,9 @@ class Config : public BasicConfig std::string SSL_VERIFY_FILE; std::string SSL_VERIFY_DIR; + // Thread pool configuration + std::size_t WORKERS = 0; + // These override the command line client settings boost::optional rpc_ip; boost::optional rpc_port; diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index 638a53403a6..29031f21a3d 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -69,6 +69,7 @@ struct ConfigSection #define SECTION_VALIDATORS "validators" #define SECTION_VALIDATOR_TOKEN "validator_token" #define SECTION_VETO_AMENDMENTS "veto_amendments" +#define SECTION_WORKERS "workers" } // ripple diff --git a/src/ripple/core/JobQueue.h b/src/ripple/core/JobQueue.h index 693d088cc14..cf2a8731973 100644 --- a/src/ripple/core/JobQueue.h +++ b/src/ripple/core/JobQueue.h @@ -163,7 +163,8 @@ class JobQueue /** Set the number of thread serving the job queue to precisely this number. */ - void setThreadCount (int c, bool const standaloneMode); + void setThreadCount (int c, bool const standaloneMode, + bool const validator=true); /** Return a scoped LoadEvent. */ diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index 2deebae6f68..6e61c518da7 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -413,6 +413,9 @@ void Config::loadFromString (std::string const& fileContents) if (getSingleSection (secConfig, SECTION_DEBUG_LOGFILE, strTemp, j_)) DEBUG_LOGFILE = strTemp; + if (getSingleSection (secConfig, SECTION_WORKERS, strTemp, j_)) + WORKERS = beast::lexicalCastThrow (strTemp); + // Do not load trusted validator configuration for standalone mode if (! RUN_STANDALONE) { diff --git a/src/ripple/core/impl/JobQueue.cpp b/src/ripple/core/impl/JobQueue.cpp index cc49adbd13c..ffb919a4372 100644 --- a/src/ripple/core/impl/JobQueue.cpp +++ b/src/ripple/core/impl/JobQueue.cpp @@ -152,7 +152,8 @@ JobQueue::getJobCountGE (JobType t) const } void -JobQueue::setThreadCount (int c, bool const standaloneMode) +JobQueue::setThreadCount (int c, bool const standaloneMode, + bool const validator) { if (standaloneMode) { @@ -161,10 +162,18 @@ JobQueue::setThreadCount (int c, bool const standaloneMode) else if (c == 0) { c = static_cast(std::thread::hardware_concurrency()); - c = 2 + std::min (c, 4); // I/O will bottleneck - - JLOG(m_journal.info()) << "Auto-tuning to " << c << - " validation/transaction/proposal threads"; + if (validator) + c = 2 + std::min(c, 4); // I/O will bottleneck + else + c *= 2; // Tested to improve stability under high RPC load. + JLOG (m_journal.info()) << "Auto-tuning to " << c << + " validation/transaction/proposal threads for " << + (validator ? "" : "non-") << "validator."; + } + else + { + JLOG (m_journal.info()) << "Configured " << c << + " validation/transaction/proposal threads."; } m_workers.setNumberOfThreads (c);