http: add overload action for closing new streams (#4126)

Add an overload action in the http connection manager to immediately close new streams in case of envoy overload (issue #373).

Signed-off-by: Elisha Ziskind <eziskind@google.com>

docs/root/configuration/http_conn_man/stats.rst

@@ -54,6 +54,7 @@ statistics:
    downstream_rq_ws_on_non_ws_route, Counter, Total WebSocket upgrade requests rejected by non WebSocket routes
    downstream_rq_time, Histogram, Request time milliseconds
    downstream_rq_idle_timeout, Counter, Total requests closed due to idle timeout
+   downstream_rq_overload_close, Counter, Total requests closed due to envoy overload
    rs_too_large, Counter, Total response errors due to buffering an overly large body
 
 Per user agent statistics

docs/root/configuration/overload_manager/overload_manager.rst

@@ -3,7 +3,7 @@
 Overload manager
 ================
 
-The :ref:`overload manager <arch_overview_overload_manager>` is configured in the Boostrap
+The :ref:`overload manager <arch_overview_overload_manager>` is configured in the Bootstrap
 :ref:`overload_manager <envoy_api_field_config.bootstrap.v2.Bootstrap.overload_manager>`
 field.
 
@@ -14,6 +14,17 @@ The overload manager uses Envoy's :ref:`extension <extending>` framework for def
 resource monitors. Envoy's builtin resource monitors are listed
 :ref:`here <config_resource_monitors>`.
 
+Overload actions
+----------------
+
+The following overload actions are supported:
+
+.. csv-table::
+  :header: Name, Description
+  :widths: 1, 2
+
+  envoy.overload_actions.stop_accepting_requests, Envoy will immediately respond with a 503 response code to new requests
+
 Statistics
 ----------
 

include/envoy/server/BUILD

@@ -147,6 +147,7 @@ envoy_cc_library(
         "//include/envoy/network:drain_decision_interface",
         "//include/envoy/ratelimit:ratelimit_interface",
         "//include/envoy/runtime:runtime_interface",
+        "//include/envoy/server:overload_manager_interface",
         "//include/envoy/singleton:manager_interface",
         "//include/envoy/thread_local:thread_local_interface",
         "//include/envoy/tracing:http_tracer_interface",

include/envoy/server/filter_config.h

@@ -12,6 +12,7 @@
 #include "envoy/ratelimit/ratelimit.h"
 #include "envoy/runtime/runtime.h"
 #include "envoy/server/admin.h"
+#include "envoy/server/overload_manager.h"
 #include "envoy/singleton/manager.h"
 #include "envoy/stats/scope.h"
 #include "envoy/thread_local/thread_local.h"
@@ -134,6 +135,11 @@ class FactoryContext {
    * @return TimeSource& a reference to the time source.
    */
   virtual TimeSource& timeSource() PURE;
+
+  /**
+   * @return OverloadManager& the overload manager for the server.
+   */
+  virtual OverloadManager& overloadManager() PURE;
 };
 
 class ListenerFactoryContext : public FactoryContext {

include/envoy/server/overload_manager.h

@@ -5,6 +5,9 @@
 #include "envoy/common/pure.h"
 #include "envoy/thread_local/thread_local.h"
 
+#include "common/common/macros.h"
+#include "common/singleton/const_singleton.h"
+
 namespace Envoy {
 namespace Server {
 
@@ -50,6 +53,17 @@ class ThreadLocalOverloadState : public ThreadLocal::ThreadLocalObject {
   std::unordered_map<std::string, OverloadActionState> actions_;
 };
 
+/**
+ * Well-known overload action names.
+ */
+class OverloadActionNameValues {
+public:
+  // Overload action to stop accepting new requests.
+  const std::string StopAcceptingRequests = "envoy.overload_actions.stop_accepting_requests";
+};
+
+typedef ConstSingleton<OverloadActionNameValues> OverloadActionNames;
+
 /**
  * The OverloadManager protects the Envoy instance from being overwhelmed by client
  * requests. It monitors a set of resources and notifies registered listeners if
@@ -81,6 +95,17 @@ class OverloadManager {
    * an alternative to registering a callback for overload action state changes.
    */
   virtual ThreadLocalOverloadState& getThreadLocalOverloadState() PURE;
+
+  /**
+   * Convenience method to get a statically allocated reference to the inactive overload
+   * action state. Useful for code that needs to initialize a reference either to an
+   * entry in the ThreadLocalOverloadState map (if overload behavior is enabled) or to
+   * some other static memory location set to the inactive state (if overload behavior
+   * is disabled).
+   */
+  static const OverloadActionState& getInactiveState() {
+    CONSTRUCT_ON_FIRST_USE(OverloadActionState, OverloadActionState::Inactive);
+  }
 };
 
 } // namespace Server

source/common/http/BUILD

@@ -123,6 +123,7 @@ envoy_cc_library(
         "//include/envoy/network:filter_interface",
         "//include/envoy/router:rds_interface",
         "//include/envoy/runtime:runtime_interface",
+        "//include/envoy/server:overload_manager_interface",
         "//include/envoy/ssl:connection_interface",
         "//include/envoy/stats:stats_interface",
         "//include/envoy/stats:stats_macros",

source/common/http/conn_manager_config.h

@@ -57,6 +57,7 @@ namespace Http {
   COUNTER  (downstream_rq_5xx)                                                                     \
   HISTOGRAM(downstream_rq_time)                                                                    \
   COUNTER  (downstream_rq_idle_timeout)                                                            \
+  COUNTER  (downstream_rq_overload_close)                                                          \
   COUNTER  (rs_too_large)
 // clang-format on
 

source/common/http/conn_manager_impl.cc

@@ -58,12 +58,17 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config,
                                              Runtime::RandomGenerator& random_generator,
                                              Tracing::HttpTracer& tracer, Runtime::Loader& runtime,
                                              const LocalInfo::LocalInfo& local_info,
-                                             Upstream::ClusterManager& cluster_manager)
+                                             Upstream::ClusterManager& cluster_manager,
+                                             Server::OverloadManager* overload_manager)
     : config_(config), stats_(config_.stats()),
       conn_length_(new Stats::Timespan(stats_.named_.downstream_cx_length_ms_)),
       drain_close_(drain_close), random_generator_(random_generator), tracer_(tracer),
       runtime_(runtime), local_info_(local_info), cluster_manager_(cluster_manager),
-      listener_stats_(config_.listenerStats()) {}
+      listener_stats_(config_.listenerStats()),
+      overload_stop_accepting_requests_(
+          overload_manager ? overload_manager->getThreadLocalOverloadState().getState(
+                                 Server::OverloadActionNames::get().StopAcceptingRequests)
+                           : Server::OverloadManager::getInactiveState()) {}
 
 const HeaderMapImpl& ConnectionManagerImpl::continueHeader() {
   CONSTRUCT_ON_FIRST_USE(HeaderMapImpl,
@@ -488,6 +493,15 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(HeaderMapPtr&& headers,
     is_head_request_ = true;
   }
 
+  // Drop new requests when overloaded as soon as we have decoded the headers.
+  if (connection_manager_.overload_stop_accepting_requests_ ==
+      Server::OverloadActionState::Active) {
+    connection_manager_.stats_.named_.downstream_rq_overload_close_.inc();
+    sendLocalReply(Grpc::Common::hasGrpcContentType(*request_headers_),
+                   Http::Code::ServiceUnavailable, "envoy overloaded", nullptr, is_head_request_);
+    return;
+  }
+
   const bool upgrade_rejected = createFilterChain() == false;
 
   maybeEndDecode(end_stream);

source/common/http/conn_manager_impl.h

@@ -18,6 +18,7 @@
 #include "envoy/network/filter.h"
 #include "envoy/router/rds.h"
 #include "envoy/runtime/runtime.h"
+#include "envoy/server/overload_manager.h"
 #include "envoy/ssl/connection.h"
 #include "envoy/stats/scope.h"
 #include "envoy/stats/stats_macros.h"
@@ -49,7 +50,8 @@ class ConnectionManagerImpl : Logger::Loggable<Logger::Id::http>,
   ConnectionManagerImpl(ConnectionManagerConfig& config, const Network::DrainDecision& drain_close,
                         Runtime::RandomGenerator& random_generator, Tracing::HttpTracer& tracer,
                         Runtime::Loader& runtime, const LocalInfo::LocalInfo& local_info,
-                        Upstream::ClusterManager& cluster_manager);
+                        Upstream::ClusterManager& cluster_manager,
+                        Server::OverloadManager* overload_manager);
   ~ConnectionManagerImpl();
 
   static ConnectionManagerStats generateStats(const std::string& prefix, Stats::Scope& scope);
@@ -456,6 +458,7 @@ class ConnectionManagerImpl : Logger::Loggable<Logger::Id::http>,
   WebSocketProxyPtr ws_connection_;
   Network::ReadFilterCallbacks* read_callbacks_{};
   ConnectionManagerListenerStats& listener_stats_;
+  const Server::OverloadActionState& overload_stop_accepting_requests_;
 };
 
 } // namespace Http

source/extensions/filters/network/http_connection_manager/config.cc

@@ -76,7 +76,8 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped(
           date_provider](Network::FilterManager& filter_manager) -> void {
     filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl(
         *filter_config, context.drainDecision(), context.random(), context.httpTracer(),
-        context.runtime(), context.localInfo(), context.clusterManager())});
+        context.runtime(), context.localInfo(), context.clusterManager(),
+        &context.overloadManager())});
   };
 }
 

source/server/http/admin.cc

@@ -972,9 +972,11 @@ Http::ServerConnectionPtr AdminImpl::createCodec(Network::Connection& connection
 
 bool AdminImpl::createNetworkFilterChain(Network::Connection& connection,
                                          const std::vector<Network::FilterFactoryCb>&) {
+  // Don't pass in the overload manager so that the admin interface is accessible even when
+  // the envoy is overloaded.
   connection.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl(
       *this, server_.drainManager(), server_.random(), server_.httpTracer(), server_.runtime(),
-      server_.localInfo(), server_.clusterManager())});
+      server_.localInfo(), server_.clusterManager(), nullptr)});
   return true;
 }
 

source/server/listener_manager_impl.h

@@ -268,6 +268,7 @@ class ListenerImpl : public Network::ListenerConfig,
   Envoy::Runtime::Loader& runtime() override { return parent_.server_.runtime(); }
   Stats::Scope& scope() override { return *global_scope_; }
   Singleton::Manager& singletonManager() override { return parent_.server_.singletonManager(); }
+  OverloadManager& overloadManager() override { return parent_.server_.overloadManager(); }
   ThreadLocal::Instance& threadLocal() override { return parent_.server_.threadLocal(); }
   Admin& admin() override { return parent_.server_.admin(); }
   const envoy::api::v2::core::Metadata& listenerMetadata() const override {

source/server/overload_manager_impl.cc

@@ -58,6 +58,8 @@ OverloadAction::OverloadAction(const envoy::config::overload::v2alpha::OverloadA
           fmt::format("Duplicate trigger resource for overload action {}", config.name()));
     }
   }
+
+  active_gauge_.set(0);
 }
 
 bool OverloadAction::updateResourcePressure(const std::string& name, double pressure) {

test/common/http/conn_manager_impl_fuzz_test.cc

@@ -390,7 +390,7 @@ DEFINE_PROTO_FUZZER(const test::common::http::ConnManagerImplTestCase& input) {
       std::make_shared<Network::Address::Ipv4Instance>("0.0.0.0");
 
   ConnectionManagerImpl conn_manager(config, drain_close, random, tracer, runtime, local_info,
-                                     cluster_manager);
+                                     cluster_manager, nullptr);
   conn_manager.initializeReadFilterCallbacks(filter_callbacks);
 
   std::vector<FuzzStreamPtr> streams;