HCM: add support for IP detection extensions

This is a follow-up to:

envoyproxy#14432 (comment)

After that PR, it's no longer possible (unless you do a dynamic_cast)
to set the remote address from a filter. This is something that we
need to do because we have specialized logic for this (XFF doesn't
work for us).

So this adds an extension point which will allow us to push that logic
down to ConnectionManagerUtility::mutateRequestHeaders() where it
belongs.

Signed-off-by: Raul Gutierrez Segales <rgs@pinterest.com>

api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto

@@ -35,7 +35,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // HTTP connection manager :ref:`configuration overview <config_http_conn_man>`.
 // [#extension: envoy.filters.network.http_connection_manager]
 
-// [#next-free-field: 43]
+// [#next-free-field: 44]
 message HttpConnectionManager {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager";
@@ -583,6 +583,11 @@ message HttpConnectionManager {
   // *not* the deprecated but similarly named :ref:`stream_error_on_invalid_http_messaging
   // <envoy_v3_api_field_config.core.v3.Http2ProtocolOptions.stream_error_on_invalid_http_messaging>`
   google.protobuf.BoolValue stream_error_on_invalid_http_message = 40;
+
+  // The configuration of the IP detection extension.
+  //
+  // If not set, Envoy uses the default remote IP detection.
+  IPDetectionExtension ip_detection_extension = 43;
 }
 
 // The configuration to customize local reply returned by Envoy.
@@ -851,3 +856,8 @@ message RequestIDExtension {
   // Request ID extension specific configuration.
   google.protobuf.Any typed_config = 1;
 }
+
+message IPDetectionExtension {
+  // IP detection extension specific configuration.
+  google.protobuf.Any typed_config = 1;
+}

docs/root/version_history/current.rst

@@ -54,6 +54,7 @@ New Features
 * access log: added the :ref:`formatters <envoy_v3_api_field_config.core.v3.SubstitutionFormatString.formatters>` extension point for custom formatters (command operators).
 * access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES% and %RESPONSE_TRAILERS_BYTES%.
 * dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash.
+* http: added support for IP detection extensions.
 * http: added support for :ref:`:ref:`preconnecting <envoy_v3_api_msg_config.cluster.v3.Cluster.PreconnectPolicy>`. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1.
 * http: change frame flood and abuse checks to the upstream HTTP/2 codec to ON by default. It can be disabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to false.
 * overload: add support for scaling :ref:`transport connection timeouts<envoy_v3_api_enum_value_config.overload.v3.ScaleTimersOverloadActionConfig.TimerType.TRANSPORT_SOCKET_CONNECT>`. This can be used to reduce the TLS handshake timeout in response to overload.

include/envoy/http/BUILD

@@ -140,3 +140,12 @@ envoy_cc_library(
         ":header_map_interface",
     ],
 )
+
+envoy_cc_library(
+    name = "ip_detection_extension_interface",
+    hdrs = ["ip_detection_extension.h"],
+    deps = [
+        ":header_map_interface",
+        "//include/envoy/config:typed_config_interface",
+    ],
+)

include/envoy/http/ip_detection_extension.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "envoy/common/pure.h"
+#include "envoy/http/header_map.h"
+
+namespace Envoy {
+namespace Http {
+
+/**
+ * Interface class for IP detection extensions.
+ */
+class IPDetectionExtension {
+public:
+  virtual ~IPDetectionExtension() = default;
+
+  /**
+   * Detect the final remote address if any.
+   *
+   * @param request_headers supplies the incoming request headers.
+   */
+  virtual Network::Address::InstanceConstSharedPtr
+  detect(Http::RequestHeaderMap& request_headers) PURE;
+};
+
+using IPDetectionExtensionSharedPtr = std::shared_ptr<IPDetectionExtension>;
+
+/*
+ * A factory for creating IP detection extensions.
+ */
+class IPDetectionExtensionFactory : public Envoy::Config::TypedFactory {
+public:
+  ~IPDetectionExtensionFactory() override = default;
+
+  /**
+   * Creates a particular Extension implementation.
+   *
+   * @param config supplies the configuration for the IP detection extension.
+   * @return IPDetectionExtensionSharedPtr the extension instance.
+   */
+  virtual IPDetectionExtensionSharedPtr createExtension(const Protobuf::Message& config) const PURE;
+};
+
+using IPDetectionExtensionFactoryPtr = std::unique_ptr<IPDetectionExtensionFactory>;
+
+} // namespace Http
+} // namespace Envoy

source/common/http/BUILD

@@ -155,6 +155,7 @@ envoy_cc_library(
         ":date_provider_lib",
         "//include/envoy/config:config_provider_interface",
         "//include/envoy/http:filter_interface",
+        "//include/envoy/http:ip_detection_extension_interface",
         "//include/envoy/http:request_id_extension_interface",
         "//include/envoy/router:rds_interface",
         "//source/common/local_reply:local_reply_lib",

source/common/http/conn_manager_config.h

@@ -3,6 +3,7 @@
 #include "envoy/config/config_provider.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
 #include "envoy/http/filter.h"
+#include "envoy/http/ip_detection_extension.h"
 #include "envoy/http/request_id_extension.h"
 #include "envoy/router/rds.h"
 #include "envoy/stats/scope.h"
@@ -466,6 +467,11 @@ class ConnectionManagerConfig {
    * @return LocalReply configuration which supplies mapping for local reply generated by Envoy.
    */
   virtual const LocalReply::LocalReply& localReply() const PURE;
+
+  /**
+   * @return IPDetectionExtensionSharedPtr The IP detection extension if available.
+   */
+  virtual IPDetectionExtensionSharedPtr ipDetectionExtension() PURE;
 };
 } // namespace Http
 } // namespace Envoy

source/common/http/conn_manager_utility.cc

@@ -95,7 +95,7 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
   // peer. Cases where we don't "use remote address" include trusted double proxy where we expect
   // our peer to have already properly set XFF, etc.
   Network::Address::InstanceConstSharedPtr final_remote_address;
-  bool single_xff_address;
+  bool single_xff_address = false;
   const uint32_t xff_num_trusted_hops = config.xffNumTrustedHops();
 
   if (config.useRemoteAddress()) {
@@ -127,12 +127,23 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
           connection.ssl() ? Headers::get().SchemeValues.Https : Headers::get().SchemeValues.Http);
     }
   } else {
-    // If we are not using remote address, attempt to pull a valid IPv4 or IPv6 address out of XFF.
+    // If we are not using remote address, attempt to pull a valid IPv4 or IPv6 address out of XFF
+    // or through an extension. An extension might be needed when XFF doesn't work (e.g. an
+    // irregular network).
+    //
     // If we find one, it will be used as the downstream address for logging. It may or may not be
     // used for determining internal/external status (see below).
-    auto ret = Utility::getLastAddressFromXFF(request_headers, xff_num_trusted_hops);
-    final_remote_address = ret.address_;
-    single_xff_address = ret.single_address_;
+    auto ip_detection_extension = config.ipDetectionExtension();
+    if (ip_detection_extension) {
+      final_remote_address = ip_detection_extension->detect(request_headers);
+    }
+
+    // If there's no extension or it failed to detect, give XFF a try.
+    if (!final_remote_address) {
+      auto ret = Utility::getLastAddressFromXFF(request_headers, xff_num_trusted_hops);
+      final_remote_address = ret.address_;
+      single_xff_address = ret.single_address_;
+    }
   }
 
   // If the x-forwarded-proto header is not set, set it here, since Envoy uses it for determining
@@ -170,30 +181,7 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
     request_headers.setReferenceEnvoyInternalRequest(
         Headers::get().EnvoyInternalRequestValues.True);
   } else {
-    if (edge_request) {
-      request_headers.removeEnvoyDecoratorOperation();
-      request_headers.removeEnvoyDownstreamServiceCluster();
-      request_headers.removeEnvoyDownstreamServiceNode();
-    }
-
-    request_headers.removeEnvoyRetriableStatusCodes();
-    request_headers.removeEnvoyRetriableHeaderNames();
-    request_headers.removeEnvoyRetryOn();
-    request_headers.removeEnvoyRetryGrpcOn();
-    request_headers.removeEnvoyMaxRetries();
-    request_headers.removeEnvoyUpstreamAltStatName();
-    request_headers.removeEnvoyUpstreamRequestTimeoutMs();
-    request_headers.removeEnvoyUpstreamRequestPerTryTimeoutMs();
-    request_headers.removeEnvoyUpstreamRequestTimeoutAltResponse();
-    request_headers.removeEnvoyExpectedRequestTimeoutMs();
-    request_headers.removeEnvoyForceTrace();
-    request_headers.removeEnvoyIpTags();
-    request_headers.removeEnvoyOriginalUrl();
-    request_headers.removeEnvoyHedgeOnPerTryTimeout();
-
-    for (const LowerCaseString& header : route_config.internalOnlyHeaders()) {
-      request_headers.remove(header);
-    }
+    cleanInternalHeaders(request_headers, edge_request, route_config.internalOnlyHeaders());
   }
 
   if (config.userAgent()) {
@@ -236,6 +224,35 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
   return final_remote_address;
 }
 
+void ConnectionManagerUtility::cleanInternalHeaders(
+    RequestHeaderMap& request_headers, bool edge_request,
+    const std::list<Http::LowerCaseString>& internal_only_headers) {
+  if (edge_request) {
+    request_headers.removeEnvoyDecoratorOperation();
+    request_headers.removeEnvoyDownstreamServiceCluster();
+    request_headers.removeEnvoyDownstreamServiceNode();
+  }
+
+  request_headers.removeEnvoyRetriableStatusCodes();
+  request_headers.removeEnvoyRetriableHeaderNames();
+  request_headers.removeEnvoyRetryOn();
+  request_headers.removeEnvoyRetryGrpcOn();
+  request_headers.removeEnvoyMaxRetries();
+  request_headers.removeEnvoyUpstreamAltStatName();
+  request_headers.removeEnvoyUpstreamRequestTimeoutMs();
+  request_headers.removeEnvoyUpstreamRequestPerTryTimeoutMs();
+  request_headers.removeEnvoyUpstreamRequestTimeoutAltResponse();
+  request_headers.removeEnvoyExpectedRequestTimeoutMs();
+  request_headers.removeEnvoyForceTrace();
+  request_headers.removeEnvoyIpTags();
+  request_headers.removeEnvoyOriginalUrl();
+  request_headers.removeEnvoyHedgeOnPerTryTimeout();
+
+  for (const LowerCaseString& header : internal_only_headers) {
+    request_headers.remove(header);
+  }
+}
+
 void ConnectionManagerUtility::mutateTracingRequestHeader(RequestHeaderMap& request_headers,
                                                           Runtime::Loader& runtime,
                                                           ConnectionManagerConfig& config,

source/common/http/conn_manager_utility.h

@@ -87,6 +87,8 @@ class ConnectionManagerUtility {
   static void mutateXfccRequestHeader(RequestHeaderMap& request_headers,
                                       Network::Connection& connection,
                                       ConnectionManagerConfig& config);
+  static void cleanInternalHeaders(RequestHeaderMap& request_headers, bool edge_request,
+                                   const std::list<Http::LowerCaseString>& internal_only_headers);
 };
 
 } // namespace Http

source/extensions/filters/network/http_connection_manager/BUILD

@@ -24,6 +24,7 @@ envoy_cc_extension(
         "//include/envoy/filesystem:filesystem_interface",
         "//include/envoy/http:codec_interface",
         "//include/envoy/http:filter_interface",
+        "//include/envoy/http:ip_detection_extension_interface",
         "//include/envoy/http:request_id_extension_interface",
         "//include/envoy/registry",
         "//include/envoy/router:route_config_provider_manager_interface",

source/extensions/filters/network/http_connection_manager/config.cc

@@ -287,6 +287,18 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
         Http::RequestIDExtensionFactory::defaultInstance(context_.api().randomGenerator());
   }
 
+  // Check if we are provided with an IP detection extension.
+  if (config.ip_detection_extension().has_typed_config()) {
+    auto& typed_config = config.ip_detection_extension().typed_config();
+    const std::string type{TypeUtil::typeUrlToDescriptorFullName(typed_config.type_url())};
+    auto* ip_detection_extension_factory =
+        Registry::FactoryRegistry<Http::IPDetectionExtensionFactory>::getFactoryByType(type);
+    if (!ip_detection_extension_factory) {
+      throw EnvoyException("IP detection extension not found");
+    }
+    ip_detection_extension_ = ip_detection_extension_factory->createExtension(typed_config);
+  }
+
   // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will
   // be managed by the scoped routing logic instead.
   switch (config.route_specifier_case()) {

source/extensions/filters/network/http_connection_manager/config.h

@@ -13,6 +13,7 @@
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h"
 #include "envoy/filter/http/filter_config_provider.h"
 #include "envoy/http/filter.h"
+#include "envoy/http/ip_detection_extension.h"
 #include "envoy/http/request_id_extension.h"
 #include "envoy/router/route_config_provider_manager.h"
 #include "envoy/tracing/http_tracer_manager.h"
@@ -177,6 +178,9 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   }
   std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; }
   const LocalReply::LocalReply& localReply() const override { return *local_reply_; }
+  Http::IPDetectionExtensionSharedPtr ipDetectionExtension() override {
+    return ip_detection_extension_;
+  }
 
 private:
   enum class CodecType { HTTP1, HTTP2, HTTP3, AUTO };
@@ -255,6 +259,7 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   const envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction
       headers_with_underscores_action_;
   const LocalReply::LocalReplyPtr local_reply_;
+  Http::IPDetectionExtensionSharedPtr ip_detection_extension_{nullptr};
 
   // Default idle timeout is 5 minutes if nothing is specified in the HCM config.
   static const uint64_t StreamIdleTimeoutMs = 5 * 60 * 1000;

source/server/admin/admin.h

@@ -180,6 +180,7 @@ class AdminImpl : public Admin,
     return envoy::config::core::v3::HttpProtocolOptions::ALLOW;
   }
   const LocalReply::LocalReply& localReply() const override { return *local_reply_; }
+  Http::IPDetectionExtensionSharedPtr ipDetectionExtension() override { return nullptr; };
   Http::Code request(absl::string_view path_and_query, absl::string_view method,
                      Http::ResponseHeaderMap& response_headers, std::string& body) override;
   void closeSocket();