diff --git a/.bazelrc b/.bazelrc index 4ce341e00ebb..d43694e4ccf9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -246,7 +246,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:9400637f4aa0232465407447bfda0d3da13549fb +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:11efa5680d987fff33fde4af3cc5ece105015d04 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 6c5f03d7235f..77d7228bfbd0 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/envoy-ci/envoy-build:9400637f4aa0232465407447bfda0d3da13549fb +FROM gcr.io/envoy-ci/envoy-build:11efa5680d987fff33fde4af3cc5ece105015d04 ARG USERNAME=vscode ARG USER_UID=501 diff --git a/CODEOWNERS b/CODEOWNERS index 7d1e02896b5a..7f684161d704 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -160,3 +160,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/filters/common/local_ratelimit @mattklein123 @rgs1 # HTTP Kill Request /*/extensions/filters/http/kill_request @qqustc @htuch +# Rate limit expression descriptor +/*/extensions/rate_limit_descriptors/expr @kyessenov @lizan diff --git a/RELEASES.md b/RELEASES.md index a1d7929ad673..085c58a2ce52 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -67,7 +67,7 @@ deadline of 3 weeks. | 1.14.0 | 2020/03/31 | 2020/04/08 | +8 days | 2021/04/08 | | 1.15.0 | 2020/06/30 | 2020/07/07 | +7 days | 2021/07/07 | | 1.16.0 | 2020/09/30 | 2020/10/08 | +8 days | 2021/10/08 | -| 1.17.0 | 2020/12/31 | | | | - +| 1.17.0 | 2020/12/31 | 2021/01/11 | +11 days | 2022/01/11 | +| 1.18.0 | 2021/03/31 | | | | [repokitteh]: https://github.com/repokitteh diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md index 3eae104c8c35..fff64a494971 100644 --- a/REPO_LAYOUT.md +++ b/REPO_LAYOUT.md @@ -111,6 +111,9 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE `Envoy::Extensions::Upstreams` namespace. * [watchdog](/source/extensions/watchdog): Watchdog extensions use the `Envoy::Extensions::Watchdog` namespace. + * [descriptors](/source/extensions/rate_limit_descriptors): Rate limit + descriptor extensions use the `Envoy::Extensions::RateLimitDescriptors` + namespace. * Each extension is contained wholly in its own namespace. E.g., `Envoy::Extensions::NetworkFilters::Echo`. * Common code that is used by multiple extensions should be in a `common/` directory as close to diff --git a/VERSION b/VERSION index ee8855caa4a7..ee017091ff37 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.17.0-dev +1.18.0-dev diff --git a/api/BUILD b/api/BUILD index f7c75a2d7fae..6b8fc64edab1 100644 --- a/api/BUILD +++ b/api/BUILD @@ -239,6 +239,7 @@ proto_library( "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/network/socket_interface/v3:pkg", + "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg", "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg", "//envoy/extensions/retry/priority/previous_priorities/v3:pkg", "//envoy/extensions/stat_sinks/wasm/v3:pkg", diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index db188a572ae0..b9a807d82edb 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -49,10 +49,7 @@ message ExtAuthz { // `. bool failure_mode_allow = 2; - // Sets the package version the gRPC service should use. This is particularly - // useful when transitioning from alpha to release versions assuming that both definitions are - // semantically compatible. Deprecation note: This field is deprecated and should only be used for - // version upgrade. See release notes for more details. + // [#not-implemented-hide: Support for this field has been removed.] bool use_alpha = 4 [deprecated = true, (envoy.annotations.disallowed_by_default) = true]; // Enables filter to buffer the client request body and send it within the authorization request. diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 2d85fd0dc7db..53b351b8d3aa 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1544,7 +1544,7 @@ message VirtualCluster { message RateLimit { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit"; - // [#next-free-field: 9] + // [#next-free-field: 10] message Action { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit.Action"; @@ -1742,6 +1742,9 @@ message RateLimit { // Rate limit on metadata. MetaData metadata = 8; + + // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + core.v3.TypedExtensionConfig extension = 9; } } diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index 166b19671917..577282595d84 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -1493,7 +1493,7 @@ message VirtualCluster { message RateLimit { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RateLimit"; - // [#next-free-field: 9] + // [#next-free-field: 10] message Action { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RateLimit.Action"; @@ -1694,6 +1694,9 @@ message RateLimit { // Rate limit on metadata. MetaData metadata = 8; + + // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + core.v4alpha.TypedExtensionConfig extension = 9; } } diff --git a/api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto index 1e64376add9b..9f5c952a57ea 100644 --- a/api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto +++ b/api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto @@ -21,7 +21,22 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.filters.http.ext_proc] // The External Processing filter allows an external service to act on HTTP traffic in a flexible way. -// It communicates with an external gRPC service that can use it to do a variety of things + +// **Current Implementation Status:** +// At this time, the filter will send a "request_headers" message to the server when the +// filter is invoked from the downstream, and apply any header mutations returned by the +// server. No other part of the protocol is implemented yet. + +// As designed, the filter supports up to six different processing steps, which are in the +// process of being implemented: +// * Request headers: IMPLEMENTED +// * Request body: NOT IMPLEMENTED +// * Request trailers: NOT IMPLEMENTED +// * Response headers: NOT IMPLEMENTED +// * Response body: NOT IMPLEMENTED +// * Response trailers: NOT IMPLEMENTED + +// The filter communicates with an external gRPC service that can use it to do a variety of things // with the request and response: // // * Access and modify the HTTP headers on the request, response, or both diff --git a/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD b/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD new file mode 100644 index 000000000000..facd82ce6de2 --- /dev/null +++ b/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", + ], +) diff --git a/api/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto b/api/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto new file mode 100644 index 000000000000..76d3505cba04 --- /dev/null +++ b/api/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package envoy.extensions.rate_limit_descriptors.expr.v3; + +import "google/api/expr/v1alpha1/syntax.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.rate_limit_descriptors.expr.v3"; +option java_outer_classname = "ExprProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Rate limit descriptor expression] +// [#extension: envoy.rate_limit_descriptors.expr] + +// The following descriptor entry is appended with a value computed +// from a symbolic Common Expression Language expression. +// See :ref:`attributes ` for the set of +// available attributes. +// +// .. code-block:: cpp +// +// ("", "") +message Descriptor { + // The key to use in the descriptor entry. + string descriptor_key = 1 [(validate.rules).string = {min_len: 1}]; + + // If set to true, Envoy skips the descriptor if the expression evaluates to an error. + // By default, the rate limit is not applied when an expression produces an error. + bool skip_if_error = 2; + + oneof expr_specifier { + // Expression in a text form, e.g. "connection.requested_server_name". + string text = 3 [(validate.rules).string = {min_len: 1}]; + + // Parsed expression in AST form. + google.api.expr.v1alpha1.Expr parsed = 4; + } +} diff --git a/api/envoy/service/ext_proc/v3alpha/external_processor.proto b/api/envoy/service/ext_proc/v3alpha/external_processor.proto index cac1dde34d9e..5b0696bfc3b0 100644 --- a/api/envoy/service/ext_proc/v3alpha/external_processor.proto +++ b/api/envoy/service/ext_proc/v3alpha/external_processor.proto @@ -6,7 +6,6 @@ import "envoy/config/core/v3/base.proto"; import "envoy/extensions/filters/http/ext_proc/v3alpha/processing_mode.proto"; import "envoy/type/v3/http_status.proto"; -import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "udpa/annotations/status.proto"; @@ -289,10 +288,13 @@ message GrpcStatus { // Change HTTP headers or trailers by appending, replacing, or removing // headers. message HeaderMutation { - // Add or replace HTTP headers. + // Add or replace HTTP headers. Attempts to set the value of + // any "x-envoy" header, and attempts to set the ":method", + // ":authority", ":scheme", or "host" headers will be ignored. repeated config.core.v3.HeaderValueOption set_headers = 1; - // Remove these HTTP headers. + // Remove these HTTP headers. Attempts to remove system headers -- + // any header starting with ":", plus "host" -- will be ignored. repeated string remove_headers = 2; } diff --git a/api/envoy/type/matcher/v3/http_inputs.proto b/api/envoy/type/matcher/v3/http_inputs.proto new file mode 100644 index 000000000000..48eb4c43154a --- /dev/null +++ b/api/envoy/type/matcher/v3/http_inputs.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package envoy.type.matcher.v3; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v3"; +option java_outer_classname = "HttpInputsProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Common HTTP Inputs] + +// Match input indicates that matching should be done on a specific request header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the request contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpRequestHeaderMatchInput { + // The request header to match on. + string header_name = 1; +} + +// Match input indicating that matching should be done on a specific response header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the response contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpResponseHeaderMatchInput { + // The response header to match on. + string header_name = 1; +} diff --git a/api/envoy/type/matcher/v4alpha/http_inputs.proto b/api/envoy/type/matcher/v4alpha/http_inputs.proto new file mode 100644 index 000000000000..f15e9ceb70e2 --- /dev/null +++ b/api/envoy/type/matcher/v4alpha/http_inputs.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package envoy.type.matcher.v4alpha; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v4alpha"; +option java_outer_classname = "HttpInputsProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Common HTTP Inputs] + +// Match input indicates that matching should be done on a specific request header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the request contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpRequestHeaderMatchInput { + option (udpa.annotations.versioning).previous_message_type = + "envoy.type.matcher.v3.HttpRequestHeaderMatchInput"; + + // The request header to match on. + string header_name = 1; +} + +// Match input indicating that matching should be done on a specific response header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the response contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpResponseHeaderMatchInput { + option (udpa.annotations.versioning).previous_message_type = + "envoy.type.matcher.v3.HttpResponseHeaderMatchInput"; + + // The response header to match on. + string header_name = 1; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 41addf767671..6f6762722148 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -122,6 +122,7 @@ proto_library( "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/network/socket_interface/v3:pkg", + "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg", "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg", "//envoy/extensions/retry/priority/previous_priorities/v3:pkg", "//envoy/extensions/stat_sinks/wasm/v3:pkg", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 95aa885b088a..e6f1b1afe8dd 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -49,11 +49,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "envoy-build-tools", project_desc = "Common build tools shared by the Envoy/UDPA ecosystem", project_url = "https://github.com/envoyproxy/envoy-build-tools", - version = "3ff9995a5dd3d0e703e602ca3ebd9366de2b5752", - sha256 = "a4c74ce9a62b1c907329d248d4c225abfae8646bc77db8d72de65726632d3571", + version = "f2a7f9ba09660beacfebcd37fc977480ec9a8f50", + sha256 = "ca9975f6d5370843167b9646028ca7a0b546f8821f217c9d9d2e033a94a35f31", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2020-12-21", + release_date = "2021-01-04", use_category = ["build"], ), boringssl = dict( @@ -713,6 +713,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( extensions = [ "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", + "envoy.rate_limit_descriptors.expr", "envoy.filters.http.rbac", "envoy.filters.http.wasm", "envoy.filters.network.rbac", @@ -734,6 +735,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( extensions = [ "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", + "envoy.rate_limit_descriptors.expr", "envoy.filters.http.rbac", "envoy.filters.http.wasm", "envoy.filters.network.rbac", @@ -912,6 +914,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( extensions = [ "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", + "envoy.rate_limit_descriptors.expr", "envoy.filters.http.wasm", "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", @@ -931,6 +934,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( extensions = [ "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", + "envoy.rate_limit_descriptors.expr", "envoy.filters.http.wasm", "envoy.filters.network.wasm", "envoy.stat_sinks.wasm", diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index db70d213b89f..c1240c53af72 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -24,3 +24,4 @@ Extensions upstream/upstream wasm/wasm watchdog/watchdog + descriptors/descriptors diff --git a/docs/root/api-v3/config/descriptors/descriptors.rst b/docs/root/api-v3/config/descriptors/descriptors.rst new file mode 100644 index 000000000000..69ac0d237832 --- /dev/null +++ b/docs/root/api-v3/config/descriptors/descriptors.rst @@ -0,0 +1,8 @@ +Rate limit descriptors +====================== + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../extensions/rate_limit_descriptors/expr/v3/* diff --git a/docs/root/api-v3/types/types.rst b/docs/root/api-v3/types/types.rst index 3e6af53865bd..4321708bdb82 100644 --- a/docs/root/api-v3/types/types.rst +++ b/docs/root/api-v3/types/types.rst @@ -21,5 +21,6 @@ Types ../type/matcher/v3/string.proto ../type/matcher/v3/struct.proto ../type/matcher/v3/value.proto + ../type/matcher/v3/http_inputs.proto ../type/metadata/v3/metadata.proto ../type/tracing/v3/custom_tag.proto diff --git a/docs/root/configuration/http/http_filters/ext_proc_filter.rst b/docs/root/configuration/http/http_filters/ext_proc_filter.rst index 851dea86e1c4..f00eb27d1bf1 100644 --- a/docs/root/configuration/http/http_filters/ext_proc_filter.rst +++ b/docs/root/configuration/http/http_filters/ext_proc_filter.rst @@ -17,3 +17,24 @@ messages, and the server must reply with :ref:`ProcessingResponse `. This filter is a work in progress. In its current state, it actually does nothing. + +Statistics +---------- +This filter outputs statistics in the +*http..ext_proc.* namespace. The :ref:`stat prefix +` +comes from the owning HTTP connection manager. + +The following statistics are supported: + +.. csv-table:: + :header: Name, Type, Description + :widths: auto + + streams_started, Counter, The number of gRPC streams that have been started to send to the external processing service + streams_msgs_sent, Counter, The number of messages sent on those streams + streams_msgs_received, Counter, The number of messages received on those streams + spurious_msgs_received, Counter, The number of unexpected messages received that violated the protocol + streams_closed, Counter, The number of streams successfully closed on either end + streams_failed, Counter, The number of times a stream produced a gRPC error + failure_mode_allowed, Counter, The number of times an error was ignored due to configuration diff --git a/docs/root/configuration/http/http_filters/rate_limit_filter.rst b/docs/root/configuration/http/http_filters/rate_limit_filter.rst index b1d6d6b11c5a..b1f27c63c1ec 100644 --- a/docs/root/configuration/http/http_filters/rate_limit_filter.rst +++ b/docs/root/configuration/http/http_filters/rate_limit_filter.rst @@ -121,6 +121,23 @@ the rate limit override of 42 requests per hour will be appended to the rate lim requests_per_unit: 42 unit: HOUR +Descriptor extensions +--------------------- + +Rate limit descriptors are extensible with custom descriptors. For example, :ref:`computed descriptors +` extension allows using any of the +:ref:`request attributes ` as a descriptor value: + +.. code-block:: yaml + + actions: + - extension: + name: custom + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + text: request.method + Statistics ---------- diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index e06357291c2e..3ea3bcdfa5c1 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -1,123 +1,26 @@ -1.17.0 (pending) +1.18.0 (Pending) ================ Incompatible Behavior Changes ----------------------------- *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* -* config: v2 is now fatal-by-default. This may be overridden by setting :option:`--bootstrap-version` 2 on the CLI for a v2 bootstrap file and also enabling the runtime `envoy.reloadable_features.enable_deprecated_v2_api` feature. See - the :ref:`FAQ entry ` for further details. - Minor Behavior Changes ---------------------- *Changes that may cause incompatibilities for some users, but should not for most* -* build: the Alpine based debug images are no longer built in CI, use Ubuntu based images instead. -* cluster manager: the cluster which can't extract secret entity by SDS to be warming and never activate. This feature is disabled by default and is controlled by runtime guard `envoy.reloadable_features.cluster_keep_warming_no_secret_entity`. -* decompressor: set the default value of window_bits of the decompressor to 15 to be able to decompress responses compressed by a compressor with any window size. -* expr filter: added `connection.termination_details` property support. -* ext_authz filter: disable `envoy.reloadable_features.ext_authz_measure_timeout_on_check_created` by default. -* ext_authz filter: the deprecated field :ref:`use_alpha ` is no longer supported and cannot be set anymore. -* formatter: the :ref:`text_format ` field no longer requires at least one byte, and may now be the empty string. It has also become deprecated: see Deprecated section. -* grpc_web filter: if a `grpc-accept-encoding` header is present it's passed as-is to the upstream and if it isn't `grpc-accept-encoding:identity` is sent instead. The header was always overwriten with `grpc-accept-encoding:identity,deflate,gzip` before. -* http: upstream protocol will now only be logged if an upstream stream was established. -* jwt_authn filter: added support of Jwt time constraint verification with a clock skew (default to 60 seconds) and added a filter config field :ref:`clock_skew_seconds ` to configure it. -* kill_request: enable a way to configure kill header name in KillRequest proto. -* listener: injection of the :ref:`TLS inspector ` has been disabled by default. This feature is controlled by the runtime guard `envoy.reloadable_features.disable_tls_inspector_injection`. -* lua: add `always_wrap_body` argument to `body()` API to always return a :ref:`buffer object ` even if the body is empty. -* memory: enable new tcmalloc with restartable sequences for aarch64 builds. -* mongo proxy metrics: swapped network connection remote and local closed counters previously set reversed (`cx_destroy_local_with_active_rq` and `cx_destroy_remote_with_active_rq`). -* outlier detection: added :ref:`max_ejection_time ` to limit ejection time growth when a node stays unhealthy for extended period of time. By default :ref:`max_ejection_time ` limits ejection time to 5 minutes. Additionally, when the node stays healthy, ejection time decreases. See :ref:`ejection algorithm` for more info. Previously, ejection time could grow without limit and never decreased. -* performance: improve performance when handling large HTTP/1 bodies. -* tcp_proxy: now waits for HTTP tunnel to be established before start streaming the downstream data, the runtime guard `envoy.reloadable_features.http_upstream_wait_connect_response` can be set to "false" to disable this behavior. -* tls: removed RSA key transport and SHA-1 cipher suites from the client-side defaults. -* watchdog: the watchdog action :ref:`abort_action ` is now the default action to terminate the process if watchdog kill / multikill is enabled. -* xds: to support TTLs, heartbeating has been added to xDS. As a result, responses that contain empty resources without updating the version will no longer be propagated to the - subscribers. To undo this for VHDS (which is the only subscriber that wants empty resources), the `envoy.reloadable_features.vhds_heartbeats` can be set to "false". - Bug Fixes --------- *Changes expected to improve the state of the world and are unlikely to have negative effects* * active http health checks: properly handles HTTP/2 GOAWAY frames from the upstream. Previously a GOAWAY frame due to a graceful listener drain could cause improper failed health checks due to streams being refused by the upstream on a connection that is going away. To revert to old GOAWAY handling behavior, set the runtime feature `envoy.reloadable_features.health_check.graceful_goaway_handling` to false. -* config: validate that upgrade configs have a non-empty :ref:`upgrade_type `, fixing a bug where an errant "-" could result in unexpected behavior. -* dns: fix a bug where custom resolvers provided in configuration were not preserved after network issues. -* dns_filter: correctly associate DNS response IDs when multiple queries are received. -* grpc mux: fix sending node again after stream is reset when ::ref:`set_node_on_first_message_only ` is set. -* http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. -* http: reject requests with missing required headers after filter chain processing. -* http: sending CONNECT_ERROR for HTTP/2 where appropriate during CONNECT requests. -* proxy_proto: fixed a bug where the wrong downstream address got sent to upstream connections. -* proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy. -* sds: fix a bug that clusters sharing same sds target are marked active immediately. -* tls: fix detection of the upstream connection close event. -* tls: fix read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers. -* udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash. -* watchdog: touch the watchdog before most event loop operations to avoid misses when handling bursts of callbacks. Removed Config or Runtime ------------------------- *Normally occurs at the end of the* :ref:`deprecation period ` -* dispatcher: removed legacy socket read/write resumption code path and runtime guard `envoy.reloadable_features.activate_fds_next_event_loop`. -* ext_authz: removed auto ignore case in HTTP-based `ext_authz` header matching and the runtime guard `envoy.reloadable_features.ext_authz_http_service_enable_case_sensitive_string_matcher`. To ignore case, set the :ref:`ignore_case ` field to true. -* http: flip default HTTP/1 and HTTP/2 server codec implementations to new codecs that remove the use of exceptions for control flow. To revert to old codec behavior, set the runtime feature `envoy.reloadable_features.new_codec_behavior` to false. -* http: removed `envoy.reloadable_features.http1_flood_protection` and legacy code path for turning flood protection off. -* http: removed `envoy.reloadable_features.new_codec_behavior` and legacy codecs. - New Features ------------ -* compression: the :ref:`compressor ` filter adds support for compressing request payloads. Its configuration is unified with the :ref:`decompressor ` filter with two new fields for different directions - :ref:`requests ` and :ref:`responses `. The latter deprecates the old response-specific fields and, if used, roots the response-specific stats in `.compressor...response.*` instead of `.compressor...*`. -* config: added ability to flush stats when the admin's :ref:`/stats endpoint ` is hit instead of on a timer via :ref:`stats_flush_on_admin `. -* config: added new runtime feature `envoy.features.enable_all_deprecated_features` that allows the use of all deprecated features. -* crash support: added the ability to dump L4 connection data on crash. -* formatter: added new :ref:`text_format_source ` field to support format strings both inline and from a file. -* formatter: added support for custom date formatting to :ref:`%DOWNSTREAM_PEER_CERT_V_START% ` and :ref:`%DOWNSTREAM_PEER_CERT_V_END% `, similar to :ref:`%START_TIME% `. -* grpc: implemented header value syntax support when defining :ref:`initial metadata ` for gRPC-based `ext_authz` :ref:`HTTP ` and :ref:`network ` filters, and :ref:`ratelimit ` filters. -* grpc-json: added support for configuring :ref:`unescaping behavior ` for path components. -* hds: added support for delta updates in the :ref:`HealthCheckSpecifier `, making only the Endpoints and Health Checkers that changed be reconstructed on receiving a new message, rather than the entire HDS. -* health_check: added option to use :ref:`no_traffic_healthy_interval ` which allows a different no traffic interval when the host is healthy. -* http: added HCM :ref:`timeout config field ` to control how long a downstream has to finish sending headers before the stream is cancelled. -* http: added frame flood and abuse checks to the upstream HTTP/2 codec. This check is off by default and can be enabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to true. -* http: added :ref:`stripping any port from host header ` support. -* http: clusters now support selecting HTTP/1 or HTTP/2 based on ALPN, configurable via :ref:`alpn_config ` in the :ref:`http_protocol_options ` message. -* jwt_authn: added support for :ref:`per-route config `. -* jwt_authn: changed config field :ref:`issuer ` to be optional to comply with JWT `RFC `_ requirements. -* kill_request: added new :ref:`HTTP kill request filter `. -* listener: added an optional :ref:`default filter chain `. If this field is supplied, and none of the :ref:`filter_chains ` matches, this default filter chain is used to serve the connection. -* listener: added back the :ref:`use_original_dst field `. -* listener: added the :ref:`Listener.bind_to_port field `. -* log: added a new custom flag ``%_`` to the log pattern to print the actual message to log, but with escaped newlines. -* lua: added `downstreamDirectRemoteAddress()` and `downstreamLocalAddress()` APIs to :ref:`streamInfo() `. -* mongo_proxy: the list of commands to produce metrics for is now :ref:`configurable `. -* network: added a :ref:`timeout ` for incoming connections completing transport-level negotiation, including TLS and ALTS hanshakes. -* overload: add :ref:`envoy.overload_actions.reduce_timeouts ` overload action to enable scaling timeouts down with load. Scaling support :ref:`is limited ` to the HTTP connection and stream idle timeouts. -* ratelimit: added support for use of various :ref:`metadata ` as a ratelimit action. -* ratelimit: added :ref:`disable_x_envoy_ratelimited_header ` option to disable `X-Envoy-RateLimited` header. -* ratelimit: added :ref:`body ` field to support custom response bodies for non-OK responses from the external ratelimit service. -* ratelimit: added :ref:`dynamic_metadata ` field to support setting dynamic metadata from the ratelimit service. -* router: added support for regex rewrites during HTTP redirects using :ref:`regex_rewrite `. -* sds: improved support for atomic :ref:`key rotations ` and added configurable rotation triggers for - :ref:`TlsCertificate ` and - :ref:`CertificateValidationContext `. -* signal: added an extension point for custom actions to run on the thread that has encountered a fatal error. Actions are configurable via :ref:`fatal_actions `. -* start_tls: :ref:`transport socket` which starts in clear-text but may programatically be converted to use tls. -* tcp: added a new :ref:`envoy.overload_actions.reject_incoming_connections ` action to reject incoming TCP connections. -* thrift_proxy: added a new :ref: `payload_passthrough ` option to skip decoding body in the Thrift message. -* tls: added support for RSA certificates with 4096-bit keys in FIPS mode. -* tracing: added SkyWalking tracer. -* tracing: added support for setting the hostname used when sending spans to a Zipkin collector using the :ref:`collector_hostname ` field. -* xds: added support for resource TTLs. A TTL is specified on the :ref:`Resource `. For SotW, a :ref:`Resource ` can be embedded - in the list of resources to specify the TTL. Deprecated ----------- -* cluster: HTTP configuration for upstream clusters has beem reworked. HTTP-specific configuration is now done in the new :ref:`http_protocol_options ` message, configured via the cluster's :ref:`extension_protocol_options`. This replaces explicit HTTP configuration in cluster config, including :ref:`upstream_http_protocol_options` :ref:`common_http_protocol_options` :ref:`http_protocol_options` :ref:`http2_protocol_options` and :ref:`protocol_selection`. Examples of before-and-after configuration can be found in the :ref:`http_protocol_options docs ` and all of Envoy's example configurations have been updated to the new style of config. -* compression: the fields :ref:`content_length `, :ref:`content_type `, :ref:`disable_on_etag_header `, :ref:`remove_accept_encoding_header ` and :ref:`runtime_enabled ` of the :ref:`Compressor ` message have been deprecated in favor of :ref:`response_direction_config `. -* formatter: :ref:`text_format ` is now deprecated in favor of :ref:`text_format_source `. To migrate existing text format strings, use the :ref:`inline_string ` field. -* gzip: :ref:`HTTP Gzip filter ` is rejected now unless explicitly allowed with :ref:`runtime override ` `envoy.deprecated_features.allow_deprecated_gzip_http_filter` set to `true`. -* listener: :ref:`use_proxy_proto ` has been deprecated in favor of adding a :ref:`PROXY protocol listener filter ` explicitly. -* logging: the `--log-format-prefix-with-location` option is removed. -* ratelimit: the :ref:`dynamic metadata ` action is deprecated in favor of the more generic :ref:`metadata ` action. -* stats: the `--use-fake-symbol-table` option is removed. -* tracing: OpenCensus :ref:`Zipkin configuration ` is now deprecated, the preferred Zipkin export is via Envoy's :ref:`native Zipkin tracer `. +---------- \ No newline at end of file diff --git a/docs/root/version_history/v1.17.0.rst b/docs/root/version_history/v1.17.0.rst new file mode 100644 index 000000000000..dc4a84c02a2f --- /dev/null +++ b/docs/root/version_history/v1.17.0.rst @@ -0,0 +1,121 @@ +1.17.0 (January 11, 2021) +========================= + +Incompatible Behavior Changes +----------------------------- +*Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* + +* config: v2 is now fatal-by-default. This may be overridden by setting :option:`--bootstrap-version` 2 on the CLI for a v2 bootstrap file and also enabling the runtime `envoy.reloadable_features.enable_deprecated_v2_api` feature. See + the :ref:`FAQ entry ` for further details. + +Minor Behavior Changes +---------------------- +*Changes that may cause incompatibilities for some users, but should not for most* + +* build: the Alpine based debug images are no longer built in CI, use Ubuntu based images instead. +* decompressor: set the default value of window_bits of the decompressor to 15 to be able to decompress responses compressed by a compressor with any window size. +* expr filter: added `connection.termination_details` property support. +* formatter: the :ref:`text_format ` field no longer requires at least one byte, and may now be the empty string. It has also become :ref:`deprecated <1_17_deprecated>`. +* grpc_web filter: if a `grpc-accept-encoding` header is present it's passed as-is to the upstream and if it isn't `grpc-accept-encoding:identity` is sent instead. The header was always overwriten with `grpc-accept-encoding:identity,deflate,gzip` before. +* http: upstream protocol will now only be logged if an upstream stream was established. +* jwt_authn filter: added support of JWT time constraint verification with a clock skew (default to 60 seconds) and added a filter config field :ref:`clock_skew_seconds ` to configure it. +* listener: injection of the :ref:`TLS inspector ` has been disabled by default. This feature is controlled by the runtime guard `envoy.reloadable_features.disable_tls_inspector_injection`. +* lua: added `always_wrap_body` argument to `body()` API to always return a :ref:`buffer object ` even if the body is empty. +* memory: enabled new tcmalloc with restartable sequences for aarch64 builds. +* mongo proxy metrics: swapped network connection remote and local closed counters previously set reversed (`cx_destroy_local_with_active_rq` and `cx_destroy_remote_with_active_rq`). +* outlier detection: added :ref:`max_ejection_time ` to limit ejection time growth when a node stays unhealthy for extended period of time. By default :ref:`max_ejection_time ` limits ejection time to 5 minutes. Additionally, when the node stays healthy, ejection time decreases. See :ref:`ejection algorithm` for more info. Previously, ejection time could grow without limit and never decreased. +* performance: improved performance when handling large HTTP/1 bodies. +* tcp_proxy: now waits for HTTP tunnel to be established before start streaming the downstream data, the runtime guard `envoy.reloadable_features.http_upstream_wait_connect_response` can be set to "false" to disable this behavior. +* tls: removed RSA key transport and SHA-1 cipher suites from the client-side defaults. +* watchdog: the watchdog action :ref:`abort_action ` is now the default action to terminate the process if watchdog kill / multikill is enabled. +* xds: to support TTLs, heartbeating has been added to xDS. As a result, responses that contain empty resources without updating the version will no longer be propagated to the + subscribers. To undo this for VHDS (which is the only subscriber that wants empty resources), the `envoy.reloadable_features.vhds_heartbeats` can be set to "false". + +Bug Fixes +--------- +*Changes expected to improve the state of the world and are unlikely to have negative effects* + +* config: validate that upgrade configs have a non-empty :ref:`upgrade_type `, fixing a bug where an errant "-" could result in unexpected behavior. +* dns: fixed a bug where custom resolvers provided in configuration were not preserved after network issues. +* dns_filter: correctly associate DNS response IDs when multiple queries are received. +* grpc mux: fixed sending node again after stream is reset when :ref:`set_node_on_first_message_only ` is set. +* http: fixed URL parsing for HTTP/1.1 fully qualified URLs and connect requests containing IPv6 addresses. +* http: reject requests with missing required headers after filter chain processing. +* http: sending CONNECT_ERROR for HTTP/2 where appropriate during CONNECT requests. +* proxy_proto: fixed a bug where the wrong downstream address got sent to upstream connections. +* proxy_proto: fixed a bug where network filters would not have the correct downstreamRemoteAddress() when accessed from the StreamInfo. This could result in incorrect enforcement of RBAC rules in the RBAC network filter (but not in the RBAC HTTP filter), or incorrect access log addresses from tcp_proxy. +* sds: fixed a bug that clusters sharing same sds target are marked active immediately. +* tls: fixed detection of the upstream connection close event. +* tls: fixed read resumption after triggering buffer high-watermark and all remaining request/response bytes are stored in the SSL connection's internal buffers. +* udp: fixed issue in which receiving truncated UDP datagrams would cause Envoy to crash. +* watchdog: touch the watchdog before most event loop operations to avoid misses when handling bursts of callbacks. + +Removed Config or Runtime +------------------------- +*Normally occurs at the end of the* :ref:`deprecation period ` + +* dispatcher: removed legacy socket read/write resumption code path and runtime guard `envoy.reloadable_features.activate_fds_next_event_loop`. +* ext_authz: removed auto ignore case in HTTP-based `ext_authz` header matching and the runtime guard `envoy.reloadable_features.ext_authz_http_service_enable_case_sensitive_string_matcher`. To ignore case, set the :ref:`ignore_case ` field to true. +* ext_authz: the deprecated field `use_alpha` is no longer supported and cannot be set anymore. +* http: removed `envoy.reloadable_features.http1_flood_protection` and legacy code path for turning flood protection off. +* http: removed `envoy.reloadable_features.new_codec_behavior` and legacy codecs. + +New Features +------------ +* compression: the :ref:`compressor ` filter added support for compressing request payloads. Its configuration is unified with the :ref:`decompressor ` filter with two new fields for different directions - :ref:`requests ` and :ref:`responses `. The latter deprecates the old response-specific fields and, if used, roots the response-specific stats in `.compressor...response.*` instead of `.compressor...*`. +* config: added ability to flush stats when the admin's :ref:`/stats endpoint ` is hit instead of on a timer via :ref:`stats_flush_on_admin `. +* config: added new runtime feature `envoy.features.enable_all_deprecated_features` that allows the use of all deprecated features. +* crash support: added the ability to dump L4 connection data on crash. +* formatter: added new :ref:`text_format_source ` field to support format strings both inline and from a file. +* formatter: added support for custom date formatting to :ref:`%DOWNSTREAM_PEER_CERT_V_START% ` and :ref:`%DOWNSTREAM_PEER_CERT_V_END% `, similar to :ref:`%START_TIME% `. +* grpc: implemented header value syntax support when defining :ref:`initial metadata ` for gRPC-based `ext_authz` :ref:`HTTP ` and :ref:`network ` filters, and :ref:`ratelimit ` filters. +* grpc-json: added support for configuring :ref:`unescaping behavior ` for path components. +* hds: added support for delta updates in the :ref:`HealthCheckSpecifier `, making only the Endpoints and Health Checkers that changed be reconstructed on receiving a new message, rather than the entire HDS. +* health_check: added option to use :ref:`no_traffic_healthy_interval ` which allows a different no traffic interval when the host is healthy. +* http: added HCM :ref:`request_headers_timeout config field ` to control how long a downstream has to finish sending headers before the stream is cancelled. +* http: added frame flood and abuse checks to the upstream HTTP/2 codec. This check is off by default and can be enabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to true. +* http: added :ref:`stripping any port from host header ` support. +* http: clusters added support for selecting HTTP/1 or HTTP/2 based on ALPN, configurable via :ref:`alpn_config ` in the :ref:`http_protocol_options ` message. +* jwt_authn: added support for :ref:`per-route config `. +* jwt_authn: changed config field :ref:`issuer ` to be optional to comply with JWT `RFC `_ requirements. +* kill_request: added new :ref:`HTTP kill request filter `. +* listener: added an optional :ref:`default filter chain `. If this field is supplied, and none of the :ref:`filter_chains ` matches, this default filter chain is used to serve the connection. +* listener: added back the :ref:`use_original_dst field `. +* listener: added the :ref:`Listener.bind_to_port field `. +* log: added a new custom flag ``%_`` to the log pattern to print the actual message to log, but with escaped newlines. +* lua: added `downstreamDirectRemoteAddress()` and `downstreamLocalAddress()` APIs to :ref:`streamInfo() `. +* mongo_proxy: the list of commands to produce metrics for is now :ref:`configurable `. +* network: added a :ref:`transport_socket_connect_timeout config field ` for incoming connections completing transport-level negotiation, including TLS and ALTS hanshakes. +* overload: added :ref:`envoy.overload_actions.reduce_timeouts ` overload action to enable scaling timeouts down with load. Scaling support :ref:`is limited ` to the HTTP connection and stream idle timeouts. +* ratelimit: added support for use of various :ref:`metadata ` as a ratelimit action. +* ratelimit: added :ref:`disable_x_envoy_ratelimited_header ` option to disable `X-Envoy-RateLimited` header. +* ratelimit: added :ref:`body ` field to support custom response bodies for non-OK responses from the external ratelimit service. +* ratelimit: added :ref:`descriptor extensions `. +* ratelimit: added :ref:`computed descriptors `. +* ratelimit: added :ref:`dynamic_metadata ` field to support setting dynamic metadata from the ratelimit service. +* router: added support for regex rewrites during HTTP redirects using :ref:`regex_rewrite `. +* sds: improved support for atomic :ref:`key rotations ` and added configurable rotation triggers for + :ref:`TlsCertificate ` and + :ref:`CertificateValidationContext `. +* signal: added an extension point for custom actions to run on the thread that has encountered a fatal error. Actions are configurable via :ref:`fatal_actions `. +* start_tls: added new :ref:`transport socket` which starts in clear-text but may programatically be converted to use tls. +* tcp: added a new :ref:`envoy.overload_actions.reject_incoming_connections ` action to reject incoming TCP connections. +* thrift_proxy: added a new :ref:`payload_passthrough ` option to skip decoding body in the Thrift message. +* tls: added support for RSA certificates with 4096-bit keys in FIPS mode. +* tracing: added :ref:`SkyWalking tracer `. +* tracing: added support for setting the hostname used when sending spans to a Zipkin collector using the :ref:`collector_hostname ` field. +* xds: added support for resource TTLs. A TTL is specified on the :ref:`Resource `. For SotW, a :ref:`Resource ` can be embedded in the list of resources to specify the TTL. + +.. _1_17_deprecated: + +Deprecated +---------- +* cluster: HTTP configuration for upstream clusters has been reworked. HTTP-specific configuration is now done in the new :ref:`http_protocol_options ` message, configured via the cluster's :ref:`extension_protocol_options`. This replaces explicit HTTP configuration in cluster config, including :ref:`upstream_http_protocol_options` :ref:`common_http_protocol_options` :ref:`http_protocol_options` :ref:`http2_protocol_options` and :ref:`protocol_selection`. Examples of before-and-after configuration can be found in the :ref:`http_protocol_options docs ` and all of Envoy's example configurations have been updated to the new style of config. +* compression: the fields :ref:`content_length `, :ref:`content_type `, :ref:`disable_on_etag_header `, :ref:`remove_accept_encoding_header ` and :ref:`runtime_enabled ` of the :ref:`Compressor ` message have been deprecated in favor of :ref:`response_direction_config `. +* formatter: :ref:`text_format ` is now deprecated in favor of :ref:`text_format_source `. To migrate existing text format strings, use the :ref:`inline_string ` field. +* gzip: :ref:`HTTP Gzip filter ` is rejected now unless explicitly allowed with :ref:`runtime override ` `envoy.deprecated_features.allow_deprecated_gzip_http_filter` set to `true`. Use the :ref:`compressor filter `. +* listener: :ref:`use_proxy_proto ` has been deprecated in favor of adding a :ref:`PROXY protocol listener filter ` explicitly. +* logging: the `--log-format-prefix-with-location` option is removed. +* ratelimit: the :ref:`dynamic metadata ` action is deprecated in favor of the more generic :ref:`metadata ` action. +* stats: the `--use-fake-symbol-table` option is removed. +* tracing: OpenCensus :ref:`Zipkin configuration ` is now deprecated, the preferred Zipkin export is via Envoy's :ref:`native Zipkin tracer `. diff --git a/docs/root/version_history/version_history.rst b/docs/root/version_history/version_history.rst index b25ef731208a..32c63cf7a0dc 100644 --- a/docs/root/version_history/version_history.rst +++ b/docs/root/version_history/version_history.rst @@ -7,6 +7,7 @@ Version history :titlesonly: current + v1.17.0 v1.16.2 v1.16.1 v1.16.0 diff --git a/examples/wasm-cc/docker-compose-wasm.yaml b/examples/wasm-cc/docker-compose-wasm.yaml index 747530edcabd..7dd2490bb4ed 100644 --- a/examples/wasm-cc/docker-compose-wasm.yaml +++ b/examples/wasm-cc/docker-compose-wasm.yaml @@ -3,7 +3,7 @@ version: "3.7" services: wasm_compile_update: - image: envoyproxy/envoy-build-ubuntu:9400637f4aa0232465407447bfda0d3da13549fb + image: envoyproxy/envoy-build-ubuntu:11efa5680d987fff33fde4af3cc5ece105015d04 command: | bash -c "bazel build //examples/wasm-cc:envoy_filter_http_wasm_updated_example.wasm \ && cp -a bazel-bin/examples/wasm-cc/* /build" @@ -13,7 +13,7 @@ services: - ./lib:/build wasm_compile: - image: envoyproxy/envoy-build-ubuntu:9400637f4aa0232465407447bfda0d3da13549fb + image: envoyproxy/envoy-build-ubuntu:11efa5680d987fff33fde4af3cc5ece105015d04 command: | bash -c "bazel build //examples/wasm-cc:envoy_filter_http_wasm_example.wasm \ && cp -a bazel-bin/examples/wasm-cc/* /build" diff --git a/generated_api_shadow/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/generated_api_shadow/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index db188a572ae0..b9a807d82edb 100644 --- a/generated_api_shadow/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/generated_api_shadow/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -49,10 +49,7 @@ message ExtAuthz { // `. bool failure_mode_allow = 2; - // Sets the package version the gRPC service should use. This is particularly - // useful when transitioning from alpha to release versions assuming that both definitions are - // semantically compatible. Deprecation note: This field is deprecated and should only be used for - // version upgrade. See release notes for more details. + // [#not-implemented-hide: Support for this field has been removed.] bool use_alpha = 4 [deprecated = true, (envoy.annotations.disallowed_by_default) = true]; // Enables filter to buffer the client request body and send it within the authorization request. diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index e90d3bce7e05..bfb296e47836 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -1556,7 +1556,7 @@ message VirtualCluster { message RateLimit { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit"; - // [#next-free-field: 9] + // [#next-free-field: 10] message Action { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit.Action"; @@ -1754,6 +1754,9 @@ message RateLimit { // Rate limit on metadata. MetaData metadata = 8; + + // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + core.v3.TypedExtensionConfig extension = 9; } } diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index 935610424b1a..586527865d5b 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -1560,7 +1560,7 @@ message VirtualCluster { message RateLimit { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RateLimit"; - // [#next-free-field: 9] + // [#next-free-field: 10] message Action { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RateLimit.Action"; @@ -1764,6 +1764,9 @@ message RateLimit { // Rate limit on metadata. MetaData metadata = 8; + + // Rate limit descriptor extension. See the rate limit descriptor extensions documentation. + core.v4alpha.TypedExtensionConfig extension = 9; } } diff --git a/generated_api_shadow/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto b/generated_api_shadow/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto index 1e64376add9b..9f5c952a57ea 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto @@ -21,7 +21,22 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.filters.http.ext_proc] // The External Processing filter allows an external service to act on HTTP traffic in a flexible way. -// It communicates with an external gRPC service that can use it to do a variety of things + +// **Current Implementation Status:** +// At this time, the filter will send a "request_headers" message to the server when the +// filter is invoked from the downstream, and apply any header mutations returned by the +// server. No other part of the protocol is implemented yet. + +// As designed, the filter supports up to six different processing steps, which are in the +// process of being implemented: +// * Request headers: IMPLEMENTED +// * Request body: NOT IMPLEMENTED +// * Request trailers: NOT IMPLEMENTED +// * Response headers: NOT IMPLEMENTED +// * Response body: NOT IMPLEMENTED +// * Response trailers: NOT IMPLEMENTED + +// The filter communicates with an external gRPC service that can use it to do a variety of things // with the request and response: // // * Access and modify the HTTP headers on the request, response, or both diff --git a/generated_api_shadow/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD b/generated_api_shadow/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD new file mode 100644 index 000000000000..facd82ce6de2 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", + ], +) diff --git a/generated_api_shadow/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto b/generated_api_shadow/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto new file mode 100644 index 000000000000..76d3505cba04 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package envoy.extensions.rate_limit_descriptors.expr.v3; + +import "google/api/expr/v1alpha1/syntax.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.rate_limit_descriptors.expr.v3"; +option java_outer_classname = "ExprProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Rate limit descriptor expression] +// [#extension: envoy.rate_limit_descriptors.expr] + +// The following descriptor entry is appended with a value computed +// from a symbolic Common Expression Language expression. +// See :ref:`attributes ` for the set of +// available attributes. +// +// .. code-block:: cpp +// +// ("", "") +message Descriptor { + // The key to use in the descriptor entry. + string descriptor_key = 1 [(validate.rules).string = {min_len: 1}]; + + // If set to true, Envoy skips the descriptor if the expression evaluates to an error. + // By default, the rate limit is not applied when an expression produces an error. + bool skip_if_error = 2; + + oneof expr_specifier { + // Expression in a text form, e.g. "connection.requested_server_name". + string text = 3 [(validate.rules).string = {min_len: 1}]; + + // Parsed expression in AST form. + google.api.expr.v1alpha1.Expr parsed = 4; + } +} diff --git a/generated_api_shadow/envoy/service/ext_proc/v3alpha/external_processor.proto b/generated_api_shadow/envoy/service/ext_proc/v3alpha/external_processor.proto index cac1dde34d9e..5b0696bfc3b0 100644 --- a/generated_api_shadow/envoy/service/ext_proc/v3alpha/external_processor.proto +++ b/generated_api_shadow/envoy/service/ext_proc/v3alpha/external_processor.proto @@ -6,7 +6,6 @@ import "envoy/config/core/v3/base.proto"; import "envoy/extensions/filters/http/ext_proc/v3alpha/processing_mode.proto"; import "envoy/type/v3/http_status.proto"; -import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "udpa/annotations/status.proto"; @@ -289,10 +288,13 @@ message GrpcStatus { // Change HTTP headers or trailers by appending, replacing, or removing // headers. message HeaderMutation { - // Add or replace HTTP headers. + // Add or replace HTTP headers. Attempts to set the value of + // any "x-envoy" header, and attempts to set the ":method", + // ":authority", ":scheme", or "host" headers will be ignored. repeated config.core.v3.HeaderValueOption set_headers = 1; - // Remove these HTTP headers. + // Remove these HTTP headers. Attempts to remove system headers -- + // any header starting with ":", plus "host" -- will be ignored. repeated string remove_headers = 2; } diff --git a/generated_api_shadow/envoy/type/matcher/v3/http_inputs.proto b/generated_api_shadow/envoy/type/matcher/v3/http_inputs.proto new file mode 100644 index 000000000000..48eb4c43154a --- /dev/null +++ b/generated_api_shadow/envoy/type/matcher/v3/http_inputs.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package envoy.type.matcher.v3; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v3"; +option java_outer_classname = "HttpInputsProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Common HTTP Inputs] + +// Match input indicates that matching should be done on a specific request header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the request contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpRequestHeaderMatchInput { + // The request header to match on. + string header_name = 1; +} + +// Match input indicating that matching should be done on a specific response header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the response contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpResponseHeaderMatchInput { + // The response header to match on. + string header_name = 1; +} diff --git a/generated_api_shadow/envoy/type/matcher/v4alpha/http_inputs.proto b/generated_api_shadow/envoy/type/matcher/v4alpha/http_inputs.proto new file mode 100644 index 000000000000..f15e9ceb70e2 --- /dev/null +++ b/generated_api_shadow/envoy/type/matcher/v4alpha/http_inputs.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package envoy.type.matcher.v4alpha; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.type.matcher.v4alpha"; +option java_outer_classname = "HttpInputsProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Common HTTP Inputs] + +// Match input indicates that matching should be done on a specific request header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the request contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpRequestHeaderMatchInput { + option (udpa.annotations.versioning).previous_message_type = + "envoy.type.matcher.v3.HttpRequestHeaderMatchInput"; + + // The request header to match on. + string header_name = 1; +} + +// Match input indicating that matching should be done on a specific response header. +// The resulting input string will be all headers for the given key joined by a comma, +// e.g. if the response contains two 'foo' headers with value 'bar' and 'baz', the input +// string will be 'bar,baz'. +// [#comment:TODO(snowp): Link to unified matching docs.] +message HttpResponseHeaderMatchInput { + option (udpa.annotations.versioning).previous_message_type = + "envoy.type.matcher.v3.HttpResponseHeaderMatchInput"; + + // The response header to match on. + string header_name = 1; +} diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index 4157dfc6819f..ecb39f22dcb0 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -411,6 +411,12 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { */ virtual void encode100ContinueHeaders(ResponseHeaderMapPtr&& headers) PURE; + /** + * Returns the 100-Continue headers provided to encode100ContinueHeaders. Returns absl::nullopt if + * no headers have been provided yet. + */ + virtual ResponseHeaderMapOptRef continueHeaders() const PURE; + /** * Called with headers to be encoded, optionally indicating end of stream. * @@ -427,6 +433,12 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { virtual void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) PURE; + /** + * Returns the headers provided to encodeHeaders. Returns absl::nullopt if no headers have been + * provided yet. + */ + virtual ResponseHeaderMapOptRef responseHeaders() const PURE; + /** * Called with data to be encoded, optionally indicating end of stream. * @param data supplies the data to be encoded. @@ -440,6 +452,12 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { */ virtual void encodeTrailers(ResponseTrailerMapPtr&& trailers) PURE; + /** + * Returns the trailers provided to encodeTrailers. Returns absl::nullopt if no headers have been + * provided yet. + */ + virtual ResponseTrailerMapOptRef responseTrailers() const PURE; + /** * Called with metadata to be encoded. * @@ -867,6 +885,8 @@ using StreamFilterSharedPtr = std::shared_ptr; class HttpMatchingData { public: + static absl::string_view name() { return "http"; } + virtual ~HttpMatchingData() = default; virtual RequestHeaderMapOptConstRef requestHeaders() const PURE; diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index 8df40a404e82..e3262e3ecfb0 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -786,6 +786,7 @@ class ResponseTrailerMap public CustomInlineHeaderBase {}; using ResponseTrailerMapPtr = std::unique_ptr; using ResponseTrailerMapOptRef = OptRef; +using ResponseTrailerMapOptConstRef = OptRef; /** * Convenient container type for storing Http::LowerCaseString and std::string key/value pairs. diff --git a/include/envoy/matcher/BUILD b/include/envoy/matcher/BUILD index 838e7fd5e0c1..fadfc109afb0 100644 --- a/include/envoy/matcher/BUILD +++ b/include/envoy/matcher/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( ], deps = [ "//include/envoy/config:typed_config_interface", + "//include/envoy/protobuf:message_validator_interface", "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/include/envoy/matcher/matcher.h b/include/envoy/matcher/matcher.h index 613274ce389a..678eed64023b 100644 --- a/include/envoy/matcher/matcher.h +++ b/include/envoy/matcher/matcher.h @@ -7,6 +7,7 @@ #include "envoy/config/common/matcher/v3/matcher.pb.h" #include "envoy/config/core/v3/extension.pb.h" #include "envoy/config/typed_config.h" +#include "envoy/protobuf/message_validator.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -209,7 +210,9 @@ template class DataInputFactory : public Config::TypedFactory { /** * Creates a DataInput from the provided config. */ - virtual DataInputPtr createDataInput(const Protobuf::Message& config) PURE; + virtual DataInputPtr + createDataInput(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validation_visitor) PURE; /** * The category of this factory depends on the DataType, so we require a name() function to exist diff --git a/include/envoy/ratelimit/BUILD b/include/envoy/ratelimit/BUILD index 615b69fa3107..96c325023043 100644 --- a/include/envoy/ratelimit/BUILD +++ b/include/envoy/ratelimit/BUILD @@ -12,6 +12,10 @@ envoy_cc_library( name = "ratelimit_interface", hdrs = ["ratelimit.h"], deps = [ + "//include/envoy/config:typed_config_interface", + "//include/envoy/http:header_map_interface", + "//include/envoy/protobuf:message_validator_interface", + "//include/envoy/stream_info:stream_info_interface", "@envoy_api//envoy/type/v3:pkg_cc_proto", ], ) diff --git a/include/envoy/ratelimit/ratelimit.h b/include/envoy/ratelimit/ratelimit.h index f23c8170ef68..e10641400310 100644 --- a/include/envoy/ratelimit/ratelimit.h +++ b/include/envoy/ratelimit/ratelimit.h @@ -3,6 +3,10 @@ #include #include +#include "envoy/config/typed_config.h" +#include "envoy/http/header_map.h" +#include "envoy/protobuf/message_validator.h" +#include "envoy/stream_info/stream_info.h" #include "envoy/type/v3/ratelimit_unit.pb.h" #include "absl/types/optional.h" @@ -34,5 +38,50 @@ struct Descriptor { absl::optional limit_ = absl::nullopt; }; +/** + * Base interface for generic rate limit descriptor producer. + */ +class DescriptorProducer { +public: + virtual ~DescriptorProducer() = default; + + /** + * Potentially append a descriptor entry to the end of descriptor. + * @param descriptor supplies the descriptor to optionally fill. + * @param local_service_cluster supplies the name of the local service cluster. + * @param headers supplies the header for the request. + * @param info stream info associated with the request + * @return true if the producer populated the descriptor. + */ + virtual bool populateDescriptor(Descriptor& descriptor, const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const PURE; +}; + +using DescriptorProducerPtr = std::unique_ptr; + +/** + * Implemented by each custom rate limit descriptor extension and registered via + * Registry::registerFactory() or the convenience class RegisterFactory. + */ +class DescriptorProducerFactory : public Config::TypedFactory { +public: + ~DescriptorProducerFactory() override = default; + + /** + * Creates a particular DescriptorProducer implementation. + * + * @param config supplies the configuration for the descriptor extension. + * @param validator configuration validation visitor. + * @return DescriptorProducerPtr the rate limit descriptor producer which will be used to populate + * rate limit descriptors. + */ + virtual DescriptorProducerPtr + createDescriptorProducerFromProto(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validator) PURE; + + std::string category() const override { return "envoy.rate_limit_descriptors"; } +}; + } // namespace RateLimit } // namespace Envoy diff --git a/include/envoy/router/BUILD b/include/envoy/router/BUILD index ed2a65c67c8a..754412a7ee34 100644 --- a/include/envoy/router/BUILD +++ b/include/envoy/router/BUILD @@ -97,7 +97,6 @@ envoy_cc_library( hdrs = ["router_ratelimit.h"], deps = [ "//include/envoy/http:filter_interface", - "//include/envoy/http:header_map_interface", "//include/envoy/ratelimit:ratelimit_interface", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/include/envoy/router/router_ratelimit.h b/include/envoy/router/router_ratelimit.h index 1e6910c3b9ba..8a4ef25f6d6d 100644 --- a/include/envoy/router/router_ratelimit.h +++ b/include/envoy/router/router_ratelimit.h @@ -7,7 +7,6 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/http/filter.h" -#include "envoy/http/header_map.h" #include "envoy/ratelimit/ratelimit.h" namespace Envoy { @@ -32,32 +31,6 @@ class RateLimitOverrideAction { using RateLimitOverrideActionPtr = std::unique_ptr; -/** - * Base interface for generic rate limit action. - */ -class RateLimitAction { -public: - virtual ~RateLimitAction() = default; - - /** - * Potentially append a descriptor entry to the end of descriptor. - * @param route supplies the target route for the request. - * @param descriptor supplies the descriptor to optionally fill. - * @param local_service_cluster supplies the name of the local service cluster. - * @param headers supplies the header for the request. - * @param remote_address supplies the trusted downstream address for the connection. - * @param dynamic_metadata supplies the dynamic metadata for the request - * @return true if the RateLimitAction populated the descriptor. - */ - virtual bool - populateDescriptor(const RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const PURE; -}; - -using RateLimitActionPtr = std::unique_ptr; - /** * Rate limit configuration. */ @@ -77,18 +50,15 @@ class RateLimitPolicyEntry { /** * Potentially populate the descriptor array with new descriptors to query. - * @param route supplies the target route for the request. * @param descriptors supplies the descriptor array to optionally fill. * @param local_service_cluster supplies the name of the local service cluster. * @param headers supplies the header for the request. - * @param remote_address supplies the trusted downstream address for the connection. - * @param dynamic_metadata supplies the dynamic metadata for the request. + * @param info stream info associated with the request */ - virtual void - populateDescriptors(const RouteEntry& route, std::vector& descriptors, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const PURE; + virtual void populateDescriptors(std::vector& descriptors, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const PURE; }; /** diff --git a/source/common/http/BUILD b/source/common/http/BUILD index e19d26c058f4..a1917b42809c 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -184,6 +184,8 @@ envoy_cc_library( "//source/common/local_reply:local_reply_lib", "//source/common/matcher:matcher_lib", "@envoy_api//envoy/extensions/filters/common/matcher/action/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", ], ) @@ -247,6 +249,7 @@ envoy_cc_library( "//source/common/http/http2:codec_lib", "//source/common/http/http3:quic_codec_factory_lib", "//source/common/http/http3:well_known_names", + "//source/common/http/match_wrapper:config", "//source/common/network:utility_lib", "//source/common/router:config_lib", "//source/common/router:scoped_rds_lib", diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 00fc7d09a5dc..9cee1e0399e7 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -31,6 +31,7 @@ #include "envoy/upstream/load_balancer.h" #include "envoy/upstream/upstream.h" +#include "common/common/assert.h" #include "common/common/empty_string.h" #include "common/common/linked_object.h" #include "common/http/message_impl.h" @@ -398,10 +399,13 @@ class AsyncStreamImpl : public AsyncClient::Stream, // The async client won't pause if sending an Expect: 100-Continue so simply // swallows any incoming encode100Continue. void encode100ContinueHeaders(ResponseHeaderMapPtr&&) override {} + ResponseHeaderMapOptRef continueHeaders() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) override; + ResponseHeaderMapOptRef responseHeaders() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(ResponseTrailerMapPtr&& trailers) override; + ResponseTrailerMapOptRef responseTrailers() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void encodeMetadata(MetadataMapPtr&&) override {} void onDecoderFilterAboveWriteBufferHighWatermark() override { ++high_watermark_calls_; } void onDecoderFilterBelowWriteBufferLowWatermark() override { diff --git a/source/common/http/filter_manager.cc b/source/common/http/filter_manager.cc index fe579cebefdd..bde33e6d4520 100644 --- a/source/common/http/filter_manager.cc +++ b/source/common/http/filter_manager.cc @@ -15,6 +15,8 @@ namespace Envoy { namespace Http { namespace { +REGISTER_FACTORY(HttpRequestHeadersDataInputFactory, Matcher::DataInputFactory); +REGISTER_FACTORY(SkipActionFactory, Matcher::ActionFactory); template using FilterList = std::list>; @@ -368,6 +370,10 @@ void ActiveStreamDecoderFilter::encode100ContinueHeaders(ResponseHeaderMapPtr&& } } +ResponseHeaderMapOptRef ActiveStreamDecoderFilter::continueHeaders() const { + return parent_.filter_manager_callbacks_.continueHeaders(); +} + void ActiveStreamDecoderFilter::encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) { parent_.stream_info_.setResponseCodeDetails(details); @@ -375,6 +381,10 @@ void ActiveStreamDecoderFilter::encodeHeaders(ResponseHeaderMapPtr&& headers, bo parent_.encodeHeaders(nullptr, *parent_.filter_manager_callbacks_.responseHeaders(), end_stream); } +ResponseHeaderMapOptRef ActiveStreamDecoderFilter::responseHeaders() const { + return parent_.filter_manager_callbacks_.responseHeaders(); +} + void ActiveStreamDecoderFilter::encodeData(Buffer::Instance& data, bool end_stream) { parent_.encodeData(nullptr, data, end_stream, FilterManager::FilterIterationStartState::CanStartFromCurrent); @@ -385,6 +395,10 @@ void ActiveStreamDecoderFilter::encodeTrailers(ResponseTrailerMapPtr&& trailers) parent_.encodeTrailers(nullptr, *parent_.filter_manager_callbacks_.responseTrailers()); } +ResponseTrailerMapOptRef ActiveStreamDecoderFilter::responseTrailers() const { + return parent_.filter_manager_callbacks_.responseTrailers(); +} + void ActiveStreamDecoderFilter::encodeMetadata(MetadataMapPtr&& metadata_map_ptr) { parent_.encodeMetadata(nullptr, std::move(metadata_map_ptr)); } diff --git a/source/common/http/filter_manager.h b/source/common/http/filter_manager.h index 3047772768b4..06b9e4ee65ce 100644 --- a/source/common/http/filter_manager.h +++ b/source/common/http/filter_manager.h @@ -4,10 +4,14 @@ #include "envoy/common/optref.h" #include "envoy/extensions/filters/common/matcher/action/v3/skip_action.pb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h" #include "envoy/http/filter.h" #include "envoy/http/header_map.h" #include "envoy/matcher/matcher.h" #include "envoy/network/socket.h" +#include "envoy/protobuf/message_validator.h" +#include "envoy/type/matcher/v3/http_inputs.pb.validate.h" #include "common/buffer/watermark_buffer.h" #include "common/common/dump_state_utils.h" @@ -18,6 +22,7 @@ #include "common/http/headers.h" #include "common/local_reply/local_reply.h" #include "common/matcher/matcher.h" +#include "common/protobuf/utility.h" namespace Envoy { namespace Http { @@ -97,6 +102,36 @@ class HttpRequestHeadersDataInput : public HttpHeadersDataInputBase +class HttpHeadersDataInputFactoryBase : public Matcher::DataInputFactory { +public: + explicit HttpHeadersDataInputFactoryBase(const std::string& name) : name_(name) {} + + std::string name() const override { return name_; } + + Matcher::DataInputPtr + createDataInput(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validation_visitor) override { + const auto& typed_config = + MessageUtil::downcastAndValidate(config, validation_visitor); + + return std::make_unique(typed_config.header_name()); + }; + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + +private: + const std::string name_; +}; + +class HttpRequestHeadersDataInputFactory + : public HttpHeadersDataInputFactoryBase< + HttpRequestHeadersDataInput, envoy::type::matcher::v3::HttpRequestHeaderMatchInput> { +public: + HttpRequestHeadersDataInputFactory() : HttpHeadersDataInputFactoryBase("request-headers") {} +}; + class HttpResponseHeadersDataInput : public HttpHeadersDataInputBase { public: explicit HttpResponseHeadersDataInput(const std::string& name) : HttpHeadersDataInputBase(name) {} @@ -107,9 +142,26 @@ class HttpResponseHeadersDataInput : public HttpHeadersDataInputBase { +public: + HttpResponseHeadersDataInputFactory() : HttpHeadersDataInputFactoryBase("response-headers") {} +}; + class SkipAction : public Matcher::ActionBase< envoy::extensions::filters::common::matcher::action::v3::SkipFilter> {}; +class SkipActionFactory : public Matcher::ActionFactory { +public: + std::string name() const override { return "skip"; } + Matcher::ActionFactoryCb createActionFactoryCb(const Protobuf::Message&) override { + return []() { return std::make_unique(); }; + } + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } +}; /** * Base class wrapper for both stream encoder and decoder filters. * @@ -285,10 +337,13 @@ struct ActiveStreamDecoderFilter : public ActiveStreamFilterBase, const absl::optional grpc_status, absl::string_view details) override; void encode100ContinueHeaders(ResponseHeaderMapPtr&& headers) override; + ResponseHeaderMapOptRef continueHeaders() const override; void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) override; + ResponseHeaderMapOptRef responseHeaders() const override; void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(ResponseTrailerMapPtr&& trailers) override; + ResponseTrailerMapOptRef responseTrailers() const override; void encodeMetadata(MetadataMapPtr&& metadata_map_ptr) override; void onDecoderFilterAboveWriteBufferHighWatermark() override; void onDecoderFilterBelowWriteBufferLowWatermark() override; @@ -608,7 +663,10 @@ class OverridableRemoteSocketAddressSetterStreamInfo : public StreamInfo::Stream void setDownstreamRemoteAddress( const Network::Address::InstanceConstSharedPtr& downstream_remote_address) { - ASSERT(overridden_downstream_remote_address_ == nullptr); + // TODO(rgs1): we should assert overridden_downstream_remote_address_ is nullptr, + // but we are currently relaxing this as a workaround to: + // + // https://github.com/envoyproxy/envoy/pull/14432#issuecomment-758167614 overridden_downstream_remote_address_ = downstream_remote_address; } diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index 41c92645963d..4c2ff9904af2 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -309,5 +309,10 @@ Http::Status HeaderUtility::checkRequiredHeaders(const Http::RequestHeaderMap& h return Http::okStatus(); } +bool HeaderUtility::isRemovableHeader(absl::string_view header) { + return (header.empty() || header[0] != ':') && + !absl::EqualsIgnoreCase(header, Headers::get().HostLegacy.get()); +} + } // namespace Http } // namespace Envoy diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index 291200134552..6ccda5423b3f 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -186,6 +186,13 @@ class HeaderUtility { * missing. */ static Http::Status checkRequiredHeaders(const Http::RequestHeaderMap& headers); + + /** + * Returns true if a header may be safely removed without causing additional + * problems. Effectively, header names beginning with ":" and the "host" header + * may not be removed. + */ + static bool isRemovableHeader(absl::string_view header); }; } // namespace Http } // namespace Envoy diff --git a/source/common/http/match_wrapper/BUILD b/source/common/http/match_wrapper/BUILD new file mode 100644 index 000000000000..1350d3db39e0 --- /dev/null +++ b/source/common/http/match_wrapper/BUILD @@ -0,0 +1,23 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//source/common/matcher:matcher_lib", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/common:factory_base_lib", + "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto", + ], +) diff --git a/source/common/http/match_wrapper/config.cc b/source/common/http/match_wrapper/config.cc new file mode 100644 index 000000000000..7d5b75f9f47a --- /dev/null +++ b/source/common/http/match_wrapper/config.cc @@ -0,0 +1,90 @@ +#include "common/http/match_wrapper/config.h" + +#include "envoy/http/filter.h" +#include "envoy/matcher/matcher.h" +#include "envoy/registry/registry.h" + +#include "common/config/utility.h" +#include "common/matcher/matcher.h" + +namespace Envoy { +namespace Common { +namespace Http { +namespace MatchWrapper { + +namespace { +struct DelegatingFactoryCallbacks : public Envoy::Http::FilterChainFactoryCallbacks { + DelegatingFactoryCallbacks(Envoy::Http::FilterChainFactoryCallbacks& delegated_callbacks, + Matcher::MatchTreeSharedPtr match_tree) + : delegated_callbacks_(delegated_callbacks), match_tree_(std::move(match_tree)) {} + + void addStreamDecoderFilter(Envoy::Http::StreamDecoderFilterSharedPtr filter) override { + delegated_callbacks_.addStreamDecoderFilter(std::move(filter), match_tree_); + } + void addStreamDecoderFilter( + Envoy::Http::StreamDecoderFilterSharedPtr filter, + Matcher::MatchTreeSharedPtr match_tree) override { + delegated_callbacks_.addStreamDecoderFilter(std::move(filter), std::move(match_tree)); + } + void addStreamEncoderFilter(Envoy::Http::StreamEncoderFilterSharedPtr filter) override { + delegated_callbacks_.addStreamEncoderFilter(std::move(filter), match_tree_); + } + void addStreamEncoderFilter( + Envoy::Http::StreamEncoderFilterSharedPtr filter, + Matcher::MatchTreeSharedPtr match_tree) override { + delegated_callbacks_.addStreamEncoderFilter(std::move(filter), std::move(match_tree)); + } + void addStreamFilter(Envoy::Http::StreamFilterSharedPtr filter) override { + delegated_callbacks_.addStreamFilter(std::move(filter), match_tree_); + } + void + addStreamFilter(Envoy::Http::StreamFilterSharedPtr filter, + Matcher::MatchTreeSharedPtr match_tree) override { + delegated_callbacks_.addStreamFilter(std::move(filter), std::move(match_tree)); + } + void addAccessLogHandler(AccessLog::InstanceSharedPtr handler) override { + delegated_callbacks_.addAccessLogHandler(std::move(handler)); + } + + Envoy::Http::FilterChainFactoryCallbacks& delegated_callbacks_; + Matcher::MatchTreeSharedPtr match_tree_; +}; +} // namespace + +Envoy::Http::FilterFactoryCb MatchWrapperConfig::createFilterFactoryFromProtoTyped( + const envoy::extensions::common::matching::v3::ExtensionWithMatcher& proto_config, + const std::string& prefix, Server::Configuration::FactoryContext& context) { + + ASSERT(proto_config.has_extension_config()); + auto& factory = + Config::Utility::getAndCheckFactory( + proto_config.extension_config()); + + auto message = Config::Utility::translateAnyToFactoryConfig( + proto_config.extension_config().typed_config(), context.messageValidationVisitor(), factory); + auto filter_factory = factory.createFilterFactoryFromProto(*message, prefix, context); + + auto match_tree = + Matcher::MatchTreeFactory(context.messageValidationVisitor()) + .create(proto_config.matcher()); + + return [filter_factory, match_tree](Envoy::Http::FilterChainFactoryCallbacks& callbacks) -> void { + DelegatingFactoryCallbacks delegated_callbacks(callbacks, match_tree); + + return filter_factory(delegated_callbacks); + }; +} + +/** + * Static registration for the match wrapper filter. @see RegisterFactory. + * Note that we register this as a filter in order to serve as a drop in wrapper for other HTTP + * filters. While not a real filter, by being registered as one all the code paths that look up HTTP + * filters will look up this filter factory instead, which does the work to create and associate a + * match tree with the underlying filter. + */ +REGISTER_FACTORY(MatchWrapperConfig, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace MatchWrapper +} // namespace Http +} // namespace Common +} // namespace Envoy diff --git a/source/common/http/match_wrapper/config.h b/source/common/http/match_wrapper/config.h new file mode 100644 index 000000000000..c1417aaa6f24 --- /dev/null +++ b/source/common/http/match_wrapper/config.h @@ -0,0 +1,28 @@ +#pragma once + +#include "envoy/extensions/common/matching/v3/extension_matcher.pb.validate.h" +#include "envoy/server/filter_config.h" + +#include "extensions/filters/http/common/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Common { +namespace Http { +namespace MatchWrapper { + +class MatchWrapperConfig : public Extensions::HttpFilters::Common::FactoryBase< + envoy::extensions::common::matching::v3::ExtensionWithMatcher> { +public: + MatchWrapperConfig() : FactoryBase("match-wrapper") {} + +private: + Envoy::Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::extensions::common::matching::v3::ExtensionWithMatcher& proto_config, + const std::string&, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace MatchWrapper +} // namespace Http +} // namespace Common +} // namespace Envoy diff --git a/source/common/matcher/matcher.h b/source/common/matcher/matcher.h index 4e21e17ea463..44ae0a26e787 100644 --- a/source/common/matcher/matcher.h +++ b/source/common/matcher/matcher.h @@ -160,7 +160,7 @@ template class MatchTreeFactory { auto& factory = Config::Utility::getAndCheckFactory>(config); ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( config.typed_config(), validation_visitor_, factory); - return factory.createDataInput(*message); + return factory.createDataInput(*message, validation_visitor_); } InputMatcherPtr createInputMatcher( diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 0caa0c864137..2920805d7c8b 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -335,11 +335,13 @@ envoy_cc_library( hdrs = ["router_ratelimit.h"], deps = [ ":config_utility_lib", + "//include/envoy/ratelimit:ratelimit_interface", "//include/envoy/router:router_interface", "//include/envoy/router:router_ratelimit_interface", "//source/common/common:assert_lib", "//source/common/common:empty_string", "//source/common/config:metadata_lib", + "//source/common/config:utility_lib", "//source/common/http:header_utility_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", @@ -366,6 +368,7 @@ envoy_cc_library( deps = [ ":header_formatter_lib", "//include/envoy/http:header_map_interface", + "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index a62ea02ca9e1..469d8530a6d9 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -341,7 +341,7 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, retry_policy_(buildRetryPolicy(vhost.retryPolicy(), route.route(), validator)), internal_redirect_policy_( buildInternalRedirectPolicy(route.route(), validator, route.name())), - rate_limit_policy_(route.route().rate_limits()), + rate_limit_policy_(route.route().rate_limits(), validator), priority_(ConfigUtility::parsePriority(route.route().priority())), config_headers_(Http::HeaderUtility::buildHeaderDataVector(route.match().headers())), total_cluster_weight_( @@ -1119,7 +1119,8 @@ VirtualHostImpl::VirtualHostImpl( vcluster_scope_(Stats::Utility::scopeFromStatNames( scope, {stat_name_storage_.statName(), factory_context.routerContext().virtualClusterStatNames().vcluster_})), - rate_limit_policy_(virtual_host.rate_limits()), global_route_config_(global_route_config), + rate_limit_policy_(virtual_host.rate_limits(), validator), + global_route_config_(global_route_config), request_headers_parser_(HeaderParser::configure(virtual_host.request_headers_to_add(), virtual_host.request_headers_to_remove())), response_headers_parser_(HeaderParser::configure(virtual_host.response_headers_to_add(), diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index 1e4a9f4e3098..557a888c53fe 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -7,6 +7,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "common/common/assert.h" +#include "common/http/header_utility.h" #include "common/http/headers.h" #include "common/protobuf/utility.h" @@ -258,7 +259,7 @@ HeaderParserPtr HeaderParser::configure( // We reject :-prefix (e.g. :path) removal here. This is dangerous, since other aspects of // request finalization assume their existence and they are needed for well-formedness in most // cases. - if (header[0] == ':' || Http::LowerCaseString(header).get() == "host") { + if (!Http::HeaderUtility::isRemovableHeader(header)) { throw EnvoyException(":-prefixed or host headers may not be removed"); } header_parser->headers_to_remove_.emplace_back(header); diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index 0774f8340be5..cfe882a60f42 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -11,6 +11,7 @@ #include "common/common/assert.h" #include "common/common/empty_string.h" #include "common/config/metadata.h" +#include "common/config/utility.h" #include "common/protobuf/utility.h" namespace Envoy { @@ -43,30 +44,24 @@ bool DynamicMetadataRateLimitOverride::populateOverride( return false; } -bool SourceClusterAction::populateDescriptor(const Router::RouteEntry&, - RateLimit::Descriptor& descriptor, +bool SourceClusterAction::populateDescriptor(RateLimit::Descriptor& descriptor, const std::string& local_service_cluster, - const Http::HeaderMap&, - const Network::Address::Instance&, - const envoy::config::core::v3::Metadata*) const { + const Http::RequestHeaderMap&, + const StreamInfo::StreamInfo&) const { descriptor.entries_.push_back({"source_cluster", local_service_cluster}); return true; } -bool DestinationClusterAction::populateDescriptor(const Router::RouteEntry& route, - RateLimit::Descriptor& descriptor, - const std::string&, const Http::HeaderMap&, - const Network::Address::Instance&, - const envoy::config::core::v3::Metadata*) const { - descriptor.entries_.push_back({"destination_cluster", route.clusterName()}); +bool DestinationClusterAction::populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string&, const Http::RequestHeaderMap&, + const StreamInfo::StreamInfo& info) const { + descriptor.entries_.push_back({"destination_cluster", info.routeEntry()->clusterName()}); return true; } -bool RequestHeadersAction::populateDescriptor(const Router::RouteEntry&, - RateLimit::Descriptor& descriptor, const std::string&, - const Http::HeaderMap& headers, - const Network::Address::Instance&, - const envoy::config::core::v3::Metadata*) const { +bool RequestHeadersAction::populateDescriptor(RateLimit::Descriptor& descriptor, const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo&) const { const auto header_value = headers.get(header_name_); // If header is not present in the request and if skip_if_absent is true skip this descriptor, @@ -81,23 +76,22 @@ bool RequestHeadersAction::populateDescriptor(const Router::RouteEntry&, return true; } -bool RemoteAddressAction::populateDescriptor(const Router::RouteEntry&, - RateLimit::Descriptor& descriptor, const std::string&, - const Http::HeaderMap&, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata*) const { - if (remote_address.type() != Network::Address::Type::Ip) { +bool RemoteAddressAction::populateDescriptor(RateLimit::Descriptor& descriptor, const std::string&, + const Http::RequestHeaderMap&, + const StreamInfo::StreamInfo& info) const { + const Network::Address::InstanceConstSharedPtr& remote_address = + info.downstreamAddressProvider().remoteAddress(); + if (remote_address->type() != Network::Address::Type::Ip) { return false; } - descriptor.entries_.push_back({"remote_address", remote_address.ip()->addressAsString()}); + descriptor.entries_.push_back({"remote_address", remote_address->ip()->addressAsString()}); return true; } -bool GenericKeyAction::populateDescriptor(const Router::RouteEntry&, - RateLimit::Descriptor& descriptor, const std::string&, - const Http::HeaderMap&, const Network::Address::Instance&, - const envoy::config::core::v3::Metadata*) const { +bool GenericKeyAction::populateDescriptor(RateLimit::Descriptor& descriptor, const std::string&, + const Http::RequestHeaderMap&, + const StreamInfo::StreamInfo&) const { descriptor.entries_.push_back({descriptor_key_, descriptor_value_}); return true; } @@ -112,18 +106,17 @@ MetaDataAction::MetaDataAction( default_value_(action.default_value()), source_(envoy::config::route::v3::RateLimit::Action::MetaData::DYNAMIC) {} -bool MetaDataAction::populateDescriptor( - const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, const std::string&, - const Http::HeaderMap&, const Network::Address::Instance&, - const envoy::config::core::v3::Metadata* dynamic_metadata) const { +bool MetaDataAction::populateDescriptor(RateLimit::Descriptor& descriptor, const std::string&, + const Http::RequestHeaderMap&, + const StreamInfo::StreamInfo& info) const { const envoy::config::core::v3::Metadata* metadata_source; switch (source_) { case envoy::config::route::v3::RateLimit::Action::MetaData::DYNAMIC: - metadata_source = dynamic_metadata; + metadata_source = &info.dynamicMetadata(); break; case envoy::config::route::v3::RateLimit::Action::MetaData::ROUTE_ENTRY: - metadata_source = &route.metadata(); + metadata_source = &info.routeEntry()->metadata(); break; default: NOT_REACHED_GCOVR_EXCL_LINE; @@ -149,11 +142,10 @@ HeaderValueMatchAction::HeaderValueMatchAction( expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)), action_headers_(Http::HeaderUtility::buildHeaderDataVector(action.headers())) {} -bool HeaderValueMatchAction::populateDescriptor(const Router::RouteEntry&, - RateLimit::Descriptor& descriptor, - const std::string&, const Http::HeaderMap& headers, - const Network::Address::Instance&, - const envoy::config::core::v3::Metadata*) const { +bool HeaderValueMatchAction::populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo&) const { if (expect_match_ == Http::HeaderUtility::matchHeaders(headers, action_headers_)) { descriptor.entries_.push_back({"header_match", descriptor_value_}); return true; @@ -163,7 +155,8 @@ bool HeaderValueMatchAction::populateDescriptor(const Router::RouteEntry&, } RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl( - const envoy::config::route::v3::RateLimit& config) + const envoy::config::route::v3::RateLimit& config, + ProtobufMessage::ValidationVisitor& validator) : disable_key_(config.disable_key()), stage_(static_cast(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, stage, 0))) { for (const auto& action : config.actions()) { @@ -192,6 +185,25 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl( case envoy::config::route::v3::RateLimit::Action::ActionSpecifierCase::kHeaderValueMatch: actions_.emplace_back(new HeaderValueMatchAction(action.header_value_match())); break; + case envoy::config::route::v3::RateLimit::Action::ActionSpecifierCase::kExtension: { + auto* factory = Envoy::Config::Utility::getFactory( + action.extension()); + if (!factory) { + throw EnvoyException( + absl::StrCat("Rate limit descriptor extension not found: ", action.extension().name())); + } + auto message = Envoy::Config::Utility::translateAnyToFactoryConfig( + action.extension().typed_config(), validator, *factory); + RateLimit::DescriptorProducerPtr producer = + factory->createDescriptorProducerFromProto(*message, validator); + if (producer) { + actions_.emplace_back(std::move(producer)); + } else { + throw EnvoyException( + absl::StrCat("Rate limit descriptor extension failed: ", action.extension().name())); + } + break; + } default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -208,23 +220,21 @@ RateLimitPolicyEntryImpl::RateLimitPolicyEntryImpl( } } -void RateLimitPolicyEntryImpl::populateDescriptors( - const Router::RouteEntry& route, std::vector& descriptors, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const { +void RateLimitPolicyEntryImpl::populateDescriptors(std::vector& descriptors, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const { RateLimit::Descriptor descriptor; bool result = true; - for (const RateLimitActionPtr& action : actions_) { - result = result && action->populateDescriptor(route, descriptor, local_service_cluster, headers, - remote_address, dynamic_metadata); + for (const RateLimit::DescriptorProducerPtr& action : actions_) { + result = result && action->populateDescriptor(descriptor, local_service_cluster, headers, info); if (!result) { break; } } if (limit_override_) { - limit_override_.value()->populateOverride(descriptor, dynamic_metadata); + limit_override_.value()->populateOverride(descriptor, &info.dynamicMetadata()); } if (result) { @@ -233,11 +243,12 @@ void RateLimitPolicyEntryImpl::populateDescriptors( } RateLimitPolicyImpl::RateLimitPolicyImpl( - const Protobuf::RepeatedPtrField& rate_limits) + const Protobuf::RepeatedPtrField& rate_limits, + ProtobufMessage::ValidationVisitor& validator) : rate_limit_entries_reference_(RateLimitPolicyImpl::MAX_STAGE_NUMBER + 1) { for (const auto& rate_limit : rate_limits) { std::unique_ptr rate_limit_policy_entry( - new RateLimitPolicyEntryImpl(rate_limit)); + new RateLimitPolicyEntryImpl(rate_limit, validator)); uint64_t stage = rate_limit_policy_entry->stage(); ASSERT(stage < rate_limit_entries_reference_.size()); rate_limit_entries_reference_[stage].emplace_back(*rate_limit_policy_entry); diff --git a/source/common/router/router_ratelimit.h b/source/common/router/router_ratelimit.h index 912606fc0da8..7aa0d6ca0401 100644 --- a/source/common/router/router_ratelimit.h +++ b/source/common/router/router_ratelimit.h @@ -7,6 +7,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/ratelimit/ratelimit.h" #include "envoy/router/router.h" #include "envoy/router/router_ratelimit.h" @@ -38,41 +39,41 @@ class DynamicMetadataRateLimitOverride : public RateLimitOverrideAction { /** * Action for source cluster rate limiting. */ -class SourceClusterAction : public RateLimitAction { +class SourceClusterAction : public RateLimit::DescriptorProducer { public: - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; }; /** * Action for destination cluster rate limiting. */ -class DestinationClusterAction : public RateLimitAction { +class DestinationClusterAction : public RateLimit::DescriptorProducer { public: - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; }; /** * Action for request headers rate limiting. */ -class RequestHeadersAction : public RateLimitAction { +class RequestHeadersAction : public RateLimit::DescriptorProducer { public: RequestHeadersAction(const envoy::config::route::v3::RateLimit::Action::RequestHeaders& action) : header_name_(action.header_name()), descriptor_key_(action.descriptor_key()), skip_if_absent_(action.skip_if_absent()) {} - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; private: const Http::LowerCaseString header_name_; @@ -83,30 +84,30 @@ class RequestHeadersAction : public RateLimitAction { /** * Action for remote address rate limiting. */ -class RemoteAddressAction : public RateLimitAction { +class RemoteAddressAction : public RateLimit::DescriptorProducer { public: - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; }; /** * Action for generic key rate limiting. */ -class GenericKeyAction : public RateLimitAction { +class GenericKeyAction : public RateLimit::DescriptorProducer { public: GenericKeyAction(const envoy::config::route::v3::RateLimit::Action::GenericKey& action) : descriptor_value_(action.descriptor_value()), descriptor_key_(!action.descriptor_key().empty() ? action.descriptor_key() : "generic_key") {} - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; private: const std::string descriptor_value_; @@ -116,16 +117,16 @@ class GenericKeyAction : public RateLimitAction { /** * Action for metadata rate limiting. */ -class MetaDataAction : public RateLimitAction { +class MetaDataAction : public RateLimit::DescriptorProducer { public: MetaDataAction(const envoy::config::route::v3::RateLimit::Action::MetaData& action); // for maintaining backward compatibility with the deprecated DynamicMetaData action MetaDataAction(const envoy::config::route::v3::RateLimit::Action::DynamicMetaData& action); - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; private: const Envoy::Config::MetadataKey metadata_key_; @@ -137,16 +138,16 @@ class MetaDataAction : public RateLimitAction { /** * Action for header value match rate limiting. */ -class HeaderValueMatchAction : public RateLimitAction { +class HeaderValueMatchAction : public RateLimit::DescriptorProducer { public: HeaderValueMatchAction( const envoy::config::route::v3::RateLimit::Action::HeaderValueMatch& action); - // Router::RateLimitAction - bool populateDescriptor(const Router::RouteEntry& route, RateLimit::Descriptor& descriptor, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, + const std::string& local_service_cluster, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override; private: const std::string descriptor_value_; @@ -159,22 +160,20 @@ class HeaderValueMatchAction : public RateLimitAction { */ class RateLimitPolicyEntryImpl : public RateLimitPolicyEntry { public: - RateLimitPolicyEntryImpl(const envoy::config::route::v3::RateLimit& config); + RateLimitPolicyEntryImpl(const envoy::config::route::v3::RateLimit& config, + ProtobufMessage::ValidationVisitor& validator); // Router::RateLimitPolicyEntry uint64_t stage() const override { return stage_; } const std::string& disableKey() const override { return disable_key_; } - void - populateDescriptors(const Router::RouteEntry& route, - std::vector& descriptors, - const std::string& local_service_cluster, const Http::HeaderMap&, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata) const override; + void populateDescriptors(std::vector& descriptors, + const std::string& local_service_cluster, const Http::RequestHeaderMap&, + const StreamInfo::StreamInfo& info) const override; private: const std::string disable_key_; uint64_t stage_; - std::vector actions_; + std::vector actions_; absl::optional limit_override_ = absl::nullopt; }; @@ -184,7 +183,8 @@ class RateLimitPolicyEntryImpl : public RateLimitPolicyEntry { class RateLimitPolicyImpl : public RateLimitPolicy { public: RateLimitPolicyImpl( - const Protobuf::RepeatedPtrField& rate_limits); + const Protobuf::RepeatedPtrField& rate_limits, + ProtobufMessage::ValidationVisitor& validator); // Router::RateLimitPolicy const std::vector>& diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 52f127950c64..1fa9f6293ea7 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -229,6 +229,12 @@ EXTENSIONS = { "envoy.wasm.runtime.v8": "//source/extensions/wasm_runtime/v8:config", "envoy.wasm.runtime.wavm": "//source/extensions/wasm_runtime/wavm:config", "envoy.wasm.runtime.wasmtime": "//source/extensions/wasm_runtime/wasmtime:config", + + # + # Rate limit descriptors + # + + "envoy.rate_limit_descriptors.expr": "//source/extensions/rate_limit_descriptors/expr:config", } # These can be changed to ["//visibility:public"], for downstream builds which diff --git a/source/extensions/filters/common/expr/evaluator.cc b/source/extensions/filters/common/expr/evaluator.cc index d2c376684f72..4a7ed3c9fc4c 100644 --- a/source/extensions/filters/common/expr/evaluator.cc +++ b/source/extensions/filters/common/expr/evaluator.cc @@ -94,6 +94,31 @@ bool matches(const Expression& expr, const StreamInfo::StreamInfo& info, return result.IsBool() ? result.BoolOrDie() : false; } +std::string print(CelValue value) { + switch (value.type()) { + case CelValue::Type::kBool: + return value.BoolOrDie() ? "true" : "false"; + case CelValue::Type::kInt64: + return absl::StrCat(value.Int64OrDie()); + case CelValue::Type::kUint64: + return absl::StrCat(value.Uint64OrDie()); + case CelValue::Type::kDouble: + return absl::StrCat(value.DoubleOrDie()); + case CelValue::Type::kString: + return std::string(value.StringOrDie().value()); + case CelValue::Type::kBytes: + return std::string(value.BytesOrDie().value()); + case CelValue::Type::kMessage: + return value.IsNull() ? "NULL" : value.MessageOrDie()->ShortDebugString(); + case CelValue::Type::kDuration: + return absl::FormatDuration(value.DurationOrDie()); + case CelValue::Type::kTimestamp: + return absl::FormatTime(value.TimestampOrDie(), absl::UTCTimeZone()); + default: + return absl::StrCat(CelValue::TypeName(value.type()), " value"); + } +} + } // namespace Expr } // namespace Common } // namespace Filters diff --git a/source/extensions/filters/common/expr/evaluator.h b/source/extensions/filters/common/expr/evaluator.h index 37fdf63ab1bf..32ee98572f35 100644 --- a/source/extensions/filters/common/expr/evaluator.h +++ b/source/extensions/filters/common/expr/evaluator.h @@ -52,6 +52,9 @@ absl::optional evaluate(const Expression& expr, Protobuf::Arena& arena bool matches(const Expression& expr, const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& headers); +// Returns a string for a CelValue. +std::string print(CelValue value); + // Thrown when there is an CEL library error. class CelException : public EnvoyException { public: diff --git a/source/extensions/filters/http/admission_control/config.cc b/source/extensions/filters/http/admission_control/config.cc index fcd8c42fcc9b..27b7f9f1eb25 100644 --- a/source/extensions/filters/http/admission_control/config.cc +++ b/source/extensions/filters/http/admission_control/config.cc @@ -22,8 +22,8 @@ Http::FilterFactoryCb AdmissionControlFilterFactory::createFilterFactoryFromProt const envoy::extensions::filters::http::admission_control::v3alpha::AdmissionControl& config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { - if (config.has_sr_threshold() && config.sr_threshold().default_value().value() == 0) { - throw EnvoyException("Success Rate Threshold cannot be zero percent"); + if (config.has_sr_threshold() && config.sr_threshold().default_value().value() < 1.0) { + throw EnvoyException("Success rate threshold cannot be less than 1.0%."); } const std::string prefix = stats_prefix + "admission_control."; diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 18a5e1e0c885..37247272d879 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -212,8 +212,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { for (const auto& header : response->headers_to_remove) { // We don't allow removing any :-prefixed headers, nor Host, as removing // them would make the request malformed. - if (absl::StartsWithIgnoreCase(absl::string_view(header.get()), ":") || - header == Http::Headers::get().HostLegacy) { + if (!Http::HeaderUtility::isRemovableHeader(header.get())) { continue; } ENVOY_STREAM_LOG(trace, "'{}'", *callbacks_, header.get()); diff --git a/source/extensions/filters/http/ext_proc/BUILD b/source/extensions/filters/http/ext_proc/BUILD index 93b55a5586f8..e9b25c691e39 100644 --- a/source/extensions/filters/http/ext_proc/BUILD +++ b/source/extensions/filters/http/ext_proc/BUILD @@ -14,8 +14,13 @@ envoy_cc_library( srcs = ["ext_proc.cc"], hdrs = ["ext_proc.h"], deps = [ + ":client_interface", + ":mutation_utils_lib", "//include/envoy/http:filter_interface", + "//include/envoy/http:header_map_interface", + "//include/envoy/stats:stats_macros", "//source/extensions/filters/http/common:pass_through_filter_lib", + "@com_google_absl//absl/strings:str_format", "@envoy_api//envoy/extensions/filters/http/ext_proc/v3alpha:pkg_cc_proto", ], ) @@ -27,6 +32,7 @@ envoy_cc_extension( security_posture = "unknown", status = "alpha", deps = [ + ":client_lib", ":ext_proc", "//source/extensions/filters/http:well_known_names", "//source/extensions/filters/http/common:factory_base_lib", @@ -43,6 +49,18 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "mutation_utils_lib", + srcs = ["mutation_utils.cc"], + hdrs = ["mutation_utils.h"], + deps = [ + "//include/envoy/http:header_map_interface", + "//source/common/http:header_utility_lib", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/service/ext_proc/v3alpha:pkg_cc_proto", + ], +) + envoy_cc_library( name = "client_lib", srcs = ["client_impl.cc"], diff --git a/source/extensions/filters/http/ext_proc/config.cc b/source/extensions/filters/http/ext_proc/config.cc index 83d6ac7de14c..7db245641a54 100644 --- a/source/extensions/filters/http/ext_proc/config.cc +++ b/source/extensions/filters/http/ext_proc/config.cc @@ -2,6 +2,7 @@ #include +#include "extensions/filters/http/ext_proc/client_impl.h" #include "extensions/filters/http/ext_proc/ext_proc.h" namespace Envoy { @@ -11,11 +12,19 @@ namespace ExternalProcessing { Http::FilterFactoryCb ExternalProcessingFilterConfig::createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor& proto_config, - const std::string&, Server::Configuration::FactoryContext&) { - const auto filter_config = std::make_shared(proto_config); + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + const uint32_t timeout_ms = + PROTOBUF_GET_MS_OR_DEFAULT(proto_config.grpc_service(), timeout, DefaultTimeout); + const auto filter_config = std::make_shared( + proto_config, std::chrono::milliseconds(timeout_ms), context.scope(), stats_prefix); - return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) { - callbacks.addStreamFilter(Http::StreamFilterSharedPtr{std::make_shared(filter_config)}); + return [filter_config, grpc_service = proto_config.grpc_service(), + &context](Http::FilterChainFactoryCallbacks& callbacks) { + auto client = std::make_unique( + context.clusterManager().grpcAsyncClientManager(), grpc_service, context.scope()); + + callbacks.addStreamFilter( + Http::StreamFilterSharedPtr{std::make_shared(filter_config, std::move(client))}); }; } diff --git a/source/extensions/filters/http/ext_proc/config.h b/source/extensions/filters/http/ext_proc/config.h index c6e961ffa9e3..d6b0df8ded2c 100644 --- a/source/extensions/filters/http/ext_proc/config.h +++ b/source/extensions/filters/http/ext_proc/config.h @@ -21,6 +21,8 @@ class ExternalProcessingFilterConfig ExternalProcessingFilterConfig() : FactoryBase(HttpFilterNames::get().ExternalProcessing) {} private: + static constexpr uint64_t DefaultTimeout = 200; + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index 7beb30461201..807a84de4ac3 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -1,11 +1,133 @@ #include "extensions/filters/http/ext_proc/ext_proc.h" +#include "extensions/filters/http/ext_proc/mutation_utils.h" + +#include "absl/strings/str_format.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { namespace ExternalProcessing { -void Filter::onDestroy() {} +using envoy::service::ext_proc::v3alpha::ProcessingRequest; +using envoy::service::ext_proc::v3alpha::ProcessingResponse; + +using Http::FilterHeadersStatus; +using Http::RequestHeaderMap; + +static const std::string kErrorPrefix = "ext_proc error"; + +void Filter::closeStream() { + if (!stream_closed_) { + if (stream_) { + ENVOY_LOG(debug, "Closing gRPC stream to processing server"); + stream_->close(); + stats_.streams_closed_.inc(); + } + stream_closed_ = true; + } +} + +void Filter::onDestroy() { closeStream(); } + +FilterHeadersStatus Filter::decodeHeaders(RequestHeaderMap& headers, bool end_of_stream) { + // We're at the start, so start the stream and send a headers message + request_headers_ = &headers; + stream_ = client_->start(*this, config_->grpcTimeout()); + stats_.streams_started_.inc(); + ProcessingRequest req; + auto* headers_req = req.mutable_request_headers(); + MutationUtils::buildHttpHeaders(headers, *headers_req->mutable_headers()); + headers_req->set_end_of_stream(end_of_stream); + request_state_ = FilterState::HEADERS; + stream_->send(std::move(req), false); + stats_.stream_msgs_sent_.inc(); + + // Wait until we have a gRPC response before allowing any more callbacks + return FilterHeadersStatus::StopAllIterationAndWatermark; +} + +void Filter::onReceiveMessage( + std::unique_ptr&& r) { + auto response = std::move(r); + bool message_valid = false; + ENVOY_LOG(debug, "Received gRPC message. State = {}", request_state_); + + // This next section will grow as we support the rest of the protocol + if (request_state_ == FilterState::HEADERS) { + if (response->has_request_headers()) { + ENVOY_LOG(debug, "applying request_headers response"); + message_valid = true; + const auto& headers_response = response->request_headers(); + if (headers_response.has_response()) { + const auto& common_response = headers_response.response(); + if (common_response.has_header_mutation()) { + MutationUtils::applyHeaderMutations(common_response.header_mutation(), *request_headers_); + } + } + } else if (response->has_immediate_response()) { + // To be implemented later. Leave stream open to allow people to implement + // correct servers that don't break us. + message_valid = true; + } + request_state_ = FilterState::IDLE; + decoder_callbacks_->continueDecoding(); + } + + if (message_valid) { + stats_.stream_msgs_received_.inc(); + } else { + stats_.spurious_msgs_received_.inc(); + // Ignore messages received out of order. However, close the stream to + // protect ourselves since the server is not following the protocol. + ENVOY_LOG(warn, "Spurious response message received on gRPC stream"); + closeStream(); + } +} + +void Filter::onGrpcError(Grpc::Status::GrpcStatus status) { + ENVOY_LOG(debug, "Received gRPC error on stream: {}", status); + stream_closed_ = true; + stats_.streams_failed_.inc(); + if (config_->failureModeAllow()) { + // Ignore this and treat as a successful close + onGrpcClose(); + stats_.failure_mode_allowed_.inc(); + } else { + // Use a switch here now because there will be more than two + // cases very soon. + switch (request_state_) { + case FilterState::HEADERS: + request_state_ = FilterState::IDLE; + decoder_callbacks_->sendLocalReply( + Http::Code::InternalServerError, "", nullptr, absl::nullopt, + absl::StrFormat("%s: gRPC error %i", kErrorPrefix, status)); + break; + default: + // Nothing else to do + break; + } + } +} + +void Filter::onGrpcClose() { + ENVOY_LOG(debug, "Received gRPC stream close"); + stream_closed_ = true; + stats_.streams_closed_.inc(); + // Successful close. We can ignore the stream for the rest of our request + // and response processing. + // Use a switch here now because there will be more than two + // cases very soon. + switch (request_state_) { + case FilterState::HEADERS: + request_state_ = FilterState::IDLE; + decoder_callbacks_->continueDecoding(); + break; + default: + // Nothing to do otherwise + break; + } +} } // namespace ExternalProcessing } // namespace HttpFilters diff --git a/source/extensions/filters/http/ext_proc/ext_proc.h b/source/extensions/filters/http/ext_proc/ext_proc.h index 069cc3008c17..5ccfb49453df 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.h +++ b/source/extensions/filters/http/ext_proc/ext_proc.h @@ -1,41 +1,121 @@ #pragma once +#include #include #include "envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.pb.h" #include "envoy/grpc/async_client.h" #include "envoy/http/filter.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" #include "common/common/logger.h" #include "extensions/filters/http/common/pass_through_filter.h" +#include "extensions/filters/http/ext_proc/client.h" namespace Envoy { namespace Extensions { namespace HttpFilters { namespace ExternalProcessing { +#define ALL_EXT_PROC_FILTER_STATS(COUNTER) \ + COUNTER(streams_started) \ + COUNTER(stream_msgs_sent) \ + COUNTER(stream_msgs_received) \ + COUNTER(spurious_msgs_received) \ + COUNTER(streams_closed) \ + COUNTER(streams_failed) \ + COUNTER(failure_mode_allowed) + +struct ExtProcFilterStats { + ALL_EXT_PROC_FILTER_STATS(GENERATE_COUNTER_STRUCT) +}; + class FilterConfig { public: - FilterConfig(const envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor& config) - : failure_mode_allow_(config.failure_mode_allow()) {} + FilterConfig(const envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor& config, + const std::chrono::milliseconds grpc_timeout, Stats::Scope& scope, + const std::string& stats_prefix) + : failure_mode_allow_(config.failure_mode_allow()), grpc_timeout_(grpc_timeout), + stats_(generateStats(stats_prefix, config.stat_prefix(), scope)) {} bool failureModeAllow() const { return failure_mode_allow_; } + const std::chrono::milliseconds& grpcTimeout() const { return grpc_timeout_; } + + const ExtProcFilterStats& stats() const { return stats_; } + private: + ExtProcFilterStats generateStats(const std::string& prefix, + const std::string& filter_stats_prefix, Stats::Scope& scope) { + const std::string final_prefix = absl::StrCat(prefix, "ext_proc.", filter_stats_prefix); + return {ALL_EXT_PROC_FILTER_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; + } + const bool failure_mode_allow_; + const std::chrono::milliseconds grpc_timeout_; + + ExtProcFilterStats stats_; }; using FilterConfigSharedPtr = std::shared_ptr; -class Filter : public Logger::Loggable, public Http::PassThroughFilter { +class Filter : public Logger::Loggable, + public Http::PassThroughFilter, + public ExternalProcessorCallbacks { + // The state of filter execution -- this is used to determine + // how to handle gRPC callbacks. + enum class FilterState { + // The filter is not waiting for anything, so any response on the + // gRPC stream is spurious and will result in the filter closing + // the stream. + IDLE, + // The filter is waiting for a "request_headers" or a "response_headers" message. + // Any other response on the gRPC stream will be treated as spurious. + HEADERS, + }; + public: - Filter(const FilterConfigSharedPtr& config) : config_(config) {} + Filter(const FilterConfigSharedPtr& config, ExternalProcessorClientPtr&& client) + : config_(config), client_(std::move(client)), stats_(config->stats()) {} void onDestroy() override; + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { + decoder_callbacks_ = &callbacks; + } + + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, + bool end_stream) override; + + // ExternalProcessorCallbacks + + void onReceiveMessage( + std::unique_ptr&& response) override; + + void onGrpcError(Grpc::Status::GrpcStatus error) override; + + void onGrpcClose() override; + private: - FilterConfigSharedPtr config_; + void closeStream(); + + const FilterConfigSharedPtr config_; + const ExternalProcessorClientPtr client_; + ExtProcFilterStats stats_; + + Http::StreamDecoderFilterCallbacks* decoder_callbacks_ = nullptr; + + // The state of the request-processing, or "decoding" side of the filter. + // We maintain separate states for encoding and decoding since they may + // be interleaved. + FilterState request_state_ = FilterState::IDLE; + + ExternalProcessorStreamPtr stream_; + bool stream_closed_ = false; + + Http::HeaderMap* request_headers_ = nullptr; }; } // namespace ExternalProcessing diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.cc b/source/extensions/filters/http/ext_proc/mutation_utils.cc new file mode 100644 index 000000000000..4f31b45a4751 --- /dev/null +++ b/source/extensions/filters/http/ext_proc/mutation_utils.cc @@ -0,0 +1,68 @@ +#include "extensions/filters/http/ext_proc/mutation_utils.h" + +#include "envoy/http/header_map.h" + +#include "common/http/header_utility.h" +#include "common/http/headers.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +using Http::Headers; +using Http::LowerCaseString; + +void MutationUtils::buildHttpHeaders(const Http::HeaderMap& headers_in, + envoy::config::core::v3::HeaderMap& headers_out) { + headers_in.iterate([&headers_out](const Http::HeaderEntry& e) -> Http::HeaderMap::Iterate { + auto* new_header = headers_out.add_headers(); + new_header->set_key(std::string(e.key().getStringView())); + new_header->set_value(std::string(e.value().getStringView())); + return Http::HeaderMap::Iterate::Continue; + }); +} + +void MutationUtils::applyHeaderMutations( + const envoy::service::ext_proc::v3alpha::HeaderMutation& mutation, Http::HeaderMap& headers) { + for (const auto& remove_header : mutation.remove_headers()) { + if (Http::HeaderUtility::isRemovableHeader(remove_header)) { + headers.remove(LowerCaseString(remove_header)); + } + } + + for (const auto& sh : mutation.set_headers()) { + if (!sh.has_header()) { + continue; + } + if (isSettableHeader(sh.header().key())) { + // Make "false" the default. This is logical and matches the ext_authz + // filter. However, the router handles this same protobuf and uses "true" + // as the default instead. + const bool append = PROTOBUF_GET_WRAPPED_OR_DEFAULT(sh, append, false); + if (append) { + headers.addCopy(LowerCaseString(sh.header().key()), sh.header().value()); + } else { + headers.setCopy(LowerCaseString(sh.header().key()), sh.header().value()); + } + } + } +} + +// Ignore attempts to set certain sensitive headers that can break later processing. +// We may re-enable some of these after further testing. This logic is specific +// to the ext_proc filter so it is not shared with HeaderUtils. +bool MutationUtils::isSettableHeader(absl::string_view key) { + const auto& headers = Headers::get(); + return !absl::EqualsIgnoreCase(key, headers.HostLegacy.get()) && + !absl::EqualsIgnoreCase(key, headers.Host.get()) && + !absl::EqualsIgnoreCase(key, headers.Method.get()) && + !absl::EqualsIgnoreCase(key, headers.Scheme.get()) && + !absl::StartsWithIgnoreCase(key, headers.prefix()); +} + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/http/ext_proc/mutation_utils.h b/source/extensions/filters/http/ext_proc/mutation_utils.h new file mode 100644 index 000000000000..cad4929a3d61 --- /dev/null +++ b/source/extensions/filters/http/ext_proc/mutation_utils.h @@ -0,0 +1,29 @@ +#pragma once + +#include "envoy/http/header_map.h" +#include "envoy/service/ext_proc/v3alpha/external_processor.pb.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +class MutationUtils { +public: + // Convert a header map until a protobuf + static void buildHttpHeaders(const Http::HeaderMap& headers_in, + envoy::config::core::v3::HeaderMap& headers_out); + + // Modify header map based on a set of mutations from a protobuf + static void + applyHeaderMutations(const envoy::service::ext_proc::v3alpha::HeaderMutation& mutation, + Http::HeaderMap& headers); + +private: + static bool isSettableHeader(absl::string_view key); +}; + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/http/ratelimit/ratelimit.cc b/source/extensions/filters/http/ratelimit/ratelimit.cc index 2d8b6ab6b37a..07c11b088e2d 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.cc +++ b/source/extensions/filters/http/ratelimit/ratelimit.cc @@ -49,7 +49,7 @@ void Filter::initiateCall(const Http::RequestHeaderMap& headers) { const Router::RouteEntry* route_entry = route->routeEntry(); // Get all applicable rate limit policy entries for the route. - populateRateLimitDescriptors(route_entry->rateLimitPolicy(), descriptors, route_entry, headers); + populateRateLimitDescriptors(route_entry->rateLimitPolicy(), descriptors, headers); VhRateLimitOptions vh_rate_limit_option = getVirtualHostRateLimitOption(route); @@ -58,12 +58,12 @@ void Filter::initiateCall(const Http::RequestHeaderMap& headers) { break; case VhRateLimitOptions::Include: populateRateLimitDescriptors(route_entry->virtualHost().rateLimitPolicy(), descriptors, - route_entry, headers); + headers); break; case VhRateLimitOptions::Override: if (route_entry->rateLimitPolicy().empty()) { populateRateLimitDescriptors(route_entry->virtualHost().rateLimitPolicy(), descriptors, - route_entry, headers); + headers); } break; default: @@ -229,8 +229,7 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, void Filter::populateRateLimitDescriptors(const Router::RateLimitPolicy& rate_limit_policy, std::vector& descriptors, - const Router::RouteEntry* route_entry, - const Http::HeaderMap& headers) const { + const Http::RequestHeaderMap& headers) const { for (const Router::RateLimitPolicyEntry& rate_limit : rate_limit_policy.getApplicableRateLimit(config_->stage())) { const std::string& disable_key = rate_limit.disableKey(); @@ -239,10 +238,8 @@ void Filter::populateRateLimitDescriptors(const Router::RateLimitPolicy& rate_li fmt::format("ratelimit.{}.http_filter_enabled", disable_key), 100)) { continue; } - rate_limit.populateDescriptors( - *route_entry, descriptors, config_->localInfo().clusterName(), headers, - *callbacks_->streamInfo().downstreamAddressProvider().remoteAddress(), - &callbacks_->streamInfo().dynamicMetadata()); + rate_limit.populateDescriptors(descriptors, config_->localInfo().clusterName(), headers, + callbacks_->streamInfo()); } } diff --git a/source/extensions/filters/http/ratelimit/ratelimit.h b/source/extensions/filters/http/ratelimit/ratelimit.h index e555f566bec1..fc0341e050e3 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.h +++ b/source/extensions/filters/http/ratelimit/ratelimit.h @@ -156,8 +156,7 @@ class Filter : public Http::StreamFilter, public Filters::Common::RateLimit::Req void initiateCall(const Http::RequestHeaderMap& headers); void populateRateLimitDescriptors(const Router::RateLimitPolicy& rate_limit_policy, std::vector& descriptors, - const Router::RouteEntry* route_entry, - const Http::HeaderMap& headers) const; + const Http::RequestHeaderMap& headers) const; void populateResponseHeaders(Http::HeaderMap& response_headers, bool from_local_reply); void appendRequestHeaders(Http::HeaderMapPtr& request_headers_to_add); VhRateLimitOptions getVirtualHostRateLimitOption(const Router::RouteConstSharedPtr& route); diff --git a/source/extensions/rate_limit_descriptors/expr/BUILD b/source/extensions/rate_limit_descriptors/expr/BUILD new file mode 100644 index 000000000000..720c3bf5293b --- /dev/null +++ b/source/extensions/rate_limit_descriptors/expr/BUILD @@ -0,0 +1,36 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), + security_posture = "unknown", + deps = [ + "//include/envoy/ratelimit:ratelimit_interface", + "//include/envoy/registry", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/common/expr:evaluator_lib", + "@envoy_api//envoy/extensions/rate_limit_descriptors/expr/v3:pkg_cc_proto", + ] + select( + { + "//bazel:windows_x86_64": [], + "//conditions:default": [ + "@com_google_cel_cpp//parser", + ], + }, + ), +) diff --git a/source/extensions/rate_limit_descriptors/expr/config.cc b/source/extensions/rate_limit_descriptors/expr/config.cc new file mode 100644 index 000000000000..aeebb5f96e23 --- /dev/null +++ b/source/extensions/rate_limit_descriptors/expr/config.cc @@ -0,0 +1,99 @@ +#include "extensions/rate_limit_descriptors/expr/config.h" + +#include "envoy/extensions/rate_limit_descriptors/expr/v3/expr.pb.h" +#include "envoy/extensions/rate_limit_descriptors/expr/v3/expr.pb.validate.h" + +#include "common/protobuf/utility.h" + +#if defined(USE_CEL_PARSER) +#include "parser/parser.h" +#endif + +namespace Envoy { +namespace Extensions { +namespace RateLimitDescriptors { +namespace Expr { + +namespace { + +/** + * Descriptor producer for a symbolic expression descriptor. + */ +class ExpressionDescriptor : public RateLimit::DescriptorProducer { +public: + ExpressionDescriptor( + const envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor& config, + Filters::Common::Expr::Builder& builder, const google::api::expr::v1alpha1::Expr& input_expr) + : input_expr_(input_expr), descriptor_key_(config.descriptor_key()), + skip_if_error_(config.skip_if_error()) { + compiled_expr_ = Extensions::Filters::Common::Expr::createExpression(builder, input_expr_); + } + + // Ratelimit::DescriptorProducer + bool populateDescriptor(RateLimit::Descriptor& descriptor, const std::string&, + const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info) const override { + ProtobufWkt::Arena arena; + const auto result = Filters::Common::Expr::evaluate(*compiled_expr_.get(), arena, info, + &headers, nullptr, nullptr); + if (!result.has_value() || result.value().IsError()) { + // If result is an error and if skip_if_error is true skip this descriptor, + // while calling rate limiting service. If skip_if_error is false, do not call rate limiting + // service. + return skip_if_error_; + } + descriptor.entries_.push_back({descriptor_key_, Filters::Common::Expr::print(result.value())}); + return true; + } + +private: + const google::api::expr::v1alpha1::Expr input_expr_; + const std::string descriptor_key_; + const bool skip_if_error_; + Extensions::Filters::Common::Expr::ExpressionPtr compiled_expr_; +}; + +} // namespace + +std::string ExprDescriptorFactory::name() const { return "envoy.rate_limit_descriptors.expr"; } + +ProtobufTypes::MessagePtr ExprDescriptorFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +RateLimit::DescriptorProducerPtr ExprDescriptorFactory::createDescriptorProducerFromProto( + const Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validator) { + const auto& config = MessageUtil::downcastAndValidate< + const envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor&>(message, validator); + switch (config.expr_specifier_case()) { +#if defined(USE_CEL_PARSER) + case envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor::kText: { + auto parse_status = google::api::expr::parser::Parse(config.text()); + if (!parse_status.ok()) { + throw EnvoyException("Unable to parse descriptor expression: " + + parse_status.status().ToString()); + } + return std::make_unique(config, getOrCreateBuilder(), + parse_status.value().expr()); + } +#endif + case envoy::extensions::rate_limit_descriptors::expr::v3::Descriptor::kParsed: + return std::make_unique(config, getOrCreateBuilder(), config.parsed()); + default: + return nullptr; + } +} + +Filters::Common::Expr::Builder& ExprDescriptorFactory::getOrCreateBuilder() { + if (expr_builder_ == nullptr) { + expr_builder_ = Filters::Common::Expr::createBuilder(nullptr); + } + return *expr_builder_; +} + +REGISTER_FACTORY(ExprDescriptorFactory, RateLimit::DescriptorProducerFactory); + +} // namespace Expr +} // namespace RateLimitDescriptors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/rate_limit_descriptors/expr/config.h b/source/extensions/rate_limit_descriptors/expr/config.h new file mode 100644 index 000000000000..6d3967bb8285 --- /dev/null +++ b/source/extensions/rate_limit_descriptors/expr/config.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/ratelimit/ratelimit.h" +#include "envoy/registry/registry.h" + +#include "extensions/filters/common/expr/evaluator.h" + +namespace Envoy { +namespace Extensions { +namespace RateLimitDescriptors { +namespace Expr { + +/** + * Config registration for the computed rate limit descriptor. + * @see DescriptorProducerFactory. + */ +class ExprDescriptorFactory : public RateLimit::DescriptorProducerFactory { +public: + std::string name() const override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + RateLimit::DescriptorProducerPtr + createDescriptorProducerFromProto(const Protobuf::Message& message, + ProtobufMessage::ValidationVisitor& validator) override; + +private: + Filters::Common::Expr::Builder& getOrCreateBuilder(); + Filters::Common::Expr::BuilderPtr expr_builder_; +}; + +} // namespace Expr +} // namespace RateLimitDescriptors +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index c533e0cc9c4f..77c52874becb 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -250,6 +250,11 @@ class ServerContextImpl : public ContextImpl, public Envoy::Ssl::ServerContext { ServerContextImpl(Stats::Scope& scope, const Envoy::Ssl::ServerContextConfig& config, const std::vector& server_names, TimeSource& time_source); + // Select the TLS certificate context in SSL_CTX_set_select_certificate_cb() callback with + // ClientHello details. This is made public for use by custom TLS extensions who want to + // manually create and use this as a client hello callback. + enum ssl_select_cert_result_t selectTlsContext(const SSL_CLIENT_HELLO* ssl_client_hello); + private: using SessionContextID = std::array; @@ -259,9 +264,6 @@ class ServerContextImpl : public ContextImpl, public Envoy::Ssl::ServerContext { HMAC_CTX* hmac_ctx, int encrypt); bool isClientEcdsaCapable(const SSL_CLIENT_HELLO* ssl_client_hello); bool isClientOcspCapable(const SSL_CLIENT_HELLO* ssl_client_hello); - // Select the TLS certificate context in SSL_CTX_set_select_certificate_cb() callback with - // ClientHello details. - enum ssl_select_cert_result_t selectTlsContext(const SSL_CLIENT_HELLO* ssl_client_hello); OcspStapleAction ocspStapleAction(const ServerContextImpl::TlsContext& ctx, bool client_ocsp_capable); diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index 1d6548a29211..4e60542f0be0 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -692,5 +692,14 @@ TEST(PercentEncoding, ShouldCloseConnection) { Protocol::Http11, TestRequestHeaderMapImpl{{"proxy-connection", "foo,close"}})); } +TEST(RequiredHeaders, IsRemovableHeader) { + EXPECT_FALSE(HeaderUtility::isRemovableHeader(":path")); + EXPECT_FALSE(HeaderUtility::isRemovableHeader("host")); + EXPECT_FALSE(HeaderUtility::isRemovableHeader("Host")); + EXPECT_TRUE(HeaderUtility::isRemovableHeader("")); + EXPECT_TRUE(HeaderUtility::isRemovableHeader("hostname")); + EXPECT_TRUE(HeaderUtility::isRemovableHeader("Content-Type")); +} + } // namespace Http } // namespace Envoy diff --git a/test/common/http/match_wrapper/BUILD b/test/common/http/match_wrapper/BUILD new file mode 100644 index 000000000000..52425ab72baa --- /dev/null +++ b/test/common/http/match_wrapper/BUILD @@ -0,0 +1,19 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + deps = [ + "//source/common/http/match_wrapper:config", + "//test/mocks/server:factory_context_mocks", + "//test/test_common:registry_lib", + ], +) diff --git a/test/common/http/match_wrapper/config_test.cc b/test/common/http/match_wrapper/config_test.cc new file mode 100644 index 000000000000..d29207ff3c6e --- /dev/null +++ b/test/common/http/match_wrapper/config_test.cc @@ -0,0 +1,91 @@ +#include "envoy/http/filter.h" +#include "envoy/server/factory_context.h" +#include "envoy/server/filter_config.h" + +#include "common/http/match_wrapper/config.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/registry.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Common { +namespace Http { +namespace MatchWrapper { +namespace { + +struct TestFactory : public Envoy::Server::Configuration::NamedHttpFilterConfigFactory { + std::string name() const override { return "test"; } + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + Envoy::Http::FilterFactoryCb + createFilterFactoryFromProto(const Protobuf::Message&, const std::string&, + Server::Configuration::FactoryContext&) override { + return [](auto& callbacks) { + callbacks.addStreamDecoderFilter(nullptr); + callbacks.addStreamEncoderFilter(nullptr); + callbacks.addStreamFilter(nullptr); + + callbacks.addStreamDecoderFilter(nullptr, nullptr); + callbacks.addStreamEncoderFilter(nullptr, nullptr); + callbacks.addStreamFilter(nullptr, nullptr); + + callbacks.addAccessLogHandler(nullptr); + }; + } +}; + +TEST(MatchWrapper, WithMatcher) { + TestFactory test_factory; + Envoy::Registry::InjectFactory + inject_factory(test_factory); + + NiceMock factory_context; + + const auto config = + TestUtility::parseYaml(R"EOF( +extension_config: + name: test + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue +matcher: + matcher_tree: + input: + name: request-headers + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: default-matcher-header + exact_match_map: + map: + match: + action: + name: skip + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter +)EOF"); + + MatchWrapperConfig match_wrapper_config; + auto cb = match_wrapper_config.createFilterFactoryFromProto(config, "", factory_context); + + Envoy::Http::MockFilterChainFactoryCallbacks factory_callbacks; + testing::InSequence s; + + // This matches the sequence of calls in the filter factory above: the ones that call the overload + // without a match tree has a match tree added, the other one does not. + EXPECT_CALL(factory_callbacks, addStreamDecoderFilter(_, testing::NotNull())); + EXPECT_CALL(factory_callbacks, addStreamEncoderFilter(_, testing::NotNull())); + EXPECT_CALL(factory_callbacks, addStreamFilter(_, testing::NotNull())); + EXPECT_CALL(factory_callbacks, addStreamDecoderFilter(_, testing::IsNull())); + EXPECT_CALL(factory_callbacks, addStreamEncoderFilter(_, testing::IsNull())); + EXPECT_CALL(factory_callbacks, addStreamFilter(_, testing::IsNull())); + EXPECT_CALL(factory_callbacks, addAccessLogHandler(_)); + cb(factory_callbacks); +} + +} // namespace +} // namespace MatchWrapper +} // namespace Http +} // namespace Common +} // namespace Envoy \ No newline at end of file diff --git a/test/common/matcher/test_utility.h b/test/common/matcher/test_utility.h index 623d8158c55f..1348d9cd80d4 100644 --- a/test/common/matcher/test_utility.h +++ b/test/common/matcher/test_utility.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/matcher/matcher.h" +#include "envoy/protobuf/message_validator.h" #include "common/matcher/matcher.h" @@ -29,7 +30,8 @@ class TestDataInputFactory : public DataInputFactory { TestDataInputFactory(absl::string_view factory_name, absl::string_view data) : factory_name_(std::string(factory_name)), value_(std::string(data)), injection_(*this) {} - DataInputPtr createDataInput(const Protobuf::Message&) override { + DataInputPtr createDataInput(const Protobuf::Message&, + ProtobufMessage::ValidationVisitor&) override { return std::make_unique( DataInputGetResult{DataInputGetResult::DataAvailability::AllDataAvailable, value_}); } diff --git a/test/common/router/router_ratelimit_test.cc b/test/common/router/router_ratelimit_test.cc index d8ab967f72de..4fab81660417 100644 --- a/test/common/router/router_ratelimit_test.cc +++ b/test/common/router/router_ratelimit_test.cc @@ -82,15 +82,16 @@ class RateLimitConfiguration : public testing::Test { TestUtility::loadFromYaml(yaml, route_config); config_ = std::make_unique(route_config, factory_context_, any_validation_visitor_, true); + stream_info_.downstream_address_provider_->setRemoteAddress(default_remote_address_); } NiceMock factory_context_; ProtobufMessage::NullValidationVisitorImpl any_validation_visitor_; std::unique_ptr config_; Http::TestRequestHeaderMapImpl header_; - const RouteEntry* route_; - Network::Address::Ipv4Instance default_remote_address_{"10.0.0.1"}; - const envoy::config::core::v3::Metadata* dynamic_metadata_; + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv4Instance("10.0.0.1")}; + NiceMock stream_info_; }; TEST_F(RateLimitConfiguration, NoApplicableRateLimit) { @@ -116,8 +117,7 @@ TEST_F(RateLimitConfiguration, NoApplicableRateLimit) { factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); setupTest(yaml); - NiceMock stream_info; - EXPECT_EQ(0U, config_->route(genHeaders("www.lyft.com", "/bar", "GET"), stream_info, 0) + EXPECT_EQ(0U, config_->route(genHeaders("www.lyft.com", "/bar", "GET"), stream_info_, 0) ->routeEntry() ->rateLimitPolicy() .getApplicableRateLimit(0) @@ -139,11 +139,12 @@ TEST_F(RateLimitConfiguration, NoRateLimitPolicy) { factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); setupTest(yaml); + auto* route = + config_->route(genHeaders("www.lyft.com", "/bar", "GET"), stream_info_, 0)->routeEntry(); + ON_CALL(Const(stream_info_), routeEntry()).WillByDefault(testing::Return(route)); - NiceMock stream_info; - route_ = config_->route(genHeaders("www.lyft.com", "/bar", "GET"), stream_info, 0)->routeEntry(); - EXPECT_EQ(0U, route_->rateLimitPolicy().getApplicableRateLimit(0).size()); - EXPECT_TRUE(route_->rateLimitPolicy().empty()); + EXPECT_EQ(0U, route->rateLimitPolicy().getApplicableRateLimit(0).size()); + EXPECT_TRUE(route->rateLimitPolicy().empty()); } TEST_F(RateLimitConfiguration, TestGetApplicationRateLimit) { @@ -164,18 +165,18 @@ TEST_F(RateLimitConfiguration, TestGetApplicationRateLimit) { factory_context_.cluster_manager_.initializeClusters({"www2"}, {}); setupTest(yaml); + auto* route = + config_->route(genHeaders("www.lyft.com", "/foo", "GET"), stream_info_, 0)->routeEntry(); + ON_CALL(Const(stream_info_), routeEntry()).WillByDefault(testing::Return(route)); - NiceMock stream_info; - route_ = config_->route(genHeaders("www.lyft.com", "/foo", "GET"), stream_info, 0)->routeEntry(); - EXPECT_FALSE(route_->rateLimitPolicy().empty()); + EXPECT_FALSE(route->rateLimitPolicy().empty()); std::vector> rate_limits = - route_->rateLimitPolicy().getApplicableRateLimit(0); + route->rateLimitPolicy().getApplicableRateLimit(0); EXPECT_EQ(1U, rate_limits.size()); std::vector descriptors; for (const RateLimitPolicyEntry& rate_limit : rate_limits) { - rate_limit.populateDescriptors(*route_, descriptors, "", header_, default_remote_address_, - dynamic_metadata_); + rate_limit.populateDescriptors(descriptors, "", header_, stream_info_); } EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), testing::ContainerEq(descriptors)); @@ -199,17 +200,17 @@ TEST_F(RateLimitConfiguration, TestVirtualHost) { factory_context_.cluster_manager_.initializeClusters({"www2test"}, {}); setupTest(yaml); + auto* route = + config_->route(genHeaders("www.lyft.com", "/bar", "GET"), stream_info_, 0)->routeEntry(); + ON_CALL(Const(stream_info_), routeEntry()).WillByDefault(testing::Return(route)); - NiceMock stream_info; - route_ = config_->route(genHeaders("www.lyft.com", "/bar", "GET"), stream_info, 0)->routeEntry(); std::vector> rate_limits = - route_->virtualHost().rateLimitPolicy().getApplicableRateLimit(0); + route->virtualHost().rateLimitPolicy().getApplicableRateLimit(0); EXPECT_EQ(1U, rate_limits.size()); std::vector descriptors; for (const RateLimitPolicyEntry& rate_limit : rate_limits) { - rate_limit.populateDescriptors(*route_, descriptors, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit.populateDescriptors(descriptors, "service_cluster", header_, stream_info_); } EXPECT_THAT(std::vector({{{{"destination_cluster", "www2test"}}}}), testing::ContainerEq(descriptors)); @@ -239,17 +240,17 @@ TEST_F(RateLimitConfiguration, Stages) { factory_context_.cluster_manager_.initializeClusters({"www2test"}, {}); setupTest(yaml); + auto* route = + config_->route(genHeaders("www.lyft.com", "/foo", "GET"), stream_info_, 0)->routeEntry(); + ON_CALL(Const(stream_info_), routeEntry()).WillByDefault(testing::Return(route)); - NiceMock stream_info; - route_ = config_->route(genHeaders("www.lyft.com", "/foo", "GET"), stream_info, 0)->routeEntry(); std::vector> rate_limits = - route_->rateLimitPolicy().getApplicableRateLimit(0); + route->rateLimitPolicy().getApplicableRateLimit(0); EXPECT_EQ(2U, rate_limits.size()); std::vector descriptors; for (const RateLimitPolicyEntry& rate_limit : rate_limits) { - rate_limit.populateDescriptors(*route_, descriptors, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit.populateDescriptors(descriptors, "service_cluster", header_, stream_info_); } EXPECT_THAT(std::vector( {{{{"destination_cluster", "www2test"}}}, @@ -257,33 +258,36 @@ TEST_F(RateLimitConfiguration, Stages) { testing::ContainerEq(descriptors)); descriptors.clear(); - rate_limits = route_->rateLimitPolicy().getApplicableRateLimit(1UL); + rate_limits = route->rateLimitPolicy().getApplicableRateLimit(1UL); EXPECT_EQ(1U, rate_limits.size()); for (const RateLimitPolicyEntry& rate_limit : rate_limits) { - rate_limit.populateDescriptors(*route_, descriptors, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit.populateDescriptors(descriptors, "service_cluster", header_, stream_info_); } EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), testing::ContainerEq(descriptors)); - rate_limits = route_->rateLimitPolicy().getApplicableRateLimit(10UL); + rate_limits = route->rateLimitPolicy().getApplicableRateLimit(10UL); EXPECT_TRUE(rate_limits.empty()); } class RateLimitPolicyEntryTest : public testing::Test { public: void setupTest(const std::string& yaml) { - rate_limit_entry_ = std::make_unique(parseRateLimitFromV3Yaml(yaml)); + rate_limit_entry_ = std::make_unique( + parseRateLimitFromV3Yaml(yaml), ProtobufMessage::getStrictValidationVisitor()); descriptors_.clear(); + stream_info_.downstream_address_provider_->setRemoteAddress(default_remote_address_); + ON_CALL(Const(stream_info_), routeEntry()).WillByDefault(testing::Return(&route_)); } std::unique_ptr rate_limit_entry_; Http::TestRequestHeaderMapImpl header_; NiceMock route_; std::vector descriptors_; - Network::Address::Ipv4Instance default_remote_address_{"10.0.0.1"}; - const envoy::config::core::v3::Metadata* dynamic_metadata_; + Network::Address::InstanceConstSharedPtr default_remote_address_{ + new Network::Address::Ipv4Instance("10.0.0.1")}; + NiceMock stream_info_; }; TEST_F(RateLimitPolicyEntryTest, RateLimitPolicyEntryMembers) { @@ -308,8 +312,7 @@ TEST_F(RateLimitPolicyEntryTest, RemoteAddress) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"remote_address", "10.0.0.1"}}}}), testing::ContainerEq(descriptors_)); } @@ -323,9 +326,9 @@ TEST_F(RateLimitPolicyEntryTest, PipeAddress) { setupTest(yaml); - Network::Address::PipeInstance pipe_address("/hello"); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, pipe_address, - dynamic_metadata_); + stream_info_.downstream_address_provider_->setRemoteAddress( + std::make_shared("/hello")); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -337,8 +340,7 @@ TEST_F(RateLimitPolicyEntryTest, SourceService) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header_, stream_info_); EXPECT_THAT( std::vector({{{{"source_cluster", "service_cluster"}}}}), testing::ContainerEq(descriptors_)); @@ -352,8 +354,7 @@ TEST_F(RateLimitPolicyEntryTest, DestinationService) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header_, stream_info_); EXPECT_THAT( std::vector({{{{"destination_cluster", "fake_cluster"}}}}), testing::ContainerEq(descriptors_)); @@ -370,8 +371,7 @@ TEST_F(RateLimitPolicyEntryTest, RequestHeaders) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "test_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header, stream_info_); EXPECT_THAT(std::vector({{{{"my_header_name", "test_value"}}}}), testing::ContainerEq(descriptors_)); } @@ -394,8 +394,7 @@ TEST_F(RateLimitPolicyEntryTest, RequestHeadersWithSkipIfAbsent) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "test_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header, stream_info_); EXPECT_THAT(std::vector({{{{"my_header_name", "test_value"}}}}), testing::ContainerEq(descriptors_)); } @@ -418,8 +417,7 @@ TEST_F(RateLimitPolicyEntryTest, RequestHeadersWithDefaultSkipIfAbsent) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-test", "test_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -434,8 +432,7 @@ TEST_F(RateLimitPolicyEntryTest, RequestHeadersNoMatch) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "test_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -448,8 +445,7 @@ TEST_F(RateLimitPolicyEntryTest, RateLimitKey) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"generic_key", "fake_key"}}}}), testing::ContainerEq(descriptors_)); } @@ -464,8 +460,7 @@ TEST_F(RateLimitPolicyEntryTest, GenericKeyWithSetDescriptorKey) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), testing::ContainerEq(descriptors_)); } @@ -480,8 +475,7 @@ TEST_F(RateLimitPolicyEntryTest, GenericKeyWithEmptyDescriptorKey) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"generic_key", "fake_value"}}}}), testing::ContainerEq(descriptors_)); } @@ -508,11 +502,8 @@ TEST_F(RateLimitPolicyEntryTest, DEPRECATED_FEATURE_TEST(DynamicMetaDataMatch)) prop: foo )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), testing::ContainerEq(descriptors_)); @@ -540,11 +531,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataMatchDynamicSourceByDefault) { prop: foo )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), testing::ContainerEq(descriptors_)); @@ -573,11 +561,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataMatchDynamicSource) { prop: foo )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), testing::ContainerEq(descriptors_)); @@ -608,8 +593,7 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataMatchRouteEntrySource) { TestUtility::loadFromYaml(metadata_yaml, route_.metadata_); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"fake_key", "foo"}}}}), testing::ContainerEq(descriptors_)); @@ -638,11 +622,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataNoMatchWithDefaultValue) { prop: foo )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"fake_key", "fake_value"}}}}), testing::ContainerEq(descriptors_)); @@ -669,11 +650,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataNoMatch) { prop: foo )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -699,11 +677,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataEmptyValue) { prop: "" )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -730,11 +705,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataAndDefaultValueEmpty) { prop: "" )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -761,11 +733,8 @@ TEST_F(RateLimitPolicyEntryTest, MetaDataNonStringNoMatch) { foo: bar )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -783,8 +752,7 @@ TEST_F(RateLimitPolicyEntryTest, HeaderValueMatch) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "test_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header, stream_info_); EXPECT_THAT(std::vector({{{{"header_match", "fake_value"}}}}), testing::ContainerEq(descriptors_)); } @@ -802,8 +770,7 @@ TEST_F(RateLimitPolicyEntryTest, HeaderValueMatchNoMatch) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "not_same_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -821,8 +788,7 @@ TEST_F(RateLimitPolicyEntryTest, HeaderValueMatchHeadersNotPresent) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "not_same_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header, stream_info_); EXPECT_THAT(std::vector({{{{"header_match", "fake_value"}}}}), testing::ContainerEq(descriptors_)); } @@ -841,8 +807,7 @@ TEST_F(RateLimitPolicyEntryTest, HeaderValueMatchHeadersPresent) { setupTest(yaml); Http::TestRequestHeaderMapImpl header{{"x-header-name", "test_value"}}; - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header, default_remote_address_, - dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "", header, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -855,8 +820,7 @@ TEST_F(RateLimitPolicyEntryTest, CompoundActions) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header_, stream_info_); EXPECT_THAT( std::vector( {{{{"destination_cluster", "fake_cluster"}, {"source_cluster", "service_cluster"}}}}), @@ -876,8 +840,7 @@ TEST_F(RateLimitPolicyEntryTest, CompoundActionsNoDescriptor) { setupTest(yaml); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "service_cluster", header_, - default_remote_address_, dynamic_metadata_); + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header_, stream_info_); EXPECT_TRUE(descriptors_.empty()); } @@ -904,10 +867,8 @@ TEST_F(RateLimitPolicyEntryTest, DynamicMetadataRateLimitOverride) { unit: HOUR )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT( std::vector( {{{{"generic_key", "limited_fake_key"}}, {{42, envoy::type::v3::RateLimitUnit::HOUR}}}}), @@ -937,10 +898,8 @@ TEST_F(RateLimitPolicyEntryTest, DynamicMetadataRateLimitOverrideNotFound) { unit: HOUR )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"generic_key", "limited_fake_key"}}}}), testing::ContainerEq(descriptors_)); } @@ -966,10 +925,8 @@ TEST_F(RateLimitPolicyEntryTest, DynamicMetadataRateLimitOverrideWrongType) { test: some_string )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"generic_key", "limited_fake_key"}}}}), testing::ContainerEq(descriptors_)); } @@ -997,10 +954,8 @@ TEST_F(RateLimitPolicyEntryTest, DynamicMetadataRateLimitOverrideWrongUnit) { unit: NOT_A_UNIT )EOF"; - envoy::config::core::v3::Metadata metadata; - TestUtility::loadFromYaml(metadata_yaml, metadata); - rate_limit_entry_->populateDescriptors(route_, descriptors_, "", header_, default_remote_address_, - &metadata); + TestUtility::loadFromYaml(metadata_yaml, stream_info_.dynamicMetadata()); + rate_limit_entry_->populateDescriptors(descriptors_, "", header_, stream_info_); EXPECT_THAT(std::vector({{{{"generic_key", "limited_fake_key"}}}}), testing::ContainerEq(descriptors_)); } diff --git a/test/extensions/filters/common/expr/BUILD b/test/extensions/filters/common/expr/BUILD index 2ee719111d5a..9f77992305d4 100644 --- a/test/extensions/filters/common/expr/BUILD +++ b/test/extensions/filters/common/expr/BUILD @@ -29,6 +29,17 @@ envoy_extension_cc_test( ], ) +envoy_extension_cc_test( + name = "evaluator_test", + srcs = ["evaluator_test.cc"], + extension_name = "envoy.filters.http.rbac", + deps = [ + "//source/extensions/filters/common/expr:evaluator_lib", + "//test/test_common:utility_lib", + "@com_google_cel_cpp//eval/public/structs:cel_proto_wrapper", + ], +) + envoy_proto_library( name = "evaluator_fuzz_proto", srcs = ["evaluator_fuzz.proto"], diff --git a/test/extensions/filters/common/expr/evaluator_test.cc b/test/extensions/filters/common/expr/evaluator_test.cc new file mode 100644 index 000000000000..ab4da05f04f7 --- /dev/null +++ b/test/extensions/filters/common/expr/evaluator_test.cc @@ -0,0 +1,48 @@ +#include "extensions/filters/common/expr/evaluator.h" + +#include "test/test_common/utility.h" + +#include "absl/time/time.h" +#include "eval/public/structs/cel_proto_wrapper.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Expr { +namespace { + +TEST(Evaluator, Print) { + EXPECT_EQ(print(CelValue::CreateBool(false)), "false"); + EXPECT_EQ(print(CelValue::CreateInt64(123)), "123"); + EXPECT_EQ(print(CelValue::CreateUint64(123)), "123"); + EXPECT_EQ(print(CelValue::CreateDouble(1.23)), "1.23"); + + std::string test = "test"; + EXPECT_EQ(print(CelValue::CreateString(&test)), "test"); + EXPECT_EQ(print(CelValue::CreateBytes(&test)), "test"); + + ProtobufWkt::Arena arena; + envoy::config::core::v3::Node node; + std::string node_yaml = "id: test"; + TestUtility::loadFromYaml(node_yaml, node); + EXPECT_EQ(print(CelValue::CreateNull()), "NULL"); + EXPECT_EQ(print(google::api::expr::runtime::CelProtoWrapper::CreateMessage(&node, &arena)), + "id: \"test\""); + + EXPECT_EQ(print(CelValue::CreateDuration(absl::Minutes(1))), "1m"); + absl::Time time = TestUtility::parseTime("Dec 22 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); + EXPECT_EQ(print(CelValue::CreateTimestamp(time)), "2020-12-22T01:50:34+00:00"); + + absl::Status status = absl::UnimplementedError("unimplemented"); + EXPECT_EQ(print(CelValue::CreateError(&status)), "CelError value"); +} + +} // namespace +} // namespace Expr +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/admission_control/config_test.cc b/test/extensions/filters/http/admission_control/config_test.cc index c5cc6917f8de..13d73551462e 100644 --- a/test/extensions/filters/http/admission_control/config_test.cc +++ b/test/extensions/filters/http/admission_control/config_test.cc @@ -50,7 +50,7 @@ class AdmissionControlConfigTest : public testing::Test { // Ensure the filter ingest throws an exception if it is passed a config with a default value of 0 // for sr_threshold If exception was not thrown, a default value of 0 for sr_threshold induces a -// divide by zero error +// divide by zero error. TEST_F(AdmissionControlConfigTest, ZeroSuccessRateThreshold) { AdmissionControlFilterFactory admission_control_filter_factory; const std::string yaml = R"EOF( @@ -75,7 +75,34 @@ sampling_window: 1337s NiceMock factory_context; EXPECT_THROW_WITH_MESSAGE(admission_control_filter_factory.createFilterFactoryFromProtoTyped( proto, "whatever", factory_context), - EnvoyException, "Success Rate Threshold cannot be zero percent"); + EnvoyException, "Success rate threshold cannot be less than 1.0%."); +} + +TEST_F(AdmissionControlConfigTest, SmallSuccessRateThreshold) { + AdmissionControlFilterFactory admission_control_filter_factory; + const std::string yaml = R"EOF( +enabled: + default_value: false + runtime_key: "foo.enabled" +sampling_window: 1337s +sr_threshold: + default_value: + value: 1.22e-22 + runtime_key: "foo.sr_threshold" +aggression: + default_value: 4.2 + runtime_key: "foo.aggression" +success_criteria: + http_criteria: + grpc_criteria: +)EOF"; + + AdmissionControlProto proto; + TestUtility::loadFromYamlAndValidate(yaml, proto); + NiceMock factory_context; + EXPECT_THROW_WITH_MESSAGE(admission_control_filter_factory.createFilterFactoryFromProtoTyped( + proto, "whatever", factory_context), + EnvoyException, "Success rate threshold cannot be less than 1.0%."); } // Verify the configuration when all fields are set. diff --git a/test/extensions/filters/http/common/fuzz/BUILD b/test/extensions/filters/http/common/fuzz/BUILD index ffc67eb4a232..e647e4354c4b 100644 --- a/test/extensions/filters/http/common/fuzz/BUILD +++ b/test/extensions/filters/http/common/fuzz/BUILD @@ -55,6 +55,7 @@ envoy_cc_test_library( "//test/mocks/http:http_mocks", "//test/mocks/server:factory_context_mocks", "//test/proto:bookstore_proto_cc_proto", + "//test/test_common:test_runtime_lib", "@envoy_api//envoy/extensions/filters/http/grpc_json_transcoder/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-4784906297278464 b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-4784906297278464 new file mode 100644 index 000000000000..c91cc6a64987 --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-4784906297278464 @@ -0,0 +1,23 @@ +config { + name: "envoy.filters.http.admission_control" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.http.admission_control.v3alpha.AdmissionControl" + value: "\022\000\032\002\020\002*\016\n\t\t+\000\000\000\000\000\000\000\022\001$" + } +} +data { + headers { + } + http_body { + data: "\037\000\000\000\000\000\000\000" + } +} +upstream_data { + headers { + } + http_body { + data: "=" + data: "=" + data: "?" + } +} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-6484279454466048 b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-6484279454466048 new file mode 100644 index 000000000000..2d9ce470aa1b --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-6484279454466048 @@ -0,0 +1,13 @@ +config { + name: "envoy.filters.http.ratelimit" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit" + value: "\n\001\021(\0010\001:\376\001\022\373\001\022\370\001\n\001T\022\002\n\000\032\003\n\001~\032\256\0012\253\001\032\250\001\nHtype.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit\022\\\n\001\021(\0010\001:Q\022O\022M\n\001T\022\002\n\000\032\005\"\003\n\001!\032\0022\000\032\005\"\003\n\001>\032\007\032\005\"003\n\032\002\022\000\032\n\"\010\n\001>\020\200\200\254\001\"\027envoy\000\000\000\027ters.http.rbac2\000H\001\032\005\"\003\n\001>\032\007\032\005\"003\n\032\002\022\000\032\n\"\010\n\001>\020\200\200\254\001\"\027envoy\000\000\000\027ters.http.rbac2\000H\001" + } +} +data { + http_body { + data: "\021" + data: "!" + } +} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-6550085676695552 b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-6550085676695552 new file mode 100644 index 000000000000..d443f6dee729 --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/clusterfuzz-testcase-minimized-filter_fuzz_test-6550085676695552 @@ -0,0 +1 @@ +config { name: "envoy.filters.http.wasm" } \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc index 7e773b4f1311..429e34191ba1 100644 --- a/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc +++ b/test/extensions/filters/http/common/fuzz/filter_fuzz_test.cc @@ -48,6 +48,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::http::FilterFuzzTestCase& i try { // Catch invalid header characters. TestUtility::validate(input); + ENVOY_LOG_MISC(debug, "Filter configuration: {}", input.config().DebugString()); // Fuzz filter. static UberFilterFuzzer fuzzer; fuzzer.fuzz(input.config(), input.data(), input.upstream_data()); diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.h b/test/extensions/filters/http/common/fuzz/uber_filter.h index 8bcef67d6afa..f89338c54148 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.h +++ b/test/extensions/filters/http/common/fuzz/uber_filter.h @@ -4,6 +4,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/server/factory_context.h" #include "test/mocks/stream_info/mocks.h" +#include "test/test_common/test_runtime.h" namespace Envoy { namespace Extensions { @@ -39,7 +40,9 @@ class UberFilterFuzzer : public HttpFilterFuzzer { Network::Address::InstanceConstSharedPtr addr_; NiceMock cluster_manager_; NiceMock async_request_; + envoy::config::core::v3::Metadata listener_metadata_; NiceMock stream_info_; + TestScopedRuntime scoped_runtime_; // Filter constructed from the config. Http::StreamDecoderFilterSharedPtr decoder_filter_; diff --git a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc index ba694d4632df..11a2c7bc1a1f 100644 --- a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc @@ -148,6 +148,10 @@ void UberFilterFuzzer::perFilterSetup() { ON_CALL(factory_context_, admin()).WillByDefault(testing::ReturnRef(factory_context_.admin_)); ON_CALL(factory_context_.admin_, addHandler(_, _, _, _, _)).WillByDefault(testing::Return(true)); ON_CALL(factory_context_.admin_, removeHandler(_)).WillByDefault(testing::Return(true)); + + // Prepare expectations for WASM filter. + ON_CALL(factory_context_, listenerMetadata()) + .WillByDefault(testing::ReturnRef(listener_metadata_)); } } // namespace HttpFilters diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD index ff8393e41984..8838adb851cf 100644 --- a/test/extensions/filters/http/ext_proc/BUILD +++ b/test/extensions/filters/http/ext_proc/BUILD @@ -5,6 +5,7 @@ load( load( "//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", + "envoy_extension_cc_test_library", ) licenses(["notice"]) # Apache 2 @@ -27,7 +28,10 @@ envoy_extension_cc_test( srcs = ["filter_test.cc"], extension_name = "envoy.filters.http.ext_proc", deps = [ + ":mock_server_lib", + ":utils_lib", "//source/extensions/filters/http/ext_proc", + "//test/common/http:common_lib", "//test/mocks/server:factory_context_mocks", "//test/test_common:test_runtime_lib", ], @@ -46,3 +50,51 @@ envoy_extension_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_extension_cc_test( + name = "mutation_utils_test", + srcs = ["mutation_utils_test.cc"], + extension_name = "envoy.filters.http.ext_proc", + deps = [ + ":utils_lib", + "//source/extensions/filters/http/ext_proc:mutation_utils_lib", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "ext_proc_integration_test", + srcs = ["ext_proc_integration_test.cc"], + extension_name = "envoy.filters.http.ext_proc", + deps = [ + ":utils_lib", + "//source/extensions/filters/http/ext_proc:config", + "//test/common/http:common_lib", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/filters/http/ext_proc/v3alpha:pkg_cc_proto", + "@envoy_api//envoy/service/ext_proc/v3alpha:pkg_cc_proto", + ], +) + +envoy_extension_cc_test_library( + name = "mock_server_lib", + srcs = ["mock_server.cc"], + hdrs = ["mock_server.h"], + extension_name = "envoy.filters.http.ext_proc", + deps = [ + "//source/extensions/filters/http/ext_proc:client_interface", + ], +) + +envoy_extension_cc_test_library( + name = "utils_lib", + srcs = ["utils.cc"], + hdrs = ["utils.h"], + extension_name = "envoy.filters.http.ext_proc", + deps = [ + "//include/envoy/http:header_map_interface", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/http/ext_proc/config_test.cc b/test/extensions/filters/http/ext_proc/config_test.cc index 59d3757457ab..34e38c8df433 100644 --- a/test/extensions/filters/http/ext_proc/config_test.cc +++ b/test/extensions/filters/http/ext_proc/config_test.cc @@ -36,7 +36,7 @@ TEST(HttpExtProcConfigTest, CorrectConfig) { ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); TestUtility::loadFromYaml(yaml, *proto_config); - testing::StrictMock context; + testing::NiceMock context; EXPECT_CALL(context, messageValidationVisitor()); Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "stats", context); Http::MockFilterChainFactoryCallbacks filter_callback; diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc new file mode 100644 index 000000000000..0e921e71bb09 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -0,0 +1,191 @@ +#include "envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.pb.h" +#include "envoy/network/address.h" +#include "envoy/service/ext_proc/v3alpha/external_processor.pb.h" + +#include "extensions/filters/http/ext_proc/config.h" + +#include "test/common/http/common.h" +#include "test/extensions/filters/http/ext_proc/utils.h" +#include "test/integration/http_integration.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +using envoy::service::ext_proc::v3alpha::ProcessingRequest; +using envoy::service::ext_proc::v3alpha::ProcessingResponse; +using Extensions::HttpFilters::ExternalProcessing::ExtProcTestUtility; + +using Http::LowerCaseString; + +class ExtProcIntegrationTest : public HttpIntegrationTest, + public Grpc::GrpcClientIntegrationParamTest { +protected: + ExtProcIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} + + void createUpstreams() override { + // Need to create a separate "upstream" for the gRPC server + HttpIntegrationTest::createUpstreams(); + addFakeUpstream(FakeHttpConnection::Type::HTTP2); + } + + void TearDown() override { + if (processor_connection_) { + ASSERT_TRUE(processor_connection_->close()); + ASSERT_TRUE(processor_connection_->waitForDisconnect()); + } + cleanupUpstreamAndDownstream(); + } + + void initializeConfig() { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + // This is the cluster for our gRPC server, starting by copying an existing cluster + auto* server_cluster = bootstrap.mutable_static_resources()->add_clusters(); + server_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + server_cluster->set_name("ext_proc_server"); + server_cluster->mutable_load_assignment()->set_cluster_name("ext_proc_server"); + ConfigHelper::setHttp2(*server_cluster); + + // Load configuration of the server from YAML and use a helper to add a grpc_service + // stanza pointing to the cluster that we just made + setGrpcService(*proto_config_.mutable_grpc_service(), "ext_proc_server", + fake_upstreams_.back()->localAddress()); + + // Construct a configuration proto for our filter and then re-write it + // to JSON so that we can add it to the overall config + envoy::config::listener::v3::Filter ext_proc_filter; + ext_proc_filter.set_name("envoy.filters.http.ext_proc"); + ext_proc_filter.mutable_typed_config()->PackFrom(proto_config_); + config_helper_.addFilter(MessageUtil::getJsonStringFromMessage(ext_proc_filter)); + }); + } + + void waitForFirstMessage(ProcessingRequest& request) { + ASSERT_TRUE(fake_upstreams_.back()->waitForHttpConnection(*dispatcher_, processor_connection_)); + ASSERT_TRUE(processor_connection_->waitForNewStream(*dispatcher_, processor_stream_)); + ASSERT_TRUE(processor_stream_->waitForGrpcMessage(*dispatcher_, request)); + } + + envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor proto_config_{}; + FakeHttpConnectionPtr processor_connection_; + FakeStreamPtr processor_stream_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, ExtProcIntegrationTest, + GRPC_CLIENT_INTEGRATION_PARAMS); + +TEST_P(ExtProcIntegrationTest, GetAndCloseStream) { + initializeConfig(); + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + HttpIntegrationTest::initialize(); + + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + auto response = codec_client_->makeHeaderOnlyRequest(headers); + + ProcessingRequest request_headers_msg; + waitForFirstMessage(request_headers_msg); + // Just close the stream without doing anything + processor_stream_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + processor_stream_->encodeTrailers(Http::TestResponseTrailerMapImpl{{"grpc-status", "0"}}); + + // Now expect a message to the real upstream + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + + // Respond from the upstream with a simple 200 + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request_->encodeData(100, true); + + // Now expect a response to the original request + response->waitForEndStream(); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + +TEST_P(ExtProcIntegrationTest, GetAndFailStream) { + initializeConfig(); + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + HttpIntegrationTest::initialize(); + + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + auto response = codec_client_->makeHeaderOnlyRequest(headers); + + ProcessingRequest request_headers_msg; + waitForFirstMessage(request_headers_msg); + // Fail the stream immediately + processor_stream_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "500"}}, true); + + response->waitForEndStream(); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("500", response->headers().getStatusValue()); +} + +TEST_P(ExtProcIntegrationTest, GetAndSetHeaders) { + initializeConfig(); + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + HttpIntegrationTest::initialize(); + + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + Http::TestRequestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + headers.addCopy(LowerCaseString("x-remove-this"), "yes"); + auto response = codec_client_->makeHeaderOnlyRequest(headers); + + ProcessingRequest request_headers_msg; + waitForFirstMessage(request_headers_msg); + + EXPECT_TRUE(request_headers_msg.has_request_headers()); + const auto request_headers = request_headers_msg.request_headers(); + Http::TestRequestHeaderMapImpl expected_request_headers{{":scheme", "http"}, + {":method", "GET"}, + {"host", "host"}, + {":path", "/"}, + {"x-remove-this", "yes"}}; + EXPECT_TRUE(ExtProcTestUtility::headerProtosEqualIgnoreOrder(expected_request_headers, + request_headers.headers())); + + processor_stream_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + + // Ask to change the headers + ProcessingResponse response_msg; + auto response_headers_msg = response_msg.mutable_request_headers(); + auto response_header_mutation = + response_headers_msg->mutable_response()->mutable_header_mutation(); + auto mut1 = response_header_mutation->add_set_headers(); + mut1->mutable_header()->set_key("x-new-header"); + mut1->mutable_header()->set_value("new"); + response_header_mutation->add_remove_headers("x-remove-this"); + processor_stream_->sendGrpcMessage(response_msg); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + + auto has_hdr1 = upstream_request_->headers().get(LowerCaseString("x-remove-this")); + EXPECT_TRUE(has_hdr1.empty()); + auto has_hdr2 = upstream_request_->headers().get(LowerCaseString("x-new-header")); + EXPECT_EQ(has_hdr2.size(), 1); + EXPECT_EQ(has_hdr2[0]->value(), "new"); + + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + upstream_request_->encodeData(100, true); + + response->waitForEndStream(); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/http/ext_proc/filter_test.cc b/test/extensions/filters/http/ext_proc/filter_test.cc index 7f821c6b130e..69968a23c886 100644 --- a/test/extensions/filters/http/ext_proc/filter_test.cc +++ b/test/extensions/filters/http/ext_proc/filter_test.cc @@ -1,5 +1,8 @@ #include "extensions/filters/http/ext_proc/ext_proc.h" +#include "test/common/http/common.h" +#include "test/extensions/filters/http/ext_proc/mock_server.h" +#include "test/extensions/filters/http/ext_proc/utils.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/router/mocks.h" @@ -18,21 +21,63 @@ namespace HttpFilters { namespace ExternalProcessing { namespace { -template class HttpFilterTestBase : public T { -public: - HttpFilterTestBase() = default; +using envoy::service::ext_proc::v3alpha::ProcessingRequest; +using envoy::service::ext_proc::v3alpha::ProcessingResponse; +using Http::FilterDataStatus; +using Http::FilterHeadersStatus; +using Http::FilterTrailersStatus; +using Http::LowerCaseString; + +using testing::Invoke; + +using namespace std::chrono_literals; + +class HttpFilterTest : public testing::Test { +protected: void initialize(std::string&& yaml) { + client_ = std::make_unique(); + EXPECT_CALL(*client_, start(_, _)).WillOnce(Invoke(this, &HttpFilterTest::doStart)); + envoy::extensions::filters::http::ext_proc::v3alpha::ExternalProcessor proto_config{}; if (!yaml.empty()) { TestUtility::loadFromYaml(yaml, proto_config); } - config_.reset(new FilterConfig(proto_config)); - filter_ = std::make_unique(config_); + config_.reset(new FilterConfig(proto_config, 200ms, stats_store_, "")); + filter_ = std::make_unique(config_, std::move(client_)); filter_->setEncoderFilterCallbacks(encoder_callbacks_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); } + ExternalProcessorStreamPtr doStart(ExternalProcessorCallbacks& callbacks, + const std::chrono::milliseconds& timeout) { + stream_callbacks_ = &callbacks; + stream_timeout_ = timeout; + + auto stream = std::make_unique(); + EXPECT_CALL(*stream, send(_, _)).WillRepeatedly(Invoke(this, &HttpFilterTest::doSend)); + EXPECT_CALL(*stream, close()).WillRepeatedly(Invoke(this, &HttpFilterTest::doSendClose)); + return stream; + } + + void doSend(ProcessingRequest&& request, bool end_stream) { + ASSERT_FALSE(stream_close_sent_); + last_request_ = std::move(request); + if (end_stream) { + stream_close_sent_ = true; + } + } + + void doSendClose() { + ASSERT_FALSE(stream_close_sent_); + stream_close_sent_ = true; + } + + std::unique_ptr client_; + ExternalProcessorCallbacks* stream_callbacks_ = nullptr; + ProcessingRequest last_request_; + bool stream_close_sent_ = false; + std::chrono::milliseconds stream_timeout_; NiceMock stats_store_; FilterConfigSharedPtr config_; std::unique_ptr filter_; @@ -45,34 +90,342 @@ template class HttpFilterTestBase : public T { Buffer::OwnedImpl data_; }; -class HttpFilterTest : public HttpFilterTestBase { -public: - HttpFilterTest() = default; -}; - TEST_F(HttpFilterTest, SimplestPost) { initialize(R"EOF( grpc_service: envoy_grpc: - cluster_name: "ext_authz_server" + cluster_name: "ext_proc_server" + failure_mode_allow: true + )EOF"); + + EXPECT_TRUE(config_->failureModeAllow()); + + // Create synthetic HTTP request + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + request_headers_.addCopy(LowerCaseString("content-type"), "text/plain"); + request_headers_.addCopy(LowerCaseString("content-length"), 10); + request_headers_.addCopy(LowerCaseString("x-some-other-header"), "yes"); + + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + // Verify that call was received by mock gRPC server + EXPECT_FALSE(last_request_.async_mode()); + EXPECT_FALSE(stream_close_sent_); + ASSERT_TRUE(last_request_.has_request_headers()); + const auto request_headers = last_request_.request_headers(); + EXPECT_FALSE(request_headers.end_of_stream()); + + Http::TestRequestHeaderMapImpl expected{{":path", "/"}, + {":method", "POST"}, + {":scheme", "http"}, + {"host", "host"}, + {"content-type", "text/plain"}, + {"content-length", "10"}, + {"x-some-other-header", "yes"}}; + EXPECT_TRUE( + ExtProcTestUtility::headerProtosEqualIgnoreOrder(expected, request_headers.headers())); + + // Send back a response + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + std::unique_ptr resp1 = std::make_unique(); + resp1->mutable_request_headers(); + stream_callbacks_->onReceiveMessage(std::move(resp1)); + + data_.add("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + data_.add("bar"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + filter_->onDestroy(); + EXPECT_TRUE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_received_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); +} + +TEST_F(HttpFilterTest, PostAndChangeHeaders) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + )EOF"); + + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + request_headers_.addCopy(LowerCaseString("x-some-other-header"), "yes"); + request_headers_.addCopy(LowerCaseString("x-do-we-want-this"), "no"); + + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_FALSE(last_request_.async_mode()); + EXPECT_FALSE(stream_close_sent_); + ASSERT_TRUE(last_request_.has_request_headers()); + + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + std::unique_ptr resp1 = std::make_unique(); + auto req_headers_response = resp1->mutable_request_headers(); + auto headers_mut = req_headers_response->mutable_response()->mutable_header_mutation(); + auto add1 = headers_mut->add_set_headers(); + add1->mutable_header()->set_key("x-new-header"); + add1->mutable_header()->set_value("new"); + add1->mutable_append()->set_value(false); + auto add2 = headers_mut->add_set_headers(); + add2->mutable_header()->set_key("x-some-other-header"); + add2->mutable_header()->set_value("no"); + add2->mutable_append()->set_value(true); + *headers_mut->add_remove_headers() = "x-do-we-want-this"; + stream_callbacks_->onReceiveMessage(std::move(resp1)); + + // We should now have changed the original header a bit + request_headers_.iterate([](const Http::HeaderEntry& e) -> Http::HeaderMap::Iterate { + std::cerr << e.key().getStringView() << ": " << e.value().getStringView() << '\n'; + return Http::HeaderMap::Iterate::Continue; + }); + auto get1 = request_headers_.get(LowerCaseString("x-new-header")); + EXPECT_EQ(get1.size(), 1); + EXPECT_EQ(get1[0]->key(), "x-new-header"); + EXPECT_EQ(get1[0]->value(), "new"); + auto get2 = request_headers_.get(LowerCaseString("x-some-other-header")); + EXPECT_EQ(get2.size(), 2); + EXPECT_EQ(get2[0]->key(), "x-some-other-header"); + EXPECT_EQ(get2[0]->value(), "yes"); + EXPECT_EQ(get2[1]->key(), "x-some-other-header"); + EXPECT_EQ(get2[1]->value(), "no"); + auto get3 = request_headers_.get(LowerCaseString("x-do-we-want-this")); + EXPECT_TRUE(get3.empty()); + + data_.add("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + data_.add("bar"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + filter_->onDestroy(); + EXPECT_TRUE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_received_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); +} + +TEST_F(HttpFilterTest, PostAndRespondImmediately) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + )EOF"); + + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + std::unique_ptr resp1 = std::make_unique(); + auto* immediate_response = resp1->mutable_immediate_response(); + immediate_response->mutable_status()->set_code(envoy::type::v3::StatusCode::BadRequest); + immediate_response->set_body("Bad request"); + immediate_response->set_details("Got a bad request"); + stream_callbacks_->onReceiveMessage(std::move(resp1)); + + // Immediate response processing not yet implemented -- all we can expect + // at this point is that continueDecoding is called and that the + // stream is not yet closed. + EXPECT_FALSE(stream_close_sent_); + + data_.add("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + data_.add("bar"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + filter_->onDestroy(); + EXPECT_TRUE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_received_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); +} + +TEST_F(HttpFilterTest, PostAndFail) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + )EOF"); + + EXPECT_FALSE(config_->failureModeAllow()); + + // Create synthetic HTTP request + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + EXPECT_FALSE(stream_close_sent_); + + // Oh no! The remote server had a failure! + EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::InternalServerError, _, _, _, _)); + stream_callbacks_->onGrpcError(Grpc::Status::Internal); + + data_.add("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + data_.add("bar"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + filter_->onDestroy(); + // The other side closed the stream + EXPECT_FALSE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().streams_failed_.value()); +} + +TEST_F(HttpFilterTest, PostAndIgnoreFailure) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" failure_mode_allow: true )EOF"); EXPECT_TRUE(config_->failureModeAllow()); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); + // Create synthetic HTTP request + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + EXPECT_FALSE(stream_close_sent_); + + // Oh no! The remote server had a failure which we will ignore + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + stream_callbacks_->onGrpcError(Grpc::Status::Internal); + data_.add("foo"); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, true)); - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, - filter_->encode100ContinueHeaders(response_headers_)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); data_.add("bar"); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, false)); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, true)); - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); filter_->onDestroy(); + // The other side closed the stream + EXPECT_FALSE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); + EXPECT_EQ(1, config_->stats().failure_mode_allowed_.value()); +} + +TEST_F(HttpFilterTest, PostAndClose) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + )EOF"); + + EXPECT_FALSE(config_->failureModeAllow()); + + // Create synthetic HTTP request + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_FALSE(last_request_.async_mode()); + EXPECT_FALSE(stream_close_sent_); + ASSERT_TRUE(last_request_.has_request_headers()); + + // Close the stream, which should tell the filter to keep on going + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + stream_callbacks_->onGrpcClose(); + + data_.add("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + data_.add("bar"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + filter_->onDestroy(); + + // The other side closed the stream + EXPECT_FALSE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); +} + +TEST_F(HttpFilterTest, OutOfOrder) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_proc_server" + )EOF"); + + HttpTestUtility::addDefaultHeaders(request_headers_, "POST"); + EXPECT_EQ(FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_FALSE(last_request_.async_mode()); + EXPECT_FALSE(stream_close_sent_); + ASSERT_TRUE(last_request_.has_request_headers()); + + // Return an out-of-order message. The server should close the stream + // and continue as if nothing happened. + EXPECT_CALL(decoder_callbacks_, continueDecoding()); + std::unique_ptr resp1 = std::make_unique(); + resp1->mutable_request_body(); + stream_callbacks_->onReceiveMessage(std::move(resp1)); + + data_.add("foo"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->decodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); + + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(response_headers_)); + EXPECT_EQ(FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, true)); + data_.add("bar"); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, false)); + EXPECT_EQ(FilterDataStatus::Continue, filter_->encodeData(data_, true)); + EXPECT_EQ(FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); + filter_->onDestroy(); + + // We closed the stream + EXPECT_TRUE(stream_close_sent_); + + EXPECT_EQ(1, config_->stats().streams_started_.value()); + EXPECT_EQ(1, config_->stats().stream_msgs_sent_.value()); + EXPECT_EQ(1, config_->stats().spurious_msgs_received_.value()); + EXPECT_EQ(1, config_->stats().streams_closed_.value()); } } // namespace diff --git a/test/extensions/filters/http/ext_proc/mock_server.cc b/test/extensions/filters/http/ext_proc/mock_server.cc new file mode 100644 index 000000000000..888144e38518 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/mock_server.cc @@ -0,0 +1,17 @@ +#include "test/extensions/filters/http/ext_proc/mock_server.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +MockClient::MockClient() = default; +MockClient::~MockClient() = default; + +MockStream::MockStream() = default; +MockStream::~MockStream() = default; + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/http/ext_proc/mock_server.h b/test/extensions/filters/http/ext_proc/mock_server.h new file mode 100644 index 000000000000..6e3654d5f9c8 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/mock_server.h @@ -0,0 +1,31 @@ +#pragma once + +#include "extensions/filters/http/ext_proc/client.h" + +#include "gmock/gmock.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +class MockClient : public ExternalProcessorClient { +public: + MockClient(); + ~MockClient() override; + MOCK_METHOD(ExternalProcessorStreamPtr, start, + (ExternalProcessorCallbacks&, const std::chrono::milliseconds&)); +}; + +class MockStream : public ExternalProcessorStream { +public: + MockStream(); + ~MockStream() override; + MOCK_METHOD(void, send, (envoy::service::ext_proc::v3alpha::ProcessingRequest&&, bool)); + MOCK_METHOD(void, close, ()); +}; + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/http/ext_proc/mutation_utils_test.cc b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc new file mode 100644 index 000000000000..fd31976332e3 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/mutation_utils_test.cc @@ -0,0 +1,125 @@ +#include "extensions/filters/http/ext_proc/mutation_utils.h" + +#include "test/extensions/filters/http/ext_proc/utils.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { +namespace { + +using Http::LowerCaseString; + +TEST(MutationUtils, TestBuildHeaders) { + Http::TestRequestHeaderMapImpl headers{ + {":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"content-type", "text/plain; encoding=UTF8"}, + {"x-something-else", "yes"}, + }; + LowerCaseString reference_key("x-reference"); + std::string reference_value("Foo"); + headers.addReference(reference_key, reference_value); + headers.addCopy(LowerCaseString("x-number"), 9999); + + envoy::config::core::v3::HeaderMap proto_headers; + MutationUtils::buildHttpHeaders(headers, proto_headers); + + Http::TestRequestHeaderMapImpl expected{{":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"content-type", "text/plain; encoding=UTF8"}, + {"x-something-else", "yes"}, + {"x-reference", "Foo"}, + {"x-number", "9999"}}; + EXPECT_TRUE(ExtProcTestUtility::headerProtosEqualIgnoreOrder(expected, proto_headers)); +} + +TEST(MutationUtils, TestApplyMutations) { + Http::TestRequestHeaderMapImpl headers{ + {":scheme", "https"}, + {":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"host", "localhost:1000"}, + {":authority", "localhost:1000"}, + {"content-type", "text/plain; encoding=UTF8"}, + {"x-append-this", "1"}, + {"x-replace-this", "Yes"}, + {"x-remove-this", "Yes"}, + {"x-envoy-strange-thing", "No"}, + }; + + envoy::service::ext_proc::v3alpha::HeaderMutation mutation; + auto* s = mutation.add_set_headers(); + s->mutable_append()->set_value(true); + s->mutable_header()->set_key("x-append-this"); + s->mutable_header()->set_value("2"); + s = mutation.add_set_headers(); + s->mutable_append()->set_value(true); + s->mutable_header()->set_key("x-append-this"); + s->mutable_header()->set_value("3"); + s = mutation.add_set_headers(); + s->mutable_append()->set_value(false); + s->mutable_header()->set_key("x-replace-this"); + s->mutable_header()->set_value("no"); + // Default of "append" is "false" and mutations + // are applied in order. + s = mutation.add_set_headers(); + s->mutable_header()->set_key("x-replace-this"); + s->mutable_header()->set_value("nope"); + // Incomplete structures should be ignored + mutation.add_set_headers(); + + mutation.add_remove_headers("x-remove-this"); + // Attempts to remove ":" and "host" headers should be ignored + mutation.add_remove_headers("host"); + mutation.add_remove_headers(":method"); + mutation.add_remove_headers(""); + + // Attempts to set method, host, authority, and x-envoy headers + // should be ignored until we explicitly allow them. + s = mutation.add_set_headers(); + s->mutable_header()->set_key("host"); + s->mutable_header()->set_value("invalid:123"); + s = mutation.add_set_headers(); + s->mutable_header()->set_key("Host"); + s->mutable_header()->set_value("invalid:456"); + s = mutation.add_set_headers(); + s->mutable_header()->set_key(":authority"); + s->mutable_header()->set_value("invalid:789"); + s = mutation.add_set_headers(); + s->mutable_header()->set_key(":method"); + s->mutable_header()->set_value("PATCH"); + s = mutation.add_set_headers(); + s->mutable_header()->set_key(":scheme"); + s->mutable_header()->set_value("http"); + s = mutation.add_set_headers(); + s->mutable_header()->set_key("X-Envoy-StrangeThing"); + s->mutable_header()->set_value("Yes"); + + MutationUtils::applyHeaderMutations(mutation, headers); + + Http::TestRequestHeaderMapImpl expected_headers{ + {":scheme", "https"}, + {":method", "GET"}, + {":path", "/foo/the/bar?size=123"}, + {"host", "localhost:1000"}, + {":authority", "localhost:1000"}, + {"content-type", "text/plain; encoding=UTF8"}, + {"x-append-this", "1"}, + {"x-append-this", "2"}, + {"x-append-this", "3"}, + {"x-replace-this", "nope"}, + {"x-envoy-strange-thing", "No"}, + }; + + EXPECT_TRUE(TestUtility::headerMapEqualIgnoreOrder(headers, expected_headers)); +} + +} // namespace +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/http/ext_proc/utils.cc b/test/extensions/filters/http/ext_proc/utils.cc new file mode 100644 index 000000000000..1bc4464b1de8 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/utils.cc @@ -0,0 +1,24 @@ +#include "test/extensions/filters/http/ext_proc/utils.h" + +#include "test/test_common/utility.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +bool ExtProcTestUtility::headerProtosEqualIgnoreOrder( + const Http::HeaderMap& expected, const envoy::config::core::v3::HeaderMap& actual) { + // Comparing header maps is hard because they have duplicates in them. + // So we're going to turn them into a HeaderMap and let Envoy do the work. + Http::TestRequestHeaderMapImpl actual_headers; + for (const auto& header : actual.headers()) { + actual_headers.addCopy(header.key(), header.value()); + } + return TestUtility::headerMapEqualIgnoreOrder(expected, actual_headers); +} + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/http/ext_proc/utils.h b/test/extensions/filters/http/ext_proc/utils.h new file mode 100644 index 000000000000..93ec895d0a04 --- /dev/null +++ b/test/extensions/filters/http/ext_proc/utils.h @@ -0,0 +1,21 @@ +#pragma once + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/http/header_map.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace ExternalProcessing { + +class ExtProcTestUtility { +public: + // Compare a reference header map to a proto + static bool headerProtosEqualIgnoreOrder(const Http::HeaderMap& expected, + const envoy::config::core::v3::HeaderMap& actual); +}; + +} // namespace ExternalProcessing +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/ratelimit/ratelimit_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_test.cc index 06feeac7954d..1dccd4f06538 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_test.cc @@ -167,8 +167,8 @@ TEST_F(HttpRateLimitFilterTest, NoApplicableRateLimit) { TEST_F(HttpRateLimitFilterTest, NoDescriptor) { SetUpTest(filter_config_); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)); EXPECT_CALL(*client_, limit(_, _, _, _, _)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); @@ -201,8 +201,8 @@ TEST_F(HttpRateLimitFilterTest, OkResponse) { EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -246,8 +246,8 @@ TEST_F(HttpRateLimitFilterTest, OkResponseWithHeaders) { EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -303,8 +303,8 @@ TEST_F(HttpRateLimitFilterTest, OkResponseWithFilterHeaders) { EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -359,8 +359,8 @@ TEST_F(HttpRateLimitFilterTest, ImmediateOkResponse) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, "foo", testing::ContainerEq(std::vector{ @@ -390,8 +390,8 @@ TEST_F(HttpRateLimitFilterTest, ImmediateErrorResponse) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, "foo", testing::ContainerEq(std::vector{ @@ -426,8 +426,8 @@ TEST_F(HttpRateLimitFilterTest, ErrorResponse) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -460,8 +460,8 @@ TEST_F(HttpRateLimitFilterTest, ErrorResponseWithFailureModeAllowOff) { SetUpTest(fail_close_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -492,8 +492,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponse) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -532,8 +532,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithDynamicMetadata) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -584,8 +584,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithHeaders) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -636,8 +636,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithBody) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -699,8 +699,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithBodyAndContentType) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -768,8 +768,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithFilterHeaders) { SetUpTest(enable_x_ratelimit_headers_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -820,8 +820,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseWithoutEnvoyRateLimitedHeader) { SetUpTest(disable_x_envoy_ratelimited_header_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -858,8 +858,8 @@ TEST_F(HttpRateLimitFilterTest, LimitResponseRuntimeDisabled) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -900,8 +900,8 @@ TEST_F(HttpRateLimitFilterTest, ResetDuringCall) { SetUpTest(filter_config_); InSequence s; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, _, _, _, _)) .WillOnce( WithArgs<0>(Invoke([&](Filters::Common::RateLimit::RequestCallbacks& callbacks) -> void { @@ -922,7 +922,7 @@ TEST_F(HttpRateLimitFilterTest, RouteRateLimitDisabledForRouteKey) { ON_CALL(runtime_.snapshot_, featureEnabled("ratelimit.test_key.http_filter_enabled", 100)) .WillByDefault(Return(false)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); EXPECT_CALL(*client_, limit(_, _, _, _, _)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); @@ -942,7 +942,7 @@ TEST_F(HttpRateLimitFilterTest, VirtualHostRateLimitDisabledForRouteKey) { ON_CALL(runtime_.snapshot_, featureEnabled("ratelimit.test_vh_key.http_filter_enabled", 100)) .WillByDefault(Return(false)); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); EXPECT_CALL(*client_, limit(_, _, _, _, _)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); @@ -964,8 +964,8 @@ TEST_F(HttpRateLimitFilterTest, IncorrectRequestType) { )EOF"; SetUpTest(internal_filter_config); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)).Times(0); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); EXPECT_CALL(*client_, limit(_, _, _, _, _)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); @@ -985,8 +985,8 @@ TEST_F(HttpRateLimitFilterTest, IncorrectRequestType) { SetUpTest(external_filter_config); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)).Times(0); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)).Times(0); EXPECT_CALL(*client_, limit(_, _, _, _, _)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); @@ -1011,8 +1011,8 @@ TEST_F(HttpRateLimitFilterTest, InternalRequestType) { EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -1055,8 +1055,8 @@ TEST_F(HttpRateLimitFilterTest, ExternalRequestType) { EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); @@ -1098,8 +1098,8 @@ TEST_F(HttpRateLimitFilterTest, DEPRECATED_FEATURE_TEST(ExcludeVirtualHost)) { InSequence s; EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_, includeVirtualHostRateLimits()) .WillOnce(Return(false)); @@ -1149,8 +1149,8 @@ TEST_F(HttpRateLimitFilterTest, OverrideVHRateLimitOptionWithRouteRateLimitSet) FilterConfigPerRoute per_route_config_(settings); EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_, includeVirtualHostRateLimits()) .WillOnce(Return(false)); @@ -1214,8 +1214,8 @@ TEST_F(HttpRateLimitFilterTest, OverrideVHRateLimitOptionWithoutRouteRateLimit) EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(*client_, limit(_, "foo", testing::ContainerEq(std::vector{ @@ -1262,8 +1262,8 @@ TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithOnlyVHRateLimitSet) EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_two_)); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_two_)); EXPECT_CALL(*client_, limit(_, "foo", @@ -1299,8 +1299,8 @@ TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithRouteAndVHRateLimitS FilterConfigPerRoute per_route_config_(settings); EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_, includeVirtualHostRateLimits()) .WillOnce(Return(false)); @@ -1312,8 +1312,8 @@ TEST_F(HttpRateLimitFilterTest, IncludeVHRateLimitOptionWithRouteAndVHRateLimitS EXPECT_CALL(filter_callbacks_.route_->route_entry_.virtual_host_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_two_)); + EXPECT_CALL(vh_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_two_)); EXPECT_CALL(*client_, limit(_, "foo", @@ -1349,8 +1349,8 @@ TEST_F(HttpRateLimitFilterTest, IgnoreVHRateLimitOptionWithRouteRateLimitSet) { FilterConfigPerRoute per_route_config_(settings); EXPECT_CALL(filter_callbacks_.route_->route_entry_.rate_limit_policy_, getApplicableRateLimit(0)); - EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _, _, _)) - .WillOnce(SetArgReferee<1>(descriptor_)); + EXPECT_CALL(route_rate_limit_, populateDescriptors(_, _, _, _)) + .WillOnce(SetArgReferee<0>(descriptor_)); EXPECT_CALL(filter_callbacks_.route_->route_entry_, includeVirtualHostRateLimits()) .WillOnce(Return(false)); diff --git a/test/extensions/rate_limit_descriptors/expr/BUILD b/test/extensions/rate_limit_descriptors/expr/BUILD new file mode 100644 index 000000000000..857126f64675 --- /dev/null +++ b/test/extensions/rate_limit_descriptors/expr/BUILD @@ -0,0 +1,34 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), + extension_name = "envoy.rate_limit_descriptors.expr", + deps = [ + "//source/common/protobuf:utility_lib", + "//source/common/router:router_ratelimit_lib", + "//source/extensions/rate_limit_descriptors/expr:config", + "//test/mocks/http:http_mocks", + "//test/mocks/ratelimit:ratelimit_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/rate_limit_descriptors/expr/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/rate_limit_descriptors/expr/config_test.cc b/test/extensions/rate_limit_descriptors/expr/config_test.cc new file mode 100644 index 000000000000..d68981ede37d --- /dev/null +++ b/test/extensions/rate_limit_descriptors/expr/config_test.cc @@ -0,0 +1,221 @@ +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/config/route/v3/route_components.pb.validate.h" +#include "envoy/extensions/rate_limit_descriptors/expr/v3/expr.pb.h" +#include "envoy/extensions/rate_limit_descriptors/expr/v3/expr.pb.validate.h" + +#include "common/protobuf/utility.h" +#include "common/router/router_ratelimit.h" + +#include "extensions/rate_limit_descriptors/expr/config.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/ratelimit/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace RateLimitDescriptors { +namespace Expr { +namespace { + +class RateLimitPolicyEntryTest : public testing::Test { +public: + void setupTest(const std::string& yaml) { + envoy::config::route::v3::RateLimit rate_limit; + TestUtility::loadFromYaml(yaml, rate_limit, false, true); + TestUtility::validate(rate_limit); + rate_limit_entry_ = std::make_unique( + rate_limit, ProtobufMessage::getStrictValidationVisitor()); + } + + std::unique_ptr rate_limit_entry_; + Http::TestRequestHeaderMapImpl header_; + std::vector descriptors_; + NiceMock stream_info_; +}; + +TEST_F(RateLimitPolicyEntryTest, MissingExtension) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom + typed_config: + "@type": type.googleapis.com/google.protobuf.Value + )EOF"; + + EXPECT_THROW_WITH_REGEX(setupTest(yaml), EnvoyException, + "Rate limit descriptor extension not found: .*"); +} + +TEST_F(RateLimitPolicyEntryTest, ExpressionUnset) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + )EOF"; + + EXPECT_THROW_WITH_REGEX(setupTest(yaml), EnvoyException, + "Rate limit descriptor extension failed: .*"); +} + +#if defined(USE_CEL_PARSER) +TEST_F(RateLimitPolicyEntryTest, ExpressionText) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + text: request.headers["x-header-name"] + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{"x-header-name", "test_value"}}; + + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header, stream_info_); + EXPECT_THAT(std::vector({{{{"my_descriptor_name", "test_value"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyEntryTest, ExpressionTextMalformed) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + text: undefined_ext(false) + )EOF"; + + EXPECT_THROW_WITH_REGEX(setupTest(yaml), EnvoyException, "failed to create an expression: .*"); +} + +TEST_F(RateLimitPolicyEntryTest, ExpressionUnparsable) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + text: ++ + )EOF"; + + EXPECT_THROW_WITH_REGEX(setupTest(yaml), EnvoyException, + "Unable to parse descriptor expression: .*"); +} +#endif + +TEST_F(RateLimitPolicyEntryTest, ExpressionParsed) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + parsed: + call_expr: + function: _==_ + args: + - select_expr: + operand: + ident_expr: + name: request + field: method + - const_expr: + string_value: GET + )EOF"; + + setupTest(yaml); + Http::TestRequestHeaderMapImpl header{{":method", "GET"}}; + + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header, stream_info_); + EXPECT_THAT(std::vector({{{{"my_descriptor_name", "true"}}}}), + testing::ContainerEq(descriptors_)); +} + +TEST_F(RateLimitPolicyEntryTest, ExpressionParsedMalformed) { + const std::string yaml = R"EOF( +actions: +- extension: + name: custom_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + parsed: + call_expr: + function: undefined_extent + args: + - const_expr: + bool_value: false + )EOF"; + + EXPECT_THROW_WITH_REGEX(setupTest(yaml), EnvoyException, "failed to create an expression: .*"); +} + +#if defined(USE_CEL_PARSER) +TEST_F(RateLimitPolicyEntryTest, ExpressionTextError) { + const std::string yaml = R"EOF( +actions: +- extension: + name: first_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: test_key + text: "'a'" +- extension: + name: second_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + text: request.headers["x-header-name"] + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header_, stream_info_); + EXPECT_TRUE(descriptors_.empty()); +} + +TEST_F(RateLimitPolicyEntryTest, ExpressionTextErrorSkip) { + const std::string yaml = R"EOF( +actions: +- extension: + name: first_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: test_key + text: "'a'" +- extension: + name: second_descriptor + typed_config: + "@type": type.googleapis.com/envoy.extensions.rate_limit_descriptors.expr.v3.Descriptor + descriptor_key: my_descriptor_name + text: request.headers["x-header-name"] + skip_if_error: true + )EOF"; + + setupTest(yaml); + + rate_limit_entry_->populateDescriptors(descriptors_, "service_cluster", header_, stream_info_); + EXPECT_THAT(std::vector({{{{"test_key", "a"}}}}), + testing::ContainerEq(descriptors_)); +} +#endif + +} // namespace +} // namespace Expr +} // namespace RateLimitDescriptors +} // namespace Extensions +} // namespace Envoy diff --git a/test/integration/BUILD b/test/integration/BUILD index dc921e3a137e..605d63e4f212 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -877,6 +877,8 @@ envoy_cc_test( "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/integration/filters:invalid_header_filter_lib", "//test/integration/filters:process_context_lib", + "//test/integration/filters:set_response_code_filter_config_proto_cc_proto", + "//test/integration/filters:set_response_code_filter_lib", "//test/integration/filters:stop_iteration_and_continue", "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", @@ -1076,6 +1078,7 @@ envoy_cc_test( "//test/integration/filters:set_response_code_filter_config_proto_cc_proto", "//test/integration/filters:set_response_code_filter_lib", "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", "@envoy_api//envoy/service/extension/v3:pkg_cc_proto", ], diff --git a/test/integration/extension_discovery_integration_test.cc b/test/integration/extension_discovery_integration_test.cc index b671e6f94bd9..28f49421dacd 100644 --- a/test/integration/extension_discovery_integration_test.cc +++ b/test/integration/extension_discovery_integration_test.cc @@ -1,3 +1,4 @@ +#include "envoy/extensions/common/matching/v3/extension_matcher.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "envoy/service/extension/v3/config_discovery.pb.h" @@ -18,6 +19,32 @@ std::string denyPrivateConfig() { )EOF"; } +std::string denyPrivateConfigWithMatcher() { + return R"EOF( + "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher + extension_config: + name: response-filter-config + typed_config: + "@type": type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig + prefix: "/private" + code: 403 + matcher: + matcher_tree: + input: + name: request-headers + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: some-header + exact_match_map: + map: + match: + action: + name: skip + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter + )EOF"; +} + std::string allowAllConfig() { return "code: 200"; } std::string invalidConfig() { return "code: 90"; } @@ -29,9 +56,10 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion()) {} void addDynamicFilter(const std::string& name, bool apply_without_warming, - bool set_default_config = true, bool rate_limit = false) { + bool set_default_config = true, bool rate_limit = false, + bool use_default_matcher = false) { config_helper_.addConfigModifier( - [this, name, apply_without_warming, set_default_config, rate_limit]( + [this, name, apply_without_warming, set_default_config, rate_limit, use_default_matcher]( envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& http_connection_manager) { auto* filter = http_connection_manager.mutable_http_filters()->Add(); @@ -39,11 +67,41 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara auto* discovery = filter->mutable_config_discovery(); discovery->add_type_urls( "type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig"); + discovery->add_type_urls( + "type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher"); if (set_default_config) { - const auto default_configuration = - TestUtility::parseYaml( - "code: 403"); - discovery->mutable_default_config()->PackFrom(default_configuration); + if (use_default_matcher) { + const auto default_configuration = TestUtility::parseYaml< + envoy::extensions::common::matching::v3::ExtensionWithMatcher>( + R"EOF( + extension_config: + name: set-response-code + typed_config: + "@type": type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig + code: 403 + matcher: + matcher_tree: + input: + name: request-headers + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: default-matcher-header + exact_match_map: + map: + match: + action: + name: skip + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter + )EOF"); + + discovery->mutable_default_config()->PackFrom(default_configuration); + } else { + const auto default_configuration = + TestUtility::parseYaml( + "code: 403"); + discovery->mutable_default_config()->PackFrom(default_configuration); + } } discovery->set_apply_default_config_without_warming(apply_without_warming); discovery->mutable_config_source()->set_resource_api_version( @@ -125,6 +183,19 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara ecds_stream_->sendGrpcMessage(response); } + void sendXdsResponseWithFullYaml(const std::string& name, const std::string& version, + const std::string& full_yaml) { + envoy::service::discovery::v3::DiscoveryResponse response; + response.set_version_info(version); + response.set_type_url("type.googleapis.com/envoy.config.core.v3.TypedExtensionConfig"); + const auto configuration = TestUtility::parseYaml(full_yaml); + envoy::config::core::v3::TypedExtensionConfig typed_config; + typed_config.set_name(name); + typed_config.mutable_typed_config()->MergeFrom(configuration); + response.add_resources()->PackFrom(typed_config); + ecds_stream_->sendGrpcMessage(response); + } + FakeUpstream& getEcdsFakeUpstream() const { return *fake_upstreams_[1]; } FakeHttpConnectionPtr ecds_connection_{nullptr}; @@ -176,6 +247,81 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccess) { } } +TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccessWithMatcher) { + on_server_init_function_ = [&]() { waitXdsStream(); }; + addDynamicFilter("foo", false); + initialize(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + registerTestServerPorts({"http"}); + sendXdsResponseWithFullYaml("foo", "1", denyPrivateConfigWithMatcher()); + test_server_->waitForCounterGe("http.config_test.extension_config_discovery.foo.config_reload", + 1); + test_server_->waitUntilListenersReady(); + test_server_->waitForGaugeGe("listener_manager.workers_started", 1); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initialized); + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + { + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "host"}}; + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + } + Http::TestRequestHeaderMapImpl banned_request_headers{ + {":method", "GET"}, {":path", "/private/key"}, {":scheme", "http"}, {":authority", "host"}}; + { + auto response = codec_client_->makeHeaderOnlyRequest(banned_request_headers); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("403", response->headers().getStatusValue()); + } + Http::TestRequestHeaderMapImpl banned_request_headers_skipped{{":method", "GET"}, + {":path", "/private/key"}, + {"some-header", "match"}, + {":scheme", "http"}, + {":authority", "host"}}; + { + auto response = codec_client_->makeHeaderOnlyRequest(banned_request_headers_skipped); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + } +} + +TEST_P(ExtensionDiscoveryIntegrationTest, BasicDefaultMatcher) { + on_server_init_function_ = [&]() { waitXdsStream(); }; + addDynamicFilter("foo", false, true, false, true); + initialize(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + registerTestServerPorts({"http"}); + sendXdsResponse("foo", "1", invalidConfig()); + test_server_->waitForCounterGe("http.config_test.extension_config_discovery.foo.config_fail", 1); + test_server_->waitUntilListenersReady(); + test_server_->waitForGaugeGe("listener_manager.workers_started", 1); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initialized); + codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); + { + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "host"}}; + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("403", response->headers().getStatusValue()); + } + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {"default-matcher-header", "match"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}}; + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + TEST_P(ExtensionDiscoveryIntegrationTest, BasicFailWithDefault) { on_server_init_function_ = [&]() { waitXdsStream(); }; addDynamicFilter("foo", false); diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index f662744274f9..dc6e3f0b214e 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -364,6 +364,58 @@ TEST_P(IntegrationTest, EnvoyProxying100ContinueWithDecodeDataPause) { testEnvoyProxying1xx(true); } +// Verifies that we can construct a match tree with a filter, and that we are able to skip +// filter invocation through the match tree. +TEST_P(IntegrationTest, MatchingHttpFilterConstruction) { + config_helper_.addFilter(R"EOF( +name: matcher +typed_config: + "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher + extension_config: + name: set-response-code + typed_config: + "@type": type.googleapis.com/test.integration.filters.SetResponseCodeFilterConfig + code: 403 + matcher: + matcher_tree: + input: + name: request-headers + typed_config: + "@type": type.googleapis.com/envoy.type.matcher.v3.HttpRequestHeaderMatchInput + header_name: match-header + exact_match_map: + map: + match: + action: + name: skip + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter +)EOF"); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + { + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 1024); + response->waitForEndStream(); + EXPECT_THAT(response->headers(), HttpStatusIs("403")); + } + + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, {":path", "/test/long/url"}, {":scheme", "http"}, + {":authority", "host"}, {"match-header", "match"}, {"content-type", "application/grpc"}}; + auto response = codec_client_->makeRequestWithBody(request_headers, 1024); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, true); + + response->waitForEndStream(); + EXPECT_THAT(response->headers(), HttpStatusIs("200")); + + codec_client_->close(); +} + // This is a regression for https://github.com/envoyproxy/envoy/issues/2715 and validates that a // pending request is not sent on a connection that has been half-closed. TEST_P(IntegrationTest, UpstreamDisconnectWithTwoRequests) { diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index fe98c1f27831..d85065dc37bd 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -219,12 +219,15 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, void encode100ContinueHeaders(ResponseHeaderMapPtr&& headers) override { encode100ContinueHeaders_(*headers); } + MOCK_METHOD(ResponseHeaderMapOptRef, continueHeaders, (), (const)); void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view details) override { stream_info_.setResponseCodeDetails(details); encodeHeaders_(*headers, end_stream); } + MOCK_METHOD(ResponseHeaderMapOptRef, responseHeaders, (), (const)); void encodeTrailers(ResponseTrailerMapPtr&& trailers) override { encodeTrailers_(*trailers); } + MOCK_METHOD(ResponseTrailerMapOptRef, responseTrailers, (), (const)); void encodeMetadata(MetadataMapPtr&& metadata_map) override { encodeMetadata_(std::move(metadata_map)); } @@ -374,9 +377,12 @@ class MockStreamFilter : public StreamFilter { // Http::MockStreamEncoderFilter MOCK_METHOD(FilterHeadersStatus, encode100ContinueHeaders, (ResponseHeaderMap & headers)); + MOCK_METHOD(ResponseHeaderMapOptRef, continueHeaders, (), (const)); MOCK_METHOD(FilterHeadersStatus, encodeHeaders, (ResponseHeaderMap & headers, bool end_stream)); + MOCK_METHOD(ResponseHeaderMapOptRef, responseHeaders, (), (const)); MOCK_METHOD(FilterDataStatus, encodeData, (Buffer::Instance & data, bool end_stream)); MOCK_METHOD(FilterTrailersStatus, encodeTrailers, (ResponseTrailerMap & trailers)); + MOCK_METHOD(ResponseTrailerMapOptRef, responseTrailers, (), (const)); MOCK_METHOD(FilterMetadataStatus, encodeMetadata, (MetadataMap & metadata_map)); MOCK_METHOD(void, setEncoderFilterCallbacks, (StreamEncoderFilterCallbacks & callbacks)); diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 39edf3a2f925..6d665cd26252 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -190,10 +190,9 @@ class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { MOCK_METHOD(uint64_t, stage, (), (const)); MOCK_METHOD(const std::string&, disableKey, (), (const)); MOCK_METHOD(void, populateDescriptors, - (const RouteEntry& route, std::vector& descriptors, - const std::string& local_service_cluster, const Http::HeaderMap& headers, - const Network::Address::Instance& remote_address, - const envoy::config::core::v3::Metadata* dynamic_metadata), + (std::vector & descriptors, + const std::string& local_service_cluster, const Http::RequestHeaderMap& headers, + const StreamInfo::StreamInfo& info), (const)); uint64_t stage_{}; diff --git a/tools/BUILD b/tools/BUILD index a3313f01becc..d9dec1be786b 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -42,7 +42,9 @@ envoy_cc_binary( "//source/common/common:assert_lib", "//source/common/protobuf:message_validator_lib", "//source/common/protobuf:utility_lib", + "//source/common/common:random_generator_lib", "//source/common/stats:isolated_store_lib", + "//test/test_common:test_version_linkstamp", "@envoy_api//envoy/config/bootstrap/v2:pkg_cc_proto", ] + envoy_cc_platform_dep("//source/exe:platform_impl_lib"), ) diff --git a/tools/bootstrap2pb.cc b/tools/bootstrap2pb.cc index cf16ce7f8581..5cb2c6c3aeaa 100644 --- a/tools/bootstrap2pb.cc +++ b/tools/bootstrap2pb.cc @@ -13,6 +13,7 @@ #include "common/api/api_impl.h" #include "common/common/assert.h" +#include "common/common/random_generator.h" #include "common/event/real_time_system.h" #include "common/protobuf/message_validator_impl.h" #include "common/protobuf/utility.h" @@ -31,8 +32,9 @@ int main(int argc, char** argv) { Envoy::PlatformImpl platform_impl_; Envoy::Stats::IsolatedStoreImpl stats_store; Envoy::Event::RealTimeSystem time_system; // NO_CHECK_FORMAT(real_time) + Envoy::Random::RandomGeneratorImpl rand; Envoy::Api::Impl api(platform_impl_.threadFactory(), stats_store, time_system, - platform_impl_.fileSystem()); + platform_impl_.fileSystem(), rand); envoy::config::bootstrap::v2::Bootstrap bootstrap; Envoy::MessageUtil::loadFromFile(argv[1], bootstrap, diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index f6ff2c9d7365..71645f4cbfd0 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -546,6 +546,7 @@ def reportError(message): # If we hit the end of this release note block block, check the prior line. if not endsWithPeriod(prior_line): reportError("The following release note does not end with a '.'\n %s" % prior_line) + prior_line = '' elif prior_line: prior_line += line