Merge branch 'master' into mpuncel/http2-hc-goaway

* master: http: support creating filters with match tree (envoyproxy#14430) [tls] Expose ServerContextImpl::selectTlsContext (envoyproxy#14592) docs: update ext_proc docs to reflect implementation status (envoyproxy#14636) filter manager: drop assert (envoyproxy#14633) kick off v1.18.0 (envoyproxy#14637) 1.17.0 release (envoyproxy#14624) Implement request header processing in ext_proc (envoyproxy#14385) http: expose encoded headers/trailers via callbacks (envoyproxy#14544) [fuzz] fix minor fuzz bugs (envoyproxy#14593) rate limit: add computed descriptors (envoyproxy#14448) tools: fill in the required args for Api::Impl (envoyproxy#14554) Bump envoy-build to current images (envoyproxy#14608) Signed-off-by: Michael Puncel <mpuncel@squareup.com>
mpuncel · Jan 12, 2021 · 64cc217 · 64cc217
2 parents eacd1f4 + dcf575a
commit 64cc217
Show file tree

Hide file tree

Showing 112 changed files with 3,208 additions and 518 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -246,7 +246,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl
 
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:9400637f4aa0232465407447bfda0d3da13549fb
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:11efa5680d987fff33fde4af3cc5ece105015d04
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker

diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
-FROM gcr.io/envoy-ci/envoy-build:9400637f4aa0232465407447bfda0d3da13549fb
+FROM gcr.io/envoy-ci/envoy-build:11efa5680d987fff33fde4af3cc5ece105015d04
 
 ARG USERNAME=vscode
 ARG USER_UID=501

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -160,3 +160,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/filters/common/local_ratelimit @mattklein123 @rgs1
 # HTTP Kill Request
 /*/extensions/filters/http/kill_request @qqustc @htuch
+# Rate limit expression descriptor
+/*/extensions/rate_limit_descriptors/expr @kyessenov @lizan
diff --git a/RELEASES.md b/RELEASES.md
@@ -67,7 +67,7 @@ deadline of 3 weeks.
 | 1.14.0  | 2020/03/31 | 2020/04/08 |   +8 days  | 2021/04/08  |
 | 1.15.0  | 2020/06/30 | 2020/07/07 |   +7 days  | 2021/07/07  |
 | 1.16.0  | 2020/09/30 | 2020/10/08 |   +8 days  | 2021/10/08  |
-| 1.17.0  | 2020/12/31 |            |            |             |
-
+| 1.17.0  | 2020/12/31 | 2021/01/11 |  +11 days  | 2022/01/11  |
+| 1.18.0  | 2021/03/31 |            |            |             |
 
 [repokitteh]: https://github.com/repokitteh
diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md
@@ -111,6 +111,9 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE
     `Envoy::Extensions::Upstreams` namespace.
   * [watchdog](/source/extensions/watchdog): Watchdog extensions use the
     `Envoy::Extensions::Watchdog` namespace.
+  * [descriptors](/source/extensions/rate_limit_descriptors): Rate limit
+    descriptor extensions use the `Envoy::Extensions::RateLimitDescriptors`
+    namespace.
 * Each extension is contained wholly in its own namespace. E.g.,
   `Envoy::Extensions::NetworkFilters::Echo`.
 * Common code that is used by multiple extensions should be in a `common/` directory as close to

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.17.0-dev
+1.18.0-dev
diff --git a/api/BUILD b/api/BUILD
@@ -239,6 +239,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
         "//envoy/extensions/network/socket_interface/v3:pkg",
+        "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg",
         "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg",
         "//envoy/extensions/retry/priority/previous_priorities/v3:pkg",
         "//envoy/extensions/stat_sinks/wasm/v3:pkg",

diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto
@@ -49,10 +49,7 @@ message ExtAuthz {
   // <config_http_filters_ext_authz_stats>`.
   bool failure_mode_allow = 2;
 
-  // Sets the package version the gRPC service should use. This is particularly
-  // useful when transitioning from alpha to release versions assuming that both definitions are
-  // semantically compatible. Deprecation note: This field is deprecated and should only be used for
-  // version upgrade. See release notes for more details.
+  // [#not-implemented-hide: Support for this field has been removed.]
   bool use_alpha = 4 [deprecated = true, (envoy.annotations.disallowed_by_default) = true];
 
   // Enables filter to buffer the client request body and send it within the authorization request.

diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto
@@ -1544,7 +1544,7 @@ message VirtualCluster {
 message RateLimit {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit";
 
-  // [#next-free-field: 9]
+  // [#next-free-field: 10]
   message Action {
     option (udpa.annotations.versioning).previous_message_type =
         "envoy.api.v2.route.RateLimit.Action";
@@ -1742,6 +1742,9 @@ message RateLimit {
 
       // Rate limit on metadata.
       MetaData metadata = 8;
+
+      // Rate limit descriptor extension. See the rate limit descriptor extensions documentation.
+      core.v3.TypedExtensionConfig extension = 9;
     }
   }
 

diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto
diff --git a/api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto b/api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto
@@ -21,7 +21,22 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // [#extension: envoy.filters.http.ext_proc]
 
 // The External Processing filter allows an external service to act on HTTP traffic in a flexible way.
-// It communicates with an external gRPC service that can use it to do a variety of things
+
+// **Current Implementation Status:**
+// At this time, the filter will send a "request_headers" message to the server when the
+// filter is invoked from the downstream, and apply any header mutations returned by the
+// server. No other part of the protocol is implemented yet.
+
+// As designed, the filter supports up to six different processing steps, which are in the
+// process of being implemented:
+// * Request headers: IMPLEMENTED
+// * Request body: NOT IMPLEMENTED
+// * Request trailers: NOT IMPLEMENTED
+// * Response headers: NOT IMPLEMENTED
+// * Response body: NOT IMPLEMENTED
+// * Response trailers: NOT IMPLEMENTED
+
+// The filter communicates with an external gRPC service that can use it to do a variety of things
 // with the request and response:
 //
 // * Access and modify the HTTP headers on the request, response, or both

diff --git a/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD b/api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD
@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+        "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto",
+    ],
+)
diff --git a/api/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto b/api/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto
@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package envoy.extensions.rate_limit_descriptors.expr.v3;
+
+import "google/api/expr/v1alpha1/syntax.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.rate_limit_descriptors.expr.v3";
+option java_outer_classname = "ExprProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Rate limit descriptor expression]
+// [#extension: envoy.rate_limit_descriptors.expr]
+
+// The following descriptor entry is appended with a value computed
+// from a symbolic Common Expression Language expression.
+// See :ref:`attributes <arch_overview_attributes>` for the set of
+// available attributes.
+//
+// .. code-block:: cpp
+//
+//   ("<descriptor_key>", "<expression_value>")
+message Descriptor {
+  // The key to use in the descriptor entry.
+  string descriptor_key = 1 [(validate.rules).string = {min_len: 1}];
+
+  // If set to true, Envoy skips the descriptor if the expression evaluates to an error.
+  // By default, the rate limit is not applied when an expression produces an error.
+  bool skip_if_error = 2;
+
+  oneof expr_specifier {
+    // Expression in a text form, e.g. "connection.requested_server_name".
+    string text = 3 [(validate.rules).string = {min_len: 1}];
+
+    // Parsed expression in AST form.
+    google.api.expr.v1alpha1.Expr parsed = 4;
+  }
+}
diff --git a/api/envoy/service/ext_proc/v3alpha/external_processor.proto b/api/envoy/service/ext_proc/v3alpha/external_processor.proto
@@ -6,7 +6,6 @@ import "envoy/config/core/v3/base.proto";
 import "envoy/extensions/filters/http/ext_proc/v3alpha/processing_mode.proto";
 import "envoy/type/v3/http_status.proto";
 
-import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 
 import "udpa/annotations/status.proto";
@@ -289,10 +288,13 @@ message GrpcStatus {
 // Change HTTP headers or trailers by appending, replacing, or removing
 // headers.
 message HeaderMutation {
-  // Add or replace HTTP headers.
+  // Add or replace HTTP headers. Attempts to set the value of
+  // any "x-envoy" header, and attempts to set the ":method",
+  // ":authority", ":scheme", or "host" headers will be ignored.
   repeated config.core.v3.HeaderValueOption set_headers = 1;
 
-  // Remove these HTTP headers.
+  // Remove these HTTP headers. Attempts to remove system headers --
+  // any header starting with ":", plus "host" -- will be ignored.
   repeated string remove_headers = 2;
 }
 

diff --git a/api/envoy/type/matcher/v3/http_inputs.proto b/api/envoy/type/matcher/v3/http_inputs.proto
@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package envoy.type.matcher.v3;
+
+import "udpa/annotations/migrate.proto";
+import "udpa/annotations/status.proto";
+import "udpa/annotations/versioning.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.type.matcher.v3";
+option java_outer_classname = "HttpInputsProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Common HTTP Inputs]
+
+// Match input indicates that matching should be done on a specific request header.
+// The resulting input string will be all headers for the given key joined by a comma,
+// e.g. if the request contains two 'foo' headers with value 'bar' and 'baz', the input
+// string will be 'bar,baz'.
+// [#comment:TODO(snowp): Link to unified matching docs.]
+message HttpRequestHeaderMatchInput {
+  // The request header to match on.
+  string header_name = 1;
+}
+
+// Match input indicating that matching should be done on a specific response header.
+// The resulting input string will be all headers for the given key joined by a comma,
+// e.g. if the response contains two 'foo' headers with value 'bar' and 'baz', the input
+// string will be 'bar,baz'.
+// [#comment:TODO(snowp): Link to unified matching docs.]
+message HttpResponseHeaderMatchInput {
+  // The response header to match on.
+  string header_name = 1;
+}
diff --git a/api/envoy/type/matcher/v4alpha/http_inputs.proto b/api/envoy/type/matcher/v4alpha/http_inputs.proto
diff --git a/api/versioning/BUILD b/api/versioning/BUILD
@@ -122,6 +122,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
         "//envoy/extensions/network/socket_interface/v3:pkg",
+        "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg",
         "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg",
         "//envoy/extensions/retry/priority/previous_priorities/v3:pkg",
         "//envoy/extensions/stat_sinks/wasm/v3:pkg",

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -49,11 +49,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "envoy-build-tools",
         project_desc = "Common build tools shared by the Envoy/UDPA ecosystem",
         project_url = "https://github.com/envoyproxy/envoy-build-tools",
-        version = "3ff9995a5dd3d0e703e602ca3ebd9366de2b5752",
-        sha256 = "a4c74ce9a62b1c907329d248d4c225abfae8646bc77db8d72de65726632d3571",
+        version = "f2a7f9ba09660beacfebcd37fc977480ec9a8f50",
+        sha256 = "ca9975f6d5370843167b9646028ca7a0b546f8821f217c9d9d2e033a94a35f31",
         strip_prefix = "envoy-build-tools-{version}",
         urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"],
-        release_date = "2020-12-21",
+        release_date = "2021-01-04",
         use_category = ["build"],
     ),
     boringssl = dict(
@@ -713,6 +713,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.rbac",
             "envoy.filters.http.wasm",
             "envoy.filters.network.rbac",
@@ -734,6 +735,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.rbac",
             "envoy.filters.http.wasm",
             "envoy.filters.network.rbac",
@@ -912,6 +914,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.wasm",
             "envoy.filters.network.wasm",
             "envoy.stat_sinks.wasm",
@@ -931,6 +934,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.wasm",
             "envoy.filters.network.wasm",
             "envoy.stat_sinks.wasm",

diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst
@@ -24,3 +24,4 @@ Extensions
   upstream/upstream
   wasm/wasm
   watchdog/watchdog
+  descriptors/descriptors
diff --git a/docs/root/api-v3/config/descriptors/descriptors.rst b/docs/root/api-v3/config/descriptors/descriptors.rst
@@ -0,0 +1,8 @@
+Rate limit descriptors
+======================
+
+.. toctree::
+  :glob:
+  :maxdepth: 2
+
+  ../../extensions/rate_limit_descriptors/expr/v3/*
diff --git a/docs/root/api-v3/types/types.rst b/docs/root/api-v3/types/types.rst
@@ -21,5 +21,6 @@ Types
   ../type/matcher/v3/string.proto
   ../type/matcher/v3/struct.proto
   ../type/matcher/v3/value.proto
+  ../type/matcher/v3/http_inputs.proto
   ../type/metadata/v3/metadata.proto
   ../type/tracing/v3/custom_tag.proto
diff --git a/docs/root/configuration/http/http_filters/ext_proc_filter.rst b/docs/root/configuration/http/http_filters/ext_proc_filter.rst
@@ -17,3 +17,24 @@ messages, and the server must reply with
 :ref:`ProcessingResponse <envoy_v3_api_msg_service.ext_proc.v3alpha.ProcessingResponse>`.
 
 This filter is a work in progress. In its current state, it actually does nothing.
+
+Statistics
+----------
+This filter outputs statistics in the
+*http.<stat_prefix>.ext_proc.* namespace. The :ref:`stat prefix
+<envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.stat_prefix>`
+comes from the owning HTTP connection manager.
+
+The following statistics are supported:
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: auto
+
+  streams_started, Counter, The number of gRPC streams that have been started to send to the external processing service
+  streams_msgs_sent, Counter, The number of messages sent on those streams
+  streams_msgs_received, Counter, The number of messages received on those streams
+  spurious_msgs_received, Counter, The number of unexpected messages received that violated the protocol
+  streams_closed, Counter, The number of streams successfully closed on either end
+  streams_failed, Counter, The number of times a stream produced a gRPC error
+  failure_mode_allowed, Counter, The number of times an error was ignored due to configuration