Merge branch 'master' of https://github.com/envoyproxy/envoy into ign…

…ore-transfer-encoding

Signed-off-by: Iacopo Rozzo <iacopo@kubermatic.com>

.bazelrc

@@ -246,7 +246,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl
 
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:9400637f4aa0232465407447bfda0d3da13549fb
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:11efa5680d987fff33fde4af3cc5ece105015d04
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM gcr.io/envoy-ci/envoy-build:9400637f4aa0232465407447bfda0d3da13549fb
+FROM gcr.io/envoy-ci/envoy-build:11efa5680d987fff33fde4af3cc5ece105015d04
 
 ARG USERNAME=vscode
 ARG USER_UID=501

CODEOWNERS

@@ -160,3 +160,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/filters/common/local_ratelimit @mattklein123 @rgs1
 # HTTP Kill Request
 /*/extensions/filters/http/kill_request @qqustc @htuch
+# Rate limit expression descriptor
+/*/extensions/rate_limit_descriptors/expr @kyessenov @lizan

RELEASES.md

@@ -67,7 +67,7 @@ deadline of 3 weeks.
 | 1.14.0  | 2020/03/31 | 2020/04/08 |   +8 days  | 2021/04/08  |
 | 1.15.0  | 2020/06/30 | 2020/07/07 |   +7 days  | 2021/07/07  |
 | 1.16.0  | 2020/09/30 | 2020/10/08 |   +8 days  | 2021/10/08  |
-| 1.17.0  | 2020/12/31 |            |            |             |
-
+| 1.17.0  | 2020/12/31 | 2021/01/11 |  +11 days  | 2022/01/11  |
+| 1.18.0  | 2021/03/31 |            |            |             |
 
 [repokitteh]: https://github.com/repokitteh

REPO_LAYOUT.md

@@ -111,6 +111,9 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE
     `Envoy::Extensions::Upstreams` namespace.
   * [watchdog](/source/extensions/watchdog): Watchdog extensions use the
     `Envoy::Extensions::Watchdog` namespace.
+  * [descriptors](/source/extensions/rate_limit_descriptors): Rate limit
+    descriptor extensions use the `Envoy::Extensions::RateLimitDescriptors`
+    namespace.
 * Each extension is contained wholly in its own namespace. E.g.,
   `Envoy::Extensions::NetworkFilters::Echo`.
 * Common code that is used by multiple extensions should be in a `common/` directory as close to

VERSION

@@ -1 +1 @@
-1.17.0-dev
+1.18.0-dev

api/BUILD

@@ -239,6 +239,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
         "//envoy/extensions/network/socket_interface/v3:pkg",
+        "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg",
         "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg",
         "//envoy/extensions/retry/priority/previous_priorities/v3:pkg",
         "//envoy/extensions/stat_sinks/wasm/v3:pkg",

api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto

@@ -49,10 +49,7 @@ message ExtAuthz {
   // <config_http_filters_ext_authz_stats>`.
   bool failure_mode_allow = 2;
 
-  // Sets the package version the gRPC service should use. This is particularly
-  // useful when transitioning from alpha to release versions assuming that both definitions are
-  // semantically compatible. Deprecation note: This field is deprecated and should only be used for
-  // version upgrade. See release notes for more details.
+  // [#not-implemented-hide: Support for this field has been removed.]
   bool use_alpha = 4 [deprecated = true, (envoy.annotations.disallowed_by_default) = true];
 
   // Enables filter to buffer the client request body and send it within the authorization request.

api/envoy/config/route/v3/route_components.proto

@@ -1544,7 +1544,7 @@ message VirtualCluster {
 message RateLimit {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit";
 
-  // [#next-free-field: 9]
+  // [#next-free-field: 10]
   message Action {
     option (udpa.annotations.versioning).previous_message_type =
         "envoy.api.v2.route.RateLimit.Action";
@@ -1742,6 +1742,9 @@ message RateLimit {
 
       // Rate limit on metadata.
       MetaData metadata = 8;
+
+      // Rate limit descriptor extension. See the rate limit descriptor extensions documentation.
+      core.v3.TypedExtensionConfig extension = 9;
     }
   }
 

api/envoy/extensions/rate_limit_descriptors/expr/v3/BUILD

@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+        "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto",
+    ],
+)

api/envoy/extensions/rate_limit_descriptors/expr/v3/expr.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package envoy.extensions.rate_limit_descriptors.expr.v3;
+
+import "google/api/expr/v1alpha1/syntax.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.rate_limit_descriptors.expr.v3";
+option java_outer_classname = "ExprProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Rate limit descriptor expression]
+// [#extension: envoy.rate_limit_descriptors.expr]
+
+// The following descriptor entry is appended with a value computed
+// from a symbolic Common Expression Language expression.
+// See :ref:`attributes <arch_overview_attributes>` for the set of
+// available attributes.
+//
+// .. code-block:: cpp
+//
+//   ("<descriptor_key>", "<expression_value>")
+message Descriptor {
+  // The key to use in the descriptor entry.
+  string descriptor_key = 1 [(validate.rules).string = {min_len: 1}];
+
+  // If set to true, Envoy skips the descriptor if the expression evaluates to an error.
+  // By default, the rate limit is not applied when an expression produces an error.
+  bool skip_if_error = 2;
+
+  oneof expr_specifier {
+    // Expression in a text form, e.g. "connection.requested_server_name".
+    string text = 3 [(validate.rules).string = {min_len: 1}];
+
+    // Parsed expression in AST form.
+    google.api.expr.v1alpha1.Expr parsed = 4;
+  }
+}

api/envoy/service/ext_proc/v3alpha/external_processor.proto

@@ -6,7 +6,6 @@ import "envoy/config/core/v3/base.proto";
 import "envoy/extensions/filters/http/ext_proc/v3alpha/processing_mode.proto";
 import "envoy/type/v3/http_status.proto";
 
-import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 
 import "udpa/annotations/status.proto";
@@ -289,10 +288,13 @@ message GrpcStatus {
 // Change HTTP headers or trailers by appending, replacing, or removing
 // headers.
 message HeaderMutation {
-  // Add or replace HTTP headers.
+  // Add or replace HTTP headers. Attempts to set the value of
+  // any "x-envoy" header, and attempts to set the ":method",
+  // ":authority", ":scheme", or "host" headers will be ignored.
   repeated config.core.v3.HeaderValueOption set_headers = 1;
 
-  // Remove these HTTP headers.
+  // Remove these HTTP headers. Attempts to remove system headers --
+  // any header starting with ":", plus "host" -- will be ignored.
   repeated string remove_headers = 2;
 }
 

api/versioning/BUILD

@@ -122,6 +122,7 @@ proto_library(
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg",
         "//envoy/extensions/network/socket_interface/v3:pkg",
+        "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg",
         "//envoy/extensions/retry/host/omit_host_metadata/v3:pkg",
         "//envoy/extensions/retry/priority/previous_priorities/v3:pkg",
         "//envoy/extensions/stat_sinks/wasm/v3:pkg",

bazel/foreign_cc/BUILD

@@ -126,6 +126,8 @@ envoy_cmake_external(
         "CURL_CA_PATH": "none",
         "CMAKE_USE_OPENSSL": "off",
         "OPENSSL_ROOT_DIR": "$EXT_BUILD_DEPS",
+        # Avoid libidn2
+        "USE_LIBIDN2": "off",
         # NGHTTP2.
         "USE_NGHTTP2": "on",
         "NGHTTP2_LIBRARY": "$EXT_BUILD_DEPS/nghttp2",

bazel/foreign_cc/curl.patch

@@ -1,14 +1,61 @@
+#commit 743021d6c7abba91c47e5be8035ff0497f2b78bd
+#Author: Jay Satiro <raysatiro@yahoo.com>
+#Date:   Tue Dec 22 15:31:03 2020 -0500
+#
+#    cmake: Add an option to disable libidn2
+#    
+#    New option USE_LIBIDN2 defaults to ON for libidn2 detection. Prior to
+#    this change libidn2 detection could not be turned off in cmake builds.
+#    
+#    Reported-by: William A Rowe Jr
+#    
+#    Fixes https://github.com/curl/curl/issues/6361
+#    Closes #xxxx
+#
+#commit e952764adbb89f37dbf227a48a55cc57c60b537d
+#Author: William A Rowe Jr <wrowe@vmware.com>
+#Date:   Wed Oct 7 14:32:49 2020 -0500
+#
+#    Correct fragile windows assumptions
+#    
+#    - Locking CMake to 3.16 breaks all features and corrections applied to
+#      CMake 3.17 and later, including the correction of the poorly designed
+#      and now abandoned Windows CRT election policy CMP0091 (see final para
+#      of the policy description here:
+#      https://cmake.org/cmake/help/v3.18/policy/CMP0091.html). Locking to
+#      rev 3.16 from ensures a more difficult transition to CMake-current
+#    
+#    - Windows curl builds previously only adjusted the Release and Debug
+#      builds, and combined with CMP0091 to break other flavors. Update any
+#      /MD* flags with /MT* present in the base and four alternate build
+#      flavors, without introducing conflicting flag values or introducing
+#      a CRT election where one is not present
+#    
+#    - Windows clang-cl builds of curl static libs are broken when using
+#      link-lld.exe because curl appended the dynamic run time flags to the
+#      static library lib.exe options. While these were ignored/no-op on
+#      Windows link.exe, they cause link-lld from LLVM/clang-cl compile
+#      toolchain to fail to parse the library command.
+#    
+#    Summary exists in this bazel-specific bug report;
+#    https://github.com/bazelbuild/rules_foreign_cc/issues/426
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index ec1cfa782..0c5a72f00 100644
+index 6a1a6fe8e..777ee122f 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -42,0 +42,5 @@
-+# revert CMake bug triggered by curl's defined max CMake policy version, see https://gitlab.kitware.com/cmake/cmake/-/issues/21288
+@@ -40,4 +40,9 @@
+ cmake_minimum_required(VERSION 3.2...3.16 FATAL_ERROR)
+
++# Revert CMake bug triggered by curl's defined max CMake policy version, see https://gitlab.kitware.com/cmake/cmake/-/issues/21288
 +if(POLICY CMP0091)
 +  cmake_policy(SET CMP0091 OLD)
 +endif()
 +
-@@ -249,3 +254,6 @@
+ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
+ include(Utilities)
+@@ -248,7 +253,10 @@ endif()
+
+ if(CURL_STATIC_CRT)
 -  set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 -  set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /MT")
 -  set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /MTd")
@@ -18,3 +65,16 @@ index ec1cfa782..0c5a72f00 100644
 +      string(REGEX REPLACE "/MD" "/MT" ${flags_var} "${${flags_var}}")
 +    endif()
 +  endforeach()
+ endif()
+
+@@ -619,5 +627,9 @@ endif()
+
+ # Check for idn
+-check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
++option(USE_LIBIDN2 "Use libidn2 for IDN support" ON)
++set(HAVE_LIBIDN2 OFF)
++if(USE_LIBIDN2)
++  check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
++endif()
+
+ # Check for symbol dlopen (same as HAVE_LIBDL)

bazel/repositories.bzl

@@ -633,12 +633,10 @@ def _com_github_curl():
         build_file_content = BUILD_ALL_CONTENT + """
 cc_library(name = "curl", visibility = ["//visibility:public"], deps = ["@envoy//bazel/foreign_cc:curl"])
 """,
-        # Patch curl 7.72.0 due to CMake's problematic implementation of policy `CMP0091`
-        # introduced in CMake 3.15 and then deprecated in CMake 3.18. Curl forcing the CMake
-        # ruleset to 3.16 breaks the Envoy windows fastbuild target.
-        # Also cure a fatal assumption creating a static library using LLVM `lld-link.exe`
-        # adding dynamic link flags, which breaks the Envoy clang-cl library archive step.
-        # Upstream patch submitted: https://github.com/curl/curl/pull/6050
+        # Patch curl 7.74.0 due to CMake's problematic implementation of policy `CMP0091`
+        # and introduction of libidn2 dependency which is inconsistently available and must
+        # not be a dynamic dependency on linux.
+        # Upstream patches submitted: https://github.com/curl/curl/pull/6050 & 6362
         # TODO(https://github.com/envoyproxy/envoy/issues/11816): This patch is obsoleted
         # by elimination of the curl dependency.
         patches = ["@envoy//bazel/foreign_cc:curl.patch"],

bazel/repository_locations.bzl

@@ -49,11 +49,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "envoy-build-tools",
         project_desc = "Common build tools shared by the Envoy/UDPA ecosystem",
         project_url = "https://github.com/envoyproxy/envoy-build-tools",
-        version = "3ff9995a5dd3d0e703e602ca3ebd9366de2b5752",
-        sha256 = "a4c74ce9a62b1c907329d248d4c225abfae8646bc77db8d72de65726632d3571",
+        version = "f2a7f9ba09660beacfebcd37fc977480ec9a8f50",
+        sha256 = "ca9975f6d5370843167b9646028ca7a0b546f8821f217c9d9d2e033a94a35f31",
         strip_prefix = "envoy-build-tools-{version}",
         urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"],
-        release_date = "2020-12-21",
+        release_date = "2021-01-04",
         use_category = ["build"],
     ),
     boringssl = dict(
@@ -713,6 +713,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.rbac",
             "envoy.filters.http.wasm",
             "envoy.filters.network.rbac",
@@ -734,6 +735,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.rbac",
             "envoy.filters.http.wasm",
             "envoy.filters.network.rbac",
@@ -912,6 +914,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.wasm",
             "envoy.filters.network.wasm",
             "envoy.stat_sinks.wasm",
@@ -931,6 +934,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         extensions = [
             "envoy.access_loggers.wasm",
             "envoy.bootstrap.wasm",
+            "envoy.rate_limit_descriptors.expr",
             "envoy.filters.http.wasm",
             "envoy.filters.network.wasm",
             "envoy.stat_sinks.wasm",

docs/root/api-v3/config/config.rst

@@ -24,3 +24,4 @@ Extensions
   upstream/upstream
   wasm/wasm
   watchdog/watchdog
+  descriptors/descriptors

docs/root/api-v3/config/descriptors/descriptors.rst

@@ -0,0 +1,8 @@
+Rate limit descriptors
+======================
+
+.. toctree::
+  :glob:
+  :maxdepth: 2
+
+  ../../extensions/rate_limit_descriptors/expr/v3/*

docs/root/configuration/http/http_filters/ext_proc_filter.rst

@@ -17,3 +17,24 @@ messages, and the server must reply with
 :ref:`ProcessingResponse <envoy_v3_api_msg_service.ext_proc.v3alpha.ProcessingResponse>`.
 
 This filter is a work in progress. In its current state, it actually does nothing.
+
+Statistics
+----------
+This filter outputs statistics in the
+*http.<stat_prefix>.ext_proc.* namespace. The :ref:`stat prefix
+<envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.stat_prefix>`
+comes from the owning HTTP connection manager.
+
+The following statistics are supported:
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: auto
+
+  streams_started, Counter, The number of gRPC streams that have been started to send to the external processing service
+  streams_msgs_sent, Counter, The number of messages sent on those streams
+  streams_msgs_received, Counter, The number of messages received on those streams
+  spurious_msgs_received, Counter, The number of unexpected messages received that violated the protocol
+  streams_closed, Counter, The number of streams successfully closed on either end
+  streams_failed, Counter, The number of times a stream produced a gRPC error
+  failure_mode_allowed, Counter, The number of times an error was ignored due to configuration