Merge branch 'master' into mpuncel/http2-hc-goaway

* master: (22 commits)
  delay health checks until transport socket secrets are ready. (envoyproxy#13516)
  test, oauth2: Make sure config test runs field validation (envoyproxy#13496)
  [http] swap codec implementations to default new (envoyproxy#13579)
  wasm: update proxy-wasm-cpp-host (envoyproxy#13606)
  postgres: do not copy and linearize received data when it is not going to be used (envoyproxy#13393)
  configs: Update configs v2 -> v3 (envoyproxy#13562)
  http2: Remove RELEASE_ASSERTs in sendPendingFrames() error handling (envoyproxy#13546)
  dependencies: track untracked implied dependencies, wrapup dashboard. (envoyproxy#13571)
  listener: add match all filter chain (envoyproxy#13449)
  fix mistakes in docstrings (envoyproxy#13603)
  ratelimit: add route entry metadata to ratelimit actions (envoyproxy#13269)
  cluster manager: avoid immediate activation for dynamic inserted cluster when initialize (envoyproxy#12783)
  ext_authz: Avoid calling check multiple times (envoyproxy#13288)
  docs: Unexclude remaining configs from validation (envoyproxy#13534)
  build: update rules_rust to allow Rustc in RBE (envoyproxy#13595)
  docs: Update sphinxext.rediraffe (envoyproxy#13589)
  Deprecate moonjit support on Windows before beta (envoyproxy#13541)
  dependencies: bump LuaJIT to 2.1 branch HEAD @ e9af1ab. (envoyproxy#13474)
  docs: add TLS stats to cluster stats doc (envoyproxy#13561)
  ci: stop building alpine-debug images in favor of ubuntu-based debug image (envoyproxy#13598)
  ...

Signed-off-by: Michael Puncel <mpuncel@squareup.com>

.azure-pipelines/pipelines.yml

@@ -250,6 +250,10 @@ stages:
         steps:
           - bash: .azure-pipelines/cleanup.sh
             displayName: "Removing tools from agent"
+          - bash: |
+              echo "disk space at beginning of build:"
+              df -h
+            displayName: "Check disk space at beginning"
           - task: DownloadBuildArtifacts@0
             inputs:
               buildType: current
@@ -275,6 +279,11 @@ stages:
               AZP_SHA1: $(Build.SourceVersion)
               DOCKERHUB_USERNAME: $(DockerUsername)
               DOCKERHUB_PASSWORD: $(DockerPassword)
+          - bash: |
+              echo "disk space at end of build:"
+              df -h
+            displayName: "Check disk space at end"
+            condition: always()
           - task: PublishBuildArtifacts@1
             inputs:
               pathtoPublish: "$(Build.StagingDirectory)/build_images"

.bazelrc

@@ -193,8 +193,6 @@ build:remote --spawn_strategy=remote,sandboxed,local
 build:remote --strategy=Javac=remote,sandboxed,local
 build:remote --strategy=Closure=remote,sandboxed,local
 build:remote --strategy=Genrule=remote,sandboxed,local
-# rules_rust is not remote runnable (yet)
-build:remote --strategy=Rustc=sandboxed,local
 build:remote --remote_timeout=7200
 build:remote --auth_enabled=true
 build:remote --remote_download_toplevel

CONTRIBUTING.md

@@ -69,7 +69,7 @@ versioning guidelines:
   cause a configuration load failure, unless the feature in question is
   explicitly overridden in
   [runtime](https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features)
-  config ([example](configs/using_deprecated_config.v2.yaml)). Finally, following the deprecation
+  config ([example](configs/using_deprecated_config.yaml)). Finally, following the deprecation
   of the API major version where the field was first
   marked deprecated, the entire implementation code will be removed from the Envoy implementation.
 * This policy means that organizations deploying master should have some time to get ready for

OWNERS.md

@@ -68,3 +68,5 @@ matter expert reviews. Feel free to loop them in as needed.
   * Redis, Python, configuration/operational questions.
 * Yuchen Dai ([lambdai](https://github.com/lambdai)) (lambdai@google.com)
   * v2 xDS, listeners, filter chain discovery service.
+* Michael Payne ([moderation](https://github.com/moderation)) (m@m17e.org)
+  * External dependencies, Envoy's supply chain and documentation.

api/bazel/external_deps.bzl

@@ -2,19 +2,38 @@ load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locati
 
 # Envoy dependencies may be annotated with the following attributes:
 DEPENDENCY_ANNOTATIONS = [
-    # List of the categories describing how the dependency is being used. This attribute is used
-    # for automatic tracking of security posture of Envoy's dependencies.
-    # Possible values are documented in the USE_CATEGORIES list below.
-    # This attribute is mandatory for each dependecy.
-    "use_category",
-
     # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID
     # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See
     # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements
     # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned.
     # This attribute is optional for components with use categories listed in the
     # USE_CATEGORIES_WITH_CPE_OPTIONAL
     "cpe",
+
+    # A list of extensions when 'use_category' contains 'dataplane_ext' or 'observability_ext'.
+    "extensions",
+
+    # Additional dependencies loaded transitively via this dependency that are not tracked in
+    # Envoy (see the external dependency at the given version for information).
+    "implied_untracked_deps",
+
+    # When the dependency was last updated in Envoy.
+    "last_updated",
+
+    # Project metadata.
+    "project_desc",
+    "project_name",
+    "project_url",
+
+    # List of the categories describing how the dependency is being used. This attribute is used
+    # for automatic tracking of security posture of Envoy's dependencies.
+    # Possible values are documented in the USE_CATEGORIES list below.
+    # This attribute is mandatory for each dependecy.
+    "use_category",
+
+    # The dependency version. This may be either a tagged release (preferred)
+    # or git SHA (as an exception when no release tagged version is suitable).
+    "version",
 ]
 
 # NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed
@@ -38,6 +57,10 @@ USE_CATEGORIES = [
     "other",
     # This dependency is used only in tests.
     "test_only",
+    # Documentation generation
+    "docs",
+    # Developer tools (not used in build or docs)
+    "devtools",
 ]
 
 # Components with these use categories are not required to specify the 'cpe'
@@ -62,47 +85,43 @@ def load_repository_locations(repository_locations_spec):
 
         if "project_name" not in location:
             _fail_missing_attribute("project_name", key)
-        mutable_location.pop("project_name")
 
         if "project_desc" not in location:
             _fail_missing_attribute("project_desc", key)
-        mutable_location.pop("project_desc")
 
         if "project_url" not in location:
             _fail_missing_attribute("project_url", key)
-        project_url = mutable_location.pop("project_url")
+        project_url = location["project_url"]
         if not project_url.startswith("https://") and not project_url.startswith("http://"):
             fail("project_url must start with https:// or http://: " + project_url)
 
         if "version" not in location:
             _fail_missing_attribute("version", key)
-        mutable_location.pop("version")
 
         if "use_category" not in location:
             _fail_missing_attribute("use_category", key)
-        use_category = mutable_location.pop("use_category")
+        use_category = location["use_category"]
 
         if "dataplane_ext" in use_category or "observability_ext" in use_category:
             if "extensions" not in location:
                 _fail_missing_attribute("extensions", key)
-            mutable_location.pop("extensions")
 
         if "last_updated" not in location:
             _fail_missing_attribute("last_updated", key)
-        last_updated = mutable_location.pop("last_updated")
+        last_updated = location["last_updated"]
 
         # Starlark doesn't have regexes.
         if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
             fail("last_updated must match YYYY-DD-MM: " + last_updated)
 
         if "cpe" in location:
-            cpe = mutable_location.pop("cpe")
+            cpe = location["cpe"]
 
             # Starlark doesn't have regexes.
             cpe_components = len(cpe.split(":"))
 
-            # We allow cpe:2.3:a:foo:* and cpe:2.3.:a:foo:bar:* only.
-            cpe_components_valid = cpe_components in [5, 6]
+            # We allow cpe:2.3:a:foo:*:* and cpe:2.3.:a:foo:bar:* only.
+            cpe_components_valid = (cpe_components == 6)
             cpe_matches = (cpe == "N/A" or (cpe.startswith("cpe:2.3:a:") and cpe.endswith(":*") and cpe_components_valid))
             if not cpe_matches:
                 fail("CPE must match cpe:2.3:a:<facet>:<facet>:*: " + cpe)
@@ -113,4 +132,9 @@ def load_repository_locations(repository_locations_spec):
             if category not in USE_CATEGORIES:
                 fail("Unknown use_category value '" + category + "' for dependecy " + key)
 
+        # Remove any extra annotations that we add, so that we don't confuse http_archive etc.
+        for annotation in DEPENDENCY_ANNOTATIONS:
+            if annotation in mutable_location:
+                mutable_location.pop(annotation)
+
     return locations

api/envoy/config/listener/v3/listener.proto

@@ -36,7 +36,7 @@ message ListenerCollection {
   repeated udpa.core.v1.CollectionEntry entries = 1;
 }
 
-// [#next-free-field: 25]
+// [#next-free-field: 26]
 message Listener {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener";
 
@@ -116,6 +116,10 @@ message Listener {
   // :ref:`FAQ entry <faq_how_to_setup_sni>`.
   repeated FilterChain filter_chains = 3;
 
+  // The default filter chain if none of the filter chain matches. If no default filter chain is supplied,
+  // the connection will be closed. The filter chain match is ignored in this field.
+  FilterChain default_filter_chain = 25;
+
   // Soft limit on size of the listener’s new connection read and write buffers.
   // If unspecified, an implementation defined default is applied (1MiB).
   google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5

api/envoy/config/listener/v3/listener_components.proto

@@ -65,6 +65,18 @@ message Filter {
 // ``www.example.com``, then ``*.example.com``, then ``*.com``, then any filter
 // chain without ``server_names`` requirements).
 //
+// A different way to reason about the filter chain matches:
+// Suppose there exists N filter chains. Prune the filter chain set using the above 8 steps.
+// In each step, filter chains which most specifically matches the attributes continue to the next step.
+// The listener guarantees at most 1 filter chain is left after all of the steps.
+//
+// Example:
+//
+// For destination port, filter chains specifying the destination port of incoming traffic are the
+// most specific match. If none of the filter chains specifies the exact destination port, the filter
+// chains which do not specify ports are the most specific match. Filter chains specifying the
+// wrong port can never be the most specific match.
+//
 // [#comment: Implemented rules are kept in the preference order, with deprecated fields
 // listed at the end, because that's how we want to list them in the docs.
 //

api/envoy/config/route/v3/route_components.proto

@@ -1510,7 +1510,7 @@ message VirtualCluster {
 message RateLimit {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RateLimit";
 
-  // [#next-free-field: 8]
+  // [#next-free-field: 9]
   message Action {
     option (udpa.annotations.versioning).previous_message_type =
         "envoy.api.v2.route.RateLimit.Action";
@@ -1627,11 +1627,15 @@ message RateLimit {
       repeated HeaderMatcher headers = 3 [(validate.rules).repeated = {min_items: 1}];
     }
 
-    // The following descriptor entry is appended when the dynamic metadata contains a key value:
+    // The following descriptor entry is appended when the
+    // :ref:`dynamic metadata <well_known_dynamic_metadata>` contains a key value:
     //
     // .. code-block:: cpp
     //
-    //   ("<descriptor_key>", "<value_queried_from_metadata>")
+    //   ("<descriptor_key>", "<value_queried_from_dynamic_metadata>")
+    //
+    // .. attention::
+    //   This action has been deprecated in favor of the :ref:`metadata <envoy_api_msg_config.route.v3.RateLimit.Action.MetaData>` action
     message DynamicMetaData {
       // The key to use in the descriptor entry.
       string descriptor_key = 1 [(validate.rules).string = {min_len: 1}];
@@ -1645,6 +1649,35 @@ message RateLimit {
       string default_value = 3;
     }
 
+    // The following descriptor entry is appended when the metadata contains a key value:
+    //
+    // .. code-block:: cpp
+    //
+    //   ("<descriptor_key>", "<value_queried_from_metadata>")
+    message MetaData {
+      enum Source {
+        // Query :ref:`dynamic metadata <well_known_dynamic_metadata>`
+        DYNAMIC = 0;
+
+        // Query :ref:`route entry metadata <envoy_api_field_config.route.v3.Route.metadata>`
+        ROUTE_ENTRY = 1;
+      }
+
+      // The key to use in the descriptor entry.
+      string descriptor_key = 1 [(validate.rules).string = {min_len: 1}];
+
+      // Metadata struct that defines the key and path to retrieve the string value. A match will
+      // only happen if the value in the metadata is of type string.
+      type.metadata.v3.MetadataKey metadata_key = 2 [(validate.rules).message = {required: true}];
+
+      // An optional value to use if *metadata_key* is empty. If not set and
+      // no value is present under the metadata_key then no descriptor is generated.
+      string default_value = 3;
+
+      // Source of metadata
+      Source source = 4 [(validate.rules).enum = {defined_only: true}];
+    }
+
     oneof action_specifier {
       option (validate.required) = true;
 
@@ -1667,7 +1700,14 @@ message RateLimit {
       HeaderValueMatch header_value_match = 6;
 
       // Rate limit on dynamic metadata.
-      DynamicMetaData dynamic_metadata = 7;
+      //
+      // .. attention::
+      //   This field has been deprecated in favor of the :ref:`metadata <envoy_api_field_config.route.v3.RateLimit.Action.metadata>` field
+      DynamicMetaData dynamic_metadata = 7
+          [deprecated = true, (envoy.annotations.disallowed_by_default) = true];
+
+      // Rate limit on metadata.
+      MetaData metadata = 8;
     }
   }