Merge branch 'master' into mpuncel/http2-hc-goaway

* master: (22 commits) ci: various improvements (envoyproxy#13660) dns: fix defunct fd bug in apple resolver (envoyproxy#13641) build: support ppc64le with wasm (envoyproxy#13657) [fuzz] Added random load balancer fuzz (envoyproxy#13400) dependencies: compute and check release dates via GitHub API. (envoyproxy#13582) mac ci: try ignoring update failure (envoyproxy#13658) watchdog: Optimize WatchdogImpl::touch in preparation to more frequent petting of the watchdog. (envoyproxy#13103) typos: fix a couple 'enovy' mispellings (envoyproxy#13645) lua: Expose stream info downstreamLocalAddress and downstreamDirectRemoteAddress for Lua filter (envoyproxy#13536) tap: fix upstream streamed transport socket taps (envoyproxy#13638) Revert "delay health checks until transport socket secrets are ready. (envoyproxy#13516)" (envoyproxy#13639) Watchdog: use abort action as a default if killing is enabled. (envoyproxy#13523) [fuzz] Fixed divide by zero bug (envoyproxy#13545) wasm: flip the meaning of the "repository" in envoy_wasm_cc_binary(). (envoyproxy#13621) fix: record recovered local address (envoyproxy#13581) docs: fix incorrect compressor filter doc (envoyproxy#13611) docs: clean up docs for azp migration (envoyproxy#13558) wasm: fix building Wasm example. (envoyproxy#13619) test: Refactor flood tests into a separate test file (envoyproxy#13556) wasm: re-enable tests with precompiled modules. (envoyproxy#13583) ... Signed-off-by: Michael Puncel <mpuncel@squareup.com>
mpuncel · Oct 21, 2020 · cd06c28 · cd06c28
2 parents 01fa365 + f95f539
commit cd06c28
Show file tree

Hide file tree

Showing 163 changed files with 3,336 additions and 2,133 deletions.
diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml
@@ -70,26 +70,40 @@ stages:
             inputs:
               pathtoPublish: "$(Build.SourcesDirectory)/generated/docs"
               artifactName: docs
-            condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
           - task: InstallSSHKey@0
             inputs:
               hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
               sshPublicKey: "$(DocsPublicKey)"
               sshPassphrase: "$(SshDeployKeyPassphrase)"
               sshKeySecureFile: "$(DocsPrivateKey)"
-            condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
+            condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
 
           - script: docs/publish.sh
             displayName: "Publish to GitHub"
             workingDirectory: $(Build.SourcesDirectory)
             env:
               AZP_BRANCH: $(Build.SourceBranch)
               AZP_SHA1: $(Build.SourceVersion)
-            condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
+            condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
+
+      - job: dependencies
+        dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+        pool:
+          vmImage: "ubuntu-18.04"
+        steps:
+          - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps'
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)
+            displayName: "Verify dependency information"
 
   - stage: sync
-    condition: and(succeeded(), eq(variables['PostSubmit'], true))
+    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
     dependsOn: []
     jobs:
       - job: filter_example
@@ -130,6 +144,8 @@ stages:
 
       - job: go_control_plane
         dependsOn: []
+        pool:
+          vmImage: "ubuntu-18.04"
         steps:
           - task: InstallSSHKey@0
             inputs:
@@ -152,12 +168,11 @@ stages:
 
   - stage: linux_x64
     dependsOn: ["precheck"]
-    # For master builds, continue even if precheck fails
-    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
+    # For post-submit builds, continue even if precheck fails
+    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
     jobs:
       - job: release
-        # For master builds, continue even if format fails
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "ubuntu-18.04"
         steps:
@@ -167,11 +182,11 @@ stages:
 
   - stage: linux_arm64
     dependsOn: ["precheck"]
-    # For master builds, continue even if precheck fails
-    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
+    # For post-submit builds, continue even if precheck fails
+    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
     jobs:
       - job: release
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool: "arm-large"
         steps:
           - template: bazel.yml
@@ -203,7 +218,7 @@ stages:
               CI_TARGET: "bazel.tsan"
             compile_time_options:
               CI_TARGET: "bazel.compile_time_options"
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "ubuntu-18.04"
         steps:
@@ -214,7 +229,7 @@ stages:
       - job: coverage
         displayName: "linux_x64"
         dependsOn: []
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool: "x64-large"
         strategy:
           maxParallel: 2
@@ -313,7 +328,7 @@ stages:
     dependsOn: ["precheck"]
     jobs:
       - job: test
-        timeoutInMinutes: 360
+        timeoutInMinutes: 180
         pool:
           vmImage: "macos-latest"
         steps:
@@ -347,7 +362,7 @@ stages:
     dependsOn: ["precheck"]
     jobs:
       - job: release
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "windows-latest"
         steps:
@@ -368,7 +383,7 @@ stages:
 
       - job: docker
         dependsOn: ["release"]
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "windows-latest"
         steps:

diff --git a/.bazelrc b/.bazelrc
@@ -3,10 +3,10 @@
 # Bazel doesn't need more than 200MB of memory for local build based on memory profiling:
 # https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling
 # The default JVM max heapsize is 1/4 of physical memory up to 32GB which could be large
-# enough to consume all memory constrained by cgroup in large host, which is the case in CircleCI.
+# enough to consume all memory constrained by cgroup in large host.
 # Limiting JVM heapsize here to let it do GC more when approaching the limit to
 # leave room for compiler/linker.
-# The number 2G is choosed heuristically to both support in CircleCI and large enough for RBE.
+# The number 2G is chosen heuristically to both support large VM and small VM with RBE.
 # Startup options cannot be selected via config.
 startup --host_jvm_args=-Xmx2g
 

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -142,7 +142,6 @@ extensions/filters/common/original_src @snowp @klarose
 /*/extensions/filters/http/decompressor @rojkov @dio
 # Watchdog Extensions
 /*/extensions/watchdog/profile_action @kbaichoo @antoniovicente
-/*/extensions/watchdog/abort_action @kbaichoo @antoniovicente
 # Core upstream code
 extensions/upstreams/http @alyssawilk @snowp @mattklein123
 extensions/upstreams/http/http @alyssawilk @snowp @mattklein123

diff --git a/DEPENDENCY_POLICY.md b/DEPENDENCY_POLICY.md
@@ -45,8 +45,11 @@ Dependency declarations must:
   version is being used.
 * Provide accurate entries for `use_category`. Please think carefully about whether there are data
   or control plane implications of the dependency.
-* Reflect the date (YYYY-MM-DD) at which they were last updated in the `last_updated` field. This
-  date is preferably the date at which the PR is created.
+* Reflect the UTC date (YYYY-MM-DD format) for the dependency release. This is when
+  the dependency was updated in its repository. For dependencies that have
+  releases, this is the date of the release. For dependencies without releases
+  or for scenarios where we temporarily need to use a commit, this date should
+  be the date of the commit in UTC.
 * CPEs are compulsory for all dependencies that are not purely build/test.
   [CPEs](https://en.wikipedia.org/wiki/Common_Platform_Enumeration) provide metadata that allow us
   to correlate with related CVEs in dashboards and other tooling, and also provide a machine

diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -1,6 +1,7 @@
 # Developer documentation
 
-Envoy is built using the Bazel build system. CircleCI builds, tests, and runs coverage against all pull requests and the master branch.
+Envoy is built using the Bazel build system. Our CI on Azure Pipelines builds, tests, and runs coverage against
+all pull requests and the master branch.
 
 To get started building Envoy locally, see the [Bazel quick start](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#quick-start-bazel-build-for-developers).
 To run tests, there are Bazel [targets](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#testing-envoy-with-bazel) for Google Test.

diff --git a/README.md b/README.md
@@ -10,7 +10,6 @@ involved and how Envoy plays a role, read the CNCF
 
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1266/badge)](https://bestpractices.coreinfrastructure.org/projects/1266)
 [![Azure Pipelines](https://dev.azure.com/cncf/envoy/_apis/build/status/11?branchName=master)](https://dev.azure.com/cncf/envoy/_build/latest?definitionId=11&branchName=master)
-[![CircleCI](https://circleci.com/gh/envoyproxy/envoy/tree/master.svg?style=shield)](https://circleci.com/gh/envoyproxy/envoy/tree/master)
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/envoy.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:envoy)
 [![Jenkins](https://powerci.osuosl.org/buildStatus/icon?job=build-envoy-static-master&subject=ppc64le%20build)](https://powerci.osuosl.org/job/build-envoy-static-master/)
 

diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md
@@ -4,7 +4,8 @@ This is a high level overview of how the repository is laid out to both aid in c
 as well as to clearly specify how extensions are added to the repository. The top level directories
 are:
 
-* [.circleci/](.circleci/): Configuration for [CircleCI](https://circleci.com/gh/envoyproxy).
+* [.azure-pipelines/](.azure-pipelines/): Configuration for
+[Azure Pipelines](https://azure.microsoft.com/en-us/services/devops/pipelines/).
 * [api/](api/): Envoy data plane API.
 * [bazel/](bazel/): Configuration for Envoy's use of [Bazel](https://bazel.build/).
 * [ci/](ci/): Scripts used both during CI as well as to build Docker containers.

diff --git a/STYLE.md b/STYLE.md
@@ -1,7 +1,7 @@
 # C++ coding style
 
 * The Envoy source code is formatted using clang-format. Thus all white spaces, etc.
-  issues are taken care of automatically. The CircleCI tests will automatically check
+  issues are taken care of automatically. The Azure Pipelines will automatically check
   the code format and fail. There are make targets that can both check the format
   (check_format) as well as fix the code format for you (fix_format). Errors in
   .clang-tidy are enforced while other warnings are suggestions. Note that code and

diff --git a/api/BUILD b/api/BUILD
@@ -248,7 +248,6 @@ proto_library(
         "//envoy/extensions/upstreams/http/http/v3:pkg",
         "//envoy/extensions/upstreams/http/tcp/v3:pkg",
         "//envoy/extensions/wasm/v3:pkg",
-        "//envoy/extensions/watchdog/abort_action/v3alpha:pkg",
         "//envoy/extensions/watchdog/profile_action/v3alpha:pkg",
         "//envoy/service/accesslog/v3:pkg",
         "//envoy/service/auth/v3:pkg",
@@ -272,6 +271,7 @@ proto_library(
         "//envoy/type/metadata/v3:pkg",
         "//envoy/type/tracing/v3:pkg",
         "//envoy/type/v3:pkg",
+        "//envoy/watchdog/v3alpha:pkg",
     ],
 )
 

diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md
@@ -50,11 +50,11 @@ generated RST files are also viewable in `generated/rst`.
 
 Note also that the generated documentation can be viewed in CI:
 
-1. Open docs job in CircleCI.
-2. Navigate to "artifacts" tab.
-3. Expand files and click on `index.html`.
+1. Open docs job in Azure Pipelines.
+2. Navigate to "Upload Docs to GCS" log.
+3. Click on the link there.
 
-If you do not see an artifacts tab this is a bug in CircleCI. Try logging out and logging back in.
+If you do not see "Upload Docs to GCS" or it is failing, that means the docs are not built correctly.
 
 ### Documentation guidelines
 

diff --git a/api/bazel/external_deps.bzl b/api/bazel/external_deps.bzl
@@ -17,14 +17,18 @@ DEPENDENCY_ANNOTATIONS = [
     # Envoy (see the external dependency at the given version for information).
     "implied_untracked_deps",
 
-    # When the dependency was last updated in Envoy.
-    "last_updated",
-
     # Project metadata.
     "project_desc",
     "project_name",
     "project_url",
 
+    # Reflects the UTC date (YYYY-MM-DD format) for the dependency release. This
+    # is when the dependency was updated in its repository. For dependencies
+    # that have releases, this is the date of the release. For dependencies
+    # without releases or for scenarios where we temporarily need to use a
+    # commit, this date should be the date of the commit in UTC.
+    "release_date",
+
     # List of the categories describing how the dependency is being used. This attribute is used
     # for automatic tracking of security posture of Envoy's dependencies.
     # Possible values are documented in the USE_CATEGORIES list below.
@@ -63,8 +67,7 @@ USE_CATEGORIES = [
     "devtools",
 ]
 
-# Components with these use categories are not required to specify the 'cpe'
-# and 'last_updated' annotation.
+# Components with these use categories are not required to specify the 'cpe'.
 USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"]
 
 def _fail_missing_attribute(attr, key):
@@ -106,13 +109,13 @@ def load_repository_locations(repository_locations_spec):
             if "extensions" not in location:
                 _fail_missing_attribute("extensions", key)
 
-        if "last_updated" not in location:
-            _fail_missing_attribute("last_updated", key)
-        last_updated = location["last_updated"]
+        if "release_date" not in location:
+            _fail_missing_attribute("release_date", key)
+        release_date = location["release_date"]
 
         # Starlark doesn't have regexes.
-        if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
-            fail("last_updated must match YYYY-DD-MM: " + last_updated)
+        if len(release_date) != 10 or release_date[4] != "-" or release_date[7] != "-":
+            fail("release_date must match YYYY-DD-MM: " + release_date)
 
         if "cpe" in location:
             cpe = location["cpe"]

diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl
@@ -7,7 +7,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         version = "1.0.3",
         sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
         urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz"],
-        last_updated = "2020-08-27",
+        release_date = "2020-08-27",
         use_category = ["api"],
     ),
     com_envoyproxy_protoc_gen_validate = dict(
@@ -18,7 +18,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "e368733c9fb7f8489591ffaf269170d7658cc0cd1ee322b601512b769446d3c8",
         strip_prefix = "protoc-gen-validate-{version}",
         urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/{version}.tar.gz"],
-        last_updated = "2020-06-09",
+        release_date = "2020-06-08",
         use_category = ["api"],
     ),
     com_github_cncf_udpa = dict(
@@ -29,7 +29,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "83a7dcc316d741031f34c0409021432b74a39c4811845a177133f02f948fe2d8",
         strip_prefix = "udpa-{version}",
         urls = ["https://github.com/cncf/udpa/archive/v{version}.tar.gz"],
-        last_updated = "2020-09-23",
+        release_date = "2020-06-29",
         use_category = ["api"],
     ),
     com_github_openzipkin_zipkinapi = dict(
@@ -40,7 +40,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b",
         strip_prefix = "zipkin-api-{version}",
         urls = ["https://github.com/openzipkin/zipkin-api/archive/{version}.tar.gz"],
-        last_updated = "2020-09-23",
+        release_date = "2019-08-23",
         use_category = ["api"],
     ),
     com_google_googleapis = dict(
@@ -52,7 +52,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d1405",
         strip_prefix = "googleapis-{version}",
         urls = ["https://github.com/googleapis/googleapis/archive/{version}.tar.gz"],
-        last_updated = "2019-12-02",
+        release_date = "2019-12-02",
         use_category = ["api"],
     ),
     opencensus_proto = dict(
@@ -63,7 +63,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0",
         strip_prefix = "opencensus-proto-{version}/src",
         urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v{version}.tar.gz"],
-        last_updated = "2020-06-20",
+        release_date = "2020-07-21",
         use_category = ["api"],
     ),
     prometheus_metrics_model = dict(
@@ -74,7 +74,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "6748b42f6879ad4d045c71019d2512c94be3dd86f60965e9e31e44a3f464323e",
         strip_prefix = "client_model-{version}",
         urls = ["https://github.com/prometheus/client_model/archive/{version}.tar.gz"],
-        last_updated = "2020-06-23",
+        release_date = "2020-06-23",
         use_category = ["api"],
     ),
     rules_proto = dict(
@@ -85,7 +85,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "aa1ee19226f707d44bee44c720915199c20c84a23318bb0597ed4e5c873ccbd5",
         strip_prefix = "rules_proto-{version}",
         urls = ["https://github.com/bazelbuild/rules_proto/archive/{version}.tar.gz"],
-        last_updated = "2020-08-17",
+        release_date = "2020-08-17",
         use_category = ["api"],
     ),
 )
diff --git a/api/envoy/config/common/tap/v2alpha/BUILD b/api/envoy/config/common/tap/v2alpha/BUILD
@@ -6,7 +6,6 @@ licenses(["notice"])  # Apache 2
 
 api_proto_package(
     deps = [
-        "//envoy/api/v2/core:pkg",
         "//envoy/service/tap/v2alpha:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
     ],

diff --git a/api/envoy/config/common/tap/v2alpha/common.proto b/api/envoy/config/common/tap/v2alpha/common.proto
@@ -2,7 +2,6 @@ syntax = "proto3";
 
 package envoy.config.common.tap.v2alpha;
 
-import "envoy/api/v2/core/config_source.proto";
 import "envoy/service/tap/v2alpha/common.proto";
 
 import "udpa/annotations/migrate.proto";
@@ -19,15 +18,6 @@ option (udpa.annotations.file_status).package_version_status = FROZEN;
 
 // Common configuration for all tap extensions.
 message CommonExtensionConfig {
-  // [#not-implemented-hide:]
-  message TapDSConfig {
-    // Configuration for the source of TapDS updates for this Cluster.
-    api.v2.core.ConfigSource config_source = 1 [(validate.rules).message = {required: true}];
-
-    // Tap config to request from XDS server.
-    string name = 2 [(validate.rules).string = {min_bytes: 1}];
-  }
-
   oneof config_type {
     option (validate.required) = true;
 
@@ -37,9 +27,6 @@ message CommonExtensionConfig {
     // If specified, the tap filter will be configured via a static configuration that cannot be
     // changed.
     service.tap.v2alpha.TapConfig static_config = 2;
-
-    // [#not-implemented-hide:] Configuration to use for TapDS updates for the filter.
-    TapDSConfig tapds_config = 3;
   }
 }