merge feature/hcp-telemetry

.changelog/16916.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+hcp: Add support for linking existing Consul clusters to HCP management plane.
+```

.changelog/17066.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+command: Allow creating ACL Token TTL with greater than 24 hours with the -expires-ttl flag.
+```

.changelog/17086.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+command: Adds ACL enabled to status output on agent startup.
+```

.changelog/17115.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+gateway: Change status condition reason for invalid certificate on a listener from "Accepted" to "ResolvedRefs".
+```

.changelog/17138.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+ca: automatically set up Vault's auto-tidy setting for tidy_expired_issuers when using Vault as a CA provider.
+```
+

.changelog/17171.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: add a configurable maximimum age (default: 7 days) to prevent servers re-joining a cluster with stale data 
+```

.changelog/17179.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: ensure that merged central configs of peered upstreams for partitioned downstreams work
+```

.changelog/17183.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+* cli: Add `-filter` option to `consul config list` for filtering config entries.
+```
+```release-note:improvement
+* api: Support filtering for config entries.
+```
+

.changelog/17185.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+xds: Fix possible panic that can when generating clusters before the root certificates have been fetched.
+```

.changelog/17235.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: Fix issue where peer streams could incorrectly deregister services in various scenarios.
+```

.changelog/17236.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot`
+```

.changelog/17240.txt

@@ -0,0 +1,12 @@
+```release-note:security
+Upgrade to use Go 1.20.4.
+This resolves vulnerabilities [CVE-2023-24537](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`go/scanner`),
+[CVE-2023-24538](https://github.com/advisories/GHSA-v4m2-x4rp-hv22)(`html/template`),
+[CVE-2023-24534](https://github.com/advisories/GHSA-8v5j-pwr7-w5f8)(`net/textproto`) and
+[CVE-2023-24536](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`mime/multipart`).
+Also, `golang.org/x/net` has been updated to v0.7.0 to resolve CVEs [CVE-2022-41721
+](https://github.com/advisories/GHSA-fxg5-wq6x-vr4w
+), [CVE-2022-27664](https://github.com/advisories/GHSA-69cg-p879-7622) and [CVE-2022-41723
+](https://github.com/advisories/GHSA-vvpx-j8f3-3w6h
+.)
+```

.changelog/17241.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+connect: Fix multiple inefficient behaviors when querying service health.
+```

.changelog/17270.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+grpc: ensure grpc resolver correctly uses lan/wan addresses on servers
+```

.changelog/17317.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+connect: fix a bug with Envoy potentially starting with incomplete configuration by not waiting enough for initial xDS configuration.
+```

.changelog/17327.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+ xds: rename envoy_hcp_metrics_bind_socket_dir to envoy_telemetry_collector_bind_socket_dir to remove HCP naming references.
+ ```

.github/workflows/backport-assistant.yml

@@ -19,7 +19,7 @@ jobs:
   backport:
     if: github.event.pull_request.merged
     runs-on: ubuntu-latest
-    container: hashicorpdev/backport-assistant:0.3.0
+    container: hashicorpdev/backport-assistant:0.3.3
     steps:
       - name: Run Backport Assistant for release branches
         run: |
@@ -28,3 +28,16 @@ jobs:
           BACKPORT_LABEL_REGEXP: "backport/(?P<target>\\d+\\.\\d+)"
           BACKPORT_TARGET_TEMPLATE: "release/{{.target}}.x"
           GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+  handle-failure:
+    needs:
+      - backport
+    if: always() && needs.backport.result == 'failure'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Comment on PR
+        run: |
+          github_message="Backport failed @${{ github.event.sender.login }}. Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl -s -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \
+            -X POST \
+            -d "{ \"body\": \"${github_message}\"}" \
+            "https://api.github.com/repos/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}/comments"

.github/workflows/build.yml

@@ -82,15 +82,15 @@ jobs:
     strategy:
       matrix:
         include:
-          - {go: "1.20.1", goos: "linux", goarch: "386"}
-          - {go: "1.20.1", goos: "linux", goarch: "amd64"}
-          - {go: "1.20.1", goos: "linux", goarch: "arm"}
-          - {go: "1.20.1", goos: "linux", goarch: "arm64"}
-          - {go: "1.20.1", goos: "freebsd", goarch: "386"}
-          - {go: "1.20.1", goos: "freebsd", goarch: "amd64"}
-          - {go: "1.20.1", goos: "windows", goarch: "386"}
-          - {go: "1.20.1", goos: "windows", goarch: "amd64"}
-          - {go: "1.20.1", goos: "solaris", goarch: "amd64"}
+          - {go: "1.20.4", goos: "linux", goarch: "386"}
+          - {go: "1.20.4", goos: "linux", goarch: "amd64"}
+          - {go: "1.20.4", goos: "linux", goarch: "arm"}
+          - {go: "1.20.4", goos: "linux", goarch: "arm64"}
+          - {go: "1.20.4", goos: "freebsd", goarch: "386"}
+          - {go: "1.20.4", goos: "freebsd", goarch: "amd64"}
+          - {go: "1.20.4", goos: "windows", goarch: "386"}
+          - {go: "1.20.4", goos: "windows", goarch: "amd64"}
+          - {go: "1.20.4", goos: "solaris", goarch: "amd64"}
       fail-fast: true
 
     name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build
@@ -179,7 +179,7 @@ jobs:
       matrix:
         goos: [ darwin ]
         goarch: [ "amd64", "arm64" ]
-        go: [ "1.20.1" ]
+        go: [ "1.20.4" ]
       fail-fast: true
 
     name: Go ${{ matrix.go }} ${{ matrix.goos }} ${{ matrix.goarch }} build

.github/workflows/go-tests.yml

@@ -21,7 +21,6 @@ permissions:
 
 env:
   TEST_RESULTS: /tmp/test-results
-  GOTESTSUM_VERSION: 1.8.2
 
 jobs:
   setup:
@@ -215,6 +214,7 @@ jobs:
   #   secrets:
   #     elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
   #     consul-license: ${{secrets.CONSUL_LICENSE}}
+  #     datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-oss:
     needs: 
@@ -227,9 +227,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: ""
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-enterprise:
     if: ${{ endsWith(github.repository, '-enterprise') }}
@@ -243,9 +247,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-race:
     needs: 
@@ -259,9 +267,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-32bit:
     needs: 
@@ -275,9 +287,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-envoyextensions:
     needs:
@@ -289,9 +305,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-troubleshoot:
     needs:
@@ -303,9 +323,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-api-1-19:
     needs: 
@@ -317,9 +341,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-api-1-20:
     needs: 
@@ -331,9 +359,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-sdk-1-19:
     needs: 
@@ -345,9 +377,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   go-test-sdk-1-20:
     needs: 
@@ -359,9 +395,13 @@ jobs:
       runs-on: ${{ needs.setup.outputs.compute-xl }}
       repository-name: ${{ github.repository }}
       go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}"
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read    
     secrets:
       elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
       consul-license: ${{secrets.CONSUL_LICENSE}}
+      datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}"
 
   noop:
     runs-on: ubuntu-latest

.github/workflows/reusable-unit-split.yml

@@ -42,13 +42,16 @@ on:
         required: true
       consul-license:
         required: true
+      datadog-api-key:
+        required: true
 env:
   TEST_RESULTS: /tmp/test-results
   GOTESTSUM_VERSION: 1.8.2
   GOARCH: ${{inputs.go-arch}}
   TOTAL_RUNNERS: ${{inputs.runner-count}}
   CONSUL_LICENSE: ${{secrets.consul-license}}
   GOTAGS: ${{ inputs.go-tags}}
+  DATADOG_API_KEY: ${{secrets.datadog-api-key}}
 
 jobs:
   set-test-package-matrix:
@@ -128,6 +131,36 @@ jobs:
           -tags="${{env.GOTAGS}}" -p 2 \
           ${GO_TEST_FLAGS-} \
           -cover -coverprofile=coverage.txt
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Authenticate to Vault
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: vault-auth
+        run: vault-auth
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Fetch Secrets
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: secrets
+        uses: hashicorp/vault-action@v2.5.0
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
+
+      - name: prepare datadog-ci
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
+          chmod +x /usr/local/bin/datadog-ci
+
+      - name: upload coverage
+        env:
+          DD_ENV: ci
+        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml
+
       - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
         with:
           name: test-results