Merge branch 'main' into patch-10

.github/actions/install-external-tools/action.yml

@@ -21,15 +21,15 @@ runs:
     - uses: ./.github/actions/set-up-staticcheck
       # We assume that the Go toolchain will be managed by the caller workflow so we don't set one
       # up here.
-    - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+    - run: ./.github/scripts/retry-command.sh go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
       shell: bash
-    - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+    - run: ./.github/scripts/retry-command.sh go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.4.0
       shell: bash
-    - run: go install github.com/favadi/protoc-go-inject-tag@latest
+    - run: ./.github/scripts/retry-command.sh go install github.com/favadi/protoc-go-inject-tag@latest
       shell: bash
-    - run: go install golang.org/x/tools/cmd/goimports@latest
+    - run: ./.github/scripts/retry-command.sh go install golang.org/x/tools/cmd/goimports@latest
       shell: bash
-    - run: go install github.com/golangci/revgrep/cmd/revgrep@latest
+    - run: ./.github/scripts/retry-command.sh go install github.com/golangci/revgrep/cmd/revgrep@latest
       shell: bash
-    - run: go install github.com/loggerhead/enumer@latest
+    - run: ./.github/scripts/retry-command.sh go install github.com/loggerhead/enumer@latest
       shell: bash

.github/actions/set-up-buf/action.yml

@@ -34,7 +34,7 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        VERSION=$(gh release list -R bufbuild/buf --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
+        VERSION=$(./.github/scripts/retry-command.sh gh release list -R bufbuild/buf --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
 
         mkdir -p $(dirname ${{ inputs.destination }})
         DESTINATION="$(readlink -f "${{ inputs.destination }}")"
@@ -60,7 +60,7 @@ runs:
         fi
 
         mkdir -p tmp
-        gh release download "$VERSION" -p "buf-${OS}-${ARCH}.tar.gz" -O tmp/buf.tgz -R bufbuild/buf
+        ./.github/scripts/retry-command.sh gh release download "$VERSION" --clobber -p "buf-${OS}-${ARCH}.tar.gz" -O tmp/buf.tgz -R bufbuild/buf
         pushd tmp && tar -xvf buf.tgz && popd
         mv tmp/buf/bin/buf "$DESTINATION"
         rm -rf tmp

.github/actions/set-up-gofumpt/action.yml

@@ -34,7 +34,7 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        VERSION=$(gh release list -R mvdan/gofumpt --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
+        VERSION=$(./.github/scripts/retry-command.sh gh release list -R mvdan/gofumpt --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
 
         mkdir -p $(dirname ${{ inputs.destination }})
         DESTINATION="$(readlink -f "${{ inputs.destination }}")"
@@ -56,6 +56,6 @@ runs:
           export OS="darwin"
         fi
 
-        gh release download "$VERSION" -p "gofumpt_*_${OS}_${ARCH}" -O gofumpt -R mvdan/gofumpt
+        ./.github/scripts/retry-command.sh gh release download "$VERSION" --clobber -p "gofumpt_*_${OS}_${ARCH}" -O gofumpt -R mvdan/gofumpt
         chmod +x gofumpt
         mv gofumpt "$DESTINATION"

.github/actions/set-up-gosimports/action.yml

@@ -34,7 +34,7 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        VERSION=$(gh release list -R rinchsan/gosimports --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
+        VERSION=$(./.github/scripts/retry-command.sh gh release list -R rinchsan/gosimports --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
 
         mkdir -p $(dirname ${{ inputs.destination }})
         DESTINATION="$(readlink -f "${{ inputs.destination }}")"
@@ -57,7 +57,7 @@ runs:
         fi
 
         mkdir -p tmp
-        gh release download "$VERSION" -p "gosimports_*_${OS}_${ARCH}.tar.gz" -O tmp/gosimports.tgz -R rinchsan/gosimports
+        ./.github/scripts/retry-command.sh gh release download "$VERSION" --clobber -p "gosimports_*_${OS}_${ARCH}.tar.gz" -O tmp/gosimports.tgz -R rinchsan/gosimports
         pushd tmp && tar -xvf gosimports.tgz && popd
         mv tmp/gosimports "$DESTINATION"
         rm -rf tmp

.github/actions/set-up-gotestsum/action.yml

@@ -34,7 +34,7 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        VERSION=$(gh release list -R gotestyourself/gotestsum --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
+        VERSION=$(./.github/scripts/retry-command.sh gh release list -R gotestyourself/gotestsum --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
 
         mkdir -p $(dirname ${{ inputs.destination }})
         DESTINATION="$(readlink -f "${{ inputs.destination }}")"
@@ -54,7 +54,7 @@ runs:
         fi
 
         mkdir -p tmp
-        gh release download "$VERSION" -p "*${OS}_${ARCH}.tar.gz" -O tmp/gotestsum.tgz -R gotestyourself/gotestsum
+        ./.github/scripts/retry-command.sh gh release download "$VERSION" --clobber -p "*${OS}_${ARCH}.tar.gz" -O tmp/gotestsum.tgz -R gotestyourself/gotestsum
         pushd tmp && tar -xvf gotestsum.tgz && popd
         mv tmp/gotestsum "$DESTINATION"
         rm -rf tmp

.github/actions/set-up-misspell/action.yml

@@ -34,7 +34,7 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        VERSION=$(gh release list -R golangci/misspell --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
+        VERSION=$(./.github/scripts/retry-command.sh gh release list -R golangci/misspell --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -f1)
 
         mkdir -p $(dirname ${{ inputs.destination }})
         DESTINATION="$(readlink -f "${{ inputs.destination }}")"
@@ -57,7 +57,7 @@ runs:
         fi
 
         mkdir -p tmp
-        gh release download "$VERSION" -p "misspell_*_${OS}_${ARCH}.tar.gz" -O tmp/misspell.tgz -R golangci/misspell
+        ./.github/scripts/retry-command.sh gh release download "$VERSION" --clobber -p "misspell_*_${OS}_${ARCH}.tar.gz" -O tmp/misspell.tgz -R golangci/misspell
         pushd tmp && tar -xvf misspell.tgz && popd
         mv tmp/misspell_"$(echo "$VERSION" | tr -d v)"_${OS}_${ARCH}/misspell "$DESTINATION"
         rm -rf tmp

.github/actions/set-up-staticcheck/action.yml

@@ -34,7 +34,7 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        VERSION=$(gh release list -R dominikh/go-tools --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -d " " -f2)
+        VERSION=$(./.github/scripts/retry-command.sh gh release list -R dominikh/go-tools --exclude-drafts --exclude-pre-releases | grep ${{ inputs.version }} | cut -d " " -f2)
 
         mkdir -p $(dirname ${{ inputs.destination }})
         DESTINATION="$(readlink -f "${{ inputs.destination }}")"
@@ -57,7 +57,7 @@ runs:
         fi
 
         mkdir -p tmp
-        gh release download "$VERSION" -p "staticcheck_${OS}_${ARCH}.tar.gz" -O tmp/staticcheck.tgz -R dominikh/go-tools
+        ./.github/scripts/retry-command.sh gh release download "$VERSION" --clobber -p "staticcheck_${OS}_${ARCH}.tar.gz" -O tmp/staticcheck.tgz -R dominikh/go-tools
         pushd tmp && tar -xvf staticcheck.tgz && popd
         mv tmp/staticcheck/staticcheck "$DESTINATION"
         rm -rf tmp

.github/scripts/retry-command.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+set -uo pipefail
+
+tries=5
+count=0
+
+until "$@"
+do
+  if [ $count -eq $tries ]; then
+    echo "tried $count times, exiting"
+    exit 1
+  fi
+  ((count++))
+  echo "trying again, attempt $count"
+  sleep $count
+done

.github/workflows/enos-release-testing-oss.yml

@@ -64,7 +64,7 @@ jobs:
     secrets: inherit
 
   save-metadata:
-    runs-on: linux
+    runs-on: ubuntu-latest
     if: always()
     needs: test
     steps:

.github/workflows/test-go.yml

@@ -421,8 +421,9 @@ jobs:
             package_parallelism="-p 2"
           fi
 
-          # If running Go Test 32bit nightly tests, add a flag to rerun failed tests
-          if [[  "${{inputs.name}}" == 'i386' ]]; then
+          # If running Go tests on the enterprise repo, add a flag to rerun failed tests.
+          # This is to address the issues with flaky tests affecting the reliability of CI.
+          if [[  "${{github.repository}}" == 'hashicorp/vault-enterprise' ]]; then
               RERUN_FAILS="--rerun-fails"
           fi
 

CHANGELOG.md

@@ -2,6 +2,31 @@
 - [v1.0.0 - v1.9.10](CHANGELOG-pre-v1.10.md)
 - [v0.11.6 and earlier](CHANGELOG-v0.md)
 
+## 1.17.2
+### July 10, 2024
+
+CHANGES:
+
+* core: Bump Go version to 1.22.5
+* secrets/azure: Update plugin to v0.19.2 [[GH-27652](https://github.com/hashicorp/vault/pull/27652)]
+
+FEATURES:
+
+* **AWS secrets engine STS session tags support**: Adds support for setting STS
+session tags when generating temporary credentials using the AWS secrets
+engine. [[GH-27620](https://github.com/hashicorp/vault/pull/27620)]
+
+BUG FIXES:
+
+* cli: Fixed issue with `vault hcp connect` where HCP resources with uppercase letters were inaccessible when entering the correct project name. [[GH-27694](https://github.com/hashicorp/vault/pull/27694)]
+* core (enterprise): Fix HTTP redirects in namespaces to use the correct path and (in the case of event subscriptions) the correct URI scheme. [[GH-27660](https://github.com/hashicorp/vault/pull/27660)]
+* core/config: fix issue when using `proxy_protocol_behavior` with `deny_unauthorized`, 
+which causes the Vault TCP listener to close after receiving an untrusted upstream proxy connection. [[GH-27589](https://github.com/hashicorp/vault/pull/27589)]
+* core: Fixed an issue with performance standbys not being able to handle rotate root requests. [[GH-27631](https://github.com/hashicorp/vault/pull/27631)]
+* secrets/transit (enterprise): Fix an issue that caused input data be returned as part of generated CMAC values.
+* ui: Display an error and force a timeout when TOTP passcode is incorrect [[GH-27574](https://github.com/hashicorp/vault/pull/27574)]
+* ui: Ensure token expired banner displays when batch token expires [[GH-27479](https://github.com/hashicorp/vault/pull/27479)]
+
 ## 1.17.1
 ### June 26, 2024
 
@@ -227,6 +252,25 @@ autopilot to fail to discover new server versions and so not trigger an upgrade.
 * ui: fixed a bug where the replication pages did not update display when navigating between DR and performance [[GH-26325](https://github.com/hashicorp/vault/pull/26325)]
 * ui: fixes undefined start time in filename for downloaded client count attribution csv [[GH-26485](https://github.com/hashicorp/vault/pull/26485)]
 
+## 1.16.6 Enterprise
+### July 10, 2024
+**Enterprise LTS:** Vault Enterprise 1.16 is a [Long-Term Support (LTS)](https://developer.hashicorp.com/vault/docs/enterprise/lts) release.
+
+CHANGES:
+
+* core: Bump Go version to 1.22.5.
+* auth/jwt: Revert [GH-295](https://github.com/hashicorp/vault-plugin-auth-jwt/pull/295) which changed the way JWT `aud` claims were validated.
+
+BUG FIXES:
+
+* agent: Correctly constructs kv-v2 secret paths in nested namespaces. [[GH-26863](https://github.com/hashicorp/vault/pull/26863)]
+* core (enterprise): Fix HTTP redirects in namespaces to use the correct path and (in the case of event subscriptions) the correct URI scheme. [[GH-27660](https://github.com/hashicorp/vault/pull/27660)]
+* core/config: fix issue when using `proxy_protocol_behavior` with `deny_unauthorized`, 
+which causes the Vault TCP listener to close after receiving an untrusted upstream proxy connection. [[GH-27589](https://github.com/hashicorp/vault/pull/27589)]
+* core: Fixed an issue with performance standbys not being able to handle rotate root requests. [[GH-27631](https://github.com/hashicorp/vault/pull/27631)]
+* ui: Display an error and force a timeout when TOTP passcode is incorrect [[GH-27574](https://github.com/hashicorp/vault/pull/27574)]
+* ui: Ensure token expired banner displays when batch token expires [[GH-27479](https://github.com/hashicorp/vault/pull/27479)]
+
 ## 1.16.5 Enterprise
 ### June 26, 2024
 
@@ -728,6 +772,23 @@ leading to failure to complete merkle sync without a full re-index. [[GH-23013](
 * ui: remove user_lockout_config settings for unsupported methods [[GH-25867](https://github.com/hashicorp/vault/pull/25867)]
 * ui: show error from API when seal fails [[GH-23921](https://github.com/hashicorp/vault/pull/23921)]
 
+## 1.15.12 Enterprise
+### July 10, 2024
+
+CHANGES:
+
+* core: Bump Go version to 1.22.5.
+* auth/jwt: Revert [GH-295](https://github.com/hashicorp/vault-plugin-auth-jwt/pull/295) which changed the way JWT `aud` claims were validated.
+
+BUG FIXES:
+
+* core (enterprise): Fix HTTP redirects in namespaces to use the correct path and (in the case of event subscriptions) the correct URI scheme. [[GH-27660](https://github.com/hashicorp/vault/pull/27660)]
+* core/config: fix issue when using `proxy_protocol_behavior` with `deny_unauthorized`, 
+which causes the Vault TCP listener to close after receiving an untrusted upstream proxy connection. [[GH-27589](https://github.com/hashicorp/vault/pull/27589)]
+* core: Fixed an issue with performance standbys not being able to handle rotate root requests. [[GH-27631](https://github.com/hashicorp/vault/pull/27631)]
+* ui: Display an error and force a timeout when TOTP passcode is incorrect [[GH-27574](https://github.com/hashicorp/vault/pull/27574)]
+* ui: Ensure token expired banner displays when batch token expires [[GH-27479](https://github.com/hashicorp/vault/pull/27479)]
+
 ## 1.15.11 Enterprise
 ### June 26, 2024
 

Dockerfile

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: BUSL-1.1
 
 ## DOCKERHUB DOCKERFILE ##
-FROM alpine:3.18 as default
+FROM alpine:3 as default
 
 ARG BIN_NAME
 # NAME and PRODUCT_VERSION are the name of the software in releases.hashicorp.com
@@ -75,7 +75,7 @@ CMD ["server", "-dev"]
 
 
 ## UBI DOCKERFILE ##
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10 as ubi
+FROM registry.access.redhat.com/ubi8/ubi-minimal as ubi
 
 ARG BIN_NAME
 # PRODUCT_VERSION is the version built dist/$TARGETOS/$TARGETARCH/$BIN_NAME,

README.md

@@ -72,9 +72,12 @@ If you wish to work on Vault itself or any of its built-in systems, you'll
 first need [Go](https://www.golang.org) installed on your machine.
 
 For local dev first make sure Go is properly installed, including setting up a
-[GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that `$GOPATH/bin` is in
-your path as some distributions bundle the old version of build tools. Next, clone this
-repository. Vault uses [Go Modules](https://github.com/golang/go/wiki/Modules),
+[GOPATH](https://golang.org/doc/code.html#GOPATH), then setting the 
+[GOBIN](https://pkg.go.dev/cmd/go#hdr-Environment_variables) variable to `$GOPATH/bin`. 
+Ensure that `$GOPATH/bin` is in your path as some distributions bundle the old version 
+of build tools. 
+
+Next, clone this repository. Vault uses [Go Modules](https://github.com/golang/go/wiki/Modules),
 so it is recommended that you clone the repository ***outside*** of the GOPATH.
 You can then download any required build tools by bootstrapping your environment:
 
@@ -120,6 +123,15 @@ $ make test TEST=./vault
 ...
 ```
 
+### Troubleshooting
+
+If you encounter an error like `could not read Username for 'https://github.com'` you may need to adjust your git config like so:
+
+```sh
+$ git config --global --add url."git@github.com:".insteadOf "https://github.com/"
+```
+
+
 ### Importing Vault
 
 This repository publishes two libraries that may be imported by other projects:

api/auth/azure/azure.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"time"
@@ -175,7 +175,7 @@ func (a *AzureAuth) getJWT() (string, error) {
 	}
 	defer resp.Body.Close()
 
-	responseBytes, err := ioutil.ReadAll(resp.Body)
+	responseBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf("error reading response body from Azure token endpoint: %w", err)
 	}
@@ -222,7 +222,7 @@ func getMetadata() (metadataJSON, error) {
 	}
 	defer resp.Body.Close()
 
-	responseBytes, err := ioutil.ReadAll(resp.Body)
+	responseBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return metadataJSON{}, fmt.Errorf("error reading response body from metadata endpoint: %w", err)
 	}

api/auth/gcp/gcp.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"time"
@@ -181,7 +181,7 @@ func (a *GCPAuth) getJWTFromMetadataService(vaultAddress string) (string, error)
 	defer resp.Body.Close()
 
 	// get jwt from response
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	jwt := string(body)
 	if err != nil {
 		return "", fmt.Errorf("error reading response from metadata service: %w", err)

api/sudo_paths.go

@@ -28,6 +28,7 @@ var sudoPaths = map[string]*regexp.Regexp{
 	"/sys/config/ui/headers":                        regexp.MustCompile(`^/sys/config/ui/headers/?$`),
 	"/sys/config/ui/headers/{header}":               regexp.MustCompile(`^/sys/config/ui/headers/.+$`),
 	"/sys/internal/inspect/router/{tag}":            regexp.MustCompile(`^/sys/internal/inspect/router/.+$`),
+	"/sys/internal/counters/activity/export":        regexp.MustCompile(`^/sys/internal/counters/activity/export$`),
 	"/sys/leases":                                   regexp.MustCompile(`^/sys/leases$`),
 	// This entry is a bit wrong... sys/leases/lookup does NOT require sudo. But sys/leases/lookup/ with a trailing
 	// slash DOES require sudo. But the part of the Vault CLI that uses this logic doesn't pass operation-appropriate

api/sys_raft.go

@@ -264,7 +264,7 @@ func (c *Sys) RaftSnapshotWithContext(ctx context.Context, snapWriter io.Writer)
 				continue
 			}
 			var b []byte
-			b, err = ioutil.ReadAll(t)
+			b, err = io.ReadAll(t)
 			if err != nil || len(b) == 0 {
 				return
 			}

audit/backend_file.go

@@ -76,12 +76,12 @@ func newFileBackend(conf *BackendConfig, headersConfig HeaderFormatter) (*FileBa
 		return nil, err
 	}
 
-	var opt []event.Option
+	sinkOpts := []event.Option{event.WithLogger(conf.Logger)}
 	if mode, ok := conf.Config[optionMode]; ok {
-		opt = append(opt, event.WithFileMode(mode))
+		sinkOpts = append(sinkOpts, event.WithFileMode(mode))
 	}
 
-	err = b.configureSinkNode(conf.MountPath, filePath, cfg.requiredFormat, opt...)
+	err = b.configureSinkNode(conf.MountPath, filePath, cfg.requiredFormat, sinkOpts...)
 	if err != nil {
 		return nil, err
 	}

audit/backend_noop.go

@@ -244,7 +244,7 @@ func (n *noopWrapper) Process(ctx context.Context, e *eventlogger.Event) (*event
 	// formatted headers that would have made it to the logs via the sink node.
 	// They only appear in requests.
 	if a.Subtype == RequestType {
-		reqEntry := &RequestEntry{}
+		reqEntry := &Entry{}
 		err = json.Unmarshal(b, &reqEntry)
 		if err != nil {
 			return nil, fmt.Errorf("unable to parse formatted audit entry data: %w", err)

audit/backend_socket.go

@@ -70,6 +70,7 @@ func newSocketBackend(conf *BackendConfig, headersConfig HeaderFormatter) (*Sock
 	sinkOpts := []event.Option{
 		event.WithSocketType(socketType),
 		event.WithMaxDuration(writeDeadline),
+		event.WithLogger(conf.Logger),
 	}
 
 	err = event.ValidateOptions(sinkOpts...)