Merge branch 'main' into dependabot/github_actions/actions/setup-go-5…

….0.2
hashicorp · Aug 15, 2024 · 4224e08 · 4224e08
2 parents c56ec72 + 5e08bd4
commit 4224e08
Show file tree

Hide file tree

Showing 51 changed files with 1,717 additions and 213 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -2,11 +2,9 @@
 What does this PR do?
 
 ### TODO only if you're a HashiCorp employee
-- [ ] **Labels:** If this PR is the CE portion of an ENT change, and that ENT change is
-  getting backported to N-2, use the new style `backport/ent/x.x.x+ent` labels
-  instead of the old style `backport/x.x.x` labels.
-- [ ] **Labels:** If this PR is a CE only change, it can only be backported to N, so use
-  the normal `backport/x.x.x` label (there should be only 1).
+- [ ] **Backport Labels:** If this PR is in the ENT repo and needs to be backported, backport  
+  to N, N-1, and N-2, using the `backport/ent/x.x.x+ent` labels. If this PR is in the CE repo, you should only backport to N, using the `backport/x.x.x` label, not the enterprise labels.
+    - [ ] If this fixes a critical security vulnerability or [severity 1](https://www.hashicorp.com/customer-success/enterprise-support) bug, it will also need to be backported to the current [LTS versions](https://developer.hashicorp.com/vault/docs/enterprise/lts#why-is-there-a-risk-to-updating-to-a-non-lts-vault-enterprise-version) of Vault. To ensure this, use **all** available enterprise labels.
 - [ ] **ENT Breakage:** If this PR either 1) removes a public function OR 2) changes the signature
   of a public function, even if that change is in a CE file, _double check_ that
   applying the patch for this PR to the ENT repo and running tests doesn't

diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml
@@ -15,12 +15,13 @@ on:
 
 jobs:
   scan:
-    runs-on: ${{ fromJSON(vars.RUNNER_XL) }}
+    runs-on: ${{ github.repository == 'hashicorp/vault' && 'ubuntu-latest' || fromJSON('["self-hosted","ondemand","os=linux","type=c6a.4xlarge"]') }}
     # The first check ensures this doesn't run on community-contributed PRs, who
     # won't have the permissions to run this job.
-    if: ${{ (github.repository != 'hashicorp/vault' || (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name))
-          && (github.actor != 'dependabot[bot]') && ( github.actor != 'hc-github-team-secure-vault-core') }}
-
+    if: |
+      (startsWith(github.repository, 'hashicorp/vault') || (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)) &&
+      github.actor != 'dependabot[bot]' &&
+      github.actor != 'hc-github-team-secure-vault-core'
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 

diff --git a/Dockerfile b/Dockerfile
@@ -25,7 +25,7 @@ LABEL name="Vault" \
       description="Vault is a tool for securely accessing secrets. A secret is anything that you want to tightly control access to, such as API keys, passwords, certificates, and more. Vault provides a unified interface to any secret, while providing tight access control and recording a detailed audit log."
 
 # Copy the license file as per Legal requirement
-COPY LICENSE /licenses/LICENSE.txt
+COPY LICENSE /usr/share/doc/$NAME/LICENSE.txt
 
 # Set ARGs as ENV so that they can be used in ENTRYPOINT/CMD
 ENV NAME=$NAME
@@ -97,7 +97,7 @@ LABEL name="Vault" \
       description="Vault is a tool for securely accessing secrets. A secret is anything that you want to tightly control access to, such as API keys, passwords, certificates, and more. Vault provides a unified interface to any secret, while providing tight access control and recording a detailed audit log."
 
 # Copy the license file as per Legal requirement
-COPY LICENSE /licenses/LICENSE.txt
+COPY LICENSE /usr/share/doc/$NAME/LICENSE.txt
 
 # Set ARGs as ENV so that they can be used in ENTRYPOINT/CMD
 ENV NAME=$NAME

diff --git a/changelog/28068.txt b/changelog/28068.txt
@@ -0,0 +1,7 @@
+```release-note:improvement
+cli: `vault operator usage` will now include a warning if the specified usage period contains estimated client counts.
+```
+
+```release-note:improvement
+activity: `/sys/internal/counters/activity` will now include a warning if the specified usage period contains estimated client counts.
+```
diff --git a/changelog/28076.txt b/changelog/28076.txt
@@ -0,0 +1,3 @@
+```release-note:improvement
+website/docs: corrected invalid json in sample payload for azure secrets engine create/update role
+```
diff --git a/changelog/28093.txt b/changelog/28093.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+ui: fixes toast (flash) alert message saying "created" when deleting a kv v2 secret
+```
diff --git a/command/operator_usage.go b/command/operator_usage.go
@@ -142,6 +142,12 @@ func (c *OperatorUsageCommand) Run(args []string) int {
 	colConfig.Empty = " " // Do not show n/a on intentional blank lines
 	colConfig.Glue = "   "
 	c.UI.Output(tableOutput(out, colConfig))
+
+	// Also, output the warnings returned, if any:
+	for _, warning := range resp.Warnings {
+		c.UI.Warn(warning)
+	}
+
 	return 0
 }
 

diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl
@@ -245,7 +245,7 @@ scenario "autopilot" {
       config_mode          = matrix.config_mode
       enable_audit_devices = var.vault_enable_audit_devices
       hosts                = step.create_vault_cluster_targets.hosts
-      install_dir          = global.vault_install_dir[matrix.artifact_type]
+      install_dir          = local.vault_install_dir
       ip_version           = matrix.ip_version
       license              = matrix.edition != "ce" ? step.read_license.license : null
       packages             = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]])
@@ -315,7 +315,7 @@ scenario "autopilot" {
       hosts             = step.create_vault_cluster.hosts
       ip_version        = matrix.ip_version
       vault_addr        = step.create_vault_cluster.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
   }
@@ -343,7 +343,7 @@ scenario "autopilot" {
       hosts             = step.create_vault_cluster.hosts
       leader_host       = step.get_vault_cluster_ips.leader_host
       vault_addr        = step.create_vault_cluster.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
   }
@@ -381,7 +381,7 @@ scenario "autopilot" {
       force_unseal                = matrix.seal == "shamir"
       hosts                       = step.create_vault_cluster_upgrade_targets.hosts
       initialize_cluster          = false
-      install_dir                 = global.vault_install_dir[matrix.artifact_type]
+      install_dir                 = local.vault_install_dir
       ip_version                  = matrix.ip_version
       license                     = matrix.edition != "ce" ? step.read_license.license : null
       local_artifact_path         = local.artifact_path
@@ -421,7 +421,7 @@ scenario "autopilot" {
     variables {
       hosts             = step.upgrade_vault_cluster_with_autopilot.hosts
       vault_addr        = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
     }
   }
 
@@ -443,7 +443,7 @@ scenario "autopilot" {
       hosts             = step.upgrade_vault_cluster_with_autopilot.hosts
       ip_version        = matrix.ip_version
       vault_addr        = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
       vault_root_token  = step.upgrade_vault_cluster_with_autopilot.root_token
     }
   }
@@ -471,7 +471,7 @@ scenario "autopilot" {
       vault_addr                      = step.create_vault_cluster.api_addr_localhost
       vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
       vault_autopilot_upgrade_status  = "await-server-removal"
-      vault_install_dir               = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir               = local.vault_install_dir
       vault_root_token                = step.upgrade_vault_cluster_with_autopilot.root_token
     }
   }
@@ -500,7 +500,7 @@ scenario "autopilot" {
       ip_version        = matrix.ip_version
       timeout           = 120 // seconds
       vault_addr        = step.create_vault_cluster.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
   }
@@ -530,7 +530,7 @@ scenario "autopilot" {
       hosts             = step.upgrade_vault_cluster_with_autopilot.hosts
       ip_version        = matrix.ip_version
       vault_addr        = step.create_vault_cluster.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
   }
@@ -554,7 +554,7 @@ scenario "autopilot" {
     variables {
       hosts             = step.get_updated_vault_cluster_ips.follower_hosts
       vault_addr        = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir = local.vault_install_dir
     }
   }
 
@@ -585,7 +585,7 @@ scenario "autopilot" {
       operator_instance       = step.get_updated_vault_cluster_ips.leader_public_ip
       vault_addr              = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
       vault_cluster_addr_port = step.upgrade_vault_cluster_with_autopilot.cluster_port
-      vault_install_dir       = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir       = local.vault_install_dir
       vault_root_token        = step.create_vault_cluster.root_token
     }
   }
@@ -631,7 +631,7 @@ scenario "autopilot" {
       vault_addr                      = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
       vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
       vault_autopilot_upgrade_status  = "idle"
-      vault_install_dir               = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir               = local.vault_install_dir
       vault_root_token                = step.create_vault_cluster.root_token
     }
   }
@@ -689,7 +689,7 @@ scenario "autopilot" {
       hosts                 = step.upgrade_vault_cluster_with_autopilot.hosts
       vault_addr            = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
       vault_edition         = matrix.edition
-      vault_install_dir     = global.vault_install_dir[matrix.artifact_type]
+      vault_install_dir     = local.vault_install_dir
       vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
       vault_revision        = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
       vault_build_date      = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
@@ -719,10 +719,8 @@ scenario "autopilot" {
     }
   }
 
-  step "verify_undo_logs_status" {
-    skip_step = true
-    // NOTE: temporarily disable undo logs checking until it is fixed. See VAULT-20259
-    // skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0")
+  step "verify_undo_logs_enabled_on_primary" {
+    skip_step   = semverconstraint(var.vault_product_version, "<1.13.0-0")
     module      = module.vault_verify_undo_logs
     description = <<-EOF
       Verifies that undo logs is correctly enabled on newly upgraded target hosts. For this it will
@@ -743,8 +741,30 @@ scenario "autopilot" {
     }
 
     variables {
-      hosts             = step.upgrade_vault_cluster_with_autopilot.hosts
-      vault_install_dir = global.vault_install_dir[matrix.artifact_type]
+      expected_state    = 1 # Enabled
+      hosts             = step.get_updated_vault_cluster_ips.leader_hosts
+      timeout           = 180 # Seconds
+      vault_addr        = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "verify_undo_logs_disabled_on_followers" {
+    skip_step  = semverconstraint(var.vault_product_version, "<1.13.0-0")
+    module     = module.vault_verify_undo_logs
+    depends_on = [step.verify_undo_logs_enabled_on_primary]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      expected_state    = 0 # Disabled
+      hosts             = step.get_updated_vault_cluster_ips.follower_hosts
+      timeout           = 10 # Seconds
+      vault_addr        = step.upgrade_vault_cluster_with_autopilot.api_addr_localhost
+      vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
   }

diff --git a/enos/modules/vault_get_cluster_ips/main.tf b/enos/modules/vault_get_cluster_ips/main.tf
@@ -168,6 +168,10 @@ output "leader_host" {
   value = local.leader_host
 }
 
+output "leader_hosts" {
+  value = { 0 : local.leader_host }
+}
+
 output "leader_ipv6" {
   value = local.leader_ipv6
 }

diff --git a/enos/modules/vault_verify_undo_logs/main.tf b/enos/modules/vault_verify_undo_logs/main.tf
@@ -9,13 +9,35 @@ terraform {
   }
 }
 
+variable "expected_state" {
+  type        = number
+  description = "The expected state to have in vault.core.replication.write_undo_logs telemetry. Must be either 1 for enabled or 0 for disabled."
+
+  validation {
+    condition     = contains([0, 1], var.expected_state)
+    error_message = "The expected_state must be either 0 or 1"
+  }
+}
+
 variable "hosts" {
   type = map(object({
     ipv6       = string
     private_ip = string
     public_ip  = string
   }))
-  description = "The vault cluster instances that were created"
+  description = "The vault cluster target hosts to check"
+}
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
 }
 
 variable "vault_addr" {
@@ -37,6 +59,9 @@ resource "enos_remote_exec" "smoke-verify-undo-logs" {
   for_each = var.hosts
 
   environment = {
+    EXPECTED_STATE    = var.expected_state
+    RETRY_INTERVAL    = var.retry_interval
+    TIMEOUT_SECONDS   = var.timeout
     VAULT_ADDR        = var.vault_addr
     VAULT_INSTALL_DIR = var.vault_install_dir
     VAULT_TOKEN       = var.vault_root_token

diff --git a/enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh b/enos/modules/vault_verify_undo_logs/scripts/smoke-verify-undo-logs.sh
@@ -7,30 +7,29 @@ function fail() {
   exit 1
 }
 
+[[ -z "$EXPECTED_STATE" ]] && fail "EXPECTED_STAE env variable has not been set"
+[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set"
+[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
 [[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
 [[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
 [[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
 
 binpath=${VAULT_INSTALL_DIR}/vault
 test -x "$binpath" || fail "unable to locate vault binary at $binpath"
 
-count=0
-retries=5
-while :; do
+begin_time=$(date +%s)
+end_time=$((begin_time + TIMEOUT_SECONDS))
+while [ "$(date +%s)" -lt "$end_time" ]; do
   state=$($binpath read sys/metrics -format=json | jq -r '.data.Gauges[] | select(.Name == "vault.core.replication.write_undo_logs")')
   target_undo_logs_status="$(jq -r '.Value' <<< "$state")"
 
-  if [ "$target_undo_logs_status" == "1" ]; then
+  if [ "$target_undo_logs_status" == "$EXPECTED_STATE" ]; then
+    echo "vault.core.replication.write_undo_logs has expected Value: \"${EXPECTED_STATE}\""
     exit 0
   fi
 
-  wait=$((2 ** count))
-  count=$((count + 1))
-  if [ "$count" -lt "$retries" ]; then
-    echo "Waiting for vault.core.replication.write_undo_logs to have Value:1"
-    echo "$state"
-    sleep "$wait"
-  else
-    fail "Timed out waiting for vault.core.replication.write_undo_logs to have Value:1"
-  fi
+  echo "Waiting for vault.core.replication.write_undo_logs to have Value: \"${EXPECTED_STATE}\""
+  sleep "$RETRY_INTERVAL"
 done
+
+fail "Timed out waiting for vault.core.replication.write_undo_logs to have Value: \"${EXPECTED_STATE}\""
diff --git a/enos/modules/vault_verify_version/scripts/verify-cli-version.sh b/enos/modules/vault_verify_version/scripts/verify-cli-version.sh
@@ -43,5 +43,13 @@ version_output=$("$binpath" version)
 if [[ "$version_output" == "$version_expected_nosha" ]] || [[ "$version_output" == "$version_expected" ]]; then
   echo "Version verification succeeded!"
 else
-  fail "expected Version=$version_expected or $version_expected_nosha, got: $version_output"
+  msg="$(printf "\nThe Vault cluster did not match the expected version, expected:\n%s\nor\n%s\ngot:\n%s" "$version_expected" "$version_expected_nosha" "$version_output")"
+  if type diff &> /dev/null; then
+    # Diff exits non-zero if we have a diff, which we want, so we'll guard against failing early.
+    if ! version_diff=$(diff  <(echo "$version_expected" ) <(echo "$version_output") -u -L expected -L got); then
+      msg="$(printf "\nThe Vault cluster did not match the expected version:\n%s" "$version_diff")"
+    fi
+  fi
+
+  fail "$msg"
 fi