Acceptance tests fixes (hashicorp#858)

* Pass either build URL or build number to terraform resources for GCP/AWS/Azure. On GKE, we can't pass build URL because / is not a valid character in labels * Remove scripting logic that makes tests fail early (we don't need it in nightly acceptance tests) * Fix metrics merging test on GKE. We are running tests there with enterprise license enabled, but since we're setting the consul image to OSS one, we should also unset enterprise license values so that we don't try to run the enterprise license commands against OSS binary, where they don't exist
rrondeau · Mar 12, 2021 · c7c8c33 · c7c8c33
1 parent c53aea9
commit c7c8c33
Show file tree

Hide file tree

Showing 8 changed files with 74 additions and 71 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -184,7 +184,14 @@ jobs:
             terraform init
             echo "${GOOGLE_CREDENTIALS}" | gcloud auth activate-service-account --key-file=-
 
-            terraform apply -var project=${CLOUDSDK_CORE_PROJECT} -var init_cli=true -var cluster_count=2 -auto-approve
+            # On GKE, we're setting the build number instead of build URL because label values
+            # cannot contain '/'.
+            terraform apply \
+              -var project=${CLOUDSDK_CORE_PROJECT} \
+              -var init_cli=true \
+              -var cluster_count=2 \
+              -var labels="{\"build_number\": \"$CIRCLE_BUILD_NUM\"}" \
+              -auto-approve
 
       # Restore go module cache if there is one
       - restore_cache:
@@ -206,30 +213,16 @@ jobs:
             # The license expires 15-Oct-2025.
             KUBECONFIG=$primary_kubeconfig kubectl create secret generic ent-license --from-literal=key="${CONSUL_ENT_LICENSE}"
 
-            # We have to run the tests for each package separately so that we can
-            # exit early if any test fails (-failfast only works within a single
-            # package).
-            exit_code=0
-            for pkg in $(go list ./...)
-            do
-              if ! gotestsum --no-summary=all --jsonfile=jsonfile-${pkg////-} -- $pkg -p 1 -timeout 30m -failfast \
-                    -enable-enterprise \
-                    -enterprise-license-secret-name=ent-license \
-                    -enterprise-license-secret-key=key \
-                    -enable-pod-security-policies \
-                    -enable-multi-cluster \
-                    -kubeconfig="$primary_kubeconfig" \
-                    -secondary-kubeconfig="$secondary_kubeconfig" \
-                    -debug-directory="$TEST_RESULTS/debug" \
-                    -consul-k8s-image=docker.mirror.hashicorp.services/hashicorpdev/consul-k8s:latest
-              then
-                echo "Tests in ${pkg} failed, aborting early"
-                exit_code=1
-                break
-              fi
-            done
-            gotestsum --raw-command --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- cat jsonfile*
-            exit $exit_code
+            gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- ./... -p 1 -timeout 40m -failfast \
+              -enable-enterprise \
+              -enterprise-license-secret-name=ent-license \
+              -enterprise-license-secret-key=key \
+              -enable-pod-security-policies \
+              -enable-multi-cluster \
+              -kubeconfig="$primary_kubeconfig" \
+              -secondary-kubeconfig="$secondary_kubeconfig" \
+              -debug-directory="$TEST_RESULTS/debug" \
+              -consul-k8s-image=docker.mirror.hashicorp.services/hashicorpdev/consul-k8s:latest
 
       - store_test_results:
           path: /tmp/test-results
@@ -263,7 +256,12 @@ jobs:
           command: |
             terraform init
 
-            terraform apply -var client_id="$ARM_CLIENT_ID" -var client_secret="$ARM_CLIENT_SECRET" -var cluster_count=2 -auto-approve
+            terraform apply \
+              -var client_id="$ARM_CLIENT_ID" \
+              -var client_secret="$ARM_CLIENT_SECRET" \
+              -var cluster_count=2 \
+              -var tags="{\"build_url\": \"$CIRCLE_BUILD_URL\"}" \
+              -auto-approve
 
       # Restore go module cache if there is one
       - restore_cache:
@@ -280,27 +278,13 @@ jobs:
             export primary_kubeconfig=$(terraform output -state ../../terraform/aks/terraform.tfstate -json | jq -r .kubeconfigs.value[0])
             export secondary_kubeconfig=$(terraform output -state ../../terraform/aks/terraform.tfstate -json | jq -r .kubeconfigs.value[1])
 
-            # We have to run the tests for each package separately so that we can
-            # exit early if any test fails (-failfast only works within a single
-            # package).
-            exit_code=0
-            for pkg in $(go list ./...)
-            do
-              if ! gotestsum --no-summary=all --jsonfile=jsonfile-${pkg////-} -- $pkg -p 1 -timeout 40m -failfast \
-                    -enable-enterprise \
-                    -enable-multi-cluster \
-                    -kubeconfig="$primary_kubeconfig" \
-                    -secondary-kubeconfig="$secondary_kubeconfig" \
-                    -debug-directory="$TEST_RESULTS/debug" \
-                    -consul-k8s-image=docker.mirror.hashicorp.services/hashicorpdev/consul-k8s:latest
-              then
-                echo "Tests in ${pkg} failed, aborting early"
-                exit_code=1
-                break
-              fi
-            done
-            gotestsum --raw-command --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- cat jsonfile*
-            exit $exit_code
+            gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- ./... -p 1 -timeout 40m -failfast \
+              -enable-enterprise \
+              -enable-multi-cluster \
+              -kubeconfig="$primary_kubeconfig" \
+              -secondary-kubeconfig="$secondary_kubeconfig" \
+              -debug-directory="$TEST_RESULTS/debug" \
+              -consul-k8s-image=docker.mirror.hashicorp.services/hashicorpdev/consul-k8s:latest
 
       - store_test_results:
           path: /tmp/test-results
@@ -345,7 +329,7 @@ jobs:
           command: |
             terraform init
 
-            terraform apply -var cluster_count=2 -auto-approve
+            terraform apply -var cluster_count=2 -var tags="{\"build_url\": \"$CIRCLE_BUILD_URL\"}" -auto-approve
 
       # Restore go module cache if there is one
       - restore_cache:
@@ -367,27 +351,13 @@ jobs:
             chmod 600 "$primary_kubeconfig"
             chmod 600 "$secondary_kubeconfig"
 
-            # We have to run the tests for each package separately so that we can
-            # exit early if any test fails (-failfast only works within a single
-            # package).
-            exit_code=0
-            for pkg in $(go list ./...)
-            do
-              if ! gotestsum --no-summary=all --jsonfile=jsonfile-${pkg////-} -- $pkg -p 1 -timeout 40m -failfast \
-                    -enable-enterprise \
-                    -enable-multi-cluster \
-                    -kubeconfig="$primary_kubeconfig" \
-                    -secondary-kubeconfig="$secondary_kubeconfig" \
-                    -debug-directory="$TEST_RESULTS/debug" \
-                    -consul-k8s-image=docker.mirror.hashicorp.services/hashicorpdev/consul-k8s:latest
-              then
-                echo "Tests in ${pkg} failed, aborting early"
-                exit_code=1
-                break
-              fi
-            done
-            gotestsum --raw-command --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- cat jsonfile*
-            exit $exit_code
+            gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- ./... -p 1 -timeout 40m -failfast \
+              -enable-enterprise \
+              -enable-multi-cluster \
+              -kubeconfig="$primary_kubeconfig" \
+              -secondary-kubeconfig="$secondary_kubeconfig" \
+              -debug-directory="$TEST_RESULTS/debug" \
+              -consul-k8s-image=docker.mirror.hashicorp.services/hashicorpdev/consul-k8s:latest
 
       - store_test_results:
           path: /tmp/test-results

diff --git a/test/acceptance/tests/metrics/metrics_test.go b/test/acceptance/tests/metrics/metrics_test.go
@@ -94,7 +94,12 @@ func TestAppMetrics(t *testing.T) {
 		"global.metrics.enabled": "true",
 
 		// This image is required till Consul 1.10 is released and the CI config is updated.
-		"global.image": "docker.mirror.hashicorp.services/hashicorpdev/consul:latest",
+		// Note we need to set ent license to empty explicitly so that if tests have ent license
+		// globally provided this test won't try to apply it because this image is not an enterprise image.
+		// TODO: Remove this setting and ent license settings once Consul alpha is released.
+		"global.image":                        "docker.mirror.hashicorp.services/hashicorpdev/consul:latest",
+		"server.enterpriseLicense.secretName": "",
+		"server.enterpriseLicense.secretKey":  "",
 
 		"connectInject.enabled":                      "true",
 		"connectInject.metrics.defaultEnableMerging": "true",

diff --git a/test/terraform/aks/main.tf b/test/terraform/aks/main.tf
@@ -14,6 +14,8 @@ resource "azurerm_resource_group" "default" {
   count    = var.cluster_count
   name     = "consul-k8s-${random_id.suffix[count.index].dec}"
   location = var.location
+
+  tags = var.tags
 }
 
 resource "azurerm_kubernetes_cluster" "default" {
@@ -39,6 +41,8 @@ resource "azurerm_kubernetes_cluster" "default" {
   role_based_access_control {
     enabled = true
   }
+
+  tags = var.tags
 }
 
 resource "local_file" "kubeconfigs" {

diff --git a/test/terraform/aks/variables.tf b/test/terraform/aks/variables.tf
@@ -17,3 +17,9 @@ variable "cluster_count" {
   default     = 1
   description = "The number of Kubernetes clusters to create."
 }
+
+variable "tags" {
+  type = map
+  default = {}
+  description = "Tags to attach to the created resources."
+}
diff --git a/test/terraform/eks/main.tf b/test/terraform/eks/main.tf
@@ -43,6 +43,8 @@ module "vpc" {
     "kubernetes.io/cluster/consul-k8s-${random_id.suffix[count.index].dec}" = "shared"
     "kubernetes.io/role/internal-elb"                                       = "1"
   }
+
+  tags = var.tags
 }
 
 module "eks" {
@@ -70,6 +72,8 @@ module "eks" {
   manage_aws_auth    = false
   write_kubeconfig   = true
   config_output_path = pathexpand("~/.kube/consul-k8s-${random_id.suffix[count.index].dec}")
+
+  tags = var.tags
 }
 
 data "aws_eks_cluster" "cluster" {

diff --git a/test/terraform/eks/variables.tf b/test/terraform/eks/variables.tf
@@ -11,4 +11,10 @@ variable "cluster_count" {
 variable "role_arn" {
   default     = ""
   description = "AWS role for the AWS provider to assume when running these templates."
-}
+}
+
+variable "tags" {
+  type = map
+  default = {}
+  description = "Tags to attach to the created resources."
+}
diff --git a/test/terraform/gke/main.tf b/test/terraform/gke/main.tf
@@ -28,6 +28,8 @@ resource "google_container_cluster" "cluster" {
   pod_security_policy_config {
     enabled = true
   }
+
+  resource_labels = var.labels
 }
 
 resource "null_resource" "kubectl" {

diff --git a/test/terraform/gke/variables.tf b/test/terraform/gke/variables.tf
@@ -20,3 +20,9 @@ variable "cluster_count" {
   default     = 1
   description = "The number of Kubernetes clusters to create."
 }
+
+variable "labels" {
+  type = map
+  default = {}
+  description = "Labels to attach to the created resources."
+}