Mw/net 4260 phase 2 automate the k8s sameness tests (#2579)

* add kustomize files
- These reflect the different test cases
- sameness.yaml defines the ordered list of failovers
- static-server responds with a unique name so we can track failover order
- static-client includes both DNS and CURL in the image used so we can exec in for testing

* add sameness tests
- We do a bunch of infra setup for peering and partitions, but after the initial setup only partitions are tested
- We test service failover, dns failover and PQ failover scenarios

* add 4 kind clusters to make target
- The sameness tests require 4 kind clusters, so the make target will now spin up 4 kind clusters
- not all tests need 4 kind clusters, but the entire suite of tests can be run with 4

* increase kubectl timeout to 90s
- add variable for configuring timeout
- timeout was triggering locally on intel mac machine, so this timeout should cover our devs lowest performing machines

* add sameness test to test packages

* Fix comments on partition connect test

Makefile

@@ -130,22 +130,26 @@ kind-cni-calico:
 	kubectl create -f $(CURDIR)/acceptance/framework/environment/cni-kind/custom-resources.yaml
 	@sleep 20
 
-# Helper target for doing local cni acceptance testing
-kind-cni:
+kind-delete:
 	kind delete cluster --name dc1
 	kind delete cluster --name dc2
+	kind delete cluster --name dc3
+	kind delete cluster --name dc4
+
+
+# Helper target for doing local cni acceptance testing
+kind-cni: kind-delete
 	kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc1 --image $(KIND_NODE_IMAGE)
 	make kind-cni-calico
 	kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc2 --image $(KIND_NODE_IMAGE)
 	make kind-cni-calico
 
 # Helper target for doing local acceptance testing
-kind:
-	kind delete cluster --name dc1
-	kind delete cluster --name dc2
+kind: kind-delete
 	kind create cluster --name dc1 --image $(KIND_NODE_IMAGE)
 	kind create cluster --name dc2 --image $(KIND_NODE_IMAGE)
-
+	kind create cluster --name dc3 --image $(KIND_NODE_IMAGE)
+	kind create cluster --name dc4 --image $(KIND_NODE_IMAGE)
 
 # ===========> Shared Targets
 

acceptance/ci-inputs/kind_acceptance_test_packages.yaml

@@ -3,7 +3,8 @@
 
 - {runner: 0, test-packages: "partitions"}
 - {runner: 1, test-packages: "peering"}
-- {runner: 2, test-packages: "connect snapshot-agent wan-federation"}
-- {runner: 3, test-packages: "cli vault metrics"}
-- {runner: 4, test-packages: "api-gateway ingress-gateway sync example consul-dns"}
-- {runner: 5, test-packages: "config-entries terminating-gateway basic"}
+- {runner: 2, test-packages: "sameness"}
+- {runner: 3, test-packages: "connect snapshot-agent wan-federation"}
+- {runner: 4, test-packages: "cli vault metrics"}
+- {runner: 5, test-packages: "api-gateway ingress-gateway sync example consul-dns"}
+- {runner: 6, test-packages: "config-entries terminating-gateway basic"}

acceptance/framework/k8s/kubectl.go

@@ -4,6 +4,7 @@
 package k8s
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -16,6 +17,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const (
+	kubectlTimeout = "--timeout=120s"
+)
+
 // kubeAPIConnectErrs are errors that sometimes occur when talking to the
 // Kubernetes API related to connection issues.
 var kubeAPIConnectErrs = []string{
@@ -97,7 +102,7 @@ func KubectlApplyK(t *testing.T, options *k8s.KubectlOptions, kustomizeDir strin
 // deletes it from the cluster by running 'kubectl delete -f'.
 // If there's an error deleting the file, fail the test.
 func KubectlDelete(t *testing.T, options *k8s.KubectlOptions, configPath string) {
-	_, err := RunKubectlAndGetOutputE(t, options, "delete", "--timeout=60s", "-f", configPath)
+	_, err := RunKubectlAndGetOutputE(t, options, "delete", kubectlTimeout, "-f", configPath)
 	require.NoError(t, err)
 }
 
@@ -107,7 +112,13 @@ func KubectlDelete(t *testing.T, options *k8s.KubectlOptions, configPath string)
 func KubectlDeleteK(t *testing.T, options *k8s.KubectlOptions, kustomizeDir string) {
 	// Ignore not found errors because Kubernetes automatically cleans up the kube secrets that we deployed
 	// referencing the ServiceAccount when it is deleted.
-	_, err := RunKubectlAndGetOutputE(t, options, "delete", "--timeout=60s", "--ignore-not-found", "-k", kustomizeDir)
+	_, err := RunKubectlAndGetOutputE(t, options, "delete", kubectlTimeout, "--ignore-not-found", "-k", kustomizeDir)
+	require.NoError(t, err)
+}
+
+// KubectlScale takes a deployment and scales it to the provided number of replicas.
+func KubectlScale(t *testing.T, options *k8s.KubectlOptions, deployment string, replicas int) {
+	_, err := RunKubectlAndGetOutputE(t, options, "scale", kubectlTimeout, fmt.Sprintf("--replicas=%d", replicas), deployment)
 	require.NoError(t, err)
 }
 

acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml

@@ -0,0 +1,5 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - sameness.yaml

acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml

@@ -0,0 +1,12 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: SamenessGroup
+metadata:
+  name: mine
+spec:
+  members:
+    - partition: default
+    - partition: ap1
+    - peer: cluster-01-a
+    - peer: cluster-01-b
+    - peer: cluster-02-a
+    - peer: cluster-03-a

acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml

@@ -0,0 +1,9 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ExportedServices
+metadata:
+  name: ap1
+spec:
+  services: []

acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml

@@ -0,0 +1,5 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - exportedservices-ap1.yaml

acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml

@@ -0,0 +1,12 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceIntentions
+metadata:
+  name: static-server
+spec:
+  destination:
+    name: static-server
+  sources:
+    - name: static-client
+      namespace: ns1
+      samenessGroup: mine
+      action: allow

acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml

@@ -0,0 +1,7 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - intentions.yaml
+  - payment-service-resolver.yaml
+  - service-defaults.yaml

acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml

@@ -0,0 +1,13 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceResolver
+metadata:
+  name: static-server
+spec:
+  connectTimeout: 15s
+  failover:
+    '*':
+      samenessGroup: mine
+      policy:
+        mode: order-by-locality
+        regions:
+          - us-west-2

acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml

@@ -0,0 +1,6 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ServiceDefaults
+metadata:
+  name: static-server
+spec:
+  protocol: http

acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml

@@ -0,0 +1,5 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - mesh.yaml

acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml

@@ -0,0 +1,7 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: Mesh
+metadata:
+  name: mesh
+spec:
+  peering:
+    peerThroughMeshGateways: true

acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - ../../../../bases/sameness/exportedservices-ap1
+
+patchesStrategicMerge:
+- patch.yaml

acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml

@@ -0,0 +1,16 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ExportedServices
+metadata:
+  name: ap1
+spec:
+  services:
+    - name: static-server
+      namespace: ns2
+      consumers:
+        - samenessGroup: mine
+    - name: mesh-gateway
+      consumers:
+        - samenessGroup: mine

acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - ../../../../bases/exportedservices-default
+
+patchesStrategicMerge:
+- patch.yaml

acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml

@@ -0,0 +1,16 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ExportedServices
+metadata:
+  name: default
+spec:
+  services:
+    - name: static-server
+      namespace: ns2
+      consumers:
+        - samenessGroup: mine
+    - name: mesh-gateway
+      consumers:
+        - samenessGroup: mine

acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - ../../../../bases/static-client
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml

@@ -0,0 +1,22 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-client
+spec:
+  template:
+    metadata:
+      annotations:
+        'consul.hashicorp.com/connect-inject': 'true'
+        'consul.hashicorp.com/connect-service-upstreams': 'static-server.ns2.default:8080'
+    spec:
+      containers:
+        - name: static-client
+          image: anubhavmishra/tiny-tools:latest
+          # Just spin & wait forever, we'll use `kubectl exec` to demo
+          command: ['/bin/sh', '-c', '--']
+          args: ['while true; do sleep 30; done;']
+      # If ACLs are enabled, the serviceAccountName must match the Consul service name.
+      serviceAccountName: static-client

acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - ../../../../bases/static-client
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml

@@ -0,0 +1,22 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-client
+spec:
+  template:
+    metadata:
+      annotations:
+        'consul.hashicorp.com/connect-inject': 'true'
+        'consul.hashicorp.com/connect-service-upstreams': 'static-server.ns2.ap1:8080'
+    spec:
+      containers:
+        - name: static-client
+          image: anubhavmishra/tiny-tools:latest
+          # Just spin & wait forever, we'll use `kubectl exec` to demo
+          command: ['/bin/sh', '-c', '--']
+          args: ['while true; do sleep 30; done;']
+      # If ACLs are enabled, the serviceAccountName must match the Consul service name.
+      serviceAccountName: static-client

acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - ../../../../bases/static-server
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml

@@ -0,0 +1,23 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-server
+spec:
+  template:
+    metadata:
+      annotations:
+        "consul.hashicorp.com/connect-inject": "true"
+    spec:
+      containers:
+        - name: static-server
+          image: docker.mirror.hashicorp.services/hashicorp/http-echo:alpine
+          args:
+            - -text="cluster-01-a"
+            - -listen=:8080
+          ports:
+            - containerPort: 8080
+              name: http
+      serviceAccountName: static-server

acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+resources:
+  - ../../../../bases/static-server
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml

@@ -0,0 +1,23 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-server
+spec:
+  template:
+    metadata:
+      annotations:
+        "consul.hashicorp.com/connect-inject": "true"
+    spec:
+      containers:
+        - name: static-server
+          image: docker.mirror.hashicorp.services/hashicorp/http-echo:alpine
+          args:
+            - -text="cluster-01-b"
+            - -listen=:8080
+          ports:
+            - containerPort: 8080
+              name: http
+      serviceAccountName: static-server

acceptance/tests/partitions/partitions_connect_test.go

@@ -108,6 +108,7 @@ func TestPartitions_Connect(t *testing.T) {
 				"dns.enableRedirection": strconv.FormatBool(cfg.EnableTransparentProxy),
 			}
 
+			// Setup the default partition
 			defaultPartitionHelmValues := make(map[string]string)
 
 			// On Kind, there are no load balancers but since all clusters
@@ -129,6 +130,7 @@ func TestPartitions_Connect(t *testing.T) {
 			serverConsulCluster := consul.NewHelmCluster(t, defaultPartitionHelmValues, defaultPartitionClusterContext, cfg, releaseName)
 			serverConsulCluster.Create(t)
 
+			// Copy secrets from the default partition to the secondary partition
 			// Get the TLS CA certificate and key secret from the server cluster and apply it to the client cluster.
 			caCertSecretName := fmt.Sprintf("%s-consul-ca-cert", releaseName)
 
@@ -146,7 +148,7 @@ func TestPartitions_Connect(t *testing.T) {
 
 			k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, secondaryPartitionClusterContext)
 
-			// Create client cluster.
+			// Create secondary partition cluster.
 			secondaryPartitionHelmValues := map[string]string{
 				"global.enabled": "false",