From 963ec0214c59be79d0f9ae84836254724c45ef96 Mon Sep 17 00:00:00 2001 From: Michael Wilkerson <62034708+wilkermichael@users.noreply.github.com> Date: Thu, 27 Jul 2023 13:48:26 -0700 Subject: [PATCH] Mw/net 4260 phase 2 automate the k8s sameness tests (#2579) * add kustomize files - These reflect the different test cases - sameness.yaml defines the ordered list of failovers - static-server responds with a unique name so we can track failover order - static-client includes both DNS and CURL in the image used so we can exec in for testing * add sameness tests - We do a bunch of infra setup for peering and partitions, but after the initial setup only partitions are tested - We test service failover, dns failover and PQ failover scenarios * add 4 kind clusters to make target - The sameness tests require 4 kind clusters, so the make target will now spin up 4 kind clusters - not all tests need 4 kind clusters, but the entire suite of tests can be run with 4 * increase kubectl timeout to 90s - add variable for configuring timeout - timeout was triggering locally on intel mac machine, so this timeout should cover our devs lowest performing machines * add sameness test to test packages * Fix comments on partition connect test --- Makefile | 16 +- .../kind_acceptance_test_packages.yaml | 9 +- acceptance/framework/k8s/kubectl.go | 15 +- .../sameness/default-ns/kustomization.yaml | 5 + .../bases/sameness/default-ns/sameness.yaml | 12 + .../exportedservices-ap1.yaml | 9 + .../exportedservices-ap1/kustomization.yaml | 5 + .../sameness/override-ns/intentions.yaml | 12 + .../sameness/override-ns/kustomization.yaml | 7 + .../override-ns/payment-service-resolver.yaml | 13 + .../override-ns/service-defaults.yaml | 6 + .../bases/sameness/peering/kustomization.yaml | 5 + .../fixtures/bases/sameness/peering/mesh.yaml | 7 + .../ap1-partition/kustomization.yaml | 8 + .../ap1-partition/patch.yaml | 16 + .../default-partition/kustomization.yaml | 8 + .../default-partition/patch.yaml | 16 + .../static-client/default/kustomization.yaml | 8 + .../sameness/static-client/default/patch.yaml | 22 + .../partition/kustomization.yaml | 8 + .../static-client/partition/patch.yaml | 22 + .../static-server/default/kustomization.yaml | 8 + .../sameness/static-server/default/patch.yaml | 23 + .../partition/kustomization.yaml | 8 + .../static-server/partition/patch.yaml | 23 + .../partitions/partitions_connect_test.go | 4 +- acceptance/tests/sameness/main_test.go | 28 ++ acceptance/tests/sameness/sameness_test.go | 454 ++++++++++++++++++ 28 files changed, 764 insertions(+), 13 deletions(-) create mode 100644 acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml create mode 100644 acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml create mode 100644 acceptance/tests/sameness/main_test.go create mode 100644 acceptance/tests/sameness/sameness_test.go diff --git a/Makefile b/Makefile index 4289b81b9b..e2c39de2ea 100644 --- a/Makefile +++ b/Makefile @@ -130,22 +130,26 @@ kind-cni-calico: kubectl create -f $(CURDIR)/acceptance/framework/environment/cni-kind/custom-resources.yaml @sleep 20 -# Helper target for doing local cni acceptance testing -kind-cni: +kind-delete: kind delete cluster --name dc1 kind delete cluster --name dc2 + kind delete cluster --name dc3 + kind delete cluster --name dc4 + + +# Helper target for doing local cni acceptance testing +kind-cni: kind-delete kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc1 --image $(KIND_NODE_IMAGE) make kind-cni-calico kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc2 --image $(KIND_NODE_IMAGE) make kind-cni-calico # Helper target for doing local acceptance testing -kind: - kind delete cluster --name dc1 - kind delete cluster --name dc2 +kind: kind-delete kind create cluster --name dc1 --image $(KIND_NODE_IMAGE) kind create cluster --name dc2 --image $(KIND_NODE_IMAGE) - + kind create cluster --name dc3 --image $(KIND_NODE_IMAGE) + kind create cluster --name dc4 --image $(KIND_NODE_IMAGE) # ===========> Shared Targets diff --git a/acceptance/ci-inputs/kind_acceptance_test_packages.yaml b/acceptance/ci-inputs/kind_acceptance_test_packages.yaml index 8677b83c4e..e0e126bbda 100644 --- a/acceptance/ci-inputs/kind_acceptance_test_packages.yaml +++ b/acceptance/ci-inputs/kind_acceptance_test_packages.yaml @@ -3,7 +3,8 @@ - {runner: 0, test-packages: "partitions"} - {runner: 1, test-packages: "peering"} -- {runner: 2, test-packages: "connect snapshot-agent wan-federation"} -- {runner: 3, test-packages: "cli vault metrics"} -- {runner: 4, test-packages: "api-gateway ingress-gateway sync example consul-dns"} -- {runner: 5, test-packages: "config-entries terminating-gateway basic"} \ No newline at end of file +- {runner: 2, test-packages: "sameness"} +- {runner: 3, test-packages: "connect snapshot-agent wan-federation"} +- {runner: 4, test-packages: "cli vault metrics"} +- {runner: 5, test-packages: "api-gateway ingress-gateway sync example consul-dns"} +- {runner: 6, test-packages: "config-entries terminating-gateway basic"} diff --git a/acceptance/framework/k8s/kubectl.go b/acceptance/framework/k8s/kubectl.go index ea90212e04..acfedbdb3d 100644 --- a/acceptance/framework/k8s/kubectl.go +++ b/acceptance/framework/k8s/kubectl.go @@ -4,6 +4,7 @@ package k8s import ( + "fmt" "strings" "testing" "time" @@ -16,6 +17,10 @@ import ( "github.com/stretchr/testify/require" ) +const ( + kubectlTimeout = "--timeout=120s" +) + // kubeAPIConnectErrs are errors that sometimes occur when talking to the // Kubernetes API related to connection issues. var kubeAPIConnectErrs = []string{ @@ -97,7 +102,7 @@ func KubectlApplyK(t *testing.T, options *k8s.KubectlOptions, kustomizeDir strin // deletes it from the cluster by running 'kubectl delete -f'. // If there's an error deleting the file, fail the test. func KubectlDelete(t *testing.T, options *k8s.KubectlOptions, configPath string) { - _, err := RunKubectlAndGetOutputE(t, options, "delete", "--timeout=60s", "-f", configPath) + _, err := RunKubectlAndGetOutputE(t, options, "delete", kubectlTimeout, "-f", configPath) require.NoError(t, err) } @@ -107,7 +112,13 @@ func KubectlDelete(t *testing.T, options *k8s.KubectlOptions, configPath string) func KubectlDeleteK(t *testing.T, options *k8s.KubectlOptions, kustomizeDir string) { // Ignore not found errors because Kubernetes automatically cleans up the kube secrets that we deployed // referencing the ServiceAccount when it is deleted. - _, err := RunKubectlAndGetOutputE(t, options, "delete", "--timeout=60s", "--ignore-not-found", "-k", kustomizeDir) + _, err := RunKubectlAndGetOutputE(t, options, "delete", kubectlTimeout, "--ignore-not-found", "-k", kustomizeDir) + require.NoError(t, err) +} + +// KubectlScale takes a deployment and scales it to the provided number of replicas. +func KubectlScale(t *testing.T, options *k8s.KubectlOptions, deployment string, replicas int) { + _, err := RunKubectlAndGetOutputE(t, options, "scale", kubectlTimeout, fmt.Sprintf("--replicas=%d", replicas), deployment) require.NoError(t, err) } diff --git a/acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml new file mode 100644 index 0000000000..3f9d23c28a --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - sameness.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml b/acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml new file mode 100644 index 0000000000..0eb7d9e008 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml @@ -0,0 +1,12 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: SamenessGroup +metadata: + name: mine +spec: + members: + - partition: default + - partition: ap1 + - peer: cluster-01-a + - peer: cluster-01-b + - peer: cluster-02-a + - peer: cluster-03-a diff --git a/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml new file mode 100644 index 0000000000..3dc494dd43 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ExportedServices +metadata: + name: ap1 +spec: + services: [] diff --git a/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml new file mode 100644 index 0000000000..1793fa6db7 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - exportedservices-ap1.yaml diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml new file mode 100644 index 0000000000..425b9fe21d --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml @@ -0,0 +1,12 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: static-server +spec: + destination: + name: static-server + sources: + - name: static-client + namespace: ns1 + samenessGroup: mine + action: allow diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml new file mode 100644 index 0000000000..adfd1c827b --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml @@ -0,0 +1,7 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - intentions.yaml + - payment-service-resolver.yaml + - service-defaults.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml new file mode 100644 index 0000000000..b2b6b68c3d --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml @@ -0,0 +1,13 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceResolver +metadata: + name: static-server +spec: + connectTimeout: 15s + failover: + '*': + samenessGroup: mine + policy: + mode: order-by-locality + regions: + - us-west-2 diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml new file mode 100644 index 0000000000..f88d143728 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml @@ -0,0 +1,6 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: static-server +spec: + protocol: http \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml new file mode 100644 index 0000000000..926e91236d --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - mesh.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml b/acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml new file mode 100644 index 0000000000..de84382d3e --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml @@ -0,0 +1,7 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: Mesh +metadata: + name: mesh +spec: + peering: + peerThroughMeshGateways: true diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml new file mode 100644 index 0000000000..2a2f47a332 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/sameness/exportedservices-ap1 + +patchesStrategicMerge: +- patch.yaml diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml new file mode 100644 index 0000000000..d71e8211ba --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml @@ -0,0 +1,16 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ExportedServices +metadata: + name: ap1 +spec: + services: + - name: static-server + namespace: ns2 + consumers: + - samenessGroup: mine + - name: mesh-gateway + consumers: + - samenessGroup: mine diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml new file mode 100644 index 0000000000..05de6151fc --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/exportedservices-default + +patchesStrategicMerge: +- patch.yaml diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml new file mode 100644 index 0000000000..9bb440637e --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml @@ -0,0 +1,16 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ExportedServices +metadata: + name: default +spec: + services: + - name: static-server + namespace: ns2 + consumers: + - samenessGroup: mine + - name: mesh-gateway + consumers: + - samenessGroup: mine diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml new file mode 100644 index 0000000000..227f223c9f --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml new file mode 100644 index 0000000000..1775e9abb1 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml @@ -0,0 +1,22 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'static-server.ns2.default:8080' + spec: + containers: + - name: static-client + image: anubhavmishra/tiny-tools:latest + # Just spin & wait forever, we'll use `kubectl exec` to demo + command: ['/bin/sh', '-c', '--'] + args: ['while true; do sleep 30; done;'] + # If ACLs are enabled, the serviceAccountName must match the Consul service name. + serviceAccountName: static-client \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml new file mode 100644 index 0000000000..227f223c9f --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml new file mode 100644 index 0000000000..c1a14c6070 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml @@ -0,0 +1,22 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'static-server.ns2.ap1:8080' + spec: + containers: + - name: static-client + image: anubhavmishra/tiny-tools:latest + # Just spin & wait forever, we'll use `kubectl exec` to demo + command: ['/bin/sh', '-c', '--'] + args: ['while true; do sleep 30; done;'] + # If ACLs are enabled, the serviceAccountName must match the Consul service name. + serviceAccountName: static-client diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml new file mode 100644 index 0000000000..c15bfe7ba7 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-server + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml new file mode 100644 index 0000000000..ca27b7ba42 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml @@ -0,0 +1,23 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + containers: + - name: static-server + image: docker.mirror.hashicorp.services/hashicorp/http-echo:alpine + args: + - -text="cluster-01-a" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + serviceAccountName: static-server diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml new file mode 100644 index 0000000000..c15bfe7ba7 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-server + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml new file mode 100644 index 0000000000..044115d1d1 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml @@ -0,0 +1,23 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + containers: + - name: static-server + image: docker.mirror.hashicorp.services/hashicorp/http-echo:alpine + args: + - -text="cluster-01-b" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + serviceAccountName: static-server diff --git a/acceptance/tests/partitions/partitions_connect_test.go b/acceptance/tests/partitions/partitions_connect_test.go index 6b103614c7..c53e5b6171 100644 --- a/acceptance/tests/partitions/partitions_connect_test.go +++ b/acceptance/tests/partitions/partitions_connect_test.go @@ -108,6 +108,7 @@ func TestPartitions_Connect(t *testing.T) { "dns.enableRedirection": strconv.FormatBool(cfg.EnableTransparentProxy), } + // Setup the default partition defaultPartitionHelmValues := make(map[string]string) // On Kind, there are no load balancers but since all clusters @@ -129,6 +130,7 @@ func TestPartitions_Connect(t *testing.T) { serverConsulCluster := consul.NewHelmCluster(t, defaultPartitionHelmValues, defaultPartitionClusterContext, cfg, releaseName) serverConsulCluster.Create(t) + // Copy secrets from the default partition to the secondary partition // Get the TLS CA certificate and key secret from the server cluster and apply it to the client cluster. caCertSecretName := fmt.Sprintf("%s-consul-ca-cert", releaseName) @@ -146,7 +148,7 @@ func TestPartitions_Connect(t *testing.T) { k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, secondaryPartitionClusterContext) - // Create client cluster. + // Create secondary partition cluster. secondaryPartitionHelmValues := map[string]string{ "global.enabled": "false", diff --git a/acceptance/tests/sameness/main_test.go b/acceptance/tests/sameness/main_test.go new file mode 100644 index 0000000000..67e6ee42b7 --- /dev/null +++ b/acceptance/tests/sameness/main_test.go @@ -0,0 +1,28 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package sameness + +import ( + "fmt" + "os" + "testing" + + testsuite "github.com/hashicorp/consul-k8s/acceptance/framework/suite" +) + +var suite testsuite.Suite + +func TestMain(m *testing.M) { + suite = testsuite.NewSuite(m) + + expectedNumberOfClusters := 4 + + if suite.Config().EnableMultiCluster && suite.Config().IsExpectedClusterCount(expectedNumberOfClusters) && suite.Config().UseKind { + os.Exit(suite.Run()) + } else { + fmt.Println(fmt.Sprintf("Skipping sameness tests because either -enable-multi-cluster is "+ + "not set, the number of clusters did not match the expected count of %d, or --useKind is false. "+ + "Sameness acceptance tests are currently only suopported on Kind clusters", expectedNumberOfClusters)) + } +} diff --git a/acceptance/tests/sameness/sameness_test.go b/acceptance/tests/sameness/sameness_test.go new file mode 100644 index 0000000000..a7a926cd42 --- /dev/null +++ b/acceptance/tests/sameness/sameness_test.go @@ -0,0 +1,454 @@ +package sameness + +import ( + "context" + "fmt" + "strconv" + "testing" + + terratestk8s "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/config" + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + primaryDatacenterPartition = "ap1" + primaryServerDatacenter = "dc1" + peer1Datacenter = "dc2" + peer2Datacenter = "dc3" + staticClientNamespace = "ns1" + staticServerNamespace = "ns2" + + keyPrimaryServer = "server" + keyPartition = "partition" + keyPeer1 = "peer1" + keyPeer2 = "peer2" + + staticServerDeployment = "deploy/static-server" + staticClientDeployment = "deploy/static-client" + + primaryServerClusterName = "cluster-01-a" + partitionClusterName = "cluster-01-b" +) + +func TestFailover_Connect(t *testing.T) { + env := suite.Environment() + cfg := suite.Config() + + if !cfg.EnableEnterprise { + t.Skipf("skipping this test because -enable-enterprise is not set") + } + + cases := []struct { + name string + ACLsEnabled bool + }{ + { + "default failover", + false, + }, + { + "secure failover", + true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + /* + Architecture Overview: + Primary Datacenter (DC1) + Default Partition + Peer -> DC2 (cluster-02-a) + Peer -> DC3 (cluster-03-a) + AP1 Partition + Peer -> DC2 (cluster-02-a) + Peer -> DC3 (cluster-03-a) + Datacenter 2 (DC2) + Default Partition + Peer -> DC1 (cluster-01-a) + Peer -> DC1 (cluster-01-b) + Peer -> DC3 (cluster-03-a) + Datacenter 3 (DC3) + Default Partition + Peer -> DC1 (cluster-01-a) + Peer -> DC1 (cluster-01-b) + Peer -> DC2 (cluster-02-a) + + + Architecture Diagram + failover scenarios from perspective of DC1 Default Partition Static-Server + +-------------------------------------------+ + | | + | DC1 | + | | + | +-----------------------------+ | +-----------------------------------+ + | | | | | DC2 | + | | +------------------+ | | Failover 2 | +------------------+ | + | | | +-------+--------+-----------------+------>| | | + | | | Static-Server | | | | | Static-Server | | + | | | +-------+---+ | | | | | + | | | | | | | | | | | + | | | | | | | | | | | + | | | +-------+---+----+-------------+ | | | | + | | +------------------+ | | | | | +------------------+ | + | | Admin Partitions: Default | | | | | | + | | Name: cluster-01-a | | | | | Admin Partitions: Default | + | | | | | | | Name: cluster-02-a | + | +-----------------------------+ | | | | | + | | | | +-----------------------------------+ + | Failover 1| | Failover 3 | + | +-------------------------------+ | | | +-----------------------------------+ + | | | | | | | DC3 | + | | +------------------+ | | | | | +------------------+ | + | | | | | | | | | | Static-Server | | + | | | Static-Server | | | | | | | | | + | | | | | | | | | | | | + | | | | | | | +---+------>| | | + | | | |<------+--+ | | | | | + | | | | | | | +------------------+ | + | | +------------------+ | | | | + | | Admin Partitions: ap1 | | | Admin Partitions: Default | + | | Name: cluster-01-b | | | Name: cluster-03-a | + | | | | | | + | +-------------------------------+ | | | + | | +-----------------------------------+ + +-------------------------------------------+ + */ + + members := map[string]*member{ + keyPrimaryServer: {context: env.DefaultContext(t), hasServer: true}, + keyPartition: {context: env.Context(t, 1), hasServer: false}, + keyPeer1: {context: env.Context(t, 2), hasServer: true}, + keyPeer2: {context: env.Context(t, 3), hasServer: true}, + } + + // Setup Namespaces. + for _, v := range members { + createNamespaces(t, cfg, v.context) + } + + // Create the Default Cluster. + commonHelmValues := map[string]string{ + "global.peering.enabled": "true", + + "global.tls.enabled": "true", + "global.tls.httpsOnly": strconv.FormatBool(c.ACLsEnabled), + + "global.enableConsulNamespaces": "true", + + "global.adminPartitions.enabled": "true", + + "global.logLevel": "debug", + + "global.acls.manageSystemACLs": strconv.FormatBool(c.ACLsEnabled), + + "connectInject.enabled": "true", + "connectInject.consulNamespaces.mirroringK8S": "true", + + "meshGateway.enabled": "true", + "meshGateway.replicas": "1", + + "dns.enabled": "true", + } + + defaultPartitionHelmValues := map[string]string{ + "global.datacenter": primaryServerDatacenter, + } + + // On Kind, there are no load balancers but since all clusters + // share the same node network (docker bridge), we can use + // a NodePort service so that we can access node(s) in a different Kind cluster. + if cfg.UseKind { + defaultPartitionHelmValues["meshGateway.service.type"] = "NodePort" + defaultPartitionHelmValues["meshGateway.service.nodePort"] = "30200" + defaultPartitionHelmValues["server.exposeService.type"] = "NodePort" + defaultPartitionHelmValues["server.exposeService.nodePort.https"] = "30000" + defaultPartitionHelmValues["server.exposeService.nodePort.grpc"] = "30100" + } + helpers.MergeMaps(defaultPartitionHelmValues, commonHelmValues) + + releaseName := helpers.RandomName() + members[keyPrimaryServer].helmCluster = consul.NewHelmCluster(t, defaultPartitionHelmValues, members[keyPrimaryServer].context, cfg, releaseName) + members[keyPrimaryServer].helmCluster.Create(t) + + // Get the TLS CA certificate and key secret from the server cluster and apply it to the client cluster. + caCertSecretName := fmt.Sprintf("%s-consul-ca-cert", releaseName) + + logger.Logf(t, "retrieving ca cert secret %s from the server cluster and applying to the client cluster", caCertSecretName) + k8s.CopySecret(t, members[keyPrimaryServer].context, members[keyPartition].context, caCertSecretName) + + // Create Secondary Partition Cluster which will apply the primary datacenter. + partitionToken := fmt.Sprintf("%s-consul-partitions-acl-token", releaseName) + if c.ACLsEnabled { + logger.Logf(t, "retrieving partition token secret %s from the server cluster and applying to the client cluster", partitionToken) + k8s.CopySecret(t, members[keyPrimaryServer].context, members[keyPartition].context, partitionToken) + } + + partitionServiceName := fmt.Sprintf("%s-consul-expose-servers", releaseName) + partitionSvcAddress := k8s.ServiceHost(t, cfg, members[keyPrimaryServer].context, partitionServiceName) + + k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, members[keyPartition].context) + + secondaryPartitionHelmValues := map[string]string{ + "global.enabled": "false", + "global.datacenter": primaryServerDatacenter, + + "global.adminPartitions.name": primaryDatacenterPartition, + + "global.tls.caCert.secretName": caCertSecretName, + "global.tls.caCert.secretKey": "tls.crt", + + "externalServers.enabled": "true", + "externalServers.hosts[0]": partitionSvcAddress, + "externalServers.tlsServerName": fmt.Sprintf("server.%s.consul", primaryServerDatacenter), + "global.server.enabled": "false", + } + + if c.ACLsEnabled { + // Setup partition token and auth method host if ACLs enabled. + secondaryPartitionHelmValues["global.acls.bootstrapToken.secretName"] = partitionToken + secondaryPartitionHelmValues["global.acls.bootstrapToken.secretKey"] = "token" + secondaryPartitionHelmValues["externalServers.k8sAuthMethodHost"] = k8sAuthMethodHost + } + + if cfg.UseKind { + secondaryPartitionHelmValues["externalServers.httpsPort"] = "30000" + secondaryPartitionHelmValues["externalServers.grpcPort"] = "30100" + secondaryPartitionHelmValues["meshGateway.service.type"] = "NodePort" + secondaryPartitionHelmValues["meshGateway.service.nodePort"] = "30200" + } + helpers.MergeMaps(secondaryPartitionHelmValues, commonHelmValues) + + members[keyPartition].helmCluster = consul.NewHelmCluster(t, secondaryPartitionHelmValues, members[keyPartition].context, cfg, releaseName) + members[keyPartition].helmCluster.Create(t) + + // Create Peer 1 Cluster. + PeerOneHelmValues := map[string]string{ + "global.datacenter": peer1Datacenter, + } + + if cfg.UseKind { + PeerOneHelmValues["server.exposeGossipAndRPCPorts"] = "true" + PeerOneHelmValues["meshGateway.service.type"] = "NodePort" + PeerOneHelmValues["meshGateway.service.nodePort"] = "30100" + } + helpers.MergeMaps(PeerOneHelmValues, commonHelmValues) + + members[keyPeer1].helmCluster = consul.NewHelmCluster(t, PeerOneHelmValues, members[keyPeer1].context, cfg, releaseName) + members[keyPeer1].helmCluster.Create(t) + + // Create Peer 2 Cluster. + PeerTwoHelmValues := map[string]string{ + "global.datacenter": peer2Datacenter, + } + + if cfg.UseKind { + PeerTwoHelmValues["server.exposeGossipAndRPCPorts"] = "true" + PeerTwoHelmValues["meshGateway.service.type"] = "NodePort" + PeerTwoHelmValues["meshGateway.service.nodePort"] = "30100" + } + helpers.MergeMaps(PeerTwoHelmValues, commonHelmValues) + + members[keyPeer2].helmCluster = consul.NewHelmCluster(t, PeerTwoHelmValues, members[keyPeer2].context, cfg, releaseName) + members[keyPeer2].helmCluster.Create(t) + + // Create a ProxyDefaults resource to configure services to use the mesh + // gateways and set server and client opts. + for k, v := range members { + logger.Logf(t, "applying resources on %s", v.context.KubectlOptions(t).ContextName) + + // Client will use the client namespace. + members[k].clientOpts = &terratestk8s.KubectlOptions{ + ContextName: v.context.KubectlOptions(t).ContextName, + ConfigPath: v.context.KubectlOptions(t).ConfigPath, + Namespace: staticClientNamespace, + } + + // Server will use the server namespace. + members[k].serverOpts = &terratestk8s.KubectlOptions{ + ContextName: v.context.KubectlOptions(t).ContextName, + ConfigPath: v.context.KubectlOptions(t).ConfigPath, + Namespace: staticServerNamespace, + } + + // Sameness Defaults need to be applied first so that the sameness group exists. + applyResources(t, cfg, "../fixtures/bases/mesh-gateway", members[k].context.KubectlOptions(t)) + applyResources(t, cfg, "../fixtures/bases/sameness/default-ns", members[k].context.KubectlOptions(t)) + applyResources(t, cfg, "../fixtures/bases/sameness/override-ns", members[k].serverOpts) + + // Only assign a client if the cluster is running a Consul server. + if v.hasServer { + members[k].client, _ = members[k].helmCluster.SetupConsulClient(t, c.ACLsEnabled) + } + } + + // TODO: Add further setup for peering, right now the rest of this test will only cover Partitions + // Create static server deployments. + logger.Log(t, "creating static-server and static-client deployments") + k8s.DeployKustomize(t, members[keyPrimaryServer].serverOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-server/default") + k8s.DeployKustomize(t, members[keyPartition].serverOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-server/partition") + + // Create static client deployments. + k8s.DeployKustomize(t, members[keyPrimaryServer].clientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-client/default") + k8s.DeployKustomize(t, members[keyPartition].clientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-client/partition") + + // Verify that both static-server and static-client have been injected and now have 2 containers in server cluster. + // Also get the server IP + for _, labelSelector := range []string{"app=static-server", "app=static-client"} { + podList, err := members[keyPrimaryServer].context.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), + metav1.ListOptions{LabelSelector: labelSelector}) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Len(t, podList.Items[0].Spec.Containers, 2) + if labelSelector == "app=static-server" { + ip := &podList.Items[0].Status.PodIP + require.NotNil(t, ip) + logger.Logf(t, "default-static-server-ip: %s", *ip) + members[keyPrimaryServer].staticServerIP = ip + } + + podList, err = members[keyPartition].context.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), + metav1.ListOptions{LabelSelector: labelSelector}) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Len(t, podList.Items[0].Spec.Containers, 2) + if labelSelector == "app=static-server" { + ip := &podList.Items[0].Status.PodIP + require.NotNil(t, ip) + logger.Logf(t, "partition-static-server-ip: %s", *ip) + members[keyPartition].staticServerIP = ip + } + } + + logger.Log(t, "creating exported services") + applyResources(t, cfg, "../fixtures/cases/sameness/exported-services/default-partition", members[keyPrimaryServer].context.KubectlOptions(t)) + applyResources(t, cfg, "../fixtures/cases/sameness/exported-services/ap1-partition", members[keyPartition].context.KubectlOptions(t)) + + // Setup DNS. + dnsService, err := members[keyPrimaryServer].context.KubernetesClient(t).CoreV1().Services("default").Get(context.Background(), fmt.Sprintf("%s-%s", releaseName, "consul-dns"), metav1.GetOptions{}) + require.NoError(t, err) + dnsIP := dnsService.Spec.ClusterIP + logger.Logf(t, "dnsIP: %s", dnsIP) + + // Setup Prepared Query. + definition := &api.PreparedQueryDefinition{ + Name: "my-query", + Service: api.ServiceQuery{ + Service: "static-server", + SamenessGroup: "mine", + Namespace: staticServerNamespace, + OnlyPassing: false, + }, + } + resp, _, err := members[keyPrimaryServer].client.PreparedQuery().Create(definition, &api.WriteOptions{}) + require.NoError(t, err) + logger.Logf(t, "PQ ID: %s", resp) + + logger.Log(t, "all infrastructure up and running") + logger.Log(t, "verifying failover scenarios") + + const dnsLookup = "static-server.service.ns2.ns.mine.sg.consul" + const dnsPQLookup = "my-query.query.consul" + + // Verify initial server. + serviceFailoverCheck(t, primaryServerClusterName, members[keyPrimaryServer]) + + // Verify initial dns. + dnsFailoverCheck(t, releaseName, dnsIP, dnsLookup, members[keyPrimaryServer], members[keyPrimaryServer]) + + // Verify initial dns with PQ. + dnsFailoverCheck(t, releaseName, dnsIP, dnsPQLookup, members[keyPrimaryServer], members[keyPrimaryServer]) + + // Scale down static-server on the server, will fail over to partition. + k8s.KubectlScale(t, members[keyPrimaryServer].serverOpts, staticServerDeployment, 0) + + // Verify failover to partition. + serviceFailoverCheck(t, partitionClusterName, members[keyPrimaryServer]) + + // Verify dns failover to partition. + dnsFailoverCheck(t, releaseName, dnsIP, dnsLookup, members[keyPrimaryServer], members[keyPartition]) + + // Verify prepared query failover. + dnsFailoverCheck(t, releaseName, dnsIP, dnsPQLookup, members[keyPrimaryServer], members[keyPartition]) + + logger.Log(t, "tests complete") + }) + } +} + +type member struct { + context environment.TestContext + helmCluster *consul.HelmCluster + client *api.Client + hasServer bool + serverOpts *terratestk8s.KubectlOptions + clientOpts *terratestk8s.KubectlOptions + staticServerIP *string +} + +func createNamespaces(t *testing.T, cfg *config.TestConfig, context environment.TestContext) { + logger.Logf(t, "creating namespaces in %s", context.KubectlOptions(t).ContextName) + k8s.RunKubectl(t, context.KubectlOptions(t), "create", "ns", staticServerNamespace) + k8s.RunKubectl(t, context.KubectlOptions(t), "create", "ns", staticClientNamespace) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.RunKubectl(t, context.KubectlOptions(t), "delete", "ns", staticClientNamespace, staticServerNamespace) + }) +} + +func applyResources(t *testing.T, cfg *config.TestConfig, kustomizeDir string, opts *terratestk8s.KubectlOptions) { + k8s.KubectlApplyK(t, opts, kustomizeDir) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, opts, kustomizeDir) + }) +} + +// serviceFailoverCheck verifies that the server failed over as expected by checking that curling the `static-server` +// using the `static-client` responds with the expected cluster name. Each static-server responds with a uniquue +// name so that we can verify failover occured as expected. +func serviceFailoverCheck(t *testing.T, expectedClusterName string, server *member) { + retry.Run(t, func(r *retry.R) { + resp, err := k8s.RunKubectlAndGetOutputE(t, server.clientOpts, "exec", "-i", + staticClientDeployment, "-c", "static-client", "--", "curl", "localhost:8080") + require.NoError(r, err) + assert.Contains(r, resp, expectedClusterName) + logger.Log(t, resp) + }) +} + +func dnsFailoverCheck(t *testing.T, releaseName string, dnsIP string, dnsQuery string, server, failover *member) { + retry.Run(t, func(r *retry.R) { + logs, err := k8s.RunKubectlAndGetOutputE(t, server.clientOpts, "exec", "-i", + staticClientDeployment, "-c", "static-client", "--", "dig", fmt.Sprintf("@%s-consul-dns.default", releaseName), dnsQuery) + require.NoError(r, err) + + // When the `dig` request is successful, a section of its response looks like the following: + // + // ;; ANSWER SECTION: + // static-server.service.mine.sg.ns2.ns.consul. 0 IN A + // + // ;; Query time: 2 msec + // ;; SERVER: #() + // ;; WHEN: Mon Aug 10 15:02:40 UTC 2020 + // ;; MSG SIZE rcvd: 98 + // + // We assert on the existence of the ANSWER SECTION, The consul-server IPs being present in + // the ANSWER SECTION and the DNS IP mentioned in the SERVER: field + + assert.Contains(r, logs, fmt.Sprintf("SERVER: %s", dnsIP)) + assert.Contains(r, logs, "ANSWER SECTION:") + assert.Contains(r, logs, *failover.staticServerIP) + }) +}