Adding Acceptance test for Auto Reload Config (#1135)

* Adding Acceptance test for Auto Reload Config

* removing auto_config_reload from acceptance test configuration

* Setting auto_reload_config to true on client and server config maps

* Add tests for port 8502 and 8300

* removing grpc from test

* fixing checksum in tests for server and client

* fixing some formating and comments

* PR Feedback

acceptance/framework/consul/cli_cluster.go

@@ -194,7 +194,7 @@ func (c *CLICluster) Destroy(t *testing.T) {
 	require.NoError(t, err)
 }
 
-func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
+func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client, string) {
 	t.Helper()
 
 	namespace := c.kubectlOptions.Namespace
@@ -264,7 +264,7 @@ func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
 	consulClient, err := api.NewClient(config)
 	require.NoError(t, err)
 
-	return consulClient
+	return consulClient, config.Address
 }
 
 func createOrUpdateNamespace(t *testing.T, client kubernetes.Interface, namespace string) {

acceptance/framework/consul/cluster.go

@@ -9,7 +9,7 @@ import (
 // Cluster represents a consul cluster object.
 type Cluster interface {
 	// SetupConsulClient returns a new Consul client.
-	SetupConsulClient(t *testing.T, secure bool) *api.Client
+	SetupConsulClient(t *testing.T, secure bool) (*api.Client, string)
 
 	// Create creates a new Consul Cluster.
 	Create(t *testing.T)

acceptance/framework/consul/helm_cluster.go

@@ -276,12 +276,38 @@ func (h *HelmCluster) Upgrade(t *testing.T, helmValues map[string]string) {
 	k8s.WaitForAllPodsToBeReady(t, h.kubernetesClient, h.helmOptions.KubectlOptions.Namespace, fmt.Sprintf("release=%s", h.releaseName))
 }
 
-func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
+func (h *HelmCluster) CreatePortForwardTunnel(t *testing.T, remotePort int) string {
+	localPort := terratestk8s.GetAvailablePort(t)
+	serverPod := fmt.Sprintf("%s-consul-server-0", h.releaseName)
+	tunnel := terratestk8s.NewTunnelWithLogger(
+		h.helmOptions.KubectlOptions,
+		terratestk8s.ResourceTypePod,
+		serverPod,
+		localPort,
+		remotePort,
+		h.logger)
+
+	// Retry creating the port forward since it can fail occasionally.
+	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 3}, t, func(r *retry.R) {
+		// NOTE: It's okay to pass in `t` to ForwardPortE despite being in a retry
+		// because we're using ForwardPortE (not ForwardPort) so the `t` won't
+		// get used to fail the test, just for logging.
+		require.NoError(r, tunnel.ForwardPortE(t))
+	})
+
+	t.Cleanup(func() {
+		tunnel.Close()
+	})
+
+	return fmt.Sprintf("127.0.0.1:%d", localPort)
+
+}
+
+func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api.Client, configAddress string) {
 	t.Helper()
 
 	namespace := h.helmOptions.KubectlOptions.Namespace
 	config := api.DefaultConfig()
-	localPort := terratestk8s.GetAvailablePort(t)
 	remotePort := 8500 // use non-secure by default
 
 	if secure {
@@ -315,32 +341,11 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) *api.Client {
 		}
 	}
 
-	serverPod := fmt.Sprintf("%s-consul-server-0", h.releaseName)
-	tunnel := terratestk8s.NewTunnelWithLogger(
-		h.helmOptions.KubectlOptions,
-		terratestk8s.ResourceTypePod,
-		serverPod,
-		localPort,
-		remotePort,
-		h.logger)
-
-	// Retry creating the port forward since it can fail occasionally.
-	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 3}, t, func(r *retry.R) {
-		// NOTE: It's okay to pass in `t` to ForwardPortE despite being in a retry
-		// because we're using ForwardPortE (not ForwardPort) so the `t` won't
-		// get used to fail the test, just for logging.
-		require.NoError(r, tunnel.ForwardPortE(t))
-	})
-
-	t.Cleanup(func() {
-		tunnel.Close()
-	})
-
-	config.Address = fmt.Sprintf("127.0.0.1:%d", localPort)
+	config.Address = h.CreatePortForwardTunnel(t, remotePort)
 	consulClient, err := api.NewClient(config)
 	require.NoError(t, err)
 
-	return consulClient
+	return consulClient, config.Address
 }
 
 // configurePodSecurityPolicies creates a simple pod security policy, a cluster role to allow access to the PSP,

acceptance/framework/vault/helpers.go

@@ -190,8 +190,7 @@ func ConfigurePKICA(t *testing.T, vaultClient *vapi.Client) {
 
 // ConfigurePKICertificates configures roles in Vault so that Consul server TLS certificates
 // can be issued by Vault.
-func ConfigurePKICertificates(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns, datacenter string) string {
-	// Create the Vault PKI Role.
+func ConfigurePKICertificates(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns, datacenter string, maxTTL string) string {
 	consulServerDNSName := consulReleaseName + "-consul-server"
 	allowedDomains := fmt.Sprintf("%s.consul,%s,%s.%s,%s.%s.svc", datacenter, consulServerDNSName, consulServerDNSName, ns, consulServerDNSName, ns)
 	params := map[string]interface{}{
@@ -200,7 +199,7 @@ func ConfigurePKICertificates(t *testing.T, vaultClient *vapi.Client, consulRele
 		"allow_localhost":    "true",
 		"allow_subdomains":   "true",
 		"generate_lease":     "true",
-		"max_ttl":            "1h",
+		"max_ttl":            maxTTL,
 	}
 
 	pkiRoleName := fmt.Sprintf("server-cert-%s", datacenter)

acceptance/tests/basic/basic_test.go

@@ -51,7 +51,7 @@ func TestBasicInstallation(t *testing.T) {
 
 			consulCluster.Create(t)
 
-			client := consulCluster.SetupConsulClient(t, c.secure)
+			client, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Create a KV entry
 			randomKey := helpers.RandomName()

acceptance/tests/connect/connect_helper.go

@@ -68,7 +68,7 @@ func (c *ConnectHelper) Setup(t *testing.T) {
 func (c *ConnectHelper) Install(t *testing.T) {
 	logger.Log(t, "Installing Consul cluster")
 	c.consulCluster.Create(t)
-	c.consulClient = c.consulCluster.SetupConsulClient(t, c.Secure)
+	c.consulClient, _ = c.consulCluster.SetupConsulClient(t, c.Secure)
 }
 
 // Upgrade uses the existing Consul cluster and upgrades it using Helm values

acceptance/tests/connect/connect_inject_namespaces_test.go

@@ -108,7 +108,7 @@ func TestConnectInjectNamespaces(t *testing.T) {
 				k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", staticClientNamespace)
 			})
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			serverQueryOpts := &api.QueryOptions{Namespace: staticServerNamespace}
 			clientQueryOpts := &api.QueryOptions{Namespace: staticClientNamespace}
@@ -313,7 +313,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) {
 			k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces")
 
 			logger.Log(t, "waiting for static-client to be registered with Consul")
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 			expectedConsulNS := staticClientNamespace
 			if !c.mirrorK8S {
 				expectedConsulNS = c.destinationNamespace

acceptance/tests/connect/connect_inject_test.go

@@ -179,7 +179,7 @@ func TestConnectInject_CleanupKilledPods(t *testing.T) {
 			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
 
 			logger.Log(t, "waiting for static-client to be registered with Consul")
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 			retry.Run(t, func(r *retry.R) {
 				for _, name := range []string{"static-client", "static-client-sidecar-proxy"} {
 					instances, _, err := consulClient.Catalog().Service(name, "", nil)
@@ -301,7 +301,7 @@ func TestConnectInject_MultiportServices(t *testing.T) {
 
 			consulCluster.Create(t)
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Check that the ACL token is deleted.
 			if c.secure {

acceptance/tests/controller/controller_namespaces_test.go

@@ -114,7 +114,7 @@ func TestControllerNamespaces(t *testing.T) {
 			defaultOpts := &api.QueryOptions{
 				Namespace: DefaultConsulNamespace,
 			}
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Test creation.
 			{

acceptance/tests/controller/controller_test.go

@@ -49,7 +49,7 @@ func TestController(t *testing.T) {
 			consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
 
 			consulCluster.Create(t)
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Test creation.
 			{

acceptance/tests/example/example_test.go

@@ -54,7 +54,7 @@ func TestExample(t *testing.T) {
 
 	// To make Consul API calls, you can get the Consul client from the consulCluster object,
 	// indicating whether the client needs to be secure or not (i.e. whether TLS and ACLs are enabled on the Consul cluster):
-	consulClient := consulCluster.SetupConsulClient(t, true)
+	consulClient, _ := consulCluster.SetupConsulClient(t, true)
 	consulServices, _, err := consulClient.Catalog().Services(nil)
 	require.NoError(t, err)
 	require.NotNil(t, consulServices)

acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go

@@ -58,7 +58,7 @@ func TestIngressGatewaySingleNamespace(t *testing.T) {
 
 			consulCluster.Create(t)
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Create the destination namespace in the non-secure case.
 			// In the secure installation, this namespace is created by the server-acl-init job.
@@ -223,7 +223,7 @@ func TestIngressGatewayNamespaceMirroring(t *testing.T) {
 			logger.Logf(t, "creating static-client in %s namespace", testNamespace)
 			k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client")
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// With the cluster up, we can create our ingress-gateway config entry.
 			logger.Log(t, "creating config entry")

acceptance/tests/ingress-gateway/ingress_gateway_test.go

@@ -66,7 +66,7 @@ func TestIngressGateway(t *testing.T) {
 
 			// With the cluster up, we can create our ingress-gateway config entry.
 			logger.Log(t, "creating config entry")
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Create config entry
 			created, _, err := consulClient.ConfigEntries().Set(&api.IngressGatewayConfigEntry{

acceptance/tests/mesh-gateway/mesh_gateway_test.go

@@ -104,8 +104,8 @@ func TestMeshGatewayDefault(t *testing.T) {
 		k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "rollout", "status", fmt.Sprintf("sts/%s-consul-server", releaseName))
 	}
 
-	primaryClient := primaryConsulCluster.SetupConsulClient(t, false)
-	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, false)
+	primaryClient, _ := primaryConsulCluster.SetupConsulClient(t, false)
+	secondaryClient, _ := secondaryConsulCluster.SetupConsulClient(t, false)
 
 	// Verify federation between servers
 	logger.Log(t, "verifying federation was successful")
@@ -258,8 +258,8 @@ func TestMeshGatewaySecure(t *testing.T) {
 				k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "rollout", "status", fmt.Sprintf("sts/%s-consul-server", releaseName))
 			}
 
-			primaryClient := primaryConsulCluster.SetupConsulClient(t, true)
-			secondaryClient := secondaryConsulCluster.SetupConsulClient(t, true)
+			primaryClient, _ := primaryConsulCluster.SetupConsulClient(t, true)
+			secondaryClient, _ := secondaryConsulCluster.SetupConsulClient(t, true)
 
 			// Verify federation between servers
 			logger.Log(t, "verifying federation was successful")

acceptance/tests/partitions/partitions_test.go

@@ -243,7 +243,7 @@ func TestPartitions(t *testing.T) {
 				k8s.RunKubectl(t, clientClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace, staticClientNamespace)
 			})
 
-			consulClient := serverConsulCluster.SetupConsulClient(t, c.ACLsAndAutoEncryptEnabled)
+			consulClient, _ := serverConsulCluster.SetupConsulClient(t, c.ACLsAndAutoEncryptEnabled)
 
 			serverQueryServerOpts := &api.QueryOptions{Namespace: staticServerNamespace, Partition: defaultPartition}
 			clientQueryServerOpts := &api.QueryOptions{Namespace: staticClientNamespace, Partition: defaultPartition}

acceptance/tests/snapshot-agent/snapshot_agent_vault_test.go

@@ -61,7 +61,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
 	vault.ConfigureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes")
 
 	vault.ConfigurePKICA(t, vaultClient)
-	certPath := vault.ConfigurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
+	certPath := vault.ConfigurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1", "1h")
 
 	vaultCASecret := vault.CASecretName(vaultReleaseName)
 

acceptance/tests/sync/sync_catalog_namespaces_test.go

@@ -98,7 +98,7 @@ func TestSyncCatalogNamespaces(t *testing.T) {
 			logger.Log(t, "creating a static-server with a service")
 			k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server")
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			logger.Log(t, "checking that the service has been synced to Consul")
 			var services map[string][]string

acceptance/tests/sync/sync_catalog_test.go

@@ -63,7 +63,7 @@ func TestSyncCatalog(t *testing.T) {
 			logger.Log(t, "creating a static-server with a service")
 			k8s.DeployKustomize(t, ctx.KubectlOptions(t), suite.Config().NoCleanupOnFailure, suite.Config().DebugDirectory, "../fixtures/bases/static-server")
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			logger.Log(t, "checking that the service has been synced to Consul")
 			var services map[string][]string

acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go

@@ -58,7 +58,7 @@ func TestTerminatingGatewaySingleNamespace(t *testing.T) {
 
 			consulCluster.Create(t)
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Create the destination namespace in the non-secure case.
 			// In the secure installation, this namespace is created by the server-acl-init job.
@@ -172,7 +172,7 @@ func TestTerminatingGatewayNamespaceMirroring(t *testing.T) {
 
 			consulCluster.Create(t)
 
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			logger.Logf(t, "creating Kubernetes namespace %s", testNamespace)
 			k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace)

acceptance/tests/terminating-gateway/terminating_gateway_test.go

@@ -64,7 +64,7 @@ func TestTerminatingGateway(t *testing.T) {
 			k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server")
 
 			// Once the cluster is up, register the external service, then create the config entry.
-			consulClient := consulCluster.SetupConsulClient(t, c.secure)
+			consulClient, _ := consulCluster.SetupConsulClient(t, c.secure)
 
 			// Register the external service
 			registerExternalService(t, consulClient, "")

acceptance/tests/vault/vault_partitions_test.go

@@ -119,7 +119,7 @@ func TestVault_Partitions(t *testing.T) {
 	vault.ConfigureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-"+secondaryPartition, "partition-init", "partition-token")
 	vault.ConfigureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes-"+secondaryPartition)
 	vault.ConfigurePKICA(t, vaultClient)
-	certPath := vault.ConfigurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
+	certPath := vault.ConfigurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1", "1h")
 
 	vaultCASecretName := vault.CASecretName(vaultReleaseName)
 

acceptance/tests/vault/vault_test.go

@@ -51,7 +51,7 @@ func TestVault(t *testing.T) {
 	vault.ConfigureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes")
 
 	vault.ConfigurePKICA(t, vaultClient)
-	certPath := vault.ConfigurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
+	certPath := vault.ConfigurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1", "1h")
 
 	vaultCASecret := vault.CASecretName(vaultReleaseName)
 
@@ -114,7 +114,7 @@ func TestVault(t *testing.T) {
 	// Validate that the gossip encryption key is set correctly.
 	logger.Log(t, "Validating the gossip key has been set correctly.")
 	consulCluster.ACLToken = bootstrapToken
-	consulClient := consulCluster.SetupConsulClient(t, true)
+	consulClient, _ := consulCluster.SetupConsulClient(t, true)
 	keys, err := consulClient.Operator().KeyringList(nil)
 	require.NoError(t, err)
 	// There are two identical keys for LAN and WAN since there is only 1 dc.