Test: Support TestConnectInject on OpenShift + CNI

* When `-enable-openshift` and `-enable-cni` are set, configure the CNI
  settings correctly for OpenShift, which must look like:

  ```
  connectInject:
    cni:
      enabled: true
      multus: true
      cniBinDir: /var/lib/cni/bin
      cniNetDir: /etc/kubernetes/cni/net.
  ```

* Add `-cni-namespace` flag to support deploying the CNI into a separate
  namespace. This is needed to testing with the CNI when restricted PSA
  enforcement is enabled on the namespace where Consul is deployed
  because the CNI cannot run in a PSA-restricted namespace.
* Add `-app-namespace` and `-secondary-app-namespace` flags to support
  deploying test applications into a specific namespace. This is needed
  to test with restricted PSA enforcement enabled on the Consul
  namespace because our test applications require a bit more privilege.
* Update the ConnectHelper to configure the NetworkAttachmentDefinition
  required to be compatible with the CNI on OpenShift.
* Add fixtures for static-client and static-server for OpenShift. This
  is necessary because the deployment configs must reference the network
  attachment definition when using the CNI on OpenShift.
* Update tests in the `acceptance/tests/connect` directory to either
  run or skip based on -enable-cni and -enable-openshift (all but two
  cases are skipped for now).

acceptance/framework/config/config.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"testing"
 
 	"github.com/hashicorp/go-version"
 	"gopkg.in/yaml.v2"
@@ -33,14 +34,18 @@ type TestConfig struct {
 	SecondaryKubeContext   string
 	SecondaryKubeNamespace string
 
+	AppNamespace          string
+	SecondaryAppNamespace string
+
 	EnableEnterprise  bool
 	EnterpriseLicense string
 
 	EnableOpenshift bool
 
 	EnablePodSecurityPolicies bool
 
-	EnableCNI bool
+	EnableCNI    bool
+	CNINamespace string
 
 	EnableTransparentProxy bool
 
@@ -101,10 +106,18 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) {
 
 	if t.EnableCNI {
 		setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true")
+		setIfNotEmpty(helmValues, "connectInject.cni.logLevel", "debug")
 		// GKE is currently the only cloud provider that uses a different CNI bin dir.
 		if t.UseGKE {
 			setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin")
 		}
+		if t.EnableOpenshift {
+			setIfNotEmpty(helmValues, "connectInject.cni.multus", "true")
+			setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/var/lib/cni/bin")
+			setIfNotEmpty(helmValues, "connectInject.cni.cniNetDir", "/etc/kubernetes/cni/net.d")
+		}
+
+		setIfNotEmpty(helmValues, "connectInject.cni.namespace", t.CNINamespace)
 	}
 
 	setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy))
@@ -169,6 +182,12 @@ func (t *TestConfig) entImage() (string, error) {
 	return fmt.Sprintf("hashicorp/consul-enterprise:%s%s-ent", consulImageVersion, preRelease), nil
 }
 
+func (c *TestConfig) SkipWhenOpenshiftAndCNI(t *testing.T) {
+	if c.EnableOpenshift && c.EnableCNI {
+		t.Skip("skipping because -enable-cni and -enable-openshift are set and this test doesn't deploy apps correctly")
+	}
+}
+
 // setIfNotEmpty sets key to val in map m if value is not empty.
 func setIfNotEmpty(m map[string]string, key, val string) {
 	if val != "" {

acceptance/framework/connhelper/connect_helper.go

@@ -44,8 +44,11 @@ type ConnectHelper struct {
 	// ReleaseName is the name of the Consul cluster.
 	ReleaseName string
 
+	// Ctx is used to deploy Consul
 	Ctx environment.TestContext
-	Cfg *config.TestConfig
+	// AppCtx is used to deploy applications. If nil, then Ctx is used.
+	AppCtx environment.TestContext
+	Cfg    *config.TestConfig
 
 	// consulCluster is the cluster to use for the test.
 	consulCluster consul.Cluster
@@ -82,6 +85,14 @@ func (c *ConnectHelper) Upgrade(t *testing.T) {
 	c.consulCluster.Upgrade(t, c.helmValues())
 }
 
+// appCtx returns the context where apps are deployed.
+func (c *ConnectHelper) appCtx() environment.TestContext {
+	if c.AppCtx != nil {
+		return c.AppCtx
+	}
+	return c.Ctx
+}
+
 // DeployClientAndServer deploys a client and server pod to the Kubernetes
 // cluster which will be used to test service mesh connectivity. If the Secure
 // flag is true, a pre-check is done to ensure that the ACL tokens for the test
@@ -108,23 +119,47 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) {
 
 	logger.Log(t, "creating static-server and static-client deployments")
 
-	k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
-	if c.Cfg.EnableTransparentProxy {
-		k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy")
+	ctx := c.appCtx()
+	opts := ctx.KubectlOptions(t)
+
+	if c.Cfg.EnableCNI && c.Cfg.EnableOpenshift {
+		// On OpenShift with the CNI, we need to create a network attachment definition in the namespace
+		// where the applications are running, and the app deployment configs need to reference that network
+		// attachment definition.
+
+		// TODO: A base fixture is the wrong place for these files
+		k8s.KubectlApply(t, opts, "../fixtures/bases/openshift/")
+		helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() {
+			k8s.KubectlDelete(t, opts, "../fixtures/bases/openshift/")
+		})
+
+		k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift")
+		if c.Cfg.EnableTransparentProxy {
+			k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy")
+		} else {
+			k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject")
+		}
 	} else {
-		k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
-	}
+		k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
+		if c.Cfg.EnableTransparentProxy {
+			k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy")
+		} else {
+			k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
+		}
 
+	}
 	// Check that both static-server and static-client have been injected and
 	// now have 2 containers.
 	retry.RunWith(
 		&retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond}, t,
 		func(r *retry.R) {
 			for _, labelSelector := range []string{"app=static-server", "app=static-client"} {
-				podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{
-					LabelSelector: labelSelector,
-					FieldSelector: `status.phase=Running`,
-				})
+				podList, err := ctx.KubernetesClient(t).CoreV1().
+					Pods(opts.Namespace).
+					List(context.Background(), metav1.ListOptions{
+						LabelSelector: labelSelector,
+						FieldSelector: `status.phase=Running`,
+					})
 				require.NoError(r, err)
 				require.Len(r, podList.Items, 1)
 				require.Len(r, podList.Items[0].Spec.Containers, 2)
@@ -136,7 +171,7 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) {
 // and intentions. This helper is primarly used to ensure that the virtual-ips are persisted to consul properly.
 func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) {
 	logger.Log(t, "creating resolver redirect")
-	options := c.Ctx.KubectlOptions(t)
+	options := c.appCtx().KubectlOptions(t)
 	kustomizeDir := "../fixtures/cases/resolver-redirect-virtualip"
 	k8s.KubectlApplyK(t, options, kustomizeDir)
 
@@ -149,10 +184,12 @@ func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) {
 // server fails when no intentions are configured.
 func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) {
 	logger.Log(t, "checking that the connection is not successful because there's no intention")
+	ctx := c.appCtx()
+	opts := ctx.KubectlOptions(t)
 	if c.Cfg.EnableTransparentProxy {
-		k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server")
+		k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://static-server")
 	} else {
-		k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234")
+		k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://localhost:1234")
 	}
 }
 
@@ -177,11 +214,13 @@ func (c *ConnectHelper) CreateIntention(t *testing.T) {
 // static-client pod once the intention is set.
 func (c *ConnectHelper) TestConnectionSuccess(t *testing.T) {
 	logger.Log(t, "checking that connection is successful")
+	ctx := c.appCtx()
+	opts := ctx.KubectlOptions(t)
 	if c.Cfg.EnableTransparentProxy {
 		// todo: add an assertion that the traffic is going through the proxy
-		k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server")
+		k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://static-server")
 	} else {
-		k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234")
+		k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://localhost:1234")
 	}
 }
 
@@ -192,8 +231,11 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) {
 	// Test that kubernetes readiness status is synced to Consul.
 	// Create a file called "unhealthy" at "/tmp/" so that the readiness probe
 	// of the static-server pod fails.
+	ctx := c.appCtx()
+	opts := ctx.KubectlOptions(t)
+
 	logger.Log(t, "testing k8s -> consul health checks sync by making the static-server unhealthy")
-	k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy")
+	k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy")
 
 	// The readiness probe should take a moment to be reflected in Consul,
 	// CheckStaticServerConnection will retry until Consul marks the service
@@ -205,20 +247,20 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) {
 	// other tests.
 	logger.Log(t, "checking that connection is unsuccessful")
 	if c.Cfg.EnableTransparentProxy {
-		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{
+		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{
 			"curl: (56) Recv failure: Connection reset by peer",
 			"curl: (52) Empty reply from server",
 			"curl: (7) Failed to connect to static-server port 80: Connection refused",
 		}, "", "http://static-server")
 	} else {
-		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{
+		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{
 			"curl: (56) Recv failure: Connection reset by peer",
 			"curl: (52) Empty reply from server",
 		}, "", "http://localhost:1234")
 	}
 
 	// Return the static-server to a "healthy state".
-	k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy")
+	k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy")
 }
 
 // helmValues uses the Secure and AutoEncrypt fields to set values for the Helm

acceptance/framework/environment/environment.go

@@ -21,8 +21,10 @@ import (
 )
 
 const (
-	DefaultContextName   = "default"
-	SecondaryContextName = "secondary"
+	DefaultContextName      = "default"
+	SecondaryContextName    = "secondary"
+	AppContextName          = "app-default"
+	SecondaryAppContextName = "app-secondary"
 )
 
 // TestEnvironment represents the infrastructure environment of the test,
@@ -59,6 +61,20 @@ func NewKubernetesEnvironmentFromConfig(config *config.TestConfig) *KubernetesEn
 		kenv.contexts[SecondaryContextName] = NewContext(config.SecondaryKubeNamespace, config.SecondaryKubeconfig, config.SecondaryKubeContext)
 	}
 
+	// Optionally, deploy apps into a separate namespace.
+	// Maybe not the right place for this?
+	if config.AppNamespace != "" {
+		kenv.contexts[AppContextName] = NewContext(config.AppNamespace, config.Kubeconfig, config.KubeContext)
+		if config.EnableMultiCluster {
+			kenv.contexts[SecondaryAppContextName] = NewContext(config.SecondaryAppNamespace, config.SecondaryKubeconfig, config.SecondaryKubeContext)
+		}
+	} else {
+		kenv.contexts[AppContextName] = defaultContext
+		if config.EnableMultiCluster {
+			kenv.contexts[SecondaryAppContextName] = kenv.contexts[SecondaryContextName]
+		}
+	}
+
 	return kenv
 }
 

acceptance/framework/flags/flags.go

@@ -23,14 +23,18 @@ type TestFlags struct {
 	flagSecondaryKubecontext string
 	flagSecondaryNamespace   string
 
+	flagAppNamespace          string
+	flagSecondaryAppNamespace string
+
 	flagEnableEnterprise  bool
 	flagEnterpriseLicense string
 
 	flagEnableOpenshift bool
 
 	flagEnablePodSecurityPolicies bool
 
-	flagEnableCNI bool
+	flagEnableCNI    bool
+	flagCNINamespace string
 
 	flagEnableTransparentProxy bool
 
@@ -75,6 +79,9 @@ func (t *TestFlags) init() {
 		"the context set as the current context will be used by default.")
 	flag.StringVar(&t.flagNamespace, "namespace", "", "The Kubernetes namespace to use for tests.")
 
+	flag.StringVar(&t.flagAppNamespace, "app-namespace", "", "The Kubernetes namespace where mesh services should be deployed.")
+	flag.StringVar(&t.flagSecondaryAppNamespace, "secondary-app-namespace", "", "The secondary Kubernetes namespace where mesh services should be deployed [multi-cluster only].")
+
 	flag.StringVar(&t.flagConsulImage, "consul-image", "", "The Consul image to use for all tests.")
 	flag.StringVar(&t.flagConsulK8sImage, "consul-k8s-image", "", "The consul-k8s image to use for all tests.")
 	flag.StringVar(&t.flagConsulDataplaneImage, "consul-dataplane-image", "", "The consul-dataplane image to use for all tests.")
@@ -112,6 +119,8 @@ func (t *TestFlags) init() {
 	flag.BoolVar(&t.flagEnableCNI, "enable-cni", false,
 		"If true, the test suite will run tests with consul-cni plugin enabled. "+
 			"In general, this will only run against tests that are mesh related (connect, mesh-gateway, peering, etc")
+	flag.StringVar(&t.flagCNINamespace, "cni-namespace", "",
+		"If configured, the CNI will be deployed in this namespace.")
 
 	flag.BoolVar(&t.flagEnableTransparentProxy, "enable-transparent-proxy", false,
 		"If true, the test suite will run tests with transparent proxy enabled. "+
@@ -172,14 +181,18 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig {
 		SecondaryKubeContext:   t.flagSecondaryKubecontext,
 		SecondaryKubeNamespace: t.flagSecondaryNamespace,
 
+		AppNamespace:          t.flagAppNamespace,
+		SecondaryAppNamespace: t.flagSecondaryAppNamespace,
+
 		EnableEnterprise:  t.flagEnableEnterprise,
 		EnterpriseLicense: t.flagEnterpriseLicense,
 
 		EnableOpenshift: t.flagEnableOpenshift,
 
 		EnablePodSecurityPolicies: t.flagEnablePodSecurityPolicies,
 
-		EnableCNI: t.flagEnableCNI,
+		EnableCNI:    t.flagEnableCNI,
+		CNINamespace: t.flagCNINamespace,
 
 		EnableTransparentProxy: t.flagEnableTransparentProxy,
 

acceptance/tests/connect/connect_external_servers_test.go

@@ -30,6 +30,8 @@ func TestConnectInject_ExternalServers(t *testing.T) {
 		caseName := fmt.Sprintf("secure: %t", secure)
 		t.Run(caseName, func(t *testing.T) {
 			cfg := suite.Config()
+			cfg.SkipWhenOpenshiftAndCNI(t)
+
 			ctx := suite.Environment().DefaultContext(t)
 
 			serverHelmValues := map[string]string{

acceptance/tests/connect/connect_inject_namespaces_test.go

@@ -34,6 +34,7 @@ func TestConnectInjectNamespaces(t *testing.T) {
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}
+	cfg.SkipWhenOpenshiftAndCNI(t)
 
 	cases := []struct {
 		name                 string
@@ -246,6 +247,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) {
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}
+	cfg.SkipWhenOpenshiftAndCNI(t)
 
 	consulDestNS := "consul-dest"
 	cases := []struct {