diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index 7151a75908..e29454edab 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "testing" "github.com/hashicorp/go-version" "gopkg.in/yaml.v2" @@ -33,6 +34,9 @@ type TestConfig struct { SecondaryKubeContext string SecondaryKubeNamespace string + AppNamespace string + SecondaryAppNamespace string + EnableEnterprise bool EnterpriseLicense string @@ -40,7 +44,8 @@ type TestConfig struct { EnablePodSecurityPolicies bool - EnableCNI bool + EnableCNI bool + CNINamespace string EnableTransparentProxy bool @@ -101,10 +106,18 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { if t.EnableCNI { setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true") + setIfNotEmpty(helmValues, "connectInject.cni.logLevel", "debug") // GKE is currently the only cloud provider that uses a different CNI bin dir. if t.UseGKE { setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin") } + if t.EnableOpenshift { + setIfNotEmpty(helmValues, "connectInject.cni.multus", "true") + setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/var/lib/cni/bin") + setIfNotEmpty(helmValues, "connectInject.cni.cniNetDir", "/etc/kubernetes/cni/net.d") + } + + setIfNotEmpty(helmValues, "connectInject.cni.namespace", t.CNINamespace) } setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy)) @@ -169,6 +182,12 @@ func (t *TestConfig) entImage() (string, error) { return fmt.Sprintf("hashicorp/consul-enterprise:%s%s-ent", consulImageVersion, preRelease), nil } +func (c *TestConfig) SkipWhenOpenshiftAndCNI(t *testing.T) { + if c.EnableOpenshift && c.EnableCNI { + t.Skip("skipping because -enable-cni and -enable-openshift are set and this test doesn't deploy apps correctly") + } +} + // setIfNotEmpty sets key to val in map m if value is not empty. func setIfNotEmpty(m map[string]string, key, val string) { if val != "" { diff --git a/acceptance/framework/connhelper/connect_helper.go b/acceptance/framework/connhelper/connect_helper.go index 670307da88..944684793c 100644 --- a/acceptance/framework/connhelper/connect_helper.go +++ b/acceptance/framework/connhelper/connect_helper.go @@ -44,8 +44,11 @@ type ConnectHelper struct { // ReleaseName is the name of the Consul cluster. ReleaseName string + // Ctx is used to deploy Consul Ctx environment.TestContext - Cfg *config.TestConfig + // AppCtx is used to deploy applications. If nil, then Ctx is used. + AppCtx environment.TestContext + Cfg *config.TestConfig // consulCluster is the cluster to use for the test. consulCluster consul.Cluster @@ -82,6 +85,14 @@ func (c *ConnectHelper) Upgrade(t *testing.T) { c.consulCluster.Upgrade(t, c.helmValues()) } +// appCtx returns the context where apps are deployed. +func (c *ConnectHelper) appCtx() environment.TestContext { + if c.AppCtx != nil { + return c.AppCtx + } + return c.Ctx +} + // DeployClientAndServer deploys a client and server pod to the Kubernetes // cluster which will be used to test service mesh connectivity. If the Secure // flag is true, a pre-check is done to ensure that the ACL tokens for the test @@ -108,23 +119,47 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") - if c.Cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + ctx := c.appCtx() + opts := ctx.KubectlOptions(t) + + if c.Cfg.EnableCNI && c.Cfg.EnableOpenshift { + // On OpenShift with the CNI, we need to create a network attachment definition in the namespace + // where the applications are running, and the app deployment configs need to reference that network + // attachment definition. + + // TODO: A base fixture is the wrong place for these files + k8s.KubectlApply(t, opts, "../fixtures/bases/openshift/") + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + k8s.KubectlDelete(t, opts, "../fixtures/bases/openshift/") + }) + + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift") + if c.Cfg.EnableTransparentProxy { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy") + } else { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject") + } } else { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") - } + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + if c.Cfg.EnableTransparentProxy { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + } else { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + } + } // Check that both static-server and static-client have been injected and // now have 2 containers. retry.RunWith( &retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond}, t, func(r *retry.R) { for _, labelSelector := range []string{"app=static-server", "app=static-client"} { - podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: labelSelector, - FieldSelector: `status.phase=Running`, - }) + podList, err := ctx.KubernetesClient(t).CoreV1(). + Pods(opts.Namespace). + List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelector, + FieldSelector: `status.phase=Running`, + }) require.NoError(r, err) require.Len(r, podList.Items, 1) require.Len(r, podList.Items[0].Spec.Containers, 2) @@ -136,7 +171,7 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { // and intentions. This helper is primarly used to ensure that the virtual-ips are persisted to consul properly. func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) { logger.Log(t, "creating resolver redirect") - options := c.Ctx.KubectlOptions(t) + options := c.appCtx().KubectlOptions(t) kustomizeDir := "../fixtures/cases/resolver-redirect-virtualip" k8s.KubectlApplyK(t, options, kustomizeDir) @@ -149,10 +184,12 @@ func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) { // server fails when no intentions are configured. func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) { logger.Log(t, "checking that the connection is not successful because there's no intention") + ctx := c.appCtx() + opts := ctx.KubectlOptions(t) if c.Cfg.EnableTransparentProxy { - k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server") + k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://static-server") } else { - k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234") + k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://localhost:1234") } } @@ -177,11 +214,13 @@ func (c *ConnectHelper) CreateIntention(t *testing.T) { // static-client pod once the intention is set. func (c *ConnectHelper) TestConnectionSuccess(t *testing.T) { logger.Log(t, "checking that connection is successful") + ctx := c.appCtx() + opts := ctx.KubectlOptions(t) if c.Cfg.EnableTransparentProxy { // todo: add an assertion that the traffic is going through the proxy - k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server") + k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://static-server") } else { - k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234") + k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://localhost:1234") } } @@ -192,8 +231,11 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) { // Test that kubernetes readiness status is synced to Consul. // Create a file called "unhealthy" at "/tmp/" so that the readiness probe // of the static-server pod fails. + ctx := c.appCtx() + opts := ctx.KubectlOptions(t) + logger.Log(t, "testing k8s -> consul health checks sync by making the static-server unhealthy") - k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy") + k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy") // The readiness probe should take a moment to be reflected in Consul, // CheckStaticServerConnection will retry until Consul marks the service @@ -205,20 +247,20 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) { // other tests. logger.Log(t, "checking that connection is unsuccessful") if c.Cfg.EnableTransparentProxy { - k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{ + k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{ "curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", "curl: (7) Failed to connect to static-server port 80: Connection refused", }, "", "http://static-server") } else { - k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{ + k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{ "curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", }, "", "http://localhost:1234") } // Return the static-server to a "healthy state". - k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy") + k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy") } // helmValues uses the Secure and AutoEncrypt fields to set values for the Helm diff --git a/acceptance/framework/environment/environment.go b/acceptance/framework/environment/environment.go index 58e4e83a5b..d46ad9bdd0 100644 --- a/acceptance/framework/environment/environment.go +++ b/acceptance/framework/environment/environment.go @@ -21,8 +21,10 @@ import ( ) const ( - DefaultContextName = "default" - SecondaryContextName = "secondary" + DefaultContextName = "default" + SecondaryContextName = "secondary" + AppContextName = "app-default" + SecondaryAppContextName = "app-secondary" ) // TestEnvironment represents the infrastructure environment of the test, @@ -59,6 +61,20 @@ func NewKubernetesEnvironmentFromConfig(config *config.TestConfig) *KubernetesEn kenv.contexts[SecondaryContextName] = NewContext(config.SecondaryKubeNamespace, config.SecondaryKubeconfig, config.SecondaryKubeContext) } + // Optionally, deploy apps into a separate namespace. + // Maybe not the right place for this? + if config.AppNamespace != "" { + kenv.contexts[AppContextName] = NewContext(config.AppNamespace, config.Kubeconfig, config.KubeContext) + if config.EnableMultiCluster { + kenv.contexts[SecondaryAppContextName] = NewContext(config.SecondaryAppNamespace, config.SecondaryKubeconfig, config.SecondaryKubeContext) + } + } else { + kenv.contexts[AppContextName] = defaultContext + if config.EnableMultiCluster { + kenv.contexts[SecondaryAppContextName] = kenv.contexts[SecondaryContextName] + } + } + return kenv } diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index 5df09f853a..5363d303cf 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -23,6 +23,9 @@ type TestFlags struct { flagSecondaryKubecontext string flagSecondaryNamespace string + flagAppNamespace string + flagSecondaryAppNamespace string + flagEnableEnterprise bool flagEnterpriseLicense string @@ -30,7 +33,8 @@ type TestFlags struct { flagEnablePodSecurityPolicies bool - flagEnableCNI bool + flagEnableCNI bool + flagCNINamespace string flagEnableTransparentProxy bool @@ -75,6 +79,9 @@ func (t *TestFlags) init() { "the context set as the current context will be used by default.") flag.StringVar(&t.flagNamespace, "namespace", "", "The Kubernetes namespace to use for tests.") + flag.StringVar(&t.flagAppNamespace, "app-namespace", "", "The Kubernetes namespace where mesh services should be deployed.") + flag.StringVar(&t.flagSecondaryAppNamespace, "secondary-app-namespace", "", "The secondary Kubernetes namespace where mesh services should be deployed [multi-cluster only].") + flag.StringVar(&t.flagConsulImage, "consul-image", "", "The Consul image to use for all tests.") flag.StringVar(&t.flagConsulK8sImage, "consul-k8s-image", "", "The consul-k8s image to use for all tests.") flag.StringVar(&t.flagConsulDataplaneImage, "consul-dataplane-image", "", "The consul-dataplane image to use for all tests.") @@ -112,6 +119,8 @@ func (t *TestFlags) init() { flag.BoolVar(&t.flagEnableCNI, "enable-cni", false, "If true, the test suite will run tests with consul-cni plugin enabled. "+ "In general, this will only run against tests that are mesh related (connect, mesh-gateway, peering, etc") + flag.StringVar(&t.flagCNINamespace, "cni-namespace", "", + "If configured, the CNI will be deployed in this namespace.") flag.BoolVar(&t.flagEnableTransparentProxy, "enable-transparent-proxy", false, "If true, the test suite will run tests with transparent proxy enabled. "+ @@ -172,6 +181,9 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { SecondaryKubeContext: t.flagSecondaryKubecontext, SecondaryKubeNamespace: t.flagSecondaryNamespace, + AppNamespace: t.flagAppNamespace, + SecondaryAppNamespace: t.flagSecondaryAppNamespace, + EnableEnterprise: t.flagEnableEnterprise, EnterpriseLicense: t.flagEnterpriseLicense, @@ -179,7 +191,8 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { EnablePodSecurityPolicies: t.flagEnablePodSecurityPolicies, - EnableCNI: t.flagEnableCNI, + EnableCNI: t.flagEnableCNI, + CNINamespace: t.flagCNINamespace, EnableTransparentProxy: t.flagEnableTransparentProxy, diff --git a/acceptance/tests/connect/connect_external_servers_test.go b/acceptance/tests/connect/connect_external_servers_test.go index a7b0f656bf..c95d791773 100644 --- a/acceptance/tests/connect/connect_external_servers_test.go +++ b/acceptance/tests/connect/connect_external_servers_test.go @@ -30,6 +30,8 @@ func TestConnectInject_ExternalServers(t *testing.T) { caseName := fmt.Sprintf("secure: %t", secure) t.Run(caseName, func(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) serverHelmValues := map[string]string{ diff --git a/acceptance/tests/connect/connect_inject_namespaces_test.go b/acceptance/tests/connect/connect_inject_namespaces_test.go index f848594cd2..04021ec391 100644 --- a/acceptance/tests/connect/connect_inject_namespaces_test.go +++ b/acceptance/tests/connect/connect_inject_namespaces_test.go @@ -34,6 +34,7 @@ func TestConnectInjectNamespaces(t *testing.T) { if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) cases := []struct { name string @@ -246,6 +247,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) { if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) consulDestNS := "consul-dest" cases := []struct { diff --git a/acceptance/tests/connect/connect_inject_test.go b/acceptance/tests/connect/connect_inject_test.go index 199bbf0f1b..62abef7dac 100644 --- a/acceptance/tests/connect/connect_inject_test.go +++ b/acceptance/tests/connect/connect_inject_test.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/consul-k8s/acceptance/framework/connhelper" "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" @@ -34,7 +35,10 @@ func TestConnectInject(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { cfg := suite.Config() - ctx := suite.Environment().DefaultContext(t) + env := suite.Environment() + ctx := env.DefaultContext(t) + appCtx := env.Context(t, environment.AppContextName) + t.Logf("appCtx = %#v", appCtx) releaseName := helpers.RandomName() connHelper := connhelper.ConnectHelper{ @@ -42,6 +46,7 @@ func TestConnectInject(t *testing.T) { Secure: c.secure, ReleaseName: releaseName, Ctx: ctx, + AppCtx: appCtx, Cfg: cfg, } @@ -67,7 +72,9 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { // This can only be tested in transparent proxy mode. t.SkipNow() } - ctx := suite.Environment().DefaultContext(t) + env := suite.Environment() + ctx := env.DefaultContext(t) + appCtx := env.Context(t, environment.AppContextName) releaseName := helpers.RandomName() connHelper := connhelper.ConnectHelper{ @@ -75,6 +82,7 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { Secure: true, ReleaseName: releaseName, Ctx: ctx, + AppCtx: appCtx, Cfg: cfg, } @@ -84,7 +92,7 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { connHelper.CreateResolverRedirect(t) connHelper.DeployClientAndServer(t) - k8s.CheckStaticServerConnectionSuccessful(t, connHelper.Ctx.KubectlOptions(t), "static-client", "http://resolver-redirect") + k8s.CheckStaticServerConnectionSuccessful(t, appCtx.KubectlOptions(t), "static-client", "http://resolver-redirect") } // Test the endpoints controller cleans up force-killed pods. @@ -93,6 +101,9 @@ func TestConnectInject_CleanupKilledPods(t *testing.T) { name := fmt.Sprintf("secure: %t", secure) t.Run(name, func(t *testing.T) { cfg := suite.Config() + + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ @@ -161,6 +172,8 @@ func TestConnectInject_MultiportServices(t *testing.T) { name := fmt.Sprintf("secure: %t", secure) t.Run(name, func(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ diff --git a/acceptance/tests/connect/connect_proxy_lifecycle_test.go b/acceptance/tests/connect/connect_proxy_lifecycle_test.go index 321c002a4c..39dd0b4ae7 100644 --- a/acceptance/tests/connect/connect_proxy_lifecycle_test.go +++ b/acceptance/tests/connect/connect_proxy_lifecycle_test.go @@ -34,6 +34,7 @@ const ( // Test the endpoints controller cleans up force-killed pods. func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) for _, testCfg := range []LifecycleShutdownConfig{ {secure: false, helmValues: map[string]string{ diff --git a/acceptance/tests/connect/permissive_mtls_test.go b/acceptance/tests/connect/permissive_mtls_test.go index 1dcc6fe911..d3d863ad61 100644 --- a/acceptance/tests/connect/permissive_mtls_test.go +++ b/acceptance/tests/connect/permissive_mtls_test.go @@ -23,6 +23,7 @@ func TestConnectInject_PermissiveMTLS(t *testing.T) { if !cfg.EnableTransparentProxy { t.Skipf("skipping this because -enable-transparent-proxy is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) ctx := suite.Environment().DefaultContext(t)