diff --git a/.changelog/2572.txt b/.changelog/2572.txt new file mode 100644 index 0000000000..4bc6c4ba50 --- /dev/null +++ b/.changelog/2572.txt @@ -0,0 +1,3 @@ +```release-note:improvement +helm: set container securityContexts to match the `restricted` Pod Security Standards policy to support running Consul in a namespace with restricted PSA enforcement enabled +``` diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index eada42af20..83114ad4a2 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "testing" "github.com/hashicorp/go-version" "gopkg.in/yaml.v2" @@ -73,7 +74,8 @@ type TestConfig struct { EnablePodSecurityPolicies bool - EnableCNI bool + EnableCNI bool + EnableRestrictedPSAEnforcement bool EnableTransparentProxy bool @@ -134,10 +136,22 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { if t.EnableCNI { setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true") + setIfNotEmpty(helmValues, "connectInject.cni.logLevel", "debug") // GKE is currently the only cloud provider that uses a different CNI bin dir. if t.UseGKE { setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin") } + if t.EnableOpenshift { + setIfNotEmpty(helmValues, "connectInject.cni.multus", "true") + setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/var/lib/cni/bin") + setIfNotEmpty(helmValues, "connectInject.cni.cniNetDir", "/etc/kubernetes/cni/net.d") + } + + if t.EnableRestrictedPSAEnforcement { + // The CNI requires privilege, so when restricted PSA enforcement is enabled on the Consul + // namespace it must be run in a different privileged namespace. + setIfNotEmpty(helmValues, "connectInject.cni.namespace", "kube-system") + } } setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy)) @@ -219,6 +233,12 @@ func (t *TestConfig) entImage() (string, error) { return fmt.Sprintf("hashicorp/consul-enterprise:%s%s-ent", consulImageVersion, preRelease), nil } +func (c *TestConfig) SkipWhenOpenshiftAndCNI(t *testing.T) { + if c.EnableOpenshift && c.EnableCNI { + t.Skip("skipping because -enable-cni and -enable-openshift are set and this test doesn't deploy apps correctly") + } +} + // setIfNotEmpty sets key to val in map m if value is not empty. func setIfNotEmpty(m map[string]string, key, val string) { if val != "" { diff --git a/acceptance/framework/config/config_test.go b/acceptance/framework/config/config_test.go index 96f0f0e7eb..df981e26fa 100644 --- a/acceptance/framework/config/config_test.go +++ b/acceptance/framework/config/config_test.go @@ -116,6 +116,7 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) { }, map[string]string{ "connectInject.cni.enabled": "true", + "connectInject.cni.logLevel": "debug", "connectInject.transparentProxy.defaultEnabled": "false", }, }, diff --git a/acceptance/framework/connhelper/connect_helper.go b/acceptance/framework/connhelper/connect_helper.go index 40c02fb091..3deeaddb99 100644 --- a/acceptance/framework/connhelper/connect_helper.go +++ b/acceptance/framework/connhelper/connect_helper.go @@ -6,9 +6,11 @@ package connhelper import ( "context" "strconv" + "strings" "testing" "time" + terratestK8s "github.com/gruntwork-io/terratest/modules/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/config" "github.com/hashicorp/consul-k8s/acceptance/framework/consul" "github.com/hashicorp/consul-k8s/acceptance/framework/environment" @@ -44,7 +46,12 @@ type ConnectHelper struct { // ReleaseName is the name of the Consul cluster. ReleaseName string + // Ctx is used to deploy Consul Ctx environment.TestContext + // UseAppNamespace is used top optionally deploy applications into a separate namespace. + // If unset, the namespace associated with Ctx is used. + UseAppNamespace bool + Cfg *config.TestConfig // consulCluster is the cluster to use for the test. @@ -82,6 +89,14 @@ func (c *ConnectHelper) Upgrade(t *testing.T) { c.consulCluster.Upgrade(t, c.helmValues()) } +func (c *ConnectHelper) KubectlOptsForApp(t *testing.T) *terratestK8s.KubectlOptions { + opts := c.Ctx.KubectlOptions(t) + if !c.UseAppNamespace { + return opts + } + return c.Ctx.KubectlOptionsForNamespace(opts.Namespace + "-apps") +} + // DeployClientAndServer deploys a client and server pod to the Kubernetes // cluster which will be used to test service mesh connectivity. If the Secure // flag is true, a pre-check is done to ensure that the ACL tokens for the test @@ -108,23 +123,46 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") - if c.Cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + c.setupAppNamespace(t) + + opts := c.KubectlOptsForApp(t) + if c.Cfg.EnableCNI && c.Cfg.EnableOpenshift { + // On OpenShift with the CNI, we need to create a network attachment definition in the namespace + // where the applications are running, and the app deployment configs need to reference that network + // attachment definition. + + // TODO: A base fixture is the wrong place for these files + k8s.KubectlApply(t, opts, "../fixtures/bases/openshift/") + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + k8s.KubectlDelete(t, opts, "../fixtures/bases/openshift/") + }) + + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift") + if c.Cfg.EnableTransparentProxy { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy") + } else { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject") + } } else { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + if c.Cfg.EnableTransparentProxy { + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + } else { + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + } } - // Check that both static-server and static-client have been injected and // now have 2 containers. retry.RunWith( &retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond}, t, func(r *retry.R) { for _, labelSelector := range []string{"app=static-server", "app=static-client"} { - podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: labelSelector, - FieldSelector: `status.phase=Running`, - }) + podList, err := c.Ctx.KubernetesClient(t).CoreV1(). + Pods(opts.Namespace). + List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelector, + FieldSelector: `status.phase=Running`, + }) require.NoError(r, err) require.Len(r, podList.Items, 1) require.Len(r, podList.Items[0].Spec.Containers, 2) @@ -132,16 +170,46 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { }) } +// setupAppNamespace creates a namespace where applications are deployed. This +// does nothing if UseAppNamespace is not set. The app namespace is relevant +// when testing with restricted PSA enforcement enabled. +func (c *ConnectHelper) setupAppNamespace(t *testing.T) { + if !c.UseAppNamespace { + return + } + opts := c.KubectlOptsForApp(t) + // If we are deploying apps in another namespace, create the namespace. + + _, err := k8s.RunKubectlAndGetOutputE(t, opts, "create", "ns", opts.Namespace) + if err != nil && strings.Contains(err.Error(), "AlreadyExists") { + return + } + require.NoError(t, err) + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + k8s.RunKubectl(t, opts, "delete", "ns", opts.Namespace) + }) + + if c.Cfg.EnableRestrictedPSAEnforcement { + // Allow anything to run in the app namespace. + k8s.RunKubectl(t, opts, "label", "--overwrite", "ns", opts.Namespace, + "pod-security.kubernetes.io/enforce=privileged", + "pod-security.kubernetes.io/enforce-version=v1.24", + ) + } + +} + // CreateResolverRedirect creates a resolver that redirects to a static-server, a corresponding k8s service, // and intentions. This helper is primarly used to ensure that the virtual-ips are persisted to consul properly. func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) { logger.Log(t, "creating resolver redirect") - options := c.Ctx.KubectlOptions(t) + opts := c.KubectlOptsForApp(t) + c.setupAppNamespace(t) kustomizeDir := "../fixtures/cases/resolver-redirect-virtualip" - k8s.KubectlApplyK(t, options, kustomizeDir) + k8s.KubectlApplyK(t, opts, kustomizeDir) helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { - k8s.KubectlDeleteK(t, options, kustomizeDir) + k8s.KubectlDeleteK(t, opts, kustomizeDir) }) } @@ -149,10 +217,11 @@ func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) { // server fails when no intentions are configured. func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) { logger.Log(t, "checking that the connection is not successful because there's no intention") + opts := c.KubectlOptsForApp(t) if c.Cfg.EnableTransparentProxy { - k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server") + k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://static-server") } else { - k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234") + k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://localhost:1234") } } @@ -177,11 +246,12 @@ func (c *ConnectHelper) CreateIntention(t *testing.T) { // static-client pod once the intention is set. func (c *ConnectHelper) TestConnectionSuccess(t *testing.T) { logger.Log(t, "checking that connection is successful") + opts := c.KubectlOptsForApp(t) if c.Cfg.EnableTransparentProxy { // todo: add an assertion that the traffic is going through the proxy - k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server") + k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://static-server") } else { - k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234") + k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://localhost:1234") } } @@ -192,8 +262,10 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) { // Test that kubernetes readiness status is synced to Consul. // Create a file called "unhealthy" at "/tmp/" so that the readiness probe // of the static-server pod fails. + opts := c.KubectlOptsForApp(t) + logger.Log(t, "testing k8s -> consul health checks sync by making the static-server unhealthy") - k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy") + k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy") // The readiness probe should take a moment to be reflected in Consul, // CheckStaticServerConnection will retry until Consul marks the service @@ -205,20 +277,20 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) { // other tests. logger.Log(t, "checking that connection is unsuccessful") if c.Cfg.EnableTransparentProxy { - k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{ + k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{ "curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", "curl: (7) Failed to connect to static-server port 80: Connection refused", }, "", "http://static-server") } else { - k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{ + k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{ "curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", }, "", "http://localhost:1234") } // Return the static-server to a "healthy state". - k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy") + k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy") } // helmValues uses the Secure and AutoEncrypt fields to set values for the Helm diff --git a/acceptance/framework/consul/helm_cluster_test.go b/acceptance/framework/consul/helm_cluster_test.go index 9c44006d43..011ca23e0f 100644 --- a/acceptance/framework/consul/helm_cluster_test.go +++ b/acceptance/framework/consul/helm_cluster_test.go @@ -8,6 +8,7 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/config" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" "github.com/stretchr/testify/require" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -80,9 +81,14 @@ func (c *ctx) Name() string { func (c *ctx) KubectlOptions(_ *testing.T) *k8s.KubectlOptions { return &k8s.KubectlOptions{} } +func (c *ctx) KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions { + return &k8s.KubectlOptions{} +} func (c *ctx) KubernetesClient(_ *testing.T) kubernetes.Interface { return fake.NewSimpleClientset() } func (c *ctx) ControllerRuntimeClient(_ *testing.T) client.Client { return runtimefake.NewClientBuilder().Build() } + +var _ environment.TestContext = (*ctx)(nil) diff --git a/acceptance/framework/environment/environment.go b/acceptance/framework/environment/environment.go index 9150f4ff03..7014a3c05f 100644 --- a/acceptance/framework/environment/environment.go +++ b/acceptance/framework/environment/environment.go @@ -35,6 +35,8 @@ type TestEnvironment interface { // for example, information about a specific Kubernetes cluster. type TestContext interface { KubectlOptions(t *testing.T) *k8s.KubectlOptions + // TODO: I don't love this. + KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions KubernetesClient(t *testing.T) kubernetes.Interface ControllerRuntimeClient(t *testing.T) client.Client } @@ -138,6 +140,14 @@ func (k kubernetesContext) KubectlOptions(t *testing.T) *k8s.KubectlOptions { return k.options } +func (k kubernetesContext) KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions { + return &k8s.KubectlOptions{ + ContextName: k.kubeContextName, + ConfigPath: k.pathToKubeConfig, + Namespace: ns, + } +} + // KubernetesClientFromOptions takes KubectlOptions and returns Kubernetes API client. func KubernetesClientFromOptions(t *testing.T, options *k8s.KubectlOptions) kubernetes.Interface { configPath, err := options.GetConfigPath(t) diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index e6a29de100..de413bbba5 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -27,7 +27,8 @@ type TestFlags struct { flagEnablePodSecurityPolicies bool - flagEnableCNI bool + flagEnableCNI bool + flagEnableRestrictedPSAEnforcement bool flagEnableTransparentProxy bool @@ -115,6 +116,13 @@ func (t *TestFlags) init() { flag.BoolVar(&t.flagEnableCNI, "enable-cni", false, "If true, the test suite will run tests with consul-cni plugin enabled. "+ "In general, this will only run against tests that are mesh related (connect, mesh-gateway, peering, etc") + flag.BoolVar(&t.flagEnableRestrictedPSAEnforcement, "enable-restricted-psa-enforcement", false, + "If true, this indicates that Consul is being run in a namespace with restricted PSA enforcement enabled. "+ + "The tests do not configure Consul's namespace with PSA enforcement enabled. This must configured before tests are run. "+ + "The CNI and test applications need more privilege than is allowed in a restricted namespace. "+ + "When set, the CNI will be deployed into the kube-system namespace, and in supported test cases, applications "+ + "are deployed, by default, into a namespace named '-apps' instead of being deployed into the "+ + "Consul namespace.") flag.BoolVar(&t.flagEnableTransparentProxy, "enable-transparent-proxy", false, "If true, the test suite will run tests with transparent proxy enabled. "+ @@ -172,6 +180,7 @@ func (t *TestFlags) Validate() error { if t.flagEnableEnterprise && t.flagEnterpriseLicense == "" { return errors.New("-enable-enterprise provided without setting env var CONSUL_ENT_LICENSE with consul license") } + return nil } @@ -195,7 +204,8 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { EnablePodSecurityPolicies: t.flagEnablePodSecurityPolicies, - EnableCNI: t.flagEnableCNI, + EnableCNI: t.flagEnableCNI, + EnableRestrictedPSAEnforcement: t.flagEnableRestrictedPSAEnforcement, EnableTransparentProxy: t.flagEnableTransparentProxy, diff --git a/acceptance/tests/connect/connect_external_servers_test.go b/acceptance/tests/connect/connect_external_servers_test.go index a7b0f656bf..c95d791773 100644 --- a/acceptance/tests/connect/connect_external_servers_test.go +++ b/acceptance/tests/connect/connect_external_servers_test.go @@ -30,6 +30,8 @@ func TestConnectInject_ExternalServers(t *testing.T) { caseName := fmt.Sprintf("secure: %t", secure) t.Run(caseName, func(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) serverHelmValues := map[string]string{ diff --git a/acceptance/tests/connect/connect_inject_namespaces_test.go b/acceptance/tests/connect/connect_inject_namespaces_test.go index f848594cd2..04021ec391 100644 --- a/acceptance/tests/connect/connect_inject_namespaces_test.go +++ b/acceptance/tests/connect/connect_inject_namespaces_test.go @@ -34,6 +34,7 @@ func TestConnectInjectNamespaces(t *testing.T) { if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) cases := []struct { name string @@ -246,6 +247,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) { if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) consulDestNS := "consul-dest" cases := []struct { diff --git a/acceptance/tests/connect/connect_inject_test.go b/acceptance/tests/connect/connect_inject_test.go index 199bbf0f1b..0704b16e54 100644 --- a/acceptance/tests/connect/connect_inject_test.go +++ b/acceptance/tests/connect/connect_inject_test.go @@ -38,11 +38,12 @@ func TestConnectInject(t *testing.T) { releaseName := helpers.RandomName() connHelper := connhelper.ConnectHelper{ - ClusterKind: consul.Helm, - Secure: c.secure, - ReleaseName: releaseName, - Ctx: ctx, - Cfg: cfg, + ClusterKind: consul.Helm, + Secure: c.secure, + ReleaseName: releaseName, + Ctx: ctx, + UseAppNamespace: cfg.EnableRestrictedPSAEnforcement, + Cfg: cfg, } connHelper.Setup(t) @@ -71,11 +72,12 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { releaseName := helpers.RandomName() connHelper := connhelper.ConnectHelper{ - ClusterKind: consul.Helm, - Secure: true, - ReleaseName: releaseName, - Ctx: ctx, - Cfg: cfg, + ClusterKind: consul.Helm, + Secure: true, + ReleaseName: releaseName, + Ctx: ctx, + UseAppNamespace: cfg.EnableRestrictedPSAEnforcement, + Cfg: cfg, } connHelper.Setup(t) @@ -84,7 +86,8 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { connHelper.CreateResolverRedirect(t) connHelper.DeployClientAndServer(t) - k8s.CheckStaticServerConnectionSuccessful(t, connHelper.Ctx.KubectlOptions(t), "static-client", "http://resolver-redirect") + opts := connHelper.KubectlOptsForApp(t) + k8s.CheckStaticServerConnectionSuccessful(t, opts, "static-client", "http://resolver-redirect") } // Test the endpoints controller cleans up force-killed pods. @@ -93,6 +96,9 @@ func TestConnectInject_CleanupKilledPods(t *testing.T) { name := fmt.Sprintf("secure: %t", secure) t.Run(name, func(t *testing.T) { cfg := suite.Config() + + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ @@ -161,6 +167,8 @@ func TestConnectInject_MultiportServices(t *testing.T) { name := fmt.Sprintf("secure: %t", secure) t.Run(name, func(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ diff --git a/acceptance/tests/connect/connect_proxy_lifecycle_test.go b/acceptance/tests/connect/connect_proxy_lifecycle_test.go index d2f6a0c383..7487847a0a 100644 --- a/acceptance/tests/connect/connect_proxy_lifecycle_test.go +++ b/acceptance/tests/connect/connect_proxy_lifecycle_test.go @@ -34,6 +34,7 @@ const ( // Test the endpoints controller cleans up force-killed pods. func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) for _, testCfg := range []LifecycleShutdownConfig{ {secure: false, helmValues: map[string]string{ diff --git a/acceptance/tests/connect/permissive_mtls_test.go b/acceptance/tests/connect/permissive_mtls_test.go index f0a55779ee..310d879d06 100644 --- a/acceptance/tests/connect/permissive_mtls_test.go +++ b/acceptance/tests/connect/permissive_mtls_test.go @@ -20,6 +20,7 @@ func TestConnectInject_PermissiveMTLS(t *testing.T) { if !cfg.EnableTransparentProxy { t.Skipf("skipping this because -enable-transparent-proxy is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) ctx := suite.Environment().DefaultContext(t) diff --git a/acceptance/tests/fixtures/bases/openshift/network-attachment.yaml b/acceptance/tests/fixtures/bases/openshift/network-attachment.yaml new file mode 100644 index 0000000000..4b3f7948ee --- /dev/null +++ b/acceptance/tests/fixtures/bases/openshift/network-attachment.yaml @@ -0,0 +1,17 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: consul-cni +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "consul-cni", + "cni_bin_dir": "/var/lib/cni/bin", + "cni_net_dir": "/etc/kubernetes/cni/net.d", + "kubeconfig": "ZZZ-consul-cni-kubeconfig", + "log_level": "debug", + "multus": true, + "name": "consul-cni", + "type": "consul-cni" + }' + diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml new file mode 100644 index 0000000000..4d4a53b87f --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml new file mode 100644 index 0000000000..8cc6d10411 --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml @@ -0,0 +1,14 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "consul.hashicorp.com/connect-service-upstreams": "static-server:1234" + "k8s.v1.cni.cncf.io/networks": '[{ "name":"consul-cni" }]' diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml new file mode 100644 index 0000000000..4d4a53b87f --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml new file mode 100644 index 0000000000..3b9c91fcc0 --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml @@ -0,0 +1,18 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# When using the CNI on OpenShift, we need to specify the +# network attachment definition for the pods to use. This assumes +# that one named 'consul-cni' was created by the acceptance tests. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "k8s.v1.cni.cncf.io/networks": '[{ "name":"consul-cni" }]' + diff --git a/acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml b/acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml new file mode 100644 index 0000000000..bc50c78adf --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../bases/static-server + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml b/acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml new file mode 100644 index 0000000000..8e2ed857f3 --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml @@ -0,0 +1,42 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "k8s.v1.cni.cncf.io/networks": '[{ "name":"consul-cni" }]' + spec: + containers: + - name: static-server + image: docker.mirror.hashicorp.services/kschoche/http-echo:latest + args: + - -text="hello world" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + livenessProbe: + httpGet: + port: 8080 + initialDelaySeconds: 1 + failureThreshold: 1 + periodSeconds: 1 + startupProbe: + httpGet: + port: 8080 + initialDelaySeconds: 1 + failureThreshold: 30 + periodSeconds: 1 + readinessProbe: + exec: + command: ['sh', '-c', 'test ! -f /tmp/unhealthy'] + initialDelaySeconds: 1 + failureThreshold: 1 + periodSeconds: 1 + serviceAccountName: static-server diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl index 1b866888c0..18f57b188c 100644 --- a/charts/consul/templates/_helpers.tpl +++ b/charts/consul/templates/_helpers.tpl @@ -15,6 +15,29 @@ as well as the global.name setting. {{- end -}} {{- end -}} +{{- define "consul.restrictedSecurityContext" -}} +{{- if not .Values.global.enablePodSecurityPolicies -}} +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault +{{- if not .Values.global.openshift.enabled -}} +{{/* +We must set runAsUser or else the root user will be used in some cases and +containers will fail to start due to runAsNonRoot above (e.g. +tls-init-cleanup). On OpenShift, runAsUser is automatically. We pick user 100 +because it is a non-root user id that exists in the consul, consul-dataplane, +and consul-k8s-control-plane images. +*/}} + runAsUser: 100 +{{- end -}} +{{- end -}} +{{- end -}} + {{- define "consul.vaultSecretTemplate" -}} | {{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} @@ -422,4 +445,4 @@ Usage: {{ template "consul.validateTelemetryCollectorCloud" . }} {{- if or (and .Values.telemetryCollector.cloud.clientSecret.secretName .Values.telemetryCollector.cloud.clientSecret.secretKey .Values.telemetryCollector.cloud.clientId.secretName .Values.telemetryCollector.cloud.clientId.secretKey (not .Values.global.cloud.resourceId.secretKey)) }} {{fail "When telemetryCollector has clientId and clientSecret .global.cloud.resourceId.secretKey must be set"}} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index 14c3961b4e..e726c9ecc9 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -94,6 +94,7 @@ spec: - containerPort: 8080 name: webhook-server protocol: TCP + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/gateway-cleanup-job.yaml b/charts/consul/templates/gateway-cleanup-job.yaml index 8731aaed81..a987c3b591 100644 --- a/charts/consul/templates/gateway-cleanup-job.yaml +++ b/charts/consul/templates/gateway-cleanup-job.yaml @@ -40,6 +40,7 @@ spec: containers: - name: gateway-cleanup image: {{ .Values.global.imageK8S }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} command: - consul-k8s-control-plane args: diff --git a/charts/consul/templates/gateway-resources-job.yaml b/charts/consul/templates/gateway-resources-job.yaml index 5fcd96cad3..3a29f75e66 100644 --- a/charts/consul/templates/gateway-resources-job.yaml +++ b/charts/consul/templates/gateway-resources-job.yaml @@ -40,6 +40,7 @@ spec: containers: - name: gateway-resources image: {{ .Values.global.imageK8S }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} command: - consul-k8s-control-plane args: diff --git a/charts/consul/templates/gossip-encryption-autogenerate-job.yaml b/charts/consul/templates/gossip-encryption-autogenerate-job.yaml index 9d296478a1..240bfe3f9c 100644 --- a/charts/consul/templates/gossip-encryption-autogenerate-job.yaml +++ b/charts/consul/templates/gossip-encryption-autogenerate-job.yaml @@ -48,6 +48,7 @@ spec: containers: - name: gossip-encryption-autogen image: "{{ .Values.global.imageK8S }}" + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} command: - "/bin/sh" - "-ec" diff --git a/charts/consul/templates/server-acl-init-cleanup-job.yaml b/charts/consul/templates/server-acl-init-cleanup-job.yaml index 4d0aa8f05d..c9f6763bd8 100644 --- a/charts/consul/templates/server-acl-init-cleanup-job.yaml +++ b/charts/consul/templates/server-acl-init-cleanup-job.yaml @@ -60,6 +60,9 @@ spec: containers: - name: server-acl-init-cleanup image: {{ .Values.global.imageK8S }} + {{- if not .Values.server.containerSecurityContext.aclInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} + {{- end }} command: - consul-k8s-control-plane args: diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml index 6625e23b38..c3d4a710e8 100644 --- a/charts/consul/templates/server-acl-init-job.yaml +++ b/charts/consul/templates/server-acl-init-job.yaml @@ -129,6 +129,9 @@ spec: containers: - name: server-acl-init-job image: {{ .Values.global.imageK8S }} + {{- if not .Values.server.containerSecurityContext.aclInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml index 0bde9b881a..04c84df71b 100644 --- a/charts/consul/templates/server-statefulset.yaml +++ b/charts/consul/templates/server-statefulset.yaml @@ -238,6 +238,7 @@ spec: volumeMounts: - name: extra-config mountPath: /consul/extra-config + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} containers: - name: consul image: "{{ default .Values.global.image .Values.server.image }}" @@ -526,9 +527,11 @@ spec: {{- toYaml .Values.server.resources | nindent 12 }} {{- end }} {{- end }} - {{- if not .Values.global.openshift.enabled }} + {{- if .Values.server.containerSecurityContext.server }} securityContext: {{- toYaml .Values.server.containerSecurityContext.server | nindent 12 }} + {{- else }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} {{- end }} {{- if .Values.server.extraContainers }} {{ toYaml .Values.server.extraContainers | nindent 8 }} diff --git a/charts/consul/templates/tls-init-cleanup-job.yaml b/charts/consul/templates/tls-init-cleanup-job.yaml index 69b5a30849..2254a38ed2 100644 --- a/charts/consul/templates/tls-init-cleanup-job.yaml +++ b/charts/consul/templates/tls-init-cleanup-job.yaml @@ -48,6 +48,9 @@ spec: containers: - name: tls-init-cleanup image: "{{ .Values.global.image }}" + {{- if not .Values.server.containerSecurityContext.tlsInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/tls-init-job.yaml b/charts/consul/templates/tls-init-job.yaml index 5839f07dbf..12d3acbad8 100644 --- a/charts/consul/templates/tls-init-job.yaml +++ b/charts/consul/templates/tls-init-job.yaml @@ -63,6 +63,9 @@ spec: containers: - name: tls-init image: "{{ .Values.global.imageK8S }}" + {{- if not .Values.server.containerSecurityContext.tlsInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/webhook-cert-manager-deployment.yaml b/charts/consul/templates/webhook-cert-manager-deployment.yaml index dd93c039d2..7ba25b330c 100644 --- a/charts/consul/templates/webhook-cert-manager-deployment.yaml +++ b/charts/consul/templates/webhook-cert-manager-deployment.yaml @@ -51,6 +51,7 @@ spec: -deployment-namespace={{ .Release.Namespace }} image: {{ .Values.global.imageK8S }} name: webhook-cert-manager + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} resources: limits: cpu: 100m diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats index 108fd9bbf8..a60884d20c 100755 --- a/charts/consul/test/unit/server-statefulset.bats +++ b/charts/consul/test/unit/server-statefulset.bats @@ -829,9 +829,9 @@ load _helpers } #-------------------------------------------------------------------- -# global.openshift.enabled & client.containerSecurityContext +# global.openshift.enabled && server.containerSecurityContext -@test "server/StatefulSet: container level securityContexts are not set when global.openshift.enabled=true" { +@test "server/StatefulSet: Can set container level securityContexts when global.openshift.enabled=true" { cd `chart_dir` local manifest=$(helm template \ -s templates/server-statefulset.yaml \ @@ -839,8 +839,72 @@ load _helpers --set 'server.containerSecurityContext.server.privileged=false' \ . | tee /dev/stderr) + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.containers | map(select(.name == "consul")) | .[0].securityContext.privileged') + [ "${actual}" = "false" ] +} + +#-------------------------------------------------------------------- +# global.openshift.enabled + +@test "server/StatefulSet: restricted container securityContexts are set when global.openshift.enabled=true" { + cd `chart_dir` + local manifest=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr) + + local expected=$(echo '{ + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "seccompProfile": { + "type": "RuntimeDefault" + } + }') + + # Check consul container local actual=$(echo "$manifest" | yq -r '.spec.template.spec.containers | map(select(.name == "consul")) | .[0].securityContext') - [ "${actual}" = "null" ] + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] + + # Check locality-init container + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.initContainers | map(select(.name == "locality-init")) | .[0].securityContext') + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] +} + +#-------------------------------------------------------------------- +# global.openshift.enabled = false + +@test "server/StatefulSet: restricted container securityContexts are set by default" { + cd `chart_dir` + local manifest=$(helm template \ + -s templates/server-statefulset.yaml \ + . | tee /dev/stderr) + + local expected=$(echo '{ + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "seccompProfile": { + "type": "RuntimeDefault" + }, + "runAsUser": 100 + }') + + # Check consul container + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.containers | map(select(.name == "consul")) | .[0].securityContext') + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] + + # Check locality-init container + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.initContainers | map(select(.name == "locality-init")) | .[0].securityContext') + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] } #--------------------------------------------------------------------