From cf2eba9518a947bb09ebff5dcd6ae42f66d2d045 Mon Sep 17 00:00:00 2001 From: Denis Molokanov Date: Mon, 2 Dec 2019 12:07:53 -0800 Subject: [PATCH] [k8s] Add pod security context to create options (#2008) * Add pod security context to create options --- .../CombinedKubernetesConfigProvider.cs | 1 + .../CreatePodParameters.cs | 15 ++++-- ...bernetesExperimentalCreatePodParameters.cs | 15 ++++-- .../deployment/KubernetesDeploymentMapper.cs | 7 +-- .../KubernetesDeploymentMapperTest.cs | 41 +++++++++++++- ...KubernetesExperimentalPodParametersTest.cs | 54 +++++++++++++++++++ kubernetes/doc/create-options.md | 21 ++++++++ 7 files changed, 143 insertions(+), 11 deletions(-) diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs index eada7c13163..84dd587d611 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs @@ -51,6 +51,7 @@ public CombinedKubernetesConfig GetCombinedConfig(IModule module, IRuntimeInfo r experimentalOptions.ForEach(parameters => createOptions.Volumes = parameters.Volumes); experimentalOptions.ForEach(parameters => createOptions.NodeSelector = parameters.NodeSelector); experimentalOptions.ForEach(parameters => createOptions.Resources = parameters.Resources); + experimentalOptions.ForEach(parameters => createOptions.SecurityContext = parameters.SecurityContext); } Option imagePullSecret = dockerConfig.AuthConfig diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs index 9dd7c7251f5..f32b20d85a2 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs @@ -18,7 +18,7 @@ public CreatePodParameters( HostConfig hostConfig, string image, IDictionary labels) - : this(env?.ToList(), exposedPorts, hostConfig, image, labels, null, null, null) + : this(env?.ToList(), exposedPorts, hostConfig, image, labels, null, null, null, null) { } @@ -31,7 +31,8 @@ public CreatePodParameters( IDictionary labels, IDictionary nodeSelector, V1ResourceRequirements resources, - IReadOnlyList volumes) + IReadOnlyList volumes, + V1PodSecurityContext securityContext) { this.Env = Option.Maybe(env); this.ExposedPorts = Option.Maybe(exposedPorts); @@ -41,6 +42,7 @@ public CreatePodParameters( this.NodeSelector = Option.Maybe(nodeSelector); this.Resources = Option.Maybe(resources); this.Volumes = Option.Maybe(volumes); + this.SecurityContext = Option.Maybe(securityContext); } internal static CreatePodParameters Create( @@ -51,8 +53,9 @@ internal static CreatePodParameters Create( IDictionary labels = null, IDictionary nodeSelector = null, V1ResourceRequirements resources = null, - IReadOnlyList volumes = null) - => new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, nodeSelector, resources, volumes); + IReadOnlyList volumes = null, + V1PodSecurityContext securityContext = null) + => new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, nodeSelector, resources, volumes, securityContext); [JsonProperty("env", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] [JsonConverter(typeof(OptionConverter>))] @@ -85,5 +88,9 @@ internal static CreatePodParameters Create( [JsonProperty("volumes", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] [JsonConverter(typeof(OptionConverter>))] public Option> Volumes { get; set; } + + [JsonProperty("securityContext", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + [JsonConverter(typeof(OptionConverter))] + public Option SecurityContext { get; set; } } } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesExperimentalCreatePodParameters.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesExperimentalCreatePodParameters.cs index e3960348559..290633f7c24 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesExperimentalCreatePodParameters.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesExperimentalCreatePodParameters.cs @@ -17,14 +17,18 @@ public class KubernetesExperimentalCreatePodParameters public Option> Volumes { get; } + public Option SecurityContext { get; } + KubernetesExperimentalCreatePodParameters( Option> nodeSelector, Option resources, - Option> volumes) + Option> volumes, + Option securityContext) { this.NodeSelector = nodeSelector; this.Resources = resources; this.Volumes = volumes; + this.SecurityContext = securityContext; } static class ExperimentalParameterNames @@ -33,6 +37,7 @@ static class ExperimentalParameterNames public const string NodeSelector = "NodeSelector"; public const string Resources = "Resources"; public const string Volumes = "Volumes"; + public const string SecurityContext = "SecurityContext"; } public static Option Parse(IDictionary other) @@ -61,7 +66,10 @@ static KubernetesExperimentalCreatePodParameters ParseParameters(JObject experim var volumes = options.Get(ExperimentalParameterNames.Volumes) .FlatMap(option => Option.Maybe(option.ToObject>())); - return new KubernetesExperimentalCreatePodParameters(nodeSelector, resources, volumes); + var securityContext = options.Get(ExperimentalParameterNames.SecurityContext) + .FlatMap(option => Option.Maybe(option.ToObject())); + + return new KubernetesExperimentalCreatePodParameters(nodeSelector, resources, volumes, securityContext); } static Dictionary PrepareSupportedOptionsStore(JObject experimental) @@ -85,7 +93,8 @@ static Dictionary PrepareSupportedOptionsStore(JObject experimen { ExperimentalParameterNames.NodeSelector, ExperimentalParameterNames.Resources, - ExperimentalParameterNames.Volumes + ExperimentalParameterNames.Volumes, + ExperimentalParameterNames.SecurityContext }; static class Events diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/deployment/KubernetesDeploymentMapper.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/deployment/KubernetesDeploymentMapper.cs index 1603135923b..7269da48d93 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/deployment/KubernetesDeploymentMapper.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/deployment/KubernetesDeploymentMapper.cs @@ -132,9 +132,10 @@ V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule .Select(pullSecretName => new V1LocalObjectReference(pullSecretName)) .ToList(); - V1PodSecurityContext securityContext = this.runAsNonRoot - ? new V1PodSecurityContext { RunAsNonRoot = true, RunAsUser = 1000 } - : null; + V1PodSecurityContext securityContext = module.Config.CreateOptions.SecurityContext.GetOrElse( + () => this.runAsNonRoot + ? new V1PodSecurityContext { RunAsNonRoot = true, RunAsUser = 1000 } + : null); return new V1PodTemplateSpec { diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesDeploymentMapperTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesDeploymentMapperTest.cs index 5cb806fadec..ad6ec584e66 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesDeploymentMapperTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesDeploymentMapperTest.cs @@ -520,6 +520,40 @@ public void RunAsNonRootAndRunAsUser1000SecurityPolicyWhenSettingSet() Assert.Equal(1000, deployment.Spec.Template.Spec.SecurityContext.RunAsUser); } + [Fact] + public void PodSecurityContextFromCreateOptionsOverridesDefaultRunAsNonRootOptionsWhenProvided() + { + var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of()); + var securityContext = new V1PodSecurityContext { RunAsNonRoot = true, RunAsUser = 0 }; + var config = new KubernetesConfig("image", CreatePodParameters.Create(securityContext: securityContext), Option.Some(new AuthConfig("user-registry1"))); + var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner); + var labels = new Dictionary(); + var mapper = CreateMapper(runAsNonRoot: true); + + var deployment = mapper.CreateDeployment(identity, module, labels); + + Assert.Equal(1, deployment.Spec.Template.Spec.ImagePullSecrets.Count); + Assert.Equal(true, deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot); + Assert.Equal(0, deployment.Spec.Template.Spec.SecurityContext.RunAsUser); + } + + [Fact] + public void ApplyPodSecurityContextFromCreateOptionsWhenProvided() + { + var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of()); + var securityContext = new V1PodSecurityContext { RunAsNonRoot = true, RunAsUser = 20001 }; + var config = new KubernetesConfig("image", CreatePodParameters.Create(securityContext: securityContext), Option.Some(new AuthConfig("user-registry1"))); + var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner); + var labels = new Dictionary(); + var mapper = CreateMapper(); + + var deployment = mapper.CreateDeployment(identity, module, labels); + + Assert.Equal(1, deployment.Spec.Template.Spec.ImagePullSecrets.Count); + Assert.Equal(true, deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot); + Assert.Equal(20001, deployment.Spec.Template.Spec.SecurityContext.RunAsUser); + } + [Fact] public void EdgeAgentEnvSettingsHaveLotsOfStuff() { @@ -564,7 +598,12 @@ public void EdgeAgentEnvSettingsHaveLotsOfStuff() Assert.Equal("False", container.Env.Single(e => e.Name == "feature2").Value); } - static KubernetesDeploymentMapper CreateMapper(string persistentVolumeName = "", string storageClassName = "", string proxyImagePullSecretName = null, bool runAsNonRoot = false, IDictionary experimentalFeatures = null) + static KubernetesDeploymentMapper CreateMapper( + string persistentVolumeName = "", + string storageClassName = "", + string proxyImagePullSecretName = null, + bool runAsNonRoot = false, + IDictionary experimentalFeatures = null) => new KubernetesDeploymentMapper( "namespace", "edgehub", diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesExperimentalPodParametersTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesExperimentalPodParametersTest.cs index 0edd648da27..65880f1787c 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesExperimentalPodParametersTest.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesExperimentalPodParametersTest.cs @@ -67,6 +67,7 @@ public void IgnoresUnsupportedOptions() Assert.False(parameters.Volumes.HasValue); Assert.False(parameters.NodeSelector.HasValue); Assert.False(parameters.Resources.HasValue); + Assert.False(parameters.SecurityContext.HasValue); } [Fact] @@ -256,5 +257,58 @@ public void ParsesVolumesExperimentalOptions() volumeSpec.VolumeMounts.ForEach(mounts => Assert.Equal(true, mounts[0].ReadOnlyProperty)); volumeSpec.VolumeMounts.ForEach(mounts => Assert.Equal(string.Empty, mounts[0].SubPath)); } + + [Fact] + public void ParsesNoneSecurityContextExperimentalOptions() + { + var experimental = new Dictionary + { + ["k8s-experimental"] = JToken.Parse("{ securityContext: null }") + }; + + var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault(); + + Assert.False(parameters.SecurityContext.HasValue); + } + + [Fact] + public void ParsesEmptySecurityContextExperimentalOptions() + { + var experimental = new Dictionary + { + ["k8s-experimental"] = JToken.Parse("{ securityContext: { } }") + }; + + var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault(); + + Assert.True(parameters.SecurityContext.HasValue); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.RunAsGroup)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.RunAsUser)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.RunAsNonRoot)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.Sysctls)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.FsGroup)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.SeLinuxOptions)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.SupplementalGroups)); + } + + [Fact] + public void ParsesSomeSecurityContextExperimentalOptions() + { + var experimental = new Dictionary + { + ["k8s-experimental"] = JToken.Parse("{ securityContext: { runAsGroup: 1001, runAsUser: 1000, runAsNonRoot: true } }") + }; + + var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault(); + + Assert.True(parameters.SecurityContext.HasValue); + parameters.SecurityContext.ForEach(securityContext => Assert.Equal(1001, securityContext.RunAsGroup)); + parameters.SecurityContext.ForEach(securityContext => Assert.Equal(1000, securityContext.RunAsUser)); + parameters.SecurityContext.ForEach(securityContext => Assert.Equal(true, securityContext.RunAsNonRoot)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.Sysctls)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.FsGroup)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.SeLinuxOptions)); + parameters.SecurityContext.ForEach(securityContext => Assert.Null(securityContext.SupplementalGroups)); + } } } diff --git a/kubernetes/doc/create-options.md b/kubernetes/doc/create-options.md index 2e47e3fadd2..887da674115 100644 --- a/kubernetes/doc/create-options.md +++ b/kubernetes/doc/create-options.md @@ -21,6 +21,7 @@ We added CreateOptions for experimental features on Kubernetes. These options "o "volumes": [{...}], "resources": [{...}], "nodeSelector": {...} + "securityContext": {...}, } } ``` @@ -118,5 +119,25 @@ A `nodeSelector` section of config used for node selection constrain. It specifi } ``` +## Apply Pod Security Context +By default EdgeAgent doesn't make any assumptions about pod security context and allows to run containers under the user this container was built. With this section user can specify exact pod security policy module will run with. + +### CreateOptions + +A `securityContext` section of config used to apply a pod security context to a module pod spec. The section has the same structure as a Kubernetes [Pod spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#podsecuritycontext-v1-core). +`EdgeAgent` doesn't do any translations or interpretations of values but simply assigns value from module deployment to `securityContext` parameter of a pod spec. + +```json +{ + "k8s-experimental": { + "securityContext": { + "fsGroup": "1", + "runAsGroup": "1001", + "runAsUser": "1000", + ... + } + } +} +``` \ No newline at end of file