Skip to content

Commit

Permalink
[k8s] Add "Cmd", "Entrypoint", and "WorkingDir" translations (#2629)
Browse files Browse the repository at this point in the history 
Make  "Cmd", "Entrypoint", and "WorkingDir" of DockerOptions available in CombinedKubernetesConfig, and translate them to k8s options on the module container.
  • Loading branch information
@darobs
darobs authored Mar 3, 2020
1 parent 934840b commit 7cbc607
Show file tree
Hide file tree
Showing 5 changed files with 291 additions and 5 deletions.
16 changes: 15 additions & 1 deletion ...ent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,14 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
using Microsoft.Azure.Devices.Edge.Agent.Docker.Models;
using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment;
using Microsoft.Azure.Devices.Edge.Util;
using Newtonsoft.Json.Linq;

public class CombinedKubernetesConfigProvider : ICombinedConfigProvider<CombinedKubernetesConfig>
{
const string CmdKey = "Cmd";
const string EntrypointKey = "Entrypoint";
const string WorkingDirKey = "WorkingDir";

readonly CombinedDockerConfigProvider dockerConfigProvider;
readonly Uri workloadUri;
readonly Uri managementUri;
Expand Down Expand Up @@ -43,7 +48,10 @@ public CombinedKubernetesConfig GetCombinedConfig(IModule module, IRuntimeInfo r
dockerConfig.CreateOptions.ExposedPorts,
hostConfig,
dockerConfig.CreateOptions.Image,
dockerConfig.CreateOptions.Labels);
dockerConfig.CreateOptions.Labels,
GetPropertiesStringArray(CmdKey, dockerConfig.CreateOptions.OtherProperties),
GetPropertiesStringArray(EntrypointKey, dockerConfig.CreateOptions.OtherProperties),
GetPropertiesString(WorkingDirKey, dockerConfig.CreateOptions.OtherProperties));

if (this.enableKubernetesExtensions)
{
Expand Down Expand Up @@ -92,5 +100,11 @@ static string BindPath(Uri uri)
? Path.GetDirectoryName(uri.LocalPath)
: uri.AbsolutePath;
}

static IReadOnlyList<string> GetPropertiesStringArray(string key, IDictionary<string, JToken> other) =>
Option.Maybe(other).FlatMap(options => options.Get(key).FlatMap(option => Option.Maybe(option.ToObject<IReadOnlyList<string>>()))).OrDefault();

static string GetPropertiesString(string key, IDictionary<string, JToken> other) =>
Option.Maybe(other).FlatMap(options => options.Get(key).FlatMap(option => Option.Maybe(option.ToObject<string>()))).OrDefault();
}
}
30 changes: 27 additions & 3 deletions edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ public CreatePodParameters(
IDictionary<string, EmptyStruct> exposedPorts,
HostConfig hostConfig,
string image,
IDictionary<string, string> labels)
: this(env?.ToList(), exposedPorts, hostConfig, image, labels, null, null, null, null)
IDictionary<string, string> labels,
IEnumerable<string> cmd,
IEnumerable<string> entrypoint,
string workingDir)
: this(env?.ToList(), exposedPorts, hostConfig, image, labels, cmd?.ToList(), entrypoint?.ToList(), workingDir, null, null, null, null)
{
}

Expand All @@ -29,6 +32,9 @@ public CreatePodParameters(
HostConfig hostConfig,
string image,
IDictionary<string, string> labels,
IReadOnlyList<string> cmd,
IReadOnlyList<string> entrypoint,
string workingDir,
IDictionary<string, string> nodeSelector,
V1ResourceRequirements resources,
IReadOnlyList<KubernetesModuleVolumeSpec> volumes,
Expand All @@ -39,6 +45,9 @@ public CreatePodParameters(
this.HostConfig = Option.Maybe(hostConfig);
this.Image = Option.Maybe(image);
this.Labels = Option.Maybe(labels);
this.Cmd = Option.Maybe(cmd);
this.Entrypoint = Option.Maybe(entrypoint);
this.WorkingDir = Option.Maybe(workingDir);
this.NodeSelector = Option.Maybe(nodeSelector);
this.Resources = Option.Maybe(resources);
this.Volumes = Option.Maybe(volumes);
Expand All @@ -51,11 +60,14 @@ internal static CreatePodParameters Create(
HostConfig hostConfig = null,
string image = null,
IDictionary<string, string> labels = null,
IReadOnlyList<string> cmd = null,
IReadOnlyList<string> entrypoint = null,
string workingDir = null,
IDictionary<string, string> nodeSelector = null,
V1ResourceRequirements resources = null,
IReadOnlyList<KubernetesModuleVolumeSpec> volumes = null,
V1PodSecurityContext securityContext = null)
=> new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, nodeSelector, resources, volumes, securityContext);
=> new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, cmd, entrypoint, workingDir, nodeSelector, resources, volumes, securityContext);

[JsonProperty("env", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
Expand Down Expand Up @@ -92,5 +104,17 @@ internal static CreatePodParameters Create(
[JsonProperty("securityContext", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<V1PodSecurityContext>))]
public Option<V1PodSecurityContext> SecurityContext { get; set; }

[JsonProperty("cmd", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
public Option<IReadOnlyList<string>> Cmd { get; }

[JsonProperty("entrypoint", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
public Option<IReadOnlyList<string>> Entrypoint { get; }

[JsonProperty("workingDir", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<string>))]
public Option<string> WorkingDir { get; }
}
}
5 changes: 4 additions & 1 deletion ...ure.Devices.Edge.Agent.Kubernetes/edgedeployment/deployment/KubernetesDeploymentMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,10 @@ V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule
VolumeMounts = volumeMounts,
SecurityContext = securityContext.OrDefault(),
Ports = exposedPorts.OrDefault(),
Resources = module.Config.CreateOptions.Resources.OrDefault()
Resources = module.Config.CreateOptions.Resources.OrDefault(),
Command = module.Config.CreateOptions.Entrypoint.Map(list => list.ToList()).OrDefault(),
Args = module.Config.CreateOptions.Cmd.Map(list => list.ToList()).OrDefault(),
WorkingDir = module.Config.CreateOptions.WorkingDir.OrDefault(),
};

return (container, volumes);
Expand Down
174 changes: 174 additions & 0 deletions ...icrosoft.Azure.Devices.Edge.Agent.Kubernetes.Test/CombinedKubernetesConfigProviderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test
using Microsoft.Azure.Devices.Edge.Agent.Docker;
using Microsoft.Azure.Devices.Edge.Util.Test.Common;
using Moq;
using Newtonsoft.Json;
using Xunit;
using CoreConstants = Microsoft.Azure.Devices.Edge.Agent.Core.Constants;

Expand Down Expand Up @@ -205,5 +206,178 @@ public void MakesKubernetesAwareAuthConfig()
Assert.True(config.ImagePullSecret.HasValue);
config.ImagePullSecret.ForEach(secret => Assert.Equal("user-docker.io", secret.Name));
}

[Fact]
public void NoExecArgumentMeansNoExecArguments()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", string.Empty));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.False(config.CreateOptions.Cmd.HasValue);
Assert.False(config.CreateOptions.Entrypoint.HasValue);
Assert.False(config.CreateOptions.WorkingDir.HasValue);
}

const string CmdCreateOptions =
@"{
""Cmd"" : [
""argument1"",
""argument2""
]
}";

[Fact]
public void CmdEntryOptionsWillExist()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", CmdCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.True(config.CreateOptions.Cmd.HasValue);
config.CreateOptions.Cmd.ForEach(cmd =>
{
Assert.Equal("argument1", cmd[0]);
Assert.Equal("argument2", cmd[1]);
});
}

const string EntryPointCreateOptions =
@"{
""Entrypoint"" : [
""a-command""
]
}";

[Fact]
public void EntrypointOptionsWillExist()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", EntryPointCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.True(config.CreateOptions.Entrypoint.HasValue);
config.CreateOptions.Entrypoint.ForEach(ep => Assert.Equal("a-command", ep[0]));
}

const string WorkingDirCreateOptions =
@"{
""WorkingDir"" : ""a-directory""
}";

[Fact]
public void WorkingDirOptionsWillExist()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", WorkingDirCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.True(config.CreateOptions.WorkingDir.HasValue);
config.CreateOptions.WorkingDir.ForEach(wd => Assert.Equal("a-directory", wd));
}

const string InvalidCmdCreateOptions =
@"{
""Cmd"" : {
""argument1"": ""argument2""
}
}";

[Fact]
public void InvalidCmdEntryOptionsThrows()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", InvalidCmdCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
// Assert
Assert.Throws<JsonSerializationException>(() => provider.GetCombinedConfig(module.Object, runtimeInfo.Object));
}

const string InvalidEntryPointCreateOptions =
@"{
""Entrypoint"" : ""a-command""
}";

[Fact]
public void InvalidEntrypointOptionsThrows()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", InvalidEntryPointCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
// Assert
Assert.Throws<JsonSerializationException>(() => provider.GetCombinedConfig(module.Object, runtimeInfo.Object));
}

const string InvalidWorkingDirCreateOptions =
@"{
""WorkingDir"" : [ ""/tmp/working"" ]
}";

[Fact]
public void InvalidWorkingDirOptionsThrows()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", InvalidWorkingDirCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
// Assert
Assert.Throws<ArgumentException>(() => provider.GetCombinedConfig(module.Object, runtimeInfo.Object));
}
}
}
71 changes: 71 additions & 0 deletions ...test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesDeploymentMapperTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,77 @@ public void ApplyPodSecurityContextFromCreateOptionsWhenProvided()
Assert.Equal(20001, deployment.Spec.Template.Spec.SecurityContext.RunAsUser);
}

[Fact]
public void NoCmdOptionsNoContainerArgs()
{
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.Null(container.Args);
Assert.Null(container.Command);
Assert.Null(container.WorkingDir);
}

[Fact]
public void CmdOptionsContainerArgs()
{
var cmd = new List<string> { "argument1", "argument2" };
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(cmd: cmd), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.NotNull(container.Args);
Assert.Equal(2, container.Args.Count);
Assert.Equal("argument1", container.Args[0]);
Assert.Equal("argument2", container.Args[1]);
}

[Fact]
public void EntrypointOptionsContainerCommands()
{
var entrypoint = new List<string> { "command", "argument-a" };
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(entrypoint: entrypoint), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.NotNull(container.Command);
Assert.Equal(2, container.Command.Count);
Assert.Equal("command", container.Command[0]);
Assert.Equal("argument-a", container.Command[1]);
}

[Fact]
public void WorkingDirOptionsContainerWorkingDir()
{
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(workingDir: "/tmp/working"), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.NotNull(container.WorkingDir);
Assert.Equal("/tmp/working", container.WorkingDir);
}

[Fact]
public void EdgeAgentEnvSettingsHaveLotsOfStuff()
{
Expand Down

0 comments on commit 7cbc607

Please sign in to comment.