Skip to content

Commit

Permalink
[k8s] Add "Cmd", "Entrypoint", and "WorkingDir" translations (Azure#2629
Browse files Browse the repository at this point in the history 
)

Make  "Cmd", "Entrypoint", and "WorkingDir" of DockerOptions available in CombinedKubernetesConfig, and translate them to k8s options on the module container.
  • Loading branch information
@darobs
darobs committed Mar 3, 2020
1 parent 0858d84 commit 9d0be52
Show file tree
Hide file tree
Showing 5 changed files with 291 additions and 5 deletions.
16 changes: 15 additions & 1 deletion ...ent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,14 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
using Microsoft.Azure.Devices.Edge.Agent.Docker.Models;
using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment;
using Microsoft.Azure.Devices.Edge.Util;
using Newtonsoft.Json.Linq;

public class CombinedKubernetesConfigProvider : ICombinedConfigProvider<CombinedKubernetesConfig>
{
const string CmdKey = "Cmd";
const string EntrypointKey = "Entrypoint";
const string WorkingDirKey = "WorkingDir";

readonly CombinedDockerConfigProvider dockerConfigProvider;
readonly Uri workloadUri;
readonly Uri managementUri;
Expand Down Expand Up @@ -43,7 +48,10 @@ public CombinedKubernetesConfig GetCombinedConfig(IModule module, IRuntimeInfo r
dockerConfig.CreateOptions.ExposedPorts,
hostConfig,
dockerConfig.CreateOptions.Image,
dockerConfig.CreateOptions.Labels);
dockerConfig.CreateOptions.Labels,
GetPropertiesStringArray(CmdKey, dockerConfig.CreateOptions.OtherProperties),
GetPropertiesStringArray(EntrypointKey, dockerConfig.CreateOptions.OtherProperties),
GetPropertiesString(WorkingDirKey, dockerConfig.CreateOptions.OtherProperties));

if (this.enableKubernetesExtensions)
{
Expand Down Expand Up @@ -92,5 +100,11 @@ static string BindPath(Uri uri)
? Path.GetDirectoryName(uri.LocalPath)
: uri.AbsolutePath;
}

static IReadOnlyList<string> GetPropertiesStringArray(string key, IDictionary<string, JToken> other) =>
Option.Maybe(other).FlatMap(options => options.Get(key).FlatMap(option => Option.Maybe(option.ToObject<IReadOnlyList<string>>()))).OrDefault();

static string GetPropertiesString(string key, IDictionary<string, JToken> other) =>
Option.Maybe(other).FlatMap(options => options.Get(key).FlatMap(option => Option.Maybe(option.ToObject<string>()))).OrDefault();
}
}
30 changes: 27 additions & 3 deletions edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ public CreatePodParameters(
IDictionary<string, EmptyStruct> exposedPorts,
HostConfig hostConfig,
string image,
IDictionary<string, string> labels)
: this(env?.ToList(), exposedPorts, hostConfig, image, labels, null, null, null, null)
IDictionary<string, string> labels,
IEnumerable<string> cmd,
IEnumerable<string> entrypoint,
string workingDir)
: this(env?.ToList(), exposedPorts, hostConfig, image, labels, cmd?.ToList(), entrypoint?.ToList(), workingDir, null, null, null, null)
{
}

Expand All @@ -29,6 +32,9 @@ public CreatePodParameters(
HostConfig hostConfig,
string image,
IDictionary<string, string> labels,
IReadOnlyList<string> cmd,
IReadOnlyList<string> entrypoint,
string workingDir,
IDictionary<string, string> nodeSelector,
V1ResourceRequirements resources,
IReadOnlyList<KubernetesModuleVolumeSpec> volumes,
Expand All @@ -39,6 +45,9 @@ public CreatePodParameters(
this.HostConfig = Option.Maybe(hostConfig);
this.Image = Option.Maybe(image);
this.Labels = Option.Maybe(labels);
this.Cmd = Option.Maybe(cmd);
this.Entrypoint = Option.Maybe(entrypoint);
this.WorkingDir = Option.Maybe(workingDir);
this.NodeSelector = Option.Maybe(nodeSelector);
this.Resources = Option.Maybe(resources);
this.Volumes = Option.Maybe(volumes);
Expand All @@ -51,11 +60,14 @@ internal static CreatePodParameters Create(
HostConfig hostConfig = null,
string image = null,
IDictionary<string, string> labels = null,
IReadOnlyList<string> cmd = null,
IReadOnlyList<string> entrypoint = null,
string workingDir = null,
IDictionary<string, string> nodeSelector = null,
V1ResourceRequirements resources = null,
IReadOnlyList<KubernetesModuleVolumeSpec> volumes = null,
V1PodSecurityContext securityContext = null)
=> new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, nodeSelector, resources, volumes, securityContext);
=> new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, cmd, entrypoint, workingDir, nodeSelector, resources, volumes, securityContext);

[JsonProperty("env", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
Expand Down Expand Up @@ -92,5 +104,17 @@ internal static CreatePodParameters Create(
[JsonProperty("securityContext", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<V1PodSecurityContext>))]
public Option<V1PodSecurityContext> SecurityContext { get; set; }

[JsonProperty("cmd", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
public Option<IReadOnlyList<string>> Cmd { get; }

[JsonProperty("entrypoint", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
public Option<IReadOnlyList<string>> Entrypoint { get; }

[JsonProperty("workingDir", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
[JsonConverter(typeof(OptionConverter<string>))]
public Option<string> WorkingDir { get; }
}
}
5 changes: 4 additions & 1 deletion ...ure.Devices.Edge.Agent.Kubernetes/edgedeployment/deployment/KubernetesDeploymentMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,10 @@ V1PodTemplateSpec GetPod(string name, IModuleIdentity identity, KubernetesModule
VolumeMounts = volumeMounts,
SecurityContext = securityContext.OrDefault(),
Ports = exposedPorts.OrDefault(),
Resources = module.Config.CreateOptions.Resources.OrDefault()
Resources = module.Config.CreateOptions.Resources.OrDefault(),
Command = module.Config.CreateOptions.Entrypoint.Map(list => list.ToList()).OrDefault(),
Args = module.Config.CreateOptions.Cmd.Map(list => list.ToList()).OrDefault(),
WorkingDir = module.Config.CreateOptions.WorkingDir.OrDefault(),
};

return (container, volumes);
Expand Down
174 changes: 174 additions & 0 deletions ...icrosoft.Azure.Devices.Edge.Agent.Kubernetes.Test/CombinedKubernetesConfigProviderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test
using Microsoft.Azure.Devices.Edge.Agent.Docker;
using Microsoft.Azure.Devices.Edge.Util.Test.Common;
using Moq;
using Newtonsoft.Json;
using Xunit;
using CoreConstants = Microsoft.Azure.Devices.Edge.Agent.Core.Constants;

Expand Down Expand Up @@ -205,5 +206,178 @@ public void MakesKubernetesAwareAuthConfig()
Assert.True(config.ImagePullSecret.HasValue);
config.ImagePullSecret.ForEach(secret => Assert.Equal("user-docker.io", secret.Name));
}

[Fact]
public void NoExecArgumentMeansNoExecArguments()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", string.Empty));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.False(config.CreateOptions.Cmd.HasValue);
Assert.False(config.CreateOptions.Entrypoint.HasValue);
Assert.False(config.CreateOptions.WorkingDir.HasValue);
}

const string CmdCreateOptions =
@"{
""Cmd"" : [
""argument1"",
""argument2""
]
}";

[Fact]
public void CmdEntryOptionsWillExist()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", CmdCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.True(config.CreateOptions.Cmd.HasValue);
config.CreateOptions.Cmd.ForEach(cmd =>
{
Assert.Equal("argument1", cmd[0]);
Assert.Equal("argument2", cmd[1]);
});
}

const string EntryPointCreateOptions =
@"{
""Entrypoint"" : [
""a-command""
]
}";

[Fact]
public void EntrypointOptionsWillExist()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", EntryPointCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.True(config.CreateOptions.Entrypoint.HasValue);
config.CreateOptions.Entrypoint.ForEach(ep => Assert.Equal("a-command", ep[0]));
}

const string WorkingDirCreateOptions =
@"{
""WorkingDir"" : ""a-directory""
}";

[Fact]
public void WorkingDirOptionsWillExist()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", WorkingDirCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
CombinedKubernetesConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object);

// Assert
Assert.True(config.CreateOptions.WorkingDir.HasValue);
config.CreateOptions.WorkingDir.ForEach(wd => Assert.Equal("a-directory", wd));
}

const string InvalidCmdCreateOptions =
@"{
""Cmd"" : {
""argument1"": ""argument2""
}
}";

[Fact]
public void InvalidCmdEntryOptionsThrows()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", InvalidCmdCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
// Assert
Assert.Throws<JsonSerializationException>(() => provider.GetCombinedConfig(module.Object, runtimeInfo.Object));
}

const string InvalidEntryPointCreateOptions =
@"{
""Entrypoint"" : ""a-command""
}";

[Fact]
public void InvalidEntrypointOptionsThrows()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", InvalidEntryPointCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
// Assert
Assert.Throws<JsonSerializationException>(() => provider.GetCombinedConfig(module.Object, runtimeInfo.Object));
}

const string InvalidWorkingDirCreateOptions =
@"{
""WorkingDir"" : [ ""/tmp/working"" ]
}";

[Fact]
public void InvalidWorkingDirOptionsThrows()
{
var runtimeInfo = new Mock<IRuntimeInfo<DockerRuntimeConfig>>();
runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty));

var module = new Mock<IModule<DockerConfig>>();
module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", InvalidWorkingDirCreateOptions));
module.SetupGet(m => m.Name).Returns("mod1");

CombinedKubernetesConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, new Uri("unix:///var/run/iotedgedworkload.sock"), new Uri("unix:///var/run/iotedgedmgmt.sock"), true);

// Act
// Assert
Assert.Throws<ArgumentException>(() => provider.GetCombinedConfig(module.Object, runtimeInfo.Object));
}
}
}
71 changes: 71 additions & 0 deletions ...test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesDeploymentMapperTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,77 @@ public void ApplyPodSecurityContextFromCreateOptionsWhenProvided()
Assert.Equal(20001, deployment.Spec.Template.Spec.SecurityContext.RunAsUser);
}

[Fact]
public void NoCmdOptionsNoContainerArgs()
{
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.Null(container.Args);
Assert.Null(container.Command);
Assert.Null(container.WorkingDir);
}

[Fact]
public void CmdOptionsContainerArgs()
{
var cmd = new List<string> { "argument1", "argument2" };
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(cmd: cmd), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.NotNull(container.Args);
Assert.Equal(2, container.Args.Count);
Assert.Equal("argument1", container.Args[0]);
Assert.Equal("argument2", container.Args[1]);
}

[Fact]
public void EntrypointOptionsContainerCommands()
{
var entrypoint = new List<string> { "command", "argument-a" };
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(entrypoint: entrypoint), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.NotNull(container.Command);
Assert.Equal(2, container.Command.Count);
Assert.Equal("command", container.Command[0]);
Assert.Equal("argument-a", container.Command[1]);
}

[Fact]
public void WorkingDirOptionsContainerWorkingDir()
{
var identity = new ModuleIdentity("hostname", "gatewayhost", "deviceid", "Module1", Mock.Of<ICredentials>());
var config = new KubernetesConfig("image", CreatePodParameters.Create(workingDir: "/tmp/working"), Option.Some(new AuthConfig("user-registry1")));
var module = new KubernetesModule("module1", "v1", "docker", ModuleStatus.Running, RestartPolicy.Always, DefaultConfigurationInfo, EnvVarsDict, config, ImagePullPolicy.OnCreate, EdgeletModuleOwner);
var labels = new Dictionary<string, string>();
var mapper = CreateMapper();

var deployment = mapper.CreateDeployment(identity, module, labels);

var container = deployment.Spec.Template.Spec.Containers.Single(c => c.Name == "module1");
Assert.NotNull(container.WorkingDir);
Assert.Equal("/tmp/working", container.WorkingDir);
}

[Fact]
public void EdgeAgentEnvSettingsHaveLotsOfStuff()
{
Expand Down

0 comments on commit 9d0be52

Please sign in to comment.