diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs new file mode 100644 index 00000000000..f1edc2c2a6c --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Runtime.InteropServices; + using global::Docker.DotNet.Models; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Docker; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Extensions.Configuration; + using Newtonsoft.Json; + + public class CombinedKubernetesConfigProvider : CombinedDockerConfigProvider + { + readonly IConfigSource configSource; + + public CombinedKubernetesConfigProvider(IEnumerable authConfigs, IConfigSource configSource) + : base(authConfigs) + { + this.configSource = Preconditions.CheckNotNull(configSource, nameof(configSource)); + } + + static CreateContainerParameters CloneOrCreateParams(CreateContainerParameters createOptions) => + createOptions != null + ? JsonConvert.DeserializeObject(JsonConvert.SerializeObject(createOptions)) + : new CreateContainerParameters(); + + public override CombinedDockerConfig GetCombinedConfig(IModule module, IRuntimeInfo runtimeInfo) + { + CombinedDockerConfig combinedConfig = base.GetCombinedConfig(module, runtimeInfo); + + // if the workload URI is a Unix domain socket then volume mount it into the container + CreateContainerParameters createOptions = CloneOrCreateParams(combinedConfig.CreateOptions); + this.MountSockets(module, createOptions); + this.InjectNetworkAliases(module, createOptions); + + return new CombinedDockerConfig(combinedConfig.Image, createOptions, combinedConfig.AuthConfig); + } + + void InjectNetworkAliases(IModule module, CreateContainerParameters createOptions) + { + if (createOptions.NetworkingConfig?.EndpointsConfig == null) + { + string networkId = this.configSource.Configuration.GetValue(Core.Constants.NetworkIdKey); + string edgeDeviceHostName = this.configSource.Configuration.GetValue(Core.Constants.EdgeDeviceHostNameKey); + + if (!string.IsNullOrWhiteSpace(networkId)) + { + var endpointSettings = new EndpointSettings(); + if (module.Name.Equals(Core.Constants.EdgeHubModuleName, StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(edgeDeviceHostName)) + { + endpointSettings.Aliases = new List + { + edgeDeviceHostName + }; + } + + IDictionary endpointsConfig = new Dictionary + { + [networkId] = endpointSettings + }; + createOptions.NetworkingConfig = new NetworkingConfig + { + EndpointsConfig = endpointsConfig + }; + } + } + } + + void MountSockets(IModule module, CreateContainerParameters createOptions) + { + var workloadUri = new Uri(this.configSource.Configuration.GetValue(Core.Constants.EdgeletWorkloadUriVariableName)); + if (string.Equals(workloadUri.Scheme, "unix", StringComparison.OrdinalIgnoreCase)) + { + SetMountOptions(createOptions, workloadUri); + } + + // If Management URI is Unix domain socket, and the module is the EdgeAgent, then mount it ino the container. + var managementUri = new Uri(this.configSource.Configuration.GetValue(Core.Constants.EdgeletManagementUriVariableName)); + if (string.Equals(managementUri.Scheme, "unix", StringComparison.OrdinalIgnoreCase) + && module.Name.Equals(Core.Constants.EdgeAgentModuleName, StringComparison.OrdinalIgnoreCase)) + { + SetMountOptions(createOptions, managementUri); + } + } + + static void SetMountOptions(CreateContainerParameters createOptions, Uri uri) + { + HostConfig hostConfig = createOptions.HostConfig ?? new HostConfig(); + IList binds = hostConfig.Binds ?? new List(); + string path = BindPath(uri); + binds.Add($"{path}:{path}"); + + hostConfig.Binds = binds; + createOptions.HostConfig = hostConfig; + } + + static string BindPath(Uri uri) + { + // On Windows we need to bind to the parent folder. We can't bind + // directly to the socket file. + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Path.GetDirectoryName(uri.LocalPath) + : uri.AbsolutePath; + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/EdgeDeploymentDefinition.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/EdgeDeploymentDefinition.cs index cabf5458522..6e07ac0f5e0 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/EdgeDeploymentDefinition.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/EdgeDeploymentDefinition.cs @@ -8,7 +8,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes using Microsoft.Azure.Devices.Edge.Util; using Newtonsoft.Json; - class EdgeDeploymentDefinition : IEdgeDeploymentDefinition + public class EdgeDeploymentDefinition : IEdgeDeploymentDefinition { [JsonProperty(PropertyName = "apiVersion")] public string ApiVersion { get; set; } diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/ImagePullSecret.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/ImagePullSecret.cs new file mode 100644 index 00000000000..d728cfe54d2 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/ImagePullSecret.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes +{ + using System; + using System.Collections.Generic; + using System.Text; + using global::Docker.DotNet.Models; + using Newtonsoft.Json; + + public class ImagePullSecret + { + class AuthEntry + { + [JsonProperty(Required = Required.Always, PropertyName = "username")] + public readonly string Username; + [JsonProperty(Required = Required.Always, PropertyName = "password")] + public readonly string Password; + [JsonProperty(Required = Required.Always, PropertyName = "auth")] + public readonly string Auth; + + public AuthEntry(string username, string password) + { + this.Username = username; + this.Password = password; + byte[] auth = Encoding.UTF8.GetBytes($"{username}:{password}"); + this.Auth = Convert.ToBase64String(auth); + } + } + + class Auth + { + [JsonProperty(Required = Required.Always, PropertyName = "auths")] + public Dictionary Auths; + + public Auth() + { + this.Auths = new Dictionary(); + } + + public Auth(string registry, AuthEntry entry) + : this() + { + this.Auths.Add(registry, entry); + } + } + + public string Name { get; } + + readonly AuthConfig dockerAuth; + + public string GenerateSecret() + { + // JSON struct is + // { "auths": + // { "" : + // { "username":"", + // "password":"", + // "email":"" (not needed) + // "auth":":'>" + // } + // } + // } + var auths = new Auth( + this.dockerAuth.ServerAddress, + new AuthEntry(this.dockerAuth.Username, this.dockerAuth.Password)); + string authString = JsonConvert.SerializeObject(auths); + return authString; + } + + public ImagePullSecret(AuthConfig dockerAuth) + { + this.dockerAuth = dockerAuth; + this.Name = $"{dockerAuth.Username.ToLower()}-{dockerAuth.ServerAddress.ToLower()}"; + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/InvalidModuleException.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/InvalidModuleException.cs new file mode 100644 index 00000000000..26bc23b2faf --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/InvalidModuleException.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes +{ + using System; + + [Serializable] + public class InvalidModuleException : Exception + { + public InvalidModuleException(string message) + : base(message) + { + } + + public InvalidModuleException(string message, Exception inner) + : base(message, inner) + { + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesCommandFactory.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesCommandFactory.cs new file mode 100644 index 00000000000..7581309737a --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesCommandFactory.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes +{ + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Core.Commands; + + public class KubernetesCommandFactory : ICommandFactory + { + public KubernetesCommandFactory() + { + } + + public Task UpdateEdgeAgentAsync(IModuleWithIdentity module, IRuntimeInfo runtimeInfo) => Task.FromResult(NullCommand.Instance as ICommand); + + public Task CreateAsync(IModuleWithIdentity module, IRuntimeInfo runtimeInfo) => + Task.FromResult((ICommand)NullCommand.Instance); + + public Task UpdateAsync(IModule current, IModuleWithIdentity next, IRuntimeInfo runtimeInfo) => + Task.FromResult((ICommand)NullCommand.Instance); + + public Task RemoveAsync(IModule module) => + Task.FromResult((ICommand)NullCommand.Instance); + + public Task StartAsync(IModule module) => + Task.FromResult((ICommand)NullCommand.Instance); + + public Task StopAsync(IModule module) => + Task.FromResult((ICommand)NullCommand.Instance); + + public Task RestartAsync(IModule module) => + Task.FromResult((ICommand)NullCommand.Instance); + + public Task WrapAsync(ICommand command) => Task.FromResult(command); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModule.cs index 49aacfcf48b..0fa3d03641c 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModule.cs @@ -1,9 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.Azure.Devices.Edge.Agent.Core +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes { using System.Collections.Generic; using System.Collections.Immutable; + using Microsoft.Azure.Devices.Edge.Agent.Core; using Microsoft.Azure.Devices.Edge.Agent.Docker; using Microsoft.Azure.Devices.Edge.Util; @@ -39,7 +40,7 @@ public KubernetesModule(IModule module) public ImagePullPolicy ImagePullPolicy { get; set; } - public TConfig Config { get; } + public TConfig Config { get; set; } public virtual bool Equals(IModule other) => this.Equals(other as KubernetesModule); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModuleIdentity.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModuleIdentity.cs index 837f3c940d0..04104b0b013 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModuleIdentity.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesModuleIdentity.cs @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.Azure.Devices.Edge.Agent.Core +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes { + using Microsoft.Azure.Devices.Edge.Agent.Core; using Microsoft.Azure.Devices.Edge.Util; public class KubernetesModuleIdentity diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/commands/KubernetesCrdCommand.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/commands/KubernetesCrdCommand.cs new file mode 100644 index 00000000000..d234e8350a9 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/commands/KubernetesCrdCommand.cs @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Commands +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using k8s; + using k8s.Models; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Core.Serde; + using Microsoft.Azure.Devices.Edge.Agent.Docker; + using Microsoft.Azure.Devices.Edge.Agent.Kubernetes; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Extensions.Logging; + using Microsoft.Rest; + using Newtonsoft.Json; + + using Constants = Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Constants; + + public class KubernetesCrdCommand : ICommand + { + readonly IKubernetes client; + readonly KubernetesModule[] modules; + readonly Option runtimeInfo; + readonly Lazy id; + readonly ICombinedConfigProvider combinedConfigProvider; + readonly string deviceNamespace; + readonly string iotHubHostname; + readonly string deviceId; + readonly TypeSpecificSerDe> deploymentSerde; + // We use the sum of the IDs of the underlying commands as the id for this group + // command. + public string Id => this.id.Value; + + public KubernetesCrdCommand(string deviceNamespace, string iotHubHostname, string deviceId, IKubernetes client, KubernetesModule[] modules, Option runtimeInfo, ICombinedConfigProvider combinedConfigProvider) + { + this.deviceNamespace = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace))); + this.iotHubHostname = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname))); + this.deviceId = KubeUtils.SanitizeK8sValue(Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId))); + this.client = Preconditions.CheckNotNull(client, nameof(client)); + this.modules = Preconditions.CheckNotNull(modules, nameof(modules)); + this.runtimeInfo = Preconditions.CheckNotNull(runtimeInfo, nameof(runtimeInfo)); + this.combinedConfigProvider = Preconditions.CheckNotNull(combinedConfigProvider, nameof(combinedConfigProvider)); + this.id = new Lazy(() => this.modules.Aggregate(string.Empty, (prev, module) => module.Name + prev)); + var deserializerTypesMap = new Dictionary> + { + [typeof(IModule)] = new Dictionary + { + ["docker"] = typeof(CombinedDockerConfig) + } + }; + + this.deploymentSerde = new TypeSpecificSerDe>(deserializerTypesMap); + } + + async Task UpdateImagePullSecrets(Dictionary imagePullSecrets, CancellationToken token) + { + foreach (KeyValuePair imagePullSecret in imagePullSecrets) + { + var secretData = new Dictionary { [Constants.K8sPullSecretData] = Encoding.UTF8.GetBytes(imagePullSecret.Value.GenerateSecret()) }; + var secretMeta = new V1ObjectMeta(name: imagePullSecret.Key, namespaceProperty: this.deviceNamespace); + var newSecret = new V1Secret("v1", secretData, type: Constants.K8sPullSecretType, kind: "Secret", metadata: secretMeta); + Option currentSecret; + try + { + currentSecret = Option.Maybe(await this.client.ReadNamespacedSecretAsync(imagePullSecret.Key, this.deviceNamespace, cancellationToken: token)); + } + catch (Exception ex) when (!ex.IsFatal()) + { + Events.FailedToFindSecret(imagePullSecret.Key, ex); + currentSecret = Option.None(); + } + + try + { + var v1Secret = await currentSecret.Match( + async s => + { + if ((s.Data != null) && s.Data.TryGetValue(Constants.K8sPullSecretData, out byte[] pullSecretData) && + pullSecretData.SequenceEqual(secretData[Constants.K8sPullSecretData])) + { + return s; + } + + return await this.client.ReplaceNamespacedSecretAsync( + newSecret, + imagePullSecret.Key, + this.deviceNamespace, + cancellationToken: token); + }, + async () => await this.client.CreateNamespacedSecretAsync(newSecret, this.deviceNamespace, cancellationToken: token)); + if (v1Secret == null) + { + throw new InvalidIdentityException("Image pull secret was not properly created"); + } + } + catch (Exception ex) when (!ex.IsFatal()) + { + Events.SecretCreateUpdateFailed(imagePullSecret.Key, ex); + } + } + } + + public async Task ExecuteAsync(CancellationToken token) + { + string resourceName = this.iotHubHostname + Constants.K8sNameDivider + this.deviceId; + string metaApiVersion = Constants.K8sApi + "/" + Constants.K8sApiVersion; + + var modulesList = new List>(); + var secrets = new Dictionary(); + foreach (var runtime in this.runtimeInfo) + { + foreach (var m in this.modules) + { + var combinedConfig = this.combinedConfigProvider.GetCombinedConfig(m, runtime); + CombinedDockerConfig dockerConfig = combinedConfig as CombinedDockerConfig; + if (dockerConfig != null) + { + var combinedModule = new KubernetesModule(m) + { + Config = new DockerConfig(dockerConfig.Image, dockerConfig.CreateOptions) + }; + modulesList.Add(combinedModule); + dockerConfig.AuthConfig.ForEach( + auth => + { + var kubernetesAuth = new ImagePullSecret(auth); + secrets[kubernetesAuth.Name] = kubernetesAuth; + }); + } + else + { + throw new InvalidModuleException("Cannot convert combined config into KubernetesModule."); + } + } + } + + Option> activeDeployment; + try + { + HttpOperationResponse currentDeployment = await this.client.GetNamespacedCustomObjectWithHttpMessagesAsync( + Constants.K8sCrdGroup, + Constants.K8sApiVersion, + this.deviceNamespace, + Constants.K8sCrdPlural, + resourceName, + cancellationToken: token); + string body = JsonConvert.SerializeObject(currentDeployment.Body); + + activeDeployment = currentDeployment.Response.IsSuccessStatusCode ? + Option.Some(this.deploymentSerde.Deserialize(body)) : + Option.None>(); + } + catch (Exception parseException) + { + Events.FindActiveDeploymentFailed(resourceName, parseException); + activeDeployment = Option.None>(); + } + + await this.UpdateImagePullSecrets(secrets, token); + + var metadata = new V1ObjectMeta(name: resourceName, namespaceProperty: this.deviceNamespace); + // need resourceVersion for Replace. + activeDeployment.ForEach(deployment => metadata.ResourceVersion = deployment.Metadata.ResourceVersion); + var customObjectDefinition = new EdgeDeploymentDefinition(metaApiVersion, Constants.K8sCrdKind, metadata, modulesList); + string customObjectString = this.deploymentSerde.Serialize(customObjectDefinition); + + // the dotnet client is apparently really picky about all names being camelCase, + object crdObject = JsonConvert.DeserializeObject(customObjectString); + + await activeDeployment.Match( + async a => + { + Events.ReplaceDeployment(customObjectString); + await this.client.ReplaceNamespacedCustomObjectWithHttpMessagesAsync( + crdObject, + Constants.K8sCrdGroup, + Constants.K8sApiVersion, + this.deviceNamespace, + Constants.K8sCrdPlural, + resourceName, + cancellationToken: token); + }, + async () => + { + Events.CreateDeployment(customObjectString); + await this.client.CreateNamespacedCustomObjectWithHttpMessagesAsync( + crdObject, + Constants.K8sCrdGroup, + Constants.K8sApiVersion, + this.deviceNamespace, + Constants.K8sCrdPlural, + cancellationToken: token); + }); + } + + public Task UndoAsync(CancellationToken token) + { + return Task.CompletedTask; + } + + public string Show() + { + IEnumerable commandDescriptions = this.modules.Select(m => $"[{m.Name}]"); + return $"Create a CRD with modules: (\n {string.Join("\n ", commandDescriptions)}\n)"; + } + + public override string ToString() => this.Show(); + + static class Events + { + const int IdStart = KubernetesEventIds.KubernetesCommand; + static readonly ILogger Log = Logger.Factory.CreateLogger>(); + + enum EventIds + { + CreateDeployment = IdStart, + FailedToFindSecret, + SecretCreateUpdateFailed, + FindActiveDeploymentFailed, + ReplaceDeployment + } + + public static void CreateDeployment(string customObjectString) + { + Log.LogDebug( + (int)EventIds.CreateDeployment, + "===================CREATE========================\n" + + customObjectString + + "\n================================================="); + } + + public static void FailedToFindSecret(string key, Exception exception) + { + Log.LogDebug((int)EventIds.FailedToFindSecret, exception, $"Failed to find image pull secret ${key}"); + } + + public static void SecretCreateUpdateFailed(string key, Exception exception) + { + Log.LogError((int)EventIds.SecretCreateUpdateFailed, exception, $"Failed to create or update image pull secret ${key}"); + } + + public static void FindActiveDeploymentFailed(string resourceName, Exception parseException) + { + Log.LogDebug((int)EventIds.FindActiveDeploymentFailed, parseException, $"Failed to find active edge deployment ${resourceName}"); + } + + public static void ReplaceDeployment(string customObjectString) + { + Log.LogDebug( + (int)EventIds.ReplaceDeployment, + "====================REPLACE======================\n" + + customObjectString + + "\n================================================="); + } + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/planners/KubernetesPlanner.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/planners/KubernetesPlanner.cs new file mode 100644 index 00000000000..5533a9213ea --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/planners/KubernetesPlanner.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Planners +{ + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Linq; + using System.Threading.Tasks; + using k8s; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Docker; + using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Commands; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Extensions.Logging; + + public class KubernetesPlanner : IPlanner + { + readonly IKubernetes client; + readonly ICommandFactory commandFactory; + readonly ICombinedConfigProvider combinedConfigProvider; + readonly string deviceNamespace; + readonly string iotHubHostname; + readonly string deviceId; + + public KubernetesPlanner( + string deviceNamespace, + string iotHubHostname, + string deviceId, + IKubernetes client, + ICommandFactory commandFactory, + ICombinedConfigProvider combinedConfigProvider) + { + this.deviceNamespace = Preconditions.CheckNonWhiteSpace(deviceNamespace, nameof(deviceNamespace)); + this.iotHubHostname = Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname)); + this.deviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId)); + this.client = Preconditions.CheckNotNull(client, nameof(client)); + this.commandFactory = Preconditions.CheckNotNull(commandFactory, nameof(commandFactory)); + this.combinedConfigProvider = Preconditions.CheckNotNull(combinedConfigProvider, nameof(combinedConfigProvider)); + } + + public async Task PlanAsync( + ModuleSet desired, + ModuleSet current, + IRuntimeInfo runtimeInfo, + IImmutableDictionary moduleIdentities) + { + Events.LogDesired(desired); + Events.LogCurrent(current); + Events.LogIdentities(moduleIdentities); + + // Check that module names sanitize and remain unique. + var groupedModules = desired.Modules.GroupBy(pair => KubeUtils.SanitizeK8sValue(pair.Key)).ToArray(); + if (groupedModules.Any(c => c.Count() > 1)) + { + string nameList = groupedModules.Where(c => c.Count() > 1).SelectMany(g => g, (pairs, pair) => pair.Key).Join(","); + throw new InvalidIdentityException($"Deployment will cause a name collision in Kubernetes namespace, modules: [{nameList}]"); + } + + // TODO: improve this so it is generic for all potential module types. + if (!desired.Modules.Values.All(p => p is IModule)) + { + throw new InvalidModuleException($"Kubernetes deployment currently only handles type={typeof(T).FullName}"); + } + + Diff moduleDifference = desired.Diff(current); + + Plan plan; + if (!moduleDifference.IsEmpty) + { + // The "Plan" here is very simple - if we have any change, publish all desired modules to a CRD. + // The CRD allows us to give the customer a Kubernetes-centric way to see the deployment + // and the status of that deployment through the "edgedeployments" API. + var k8sModules = desired.Modules.Select(m => new KubernetesModule(m.Value as IModule)); + + var crdCommand = new KubernetesCrdCommand(this.deviceNamespace, this.iotHubHostname, this.deviceId, this.client, k8sModules.ToArray(), Option.Some(runtimeInfo), this.combinedConfigProvider as ICombinedConfigProvider); + var planCommand = await this.commandFactory.WrapAsync(crdCommand); + var planList = new List + { + planCommand + }; + Events.PlanCreated(planList); + plan = new Plan(planList); + } + else + { + plan = Plan.Empty; + } + + return plan; + } + + public Task CreateShutdownPlanAsync(ModuleSet current) => Task.FromResult(Plan.Empty); + + static class Events + { + const int IdStart = KubernetesEventIds.KubernetesPlanner; + static readonly ILogger Log = Logger.Factory.CreateLogger>(); + + enum EventIds + { + PlanCreated = IdStart, + DesiredModules, + CurrentModules, + Identities, + } + + internal static void PlanCreated(IList commands) + { + Log.LogDebug((int)EventIds.PlanCreated, $"KubernetesPlanner created Plan, with {commands.Count} command(s)."); + } + + internal static void LogDesired(ModuleSet desired) + { + Log.LogDebug((int)EventIds.DesiredModules, $"List of desired modules is - {string.Join(", ", desired.Modules.Keys)}"); + } + + internal static void LogCurrent(ModuleSet current) + { + Log.LogDebug((int)EventIds.CurrentModules, $"List of current modules is - {string.Join(", ", current.Modules.Keys)}"); + } + + internal static void LogIdentities(IImmutableDictionary moduleIdentities) + { + Log.LogDebug((int)EventIds.Identities, $"List of module identities is - {string.Join(", ", moduleIdentities.Keys)}"); + } + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/CombinedKubernetesConfigProviderTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/CombinedKubernetesConfigProviderTest.cs new file mode 100644 index 00000000000..f56ff9e38bd --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/CombinedKubernetesConfigProviderTest.cs @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test +{ + using System; + using System.Collections.Generic; + using System.Runtime.InteropServices; + using global::Docker.DotNet.Models; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Docker; + using Microsoft.Extensions.Configuration; + using Moq; + using Xunit; + using CoreConstants = Microsoft.Azure.Devices.Edge.Agent.Core.Constants; + + public class CombinedKubernetesConfigProviderTest + { + [Fact] + public void TestCreateValidation() + { + Assert.Throws(() => new CombinedKubernetesConfigProvider(new[] { new AuthConfig(), }, null)); + Assert.Throws(() => new CombinedKubernetesConfigProvider(null, Mock.Of())); + } + + [Fact] + public void TestVolMount() + { + // Arrange + var runtimeInfo = new Mock>(); + runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty)); + + var module = new Mock>(); + module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest")); + module.SetupGet(m => m.Name).Returns(CoreConstants.EdgeAgentModuleName); + + var unixUris = new Dictionary + { + { CoreConstants.EdgeletWorkloadUriVariableName, "unix:///path/to/workload.sock" }, + { CoreConstants.EdgeletManagementUriVariableName, "unix:///path/to/mgmt.sock" } + }; + + var windowsUris = new Dictionary + { + { CoreConstants.EdgeletWorkloadUriVariableName, "unix:///C:/path/to/workload/sock" }, + { CoreConstants.EdgeletManagementUriVariableName, "unix:///C:/path/to/mgmt/sock" } + }; + + IConfigurationRoot configRoot = new ConfigurationBuilder().AddInMemoryCollection( + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? windowsUris : unixUris).Build(); + var configSource = Mock.Of(s => s.Configuration == configRoot); + ICombinedConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, configSource); + + // Act + CombinedDockerConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object); + + // Assert + Assert.NotNull(config.CreateOptions); + Assert.NotNull(config.CreateOptions.HostConfig); + Assert.NotNull(config.CreateOptions.HostConfig.Binds); + Assert.Equal(2, config.CreateOptions.HostConfig.Binds.Count); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Assert.Equal("C:\\path\\to\\workload:C:\\path\\to\\workload", config.CreateOptions.HostConfig.Binds[0]); + Assert.Equal("C:\\path\\to\\mgmt:C:\\path\\to\\mgmt", config.CreateOptions.HostConfig.Binds[1]); + } + else + { + Assert.Equal("/path/to/workload.sock:/path/to/workload.sock", config.CreateOptions.HostConfig.Binds[0]); + Assert.Equal("/path/to/mgmt.sock:/path/to/mgmt.sock", config.CreateOptions.HostConfig.Binds[1]); + } + } + + [Fact] + public void TestNoVolMountForNonUds() + { + // Arrange + var runtimeInfo = new Mock>(); + runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty)); + + var module = new Mock>(); + module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest")); + module.SetupGet(m => m.Name).Returns(CoreConstants.EdgeAgentModuleName); + + IConfigurationRoot configRoot = new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + { CoreConstants.EdgeletWorkloadUriVariableName, "http://localhost:2375/" }, + { CoreConstants.EdgeletManagementUriVariableName, "http://localhost:2376/" } + }).Build(); + var configSource = Mock.Of(s => s.Configuration == configRoot); + ICombinedConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, configSource); + + // Act + CombinedDockerConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object); + + // Assert + Assert.NotNull(config.CreateOptions); + Assert.Null(config.CreateOptions.HostConfig); + } + + [Fact] + public void InjectNetworkAliasTest() + { + // Arrange + var runtimeInfo = new Mock>(); + runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty)); + + var module = new Mock>(); + module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest")); + module.SetupGet(m => m.Name).Returns("mod1"); + + IConfigurationRoot configRoot = new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + { CoreConstants.EdgeletWorkloadUriVariableName, "unix:///var/run/iotedgedworkload.sock" }, + { CoreConstants.EdgeletManagementUriVariableName, "unix:///var/run/iotedgedmgmt.sock" }, + { CoreConstants.NetworkIdKey, "testnetwork1" }, + { CoreConstants.EdgeDeviceHostNameKey, "edhk1" } + }).Build(); + var configSource = Mock.Of(s => s.Configuration == configRoot); + + ICombinedConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, configSource); + + // Act + CombinedDockerConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object); + + // Assert + Assert.NotNull(config.CreateOptions); + Assert.NotNull(config.CreateOptions.NetworkingConfig); + Assert.NotNull(config.CreateOptions.NetworkingConfig.EndpointsConfig); + Assert.NotNull(config.CreateOptions.NetworkingConfig.EndpointsConfig["testnetwork1"]); + Assert.Null(config.CreateOptions.NetworkingConfig.EndpointsConfig["testnetwork1"].Aliases); + } + + [Fact] + public void InjectNetworkAliasEdgeHubTest() + { + // Arrange + var runtimeInfo = new Mock>(); + runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty)); + + var module = new Mock>(); + module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest")); + module.SetupGet(m => m.Name).Returns(CoreConstants.EdgeHubModuleName); + + IConfigurationRoot configRoot = new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + { CoreConstants.EdgeletWorkloadUriVariableName, "unix:///var/run/iotedgedworkload.sock" }, + { CoreConstants.EdgeletManagementUriVariableName, "unix:///var/run/iotedgedmgmt.sock" }, + { CoreConstants.NetworkIdKey, "testnetwork1" }, + { CoreConstants.EdgeDeviceHostNameKey, "edhk1" } + }).Build(); + var configSource = Mock.Of(s => s.Configuration == configRoot); + + ICombinedConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, configSource); + + // Act + CombinedDockerConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object); + + // Assert + Assert.NotNull(config.CreateOptions); + Assert.NotNull(config.CreateOptions.NetworkingConfig); + Assert.NotNull(config.CreateOptions.NetworkingConfig.EndpointsConfig); + Assert.NotNull(config.CreateOptions.NetworkingConfig.EndpointsConfig["testnetwork1"]); + Assert.Equal("edhk1", config.CreateOptions.NetworkingConfig.EndpointsConfig["testnetwork1"].Aliases[0]); + } + + [Fact] + public void InjectNetworkAliasHostNetworkTest() + { + // Arrange + var runtimeInfo = new Mock>(); + runtimeInfo.SetupGet(ri => ri.Config).Returns(new DockerRuntimeConfig("1.24", string.Empty)); + + string hostNetworkCreateOptions = "{\"NetworkingConfig\":{\"EndpointsConfig\":{\"host\":{}}},\"HostConfig\":{\"NetworkMode\":\"host\"}}"; + var module = new Mock>(); + module.SetupGet(m => m.Config).Returns(new DockerConfig("nginx:latest", hostNetworkCreateOptions)); + module.SetupGet(m => m.Name).Returns("mod1"); + + IConfigurationRoot configRoot = new ConfigurationBuilder().AddInMemoryCollection( + new Dictionary + { + { CoreConstants.EdgeletWorkloadUriVariableName, "unix:///var/run/iotedgedworkload.sock" }, + { CoreConstants.EdgeletManagementUriVariableName, "unix:///var/run/iotedgedmgmt.sock" }, + { CoreConstants.NetworkIdKey, "testnetwork1" }, + { CoreConstants.EdgeDeviceHostNameKey, "edhk1" } + }).Build(); + var configSource = Mock.Of(s => s.Configuration == configRoot); + + ICombinedConfigProvider provider = new CombinedKubernetesConfigProvider(new[] { new AuthConfig() }, configSource); + + // Act + CombinedDockerConfig config = provider.GetCombinedConfig(module.Object, runtimeInfo.Object); + + // Assert + Assert.NotNull(config.CreateOptions); + Assert.NotNull(config.CreateOptions.NetworkingConfig); + Assert.NotNull(config.CreateOptions.NetworkingConfig.EndpointsConfig); + Assert.False(config.CreateOptions.NetworkingConfig.EndpointsConfig.ContainsKey("testnetwork1")); + Assert.NotNull(config.CreateOptions.NetworkingConfig.EndpointsConfig["host"]); + Assert.Null(config.CreateOptions.NetworkingConfig.EndpointsConfig["host"].Aliases); + Assert.Equal("host", config.CreateOptions.HostConfig.NetworkMode); + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/ImagePullSecretTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/ImagePullSecretTest.cs new file mode 100644 index 00000000000..d6fce4f8875 --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/ImagePullSecretTest.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test +{ + using System; + using System.Collections.Generic; + using System.Text; + using global::Docker.DotNet.Models; + using Microsoft.Azure.Devices.Edge.Agent.Kubernetes; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Newtonsoft.Json.Linq; + using Xunit; + + public class ImagePullSecretTest + { + public static IEnumerable GenerateAuthConfig() + { + yield return new object[] { new AuthConfig() { Username = "one", Password = "two", ServerAddress = "three" } }; + } + + [Theory] + [Unit] + [MemberData(nameof(GenerateAuthConfig))] + public void ImagePullSecretTestGeneration(AuthConfig auth) + { + var ips = new ImagePullSecret(auth); + Assert.Equal($"{auth.Username.ToLower()}-{auth.ServerAddress.ToLower()}", ips.Name); + var generated = JObject.Parse(ips.GenerateSecret()); + + // Validate Json structure + Assert.NotNull(generated["auths"][auth.ServerAddress]); + Assert.Equal(generated["auths"][auth.ServerAddress]["username"], auth.Username); + Assert.Equal(generated["auths"][auth.ServerAddress]["password"], auth.Password); + Assert.Equal(generated["auths"][auth.ServerAddress]["auth"], Convert.ToBase64String(Encoding.UTF8.GetBytes($"{auth.Username}:{auth.Password}"))); + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesCommandFactoryTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesCommandFactoryTest.cs new file mode 100644 index 00000000000..2c5039545fd --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesCommandFactoryTest.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Core.Commands; + using Moq; + using Xunit; + + public class KubernetesCommandFactoryTest + { + [Fact] + [Unit] + public async void KubernetesCommandFactoryIsReallyBasic() + { + var mockModule = Mock.Of(); + var mockModuleIdentity = Mock.Of(); + var mockRuntime = Mock.Of(); + var mockCommand = Mock.Of(c => c.ExecuteAsync(It.IsAny()) == Task.CompletedTask); + var kcf = new KubernetesCommandFactory(); + CancellationToken ct = CancellationToken.None; + + Assert.Equal(NullCommand.Instance, await kcf.UpdateEdgeAgentAsync(mockModuleIdentity, mockRuntime)); + Assert.Equal(NullCommand.Instance, await kcf.CreateAsync(mockModuleIdentity, mockRuntime)); + Assert.Equal(NullCommand.Instance, await kcf.UpdateAsync(mockModule, mockModuleIdentity, mockRuntime)); + Assert.Equal(NullCommand.Instance, await kcf.RemoveAsync(mockModule)); + Assert.Equal(NullCommand.Instance, await kcf.StartAsync(mockModule)); + Assert.Equal(NullCommand.Instance, await kcf.StopAsync(mockModule)); + Assert.Equal(NullCommand.Instance, await kcf.RestartAsync(mockModule)); + var newCommand = await kcf.WrapAsync(mockCommand); + await newCommand.ExecuteAsync(ct); + Mock.Get(mockCommand).Verify( c => c.ExecuteAsync(ct), Times.Once); + } +} +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/commands/KubernetesCrdCommandTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/commands/KubernetesCrdCommandTest.cs new file mode 100644 index 00000000000..c70e8e28174 --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/commands/KubernetesCrdCommandTest.cs @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test.Commands +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using global::Docker.DotNet.Models; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Docker; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Moq; + using k8s; + using k8s.Models; + using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Commands; + using Xunit; + using Microsoft.Azure.Devices.Edge.Util; + using Newtonsoft.Json; + using Constants = Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Constants; + + public class KubernetesCrdCommandTest + { + const string Ns = "namespace"; + const string Hostname = "hostname"; + const string GwHostname = "gwHostname"; + const string DeviceId = "deviceId"; + static readonly IDictionary EnvVars = new Dictionary(); + static readonly DockerConfig Config1 = new DockerConfig("test-image:1"); + static readonly DockerConfig Config2 = new DockerConfig("test-image:2"); + static readonly ConfigurationInfo DefaultConfigurationInfo = new ConfigurationInfo("1"); + static readonly IRuntimeInfo RuntimeInfo = Mock.Of(); + static readonly IKubernetes DefaultClient = Mock.Of(); + static readonly ICommandFactory DefaultCommandFactory = new KubernetesCommandFactory(); + static readonly ICombinedConfigProvider DefaultConfigProvider = Mock.Of>(); + static readonly IRuntimeInfo Runtime = Mock.Of(); + + [Fact] + [Unit] + public void CrdCommandCreateValidation() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + KubernetesModule km1 = new KubernetesModule(m1 as IModule); + KubernetesModule[] modules = { km1 }; + Assert.Throws(() => new KubernetesCrdCommand(null, Hostname, DeviceId, DefaultClient, modules, Option.None(), DefaultConfigProvider)); + Assert.Throws(() => new KubernetesCrdCommand(Ns, null, DeviceId, DefaultClient, modules, Option.None(), DefaultConfigProvider)); + Assert.Throws(() => new KubernetesCrdCommand(Ns, Hostname, null, DefaultClient, modules, Option.None(), DefaultConfigProvider)); + Assert.Throws(() => new KubernetesCrdCommand(Ns, Hostname, DeviceId, null, modules, Option.None(), DefaultConfigProvider)); + Assert.Throws(() => new KubernetesCrdCommand(Ns, Hostname, DeviceId, DefaultClient, null, Option.None(), DefaultConfigProvider)); + Assert.Throws(() => new KubernetesCrdCommand(Ns, Hostname, DeviceId, DefaultClient, modules, Option.None(), null)); + } + + [Fact] + [Unit] + public async void CrdCommandExecuteAsyncInvalidModule() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + var km1 = new KubernetesModule(m1 as IModule); + KubernetesModule[] modules = { km1 }; + Option runtimeOption = Option.Maybe(Runtime); + var configProvider = new Mock>(); + configProvider.Setup(cp => cp.GetCombinedConfig(km1, Runtime)).Returns(() => null); + + var token = new CancellationToken(); + var cmd = new KubernetesCrdCommand(Ns, Hostname, DeviceId, DefaultClient, modules, runtimeOption, DefaultConfigProvider); + await Assert.ThrowsAsync(() => cmd.ExecuteAsync(token)); + } + + [Fact] + [Unit] + public async void CrdCommandExecuteWithAuthCreateNewObjects() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + var km1 = new KubernetesModule((IModule)m1); + KubernetesModule[] modules = { km1 }; + var token = new CancellationToken(); + Option runtimeOption = Option.Maybe(Runtime); + var auth = new AuthConfig() { Username = "username", Password = "password", ServerAddress = "docker.io" }; + var configProvider = new Mock>(); + configProvider.Setup(cp => cp.GetCombinedConfig(km1, Runtime)).Returns(() => new CombinedDockerConfig("test-image:1", Config1.CreateOptions, Option.Maybe(auth))); + bool getSecretCalled = false; + bool postSecretCalled = false; + bool getCrdCalled = false; + bool postCrdCalled = false; + + using (var server = new MockKubeApiServer( + resp: string.Empty, + shouldNext: httpContext => + { + string pathStr = httpContext.Request.Path.Value; + string method = httpContext.Request.Method; + if (string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase)) + { + httpContext.Response.StatusCode = 404; + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets")) + { + getSecretCalled = true; + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}")) + { + getCrdCalled = true; + } + } + else if (string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)) + { + httpContext.Response.StatusCode = 201; + httpContext.Response.Body = httpContext.Request.Body; + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets")) + { + postSecretCalled = true; + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}")) + { + postCrdCalled = true; + } + } + + return Task.FromResult(false); + })) + { + var client = new Kubernetes( + new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + var cmd = new KubernetesCrdCommand(Ns, Hostname, DeviceId, client, modules, runtimeOption, configProvider.Object); + await cmd.ExecuteAsync(token); + Assert.True(getSecretCalled, nameof(getSecretCalled)); + Assert.True(postSecretCalled, nameof(postSecretCalled)); + Assert.True(getCrdCalled, nameof(getCrdCalled)); + Assert.True(postCrdCalled, nameof(postCrdCalled)); + } + } + + [Fact] + [Unit] + public async void CrdCommandExecuteWithAuthReplaceObjects() + { + string resourceName = Hostname + Constants.K8sNameDivider + DeviceId.ToLower(); + string metaApiVersion = Constants.K8sApi + "/" + Constants.K8sApiVersion; + string secretName = "username-docker.io"; + var secretData = new Dictionary { [Constants.K8sPullSecretData] = Encoding.UTF8.GetBytes("Invalid Secret Data") }; + var secretMeta = new V1ObjectMeta(name: secretName, namespaceProperty: Ns); + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + var km1 = new KubernetesModule((IModule)m1); + KubernetesModule[] modules = { km1 }; + var token = new CancellationToken(); + Option runtimeOption = Option.Maybe(Runtime); + var auth = new AuthConfig() { Username = "username", Password = "password", ServerAddress = "docker.io" }; + var configProvider = new Mock>(); + configProvider.Setup(cp => cp.GetCombinedConfig(km1, Runtime)).Returns(() => new CombinedDockerConfig("test-image:1", Config1.CreateOptions, Option.Maybe(auth))); + var existingSecret = new V1Secret("v1", secretData, type: Constants.K8sPullSecretType, kind: "Secret", metadata: secretMeta); + var existingDeployment = new EdgeDeploymentDefinition(metaApiVersion, Constants.K8sCrdKind, new V1ObjectMeta(name: resourceName), new List>()); + bool getSecretCalled = false; + bool putSecretCalled = false; + bool getCrdCalled = false; + bool putCrdCalled = false; + + using (var server = new MockKubeApiServer( + resp: string.Empty, + shouldNext: async httpContext => + { + string pathStr = httpContext.Request.Path.Value; + string method = httpContext.Request.Method; + if (string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase)) + { + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets/{secretName}")) + { + getSecretCalled = true; + await httpContext.Response.Body.WriteAsync(JsonConvert.SerializeObject(existingSecret).ToBody(), token); + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}/{resourceName}")) + { + getCrdCalled = true; + await httpContext.Response.Body.WriteAsync(JsonConvert.SerializeObject(existingDeployment).ToBody(), token); + } + } + else if (string.Equals(method, "PUT", StringComparison.OrdinalIgnoreCase)) + { + httpContext.Response.Body = httpContext.Request.Body; + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets/{secretName}")) + { + putSecretCalled = true; + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}/{resourceName}")) + { + putCrdCalled = true; + } + } + + return false; + })) + { + var client = new Kubernetes( + new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + var cmd = new KubernetesCrdCommand(Ns, Hostname, DeviceId, client, modules, runtimeOption, configProvider.Object); + await cmd.ExecuteAsync(token); + Assert.True(getSecretCalled, nameof(getSecretCalled)); + Assert.True(putSecretCalled, nameof(putSecretCalled)); + Assert.True(getCrdCalled, nameof(getCrdCalled)); + Assert.True(putCrdCalled, nameof(putCrdCalled)); + } + } + + [Fact] + [Unit] + public async void CrdCommandExecuteTwoModulesWithSamePullSecret() + { + string resourceName = Hostname + Constants.K8sNameDivider + DeviceId.ToLower(); + string secretName = "username-docker.io"; + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + var km1 = new KubernetesModule((IModule)m1); + IModule m2 = new DockerModule("module2", "v1", ModuleStatus.Running, Core.RestartPolicy.Always, Config2, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + var km2 = new KubernetesModule((IModule)m2); + KubernetesModule[] modules = { km1, km2 }; + var token = new CancellationToken(); + Option runtimeOption = Option.Maybe(Runtime); + var auth = new AuthConfig() { Username = "username", Password = "password", ServerAddress = "docker.io" }; + var configProvider = new Mock>(); + configProvider.Setup(cp => cp.GetCombinedConfig(It.IsAny>(), Runtime)).Returns(() => new CombinedDockerConfig("test-image:1", Config1.CreateOptions, Option.Maybe(auth))); + bool getSecretCalled = false; + bool putSecretCalled = false; + int postSecretCalled = 0; + bool getCrdCalled = false; + bool putCrdCalled = false; + int postCrdCalled = 0; + Stream secretBody = Stream.Null; + + using (var server = new MockKubeApiServer( + resp: string.Empty, + shouldNext: httpContext => + { + string pathStr = httpContext.Request.Path.Value; + string method = httpContext.Request.Method; + if (string.Equals(method, "GET", StringComparison.OrdinalIgnoreCase)) + { + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets/{secretName}")) + { + if (secretBody == Stream.Null) + { + // 1st pass, secret should not exist + getSecretCalled = true; + httpContext.Response.StatusCode = 404; + } + else + { + // 2nd pass, use secret from creation. + httpContext.Response.Body = secretBody; + } + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}/{resourceName}")) + { + getCrdCalled = true; + httpContext.Response.StatusCode = 404; + } + } + else if (string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)) + { + httpContext.Response.StatusCode = 201; + httpContext.Response.Body = httpContext.Request.Body; + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets")) + { + postSecretCalled++; + secretBody = httpContext.Request.Body; // save this for next query. + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}")) + { + postCrdCalled++; + } + } + else if (string.Equals(method, "PUT", StringComparison.OrdinalIgnoreCase)) + { + httpContext.Response.Body = httpContext.Request.Body; + if (pathStr.Contains($"api/v1/namespaces/{Ns}/secrets")) + { + putSecretCalled = true; + } + else if (pathStr.Contains($"namespaces/{Ns}/{Constants.K8sCrdPlural}")) + { + putCrdCalled = true; + } + } + + return Task.FromResult(false); + })) + { + var client = new Kubernetes( + new KubernetesClientConfiguration + { + Host = server.Uri.ToString() + }); + var cmd = new KubernetesCrdCommand(Ns, Hostname, DeviceId, client, modules, runtimeOption, configProvider.Object); + await cmd.ExecuteAsync(token); + Assert.True(getSecretCalled, nameof(getSecretCalled)); + Assert.Equal(1, postSecretCalled); + Assert.False(putSecretCalled, nameof(putSecretCalled)); + Assert.True(getCrdCalled, nameof(getCrdCalled)); + Assert.Equal(1, postCrdCalled); + Assert.False(putCrdCalled, nameof(putCrdCalled)); + } + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/planners/KubernetesPlannerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/planners/KubernetesPlannerTest.cs new file mode 100644 index 00000000000..37483c5f7ae --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test/planners/KubernetesPlannerTest.cs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test.Planners +{ + using System; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Linq; + using k8s; + using Microsoft.Azure.Devices.Edge.Agent.Core; + using Microsoft.Azure.Devices.Edge.Agent.Docker; + using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Commands; + using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Planners; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Moq; + using Xunit; + + [Unit] + public class KubernetesPlannerTest + { + const string Ns = "namespace"; + const string Hostname = "hostname"; + const string DeviceId = "deviceId"; + static readonly IDictionary EnvVars = new Dictionary(); + static readonly DockerConfig Config1 = new DockerConfig("image1"); + static readonly DockerConfig Config2 = new DockerConfig("image2"); + static readonly ConfigurationInfo DefaultConfigurationInfo = new ConfigurationInfo("1"); + static readonly IRuntimeInfo RuntimeInfo = Mock.Of(); + static readonly IKubernetes DefaultClient = Mock.Of(); + static readonly ICommandFactory DefaultCommandFactory = new KubernetesCommandFactory(); + static readonly ICombinedConfigProvider DefaultConfigProvider = Mock.Of>(); + + [Fact] + [Unit] + public void ContructorThrowsOnNull() + { + Assert.Throws(() => new KubernetesPlanner(null, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(string.Empty, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, null, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, " ", DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, Hostname, null, DefaultClient, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, Hostname, string.Empty, DefaultClient, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, Hostname, DeviceId, null, DefaultCommandFactory, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, null, DefaultConfigProvider)); + Assert.Throws(() => new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, null)); + } + + [Fact] + [Unit] + public async void KubernetesPlannerNoModulesNoPlan() + { + var planner = new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider); + Plan addPlan = await planner.PlanAsync(ModuleSet.Empty, ModuleSet.Empty, RuntimeInfo, ImmutableDictionary.Empty); + Assert.Equal(Plan.Empty, addPlan); + } + + [Fact] + [Unit] + public async void KubernetesPlannerPlanFailsWithNonDistinctModules() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + IModule m2 = new DockerModule("Module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + ModuleSet addRunning = ModuleSet.Create(m1, m2); + + var planner = new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider); + + await Assert.ThrowsAsync( () => planner.PlanAsync(addRunning, ModuleSet.Empty, RuntimeInfo, ImmutableDictionary.Empty)); + } + + [Fact] + [Unit] + public async void KubernetesPlannerPlanFailsWithNonDockerModules() + { + IModule m1 = new NonDockerModule("module1", "v1", "unknown", ModuleStatus.Running, RestartPolicy.Always, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars, string.Empty); + ModuleSet addRunning = ModuleSet.Create(m1); + + var planner = new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider); + await Assert.ThrowsAsync(() => planner.PlanAsync(addRunning, ModuleSet.Empty, RuntimeInfo, ImmutableDictionary.Empty)); + } + + [Fact] + [Unit] + public async void KubernetesPlannerEmptyPlanWhenNoChanges() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + IModule m2 = new DockerModule("module2", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + ModuleSet desired = ModuleSet.Create(m1, m2); + ModuleSet current = ModuleSet.Create(m1, m2); + + var planner = new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider); + var plan = await planner.PlanAsync(desired, current, RuntimeInfo, ImmutableDictionary.Empty); + Assert.Equal(Plan.Empty, plan); + } + + [Fact] + [Unit] + public async void KubernetesPlannerPlanExistsWhenChangesMade() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + IModule m2 = new DockerModule("module2", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + ModuleSet desired = ModuleSet.Create(m1); + ModuleSet current = ModuleSet.Create(m2); + + var planner = new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider); + var plan = await planner.PlanAsync(desired, current, RuntimeInfo, ImmutableDictionary.Empty); + Assert.Single(plan.Commands); + Assert.True(plan.Commands.First() is KubernetesCrdCommand); + } + + [Fact] + [Unit] + public async void KubernetesPlannerShutdownTest() + { + IModule m1 = new DockerModule("module1", "v1", ModuleStatus.Running, RestartPolicy.Always, Config1, ImagePullPolicy.OnCreate, DefaultConfigurationInfo, EnvVars); + ModuleSet current = ModuleSet.Create(m1); + + var planner = new KubernetesPlanner(Ns, Hostname, DeviceId, DefaultClient, DefaultCommandFactory, DefaultConfigProvider); + var plan = await planner.CreateShutdownPlanAsync(current); + Assert.Equal(Plan.Empty, plan); + } + + class NonDockerModule : IModule + { + public NonDockerModule(string name, string version, string type, ModuleStatus desiredStatus, RestartPolicy restartPolicy, ImagePullPolicy imagePullPolicy, ConfigurationInfo configurationInfo, IDictionary env, string config) + { + this.Name = name; + this.Version = version; + this.Type = type; + this.DesiredStatus = desiredStatus; + this.RestartPolicy = restartPolicy; + this.ImagePullPolicy = imagePullPolicy; + this.ConfigurationInfo = configurationInfo; + this.Env = env; + this.Config = config; + } + + public bool Equals(IModule other) => throw new NotImplementedException(); + + public string Name { get; set; } + + public string Version { get; } + + public string Type { get; } + + public ModuleStatus DesiredStatus { get; } + + public RestartPolicy RestartPolicy { get; } + + public ImagePullPolicy ImagePullPolicy { get; } + + public ConfigurationInfo ConfigurationInfo { get; } + + public IDictionary Env { get; } + + public bool OnlyModuleStatusChanged(IModule other) => throw new NotImplementedException(); + + public bool Equals(IModule other) => throw new NotImplementedException(); + + public string Config { get; } + } + } +}