Nested Edge deployment and config part 1 (#2970)

1. Add gateway_hostname paraemeter under provisioning section in iotedged config.yaml.
2. Set environment variable "IOTEDGE_GATEWAYHOSTNAME" to gateway_hostname parameter in config.yaml if defined, otherwise don't set.
3. Set environment variable "EdgeDeviceHostName" to host_name parameter in config.yaml for Edge agent/hub modules only.
4. All kubernetes related tests will not set parent edge as currently nested edge is not targeted to support with kubernetes.  Similar what did in DockerModule.

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Constants.cs

@@ -57,6 +57,8 @@ public static class Constants
 
         public const string GatewayHostnameVariableName = "IOTEDGE_GATEWAYHOSTNAME";
 
+        public const string ParentEdgeHostnameVariableName = "IOTEDGE_PARENTHOSTNAME";
+
         public const string DeviceIdVariableName = "IOTEDGE_DEVICEID";
 
         public const string ModuleIdVariableName = "IOTEDGE_MODULEID";

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/IModuleIdentity.cs

@@ -5,8 +5,6 @@ public interface IModuleIdentity
     {
         string IotHubHostname { get; }
 
-        string GatewayHostname { get; }
-
         string DeviceId { get; }
 
         string ModuleId { get; }

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/ModuleIdentity.cs

@@ -5,19 +5,20 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Core
 
     public class ModuleIdentity : IModuleIdentity
     {
-        public ModuleIdentity(string iotHubHostname, string gatewayHostname, string deviceId, string moduleId, ICredentials credentials)
+        public ModuleIdentity(
+            string iotHubHostname,
+            string deviceId,
+            string moduleId,
+            ICredentials credentials)
         {
-            this.IotHubHostname = Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(this.IotHubHostname));
-            this.GatewayHostname = gatewayHostname;
-            this.DeviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(this.DeviceId));
-            this.ModuleId = Preconditions.CheckNonWhiteSpace(moduleId, nameof(this.ModuleId));
-            this.Credentials = Preconditions.CheckNotNull(credentials, nameof(this.Credentials));
+            this.IotHubHostname = Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname));
+            this.DeviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId));
+            this.ModuleId = Preconditions.CheckNonWhiteSpace(moduleId, nameof(moduleId));
+            this.Credentials = Preconditions.CheckNotNull(credentials, nameof(credentials));
         }
 
         public string IotHubHostname { get; }
 
-        public string GatewayHostname { get; }
-
         public string DeviceId { get; }
 
         public string ModuleId { get; }

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/ModuleIdentityProviderServiceBuilder.cs

@@ -1,19 +1,18 @@
 // Copyright (c) Microsoft. All rights reserved.
 namespace Microsoft.Azure.Devices.Edge.Agent.Core
 {
+    using System;
     using Microsoft.Azure.Devices.Edge.Util;
 
     public class ModuleIdentityProviderServiceBuilder
     {
-        readonly string iotHubHostName;
+        readonly string iotHubHostname;
         readonly string deviceId;
-        readonly string gatewayHostname;
 
-        public ModuleIdentityProviderServiceBuilder(string iotHubHostName, string deviceId, string gatewayHostname)
+        public ModuleIdentityProviderServiceBuilder(string iotHubHostname, string deviceId)
         {
-            this.iotHubHostName = Preconditions.CheckNonWhiteSpace(iotHubHostName, nameof(iotHubHostName));
+            this.iotHubHostname = Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname));
             this.deviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId));
-            this.gatewayHostname = gatewayHostname;
         }
 
         public IModuleIdentity Create(string moduleId, string generationId, string providerUri)
@@ -23,7 +22,7 @@ public IModuleIdentity Create(string moduleId, string generationId, string provi
             Preconditions.CheckNonWhiteSpace(providerUri, nameof(providerUri));
 
             ICredentials credentials = new IdentityProviderServiceCredentials(providerUri, generationId);
-            return new ModuleIdentity(this.iotHubHostName, this.gatewayHostname, this.deviceId, moduleId, credentials);
+            return new ModuleIdentity(this.iotHubHostname, this.deviceId, moduleId, credentials);
         }
 
         public IModuleIdentity Create(string moduleId, string generationId, string providerUri, string authScheme)
@@ -34,7 +33,7 @@ public IModuleIdentity Create(string moduleId, string generationId, string provi
             Preconditions.CheckNonWhiteSpace(authScheme, nameof(authScheme));
 
             ICredentials credentials = new IdentityProviderServiceCredentials(providerUri, generationId, authScheme);
-            return new ModuleIdentity(this.iotHubHostName, this.gatewayHostname, this.deviceId, moduleId, credentials);
+            return new ModuleIdentity(this.iotHubHostname, this.deviceId, moduleId, credentials);
         }
     }
 }

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/EdgeletCommandFactory.cs

@@ -1,23 +1,37 @@
 // Copyright (c) Microsoft. All rights reserved.
 namespace Microsoft.Azure.Devices.Edge.Agent.Edgelet
 {
+    using System;
     using System.Threading.Tasks;
     using Microsoft.Azure.Devices.Edge.Agent.Core;
     using Microsoft.Azure.Devices.Edge.Agent.Core.Commands;
     using Microsoft.Azure.Devices.Edge.Agent.Edgelet.Commands;
     using Microsoft.Azure.Devices.Edge.Util;
+    using Microsoft.Extensions.Configuration;
 
     public class EdgeletCommandFactory<T> : ICommandFactory
     {
         readonly IConfigSource configSource;
         readonly IModuleManager moduleManager;
         readonly ICombinedConfigProvider<T> combinedConfigProvider;
+        readonly string edgeDeviceHostname;
+        readonly Option<string> parentEdgeHostname;
 
-        public EdgeletCommandFactory(IModuleManager moduleManager, IConfigSource configSource, ICombinedConfigProvider<T> combinedConfigProvider)
+        public EdgeletCommandFactory(
+            IModuleManager moduleManager,
+            IConfigSource configSource,
+            ICombinedConfigProvider<T> combinedConfigProvider)
         {
             this.moduleManager = Preconditions.CheckNotNull(moduleManager, nameof(moduleManager));
             this.configSource = Preconditions.CheckNotNull(configSource, nameof(configSource));
             this.combinedConfigProvider = Preconditions.CheckNotNull(combinedConfigProvider, nameof(combinedConfigProvider));
+            this.edgeDeviceHostname = this.configSource.Configuration.GetValue<string>(Constants.EdgeDeviceHostNameKey);
+            if (string.IsNullOrWhiteSpace(this.edgeDeviceHostname))
+            {
+                throw new ArgumentException("EdgeDeviceHostname from configuration is null, empty or whitespaces.");
+            }
+
+            this.parentEdgeHostname = Option.Maybe(this.configSource.Configuration.GetValue<string>(Constants.GatewayHostnameVariableName));
         }
 
         public Task<ICommand> CreateAsync(IModuleWithIdentity module, IRuntimeInfo runtimeInfo) =>
@@ -27,7 +41,10 @@ public Task<ICommand> CreateAsync(IModuleWithIdentity module, IRuntimeInfo runti
                     module.Module,
                     module.ModuleIdentity,
                     this.configSource,
-                    this.combinedConfigProvider.GetCombinedConfig(module.Module, runtimeInfo)) as ICommand);
+                    this.combinedConfigProvider.GetCombinedConfig(module.Module, runtimeInfo),
+                    this.edgeDeviceHostname,
+                    this.parentEdgeHostname)
+                as ICommand);
 
         public Task<ICommand> UpdateAsync(IModule current, IModuleWithIdentity next, IRuntimeInfo runtimeInfo) =>
             this.UpdateAsync(Option.Some(current), next, runtimeInfo, false);
@@ -57,7 +74,10 @@ await current.Match(this.StopAsync, () => Task.FromResult<ICommand>(NullCommand.
                     next.ModuleIdentity,
                     this.configSource,
                     config,
-                    start) as ICommand);
+                    start,
+                    this.edgeDeviceHostname,
+                    this.parentEdgeHostname)
+                as ICommand);
         }
     }
 }

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/IModuleManager.cs

@@ -6,7 +6,6 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Edgelet
     using System.Threading;
     using System.Threading.Tasks;
     using Microsoft.Azure.Devices.Edge.Agent.Core;
-    using Microsoft.Azure.Devices.Edge.Agent.Core.Metrics;
     using Microsoft.Azure.Devices.Edge.Agent.Edgelet.Models;
     using Microsoft.Azure.Devices.Edge.Util;
 

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Edgelet/commands/CreateOrUpdateCommand.cs

@@ -36,22 +36,28 @@ enum Operation
 
         public string Id => this.id.Value;
 
+        public ModuleSpec ModuleSpec => this.moduleSpec;
+
         public static CreateOrUpdateCommand BuildCreate(
             IModuleManager moduleManager,
             IModule module,
             IModuleIdentity identity,
             IConfigSource configSource,
-            object settings) =>
-            Build(moduleManager, module, identity, configSource, settings, Operation.Create);
+            object settings,
+            string edgeDeviceHostname,
+            Option<string> parentEdgeHostname) =>
+            Build(moduleManager, module, identity, configSource, settings, Operation.Create, edgeDeviceHostname, parentEdgeHostname);
 
         public static CreateOrUpdateCommand BuildUpdate(
             IModuleManager moduleManager,
             IModule module,
             IModuleIdentity identity,
             IConfigSource configSource,
             object settings,
-            bool start) =>
-            Build(moduleManager, module, identity, configSource, settings, start ? Operation.UpdateAndStart : Operation.Update);
+            bool start,
+            string edgeDeviceHostname,
+            Option<string> parentEdgeHostname) =>
+            Build(moduleManager, module, identity, configSource, settings, start ? Operation.UpdateAndStart : Operation.Update, edgeDeviceHostname, parentEdgeHostname);
 
         public Task ExecuteAsync(CancellationToken token)
         {
@@ -90,7 +96,12 @@ static ModuleSpec BuildModuleSpec(IModule module, IEnumerable<EnvVar> envVars, o
             return new ModuleSpec(module.Name, module.Type, module.ImagePullPolicy, settings, envVars);
         }
 
-        static IEnumerable<EnvVar> GetEnvVars(IDictionary<string, EnvVal> moduleEnvVars, IModuleIdentity identity, IConfigSource configSource)
+        static IEnumerable<EnvVar> GetEnvVars(
+            IDictionary<string, EnvVal> moduleEnvVars,
+            IModuleIdentity identity,
+            IConfigSource configSource,
+            string edgeDeviceHostname,
+            Option<string> parentEdgeHostname)
         {
             List<EnvVar> envVars = moduleEnvVars.Select(m => new EnvVar(m.Key, m.Value.Value)).ToList();
 
@@ -118,16 +129,17 @@ static IEnumerable<EnvVar> GetEnvVars(IDictionary<string, EnvVal> moduleEnvVars,
                 envVars.Add(new EnvVar(Constants.IotHubHostnameVariableName, identity.IotHubHostname));
             }
 
-            if (!string.IsNullOrWhiteSpace(identity.GatewayHostname))
+            // In nested edge scenario, EdgeAgent and EdgeHub will use parent edge as upstream gateway,
+            // Other modules use current edge hub as upstream gateway and have parent edge hostname in environment variable.
+            if (identity.ModuleId.Equals(Constants.EdgeAgentModuleIdentityName) || identity.ModuleId.Equals(Constants.EdgeHubModuleIdentityName))
             {
-                if (identity.ModuleId.Equals(Constants.EdgeAgentModuleIdentityName) || identity.ModuleId.Equals(Constants.EdgeHubModuleIdentityName))
-                {
-                    envVars.Add(new EnvVar(Constants.EdgeDeviceHostNameKey, identity.GatewayHostname));
-                }
-                else if (!identity.ModuleId.Equals(Constants.EdgeHubModuleIdentityName))
-                {
-                    envVars.Add(new EnvVar(Constants.GatewayHostnameVariableName, identity.GatewayHostname));
-                }
+                envVars.Add(new EnvVar(Constants.EdgeDeviceHostNameKey, edgeDeviceHostname));
+                parentEdgeHostname.ForEach(value => envVars.Add(new EnvVar(Constants.GatewayHostnameVariableName, value)));
+            }
+            else
+            {
+                envVars.Add(new EnvVar(Constants.GatewayHostnameVariableName, edgeDeviceHostname));
+                parentEdgeHostname.ForEach(value => envVars.Add(new EnvVar(Constants.ParentEdgeHostnameVariableName, value)));
             }
 
             if (!string.IsNullOrWhiteSpace(identity.DeviceId))
@@ -187,15 +199,18 @@ static CreateOrUpdateCommand Build(
             IModuleIdentity identity,
             IConfigSource configSource,
             object settings,
-            Operation operation)
+            Operation operation,
+            string edgeDeviceHostname,
+            Option<string> parentEdgeHostname)
         {
             Preconditions.CheckNotNull(moduleManager, nameof(moduleManager));
             Preconditions.CheckNotNull(module, nameof(module));
             Preconditions.CheckNotNull(identity, nameof(identity));
             Preconditions.CheckNotNull(configSource, nameof(configSource));
             Preconditions.CheckNotNull(settings, nameof(settings));
+            Preconditions.CheckNonWhiteSpace(edgeDeviceHostname, nameof(edgeDeviceHostname));
 
-            IEnumerable<EnvVar> envVars = GetEnvVars(module.Env, identity, configSource);
+            IEnumerable<EnvVar> envVars = GetEnvVars(module.Env, identity, configSource, edgeDeviceHostname, parentEdgeHostname);
             ModuleSpec moduleSpec = BuildModuleSpec(module, envVars, settings);
             return new CreateOrUpdateCommand(moduleManager, moduleSpec, operation);
         }

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/ModuleIdentityLifecycleManager.cs

@@ -110,7 +110,11 @@ async Task<IImmutableDictionary<string, IModuleIdentity>> GetModuleIdentitiesAsy
                 p =>
                 {
                     string connectionString = this.GetModuleConnectionString(p);
-                    return new ModuleIdentity(this.iothubHostName, this.gatewayHostName, this.deviceId, p.Id, new ConnectionStringCredentials(connectionString));
+                    return new ModuleIdentity(
+                        this.iothubHostName,
+                        this.deviceId,
+                        p.Id,
+                        new ConnectionStringCredentials(connectionString));
                 });
 
             return moduleIdentities.ToImmutableDictionary(m => ModuleIdentityHelper.GetModuleName(m.ModuleId));