Deploy NSG and VM public IPs when internal standard LB selected (#325)

This commit deploys a Network Security Group when a Standard SKU internal Load Balancer is selected and the loadBalancerType is internal. Each VM in the backend pool is also assigned a Standard SKU public IP address to allow outbound traffic. Different Basic and Standard SKU load balancer or public IP resources in an Availability Set is not allowed, so public IPs must be Standard SKU for VMs when deploying standard internal loadbalancer. Add integration tests for Standard Internal load balancer when used with loadBalancerType `internal` and `gateway` Fixes #308 (cherry picked from commit 1553d60)
elastic · Dec 13, 2019 · eb178e7 · eb178e7
1 parent 0f42c2b
commit eb178e7
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 50 deletions.
diff --git a/README.md b/README.md
@@ -137,14 +137,18 @@ value defined in the template.
 
   <tr><td>loadBalancerInternalSku</td><td>string</td>
     <td>The internal load balancer SKU. Can be <code>Basic</code> or <code>Standard</code>.</td>
-    </td><td><code>Basic</code></td>
+    </td><td><code>Basic</code></td>. When the <code>Standard</code> load balanacer is selected,
+    and the <code>loadBalancerType</code> is <code>internal</code>, A Network Security Group is also deployed
+    and a public IP address attached to each VM network interface card in the backend pool, to allow
+    outbound internet traffic to install the Elastic Stack and dependencies.
   </tr>
 
   <tr><td>loadBalancerExternalSku</td><td>string</td>
     <td>The external load balancer SKU. Can be <code>Basic</code> or <code>Standard</code>.
       Only relevant when <code>loadBalancerType</code> is <code>external</code>. When the <code>Standard</code> 
       load balancer SKU is selected, the public IP address SKU attached to the external load balancer 
-      will also be <code>Standard</code>.
+      will also be <code>Standard</code>. A Network Security Group is also deployed, to allow inbound internet traffic
+      to the load balancer backend pool.
     </td>
     </td><td><code>Basic</code></td>
   </tr>

diff --git a/build/arm-tests/1d-0m-0c-ags-slb-kp.json b/build/arm-tests/1d-0m-0c-ags-slb-kp.json
@@ -0,0 +1,27 @@
+{
+  "description": "1 data node cluster with Standard loadbalancer and Application Gateway",
+    "isValid" : true,
+    "deploy" : true,
+    "why" : "",
+    "location" : "westeurope",
+    "parameters" : {
+      "loadBalancerType":{"value":"gateway"},
+      "loadBalancerInternalSku":{"value":"Standard"},
+      "kibana":{"value":"Yes"},
+      "vmSizeKibana":{"value":"Standard_DS1_v2"},
+      "vmSizeDataNodes":{"value":"Standard_DS1_v2"},
+      "vmDataNodeCount":{"value":1},
+      "vmDataDiskCount":{"value":0},
+      "vmDataDiskSize":{"value":"32GiB"},
+      "vmSizeClientNodes":{"value":"Standard_DS1_v2"},
+      "dataNodesAreMasterEligible":{"value":"Yes"},
+      "authenticationType":{"value":"password"},
+      "appGatewayTier": {"value":"Standard"},
+      "appGatewaySku": {"value":"Small"},
+      "appGatewayCount": {"value":1},
+      "appGatewayCertBlob": {"value":"certs/cert-with-password.pfx"},
+      "appGatewayCertPassword": {"value":"Password123"},
+      "appGatewayWafStatus": {"value":"Disabled"},
+      "appGatewayWafMode": {"value":"Detection"}
+    }
+}
diff --git a/build/arm-tests/1d-0m-0c-int-slb-kp.json b/build/arm-tests/1d-0m-0c-int-slb-kp.json
@@ -0,0 +1,20 @@
+{
+  "description": "1 data node cluster with Standard internal loadbalancer",
+    "isValid" : true,
+    "deploy" : true,
+    "why" : "",
+    "location" : "westeurope",
+    "parameters" : {
+      "loadBalancerType":{"value":"internal"},
+      "loadBalancerInternalSku":{"value":"Standard"},
+      "kibana":{"value":"Yes"},
+      "vmSizeKibana":{"value":"Standard_DS1_v2"},
+      "vmSizeDataNodes":{"value":"Standard_DS1_v2"},
+      "vmDataNodeCount":{"value":1},
+      "vmDataDiskCount":{"value":0},
+      "vmDataDiskSize":{"value":"32GiB"},
+      "vmSizeClientNodes":{"value":"Standard_DS1_v2"},
+      "dataNodesAreMasterEligible":{"value":"Yes"},
+      "authenticationType":{"value":"password"}
+    }
+}
diff --git a/docs/azure-arm-template.asciidoc b/docs/azure-arm-template.asciidoc
@@ -743,32 +743,37 @@ The following settings apply to the internal load balancer
 
 `loadBalancerInternalSku`::
 Choose between `Basic` and `Standard` load balancer SKUs for the internal load balancer. An
-internal load balancer is always deployed, to balance internal traffic to the cluster.
+internal load balancer is **always** deployed, to balance internal traffic to the cluster. When 
+the `Standard` load balancer SKU is selected and `loadBalancerType` is `internal`, a Network 
+Security Group will also be deployed, and each VM in the backend pool will be assigned 
+a `Standard` public IP address, to allow outbound internet traffic from the VMs in the backend 
+pool, which is required to install the Elastic Stack and dependencies.
 {loadbalancers}[Check the Azure documentation on Standard Load Balancers] to determine which
 options is best suited for your needs. Default is `Basic`.
 
 When coordinating nodes are deployed, they are attached to the internal load balancer
-backend pool. When no coordinating nodes are deployed, data nodes are attached to
+backend pool. When no coordinating nodes are deployed, the data nodes are attached to
 the backend pool. The load balancer receives incoming requests on port 9200 and
 round robins them across the backend pool over port 9200,
 with a TCP health probe that checks connectivity every 30 seconds, taking
-nodes out of the backend pool when health probes fail.
+nodes out of the backend pool when health probes fail. An idle timeout of 5 minutes
+is also configured.
 
 When Kibana is deployed, Kibana is configured to communicate with Elasticsearch
 through the internal load balancer.
 
 [[external-load-balancer]]
 ==== External load balancer
-An OSI layer 4 load balancer configured with a dynamically assigned public IP address
+An OSI layer 4 load balancer configured with a public IP address
 that can be used to send requests to Elasticsearch from the public internet.
 
-The following settings are applicablt to the external load balancer
+The following settings are applicable to the external load balancer
 
 `loadBalancerExternalSku`::
 Choose between `Basic` and `Standard` load balancer SKUs for the external load balancer. Only
 relevant when `loadBalancerType` is `external`. When the `Standard` load balancer SKU is selected,
 the public IP address SKU attached to the external load balancer will also be `Standard`, and a
-Network Security Group will also be deployed to allow traffic to the VMs in the backend pool.
+Network Security Group will also be deployed, to allow inbound internet traffic to the VMs in the backend pool.
 {loadbalancers}[Check the Azure documentation on Standard Load Balancers] to determine which
 options is best suited for your needs. Default is `Basic`.
 
@@ -790,7 +795,8 @@ they are attached to the external load balancer backend pool.
 When no coordinating nodes are deployed, data nodes are attached to
 the backend pool. The load balancer round robins requests across the backend pool
 over port 9200, with a TCP health probe that checks connectivity every 30 seconds, taking
-nodes out of the backend pool when health probes fail.
+nodes out of the backend pool when health probes fail. An idle timeout of 5 minutes
+is also configured.
 
 [[application-gateway]]
 ==== Application Gateway
@@ -887,8 +893,9 @@ $gatewayIp = "https://$($gatewayIpResource.DnsSettings.Fqdn):9200"
 [[azure-arm-template-kibana]]
 === Kibana
 
-Kibana can be deployed in addition to Elasticsearch, providing a visual window and UI into the data within Elasticsearch. The version of Kibana deployed is always the same
-as the version of Elasticsearch, ensuring compatibility between the products.
+Kibana can be deployed in addition to Elasticsearch, providing a visual window and UI into the 
+data within Elasticsearch. The version of Kibana deployed is always the same as the version of 
+Elasticsearch, ensuring compatibility between the products.
 
 The following parameters can be used to deploy Kibana, and control additional configuration
 
@@ -914,7 +921,8 @@ networking performance. Valid values are `Default`, `Yes`, `No`. The default is
 enables accelerated networking for the VM SKUs known to support it.
 
 `kibanaAdditionalYaml`::
-Additional configuration that will be applied to the kibana.yml configuration file before start up. Each line must be separated by a `\n` newline character, for example
+Additional configuration that will be applied to the kibana.yml configuration file before start up. 
+Each line must be separated by a `\n` newline character, for example
 +
 [source,text]
 ----
@@ -1023,7 +1031,7 @@ without the appropriate role required for authorization. In addition, securing
 the transport of data and communication between Elasticsearch, Kibana, the browser,
 or any other client by encrypting traffic is critical for data integrity.
 
-The ARM template exposes a number of features for securing a deployment to Azure.
+The ARM template exposes a number of features for securing a deployment in Azure.
 
 [[authentication-and-authorization]]
 ==== Authentication and Authorization
@@ -1044,8 +1052,7 @@ The following parameters are used to configure initial user accounts
 
 `securityBootstrapPassword`::
 Security password for {bootstrappassword}[`bootstrap.password` key] added to the Elasticsearch keystore.
-The bootstrap password is used to seed the built-in users and is necessary only for
-Elasticsearch 6.0 onwards.
+The bootstrap password is used to seed the built-in users.
 +
 If no value is supplied, a 13 character password will be generated using the
 ARM template `uniqueString()` function.
@@ -1086,8 +1093,8 @@ It has the `remote_monitoring_agent` and `remote_monitoring_collector` built-in
 Valid only for Elasticsearch 6.5.0+
 
 It is recommended after deployment to use the `elastic` superuser account to create
-the individual user accounts that will be needed for the users and applications
-that will interact with Elasticsearch and Kibana, then to use these accounts going
+additional individual user accounts that will be needed for users and applications
+to interact with Elasticsearch and Kibana, and use these accounts going
 forward.
 
 [[saml-single-sign-on]]

diff --git a/src/partials/node-resources.json b/src/partials/node-resources.json
@@ -162,7 +162,31 @@
       "Standard_M128",
       "Standard_M128m"
     ],
-    "externalNsgName": "[concat(parameters('commonVmSettings').namespacePrefix, 'external-lb-nsg')]"
+    "vmNsgName": "[concat(parameters('commonVmSettings').namespacePrefix, 'standard-lb-nsg')]",
+    "vmNsgProperties": [
+      {},
+      {
+        "securityRules": [
+          {
+            "name": "External",
+            "properties": {
+              "description": "Allows inbound traffic from Standard External LB",
+              "protocol": "Tcp",
+              "sourcePortRange": "*",
+              "destinationPortRange": "9201",
+              "sourceAddressPrefix": "*",
+              "destinationAddressPrefix": "*",
+              "access": "Allow",
+              "priority": 100,
+              "direction": "Inbound"
+            }
+          }
+        ]
+      }
+    ],
+    "standardInternalLoadBalancer": "[equals(parameters('networkSettings').internalSku, 'Standard')]",
+    "standardExternalLoadBalancer": "[equals(parameters('networkSettings').externalSku, 'Standard')]",
+    "standardInternalOrExternalLoadBalancer": "[or(variables('standardInternalLoadBalancer'), variables('standardExternalLoadBalancer'))]"
   },
   "resources": [
     {
@@ -196,7 +220,8 @@
               "imageReference": "[parameters('osSettings').imageReference]",
               "platformFaultDomainCount": "[variables('platformFaultDomainCount')]",
               "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmMasterNodeAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeMasterNodes), 'Yes', 'No'), parameters('topologySettings').vmMasterNodeAcceleratedNetworking)]",
-              "externalNsg": ""
+              "nsg": "",
+              "standardInternalLoadBalancer": false
             }
           },
           "elasticTags": {
@@ -206,40 +231,23 @@
       }
     },
     {
-      "condition": "[equals(parameters('networkSettings').externalSku, 'Standard')]",
+      "condition": "[variables('standardInternalOrExternalLoadBalancer')]",
       "apiVersion": "2019-04-01",
       "type": "Microsoft.Network/networkSecurityGroups",
-      "name": "[variables('externalNsgName')]",
+      "name": "[variables('vmNsgName')]",
       "location": "[parameters('commonVmSettings').location]",
       "tags": {
         "provider": "[toUpper(parameters('elasticTags').provider)]"
       },
-      "properties": {
-        "securityRules": [
-          {
-            "name": "External",
-            "properties": {
-              "description": "Allows inbound traffic from Standard External LB",
-              "protocol": "Tcp",
-              "sourcePortRange": "*",
-              "destinationPortRange": "9201",
-              "sourceAddressPrefix": "*",
-              "destinationAddressPrefix": "*",
-              "access": "Allow",
-              "priority": 100,
-              "direction": "Inbound"
-            }
-          }
-        ]
-      }
+      "properties": "[variables('vmNsgProperties')[if(variables('standardExternalLoadBalancer'), 1, 0)]]"
     },
     {
       "condition": "[greater(parameters('topologySettings').vmClientNodeCount, 0)]",
       "name": "client-nodes",
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2019-05-01",
       "dependsOn": [
-        "[variables('externalNsgName')]"
+        "[variables('vmNsgName')]"
       ],
       "properties": {
         "mode": "Incremental",
@@ -266,7 +274,8 @@
               "imageReference": "[parameters('osSettings').imageReference]",
               "platformFaultDomainCount": "[variables('platformFaultDomainCount')]",
               "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmClientNodeAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeClientNodes), 'Yes', 'No'), parameters('topologySettings').vmClientNodeAcceleratedNetworking)]",
-              "externalNsg": "[if(equals(parameters('networkSettings').externalSku, 'Standard'), variables('externalNsgName'), '')]"
+              "nsg": "[if(variables('standardInternalOrExternalLoadBalancer'), variables('vmNsgName'), '')]",
+              "standardInternalLoadBalancer": "[variables('standardInternalLoadBalancer')]"
             }
           },
           "elasticTags": {
@@ -280,7 +289,7 @@
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2019-05-01",
       "dependsOn": [
-        "[variables('externalNsgName')]"
+        "[variables('vmNsgName')]"
       ],
       "properties": {
         "mode": "Incremental",
@@ -307,7 +316,8 @@
               "imageReference": "[parameters('osSettings').imageReference]",
               "platformFaultDomainCount": "[variables('platformFaultDomainCount')]",
               "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmDataNodeAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeDataNodes), 'Yes', 'No'), parameters('topologySettings').vmDataNodeAcceleratedNetworking)]",
-              "externalNsg": "[if(and(equals(parameters('networkSettings').externalSku, 'Standard'), equals(parameters('topologySettings').vmClientNodeCount, 0)), variables('externalNsgName'), '')]"
+              "nsg": "[if(and(variables('standardInternalOrExternalLoadBalancer'), equals(parameters('topologySettings').vmClientNodeCount, 0)), variables('vmNsgName'), '')]",
+              "standardInternalLoadBalancer": "[variables('standardInternalLoadBalancer')]"
             }
           },
           "storageSettings": {
@@ -422,7 +432,8 @@
               "imageReference": "[parameters('osSettings').imageReference]",
               "platformFaultDomainCount": "[variables('platformFaultDomainCount')]",
               "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmLogstashAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeLogstash), 'Yes', 'No'), parameters('topologySettings').vmLogstashAcceleratedNetworking)]",
-              "externalNsg": ""
+              "nsg": "",
+              "standardInternalLoadBalancer": false
             }
           },
           "elasticTags": {

diff --git a/src/partials/vm.json b/src/partials/vm.json
@@ -63,14 +63,40 @@
       }
     },
     "osProfile": "[variables(concat(parameters('vm').shared.credentials.authenticationType, '_osProfile'))]",
-    "externalNsgs": [
-      { },
-      { "networkSecurityGroup" : { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('vm').externalNsg)]" } }
+    "publicIpName": "[concat(variables('namespace'), parameters('index'), '-ip')]",
+    "nsgIpConfigs": [
+      {},
+      {
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('vm').nsg)]"
+        }
+      },
+      {
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('vm').nsg)]"
+        },
+        "ipConfigurations": [
+          {
+            "name": "ipconfig1",
+            "properties": {
+              "primary": true,
+              "privateIPAllocationMethod": "Dynamic",
+              "subnet": {
+                "id": "[parameters('vm').shared.subnetId]"
+              },
+              "loadBalancerBackendAddressPools": "[parameters('vm').backendPools]",
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]"
+              }
+            }
+          }
+        ]
+      }
     ],
-    "externalNsg": "[variables('externalNsgs')[if(empty(parameters('vm').externalNsg), 0, 1)]]",
+    "nsgIpConfig": "[variables('nsgIpConfigs')[if(empty(parameters('vm').nsg), 0, if(parameters('vm').standardInternalLoadBalancer, 2, 1))]]",
     "nicProperties": {
       "primary": true,
-      "enableAcceleratedNetworking": "[equals(parameters('vm').acceleratedNetworking, 'Yes')]",
+      "enableAcceleratedNetworking": "[equals(parameters('vm').acceleratedNetworking, 'Yes')]",    
       "ipConfigurations": [
         {
           "name": "ipconfig1",
@@ -86,6 +112,25 @@
     }
   },
   "resources": [
+    {
+      "condition": "[and(not(empty(parameters('vm').nsg)), parameters('vm').standardInternalLoadBalancer)]",
+      "apiVersion": "2019-04-01",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "[variables('publicIpName')]",
+      "location": "[parameters('vm').shared.location]",
+      "sku": {
+        "name": "Standard"
+      },
+      "tags": {
+        "provider": "[toUpper(parameters('elasticTags').provider)]"
+      },
+      "properties": {
+        "publicIPAllocationMethod": "Static",
+        "dnsSettings": {
+          "domainNameLabel": "[concat(variables('namespace'), parameters('index'), uniqueString(resourceGroup().id, deployment().name))]"
+        }
+      }
+    },
     {
       "apiVersion": "2019-04-01",
       "type": "Microsoft.Network/networkInterfaces",
@@ -94,7 +139,10 @@
       "tags": {
         "provider": "[toUpper(parameters('elasticTags').provider)]"
       },
-      "properties": "[union(variables('nicProperties'), variables('externalNsg'))]"
+      "dependsOn": [
+        "[variables('publicIpName')]"
+      ],
+      "properties": "[union(variables('nicProperties'), variables('nsgIpConfig'))]"
     },
     {
       "apiVersion": "2019-03-01",