From eb178e7eddb44639323f9e75f804fd38300b4b67 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 13 Dec 2019 10:25:46 +1000 Subject: [PATCH] Deploy NSG and VM public IPs when internal standard LB selected (#325) This commit deploys a Network Security Group when a Standard SKU internal Load Balancer is selected and the loadBalancerType is internal. Each VM in the backend pool is also assigned a Standard SKU public IP address to allow outbound traffic. Different Basic and Standard SKU load balancer or public IP resources in an Availability Set is not allowed, so public IPs must be Standard SKU for VMs when deploying standard internal loadbalancer. Add integration tests for Standard Internal load balancer when used with loadBalancerType `internal` and `gateway` Fixes #308 (cherry picked from commit 1553d6039709e6ff7c1fa3683030799f821db873) --- README.md | 8 ++- build/arm-tests/1d-0m-0c-ags-slb-kp.json | 27 ++++++++++ build/arm-tests/1d-0m-0c-int-slb-kp.json | 20 ++++++++ docs/azure-arm-template.asciidoc | 37 ++++++++------ src/partials/node-resources.json | 65 ++++++++++++++---------- src/partials/vm.json | 60 +++++++++++++++++++--- 6 files changed, 167 insertions(+), 50 deletions(-) create mode 100644 build/arm-tests/1d-0m-0c-ags-slb-kp.json create mode 100644 build/arm-tests/1d-0m-0c-int-slb-kp.json diff --git a/README.md b/README.md index 01cd2ffb..28dbce3b 100644 --- a/README.md +++ b/README.md @@ -137,14 +137,18 @@ value defined in the template. loadBalancerInternalSkustring The internal load balancer SKU. Can be Basic or Standard. - Basic + Basic. When the Standard load balanacer is selected, + and the loadBalancerType is internal, A Network Security Group is also deployed + and a public IP address attached to each VM network interface card in the backend pool, to allow + outbound internet traffic to install the Elastic Stack and dependencies. loadBalancerExternalSkustring The external load balancer SKU. Can be Basic or Standard. Only relevant when loadBalancerType is external. When the Standard load balancer SKU is selected, the public IP address SKU attached to the external load balancer - will also be Standard. + will also be Standard. A Network Security Group is also deployed, to allow inbound internet traffic + to the load balancer backend pool. Basic diff --git a/build/arm-tests/1d-0m-0c-ags-slb-kp.json b/build/arm-tests/1d-0m-0c-ags-slb-kp.json new file mode 100644 index 00000000..cc26ba93 --- /dev/null +++ b/build/arm-tests/1d-0m-0c-ags-slb-kp.json @@ -0,0 +1,27 @@ +{ + "description": "1 data node cluster with Standard loadbalancer and Application Gateway", + "isValid" : true, + "deploy" : true, + "why" : "", + "location" : "westeurope", + "parameters" : { + "loadBalancerType":{"value":"gateway"}, + "loadBalancerInternalSku":{"value":"Standard"}, + "kibana":{"value":"Yes"}, + "vmSizeKibana":{"value":"Standard_DS1_v2"}, + "vmSizeDataNodes":{"value":"Standard_DS1_v2"}, + "vmDataNodeCount":{"value":1}, + "vmDataDiskCount":{"value":0}, + "vmDataDiskSize":{"value":"32GiB"}, + "vmSizeClientNodes":{"value":"Standard_DS1_v2"}, + "dataNodesAreMasterEligible":{"value":"Yes"}, + "authenticationType":{"value":"password"}, + "appGatewayTier": {"value":"Standard"}, + "appGatewaySku": {"value":"Small"}, + "appGatewayCount": {"value":1}, + "appGatewayCertBlob": {"value":"certs/cert-with-password.pfx"}, + "appGatewayCertPassword": {"value":"Password123"}, + "appGatewayWafStatus": {"value":"Disabled"}, + "appGatewayWafMode": {"value":"Detection"} + } +} \ No newline at end of file diff --git a/build/arm-tests/1d-0m-0c-int-slb-kp.json b/build/arm-tests/1d-0m-0c-int-slb-kp.json new file mode 100644 index 00000000..a2d4d364 --- /dev/null +++ b/build/arm-tests/1d-0m-0c-int-slb-kp.json @@ -0,0 +1,20 @@ +{ + "description": "1 data node cluster with Standard internal loadbalancer", + "isValid" : true, + "deploy" : true, + "why" : "", + "location" : "westeurope", + "parameters" : { + "loadBalancerType":{"value":"internal"}, + "loadBalancerInternalSku":{"value":"Standard"}, + "kibana":{"value":"Yes"}, + "vmSizeKibana":{"value":"Standard_DS1_v2"}, + "vmSizeDataNodes":{"value":"Standard_DS1_v2"}, + "vmDataNodeCount":{"value":1}, + "vmDataDiskCount":{"value":0}, + "vmDataDiskSize":{"value":"32GiB"}, + "vmSizeClientNodes":{"value":"Standard_DS1_v2"}, + "dataNodesAreMasterEligible":{"value":"Yes"}, + "authenticationType":{"value":"password"} + } +} \ No newline at end of file diff --git a/docs/azure-arm-template.asciidoc b/docs/azure-arm-template.asciidoc index f6c2c801..47c4526b 100644 --- a/docs/azure-arm-template.asciidoc +++ b/docs/azure-arm-template.asciidoc @@ -743,32 +743,37 @@ The following settings apply to the internal load balancer `loadBalancerInternalSku`:: Choose between `Basic` and `Standard` load balancer SKUs for the internal load balancer. An -internal load balancer is always deployed, to balance internal traffic to the cluster. +internal load balancer is **always** deployed, to balance internal traffic to the cluster. When +the `Standard` load balancer SKU is selected and `loadBalancerType` is `internal`, a Network +Security Group will also be deployed, and each VM in the backend pool will be assigned +a `Standard` public IP address, to allow outbound internet traffic from the VMs in the backend +pool, which is required to install the Elastic Stack and dependencies. {loadbalancers}[Check the Azure documentation on Standard Load Balancers] to determine which options is best suited for your needs. Default is `Basic`. When coordinating nodes are deployed, they are attached to the internal load balancer -backend pool. When no coordinating nodes are deployed, data nodes are attached to +backend pool. When no coordinating nodes are deployed, the data nodes are attached to the backend pool. The load balancer receives incoming requests on port 9200 and round robins them across the backend pool over port 9200, with a TCP health probe that checks connectivity every 30 seconds, taking -nodes out of the backend pool when health probes fail. +nodes out of the backend pool when health probes fail. An idle timeout of 5 minutes +is also configured. When Kibana is deployed, Kibana is configured to communicate with Elasticsearch through the internal load balancer. [[external-load-balancer]] ==== External load balancer -An OSI layer 4 load balancer configured with a dynamically assigned public IP address +An OSI layer 4 load balancer configured with a public IP address that can be used to send requests to Elasticsearch from the public internet. -The following settings are applicablt to the external load balancer +The following settings are applicable to the external load balancer `loadBalancerExternalSku`:: Choose between `Basic` and `Standard` load balancer SKUs for the external load balancer. Only relevant when `loadBalancerType` is `external`. When the `Standard` load balancer SKU is selected, the public IP address SKU attached to the external load balancer will also be `Standard`, and a -Network Security Group will also be deployed to allow traffic to the VMs in the backend pool. +Network Security Group will also be deployed, to allow inbound internet traffic to the VMs in the backend pool. {loadbalancers}[Check the Azure documentation on Standard Load Balancers] to determine which options is best suited for your needs. Default is `Basic`. @@ -790,7 +795,8 @@ they are attached to the external load balancer backend pool. When no coordinating nodes are deployed, data nodes are attached to the backend pool. The load balancer round robins requests across the backend pool over port 9200, with a TCP health probe that checks connectivity every 30 seconds, taking -nodes out of the backend pool when health probes fail. +nodes out of the backend pool when health probes fail. An idle timeout of 5 minutes +is also configured. [[application-gateway]] ==== Application Gateway @@ -887,8 +893,9 @@ $gatewayIp = "https://$($gatewayIpResource.DnsSettings.Fqdn):9200" [[azure-arm-template-kibana]] === Kibana -Kibana can be deployed in addition to Elasticsearch, providing a visual window and UI into the data within Elasticsearch. The version of Kibana deployed is always the same -as the version of Elasticsearch, ensuring compatibility between the products. +Kibana can be deployed in addition to Elasticsearch, providing a visual window and UI into the +data within Elasticsearch. The version of Kibana deployed is always the same as the version of +Elasticsearch, ensuring compatibility between the products. The following parameters can be used to deploy Kibana, and control additional configuration @@ -914,7 +921,8 @@ networking performance. Valid values are `Default`, `Yes`, `No`. The default is enables accelerated networking for the VM SKUs known to support it. `kibanaAdditionalYaml`:: -Additional configuration that will be applied to the kibana.yml configuration file before start up. Each line must be separated by a `\n` newline character, for example +Additional configuration that will be applied to the kibana.yml configuration file before start up. +Each line must be separated by a `\n` newline character, for example + [source,text] ---- @@ -1023,7 +1031,7 @@ without the appropriate role required for authorization. In addition, securing the transport of data and communication between Elasticsearch, Kibana, the browser, or any other client by encrypting traffic is critical for data integrity. -The ARM template exposes a number of features for securing a deployment to Azure. +The ARM template exposes a number of features for securing a deployment in Azure. [[authentication-and-authorization]] ==== Authentication and Authorization @@ -1044,8 +1052,7 @@ The following parameters are used to configure initial user accounts `securityBootstrapPassword`:: Security password for {bootstrappassword}[`bootstrap.password` key] added to the Elasticsearch keystore. -The bootstrap password is used to seed the built-in users and is necessary only for -Elasticsearch 6.0 onwards. +The bootstrap password is used to seed the built-in users. + If no value is supplied, a 13 character password will be generated using the ARM template `uniqueString()` function. @@ -1086,8 +1093,8 @@ It has the `remote_monitoring_agent` and `remote_monitoring_collector` built-in Valid only for Elasticsearch 6.5.0+ It is recommended after deployment to use the `elastic` superuser account to create -the individual user accounts that will be needed for the users and applications -that will interact with Elasticsearch and Kibana, then to use these accounts going +additional individual user accounts that will be needed for users and applications +to interact with Elasticsearch and Kibana, and use these accounts going forward. [[saml-single-sign-on]] diff --git a/src/partials/node-resources.json b/src/partials/node-resources.json index 1a5fff69..031e7b9c 100644 --- a/src/partials/node-resources.json +++ b/src/partials/node-resources.json @@ -162,7 +162,31 @@ "Standard_M128", "Standard_M128m" ], - "externalNsgName": "[concat(parameters('commonVmSettings').namespacePrefix, 'external-lb-nsg')]" + "vmNsgName": "[concat(parameters('commonVmSettings').namespacePrefix, 'standard-lb-nsg')]", + "vmNsgProperties": [ + {}, + { + "securityRules": [ + { + "name": "External", + "properties": { + "description": "Allows inbound traffic from Standard External LB", + "protocol": "Tcp", + "sourcePortRange": "*", + "destinationPortRange": "9201", + "sourceAddressPrefix": "*", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 100, + "direction": "Inbound" + } + } + ] + } + ], + "standardInternalLoadBalancer": "[equals(parameters('networkSettings').internalSku, 'Standard')]", + "standardExternalLoadBalancer": "[equals(parameters('networkSettings').externalSku, 'Standard')]", + "standardInternalOrExternalLoadBalancer": "[or(variables('standardInternalLoadBalancer'), variables('standardExternalLoadBalancer'))]" }, "resources": [ { @@ -196,7 +220,8 @@ "imageReference": "[parameters('osSettings').imageReference]", "platformFaultDomainCount": "[variables('platformFaultDomainCount')]", "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmMasterNodeAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeMasterNodes), 'Yes', 'No'), parameters('topologySettings').vmMasterNodeAcceleratedNetworking)]", - "externalNsg": "" + "nsg": "", + "standardInternalLoadBalancer": false } }, "elasticTags": { @@ -206,32 +231,15 @@ } }, { - "condition": "[equals(parameters('networkSettings').externalSku, 'Standard')]", + "condition": "[variables('standardInternalOrExternalLoadBalancer')]", "apiVersion": "2019-04-01", "type": "Microsoft.Network/networkSecurityGroups", - "name": "[variables('externalNsgName')]", + "name": "[variables('vmNsgName')]", "location": "[parameters('commonVmSettings').location]", "tags": { "provider": "[toUpper(parameters('elasticTags').provider)]" }, - "properties": { - "securityRules": [ - { - "name": "External", - "properties": { - "description": "Allows inbound traffic from Standard External LB", - "protocol": "Tcp", - "sourcePortRange": "*", - "destinationPortRange": "9201", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*", - "access": "Allow", - "priority": 100, - "direction": "Inbound" - } - } - ] - } + "properties": "[variables('vmNsgProperties')[if(variables('standardExternalLoadBalancer'), 1, 0)]]" }, { "condition": "[greater(parameters('topologySettings').vmClientNodeCount, 0)]", @@ -239,7 +247,7 @@ "type": "Microsoft.Resources/deployments", "apiVersion": "2019-05-01", "dependsOn": [ - "[variables('externalNsgName')]" + "[variables('vmNsgName')]" ], "properties": { "mode": "Incremental", @@ -266,7 +274,8 @@ "imageReference": "[parameters('osSettings').imageReference]", "platformFaultDomainCount": "[variables('platformFaultDomainCount')]", "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmClientNodeAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeClientNodes), 'Yes', 'No'), parameters('topologySettings').vmClientNodeAcceleratedNetworking)]", - "externalNsg": "[if(equals(parameters('networkSettings').externalSku, 'Standard'), variables('externalNsgName'), '')]" + "nsg": "[if(variables('standardInternalOrExternalLoadBalancer'), variables('vmNsgName'), '')]", + "standardInternalLoadBalancer": "[variables('standardInternalLoadBalancer')]" } }, "elasticTags": { @@ -280,7 +289,7 @@ "type": "Microsoft.Resources/deployments", "apiVersion": "2019-05-01", "dependsOn": [ - "[variables('externalNsgName')]" + "[variables('vmNsgName')]" ], "properties": { "mode": "Incremental", @@ -307,7 +316,8 @@ "imageReference": "[parameters('osSettings').imageReference]", "platformFaultDomainCount": "[variables('platformFaultDomainCount')]", "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmDataNodeAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeDataNodes), 'Yes', 'No'), parameters('topologySettings').vmDataNodeAcceleratedNetworking)]", - "externalNsg": "[if(and(equals(parameters('networkSettings').externalSku, 'Standard'), equals(parameters('topologySettings').vmClientNodeCount, 0)), variables('externalNsgName'), '')]" + "nsg": "[if(and(variables('standardInternalOrExternalLoadBalancer'), equals(parameters('topologySettings').vmClientNodeCount, 0)), variables('vmNsgName'), '')]", + "standardInternalLoadBalancer": "[variables('standardInternalLoadBalancer')]" } }, "storageSettings": { @@ -422,7 +432,8 @@ "imageReference": "[parameters('osSettings').imageReference]", "platformFaultDomainCount": "[variables('platformFaultDomainCount')]", "acceleratedNetworking": "[if(equals(parameters('topologySettings').vmLogstashAcceleratedNetworking, 'Default'), if(contains(variables('vmAcceleratedNetworking'), parameters('topologySettings').vmSizeLogstash), 'Yes', 'No'), parameters('topologySettings').vmLogstashAcceleratedNetworking)]", - "externalNsg": "" + "nsg": "", + "standardInternalLoadBalancer": false } }, "elasticTags": { diff --git a/src/partials/vm.json b/src/partials/vm.json index c74def55..ce13735f 100644 --- a/src/partials/vm.json +++ b/src/partials/vm.json @@ -63,14 +63,40 @@ } }, "osProfile": "[variables(concat(parameters('vm').shared.credentials.authenticationType, '_osProfile'))]", - "externalNsgs": [ - { }, - { "networkSecurityGroup" : { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('vm').externalNsg)]" } } + "publicIpName": "[concat(variables('namespace'), parameters('index'), '-ip')]", + "nsgIpConfigs": [ + {}, + { + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('vm').nsg)]" + } + }, + { + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('vm').nsg)]" + }, + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "primary": true, + "privateIPAllocationMethod": "Dynamic", + "subnet": { + "id": "[parameters('vm').shared.subnetId]" + }, + "loadBalancerBackendAddressPools": "[parameters('vm').backendPools]", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]" + } + } + } + ] + } ], - "externalNsg": "[variables('externalNsgs')[if(empty(parameters('vm').externalNsg), 0, 1)]]", + "nsgIpConfig": "[variables('nsgIpConfigs')[if(empty(parameters('vm').nsg), 0, if(parameters('vm').standardInternalLoadBalancer, 2, 1))]]", "nicProperties": { "primary": true, - "enableAcceleratedNetworking": "[equals(parameters('vm').acceleratedNetworking, 'Yes')]", + "enableAcceleratedNetworking": "[equals(parameters('vm').acceleratedNetworking, 'Yes')]", "ipConfigurations": [ { "name": "ipconfig1", @@ -86,6 +112,25 @@ } }, "resources": [ + { + "condition": "[and(not(empty(parameters('vm').nsg)), parameters('vm').standardInternalLoadBalancer)]", + "apiVersion": "2019-04-01", + "type": "Microsoft.Network/publicIPAddresses", + "name": "[variables('publicIpName')]", + "location": "[parameters('vm').shared.location]", + "sku": { + "name": "Standard" + }, + "tags": { + "provider": "[toUpper(parameters('elasticTags').provider)]" + }, + "properties": { + "publicIPAllocationMethod": "Static", + "dnsSettings": { + "domainNameLabel": "[concat(variables('namespace'), parameters('index'), uniqueString(resourceGroup().id, deployment().name))]" + } + } + }, { "apiVersion": "2019-04-01", "type": "Microsoft.Network/networkInterfaces", @@ -94,7 +139,10 @@ "tags": { "provider": "[toUpper(parameters('elasticTags').provider)]" }, - "properties": "[union(variables('nicProperties'), variables('externalNsg'))]" + "dependsOn": [ + "[variables('publicIpName')]" + ], + "properties": "[union(variables('nicProperties'), variables('nsgIpConfig'))]" }, { "apiVersion": "2019-03-01",