From 4ad479861f33dcbcb0673a0bdd5bd7d347d8dfa5 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 8 Nov 2018 14:00:05 -0800 Subject: [PATCH 01/11] add outbound nsg rules --- parts/k8s/kubernetesmasterresources.t | 28 +++++++++++++++++++++ parts/k8s/kubernetesmasterresourcesvmss.t | 30 ++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index 391018ad51..abeca29f0d 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -128,6 +128,34 @@ "sourceAddressPrefix": "*", "sourcePortRange": "*" } + }, + { + "name": "allow_vnet", + "properties": { + "access": "Allow", + "description": "Allow outbound internet to vnet", + "destinationAddressPrefix": "[parameters('masterSubnet')]", + "destinationPortRange": "*", + "direction": "Outbound", + "priority": 110, + "protocol": "*", + "sourceAddressPrefix": "VirtualNetwork", + "sourcePortRange": "*" + } + }, + { + "name": "block_outbound", + "properties": { + "access": "Deny", + "description": "Block outbound internet from master", + "destinationAddressPrefix": "*", + "destinationPortRange": "*", + "direction": "Outbound", + "priority": 120, + "protocol": "*", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } } ] }, diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index eecf6da433..853c0a9980 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -99,7 +99,35 @@ "sourceAddressPrefix": "*", "sourcePortRange": "*" } - } + }, + { + "name": "allow_vnet", + "properties": { + "access": "Allow", + "description": "Allow outbound internet to vnet", + "destinationAddressPrefix": "[parameters('masterSubnet')]", + "destinationPortRange": "*", + "direction": "Outbound", + "priority": 110, + "protocol": "*", + "sourceAddressPrefix": "VirtualNetwork", + "sourcePortRange": "*" + } + }, + { + "name": "block_outbound", + "properties": { + "access": "Deny", + "description": "Block outbound internet from master", + "destinationAddressPrefix": "*", + "destinationPortRange": "*", + "direction": "Outbound", + "priority": 120, + "protocol": "*", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } + } ] }, "type": "Microsoft.Network/networkSecurityGroups" From 04843e7b923d841cc3527ac6c237109c80901d51 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 8 Nov 2018 14:04:21 -0800 Subject: [PATCH 02/11] remove outbound check in CSE --- parts/k8s/kubernetesagentresourcesvmas.t | 2 +- parts/k8s/kubernetesagentresourcesvmss.t | 2 +- parts/k8s/kubernetesmasterresources.t | 2 +- parts/k8s/kubernetesmasterresourcesvmss.t | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index 16d1ef4ed1..f2057506c5 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -331,7 +331,7 @@ {{if IsOpenShift }} "script": "{{ Base64 (OpenShiftGetNodeSh .) }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index daacce8bb3..5102a8417c 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -155,7 +155,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" } } } diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index abeca29f0d..ff3b3e72b6 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -957,7 +957,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index 853c0a9980..c09d780722 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -459,7 +459,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } } From 5be6db8e2daa10ff278a0d15aec786d7df15baef Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 8 Nov 2018 17:21:58 -0800 Subject: [PATCH 03/11] overwrite azure cni version --- pkg/api/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/const.go b/pkg/api/const.go index 0b3a296b0b..83ccdf8f30 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -240,7 +240,7 @@ const ( // AzureCniPluginVerLinux specifies version of Azure CNI plugin, which has been mirrored from // https://github.com/Azure/azure-container-networking/releases/download/${AZURE_PLUGIN_VER}/azure-vnet-cni-linux-amd64-${AZURE_PLUGIN_VER}.tgz // to https://acs-mirror.azureedge.net/cni - AzureCniPluginVerLinux = "v1.0.13" + AzureCniPluginVerLinux = "v1.0.12" // AzureCniPluginVerWindows specifies version of Azure CNI plugin, which has been mirrored from // https://github.com/Azure/azure-container-networking/releases/download/${AZURE_PLUGIN_VER}/azure-vnet-cni-windows-amd64-${AZURE_PLUGIN_VER}.tgz // to https://acs-mirror.azureedge.net/cni From dd6898a39f9810d7274fbc2ae71985327487106b Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Fri, 9 Nov 2018 09:57:36 -0800 Subject: [PATCH 04/11] Revert "remove outbound check in CSE" This reverts commit 2150d6bb2ca9466461074381c963df916fa3d3ab. --- parts/k8s/kubernetesagentresourcesvmas.t | 2 +- parts/k8s/kubernetesagentresourcesvmss.t | 2 +- parts/k8s/kubernetesmasterresources.t | 2 +- parts/k8s/kubernetesmasterresourcesvmss.t | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index f2057506c5..16d1ef4ed1 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -331,7 +331,7 @@ {{if IsOpenShift }} "script": "{{ Base64 (OpenShiftGetNodeSh .) }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index 5102a8417c..daacce8bb3 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -155,7 +155,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" } } } diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index ff3b3e72b6..abeca29f0d 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -957,7 +957,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index c09d780722..853c0a9980 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -459,7 +459,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } } From 413e1e0ad869adbdc7abb85d1cb64a92a3deec4a Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Fri, 9 Nov 2018 10:08:04 -0800 Subject: [PATCH 05/11] add block outbound internet feature flag --- parts/k8s/kubernetesagentresourcesvmas.t | 2 +- parts/k8s/kubernetesagentresourcesvmss.t | 2 +- parts/k8s/kubernetesmasterresources.t | 12 +++-- parts/k8s/kubernetesmasterresourcesvmss.t | 58 ++++++++++++----------- pkg/acsengine/template_generator.go | 6 +++ pkg/api/converterfromapi.go | 1 + pkg/api/convertertoapi.go | 1 + pkg/api/types.go | 1 + pkg/api/vlabs/types.go | 1 + 9 files changed, 49 insertions(+), 35 deletions(-) diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index 16d1ef4ed1..41722082ed 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -331,7 +331,7 @@ {{if IsOpenShift }} "script": "{{ Base64 (OpenShiftGetNodeSh .) }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index daacce8bb3..3ab17f10cd 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -155,7 +155,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if CSERunInBackground }} &{{end}}\"')]" } } } diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index abeca29f0d..fdd82f9b93 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -85,7 +85,7 @@ "name": "[variables('nsgName')]", "properties": { "securityRules": [ -{{if .HasWindows}} + {{if .HasWindows}} { "name": "allow_rdp", "properties": { @@ -100,7 +100,7 @@ "sourcePortRange": "*" } }, -{{end}} + {{end}} { "name": "allow_ssh", "properties": { @@ -128,8 +128,9 @@ "sourceAddressPrefix": "*", "sourcePortRange": "*" } - }, - { + } + {{if BlockOutboundInternet}} + ,{ "name": "allow_vnet", "properties": { "access": "Allow", @@ -157,6 +158,7 @@ "sourcePortRange": "*" } } + {{end}} ] }, "type": "Microsoft.Network/networkSecurityGroups" @@ -957,7 +959,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index 853c0a9980..70d3d69f58 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -99,35 +99,37 @@ "sourceAddressPrefix": "*", "sourcePortRange": "*" } + } + {{if BlockOutboundInternet}} + ,{ + "name": "allow_vnet", + "properties": { + "access": "Allow", + "description": "Allow outbound internet to vnet", + "destinationAddressPrefix": "[parameters('masterSubnet')]", + "destinationPortRange": "*", + "direction": "Outbound", + "priority": 110, + "protocol": "*", + "sourceAddressPrefix": "VirtualNetwork", + "sourcePortRange": "*" + } }, { - "name": "allow_vnet", - "properties": { - "access": "Allow", - "description": "Allow outbound internet to vnet", - "destinationAddressPrefix": "[parameters('masterSubnet')]", - "destinationPortRange": "*", - "direction": "Outbound", - "priority": 110, - "protocol": "*", - "sourceAddressPrefix": "VirtualNetwork", - "sourcePortRange": "*" - } - }, - { - "name": "block_outbound", - "properties": { - "access": "Deny", - "description": "Block outbound internet from master", - "destinationAddressPrefix": "*", - "destinationPortRange": "*", - "direction": "Outbound", - "priority": 120, - "protocol": "*", - "sourceAddressPrefix": "*", - "sourcePortRange": "*" - } - } + "name": "block_outbound", + "properties": { + "access": "Deny", + "description": "Block outbound internet from master", + "destinationAddressPrefix": "*", + "destinationPortRange": "*", + "direction": "Outbound", + "priority": 120, + "protocol": "*", + "sourceAddressPrefix": "*", + "sourcePortRange": "*" + } + } + {{end}} ] }, "type": "Microsoft.Network/networkSecurityGroups" @@ -459,7 +461,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } } diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index 30e44ec89c..5dff9ebedb 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -428,6 +428,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } return false }, + "BlockOutboundInternet": func() bool { + if cs.Properties.FeatureFlags != nil { + return cs.Properties.FeatureFlags.BlockOutboundInternet + } + return false + }, "GetDCOSBootstrapCustomData": func() string { masterIPList := generateIPList(cs.Properties.MasterProfile.Count, cs.Properties.MasterProfile.FirstConsecutiveStaticIP) for i, v := range masterIPList { diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index c7610edb76..02ddb1ea6d 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -1188,4 +1188,5 @@ func convertAzProfileToVLabs(api *AzProfile, vlabs *vlabs.AzProfile) { func convertFeatureFlagsToVLabs(api *FeatureFlags, vlabs *vlabs.FeatureFlags) { vlabs.EnableCSERunInBackground = api.EnableCSERunInBackground + vlabs.BlockOutboundInternet = api.BlockOutboundInternet } diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index ce38c401c6..55300bc030 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -415,6 +415,7 @@ func convertVLabsAZProfile(vlabs *vlabs.AzProfile, api *AzProfile) { func convertVLabsFeatureFlags(vlabs *vlabs.FeatureFlags, api *FeatureFlags) { api.EnableCSERunInBackground = vlabs.EnableCSERunInBackground + api.BlockOutboundInternet = vlabs.BlockOutboundInternet } func convertV20160930LinuxProfile(obj *v20160930.LinuxProfile, api *LinuxProfile) { diff --git a/pkg/api/types.go b/pkg/api/types.go index 8af5a7086d..bc893141ae 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -101,6 +101,7 @@ type AzProfile struct { // FeatureFlags defines feature-flag restricted functionality type FeatureFlags struct { EnableCSERunInBackground bool `json:"enableCSERunInBackground,omitempty"` + BlockOutboundInternet bool `json:"blockOutboundInternet,omitempty"` } // ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 8b7a8350f7..2466bdb9d3 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -57,6 +57,7 @@ type AzProfile struct { // FeatureFlags defines feature-flag restricted functionality type FeatureFlags struct { EnableCSERunInBackground bool `json:"enableCSERunInBackground,omitempty"` + BlockOutboundInternet bool `json:"blockOutboundInternet,omitempty"` } // ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD From 63d728cbc3f9ca29b2a881919fbb0f6c2c9961cc Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 13 Nov 2018 10:13:02 -0800 Subject: [PATCH 06/11] revert azure cni change --- pkg/api/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/const.go b/pkg/api/const.go index 83ccdf8f30..0b3a296b0b 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -240,7 +240,7 @@ const ( // AzureCniPluginVerLinux specifies version of Azure CNI plugin, which has been mirrored from // https://github.com/Azure/azure-container-networking/releases/download/${AZURE_PLUGIN_VER}/azure-vnet-cni-linux-amd64-${AZURE_PLUGIN_VER}.tgz // to https://acs-mirror.azureedge.net/cni - AzureCniPluginVerLinux = "v1.0.12" + AzureCniPluginVerLinux = "v1.0.13" // AzureCniPluginVerWindows specifies version of Azure CNI plugin, which has been mirrored from // https://github.com/Azure/azure-container-networking/releases/download/${AZURE_PLUGIN_VER}/azure-vnet-cni-windows-amd64-${AZURE_PLUGIN_VER}.tgz // to https://acs-mirror.azureedge.net/cni From ae080f348cca6855a49e421d928fe1af87c0e78f Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 13 Nov 2018 12:34:41 -0800 Subject: [PATCH 07/11] don't wait for apiserver when no outbound --- examples/kubernetes.json | 5 ++++- parts/k8s/kubernetesconfigs.sh | 3 +-- parts/k8s/kubernetescustomscript.sh | 1 + parts/k8s/kubernetesmasterresources.t | 2 +- parts/k8s/kubernetesmasterresourcesvmss.t | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/kubernetes.json b/examples/kubernetes.json index 2923d007eb..9b802599a0 100644 --- a/examples/kubernetes.json +++ b/examples/kubernetes.json @@ -29,6 +29,9 @@ "servicePrincipalProfile": { "clientId": "", "secret": "" - } + }, + "FeatureFlags": { + "blockOutboundInternet": true + } } } \ No newline at end of file diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh index 940d21e4ba..46615b0856 100755 --- a/parts/k8s/kubernetesconfigs.sh +++ b/parts/k8s/kubernetesconfigs.sh @@ -292,7 +292,7 @@ ensurePodSecurityPolicy() { } ensureK8sControlPlane() { - if $REBOOTREQUIRED; then + if $REBOOTREQUIRED || [ "$NO_OUTBOUND" = "true" ]; then return fi wait_for_file 3600 1 $KUBECTL || exit $ERR_FILE_WATCH_TIMEOUT @@ -303,7 +303,6 @@ ensureK8sControlPlane() { else retrycmd_if_failure 120 5 25 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT fi - ensurePodSecurityPolicy } ensureEtcd() { diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index d3deca784a..efa97c100a 100755 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -119,6 +119,7 @@ if [[ ! -z "${MASTER_NODE}" ]]; then writeKubeConfig ensureEtcd ensureK8sControlPlane + ensurePodSecurityPolicy fi if [[ "${GPU_NODE}" = true ]]; then diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index fdd82f9b93..8041019048 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -959,7 +959,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index 70d3d69f58..f9ccfd4b5b 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -461,7 +461,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } } From d9666d827bf1d190396358db42560e20b782fefc Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 13 Nov 2018 14:08:40 -0800 Subject: [PATCH 08/11] revert example change --- examples/kubernetes.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/kubernetes.json b/examples/kubernetes.json index 9b802599a0..fc8043c8a9 100644 --- a/examples/kubernetes.json +++ b/examples/kubernetes.json @@ -29,9 +29,6 @@ "servicePrincipalProfile": { "clientId": "", "secret": "" - }, - "FeatureFlags": { - "blockOutboundInternet": true - } + } } } \ No newline at end of file From 3dac0a495647a696e933437c7885da68e87bf5ee Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 13 Nov 2018 14:09:08 -0800 Subject: [PATCH 09/11] space --- examples/kubernetes.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kubernetes.json b/examples/kubernetes.json index fc8043c8a9..2923d007eb 100644 --- a/examples/kubernetes.json +++ b/examples/kubernetes.json @@ -29,6 +29,6 @@ "servicePrincipalProfile": { "clientId": "", "secret": "" - } + } } } \ No newline at end of file From 70a1349d75811b0517052eb94d48523a1e20c9ca Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 13 Nov 2018 14:14:18 -0800 Subject: [PATCH 10/11] ; --- parts/k8s/kubernetesmasterresources.t | 2 +- parts/k8s/kubernetesmasterresourcesvmss.t | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index 8041019048..80b8c4a1c1 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -959,7 +959,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true;{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index f9ccfd4b5b..45fb6fdfe7 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -461,7 +461,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true;{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } } From 7bdcdcae47fad40a295171767b94d8712bbb83ed Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 13 Nov 2018 14:49:27 -0800 Subject: [PATCH 11/11] NO_OUTBOUND needs to be set in env vars --- parts/k8s/kubernetesmasterresources.t | 2 +- parts/k8s/kubernetesmasterresourcesvmss.t | 2 +- parts/k8s/kubernetesmastervars.t | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index 80b8c4a1c1..fdd82f9b93 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -959,7 +959,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true;{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index 45fb6fdfe7..70d3d69f58 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -461,7 +461,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if BlockOutboundInternet}} NO_OUTBOUND=true;{{else}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not BlockOutboundInternet}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 150 1 3 nc -vz {{if IsMooncake}}k8s.gcr.io 443 && nc -vz gcr.io 443 && nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } } diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index fa5e459778..691668990a 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -144,9 +144,9 @@ "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER={{IsHostedMaster}}')]", {{if not IsHostedMaster}} {{if IsMasterVirtualMachineScaleSets}} - "provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", + "provisionScriptParametersMaster": "[concat('MASTER_NODE=true NO_OUTBOUND={{BlockOutboundInternet}} CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{else}} - "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", + "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND={{BlockOutboundInternet}} CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} {{end}} {{end}}