Documented Azure DNS deployment (#1525)

* Updated Helm chart, docs and examples for connecting to Azure Public DNS service. Signed-off-by: Nuno Guedes <nunoguedes@hotmail.com> * Bug fixing the Windows DNS example Signed-off-by: Nuno Guedes <nunoguedes@hotmail.com> --------- Signed-off-by: Nuno Guedes <nunoguedes@hotmail.com> Signed-off-by: Yury Tsarev <yury@upbound.io> Co-authored-by: Yury Tsarev <yury@upbound.io>
k8gb-io · Jun 6, 2024 · ffaac02 · ffaac02
1 parent d660caa
commit ffaac02
Show file tree

Hide file tree

Showing 43 changed files with 938 additions and 419 deletions.
diff --git a/chart/k8gb/templates/_helpers.tpl b/chart/k8gb/templates/_helpers.tpl
@@ -72,6 +72,9 @@ Create the name of the service account to use
 {{- if .Values.rfc2136.enabled }}
 {{- print "rfc2136" -}}
 {{- end -}}
+{{- if .Values.azuredns.enabled }}
+{{- print "azure-dns" -}}
+{{- end -}}
 {{- if .Values.cloudflare.enabled }}
 {{- print "cloudflare" -}}
 {{- end -}}
@@ -108,6 +111,9 @@ k8gb-{{ .Values.k8gb.dnsZone }}-{{ .Values.k8gb.clusterGeoTag }}
               name: ns1
               key: apiKey
 {{- end }}
+{{- if .Values.azuredns.enabled -}}
+        - --azure-resource-group={{ .Values.azuredns.resourceGroup }}
+{{- end }}
 {{- if and (eq .Values.rfc2136.enabled true) (eq .Values.rfc2136.rfc2136auth.insecure.enabled true) -}}
         - --rfc2136-insecure
 {{- end -}}
@@ -156,3 +162,24 @@ k8gb-{{ .Values.k8gb.dnsZone }}-{{ .Values.k8gb.clusterGeoTag }}
 {{- define "k8gb.metrics_port" -}}
 {{ print (split ":" .Values.k8gb.metricsAddress)._1 }}
 {{- end -}}
+
+{{- define "external-dns.azure-credentials" -}}
+{{- if .Values.azuredns.enabled -}}
+{
+  "tenantId": "{{ .Values.azuredns.tenantId }}",
+  "subscriptionId": "{{ .Values.azuredns.subscriptionId }}",
+  "resourceGroup": "{{ .Values.azuredns.resourceGroup }}",
+  {{- if .Values.azuredns.aadClientId -}}
+  "aadClientId": "{{ .Values.azuredns.aadClientId }}",
+  {{- end -}}
+  {{- if .Values.azuredns.aadClientSecret -}}
+  "aadClientSecret": "{{ .Values.azuredns.aadClientSecret }}",
+  {{- end -}}
+  "useManagedIdentityExtension": {{ .Values.azuredns.useManagedIdentityExtension | default false }},
+  {{- if .Values.azuredns.userAssignedIdentityID -}}
+  "userAssignedIdentityID": "{{ .Values.azuredns.userAssignedIdentityID }}",
+  {{- end -}}
+  "useWorkloadIdentityExtension": {{ .Values.azuredns.useWorkloadIdentityExtension | default false }}
+}
+{{- end -}}
+{{- end -}}
diff --git a/chart/k8gb/templates/deployment.yaml b/chart/k8gb/templates/deployment.yaml
@@ -98,7 +98,7 @@ spec:
                   name: infoblox
                   key: INFOBLOX_WAPI_PASSWORD
             {{- end }}
-            {{- if or .Values.route53.enabled .Values.ns1.enabled .Values.rfc2136.enabled .Values.cloudflare.enabled }}
+            {{- if or .Values.route53.enabled .Values.ns1.enabled .Values.rfc2136.enabled .Values.azuredns.enabled .Values.cloudflare.enabled }}
             - name: EXTDNS_ENABLED
               value: "true"
             {{- end }}

diff --git a/chart/k8gb/templates/external-dns/external-dns-azure-auth.yaml b/chart/k8gb/templates/external-dns/external-dns-azure-auth.yaml
@@ -0,0 +1,9 @@
+{{- if and .Values.azuredns.enabled .Values.azuredns.createAuthSecret }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .Values.azuredns.authSecretName | default "external-dns-secret-azure" }}
+data:
+  azure.json: {{ include "external-dns.azure-credentials" . | b64enc }}
+{{- end }}
diff --git a/chart/k8gb/templates/external-dns/external-dns.yaml b/chart/k8gb/templates/external-dns/external-dns.yaml
@@ -1,4 +1,4 @@
-{{- if or .Values.ns1.enabled .Values.route53.enabled .Values.rfc2136.enabled .Values.cloudflare.enabled }}
+{{- if or .Values.ns1.enabled .Values.route53.enabled .Values.rfc2136.enabled .Values.azuredns.enabled .Values.cloudflare.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -88,5 +88,14 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
       {{- end }}
-
+      {{- if .Values.azuredns.enabled }}
+        volumeMounts:
+          - name: azure-config-file
+            mountPath: /etc/kubernetes/
+            readOnly: true
+      volumes:
+        - name: azure-config-file
+          secret:
+            secretName: {{ .Values.azuredns.authSecretName | default "external-dns-secret-azure" }}
+      {{- end }}
 {{- end }}
diff --git a/chart/k8gb/templates/external-dns/rbac.yaml b/chart/k8gb/templates/external-dns/rbac.yaml
@@ -1,4 +1,4 @@
-{{- if or .Values.ns1.enabled .Values.route53.enabled .Values.rfc2136.enabled .Values.cloudflare.enabled }}
+{{- if or .Values.ns1.enabled .Values.route53.enabled .Values.rfc2136.enabled .Values.azuredns.enabled .Values.cloudflare.enabled }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/chart/k8gb/values.schema.json b/chart/k8gb/values.schema.json
@@ -30,6 +30,9 @@
                 "rfc2136": {
                     "$ref": "#/definitions/Rfc2136"
                 },
+                "azuredns": {
+                    "$ref": "#/definitions/AzureDNS"
+                },                
                 "cloudflare": {
                     "$ref": "#/definitions/Cloudflare"
                 },
@@ -612,6 +615,58 @@
             ],
             "title": "Route53"
         },
+        "AzureDNS": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "createAuthSecret": {
+                    "type": "boolean",
+                    "default": true
+                },
+                "authSecretName": {
+                    "type": "string",
+                    "default": "external-dns-secret-azure"
+                },
+                "tenantId": {
+                    "type": "string",
+                    "minLength": 1
+                },
+                "subscriptionId": {
+                    "type": "string",
+                    "minLength": 1
+                },
+                "resourceGroup": {
+                    "type": "string",
+                    "minLength": 1
+                },
+                "aadClientId": {
+                    "type": "string"
+                },
+                "aadClientSecret": {
+                    "type": "string"
+                },
+                "useManagedIdentityExtension": {
+                    "type": "boolean",
+                    "default": false
+                },
+                "userAssignedIdentityID": {
+                    "type": "string"
+                },
+                "useWorkloadIdentityExtension": {
+                    "type": "boolean",
+                    "default": false
+                }
+            },
+            "required": [
+                "tenantId",
+                "subscriptionId",
+                "resourceGroup"
+            ],
+            "title": "azuredns"
+        },        
         "Cloudflare": {
             "type": "object",
             "additionalProperties": false,

diff --git a/chart/k8gb/values.yaml b/chart/k8gb/values.yaml
@@ -151,6 +151,18 @@ rfc2136:
         - kerberos-password: ad-user-pass
         - kerberos-realm: ad-domain-realm
 
+azuredns:
+  enabled: false
+  # -- Create an authentication secret for Azure DNS based on the values below
+  # alternatively, you can create the secret manually and pass its name in the `authSecretName` value
+  createAuthSecret: true
+  # -- Azure tenant ID which holds the managed identity
+  tenantId: myTenantId
+  # -- subscription id which holds the Azure DNS zone
+  subscriptionId: mySubscriptionId
+  # -- Azure Resource Group which holds the Azure DNS Zone (which is defined as 'edgeDNSZone')
+  resourceGroup: myDnsResourceGroup
+
 cloudflare:
   # -- Enable Cloudflare provider
   enabled: false

diff --git a/docs/deploy_azuredns.md b/docs/deploy_azuredns.md
@@ -0,0 +1,71 @@
+<h1 align="center" style="margin-top: 0;">Using Azure Public DNS provider</h1>
+
+This document outlines how to configure k8gb to use the Azure Public DNS provider. Azure Private DNS is not supported as it does not support NS records at this time. For private DNS scenarios in Azure, please refer to the [Windows DNS](deploy_azuredns.md) documentation and consider implementing it using VM-based DNS services such as Windows DNS or BIND.
+
+### external-dns credentials for Azure DNS
+
+In this example, we will use a registered app in Microsoft Entra ID and it's corresponding Client ID / Client Secret to authenticate with the Azure DNS zone. All of the [supported authentication fields supported by external-dns](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/azure.md#configuration-file) are supported by k8gb and can be used in the `azuredns` section of the `k8gb` Helm chart values.yaml file.
+
+## Sample solution
+
+In this sample solution we will deploy two private AKS clusters in different regions. A workload will be deployed to both clusters and exposed to the internet with the help of k8gb and Azure Public DNS.
+
+## Reference Setup
+
+The reference setup includes two private AKS clusters that can be deployed on two different regions for load balancing or to provide a failover solution.
+
+Configurable resources:
+
+* Resource groups
+* VNet and subnets
+* Managed Identity
+* Clusters
+
+## Run the sample
+
+* To run the provided sample, please use the provided Makefile [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azuredns/).
+    * Deploys all the required infrastructure and configurations
+    * Before executing, please fill all the local variables in the scripts with the correct naming for the resources in order to avoid having problems with your Azure policies
+    * Scripts will use Az CLI, please ensure that it is installed and logged when trying to execute the command
+        * [Microsoft Learn](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli "Install Az CLI")
+
+### Deploy infrastructure
+
+This action will create resource groups, vnets and private AKS clusters to run all required workloads
+
+```sh
+make deploy-infra
+```
+
+### Setup clusters
+
+Install required Ingress controller in both clusters in order to deploy K8GB and demo application
+
+```sh
+make setup-clusters
+```
+
+### Install K8gb
+
+This action will install K8gb in both clusters using the provided [sample](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azuredns/k8gb/) values.yaml for each cluster. Please ensure that the are correctly updated before execution
+
+```sh
+make deploy-k8gb
+```
+
+### Install demo app
+
+Deploys the sample Podinfo workload with failover GLSB configured using annotations in the Ingress resource [samples](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azuredns/demo/).
+Ensure that the hosts on the samples are correctly updated before execution
+
+```sh
+make deploy-demo
+```
+
+### Destroy lab
+
+* Destroys the lab environment created for this sample
+
+```sh
+make destroy-infra
+```
diff --git a/docs/deploy_windowsdns.md b/docs/deploy_windowsdns.md
@@ -1,20 +1,18 @@
-<h1 align="center" style="margin-top: 0;">Using K8GB on Azure</h1>
+<h1 align="center" style="margin-top: 0;">Using K8GB with a GSS-TSIG compatible DNS provider</h1>
 
-## Sample solution
+## Sample solution: Azure based private deployment with Windows DNS integration
 
-Sample solution will create a common hub and spoke architecture with two AKS clusters in different regions
-
-![GLSB with K8gb on Azure](/docs/examples/azure/images/k8gb_solution.png?raw=true "GLSB with K8gb on Azure")
-
-# Azure based deployment with Windows DNS integration
+In this sample solution we will create a common hub and spoke architecture with two private AKS clusters in different regions. The same pattern can be used with any other Kubernetes distribution and any other DNS provider that supports GSS-TSIG.
 
 Here we provide an example of k8gb deployment in Azure environment with Windows DNS as edgeDNS provider.
 
 ## Reference Setup
 
-The reference setup includes two AKS clusters that can deployed on two different regions for load balancing or provide a failover solution.
+The reference setup includes two private AKS clusters that can be deployed on two different regions for load balancing or to provide a failover solution.
 
-The solution design can be found [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azure/).
+![GLSB with K8gb on Windows DNS](/docs/examples/windowsdns/images/k8gb_solution.png?raw=true "GLSB with K8gb on Windows DNS")
+
+The solution design can be found [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/windowsdns/).
 
 Configurable resources:
 
@@ -30,15 +28,15 @@ Configurable resources:
     * There are several tutorials available online, but this Microsoft Learn article will probably help you out 
     * [Microsoft Learn](https://learn.microsoft.com/en-us/windows-server/identity/ad-ds/deploy/install-active-directory-domain-services--level-100- "Install Active Directory")
 
-* To run the provided sample, please use the provided Makefile [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azure/).
+* To run the provided sample, please use the provided Makefile [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/windowsdns/).
     * Deploys all the required infrastructure and configurations
-    * Before execute, please fill all the local variables in the scripts with the correct naming for the resources in order to avoid having problems with your azure policies
-    * Scripts will use AZ CLI, ensure that is installed and logged when trying to execute the command
-        * [Microsoft Learn](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli "Install AZ CLI")
+    * Before executing, please fill all the local variables in the scripts with the correct naming for the resources in order to avoid having problems with your Azure policies
+    * Scripts will use Az CLI, please ensure that it is installed and logged when trying to execute the command
+        * [Microsoft Learn](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli "Install Az CLI")
 
-### Deploy infrastructure 
+### Deploy infrastructure
 
-This action will create resource groups, vnets, peering between vnets and AKS clusters to run all required workloads
+This action will create resource groups, vnets, peering between vnets and private AKS clusters to run all required workloads
 
 ```sh
 make deploy-infra
@@ -54,19 +52,21 @@ make setup-clusters
 
 ### Configure GSS-TSIG authentication for DNS updates
 
-Before deploying K8GB and the demo workload, ensure required configurations on Windows DNS 
+Before deploying K8GB and the demo workload, ensure required configurations on Windows DNS
 
 #### Domain Controller config
 
 * Ensure that the Network Security is configured only for AES256
 
-![Network Policy - Kerberos auth](/docs/examples/azure/images/LocalSecuryPolicyNetworkKerberos.png?raw=true "Network Policy - Kerberos auth")
+![Network Policy - Kerberos auth](/docs/examples/windowsdns/images/LocalSecuryPolicyNetworkKerberos.png?raw=true "Network Policy - Kerberos auth")
+
 * Ensure that the DNS Zone has only Secure updates option enabled
 
-![DNS Secure Updates](/docs/examples/azure/images/DNSSecureUpdates.png "DNS Secure Updates")
+![DNS Secure Updates](/docs/examples/windowsdns/images/DNSSecureUpdates.png "DNS Secure Updates")
+
 * Ensure that the DNS Zone has the option "Allow zone transfers" check with the option "To any server" under the tab Zone Transfers on the zone properties
 
-![DNS Zone Transfers](/docs/examples/azure/images/DNSZoneTransfers.png "DNS Zone Transfers")
+![DNS Zone Transfers](/docs/examples/windowsdns/images/DNSZoneTransfers.png "DNS Zone Transfers")
 
 * Create a new Active Directory user
     * The user should be created with "Encryptions options" for Kerberos AES256 encryption
@@ -78,7 +78,7 @@ Before deploying K8GB and the demo workload, ensure required configurations on W
 * ExternalDNS configuration
     * For communication with WindowsDNS, ExternalDNS should be configured with the RFC2136 provider with GSS-TSIG option
     * [External DNS - RFC2126](https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/rfc2136.md "RFC2136 documentation")
-    * A sample values.yaml for K8GB configuration can be found [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azure/k8gb/).
+    * A sample values.yaml for K8GB configuration can be found [here](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/windowsdns/k8gb/).
         * Ensure that the following properties are updated with your values:
             * dnsZone
             * edgeDNSZone
@@ -87,12 +87,13 @@ Before deploying K8GB and the demo workload, ensure required configurations on W
             * kerberos-username
             * kerberos-password
             * kerberos-realm
-    * At this moment ExternalDNS doesn't provide a way to use secrets as the source for the kerberos-password setting, so if you store the manifest in a git repo, please ensure that only required persons can access it
+    * At this moment ExternalDNS doesn't provide a way to use secrets as the source for the kerberos-password setting, so you must ensure this is stored in a secure way
+
 ```yaml
 rfc2136:
   enabled: true
   rfc2136Opts:
-    - host: AD-DC.mbcpk8gb.local #when using gssTsig, use the FQDN of the host, not an IP
+    - host: AD-DC.k8gb.local #when using gssTsig, use the FQDN of the host, not an IP
     - port: 53
   rfc2136auth:
     insecure: 
@@ -107,28 +108,30 @@ rfc2136:
       gssTsigCreds:
         - kerberos-username: ad-user-account
         - kerberos-password: ad-user-account-password
-        - kerberos-realm: mbcpcloud.lab
+        - kerberos-realm: cloud.lab
 ```
 
 ### Install K8gb
 
-This action will install K8gb in both clusters using the provided [sample](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azure/k8gb/) values.yaml for each cluster. Please ensure that the are correctly updated before execution
+This action will install K8gb in both clusters using the provided [sample](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/windowsdns/k8gb/) values.yaml for each cluster. Please ensure that the are correctly updated before execution
 
 ```sh
 make deploy-k8gb
 ```
 
 ### Install demo app
 
-Deploys the sample Podinfo workload with failover GLSB configured using annotations in the Ingress resource [samples](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/azure/demo/).
+Deploys the sample Podinfo workload with failover GLSB configured using annotations in the Ingress resource [samples](https://github.com/k8gb-io/k8gb/tree/master/docs/examples/windowsdns/demo/).
 Ensure that the hosts on the samples are correctly updated before execution
 
 ```sh
 make deploy-demo
 ```
 
 ### Destroy lab
+
 * Destroys the lab environment created for this sample
+
 ```sh
 make destroy-infra
 ```
diff --git a/docs/examples/azure/demo/aks1-podinfo-values.yaml b/docs/examples/azure/demo/aks1-podinfo-values.yaml
@@ -0,0 +1,2 @@
+ui:
+  message: aks1
diff --git a/docs/examples/azure/demo/aks2-podinfo-values.yaml b/docs/examples/azure/demo/aks2-podinfo-values.yaml
@@ -0,0 +1,2 @@
+ui:
+  message: aks2