Enable AKS to use Outbound type of userDefinedRouting

charts/aks-operator-crd/templates/crds.yaml

@@ -143,6 +143,9 @@ spec:
                   type: object
                 nullable: true
                 type: array
+              outboundType:
+                nullable: true
+                type: string
               podCidr:
                 nullable: true
                 type: string

controller/aks-cluster-config-handler.go

@@ -639,6 +639,7 @@ func (h *Handler) buildUpstreamClusterState(ctx context.Context, credentials *ak
 		upstreamSpec.NetworkServiceCIDR = networkProfile.ServiceCidr
 		upstreamSpec.NetworkPolicy = to.StringPtr(string(networkProfile.NetworkPolicy))
 		upstreamSpec.NetworkPodCIDR = networkProfile.PodCidr
+		upstreamSpec.OutboundType = to.StringPtr(string(networkProfile.OutboundType))
 		upstreamSpec.LoadBalancerSKU = to.StringPtr(string(networkProfile.LoadBalancerSku))
 	}
 

examples/create-example-udr.yaml

@@ -0,0 +1,40 @@
+apiVersion: aks.cattle.io/v1
+kind: AKSClusterConfig
+metadata:
+  name: my-cluster
+spec:
+  resourceLocation: "germanywestcentral"
+  resourceGroup: "my-group"
+  clusterName: "my-cluster"
+  baseUrl: "https://management.azure.com/"
+  authBaseUrl: "https://login.microsoftonline.com"
+  azureCredentialSecret: "REPLACE_WITH_K8S_SECRETS_NAME"
+  dnsPrefix: "example-dns"
+  privateCluster: false
+  linuxAdminUsername: "rancher-user"
+  loadBalancerSku: ""
+  sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY"
+  kubernetesVersion: "1.19.9"
+  nodePools:
+  - name: "masters"
+    count: 1
+    vmSize: "Standard_DS2_v2"
+    osDiskSizeGB: 128
+    osDiskType: "Managed"
+    maxPods: 110
+    mode: "System"
+    osType: "Linux"
+  - name: "workers"
+    orchestratorVersion: "1.19.9"
+    count: 6
+    vmSize: "Standard_DS2_v2"
+    osDiskSizeGB: 128
+    osDiskType: "Managed"
+    maxPods: 110
+    mode: "User"
+    osType: "Linux"
+    enableAutoScaling: true
+    minCount: 1
+    maxCount: 6
+    availabilityZones: [ "1", "2", "3" ]
+  outboundType: "userDefinedRouting"

examples/create-example.yaml

@@ -10,9 +10,10 @@ spec:
   authBaseUrl: "https://login.microsoftonline.com"
   azureCredentialSecret: "REPLACE_WITH_K8S_SECRETS_NAME"
   dnsPrefix: "example-dns"
-  privateCluster: false,
-  linuxAdminUsername: "rancher-user",
-  sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY",
+  privateCluster: false
+  linuxAdminUsername: "rancher-user"
+  loadBalancerSku: "standard"
+  sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY"
   kubernetesVersion: "1.19.9"
   nodePools:
   - name: "masters"
@@ -36,3 +37,4 @@ spec:
     minCount: 1
     maxCount: 6
     availabilityZones: [ "1", "2", "3" ]
+  outboundType: "loadBalancer"

pkg/aks/create.go

@@ -64,6 +64,15 @@ func createManagedCluster(ctx context.Context, cred *Credentials, workplacesClie
 
 	networkProfile := &containerservice.NetworkProfile{}
 
+	switch to.String(spec.OutboundType) {
+	case string(containerservice.LoadBalancer):
+		networkProfile.OutboundType = containerservice.LoadBalancer
+	case string(containerservice.UserDefinedRouting):
+		networkProfile.OutboundType = containerservice.UserDefinedRouting
+	case "":
+		networkProfile.OutboundType = containerservice.LoadBalancer
+	}
+
 	switch to.String(spec.NetworkPolicy) {
 	case string(containerservice.NetworkPolicyAzure):
 		networkProfile.NetworkPolicy = containerservice.NetworkPolicyAzure
@@ -90,14 +99,16 @@ func createManagedCluster(ctx context.Context, cred *Credentials, workplacesClie
 		return nil, fmt.Errorf("network plugin Kubenet is not compatible with network policy Azure")
 	}
 
-	switch to.String(spec.LoadBalancerSKU) { // TODO: only standard is supported for now, find a way to validate this
-	case string(containerservice.Standard):
-		networkProfile.LoadBalancerSku = containerservice.Standard
-	case string(containerservice.Basic):
+	networkProfile.LoadBalancerSku = containerservice.Standard
+
+	if to.String(spec.LoadBalancerSKU) == string(containerservice.Basic) {
 		logrus.Warnf("loadBalancerSKU 'basic' is not supported")
 		networkProfile.LoadBalancerSku = containerservice.Basic
-	case "":
-		networkProfile.LoadBalancerSku = containerservice.Standard
+	}
+
+	// Disable standard loadbalancer for UserDefinedRouting and use routing created by user pre-defined table for egress
+	if to.String(spec.OutboundType) == string(containerservice.UserDefinedRouting) {
+		networkProfile.LoadBalancerSku = ""
 	}
 
 	virtualNetworkResourceGroup := spec.ResourceGroup

pkg/aks/create_test.go

@@ -88,17 +88,19 @@ var _ = Describe("newManagedCluster", func() {
 				ID: to.StringPtr("test-workspace-id"),
 			}, nil)
 
+		clusterSpec.LoadBalancerSKU = to.StringPtr("standard")
 		managedCluster, err := createManagedCluster(ctx, cred, workplacesClientMock, clusterSpec, "test-phase")
 		Expect(err).ToNot(HaveOccurred())
 
 		Expect(managedCluster.Tags).To(HaveKeyWithValue("test-tag", to.StringPtr("test-value")))
 		Expect(managedCluster.NetworkProfile.NetworkPolicy).To(Equal(containerservice.NetworkPolicy(to.String(clusterSpec.NetworkPolicy))))
-		Expect(managedCluster.NetworkProfile.LoadBalancerSku).To(Equal(containerservice.Standard))
+		Expect(managedCluster.NetworkProfile.LoadBalancerSku).To(Equal(containerservice.LoadBalancerSku(to.String(clusterSpec.LoadBalancerSKU))))
 		Expect(managedCluster.NetworkProfile.NetworkPlugin).To(Equal(containerservice.NetworkPlugin(to.String(clusterSpec.NetworkPlugin))))
 		Expect(managedCluster.NetworkProfile.DNSServiceIP).To(Equal(clusterSpec.NetworkDNSServiceIP))
 		Expect(managedCluster.NetworkProfile.DockerBridgeCidr).To(Equal(clusterSpec.NetworkDockerBridgeCIDR))
 		Expect(managedCluster.NetworkProfile.ServiceCidr).To(Equal(clusterSpec.NetworkServiceCIDR))
 		Expect(managedCluster.NetworkProfile.PodCidr).To(Equal(clusterSpec.NetworkPodCIDR))
+		Expect(managedCluster.NetworkProfile.OutboundType).To(Equal(containerservice.LoadBalancer))
 		agentPoolProfiles := *managedCluster.AgentPoolProfiles
 		Expect(agentPoolProfiles).To(HaveLen(1))
 		Expect(agentPoolProfiles[0].Name).To(Equal(clusterSpec.NodePools[0].Name))
@@ -158,7 +160,18 @@ var _ = Describe("newManagedCluster", func() {
 		Expect(managedCluster.NetworkProfile.LoadBalancerSku).To(Equal(containerservice.Basic))
 	})
 
-	It("should successfully create managed cluster with custom network plugin no network profile", func() {
+	It("should successfully create managed cluster with outboundtype userdefinedrouting", func() {
+		workplacesClientMock.EXPECT().Get(ctx, to.String(clusterSpec.LogAnalyticsWorkspaceGroup), to.String(clusterSpec.LogAnalyticsWorkspaceName)).
+			Return(operationalinsights.Workspace{
+				ID: to.StringPtr("test-workspace-id"),
+			}, nil)
+		clusterSpec.OutboundType = to.StringPtr("userDefinedRouting")
+		managedCluster, err := createManagedCluster(ctx, cred, workplacesClientMock, clusterSpec, "test-phase")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(managedCluster.NetworkProfile.OutboundType).To(Equal(containerservice.UserDefinedRouting))
+	})
+
+	It("should successfully create managed cluster with custom network plugin without network profile", func() {
 		workplacesClientMock.EXPECT().Get(ctx, to.String(clusterSpec.LogAnalyticsWorkspaceGroup), to.String(clusterSpec.LogAnalyticsWorkspaceName)).
 			Return(operationalinsights.Workspace{
 				ID: to.StringPtr("test-workspace-id"),
@@ -171,12 +184,12 @@ var _ = Describe("newManagedCluster", func() {
 		clusterSpec.NetworkPodCIDR = to.StringPtr("")
 
 		managedCluster, err := createManagedCluster(ctx, cred, workplacesClientMock, clusterSpec, "test-phase")
+		Expect(err).ToNot(HaveOccurred())
 		Expect(managedCluster.NetworkProfile.NetworkPlugin).To(Equal(containerservice.Kubenet))
 		Expect(managedCluster.NetworkProfile.DNSServiceIP).To(Equal(clusterSpec.NetworkDNSServiceIP))
 		Expect(managedCluster.NetworkProfile.DockerBridgeCidr).To(Equal(clusterSpec.NetworkDockerBridgeCIDR))
 		Expect(managedCluster.NetworkProfile.ServiceCidr).To(Equal(clusterSpec.NetworkServiceCIDR))
 		Expect(managedCluster.NetworkProfile.PodCidr).To(Equal(clusterSpec.NetworkPodCIDR))
-		Expect(err).ToNot(HaveOccurred())
 	})
 
 	It("should successfully create managed cluster with custom network plugin", func() {

pkg/apis/aks.cattle.io/v1/types.go

@@ -48,6 +48,7 @@ type AKSClusterConfigSpec struct {
 	NetworkServiceCIDR          *string           `json:"serviceCidr" norman:"pointer"`
 	NetworkDockerBridgeCIDR     *string           `json:"dockerBridgeCidr" norman:"pointer"`
 	NetworkPodCIDR              *string           `json:"podCidr" norman:"pointer"`
+	OutboundType                *string           `json:"outboundType" norman:"pointer"`
 	LoadBalancerSKU             *string           `json:"loadBalancerSku" norman:"pointer"`
 	NetworkPolicy               *string           `json:"networkPolicy" norman:"pointer"`
 	LinuxAdminUsername          *string           `json:"linuxAdminUsername,omitempty" norman:"pointer"`