diff --git a/charts/aks-operator-crd/templates/crds.yaml b/charts/aks-operator-crd/templates/crds.yaml index 67acc908..e507288d 100755 --- a/charts/aks-operator-crd/templates/crds.yaml +++ b/charts/aks-operator-crd/templates/crds.yaml @@ -143,6 +143,9 @@ spec: type: object nullable: true type: array + outboundType: + nullable: true + type: string podCidr: nullable: true type: string diff --git a/controller/aks-cluster-config-handler.go b/controller/aks-cluster-config-handler.go index bf2ee8d1..a6f1d4eb 100644 --- a/controller/aks-cluster-config-handler.go +++ b/controller/aks-cluster-config-handler.go @@ -639,6 +639,7 @@ func (h *Handler) buildUpstreamClusterState(ctx context.Context, credentials *ak upstreamSpec.NetworkServiceCIDR = networkProfile.ServiceCidr upstreamSpec.NetworkPolicy = to.StringPtr(string(networkProfile.NetworkPolicy)) upstreamSpec.NetworkPodCIDR = networkProfile.PodCidr + upstreamSpec.OutboundType = to.StringPtr(string(networkProfile.OutboundType)) upstreamSpec.LoadBalancerSKU = to.StringPtr(string(networkProfile.LoadBalancerSku)) } diff --git a/examples/create-example-udr.yaml b/examples/create-example-udr.yaml new file mode 100644 index 00000000..39ab2629 --- /dev/null +++ b/examples/create-example-udr.yaml @@ -0,0 +1,40 @@ +apiVersion: aks.cattle.io/v1 +kind: AKSClusterConfig +metadata: + name: my-cluster +spec: + resourceLocation: "germanywestcentral" + resourceGroup: "my-group" + clusterName: "my-cluster" + baseUrl: "https://management.azure.com/" + authBaseUrl: "https://login.microsoftonline.com" + azureCredentialSecret: "REPLACE_WITH_K8S_SECRETS_NAME" + dnsPrefix: "example-dns" + privateCluster: false + linuxAdminUsername: "rancher-user" + loadBalancerSku: "" + sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY" + kubernetesVersion: "1.19.9" + nodePools: + - name: "masters" + count: 1 + vmSize: "Standard_DS2_v2" + osDiskSizeGB: 128 + osDiskType: "Managed" + maxPods: 110 + mode: "System" + osType: "Linux" + - name: "workers" + orchestratorVersion: "1.19.9" + count: 6 + vmSize: "Standard_DS2_v2" + osDiskSizeGB: 128 + osDiskType: "Managed" + maxPods: 110 + mode: "User" + osType: "Linux" + enableAutoScaling: true + minCount: 1 + maxCount: 6 + availabilityZones: [ "1", "2", "3" ] + outboundType: "userDefinedRouting" diff --git a/examples/create-example.yaml b/examples/create-example.yaml index 0fa58e9c..2dbe0e9f 100644 --- a/examples/create-example.yaml +++ b/examples/create-example.yaml @@ -10,9 +10,10 @@ spec: authBaseUrl: "https://login.microsoftonline.com" azureCredentialSecret: "REPLACE_WITH_K8S_SECRETS_NAME" dnsPrefix: "example-dns" - privateCluster: false, - linuxAdminUsername: "rancher-user", - sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY", + privateCluster: false + linuxAdminUsername: "rancher-user" + loadBalancerSku: "standard" + sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY" kubernetesVersion: "1.19.9" nodePools: - name: "masters" @@ -36,3 +37,4 @@ spec: minCount: 1 maxCount: 6 availabilityZones: [ "1", "2", "3" ] + outboundType: "loadBalancer" diff --git a/pkg/aks/create.go b/pkg/aks/create.go index cb0f9472..50b4d481 100644 --- a/pkg/aks/create.go +++ b/pkg/aks/create.go @@ -64,6 +64,15 @@ func createManagedCluster(ctx context.Context, cred *Credentials, workplacesClie networkProfile := &containerservice.NetworkProfile{} + switch to.String(spec.OutboundType) { + case string(containerservice.LoadBalancer): + networkProfile.OutboundType = containerservice.LoadBalancer + case string(containerservice.UserDefinedRouting): + networkProfile.OutboundType = containerservice.UserDefinedRouting + case "": + networkProfile.OutboundType = containerservice.LoadBalancer + } + switch to.String(spec.NetworkPolicy) { case string(containerservice.NetworkPolicyAzure): networkProfile.NetworkPolicy = containerservice.NetworkPolicyAzure @@ -90,14 +99,16 @@ func createManagedCluster(ctx context.Context, cred *Credentials, workplacesClie return nil, fmt.Errorf("network plugin Kubenet is not compatible with network policy Azure") } - switch to.String(spec.LoadBalancerSKU) { // TODO: only standard is supported for now, find a way to validate this - case string(containerservice.Standard): - networkProfile.LoadBalancerSku = containerservice.Standard - case string(containerservice.Basic): + networkProfile.LoadBalancerSku = containerservice.Standard + + if to.String(spec.LoadBalancerSKU) == string(containerservice.Basic) { logrus.Warnf("loadBalancerSKU 'basic' is not supported") networkProfile.LoadBalancerSku = containerservice.Basic - case "": - networkProfile.LoadBalancerSku = containerservice.Standard + } + + // Disable standard loadbalancer for UserDefinedRouting and use routing created by user pre-defined table for egress + if to.String(spec.OutboundType) == string(containerservice.UserDefinedRouting) { + networkProfile.LoadBalancerSku = "" } virtualNetworkResourceGroup := spec.ResourceGroup diff --git a/pkg/aks/create_test.go b/pkg/aks/create_test.go index 34b69ac7..9562212c 100644 --- a/pkg/aks/create_test.go +++ b/pkg/aks/create_test.go @@ -88,17 +88,19 @@ var _ = Describe("newManagedCluster", func() { ID: to.StringPtr("test-workspace-id"), }, nil) + clusterSpec.LoadBalancerSKU = to.StringPtr("standard") managedCluster, err := createManagedCluster(ctx, cred, workplacesClientMock, clusterSpec, "test-phase") Expect(err).ToNot(HaveOccurred()) Expect(managedCluster.Tags).To(HaveKeyWithValue("test-tag", to.StringPtr("test-value"))) Expect(managedCluster.NetworkProfile.NetworkPolicy).To(Equal(containerservice.NetworkPolicy(to.String(clusterSpec.NetworkPolicy)))) - Expect(managedCluster.NetworkProfile.LoadBalancerSku).To(Equal(containerservice.Standard)) + Expect(managedCluster.NetworkProfile.LoadBalancerSku).To(Equal(containerservice.LoadBalancerSku(to.String(clusterSpec.LoadBalancerSKU)))) Expect(managedCluster.NetworkProfile.NetworkPlugin).To(Equal(containerservice.NetworkPlugin(to.String(clusterSpec.NetworkPlugin)))) Expect(managedCluster.NetworkProfile.DNSServiceIP).To(Equal(clusterSpec.NetworkDNSServiceIP)) Expect(managedCluster.NetworkProfile.DockerBridgeCidr).To(Equal(clusterSpec.NetworkDockerBridgeCIDR)) Expect(managedCluster.NetworkProfile.ServiceCidr).To(Equal(clusterSpec.NetworkServiceCIDR)) Expect(managedCluster.NetworkProfile.PodCidr).To(Equal(clusterSpec.NetworkPodCIDR)) + Expect(managedCluster.NetworkProfile.OutboundType).To(Equal(containerservice.LoadBalancer)) agentPoolProfiles := *managedCluster.AgentPoolProfiles Expect(agentPoolProfiles).To(HaveLen(1)) Expect(agentPoolProfiles[0].Name).To(Equal(clusterSpec.NodePools[0].Name)) @@ -158,7 +160,18 @@ var _ = Describe("newManagedCluster", func() { Expect(managedCluster.NetworkProfile.LoadBalancerSku).To(Equal(containerservice.Basic)) }) - It("should successfully create managed cluster with custom network plugin no network profile", func() { + It("should successfully create managed cluster with outboundtype userdefinedrouting", func() { + workplacesClientMock.EXPECT().Get(ctx, to.String(clusterSpec.LogAnalyticsWorkspaceGroup), to.String(clusterSpec.LogAnalyticsWorkspaceName)). + Return(operationalinsights.Workspace{ + ID: to.StringPtr("test-workspace-id"), + }, nil) + clusterSpec.OutboundType = to.StringPtr("userDefinedRouting") + managedCluster, err := createManagedCluster(ctx, cred, workplacesClientMock, clusterSpec, "test-phase") + Expect(err).ToNot(HaveOccurred()) + Expect(managedCluster.NetworkProfile.OutboundType).To(Equal(containerservice.UserDefinedRouting)) + }) + + It("should successfully create managed cluster with custom network plugin without network profile", func() { workplacesClientMock.EXPECT().Get(ctx, to.String(clusterSpec.LogAnalyticsWorkspaceGroup), to.String(clusterSpec.LogAnalyticsWorkspaceName)). Return(operationalinsights.Workspace{ ID: to.StringPtr("test-workspace-id"), @@ -171,12 +184,12 @@ var _ = Describe("newManagedCluster", func() { clusterSpec.NetworkPodCIDR = to.StringPtr("") managedCluster, err := createManagedCluster(ctx, cred, workplacesClientMock, clusterSpec, "test-phase") + Expect(err).ToNot(HaveOccurred()) Expect(managedCluster.NetworkProfile.NetworkPlugin).To(Equal(containerservice.Kubenet)) Expect(managedCluster.NetworkProfile.DNSServiceIP).To(Equal(clusterSpec.NetworkDNSServiceIP)) Expect(managedCluster.NetworkProfile.DockerBridgeCidr).To(Equal(clusterSpec.NetworkDockerBridgeCIDR)) Expect(managedCluster.NetworkProfile.ServiceCidr).To(Equal(clusterSpec.NetworkServiceCIDR)) Expect(managedCluster.NetworkProfile.PodCidr).To(Equal(clusterSpec.NetworkPodCIDR)) - Expect(err).ToNot(HaveOccurred()) }) It("should successfully create managed cluster with custom network plugin", func() { diff --git a/pkg/apis/aks.cattle.io/v1/types.go b/pkg/apis/aks.cattle.io/v1/types.go index fcfc0434..860da1e7 100644 --- a/pkg/apis/aks.cattle.io/v1/types.go +++ b/pkg/apis/aks.cattle.io/v1/types.go @@ -48,6 +48,7 @@ type AKSClusterConfigSpec struct { NetworkServiceCIDR *string `json:"serviceCidr" norman:"pointer"` NetworkDockerBridgeCIDR *string `json:"dockerBridgeCidr" norman:"pointer"` NetworkPodCIDR *string `json:"podCidr" norman:"pointer"` + OutboundType *string `json:"outboundType" norman:"pointer"` LoadBalancerSKU *string `json:"loadBalancerSku" norman:"pointer"` NetworkPolicy *string `json:"networkPolicy" norman:"pointer"` LinuxAdminUsername *string `json:"linuxAdminUsername,omitempty" norman:"pointer"` diff --git a/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go b/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go index 26d57765..751e7273 100644 --- a/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go +++ b/pkg/apis/aks.cattle.io/v1/zz_generated_deepcopy.go @@ -139,6 +139,11 @@ func (in *AKSClusterConfigSpec) DeepCopyInto(out *AKSClusterConfigSpec) { *out = new(string) **out = **in } + if in.OutboundType != nil { + in, out := &in.OutboundType, &out.OutboundType + *out = new(string) + **out = **in + } if in.LoadBalancerSKU != nil { in, out := &in.LoadBalancerSKU, &out.LoadBalancerSKU *out = new(string)