diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index fd08c0cc12..16f8910570 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -8038,6 +8038,23 @@ string

PortName is the name of service port

+ + +loadBalancerSourceRanges
+ +[]string + + + +(Optional) +

LoadBalancerSourceRanges is the loadBalancerSourceRanges of service +If specified and supported by the platform, this will restrict traffic through the cloud-provider +load-balancer will be restricted to the specified client IPs. This field will be ignored if the +cloud-provider does not support the feature.” +More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/ +Optional: Defaults to omitted

+ +

Status

diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 6d5c88ce1c..ff058ee860 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -940,6 +940,10 @@ spec: type: string loadBalancerIP: type: string + loadBalancerSourceRanges: + items: + type: string + type: array portName: type: string type: diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index f2f6092e27..4a3d94383f 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -3946,6 +3946,20 @@ func schema_pkg_apis_pingcap_v1alpha1_ServiceSpec(ref common.ReferenceCallback) Format: "", }, }, + "loadBalancerSourceRanges": { + SchemaProps: spec.SchemaProps{ + Description: "LoadBalancerSourceRanges is the loadBalancerSourceRanges of service If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature.\" More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/ Optional: Defaults to omitted", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, }, }, }, diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 1e60ab6d09..7d2e4d95a0 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -677,6 +677,15 @@ type ServiceSpec struct { // PortName is the name of service port // +optional PortName *string `json:"portName,omitempty"` + + // LoadBalancerSourceRanges is the loadBalancerSourceRanges of service + // If specified and supported by the platform, this will restrict traffic through the cloud-provider + // load-balancer will be restricted to the specified client IPs. This field will be ignored if the + // cloud-provider does not support the feature." + // More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/ + // Optional: Defaults to omitted + // +optional + LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` } // +k8s:openapi-gen=true diff --git a/pkg/apis/pingcap/v1alpha1/validation/validation.go b/pkg/apis/pingcap/v1alpha1/validation/validation.go index b2fa3871e5..9afab4092b 100644 --- a/pkg/apis/pingcap/v1alpha1/validation/validation.go +++ b/pkg/apis/pingcap/v1alpha1/validation/validation.go @@ -23,10 +23,10 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/label" corev1 "k8s.io/api/core/v1" - apivalidation "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + utilnet "k8s.io/utils/net" ) // ValidateTidbCluster validates a TidbCluster, it performs basic validation for all TidbClusters despite it is legacy @@ -42,6 +42,17 @@ func ValidateTidbCluster(tc *v1alpha1.TidbCluster) field.ErrorList { return allErrs } +func ValidateTidbMonitor(monitor *v1alpha1.TidbMonitor) field.ErrorList { + allErrs := field.ErrorList{} + // validate monitor service + if monitor.Spec.Grafana != nil { + allErrs = append(allErrs, validateService(&monitor.Spec.Grafana.Service, field.NewPath("spec"))...) + } + allErrs = append(allErrs, validateService(&monitor.Spec.Prometheus.Service, field.NewPath("spec"))...) + allErrs = append(allErrs, validateService(&monitor.Spec.Reloader.Service, field.NewPath("spec"))...) + return allErrs +} + func validateAnnotations(anns map[string]string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, apivalidation.ValidateAnnotations(anns, fldPath)...) @@ -164,6 +175,9 @@ func validateTiFlashConfig(config *v1alpha1.TiFlashConfig, path *field.Path) fie func validateTiDBSpec(spec *v1alpha1.TiDBSpec, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, validateComponentSpec(&spec.ComponentSpec, fldPath)...) + if spec.Service != nil { + allErrs = append(allErrs, validateService(&spec.Service.ServiceSpec, fldPath)...) + } return allErrs } @@ -399,3 +413,16 @@ func validateDeleteSlots(annotations map[string]string, key string, fldPath *fie } return allErrs } + +func validateService(spec *v1alpha1.ServiceSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + //validate LoadBalancerSourceRanges field from service + if len(spec.LoadBalancerSourceRanges) > 0 { + ip := spec.LoadBalancerSourceRanges + _, err := utilnet.ParseIPNets(ip...) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("spec.LoadBalancerSourceRanges"), spec.LoadBalancerSourceRanges, "service.Spec.LoadBalancerSourceRanges is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24.")) + } + } + return allErrs +} diff --git a/pkg/apis/pingcap/v1alpha1/validation/validation_test.go b/pkg/apis/pingcap/v1alpha1/validation/validation_test.go index eb84c768d9..91f14a8816 100644 --- a/pkg/apis/pingcap/v1alpha1/validation/validation_test.go +++ b/pkg/apis/pingcap/v1alpha1/validation/validation_test.go @@ -230,9 +230,95 @@ func TestValidateRequestsStorage(t *testing.T) { } } +func TestValidateService(t *testing.T) { + g := NewGomegaWithT(t) + tests := []struct { + name string + loadBalancerSourceRanges []string + expectedErrors int + }{ + { + name: "correct LoadBalancerSourceRanges", + loadBalancerSourceRanges: strings.Split("192.168.0.1/32", ","), + expectedErrors: 0, + }, + { + name: "incorrect LoadBalancerSourceRanges", + loadBalancerSourceRanges: strings.Split("192.168.0.1", ","), + expectedErrors: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc := newService() + svc.LoadBalancerSourceRanges = tt.loadBalancerSourceRanges + err := validateService(svc, field.NewPath("spec")) + r := len(err) + g.Expect(r).Should(Equal(tt.expectedErrors)) + if r > 0 { + for _, e := range err { + g.Expect(e.Detail).To(ContainSubstring("service.Spec.LoadBalancerSourceRanges is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24.")) + } + } + }) + } +} + +func TestValidateTidbMonitor(t *testing.T) { + g := NewGomegaWithT(t) + tests := []struct { + name string + loadBalancerSourceRanges []string + expectedErrors int + }{ + { + name: "correct LoadBalancerSourceRanges", + loadBalancerSourceRanges: strings.Split("192.168.0.1/24,192.168.1.1/24", ","), + expectedErrors: 0, + }, + { + name: "incorrect LoadBalancerSourceRanges", + loadBalancerSourceRanges: strings.Split("192.168.0.1,192.168.1.1", ","), + expectedErrors: 3, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + monitor := newTidbMonitor() + monitor.Spec.Prometheus.Service.LoadBalancerSourceRanges = tt.loadBalancerSourceRanges + monitor.Spec.Grafana.Service.LoadBalancerSourceRanges = tt.loadBalancerSourceRanges + monitor.Spec.Reloader.Service.LoadBalancerSourceRanges = tt.loadBalancerSourceRanges + err := ValidateTidbMonitor(monitor) + r := len(err) + g.Expect(r).Should(Equal(tt.expectedErrors)) + if r > 0 { + for _, e := range err { + g.Expect(e.Detail).To(ContainSubstring("service.Spec.LoadBalancerSourceRanges is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24.")) + } + } + }) + } +} + func newTidbCluster() *v1alpha1.TidbCluster { tc := &v1alpha1.TidbCluster{} tc.Name = "test-validate-requests-storage" tc.Namespace = "default" return tc } + +func newService() *v1alpha1.ServiceSpec { + svc := &v1alpha1.ServiceSpec{} + return svc +} + +func newTidbMonitor() *v1alpha1.TidbMonitor { + monitor := &v1alpha1.TidbMonitor{ + Spec: v1alpha1.TidbMonitorSpec{ + Grafana: &v1alpha1.GrafanaSpec{}, + Prometheus: v1alpha1.PrometheusSpec{}, + Reloader: v1alpha1.ReloaderSpec{}, + }, + } + return monitor +} diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index b81294b521..b0627d6642 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -3459,6 +3459,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = new(string) **out = **in } + if in.LoadBalancerSourceRanges != nil { + in, out := &in.LoadBalancerSourceRanges, &out.LoadBalancerSourceRanges + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 96a061ce22..35af174087 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -473,8 +473,13 @@ func getNewTiDBServiceOrNil(tc *v1alpha1.TidbCluster) *corev1.Service { Selector: tidbLabels, }, } - if svcSpec.LoadBalancerIP != nil { - tidbSvc.Spec.LoadBalancerIP = *svcSpec.LoadBalancerIP + if svcSpec.Type == corev1.ServiceTypeLoadBalancer { + if svcSpec.LoadBalancerIP != nil { + tidbSvc.Spec.LoadBalancerIP = *svcSpec.LoadBalancerIP + } + if svcSpec.LoadBalancerSourceRanges != nil { + tidbSvc.Spec.LoadBalancerSourceRanges = svcSpec.LoadBalancerSourceRanges + } } if svcSpec.ExternalTrafficPolicy != nil { tidbSvc.Spec.ExternalTrafficPolicy = *svcSpec.ExternalTrafficPolicy diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index ab8667d5f6..c32f704927 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -1347,6 +1347,10 @@ func TestTiDBInitContainers(t *testing.T) { func TestGetNewTiDBService(t *testing.T) { g := NewGomegaWithT(t) trafficPolicy := corev1.ServiceExternalTrafficPolicyTypeLocal + loadBalancerSourceRanges := []string{ + "10.0.0.0/8", + "130.211.204.1/32", + } testCases := []struct { name string tc v1alpha1.TidbCluster @@ -1499,6 +1503,7 @@ func TestGetNewTiDBService(t *testing.T) { Annotations: map[string]string{ "lb-type": "testlb", }, + LoadBalancerSourceRanges: loadBalancerSourceRanges, }, ExternalTrafficPolicy: &trafficPolicy, ExposeStatus: pointer.BoolPtr(true), @@ -1537,6 +1542,10 @@ func TestGetNewTiDBService(t *testing.T) { Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + LoadBalancerSourceRanges: []string{ + "10.0.0.0/8", + "130.211.204.1/32", + }, Ports: []corev1.ServicePort{ { Name: "mysql-client", diff --git a/pkg/monitor/monitor/util.go b/pkg/monitor/monitor/util.go index f2ecea446a..85cdf7c42b 100644 --- a/pkg/monitor/monitor/util.go +++ b/pkg/monitor/monitor/util.go @@ -690,6 +690,9 @@ func getMonitorService(monitor *v1alpha1.TidbMonitor) []*core.Service { if monitor.Spec.Prometheus.Service.LoadBalancerIP != nil { prometheusService.Spec.LoadBalancerIP = *monitor.Spec.Prometheus.Service.LoadBalancerIP } + if monitor.Spec.Prometheus.Service.LoadBalancerSourceRanges != nil { + prometheusService.Spec.LoadBalancerSourceRanges = monitor.Spec.Prometheus.Service.LoadBalancerSourceRanges + } } reloaderService := &core.Service{ @@ -718,6 +721,9 @@ func getMonitorService(monitor *v1alpha1.TidbMonitor) []*core.Service { if monitor.Spec.Reloader.Service.LoadBalancerIP != nil { reloaderService.Spec.LoadBalancerIP = *monitor.Spec.Reloader.Service.LoadBalancerIP } + if monitor.Spec.Reloader.Service.LoadBalancerSourceRanges != nil { + reloaderService.Spec.LoadBalancerSourceRanges = monitor.Spec.Reloader.Service.LoadBalancerSourceRanges + } } services = append(services, prometheusService, reloaderService) @@ -748,6 +754,9 @@ func getMonitorService(monitor *v1alpha1.TidbMonitor) []*core.Service { if monitor.Spec.Grafana.Service.LoadBalancerIP != nil { grafanaService.Spec.LoadBalancerIP = *monitor.Spec.Grafana.Service.LoadBalancerIP } + if monitor.Spec.Grafana.Service.LoadBalancerSourceRanges != nil { + grafanaService.Spec.LoadBalancerSourceRanges = monitor.Spec.Grafana.Service.LoadBalancerSourceRanges + } } services = append(services, grafanaService) diff --git a/pkg/monitor/monitor/util_test.go b/pkg/monitor/monitor/util_test.go new file mode 100644 index 0000000000..c8f3865a73 --- /dev/null +++ b/pkg/monitor/monitor/util_test.go @@ -0,0 +1,268 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitor + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + core "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" +) + +func TestGetMonitorService(t *testing.T) { + g := NewGomegaWithT(t) + testCases := []struct { + name string + monitor v1alpha1.TidbMonitor + expected []*corev1.Service + }{ + { + name: "basic", + monitor: v1alpha1.TidbMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "ns", + }, + }, + expected: []*corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-prometheus", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "tidb-cluster", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/component": "monitor", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbMonitor", + Name: "foo", + UID: "", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http-prometheus", + Protocol: "TCP", + Port: 9090, + TargetPort: intstr.IntOrString{IntVal: 9090}, + }, + }, + Selector: map[string]string{ + "app.kubernetes.io/component": "monitor", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/name": "tidb-cluster", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-monitor-reloader", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/component": "monitor", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/name": "tidb-cluster", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbMonitor", + Name: "foo", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []core.ServicePort{ + { + Name: "tcp-reloader", + Port: 9089, + Protocol: core.ProtocolTCP, + TargetPort: intstr.FromInt(9089), + }, + }, + Selector: map[string]string{ + "app.kubernetes.io/component": "monitor", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/name": "tidb-cluster", + }, + }, + }, + }, + }, + { + name: "TidbMonitor service in typical public cloud", + monitor: v1alpha1.TidbMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "ns", + }, + Spec: v1alpha1.TidbMonitorSpec{ + Prometheus: v1alpha1.PrometheusSpec{ + Service: v1alpha1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerIP: pointer.StringPtr("78.11.24.19"), + LoadBalancerSourceRanges: []string{ + "10.0.0.0/8", + "130.211.204.1/32", + }, + }, + }, + Reloader: v1alpha1.ReloaderSpec{ + Service: v1alpha1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerIP: pointer.StringPtr("78.11.24.19"), + LoadBalancerSourceRanges: []string{ + "10.0.0.0/8", + "130.211.204.1/32", + }, + }, + }, + }, + }, + expected: []*corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-prometheus", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "tidb-cluster", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/component": "monitor", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbMonitor", + Name: "foo", + UID: "", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http-prometheus", + Protocol: "TCP", + Port: 9090, + TargetPort: intstr.IntOrString{IntVal: 9090}, + }, + }, + Selector: map[string]string{ + "app.kubernetes.io/component": "monitor", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/name": "tidb-cluster", + }, + Type: "LoadBalancer", + LoadBalancerIP: "78.11.24.19", + LoadBalancerSourceRanges: []string{ + "10.0.0.0/8", + "130.211.204.1/32", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-monitor-reloader", + Namespace: "ns", + Labels: map[string]string{ + "app.kubernetes.io/component": "monitor", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/managed-by": "tidb-operator", + "app.kubernetes.io/name": "tidb-cluster", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pingcap.com/v1alpha1", + Kind: "TidbMonitor", + Name: "foo", + Controller: func(b bool) *bool { + return &b + }(true), + BlockOwnerDeletion: func(b bool) *bool { + return &b + }(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []core.ServicePort{ + { + Name: "tcp-reloader", + Port: 9089, + Protocol: core.ProtocolTCP, + TargetPort: intstr.FromInt(9089), + }, + }, + Selector: map[string]string{ + "app.kubernetes.io/component": "monitor", + "app.kubernetes.io/instance": "foo", + "app.kubernetes.io/name": "tidb-cluster", + }, + Type: "LoadBalancer", + LoadBalancerIP: "78.11.24.19", + LoadBalancerSourceRanges: []string{ + "10.0.0.0/8", + "130.211.204.1/32", + }, + }, + }, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + svc := getMonitorService(&tt.monitor) + if tt.expected == nil { + g.Expect(svc).To(BeNil()) + return + } + if diff := cmp.Diff(&tt.expected, &svc); diff != "" { + t.Errorf("unexpected plugin configuration (-want, +got): %s", diff) + } + }) + } +}