diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 6200881614..5db040d740 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -7043,6 +7043,19 @@ spec: the pd could provide it. MetricsUrl represents the url to fetch the metrics info type: string + monitor: + description: TidbMonitorRef reference to a TidbMonitor + properties: + name: + description: Name is the name of TidbMonitor object + type: string + namespace: + description: Namespace is the namespace that TidbMonitor object + locates, default to the same namespace with TidbClusterAutoScaler + type: string + required: + - name + type: object tidb: description: TidbAutoScalerSpec describes the spec for tidb auto-scaling properties: diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index bf5e202ba5..8fed6209b8 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -111,6 +111,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbInitializerStatus": schema_pkg_apis_pingcap_v1alpha1_TidbInitializerStatus(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbMonitor": schema_pkg_apis_pingcap_v1alpha1_TidbMonitor(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbMonitorList": schema_pkg_apis_pingcap_v1alpha1_TidbMonitorList(ref), + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbMonitorRef": schema_pkg_apis_pingcap_v1alpha1_TidbMonitorRef(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbMonitorSpec": schema_pkg_apis_pingcap_v1alpha1_TidbMonitorSpec(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TikvAutoScalerSpec": schema_pkg_apis_pingcap_v1alpha1_TikvAutoScalerSpec(ref), "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TxnLocalLatches": schema_pkg_apis_pingcap_v1alpha1_TxnLocalLatches(ref), @@ -5846,6 +5847,12 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerSpec(ref common.Refer Format: "", }, }, + "monitor": { + SchemaProps: spec.SchemaProps{ + Description: "TidbMonitorRef describe the target TidbMonitor, when MetricsUrl and Monitor are both set, Operator will use MetricsUrl", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbMonitorRef"), + }, + }, "tikv": { SchemaProps: spec.SchemaProps{ Description: "TiKV represents the auto-scaling spec for tikv", @@ -5863,7 +5870,7 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerSpec(ref common.Refer }, }, Dependencies: []string{ - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbAutoScalerSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterRef", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TikvAutoScalerSpec"}, + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbAutoScalerSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterRef", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbMonitorRef", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TikvAutoScalerSpec"}, } } @@ -6402,6 +6409,34 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbMonitorList(ref common.ReferenceCallba } } +func schema_pkg_apis_pingcap_v1alpha1_TidbMonitorRef(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TidbMonitorRef reference to a TidbMonitor", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "namespace": { + SchemaProps: spec.SchemaProps{ + Description: "Namespace is the namespace that TidbMonitor object locates, default to the same namespace with TidbClusterAutoScaler", + Type: []string{"string"}, + Format: "", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name is the name of TidbMonitor object", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"name"}, + }, + }, + } +} + func schema_pkg_apis_pingcap_v1alpha1_TidbMonitorSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go b/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go index 139ee7c744..a4639eabd8 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go +++ b/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go @@ -58,6 +58,11 @@ type TidbClusterAutoScalerSpec struct { // +optional MetricsUrl *string `json:"metricsUrl,omitempty"` + // TidbMonitorRef describe the target TidbMonitor, when MetricsUrl and Monitor are both set, + // Operator will use MetricsUrl + // +optional + Monitor *TidbMonitorRef `json:"monitor,omitempty"` + // TiKV represents the auto-scaling spec for tikv // +optional TiKV *TikvAutoScalerSpec `json:"tikv,omitempty"` @@ -132,6 +137,18 @@ type BasicAutoScalerSpec struct { ScaleInThreshold *int32 `json:"scaleInThreshold,omitempty"` } +// +k8s:openapi-gen=true +// TidbMonitorRef reference to a TidbMonitor +type TidbMonitorRef struct { + // Namespace is the namespace that TidbMonitor object locates, + // default to the same namespace with TidbClusterAutoScaler + // +optional + Namespace string `json:"namespace,omitempty"` + + // Name is the name of TidbMonitor object + Name string `json:"name"` +} + // TODO: sync status type TidbClusterAutoSclaerStatus struct { } diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 25aa9a0013..28786d2937 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -3870,6 +3870,11 @@ func (in *TidbClusterAutoScalerSpec) DeepCopyInto(out *TidbClusterAutoScalerSpec *out = new(string) **out = **in } + if in.Monitor != nil { + in, out := &in.Monitor, &out.Monitor + *out = new(TidbMonitorRef) + **out = **in + } if in.TiKV != nil { in, out := &in.TiKV, &out.TiKV *out = new(TikvAutoScalerSpec) @@ -4244,6 +4249,22 @@ func (in *TidbMonitorList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TidbMonitorRef) DeepCopyInto(out *TidbMonitorRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TidbMonitorRef. +func (in *TidbMonitorRef) DeepCopy() *TidbMonitorRef { + if in == nil { + return nil + } + out := new(TidbMonitorRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TidbMonitorSpec) DeepCopyInto(out *TidbMonitorSpec) { *out = *in diff --git a/pkg/autoscaler/autoscaler/autoscaler_manager.go b/pkg/autoscaler/autoscaler/autoscaler_manager.go index e74ab35fbe..3bbc7c7454 100644 --- a/pkg/autoscaler/autoscaler/autoscaler_manager.go +++ b/pkg/autoscaler/autoscaler/autoscaler_manager.go @@ -21,7 +21,6 @@ import ( informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" v1alpha1listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" - promClient "github.com/prometheus/client_golang/api" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -88,21 +87,14 @@ func (am *autoScalerManager) Sync(tac *v1alpha1.TidbClusterAutoScaler) error { } func (am *autoScalerManager) syncAutoScaling(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler) error { - if tac.Spec.MetricsUrl == nil { - return fmt.Errorf("tidbclusterAutoScaler[%s/%s]' metrics url should be defined explicitly", tac.Namespace, tac.Name) - } - c, err := promClient.NewClient(promClient.Config{Address: *tac.Spec.MetricsUrl}) - if err != nil { - return err - } defaultTAC(tac) oldTikvReplicas := tc.Spec.TiKV.Replicas - if err := am.syncTiKV(tc, tac, c); err != nil { + if err := am.syncTiKV(tc, tac); err != nil { tc.Spec.TiKV.Replicas = oldTikvReplicas klog.Errorf("tac[%s/%s] tikv sync failed, continue to sync next, err:%v", tac.Namespace, tac.Name, err) } oldTidbReplicas := tc.Spec.TiDB.Replicas - if err := am.syncTiDB(tc, tac, c); err != nil { + if err := am.syncTiDB(tc, tac); err != nil { tc.Spec.TiDB.Replicas = oldTidbReplicas klog.Errorf("tac[%s/%s] tidb sync failed, continue to sync next, err:%v", tac.Namespace, tac.Name, err) } diff --git a/pkg/autoscaler/autoscaler/calculate/calculate.go b/pkg/autoscaler/autoscaler/calculate/calculate.go index 0d1f9e8413..b01132fec1 100644 --- a/pkg/autoscaler/autoscaler/calculate/calculate.go +++ b/pkg/autoscaler/autoscaler/calculate/calculate.go @@ -36,6 +36,7 @@ const ( ) type SingleQuery struct { + Endpoint string Timestamp int64 Quary string Instances []string @@ -45,7 +46,7 @@ type SingleQuery struct { func queryMetricsFromPrometheus(tac *v1alpha1.TidbClusterAutoScaler, client promClient.Client, sq *SingleQuery, resp *Response) error { query := sq.Quary timestamp := sq.Timestamp - req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", *tac.Spec.MetricsUrl, queryPath), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", sq.Endpoint, queryPath), nil) if err != nil { return err } diff --git a/pkg/autoscaler/autoscaler/tidb_autoscaler.go b/pkg/autoscaler/autoscaler/tidb_autoscaler.go index e4bed4f69e..19179b3b61 100644 --- a/pkg/autoscaler/autoscaler/tidb_autoscaler.go +++ b/pkg/autoscaler/autoscaler/tidb_autoscaler.go @@ -25,7 +25,7 @@ import ( appsv1 "k8s.io/api/apps/v1" ) -func (am *autoScalerManager) syncTiDB(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler, client promClient.Client) error { +func (am *autoScalerManager) syncTiDB(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler) error { if tac.Spec.TiDB == nil { return nil } @@ -38,7 +38,7 @@ func (am *autoScalerManager) syncTiDB(tc *v1alpha1.TidbCluster, tac *v1alpha1.Ti } currentReplicas := tc.Spec.TiDB.Replicas instances := filterTidbInstances(tc) - targetReplicas, err := calculateTidbMetrics(tac, sts, client, instances) + targetReplicas, err := calculateTidbMetrics(tac, sts, instances) if err != nil { return err } @@ -73,7 +73,15 @@ func updateTcTiDBAnnIfScale(tac *v1alpha1.TidbClusterAutoScaler) { tac.Annotations[label.AnnTiDBLastAutoScalingTimestamp] = fmt.Sprintf("%d", time.Now().Unix()) } -func calculateTidbMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.StatefulSet, client promClient.Client, instances []string) (int32, error) { +func calculateTidbMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.StatefulSet, instances []string) (int32, error) { + ep, err := genMetricsEndpoint(tac) + if err != nil { + return -1, err + } + client, err := promClient.NewClient(promClient.Config{Address: ep}) + if err != nil { + return -1, err + } metric := calculate.FilterMetrics(tac.Spec.TiDB.Metrics) mType, err := calculate.GenMetricType(tac, metric) if err != nil { @@ -84,6 +92,7 @@ func calculateTidbMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.State return -1, err } sq := &calculate.SingleQuery{ + Endpoint: ep, Timestamp: time.Now().Unix(), Instances: instances, Metric: metric, diff --git a/pkg/autoscaler/autoscaler/tikv_autoscaler.go b/pkg/autoscaler/autoscaler/tikv_autoscaler.go index 192ebb62cb..f20de7de4c 100644 --- a/pkg/autoscaler/autoscaler/tikv_autoscaler.go +++ b/pkg/autoscaler/autoscaler/tikv_autoscaler.go @@ -25,7 +25,7 @@ import ( appsv1 "k8s.io/api/apps/v1" ) -func (am *autoScalerManager) syncTiKV(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler, client promClient.Client) error { +func (am *autoScalerManager) syncTiKV(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler) error { if tac.Spec.TiKV == nil { return nil } @@ -38,7 +38,7 @@ func (am *autoScalerManager) syncTiKV(tc *v1alpha1.TidbCluster, tac *v1alpha1.Ti } instances := filterTiKVInstances(tc) currentReplicas := int32(len(instances)) - targetReplicas, err := calculateTikvMetrics(tac, sts, client, instances) + targetReplicas, err := calculateTikvMetrics(tac, sts, instances) if err != nil { return err } @@ -87,7 +87,16 @@ func updateTcTiKVAnnIfScale(tac *v1alpha1.TidbClusterAutoScaler) { tac.Annotations[label.AnnTiKVLastAutoScalingTimestamp] = fmt.Sprintf("%d", time.Now().Unix()) } -func calculateTikvMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.StatefulSet, client promClient.Client, instances []string) (int32, error) { +func calculateTikvMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.StatefulSet, instances []string) (int32, error) { + ep, err := genMetricsEndpoint(tac) + if err != nil { + return -1, err + } + client, err := promClient.NewClient(promClient.Config{Address: ep}) + if err != nil { + return -1, err + } + metric := calculate.FilterMetrics(tac.Spec.TiKV.Metrics) mType, err := calculate.GenMetricType(tac, metric) if err != nil { @@ -99,6 +108,7 @@ func calculateTikvMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.State return -1, err } sq := &calculate.SingleQuery{ + Endpoint: ep, Timestamp: time.Now().Unix(), Instances: instances, Metric: metric, diff --git a/pkg/autoscaler/autoscaler/util.go b/pkg/autoscaler/autoscaler/util.go index ab6ceedaa3..9f21156475 100644 --- a/pkg/autoscaler/autoscaler/util.go +++ b/pkg/autoscaler/autoscaler/util.go @@ -14,6 +14,7 @@ package autoscaler import ( + "fmt" "strconv" "time" @@ -174,6 +175,12 @@ func defaultTAC(tac *v1alpha1.TidbClusterAutoScaler) { tac.Spec.TiDB.ScaleInIntervalSeconds = pointer.Int32Ptr(500) } } + + if tac.Spec.Monitor != nil { + if len(tac.Spec.Monitor.Namespace) < 1 { + tac.Spec.Monitor.Namespace = tac.Namespace + } + } } func resetAutoScalingAnn(tac *v1alpha1.TidbClusterAutoScaler) { @@ -198,3 +205,13 @@ func checkAndUpdateTacAnn(tac *v1alpha1.TidbClusterAutoScaler) { // If not satisfied, reset tac Ann resetAutoScalingAnn(tac) } + +func genMetricsEndpoint(tac *v1alpha1.TidbClusterAutoScaler) (string, error) { + if tac.Spec.MetricsUrl == nil && tac.Spec.Monitor == nil { + return "", fmt.Errorf("tac[%s/%s] metrics url or monitor should be defined explicitly", tac.Namespace, tac.Name) + } + if tac.Spec.MetricsUrl != nil { + return *tac.Spec.MetricsUrl, nil + } + return fmt.Sprintf("http://%s-prometheus.%s.svc:9090", tac.Spec.Monitor.Name, tac.Spec.Monitor.Namespace), nil +}