Add tidbmonitor e2e test (#1445) (#1527)

* add e2e test * Update monitor.go * Update monitor.go * Update monitor.go * Update monitor.go * Update monitor.go * fix labels * fix by comment * fix lint * fix lint * Update tidbcluster.go * add e2e test * Update monitor.go * add monitor error log * Update monitor.go * add e2e error log * Update monitor.go * Update monitor.go * fix e2e test * Update monitor.go * Update tidbcluster.go * Update template.go * Update tidbcluster.go * Update tidbcluster.go * Update tidbcluster.go * Update tidbcluster.go * Update monitor.go * Update monitor.go * fix monitor e2e test * inline e2e monitor * add note & fix template * Update monitor.go * Update monitor.go Co-authored-by: Song Gao <disxiaofei@163.com>
pingcap · Jan 11, 2020 · 0cc3447 · 0cc3447
1 parent c0582ba
commit 0cc3447
Show file tree

Hide file tree

Showing 7 changed files with 407 additions and 133 deletions.
diff --git a/pkg/label/label.go b/pkg/label/label.go
@@ -167,6 +167,14 @@ func NewBackupSchedule() Label {
 	}
 }
 
+// NewMonitor initialize a new label for monitor of tidb-monitor
+func NewMonitor() Label {
+	return Label{
+		NameLabelKey:      TiDBMonitorVal,
+		ManagedByLabelKey: TiDBOperator,
+	}
+}
+
 // Instance adds instance kv pair to label
 func (l Label) Instance(name string) Label {
 	l[InstanceLabelKey] = name

diff --git a/pkg/monitor/monitor/template.go b/pkg/monitor/monitor/template.go
@@ -34,6 +34,11 @@ scrape_configs:
     honor_labels: true
     kubernetes_sd_configs:
     - role: pod
+      namespaces:
+        names:
+        {{- range .ReleaseNamespaces }}
+        - {{ . }}
+        {{- end}}
     tls_config:
       insecure_skip_verify: true
     {{- if .EnableTLSCluster }}
@@ -68,19 +73,19 @@ scrape_configs:
     - source_labels: [__meta_kubernetes_pod_ip]
       action: replace
       target_label: kubernetes_pod_ip
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: replace
+      target_label: instance
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+      action: replace
+      target_label: cluster
     {{- if .EnableTLSCluster }}
     # This is a workaround of https://github.com/tikv/tikv/issues/5340 and should
     # be removed after TiKV fix this issue
     - source_labels: [__meta_kubernetes_pod_name]
       action: drop
       regex: .*\-tikv\-\d*$
     {{- end }}
-    - source_labels: [__meta_kubernetes_pod_name]
-      action: replace
-      target_label: instance
-    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
-      action: replace
-      target_label: cluster
   {{- if .EnableTLSCluster }}
   # This is a workaround of https://github.com/tikv/tikv/issues/5340 and should
   # be removed after TiKV fix this issue

diff --git a/pkg/monitor/monitor/util.go b/pkg/monitor/monitor/util.go
@@ -29,7 +29,7 @@ import (
 	"k8s.io/utils/pointer"
 )
 
-func getMonitorObjectName(monitor *v1alpha1.TidbMonitor) string {
+func GetMonitorObjectName(monitor *v1alpha1.TidbMonitor) string {
 	return fmt.Sprintf("%s-monitor", monitor.Name)
 }
 
@@ -59,7 +59,7 @@ func getMonitorConfigMap(tc *v1alpha1.TidbCluster, monitor *v1alpha1.TidbMonitor
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	cm := &core.ConfigMap{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -78,7 +78,7 @@ func getMonitorSecret(monitor *v1alpha1.TidbMonitor) *core.Secret {
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	return &core.Secret{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -94,7 +94,7 @@ func getMonitorServiceAccount(monitor *v1alpha1.TidbMonitor) *core.ServiceAccoun
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	sa := &core.ServiceAccount{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -107,7 +107,7 @@ func getMonitorClusterRole(monitor *v1alpha1.TidbMonitor) *rbac.ClusterRole {
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	return &rbac.ClusterRole{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -130,7 +130,7 @@ func getMonitorRole(monitor *v1alpha1.TidbMonitor) *rbac.Role {
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	return &rbac.Role{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -149,7 +149,7 @@ func getMonitorClusterRoleBinding(sa *core.ServiceAccount, cr *rbac.ClusterRole,
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	return &rbac.ClusterRoleBinding{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -174,7 +174,7 @@ func getMonitorRoleBinding(sa *core.ServiceAccount, role *rbac.Role, monitor *v1
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	return &rbac.RoleBinding{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -212,12 +212,13 @@ func getMonitorDeployment(sa *core.ServiceAccount, config *core.ConfigMap, secre
 }
 
 func getMonitorDeploymentSkeleton(sa *core.ServiceAccount, monitor *v1alpha1.TidbMonitor) *apps.Deployment {
-	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
+	monitorLabel := label.New().Instance(monitor.Name).Monitor()
 	replicas := int32(1)
+	labels := label.NewMonitor().Instance(monitor.Name).Monitor()
 
 	deployment := &apps.Deployment{
 		ObjectMeta: meta.ObjectMeta{
-			Name:            getMonitorObjectName(monitor),
+			Name:            GetMonitorObjectName(monitor),
 			Namespace:       monitor.Namespace,
 			Labels:          monitorLabel,
 			OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)},
@@ -229,17 +230,11 @@ func getMonitorDeploymentSkeleton(sa *core.ServiceAccount, monitor *v1alpha1.Tid
 				Type: apps.RecreateDeploymentStrategyType,
 			},
 			Selector: &meta.LabelSelector{
-				MatchLabels: map[string]string{
-					label.InstanceLabelKey:  monitor.Name,
-					label.ComponentLabelKey: label.TiDBMonitorVal,
-				},
+				MatchLabels: labels,
 			},
 			Template: core.PodTemplateSpec{
 				ObjectMeta: meta.ObjectMeta{
-					Labels: map[string]string{
-						label.InstanceLabelKey:  monitor.Name,
-						label.ComponentLabelKey: label.TiDBMonitorVal,
-					},
+					Labels: labels,
 				},
 
 				Spec: core.PodSpec{
@@ -553,7 +548,7 @@ func getMonitorVolumes(config *core.ConfigMap, monitor *v1alpha1.TidbMonitor, tc
 			Name: "monitor-data",
 			VolumeSource: core.VolumeSource{
 				PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{
-					ClaimName: getMonitorObjectName(monitor),
+					ClaimName: GetMonitorObjectName(monitor),
 				},
 			},
 		}
@@ -588,7 +583,7 @@ func getMonitorVolumes(config *core.ConfigMap, monitor *v1alpha1.TidbMonitor, tc
 			VolumeSource: core.VolumeSource{
 				ConfigMap: &core.ConfigMapVolumeSource{
 					LocalObjectReference: core.LocalObjectReference{
-						Name: getMonitorObjectName(monitor),
+						Name: GetMonitorObjectName(monitor),
 					},
 					Items: []core.KeyToPath{
 						{
@@ -632,7 +627,8 @@ func getMonitorVolumes(config *core.ConfigMap, monitor *v1alpha1.TidbMonitor, tc
 
 func getMonitorService(monitor *v1alpha1.TidbMonitor) []*core.Service {
 	var services []*core.Service
-	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
+	monitorLabel := label.New().Instance(monitor.Name).Monitor()
+	labels := label.NewMonitor().Instance(monitor.Name).Monitor()
 	prometheusService := &core.Service{
 		ObjectMeta: meta.ObjectMeta{
 			Name:            fmt.Sprintf("%s-prometheus", monitor.Name),
@@ -650,11 +646,8 @@ func getMonitorService(monitor *v1alpha1.TidbMonitor) []*core.Service {
 					TargetPort: intstr.FromInt(9090),
 				},
 			},
-			Type: monitor.Spec.Prometheus.Service.Type,
-			Selector: map[string]string{
-				label.InstanceLabelKey:  monitor.Name,
-				label.ComponentLabelKey: label.TiDBMonitorVal,
-			},
+			Type:     monitor.Spec.Prometheus.Service.Type,
+			Selector: labels,
 		},
 	}
 	reloaderService := &core.Service{
@@ -716,7 +709,7 @@ func getMonitorPVC(monitor *v1alpha1.TidbMonitor) *core.PersistentVolumeClaim {
 	monitorLabel := label.New().Instance(monitor.Name).Monitor().Labels()
 	return &core.PersistentVolumeClaim{
 		ObjectMeta: meta.ObjectMeta{
-			Name:        getMonitorObjectName(monitor),
+			Name:        GetMonitorObjectName(monitor),
 			Namespace:   monitor.Namespace,
 			Labels:      monitorLabel,
 			Annotations: monitor.Spec.Annotations,

diff --git a/tests/actions.go b/tests/actions.go
@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os/exec"
 	"path/filepath"
 	"reflect"
@@ -2136,112 +2135,18 @@ func (oa *operatorActions) checkTiKVConfigUpdated(tc *v1alpha1.TidbCluster, clus
 func (oa *operatorActions) checkPrometheus(clusterInfo *TidbClusterConfig) error {
 	ns := clusterInfo.Namespace
 	tcName := clusterInfo.ClusterName
-	var prometheusAddr string
-	if oa.fw != nil {
-		localHost, localPort, cancel, err := portforward.ForwardOnePort(oa.fw, ns, fmt.Sprintf("svc/%s-prometheus", tcName), 9090)
-		if err != nil {
-			return err
-		}
-		defer cancel()
-		prometheusAddr = fmt.Sprintf("%s:%d", localHost, localPort)
-	} else {
-		prometheusAddr = fmt.Sprintf("%s-prometheus.%s:9090", tcName, ns)
-	}
-	prometheusSvc := fmt.Sprintf("http://%s/api/v1/query?query=up", prometheusAddr)
-	resp, err := http.Get(prometheusSvc)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	response := &struct {
-		Status string `json:"status"`
-	}{}
-	err = json.Unmarshal(body, response)
-	if err != nil {
-		return err
-	}
-	if response.Status != "success" {
-		return fmt.Errorf("the prometheus's api[%s] has not ready", prometheusSvc)
-	}
-	return nil
+	return checkPrometheusCommon(tcName, ns, oa.fw)
 }
 
 func (oa *operatorActions) checkGrafanaData(clusterInfo *TidbClusterConfig) error {
 	ns := clusterInfo.Namespace
 	tcName := clusterInfo.ClusterName
-	svcName := fmt.Sprintf("%s-grafana", tcName)
-	end := time.Now()
-	start := end.Add(-time.Minute)
-	values := url.Values{}
-	values.Set("query", "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))")
-	values.Set("start", fmt.Sprintf("%d", start.Unix()))
-	values.Set("end", fmt.Sprintf("%d", end.Unix()))
-	values.Set("step", "30")
-
-	var addr string
-	if oa.fw != nil {
-		localHost, localPort, cancel, err := portforward.ForwardOnePort(oa.fw, ns, fmt.Sprintf("svc/%s-prometheus", tcName), 3000)
-		if err != nil {
-			return err
-		}
-		defer cancel()
-		addr = fmt.Sprintf("%s:%d", localHost, localPort)
-	} else {
-		addr = fmt.Sprintf("%s.%s.svc.cluster.local:3000", svcName, ns)
-	}
-
-	datasourceID, err := getDatasourceID(addr)
-	if err != nil {
-		return err
-	}
-
-	u := fmt.Sprintf("http://%s/api/datasources/proxy/%d/api/v1/query_range?%s", addr, datasourceID, values.Encode())
-	req, err := http.NewRequest(http.MethodGet, u, nil)
-	if err != nil {
-		return err
-	}
-	req.SetBasicAuth(grafanaUsername, grafanaPassword)
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	grafanaClient, err := checkGrafanaDataCommon(tcName, ns, clusterInfo.GrafanaClient, oa.fw)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-	buf, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	data := struct {
-		Status string `json:"status"`
-		Data   struct {
-			ResultType string `json:"resultType"`
-			Result     []struct {
-				Metric struct {
-					Job string `json:"job"`
-				} `json:"metric"`
-				Values []interface{} `json:"values"`
-			} `json:"result"`
-		}
-	}{}
-	if err := json.Unmarshal(buf, &data); err != nil {
-		return err
-	}
-	if data.Status != "success" || len(data.Data.Result) < 1 {
-		return fmt.Errorf("invalid response: status: %s, result: %v", data.Status, data.Data.Result)
-	}
-
-	// Grafana ready, init grafana client, no more sync logic because race condition is okay here
-	if clusterInfo.GrafanaClient == nil {
-		grafanaURL := fmt.Sprintf("http://%s.%s:3000", svcName, ns)
-		client, err := metrics.NewClient(grafanaURL, grafanaUsername, grafanaPassword)
-		if err != nil {
-			return err
-		}
-		clusterInfo.GrafanaClient = client
+	if clusterInfo.GrafanaClient == nil && grafanaClient != nil {
+		clusterInfo.GrafanaClient = grafanaClient
 	}
 	return nil
 }

diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go
@@ -75,6 +75,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	var ocfg *tests.OperatorConfig
 	var genericCli client.Client
 	var fwCancel context.CancelFunc
+	var fw portforward.PortForward
 
 	ginkgo.BeforeEach(func() {
 		ns = f.Namespace.Name
@@ -91,7 +92,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		clientRawConfig, err := e2econfig.LoadClientRawConfig()
 		framework.ExpectNoError(err, "failed to load raw config")
 		ctx, cancel := context.WithCancel(context.Background())
-		fw, err := portforward.NewPortForwarder(ctx, e2econfig.NewSimpleRESTClientGetter(clientRawConfig))
+		fw, err = portforward.NewPortForwarder(ctx, e2econfig.NewSimpleRESTClientGetter(clientRawConfig))
 		framework.ExpectNoError(err, "failed to create port forwarder")
 		fwCancel = cancel
 		cfg = e2econfig.TestConfig
@@ -708,6 +709,24 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 15*time.Second)
 		framework.ExpectNoError(err, "Expected TiDB cluster scaled in and ready")
 	})
+
+	ginkgo.It("TidbMonitor: Deploying and checking monitor", func() {
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "monitor-test", "admin", "")
+		cluster.Resources["pd.replicas"] = "1"
+		cluster.Resources["tikv.replicas"] = "1"
+		cluster.Resources["tidb.replicas"] = "1"
+		oa.DeployTidbClusterOrDie(&cluster)
+		oa.CheckTidbClusterStatusOrDie(&cluster)
+
+		tc, err := cli.PingcapV1alpha1().TidbClusters(cluster.Namespace).Get(cluster.ClusterName, metav1.GetOptions{})
+		framework.ExpectNoError(err, "Expected get tidbcluster")
+
+		tm := fixture.NewTidbMonitor("e2e-monitor", tc.Namespace, tc, true, false)
+		_, err = cli.PingcapV1alpha1().TidbMonitors(tc.Namespace).Create(tm)
+		framework.ExpectNoError(err, "Expected tidbmonitor deployed success")
+		err = tests.CheckTidbMonitor(tm, c, fw)
+		framework.ExpectNoError(err, "Expected tidbmonitor checked success")
+	})
 })
 
 func newTidbClusterConfig(cfg *tests.Config, ns, clusterName, password, tidbVersion string) tests.TidbClusterConfig {