add notify when call os.Exit

pingcap · Apr 30, 2019 · 4743c3b · 4743c3b
1 parent 852625f
commit 4743c3b
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 12 deletions.
diff --git a/tests/actions.go b/tests/actions.go
@@ -2110,7 +2110,12 @@ func (oa *operatorActions) StartValidatingAdmissionWebhookServerOrDie(info *Oper
 	}
 	err = server.ListenAndServeTLS("", "")
 	if err != nil {
-		glog.Errorf("fail to start webhook server err %v", err)
+		err = fmt.Errorf("fail to start webhook server err %v", err)
+		glog.Error(err)
+		sendErr := slack.SendErrMsg(err.Error())
+		if sendErr != nil {
+			glog.Error(sendErr)
+		}
 		os.Exit(4)
 	}
 }

diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pingcap/tidb-operator/tests/slack"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/golang/glog"
@@ -84,7 +86,7 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 		return &reviewResponse
 	}
 
-	glog.Infof("delete %s pod [%s]", pod.Labels[label.ComponentLabelKey], pod.GetName())
+	glog.Infof("delete pod %s", pod.Labels[label.ComponentLabelKey])
 
 	tc, err := versionCli.PingcapV1alpha1().TidbClusters(namespace).Get(pod.Labels[label.InstanceLabelKey], metav1.GetOptions{})
 	if err != nil {
@@ -95,14 +97,15 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 	pdClient := controller.NewDefaultPDControl().GetPDClient(tc)
 	tidbController := controller.NewDefaultTiDBControl()
 
-	// if the pod is deleting, allow the pod delete operation
-	if pod.DeletionTimestamp != nil {
-		glog.Infof("pod:[%s/%s] status is timestamp %s", namespace, name, pod.DeletionTimestamp)
-		reviewResponse.Allowed = true
-		return &reviewResponse
-	}
-
 	if pod.Labels[label.ComponentLabelKey] == "tidb" {
+
+		// if tidb pod is deleting, allow pod delete operation
+		if pod.DeletionTimestamp != nil {
+			glog.Infof("TIDB pod status is namespace %s name %s timestamp %s", namespace, name, pod.DeletionTimestamp)
+			reviewResponse.Allowed = true
+			return &reviewResponse
+		}
+
 		ordinal, err := strconv.ParseInt(strings.Split(name, "-")[len(strings.Split(name, "-"))-1], 10, 32)
 		if err != nil {
 			glog.Errorf("fail to convert string to int while deleting TiDB err %v", err)
@@ -117,13 +120,19 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 
 		if info.IsOwner && tc.Status.TiDB.StatefulSet.Replicas > 1 {
 			time.Sleep(10 * time.Second)
-			glog.Errorf("tidb is ddl owner, can't be deleted namespace %s name %s", namespace, name)
+			err := fmt.Errorf("tidb is ddl owner, can't be deleted namespace %s name %s", namespace, name)
+			glog.Error(err)
+			sendErr := slack.SendErrMsg(err.Error())
+			if sendErr != nil {
+				glog.Error(sendErr)
+			}
 			os.Exit(3)
 		} else {
 			glog.Infof("savely delete pod namespace %s name %s isowner %t", namespace, name, info.IsOwner)
 		}
 
 	} else if pod.Labels[label.ComponentLabelKey] == "pd" {
+
 		leader, err := pdClient.GetPDLeader()
 		if err != nil {
 			glog.Errorf("fail to get pd leader %v", err)
@@ -132,13 +141,19 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 
 		if leader.Name == name && tc.Status.TiDB.StatefulSet.Replicas > 1 {
 			time.Sleep(10 * time.Second)
-			glog.Errorf("pd is leader, can't be deleted namespace %s name %s", namespace, name)
+			err := fmt.Errorf("pd is leader, can't be deleted namespace %s name %s", namespace, name)
+			glog.Error(err)
+			sendErr := slack.SendErrMsg(err.Error())
+			if sendErr != nil {
+				glog.Error(sendErr)
+			}
 			os.Exit(3)
 		} else {
 			glog.Infof("savely delete pod namespace %s name %s leader name %s", namespace, name, leader.Name)
 		}
 
 	} else if pod.Labels[label.ComponentLabelKey] == "tikv" {
+
 		var storeID uint64
 		storeID = 0
 		for _, store := range tc.Status.TiKV.Stores {
@@ -169,7 +184,12 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
 
 		if beforeCount != 0 && beforeCount <= afterCount && tc.Status.TiKV.StatefulSet.Replicas > 1 {
 			time.Sleep(10 * time.Second)
-			glog.Errorf("kv leader is not zero, can't be deleted namespace %s name %s leaderCount %d", namespace, name, storeInfo.Status.LeaderCount)
+			err := fmt.Errorf("kv leader is not zero, can't be deleted namespace %s name %s leaderCount %d", namespace, name, storeInfo.Status.LeaderCount)
+			glog.Error(err)
+			sendErr := slack.SendErrMsg(err.Error())
+			if sendErr != nil {
+				glog.Error(sendErr)
+			}
 			os.Exit(3)
 		} else {
 			glog.Infof("savely delete pod namespace %s name %s before count %d after count %d", namespace, name, beforeCount, afterCount)