From 9e74eacf490432f9e4b3ea8e153b652f0fee2dab Mon Sep 17 00:00:00 2001
From: weekface <weekface@gmail.com>
Date: Wed, 25 Sep 2019 17:03:31 +0800
Subject: [PATCH] ensure pods unchanged when upgrading

---
 tests/actions.go            | 43 ++++++++++++++++++++++++++++++++++++-
 tests/cluster_info.go       |  2 +-
 tests/cmd/stability/main.go |  1 -
 3 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/tests/actions.go b/tests/actions.go
index e3389f5055e..c655eb6cd69 100644
--- a/tests/actions.go
+++ b/tests/actions.go
@@ -26,6 +26,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -457,6 +458,15 @@ func (oa *operatorActions) CleanOperatorOrDie(info *OperatorConfig) {
 
 func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error {
 	glog.Infof("upgrading tidb-operator %s", info.ReleaseName)
+
+	listOptions := metav1.ListOptions{
+		LabelSelector: labels.SelectorFromSet(
+			label.New().Labels()).String(),
+	}
+	pods1, err := oa.kubeCli.CoreV1().Pods(metav1.NamespaceAll).List(listOptions)
+	if err != nil {
+		return err
+	}
 	if err := oa.checkoutTag(info.Tag); err != nil {
 		return err
 	}
@@ -469,7 +479,38 @@ func (oa *operatorActions) UpgradeOperator(info *OperatorConfig) error {
 	if err != nil {
 		return fmt.Errorf("failed to upgrade operator to: %s, %v, %s", info.Image, err, string(res))
 	}
-	return nil
+
+	time.Sleep(5 * time.Minute)
+	pods2, err := oa.kubeCli.CoreV1().Pods(metav1.NamespaceAll).List(listOptions)
+	if err != nil {
+		return err
+	}
+
+	return ensurePodsUnchanged(pods1, pods2)
+}
+
+func ensurePodsUnchanged(pods1, pods2 *corev1.PodList) error {
+	if reflect.DeepEqual(getUID(pods1), getUID(pods2)) {
+		glog.V(4).Infof("%#v", pods1)
+		glog.V(4).Infof("%#v", pods2)
+		glog.V(4).Infof("pods unchanged after operator upgraded")
+		return nil
+	}
+
+	glog.Infof("%#v", pods1)
+	glog.Infof("%#v", pods2)
+	return fmt.Errorf("some pods changed after operator upgraded")
+}
+
+func getUID(pods *corev1.PodList) []string {
+	arr := make([]string, 0, len(pods.Items))
+
+	for _, pod := range pods.Items {
+		arr = append(arr, string(pod.UID))
+	}
+
+	sort.Strings(arr)
+	return arr
 }
 
 func (oa *operatorActions) UpgradeOperatorOrDie(info *OperatorConfig) {
diff --git a/tests/cluster_info.go b/tests/cluster_info.go
index 5075f5cfc27..36da5473b2e 100644
--- a/tests/cluster_info.go
+++ b/tests/cluster_info.go
@@ -148,6 +148,6 @@ func (tc *TidbClusterConfig) BuildSubValues(path string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	glog.Infof("subValues:\n %s", subValues)
+	glog.V(4).Infof("subValues:\n %s", subValues)
 	return subVaulesPath, nil
 }
diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go
index d801f1a1417..74d951d2494 100644
--- a/tests/cmd/stability/main.go
+++ b/tests/cmd/stability/main.go
@@ -318,7 +318,6 @@ func run() {
 		ocfg.Image = cfg.UpgradeOperatorImage
 		ocfg.Tag = cfg.UpgradeOperatorTag
 		oa.UpgradeOperatorOrDie(ocfg)
-		time.Sleep(5 * time.Minute)
 		postUpgrade := []*tests.TidbClusterConfig{
 			cluster3,
 			cluster1,