From cc0d2d8e66e0ebb50a95910161605f8b4bc11e1a Mon Sep 17 00:00:00 2001 From: Aylei Date: Fri, 24 May 2019 19:20:59 +0800 Subject: [PATCH] New stability test case for ConfigMap rollout (#499) --- .../templates/scripts/_start_pd.sh.tpl | 3 ++ .../templates/scripts/_start_tidb.sh.tpl | 3 ++ .../templates/scripts/_start_tikv.sh.tpl | 3 ++ tests/actions.go | 12 ++++- tests/cmd/e2e/main.go | 2 + tests/cmd/stability/main.go | 51 +++++++++++++++++-- tests/config.go | 2 + tests/failover.go | 6 +++ 8 files changed, 76 insertions(+), 6 deletions(-) diff --git a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl index 53bba5a31ba..a628e9741f1 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl @@ -9,6 +9,9 @@ # set -uo pipefail + +{{ .Values.pd.preStartScript }} + ANNOTATIONS="/etc/podinfo/annotations" if [[ ! -f "${ANNOTATIONS}" ]] diff --git a/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl index 4379159a7cd..ee79a7594ec 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl @@ -8,6 +8,9 @@ # runmode="normal/debug" # set -uo pipefail + +{{ .Values.tidb.preStartScript }} + ANNOTATIONS="/etc/podinfo/annotations" if [[ ! -f "${ANNOTATIONS}" ]] diff --git a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl index 4c95975146a..8345ffed6c5 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl @@ -9,6 +9,9 @@ # set -uo pipefail + +{{ .Values.tikv.preStartScript }} + ANNOTATIONS="/etc/podinfo/annotations" if [[ ! -f "${ANNOTATIONS}" ]] diff --git a/tests/actions.go b/tests/actions.go index d92f61e4540..26293d3e4aa 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -151,6 +151,7 @@ type OperatorActions interface { CheckK8sAvailableOrDie(excludeNodes map[string]string, excludePods map[string]*corev1.Pod) CheckOperatorAvailable(operatorConfig *OperatorConfig) error CheckTidbClustersAvailable(infos []*TidbClusterConfig) error + CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig) CheckOneEtcdDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string) CheckOneApiserverDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string) RegisterWebHookAndService(info *OperatorConfig) error @@ -202,6 +203,7 @@ type OperatorConfig struct { WebhookSecretName string WebhookConfigName string Context *apimachinery.CertContext + ImagePullPolicy corev1.PullPolicy } type TidbClusterConfig struct { @@ -231,6 +233,10 @@ type TidbClusterConfig struct { TiDBTokenLimit int PDLogLevel string + PDPreStartScript string + TiDBPreStartScript string + TiKVPreStartScript string + BlockWriteConfig blockwriter.Config GrafanaClient *metrics.Client } @@ -286,6 +292,9 @@ func (tc *TidbClusterConfig) TidbClusterHelmSetString(m map[string]string) strin "tidb.initSql": tc.InitSQL, "monitor.create": strconv.FormatBool(tc.Monitor), "enableConfigMapRollout": strconv.FormatBool(tc.EnableConfigMapRollout), + "pd.preStartScript": tc.PDPreStartScript, + "tikv.preStartScript": tc.TiKVPreStartScript, + "tidb.preStartScript": tc.TiDBPreStartScript, } if tc.PDMaxReplicas > 0 { @@ -327,6 +336,7 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string { "scheduler.logLevel": "2", "controllerManager.replicas": "2", "scheduler.replicas": "2", + "imagePullPolicy": string(oi.ImagePullPolicy), } if oi.SchedulerTag != "" { set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag @@ -1583,7 +1593,7 @@ func notFound(res string) bool { } func (oa *operatorActions) cloneOperatorRepo() error { - cmd := fmt.Sprintf("git clone https://github.com/pingcap/tidb-operator.git %s", oa.cfg.OperatorRepoDir) + cmd := fmt.Sprintf("git clone %s %s", oa.cfg.OperatorRepoUrl, oa.cfg.OperatorRepoDir) glog.Info(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil && !strings.Contains(string(res), "already exists") { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index 7aa143fb8fc..ea16dc50f15 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -15,6 +15,7 @@ package main import ( "fmt" + "k8s.io/api/core/v1" _ "net/http/pprof" "time" @@ -51,6 +52,7 @@ func main() { WebhookServiceName: "webhook-service", WebhookSecretName: "webhook-secret", WebhookConfigName: "webhook-config", + ImagePullPolicy: v1.PullIfNotPresent, } // start a http server in goruntine diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index f583489e2c5..aee2c4fc67e 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -15,8 +15,10 @@ package main import ( "fmt" + "k8s.io/api/core/v1" "net/http" _ "net/http/pprof" + "strconv" "time" "github.com/pingcap/tidb-operator/tests/slack" @@ -55,6 +57,7 @@ func main() { WebhookServiceName: "webhook-service", WebhookSecretName: "webhook-secret", WebhookConfigName: "webhook-config", + ImagePullPolicy: v1.PullAlways, } // TODO remove this @@ -98,8 +101,13 @@ func main() { "binlog.drainer.workerCount": "1024", "binlog.drainer.txnBatch": "512", }, - Monitor: true, - BlockWriteConfig: conf.BlockWriter, + Monitor: true, + BlockWriteConfig: conf.BlockWriter, + PDMaxReplicas: 3, + TiKVGrpcConcurrency: 4, + TiDBTokenLimit: 1000, + PDLogLevel: "info", + EnableConfigMapRollout: true, } cluster2 := &tests.TidbClusterConfig{ Namespace: clusterName2, @@ -132,9 +140,14 @@ func main() { "monitor.persistent": "true", "discovery.image": conf.OperatorImage, }, - Args: map[string]string{}, - Monitor: true, - BlockWriteConfig: conf.BlockWriter, + Args: map[string]string{}, + Monitor: true, + BlockWriteConfig: conf.BlockWriter, + PDMaxReplicas: 3, + TiKVGrpcConcurrency: 4, + TiDBTokenLimit: 1000, + PDLogLevel: "info", + EnableConfigMapRollout: false, } // cluster backup and restore @@ -201,6 +214,34 @@ func main() { oa.CheckTidbClusterStatusOrDie(cluster1) oa.CheckTidbClusterStatusOrDie(cluster2) + // cluster1: bad configuration change case + cluster1.TiDBPreStartScript = strconv.Quote("exit 1") + oa.UpgradeTidbClusterOrDie(cluster1) + cluster1.TiKVPreStartScript = strconv.Quote("exit 1") + oa.UpgradeTidbClusterOrDie(cluster1) + cluster1.PDPreStartScript = strconv.Quote("exit 1") + oa.UpgradeTidbClusterOrDie(cluster1) + + time.Sleep(30 * time.Second) + oa.CheckTidbClustersAvailableOrDie([]*tests.TidbClusterConfig{cluster1}) + + // rollback cluster1 + cluster1.PDPreStartScript = strconv.Quote("") + cluster1.TiKVPreStartScript = strconv.Quote("") + cluster1.TiDBPreStartScript = strconv.Quote("") + oa.UpgradeTidbClusterOrDie(cluster1) + oa.CheckTidbClusterStatusOrDie(cluster1) + + // cluster2: enable and normal configuration change case + cluster2.EnableConfigMapRollout = true + oa.UpgradeTidbClusterOrDie(cluster2) + oa.CheckTidbClusterStatusOrDie(cluster2) + cluster2.UpdatePdMaxReplicas(conf.PDMaxReplicas). + UpdateTiKVGrpcConcurrency(conf.TiKVGrpcConcurrency). + UpdateTiDBTokenLimit(conf.TiDBTokenLimit) + oa.UpgradeTidbClusterOrDie(cluster2) + oa.CheckTidbClusterStatusOrDie(cluster2) + // after upgrade cluster, clean webhook oa.CleanWebHookAndService(operatorCfg) diff --git a/tests/config.go b/tests/config.go index c907a224cf0..f7a2c0ae285 100644 --- a/tests/config.go +++ b/tests/config.go @@ -45,6 +45,7 @@ type Config struct { BlockWriter blockwriter.Config `yaml:"block_writer,omitempty"` // For local test + OperatorRepoUrl string `yaml:"operator_repo_url" json:"operator_repo_url"` OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"` // chart dir ChartDir string `yaml:"chart_dir" json:"chart_dir"` @@ -61,6 +62,7 @@ type Nodes struct { // NewConfig creates a new config. func NewConfig() (*Config, error) { cfg := &Config{ + OperatorRepoUrl: "https://github.com/pingcap/tidb-operator.git", PDMaxReplicas: 5, TiDBTokenLimit: 1024, diff --git a/tests/failover.go b/tests/failover.go index aeb9285d203..798a37def98 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -584,6 +584,12 @@ func (oa *operatorActions) CheckTidbClustersAvailable(infos []*TidbClusterConfig } +func (oa *operatorActions) CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig) { + if err := oa.CheckTidbClustersAvailable(infos); err != nil { + slack.NotifyAndPanic(err) + } +} + var testTableName = "testTable" func (oa *operatorActions) addDataToCluster(info *TidbClusterConfig) (bool, error) {