Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New stability test case for ConfigMap rollout #499

Merged
merged 22 commits into from
May 24, 2019
Merged

New stability test case for ConfigMap rollout #499

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
2aff5a4
Support updating configuraion on the fly
aylei May 9, 2019
e99c172
Fix helm char configmap indent
aylei May 9, 2019
53c2c0e
fix tikv configmap generation
aylei May 9, 2019
edb5311
Fix configmap indent
aylei May 10, 2019
2772248
Fix initializr job
aylei May 10, 2019
ac86729
Make intializer job as post-install hook
aylei May 10, 2019
040612c
Compatibility with legacy configmap when deleting
aylei May 10, 2019
9465dbd
Merge branch 'master' into issue-225
aylei May 12, 2019
5e819d6
Skip PD configuration check in favor of #487, add tidb configuration …
aylei May 13, 2019
10a263f
Merge branch 'master' into issue-225
tennix May 14, 2019
ada8c94
Update tests/actions.go
aylei May 14, 2019
a993c4c
Add new configuration update case for PD log level
aylei May 15, 2019
67f9772
Merge branch 'issue-225'
aylei May 15, 2019
19eb9b9
Add stability test case for configmap rollout
aylei May 17, 2019
a9cffd2
Lint codes
aylei May 17, 2019
4bbf51c
Merge master
aylei May 17, 2019
6ad6358
Merge branch 'master' into issue-486
aylei May 17, 2019
a93d83f
Delete merge error
aylei May 17, 2019
6d58259
Merge branch 'master' into issue-486
weekface May 23, 2019
875040a
Restore imagePullPolicy to IfNotPresent in e2e test
aylei May 24, 2019
ba77209
Merge branch 'issue-486' of github.com:aylei/tidb-operator into issue…
aylei May 24, 2019
c44a2fc
Merge branch 'master' into issue-486
weekface May 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
#

set -uo pipefail

{{ .Values.pd.preStartScript }}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For fault-injection


ANNOTATIONS="/etc/podinfo/annotations"

if [[ ! -f "${ANNOTATIONS}" ]]
Expand Down
3 changes: 3 additions & 0 deletions charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
# runmode="normal/debug"
#
set -uo pipefail

{{ .Values.tidb.preStartScript }}

ANNOTATIONS="/etc/podinfo/annotations"

if [[ ! -f "${ANNOTATIONS}" ]]
Expand Down
3 changes: 3 additions & 0 deletions charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
#

set -uo pipefail

{{ .Values.tikv.preStartScript }}

ANNOTATIONS="/etc/podinfo/annotations"

if [[ ! -f "${ANNOTATIONS}" ]]
Expand Down
11 changes: 10 additions & 1 deletion tests/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ type OperatorActions interface {
CheckK8sAvailableOrDie(excludeNodes map[string]string, excludePods map[string]*corev1.Pod)
CheckOperatorAvailable(operatorConfig *OperatorConfig) error
CheckTidbClustersAvailable(infos []*TidbClusterConfig) error
CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig)
CheckOneEtcdDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string)
CheckOneApiserverDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string)
RegisterWebHookAndService(info *OperatorConfig) error
Expand Down Expand Up @@ -232,6 +233,10 @@ type TidbClusterConfig struct {
TiDBTokenLimit int
PDLogLevel string

PDPreStartScript string
TiDBPreStartScript string
TiKVPreStartScript string

BlockWriteConfig blockwriter.Config
GrafanaClient *metrics.Client
}
Expand Down Expand Up @@ -287,6 +292,9 @@ func (tc *TidbClusterConfig) TidbClusterHelmSetString(m map[string]string) strin
"tidb.initSql": tc.InitSQL,
"monitor.create": strconv.FormatBool(tc.Monitor),
"enableConfigMapRollout": strconv.FormatBool(tc.EnableConfigMapRollout),
"pd.preStartScript": tc.PDPreStartScript,
"tikv.preStartScript": tc.TiKVPreStartScript,
"tidb.preStartScript": tc.TiDBPreStartScript,
}

if tc.PDMaxReplicas > 0 {
Expand Down Expand Up @@ -328,6 +336,7 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string {
"scheduler.logLevel": "2",
"controllerManager.replicas": "2",
"scheduler.replicas": "2",
"imagePullPolicy": "Always",
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typically, we will use the latest tag of operator image. Always can avoid using stale image

}
if oi.SchedulerTag != "" {
set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag
Expand Down Expand Up @@ -1593,7 +1602,7 @@ func releaseIsNotFound(err error) bool {
}

func (oa *operatorActions) cloneOperatorRepo() error {
cmd := fmt.Sprintf("git clone https://github.com/pingcap/tidb-operator.git %s", oa.cfg.OperatorRepoDir)
cmd := fmt.Sprintf("git clone %s %s", oa.cfg.OperatorRepoUrl, oa.cfg.OperatorRepoDir)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the developers modify the chart in their development branch, they have to configure the git url in order to test the chart modification

glog.Info(cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil && !strings.Contains(string(res), "already exists") {
Expand Down
49 changes: 44 additions & 5 deletions tests/cmd/stability/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"fmt"
"net/http"
_ "net/http/pprof"
"strconv"
"time"

"github.com/pingcap/tidb-operator/tests/slack"
Expand Down Expand Up @@ -98,8 +99,13 @@ func main() {
"binlog.drainer.workerCount": "1024",
"binlog.drainer.txnBatch": "512",
},
Monitor: true,
BlockWriteConfig: conf.BlockWriter,
Monitor: true,
BlockWriteConfig: conf.BlockWriter,
PDMaxReplicas: 3,
TiKVGrpcConcurrency: 4,
TiDBTokenLimit: 1000,
PDLogLevel: "info",
EnableConfigMapRollout: true,
}
cluster2 := &tests.TidbClusterConfig{
Namespace: clusterName2,
Expand Down Expand Up @@ -132,9 +138,14 @@ func main() {
"monitor.persistent": "true",
"discovery.image": conf.OperatorImage,
},
Args: map[string]string{},
Monitor: true,
BlockWriteConfig: conf.BlockWriter,
Args: map[string]string{},
Monitor: true,
BlockWriteConfig: conf.BlockWriter,
PDMaxReplicas: 3,
TiKVGrpcConcurrency: 4,
TiDBTokenLimit: 1000,
PDLogLevel: "info",
EnableConfigMapRollout: false,
}

// cluster backup and restore
Expand Down Expand Up @@ -201,6 +212,34 @@ func main() {
oa.CheckTidbClusterStatusOrDie(cluster1)
oa.CheckTidbClusterStatusOrDie(cluster2)

// cluster1: bad configuration change case
cluster1.TiDBPreStartScript = strconv.Quote("exit 1")
oa.UpgradeTidbClusterOrDie(cluster1)
cluster1.TiKVPreStartScript = strconv.Quote("exit 1")
oa.UpgradeTidbClusterOrDie(cluster1)
cluster1.PDPreStartScript = strconv.Quote("exit 1")
oa.UpgradeTidbClusterOrDie(cluster1)

time.Sleep(30 * time.Second)
oa.CheckTidbClustersAvailableOrDie([]*tests.TidbClusterConfig{cluster1})

// rollback cluster1
cluster1.PDPreStartScript = strconv.Quote("")
cluster1.TiKVPreStartScript = strconv.Quote("")
cluster1.TiDBPreStartScript = strconv.Quote("")
oa.UpgradeTidbClusterOrDie(cluster1)
oa.CheckTidbClusterStatusOrDie(cluster1)

// cluster2: enable and normal configuration change case
cluster2.EnableConfigMapRollout = true
oa.UpgradeTidbClusterOrDie(cluster2)
oa.CheckTidbClusterStatusOrDie(cluster2)
cluster2.UpdatePdMaxReplicas(conf.PDMaxReplicas).
UpdateTiKVGrpcConcurrency(conf.TiKVGrpcConcurrency).
UpdateTiDBTokenLimit(conf.TiDBTokenLimit)
oa.UpgradeTidbClusterOrDie(cluster2)
oa.CheckTidbClusterStatusOrDie(cluster2)

// after upgrade cluster, clean webhook
oa.CleanWebHookAndService(operatorCfg)

Expand Down
2 changes: 2 additions & 0 deletions tests/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type Config struct {
BlockWriter blockwriter.Config `yaml:"block_writer,omitempty"`

// For local test
OperatorRepoUrl string `yaml:"operator_repo_url" json:"operator_repo_url"`
OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"`
// chart dir
ChartDir string `yaml:"chart_dir" json:"chart_dir"`
Expand All @@ -59,6 +60,7 @@ type Nodes struct {
// NewConfig creates a new config.
func NewConfig() (*Config, error) {
cfg := &Config{
OperatorRepoUrl: "https://github.com/pingcap/tidb-operator.git",

PDMaxReplicas: 5,
TiDBTokenLimit: 1024,
Expand Down
6 changes: 6 additions & 0 deletions tests/failover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,12 @@ func (oa *operatorActions) CheckTidbClustersAvailable(infos []*TidbClusterConfig

}

func (oa *operatorActions) CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig) {
if err := oa.CheckTidbClustersAvailable(infos); err != nil {
slack.NotifyAndPanic(err)
}
}

var testTableName = "testTable"

func (oa *operatorActions) addDataToCluster(info *TidbClusterConfig) (bool, error) {
Expand Down