diff --git a/.gitignore b/.gitignore index 08008893fb2..b661d496c43 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ images/tidb-operator-e2e/tidb-operator/ tmp/ data/ .idea +cscope.files +tags +filenametags # ginkgo test coverage *.coverprofile diff --git a/pkg/label/label.go b/pkg/label/label.go index cdd529b4ab8..e453027d92d 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -14,8 +14,10 @@ package label import ( + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "strings" ) const ( @@ -138,3 +140,14 @@ func (l Label) LabelSelector() *metav1.LabelSelector { func (l Label) Labels() map[string]string { return l } + +// String converts label to a string +func (l Label) String() string { + var arr []string + + for k, v := range l { + arr = append(arr, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(arr, ",") +} diff --git a/tests/actions.go b/tests/actions.go index 6636337c225..bcbcca63259 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -21,6 +21,7 @@ import ( "strings" "time" + _ "github.com/go-sql-driver/mysql" "github.com/golang/glog" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" @@ -43,6 +44,11 @@ func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface) O } } +const ( + DefaultPollTimeout time.Duration = 10 * time.Minute + DefaultPollInterval time.Duration = 1 * time.Minute +) + type OperatorActions interface { DeployOperator(info *OperatorInfo) error CleanOperator(info *OperatorInfo) error @@ -56,14 +62,16 @@ type OperatorActions interface { ScaleTidbCluster(info *TidbClusterInfo) error UpgradeTidbCluster(info *TidbClusterInfo) error DeployAdHocBackup(info *TidbClusterInfo) error - CleanAdHocBackup(info *TidbClusterInfo) error + CheckAdHocBackup(info *TidbClusterInfo) error DeployScheduledBackup(info *TidbClusterInfo) error - CleanScheduledBackup(info *TidbClusterInfo) error - DeployIncrementalBackup(info *TidbClusterInfo) error - CleanIncrementalBackup(info *TidbClusterInfo) error - Restore(from *TidbClusterInfo, jobName string, to *TidbClusterInfo) error + CheckScheduledBackup(info *TidbClusterInfo) error + DeployIncrementalBackup(from *TidbClusterInfo, to *TidbClusterInfo) error + CheckIncrementalBackup(info *TidbClusterInfo) error + Restore(from *TidbClusterInfo, to *TidbClusterInfo) error + CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error DeployMonitor(info *TidbClusterInfo) error CleanMonitor(info *TidbClusterInfo) error + ForceDeploy(info *TidbClusterInfo) error } type FaultTriggerActions interface { @@ -204,6 +212,10 @@ func (oa *operatorActions) DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClus } func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error { + glog.Infof("begin to deploy tidb cluster") + defer func() { + glog.Infof("deploy tidb cluster end") + }() cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s", info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString()) if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { @@ -215,10 +227,13 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error { } func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error { + glog.Infof("begin to clean tidb cluster") + defer func() { + glog.Infof("clean tidb cluster end") + }() charts := []string{ info.ClusterName, fmt.Sprintf("%s-backup", info.ClusterName), - fmt.Sprintf("%s-restore", info.ClusterName), } for _, chartName := range charts { res, err := exec.Command("helm", "del", "--purge", chartName).CombinedOutput() @@ -228,10 +243,12 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error { } } - resources := []string{"cronjobs", "jobs", "pods", "pvc"} + setStr := label.New().Instance(info.ClusterName).String() + + resources := []string{"pvc"} for _, resource := range resources { - if res, err := exec.Command("kubectl", "delete", resource, "-n", info.Namespace, - "--all").CombinedOutput(); err != nil { + if res, err := exec.Command("kubectl", "delete", resource, "-n", info.Namespace, "-l", + setStr).CombinedOutput(); err != nil { return fmt.Errorf("failed to delete %s: %v, %s", resource, err, string(res)) } } @@ -245,7 +262,7 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error { } pollFn := func() (bool, error) { - if res, err := exec.Command("kubectl", "get", "po", "--output=name", "-n", info.Namespace). + if res, err := exec.Command("kubectl", "get", "po", "--output=name", "-n", info.Namespace, "-l", setStr). CombinedOutput(); err != nil || len(res) != 0 { glog.Infof("waiting for tidbcluster: %s/%s pods deleting, %v, [%s]", info.Namespace, info.ClusterName, err, string(res)) @@ -262,16 +279,19 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error { info.Namespace, info.ClusterName, err, string(res)) return false, nil } - return true, nil } - return wait.PollImmediate(1*time.Minute, 5*time.Minute, pollFn) + return wait.PollImmediate(DefaultPollInterval, DefaultPollTimeout, pollFn) } func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterInfo) error { + glog.Infof("begin to check tidb cluster") + defer func() { + glog.Infof("check tidb cluster end") + }() ns := info.Namespace tcName := info.ClusterName - if err := wait.PollImmediate(1*time.Minute, 10*time.Minute, func() (bool, error) { + if err := wait.PollImmediate(DefaultPollInterval, DefaultPollTimeout, func() (bool, error) { var tc *v1alpha1.TidbCluster var err error if tc, err = oa.cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{}); err != nil { @@ -285,20 +305,30 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterInfo) error { if b, err := oa.tikvMembersReadyFn(tc); !b && err == nil { return false, nil } + + glog.Infof("check tidb cluster begin tidbMembersReadyFn") if b, err := oa.tidbMembersReadyFn(tc); !b && err == nil { return false, nil } + + glog.Infof("check tidb cluster begin reclaimPolicySyncFn") if b, err := oa.reclaimPolicySyncFn(tc); !b && err == nil { return false, nil } + + glog.Infof("check tidb cluster begin metaSyncFn") if b, err := oa.metaSyncFn(tc); err != nil { return false, err } else if !b && err == nil { return false, nil } + + glog.Infof("check tidb cluster begin schedulerHAFn") if b, err := oa.schedulerHAFn(tc); !b && err == nil { return false, nil } + + glog.Infof("check tidb cluster begin passwordIsSet") if b, err := oa.passwordIsSet(info); !b && err == nil { return false, nil } @@ -319,19 +349,10 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) CleanAdHocBackup(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) CleanScheduledBackup(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) DeployIncrementalBackup(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) CleanIncrementalBackup(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) Restore(from *TidbClusterInfo, jobName string, to *TidbClusterInfo) error { - return nil -} -func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error { return nil } -func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error { return nil } +func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error { return nil } +func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error { return nil } +func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error { return nil } +func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error { return nil } func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, error) { tcName := tc.GetName() @@ -729,7 +750,7 @@ func (oa *operatorActions) schedulerHAFn(tc *v1alpha1.TidbCluster) (bool, error) nodeName, len(nodeMap[nodeName]), totalCount) } } - return false, nil + return true, nil } components := []string{label.PDLabelVal, label.TiKVLabelVal} @@ -811,3 +832,40 @@ func checkoutTag(tagName string) error { return nil } + +func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) CheckScheduledBackup(info *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) CheckAdHocBackup(info *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterInfo) error { + return nil + +} + +func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error { + return nil +} + +func (oa *operatorActions) ForceDeploy(info *TidbClusterInfo) error { + return nil +} diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index 46af075764e..d88878ad59f 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -79,4 +79,26 @@ func main() { if err := oa.CheckTidbClusterStatus(clusterInfo); err != nil { glog.Fatal(err) } + + restoreClusterInfo := &tests.TidbClusterInfo{ + Namespace: "tidb", + ClusterName: "demo2", + OperatorTag: "v1.0.0-beta.1-p2", + PDImage: "pingcap/pd:v2.1.3", + TiKVImage: "pingcap/tikv:v2.1.3", + TiDBImage: "pingcap/tidb:v2.1.3", + StorageClassName: "local-storage", + Password: "admin", + Args: map[string]string{}, + } + + if err := oa.CleanTidbCluster(restoreClusterInfo); err != nil { + glog.Fatal(err) + } + if err := oa.DeployTidbCluster(restoreClusterInfo); err != nil { + glog.Fatal(err) + } + if err := oa.CheckTidbClusterStatus(restoreClusterInfo); err != nil { + glog.Fatal(err) + } }