Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Yinliang/backup and restore add adhoc backup and restore functison #316

Merged
merged 4 commits into from
Mar 15, 2019
Merged

Yinliang/backup and restore add adhoc backup and restore functison #316

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
11f8766
add restore and backup process
shuijing198799 Mar 13, 2019
a7dbbd3
clean cluster bug fixed
shuijing198799 Mar 13, 2019
94e5d00
fix error tag name
shuijing198799 Mar 14, 2019
eec22e3
forget a db close, fix it
shuijing198799 Mar 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
269 changes: 239 additions & 30 deletions tests/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package tests

import (
"bytes"
"database/sql"
"fmt"
"os/exec"
Expand Down Expand Up @@ -72,6 +73,7 @@ type OperatorActions interface {
DeployMonitor(info *TidbClusterInfo) error
CleanMonitor(info *TidbClusterInfo) error
ForceDeploy(info *TidbClusterInfo) error
CreateSecret(info *TidbClusterInfo) error
}

type FaultTriggerActions interface {
Expand Down Expand Up @@ -126,16 +128,22 @@ type TidbClusterInfo struct {
}

func (tc *TidbClusterInfo) HelmSetString() string {

// add a database and table for test
initSql := `"create database record;use record;create table test(t char(32));"`

set := map[string]string{
"clusterName": tc.ClusterName,
"pd.storageClassName": tc.StorageClassName,
"tikv.storageClassName": tc.StorageClassName,
"tidb.storageClassName": tc.StorageClassName,
"tidb.password": tc.Password,
"pd.maxStoreDownTime": "5m",
"pd.image": tc.PDImage,
"tikv.image": tc.TiKVImage,
"tidb.image": tc.TiDBImage,
"clusterName": tc.ClusterName,
"pd.storageClassName": tc.StorageClassName,
"tikv.storageClassName": tc.StorageClassName,
"tidb.storageClassName": tc.StorageClassName,
"tidb.password": tc.Password,
"pd.maxStoreDownTime": "5m",
"pd.image": tc.PDImage,
"tikv.image": tc.TiKVImage,
"tidb.image": tc.TiDBImage,
"tidb.passwordSecretName": "set-secret",
"tidb.initSql": initSql,
}

for k, v := range tc.Resources {
Expand Down Expand Up @@ -212,9 +220,9 @@ func (oa *operatorActions) DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClus
}

func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
glog.Infof("begin to deploy tidb cluster")
glog.Infof("begin to deploy tidb cluster cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
defer func() {
glog.Infof("deploy tidb cluster end")
glog.Infof("deploy tidb cluster end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()
cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s",
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString())
Expand All @@ -227,13 +235,14 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
}

func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
glog.Infof("begin to clean tidb cluster")
glog.Infof("begin to clean tidb cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
defer func() {
glog.Infof("clean tidb cluster end")
glog.Infof("clean tidb cluster end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()
charts := []string{
info.ClusterName,
fmt.Sprintf("%s-backup", info.ClusterName),
fmt.Sprintf("%s-restore", info.ClusterName),
}
for _, chartName := range charts {
res, err := exec.Command("helm", "del", "--purge", chartName).CombinedOutput()
Expand Down Expand Up @@ -285,9 +294,9 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
}

func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterInfo) error {
glog.Infof("begin to check tidb cluster")
glog.Infof("begin to check tidb cluster cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
defer func() {
glog.Infof("check tidb cluster end")
glog.Infof("check tidb cluster end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()
ns := info.Namespace
tcName := info.ClusterName
Expand Down Expand Up @@ -816,12 +825,12 @@ func cloneOperatorRepo() error {
}

func checkoutTag(tagName string) error {
cmd := fmt.Sprintf(`cd /tidb-operator;
git stash -u;
git checkout %s;
mkdir -p /charts/%s;
cp -rf charts/tidb-operator /charts/%s/tidb-operator;
cp -rf charts/tidb-cluster /charts/%s/tidb-cluster;
cmd := fmt.Sprintf(`cd /tidb-operator &&
git stash -u &&
git checkout %s &&
mkdir -p /charts/%s &&
cp -rf charts/tidb-operator /charts/%s/tidb-operator &&
cp -rf charts/tidb-cluster /charts/%s/tidb-cluster &&
cp -rf charts/tidb-backup /charts/%s/tidb-backup`,
tagName, tagName, tagName, tagName, tagName)
glog.Info(cmd)
Expand All @@ -833,39 +842,239 @@ func checkoutTag(tagName string) error {
return nil
}

func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error {
func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error {
glog.Infof("begin to deploy adhoc backup cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
defer func() {
glog.Infof("deploy adhoc backup end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()
sets := map[string]string{
"clusterName": info.ClusterName,
"name": "test-backup",
"mode": "backup",
"user": "root",
"password": info.Password,
"storage.size": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup %s",
fullbackupName, info.Namespace, info.OperatorTag, setStr)
glog.Infof("install adhoc deployment [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to launch adhoc backup job: %v, %s", err, string(res))
}
return nil
}

func (oa *operatorActions) CheckScheduledBackup(info *TidbClusterInfo) error {
func (oa *operatorActions) CheckAdHocBackup(info *TidbClusterInfo) error {
glog.Infof("begin to clean adhoc backup cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
defer func() {
glog.Infof("deploy clean backup end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()

jobName := fmt.Sprintf("%s-%s", info.ClusterName, "test-backup")
fn := func() (bool, error) {
job, err := oa.kubeCli.BatchV1().Jobs(info.Namespace).Get(jobName, metav1.GetOptions{})
if err != nil {
glog.Errorf("failed to get jobs %s ,%v", jobName, err)
return false, nil
}
if job.Status.Succeeded == 0 {
glog.Error("cluster [%s] back up job is not completed, please wait! ", info.ClusterName)
return false, nil
}

return true, nil
}

err := wait.Poll(DefaultPollInterval, DefaultPollTimeout, fn)
if err != nil {
return fmt.Errorf("failed to launch scheduler backup job: %v", err)
}
return nil
}

func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error {
func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) error {
glog.Infof("begin to deploy restore cluster[%s] namespace[%s]", from.ClusterName, from.Namespace)
defer func() {
glog.Infof("deploy restore end cluster[%s] namespace[%s]", to.ClusterName, to.Namespace)
}()
sets := map[string]string{
"clusterName": to.ClusterName,
"name": "test-backup",
"mode": "restore",
"user": "root",
"password": to.Password,
"storage.size": "10Gi",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

10Gi may be too small for backup job

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this setting just suit for my cluster, We can modify it during stress testing。

}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
restoreName := fmt.Sprintf("%s-restore", from.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup %s",
restoreName, to.Namespace, to.OperatorTag, setStr)
glog.Infof("install restore [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to launch restore job: %v, %s", err, string(res))
}

return nil
}

func (oa *operatorActions) CheckAdHocBackup(info *TidbClusterInfo) error {
func (oa *operatorActions) CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error {
glog.Infof("begin to check restore backup cluster[%s] namespace[%s]", from.ClusterName, from.Namespace)
defer func() {
glog.Infof("check restore end cluster[%s] namespace[%s]", to.ClusterName, to.Namespace)
}()

jobName := fmt.Sprintf("%s-restore-test-backup", to.ClusterName)
fn := func() (bool, error) {
job, err := oa.kubeCli.BatchV1().Jobs(to.Namespace).Get(jobName, metav1.GetOptions{})
if err != nil {
glog.Errorf("failed to get jobs %s ,%v", jobName, err)
return false, nil
}
if job.Status.Succeeded == 0 {
glog.Error("cluster [%s] back up job is not completed, please wait! ", to.ClusterName)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

glog.Errorf

BTW: I highly recommand zap or our own logger.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

return false, nil
}

fromCount, err := from.QueryCount()
if err != nil {
glog.Error("cluster [%s] count err ", from.ClusterName)
return false, nil
}

toCount, err := to.QueryCount()
if err != nil {
glog.Error("cluster [%s] count err ", to.ClusterName)
return false, nil
}

if fromCount != toCount {
glog.Error("cluster [%s] count %d cluster [%s] count %d is not equal ",
from.ClusterName, fromCount, to.ClusterName, toCount)
return false, nil
}
return true, nil
}

err := wait.Poll(DefaultPollInterval, DefaultPollTimeout, fn)
if err != nil {
return fmt.Errorf("failed to launch scheduler backup job: %v", err)
}
return nil
}

func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *TidbClusterInfo) error {
func (oa *operatorActions) ForceDeploy(info *TidbClusterInfo) error {
if err := oa.CleanTidbCluster(info); err != nil {
return err
}

if err := oa.DeployTidbCluster(info); err != nil {
return err
}

return nil
}

func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterInfo) error {
func (info *TidbClusterInfo) QueryCount() (int, error) {
tableName := "test"
db, err := sql.Open("mysql", getDSN(info.Namespace, info.ClusterName, "record", info.Password))
if err != nil {
return 0, err
}
zyguan marked this conversation as resolved.
Show resolved Hide resolved

rows, err := db.Query(fmt.Sprintf("SELECT count(*) FROM %s", tableName))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

var count int
if err := db.QueryRaw(...).Scan(&count); err != nil {
...
}
return count, nil

otherwise, you have to close rows.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems all right， I‘ll change it in next PR.

if err != nil {
glog.Infof("cluster:[%s], error: %v", info.ClusterName, err)
return 0, err
}

for rows.Next() {
var count int
err := rows.Scan(&count)
if err != nil {
glog.Infof("cluster:[%s], error :%v", info.ClusterName, err)
}
return count, nil
}
return 0, fmt.Errorf("can not find count of ")
}

func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) error {
initSecretName := "set-secret"
backupSecretName := "backup-secret"
initSecret := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: initSecretName,
Namespace: info.Namespace,
},
Data: map[string][]byte{
"root": []byte(info.Password),
},
Type: corev1.SecretTypeOpaque,
}

_, err := oa.kubeCli.CoreV1().Secrets(info.Namespace).Create(&initSecret)
if err != nil && !releaseIsExist(err) {
return err
}

backupSecret := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: backupSecretName,
Namespace: info.Namespace,
},
Data: map[string][]byte{
"user": []byte("root"),
"password": []byte(info.Password),
},
Type: corev1.SecretTypeOpaque,
}

_, err = oa.kubeCli.CoreV1().Secrets(info.Namespace).Create(&backupSecret)
if err != nil && !releaseIsExist(err) {
return err
}

return nil
}

func releaseIsExist(err error) bool {
return strings.Contains(err.Error(), "already exists")
}

func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) error {
func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error {
return nil
}

func (oa *operatorActions) CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error {
func (oa *operatorActions) CheckScheduledBackup(info *TidbClusterInfo) error {
return nil
}

func (oa *operatorActions) ForceDeploy(info *TidbClusterInfo) error {
func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *TidbClusterInfo) error {
return nil
}

func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterInfo) error {
return nil
}
1 change: 1 addition & 0 deletions tests/backup/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package backup
Loading