enable customizing mydumper options in CRD backup

cmd/backup-manager/app/constants/constants.go

@@ -62,4 +62,7 @@ const (
 
 	// ServiceAccountCAPath is where is CABundle of serviceaccount locates
 	ServiceAccountCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+	// DefaultTableRegex is the default regular expression for 'db.table' matching
+	DefaultTableRegex = "^(?!(mysql|test|INFORMATION_SCHEMA|PERFORMANCE_SCHEMA|METRICS_SCHEMA|INSPECTION_SCHEMA))"
 )

cmd/backup-manager/app/export/export.go

@@ -25,6 +25,7 @@ import (
 	"github.com/mholt/archiver"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
+	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"k8s.io/klog"
 )
 
@@ -48,7 +49,7 @@ func (bo *Options) getDestBucketURI(remotePath string) string {
 	return fmt.Sprintf("%s://%s", bo.StorageType, remotePath)
 }
 
-func (bo *Options) dumpTidbClusterData() (string, error) {
+func (bo *Options) dumpTidbClusterData(backup *v1alpha1.Backup) (string, error) {
 	bfPath := bo.getBackupFullPath()
 	err := util.EnsureDirectoryExist(bfPath)
 	if err != nil {
@@ -60,12 +61,8 @@ func (bo *Options) dumpTidbClusterData() (string, error) {
 		fmt.Sprintf("--port=%d", bo.Port),
 		fmt.Sprintf("--user=%s", bo.User),
 		fmt.Sprintf("--password=%s", bo.Password),
-		"--long-query-guard=3600",
-		"--tidb-force-priority=LOW_PRIORITY",
-		"--verbose=3",
-		"--regex",
-		"^(?!(mysql|test|INFORMATION_SCHEMA|PERFORMANCE_SCHEMA|METRICS_SCHEMA|INSPECTION_SCHEMA))",
 	}
+	args = append(args, util.ConstructMydumperOptionsForBackup(backup)...)
 
 	output, err := exec.Command("/mydumper", args...).CombinedOutput()
 	if err != nil {

cmd/backup-manager/app/export/manager.go

@@ -190,7 +190,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro
 		klog.Infof("set cluster %s %s to %s success", bm, constants.TikvGCVariable, constants.TikvGCLifeTime)
 	}
 
-	backupFullPath, backupErr := bm.dumpTidbClusterData()
+	backupFullPath, backupErr := bm.dumpTidbClusterData(backup)
 	if oldTikvGCTimeDuration < tikvGCTimeDuration {
 		err = bm.SetTikvGCLifeTime(db, oldTikvGCTime)
 		if err != nil {

cmd/backup-manager/app/util/util.go

@@ -20,12 +20,12 @@ import (
 	"strings"
 
 	"github.com/Masterminds/semver"
+	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
+	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+	"github.com/pingcap/tidb-operator/pkg/backup/util"
 	"github.com/spf13/pflag"
 	"k8s.io/klog"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
-
-	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
-	"github.com/pingcap/tidb-operator/pkg/backup/util"
 )
 
 var (
@@ -36,6 +36,19 @@ var (
 	}
 	// DefaultVersion is the default tikv and br version
 	DefaultVersion = "4.0"
+	defaultOptions = []string{
+		"--long-query-guard=3600",
+		"--tidb-force-priority=LOW_PRIORITY",
+		"--verbose=3",
+		"--compress-protocol",
+		"--threads=16",
+		"--rows=10000",
+		"--skip-tz-utc",
+	}
+	defaultTableRegexOptions = []string{
+		"--regex",
+		constants.DefaultTableRegex,
+	}
 )
 
 func validCmdFlagFunc(flag *pflag.Flag) {
@@ -135,6 +148,30 @@ func ConstructBRGlobalOptionsForBackup(backup *v1alpha1.Backup) ([]string, strin
 	return args, remotePath, nil
 }
 
+// ConstructMydumperOptionsForBackup constructs mydumper options for backup
+func ConstructMydumperOptionsForBackup(backup *v1alpha1.Backup) []string {
+	var args []string
+	config := backup.Spec.Mydumper
+	if config == nil {
+		args = append(args, defaultOptions...)
+		args = append(args, defaultTableRegexOptions...)
+		return args
+	}
+
+	if len(config.Options) != 0 {
+		args = append(args, config.Options...)
+	} else {
+		args = append(args, defaultOptions...)
+	}
+
+	if config.TableRegex != nil {
+		args = append(args, "--regex", *config.TableRegex)
+	} else {
+		args = append(args, defaultTableRegexOptions...)
+	}
+	return args
+}
+
 // ConstructBRGlobalOptionsForRestore constructs BR global options for restore.
 func ConstructBRGlobalOptionsForRestore(restore *v1alpha1.Restore) ([]string, error) {
 	var args []string

cmd/backup-manager/app/util/util_test.go

@@ -0,0 +1,124 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+	"github.com/pingcap/tidb-operator/pkg/backup/constants"
+	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/pointer"
+)
+
+func TestConstructMydumperOptionsForBackup(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	type testcase struct {
+		name       string
+		hasRegex   bool
+		hasOptions bool
+	}
+
+	tests := []*testcase{
+		{
+			name:       "mydumper config is empty",
+			hasOptions: false,
+			hasRegex:   false,
+		},
+		{
+			name:       "customize mydumper options but not set table regex",
+			hasOptions: true,
+			hasRegex:   false,
+		},
+		{
+			name:       "customize mydumper table regex but not customize options",
+			hasOptions: false,
+			hasRegex:   true,
+		},
+		{
+			name:       "customize mydumper table regex and customize options",
+			hasOptions: true,
+			hasRegex:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			backup := newBackup()
+
+			customRegex := "^mysql"
+			customOptions := []string{"--long-query-guard=3000"}
+
+			var expectArgs []string
+
+			if tt.hasOptions {
+				backup.Spec.Mydumper = &v1alpha1.MydumperConfig{Options: customOptions}
+				expectArgs = append(expectArgs, customOptions...)
+			} else {
+				expectArgs = append(expectArgs, defaultOptions...)
+			}
+
+			if tt.hasRegex {
+				if backup.Spec.Mydumper == nil {
+					backup.Spec.Mydumper = &v1alpha1.MydumperConfig{TableRegex: &customRegex}
+				} else {
+					backup.Spec.Mydumper.TableRegex = &customRegex
+				}
+				expectArgs = append(expectArgs, "--regex", customRegex)
+			} else {
+				expectArgs = append(expectArgs, defaultTableRegexOptions...)
+			}
+
+			generateArgs := ConstructMydumperOptionsForBackup(backup)
+			g.Expect(apiequality.Semantic.DeepEqual(generateArgs, expectArgs)).To(Equal(true))
+		})
+	}
+}
+
+func newBackup() *v1alpha1.Backup {
+	return &v1alpha1.Backup{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Backup",
+			APIVersion: "pingcap.com/v1alpha1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-backup",
+			Namespace: corev1.NamespaceDefault,
+			UID:       types.UID("test-bk"),
+		},
+		Spec: v1alpha1.BackupSpec{
+			From: v1alpha1.TiDBAccessConfig{
+				Host:       "10.1.1.2",
+				Port:       constants.DefaultTidbPort,
+				User:       constants.DefaultTidbUser,
+				SecretName: "demo1-tidb-secret",
+			},
+			StorageProvider: v1alpha1.StorageProvider{
+				S3: &v1alpha1.S3StorageProvider{
+					Provider:   v1alpha1.S3StorageProviderTypeCeph,
+					Endpoint:   "http://10.0.0.1",
+					Bucket:     "test1-demo1",
+					SecretName: "demo",
+				},
+			},
+			StorageClassName: pointer.StringPtr("local-storage"),
+			StorageSize:      "1Gi",
+		},
+	}
+}

docs/api-references/docs.md

@@ -174,6 +174,19 @@ BRConfig
 </tr>
 <tr>
 <td>
+<code>mydumper</code></br>
+<em>
+<a href="#mydumperconfig">
+MydumperConfig
+</a>
+</em>
+</td>
+<td>
+<p>MydumperConfig is the configs for mydumper</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>tolerations</code></br>
 <em>
 <a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#toleration-v1-core">
@@ -2071,6 +2084,19 @@ BRConfig
 </tr>
 <tr>
 <td>
+<code>mydumper</code></br>
+<em>
+<a href="#mydumperconfig">
+MydumperConfig
+</a>
+</em>
+</td>
+<td>
+<p>MydumperConfig is the configs for mydumper</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>tolerations</code></br>
 <em>
 <a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#toleration-v1-core">
@@ -4321,6 +4347,46 @@ Kubernetes core/v1.PullPolicy
 </tr>
 </tbody>
 </table>
+<h3 id="mydumperconfig">MydumperConfig</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#backupspec">BackupSpec</a>)
+</p>
+<p>
+<p>MydumperConfig contains config for mydumper</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>options</code></br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<p>Options means options for backup data to remote storage with mydumper.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>tableRegex</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>TableRegex means Regular expression for &lsquo;db.table&rsquo; matching</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="networks">Networks</h3>
 <p>
 (<em>Appears on:</em>

manifests/crd.yaml

@@ -4538,6 +4538,15 @@ spec:
               - projectId
               - secretName
               type: object
+            mydumper:
+              properties:
+                options:
+                  items:
+                    type: string
+                  type: array
+                tableRegex:
+                  type: string
+              type: object
             s3:
               properties:
                 acl:
@@ -5392,6 +5401,15 @@ spec:
                   - projectId
                   - secretName
                   type: object
+                mydumper:
+                  properties:
+                    options:
+                      items:
+                        type: string
+                      type: array
+                    tableRegex:
+                      type: string
+                  type: object
                 s3:
                   properties:
                     acl: