diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index a2f476e7f4..0d4a803a86 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -79,3 +79,34 @@ config-file: |- {{ include "tidb-configmap.data" . | sha256sum | trunc 8 }} {{- end -}} +{{/* +Encapsulate pump configmap data for consistent digest calculation +*/}} +{{- define "pump-configmap.data" -}} +pump-config: |- + {{- if .Values.binlog.pump.config }} +{{ .Values.binlog.pump.config | indent 2 }} + {{- else -}} +{{ tuple "config/_pump-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }} + {{- end -}} +{{- end -}} + +{{- define "pump-configmap.data-digest" -}} +{{ include "pump-configmap.data" . | sha256sum | trunc 8 }} +{{- end -}} + +{{/* +Encapsulate drainer configmap data for consistent digest calculation +*/}} +{{- define "drainer-configmap.data" -}} +drainer-config: |- + {{- if .Values.binlog.drainer.config }} +{{ .Values.binlog.drainer.config | indent 2 }} + {{- else -}} +{{ tuple "config/_drainer-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }} + {{- end -}} +{{- end -}} + +{{- define "drainer-configmap.data-digest" -}} +{{ include "drainer-configmap.data" . | sha256sum | trunc 8 }} +{{- end -}} diff --git a/charts/tidb-cluster/templates/drainer-configmap-rollout.yaml b/charts/tidb-cluster/templates/drainer-configmap-rollout.yaml new file mode 100644 index 0000000000..6c4ce7a606 --- /dev/null +++ b/charts/tidb-cluster/templates/drainer-configmap-rollout.yaml @@ -0,0 +1,16 @@ +{{- if .Values.enableConfigMapRollout }} +{{- if .Values.binlog.drainer.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cluster.name" . }}-drainer-{{ template "drainer-configmap.data-digest" . }} + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: drainer + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +data: +{{ include "drainer-configmap.data" . | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/charts/tidb-cluster/templates/drainer-configmap.yaml b/charts/tidb-cluster/templates/drainer-configmap.yaml index 832fb12879..4bbb593f6e 100644 --- a/charts/tidb-cluster/templates/drainer-configmap.yaml +++ b/charts/tidb-cluster/templates/drainer-configmap.yaml @@ -10,6 +10,5 @@ metadata: app.kubernetes.io/component: drainer helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} data: - drainer-config: |- -{{ tuple "config/_drainer-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{ include "drainer-configmap.data" . | indent 2 }} {{- end -}} diff --git a/charts/tidb-cluster/templates/drainer-statefulset.yaml b/charts/tidb-cluster/templates/drainer-statefulset.yaml index 48a88dd805..495413cb72 100644 --- a/charts/tidb-cluster/templates/drainer-statefulset.yaml +++ b/charts/tidb-cluster/templates/drainer-statefulset.yaml @@ -58,7 +58,11 @@ spec: volumes: - name: config configMap: + {{- if .Values.enableConfigMapRollout }} + name: {{ template "cluster.name" . }}-drainer-{{ template "drainer-configmap.data-digest" . }} + {{- else }} name: {{ template "cluster.name" . }}-drainer + {{- end }} items: - key: drainer-config path: drainer.toml diff --git a/charts/tidb-cluster/templates/pump-configmap-rollout.yaml b/charts/tidb-cluster/templates/pump-configmap-rollout.yaml new file mode 100644 index 0000000000..e920d9815c --- /dev/null +++ b/charts/tidb-cluster/templates/pump-configmap-rollout.yaml @@ -0,0 +1,16 @@ +{{- if .Values.enableConfigMapRollout }} +{{- if .Values.binlog.pump.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cluster.name" . }}-pump-{{ template "pump-configmap.data-digest" . }} + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: pump + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +data: +{{ include "pump-configmap.data" . | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/tidb-cluster/templates/pump-configmap.yaml b/charts/tidb-cluster/templates/pump-configmap.yaml index 2956184188..cb139a7077 100644 --- a/charts/tidb-cluster/templates/pump-configmap.yaml +++ b/charts/tidb-cluster/templates/pump-configmap.yaml @@ -10,6 +10,5 @@ metadata: app.kubernetes.io/component: pump helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} data: - pump-config: |- -{{ tuple "config/_pump-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end -}} +{{ include "pump-configmap.data" . | indent 2 }} +{{ end }} diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml index 476b61df06..310b913861 100644 --- a/charts/tidb-cluster/templates/pump-statefulset.yaml +++ b/charts/tidb-cluster/templates/pump-statefulset.yaml @@ -58,7 +58,11 @@ spec: volumes: - name: config configMap: + {{- if .Values.enableConfigMapRollout }} + name: {{ template "cluster.name" . }}-pump-{{ template "pump-configmap.data-digest" . }} + {{- else }} name: {{ template "cluster.name" . }}-pump + {{- end }} items: - key: pump-config path: pump.toml diff --git a/charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl index 7e56bfcaa5..66b10485f6 100644 --- a/charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl @@ -26,6 +26,7 @@ done /drainer \ -L={{ .Values.binlog.drainer.logLevel | default "info" }} \ +-pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \ -addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-drainer:8249 \ -config=/etc/drainer/drainer.toml \ -disable-detect={{ .Values.binlog.drainer.disableDetect | default false }} \ diff --git a/charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl index 8896a8fb71..79c3735049 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl @@ -1,5 +1,6 @@ set -euo pipefail /pump \ +-pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \ -L={{ .Values.binlog.pump.logLevel | default "info" }} \ -advertise-addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-pump:8250 \ -config=/etc/pump/pump.toml \ diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 091c061d51..1e3d7cf904 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -423,6 +423,17 @@ binlog: # number of seconds between heartbeat ticks (in 2 seconds) heartbeatInterval: 2 + # Please refer to https://github.com/pingcap/tidb-binlog/blob/master/cmd/pump/pump.toml for the default + # pump configurations (change to the tags of your pump version), + # just follow the format in the file and configure in the 'config' section + # as below if you want to customize any configuration. + # config: | + # gc = 7 + # heartbeat-interval = 2 + # [storage] + # sync-log = true + # stop-write-at-available-space = "10Gi" + drainer: create: false image: pingcap/tidb-binlog:v3.0.1 @@ -474,6 +485,24 @@ binlog: # kafkaAddrs: "127.0.0.1:9092" # kafkaVersion: "0.8.2.0" + # Please refer to https://github.com/pingcap/tidb-binlog/blob/master/cmd/drainer/drainer.toml for the default + # drainer configurations (change to the tags of your drainer version), + # just follow the format in the file and configure in the 'config' section + # as below if you want to customize any configuration. + # config: | + # worker-count = 16 + # detect-interval = 10 + # disable-dispatch = false + # ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" + # safe-mode = false + # txn-batch = 20 + # db-type = "mysql" + # [syncer.to] + # # host = "127.0.0.1" + # # user = "root" + # # password = "" + # # port = 3306 + scheduledBackup: create: false # https://github.com/pingcap/tidb-cloud-backup diff --git a/tests/actions.go b/tests/actions.go index 334184e0ba..0c2edcc9b8 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -263,6 +263,9 @@ type TidbClusterConfig struct { BlockWriteConfig blockwriter.Config GrafanaClient *metrics.Client TopologyKey string + + pumpConfig []string + drainerConfig []string } func (tc *TidbClusterConfig) String() string { @@ -2289,13 +2292,7 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterConfig, to * glog.Infof("begin to deploy incremental backup cluster[%s] namespace[%s]", from.ClusterName, from.Namespace) sets := map[string]string{ - "binlog.pump.create": "true", - "binlog.drainer.destDBType": "mysql", - "binlog.drainer.mysql.host": fmt.Sprintf("%s-tidb.%s", to.ClusterName, to.Namespace), - "binlog.drainer.mysql.user": "root", - "binlog.drainer.mysql.password": to.Password, - "binlog.drainer.mysql.port": "4000", - "binlog.drainer.ignoreSchemas": "", + "binlog.pump.create": "true", } if withDrainer { sets["binlog.drainer.create"] = "true" @@ -2304,6 +2301,21 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterConfig, to * sets["binlog.drainer.initialCommitTs"] = ts } + from.drainerConfig = []string{ + "worker-count = 16", + "detect-interval = 10", + "disable-dispatch = false", + `ignore-schemas = ""`, + `safe-mode = false`, + `txn-batch = 20`, + `db-type = "mysql"`, + `[syncer.to]`, + fmt.Sprintf(`host = "%s-tidb.%s"`, to.ClusterName, to.Namespace), + fmt.Sprintf(`user = "%s"`, "root"), + fmt.Sprintf(`password = "%s"`, to.Password), + fmt.Sprintf(`port = %d`, 4000), + } + cmd, err := oa.getHelmUpgradeClusterCmd(from, sets) if err != nil { return err @@ -2311,7 +2323,7 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterConfig, to * glog.Infof(cmd) res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if err != nil { - return fmt.Errorf("failed to launch scheduler backup job: %v, %s", err, string(res)) + return fmt.Errorf("failed to launch incremental backup job: %v, %s", err, string(res)) } return nil } @@ -2439,7 +2451,7 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD err := wait.Poll(oa.pollInterval, DefaultPollTimeout, fn) if err != nil { - return fmt.Errorf("failed to launch scheduler backup job: %v", err) + return fmt.Errorf("failed to check incremental backup job: %v", err) } return nil diff --git a/tests/cluster_info.go b/tests/cluster_info.go index 0caea9decd..853e3c14c1 100644 --- a/tests/cluster_info.go +++ b/tests/cluster_info.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "strconv" + + "github.com/golang/glog" ) func (tc *TidbClusterConfig) set(name string, value string) (string, bool) { @@ -113,7 +115,7 @@ func (tc *TidbClusterConfig) BuildSubValues(path string) (string, error) { "[log]", `level = "info"`, } - subValues := GetSubValuesOrDie(tc.ClusterName, tc.Namespace, tc.TopologyKey, pdConfig, tikvConfig, tidbConfig) + subValues := GetSubValuesOrDie(tc.ClusterName, tc.Namespace, tc.TopologyKey, pdConfig, tikvConfig, tidbConfig, tc.pumpConfig, tc.drainerConfig) subVaulesPath := fmt.Sprintf("%s/%s.yaml", path, tc.ClusterName) _, err := os.Stat(subVaulesPath) if err != nil { @@ -134,5 +136,6 @@ func (tc *TidbClusterConfig) BuildSubValues(path string) (string, error) { if err != nil { return "", err } + glog.Infof("subValues:\n %s", subValues) return subVaulesPath, nil } diff --git a/tests/util.go b/tests/util.go index 7e8397b4b0..91356e0329 100644 --- a/tests/util.go +++ b/tests/util.go @@ -104,7 +104,9 @@ var affinityTemp string = `{{.Kind}}: topologyKey: {{.TopologyKey}} namespaces: - {{.Namespace}} -binlog: +` + +var binlogTemp string = `binlog: pump: tolerations: - key: node-role @@ -119,6 +121,10 @@ binlog: topologyKey: {{.TopologyKey}} namespaces: - {{.Namespace}} +{{if .PumpConfig}} + config: | +{{range .PumpConfig}} {{.}} +{{end}}{{end}} drainer: tolerations: - key: node-role @@ -133,6 +139,10 @@ binlog: topologyKey: {{.TopologyKey}} namespaces: - {{.Namespace}} +{{if .DrainerConfig}} + config: | +{{range .DrainerConfig}} {{.}} +{{end}}{{end}} ` type AffinityInfo struct { @@ -144,7 +154,14 @@ type AffinityInfo struct { Config []string } -func GetSubValuesOrDie(clusterName, namespace, topologyKey string, pdConfig []string, tikvConfig []string, tidbConfig []string) string { +type BinLogInfo struct { + PumpConfig []string + DrainerConfig []string + Namespace string + TopologyKey string +} + +func GetSubValuesOrDie(clusterName, namespace, topologyKey string, pdConfig []string, tikvConfig []string, tidbConfig []string, pumpConfig []string, drainerConfig []string) string { temp, err := template.New("dt-affinity").Parse(affinityTemp) if err != nil { slack.NotifyAndPanic(err) @@ -165,7 +182,23 @@ func GetSubValuesOrDie(clusterName, namespace, topologyKey string, pdConfig []st if err != nil { slack.NotifyAndPanic(err) } - return fmt.Sprintf("%s%s%s", pdbuff.String(), tikvbuff.String(), tidbbuff.String()) + subValues := fmt.Sprintf("%s%s%s", pdbuff.String(), tikvbuff.String(), tidbbuff.String()) + + if pumpConfig == nil && drainerConfig == nil { + return subValues + } + + btemp, err := template.New("binlog").Parse(binlogTemp) + if err != nil { + slack.NotifyAndPanic(err) + } + binlogbuff := new(bytes.Buffer) + err = btemp.Execute(binlogbuff, &BinLogInfo{PumpConfig: pumpConfig, DrainerConfig: drainerConfig, Namespace: namespace, TopologyKey: topologyKey}) + if err != nil { + slack.NotifyAndPanic(err) + } + subValues = fmt.Sprintf("%s%s", subValues, binlogbuff.String()) + return subValues } const (