refactor backup

charts/tidb-backup/templates/NOTES.txt

@@ -1,12 +1,13 @@
-{{- if .Values.backup.create }}
+{{- if eq .Values.mode "backup" }}
 1. Watch backup job is created
    watch kubectl get jobs --namespace {{ .Release.Namespace }}  -l app.kubernetes.io/component=backup
 2. Wait until backup pods becomes completed
    watch kubectl get po --namespace {{ .Release.Namespace }}  -l app.kubernetes.io/component=backup -o wide
-{{- end -}}
-{{- if .Values.restore.create }}
+{{- else if eq .Values.mode "restore" }}
 1. Watch restore job is created
    watch kubectl get jobs --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -o wide
 2. Wait until restore pod becomes completed
    watch kubectl get po --namespace {{ .Release.Namespace }}  -l app.kubernetes.io/component=restore
+{{- else }}
+{{ fail (printf "invalid mode: %s" .Values.mode) }}
 {{- end -}}

charts/tidb-backup/templates/backup-job.yaml

@@ -1,8 +1,8 @@
-{{- if .Values.backup.create }}
+{{- if eq .Values.mode "backup" }}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ .Values.clusterName }}-backup
+  name: {{ .Values.clusterName }}-{{ .Values.name }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
@@ -21,8 +21,8 @@ spec:
     spec:
       initContainers:
       - name: get-ts
-        image: {{ .Values.backup.binlogImage }}
-        imagePullPolicy: {{ .Values.backup.binlogImagePullPolicy | default "IfNotPresent" }}
+        image: {{ .Values.image.binlog }}
+        imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
         command:
         - /binlogctl
         - -pd-urls=http://{{ .Values.clusterName }}-pd:2379
@@ -33,8 +33,8 @@ spec:
           mountPath: "/savepoint-dir"
       containers:
       - name: backup
-        image: {{ .Values.backup.mydumperImage }}
-        imagePullPolicy: {{ .Values.backup.mydumperImagePullPolicy | default "IfNotPresent" }}
+        image: {{ .Values.image.backup }}
+        imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
         command:
         - /bin/sh
         - -c
@@ -45,35 +45,50 @@ spec:
           mountPath: "/savepoint-dir"
         - name: data
           mountPath: "/data"
-      {{- if .Values.backup.gcp }}
+      {{- if .Values.gcp }}
         - name: gcp-credentials
           mountPath: "/gcp"
           readOnly: true
       {{- end }}
         env:
-        - name: MY_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-      {{- if .Values.backup.gcp }}
+        - name: BACKUP_NAME
+          value: {{ .Values.name | quote }}
+      {{- if .Values.gcp }}
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: /gcp/credentials.json
       {{- end }}
+      {{- if .Values.ceph }}
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.ceph.secretName }}
+              key: access_key
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.ceph.secretName }}
+              key: secret_key
+      {{- end }}
+        - name: TIDB_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.secretName }}
+              key: user
         - name: TIDB_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: {{ .Values.clusterName }}-backup
-              key: user-password
+              name: {{ .Values.secretName }}
+              key: password
       restartPolicy: OnFailure
       volumes:
       - name: savepoint-dir
         emptyDir: {}
       - name: data
         persistentVolumeClaim:
-          claimName: {{ .Values.clusterName }}-backup
-    {{- if .Values.backup.gcp }}
+          claimName: {{ .Values.name }}
+    {{- if .Values.gcp }}
       - name: gcp-credentials
         secret:
-          secretName: {{ .Values.clusterName }}-backup
+          secretName: {{ .Values.gcp.secretName }}
     {{- end }}
 {{- end }}

charts/tidb-backup/templates/backup-pvc.yaml

@@ -1,8 +1,8 @@
-{{- if .Values.backup.create }}
+{{- if eq .Values.mode "backup" }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
-  name: {{ .Values.clusterName }}-backup
+  name: {{ .Values.name }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: tidb-operator
@@ -15,6 +15,6 @@ spec:
   volumeMode: Filesystem
   resources:
     requests:
-      storage: {{ .Values.backup.storage }}
-  storageClassName: {{ .Values.backup.storageClassName }}
+      storage: {{ .Values.storage.size }}
+  storageClassName: {{ .Values.storage.className }}
 {{- end }}

charts/tidb-backup/templates/restore-job.yaml

@@ -1,8 +1,8 @@
-{{- if .Values.restore.create }}
+{{- if eq .Values.mode "restore" }}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ .Values.clusterName }}-restore
+  name: {{ .Values.clusterName }}-restore-{{ .Values.name }}
   labels:
     app.kubernetes.io/name: {{ template "chart.name" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
@@ -20,7 +20,7 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: tidb-restore-job
-        image: {{ .Values.restore.image }}
+        image: {{ .Values.image.backup }}
         command:
         - /bin/sh
         - -c
@@ -29,33 +29,47 @@ spec:
         volumeMounts:
         - name: data
           mountPath: "/data"
-      {{- if .Values.restore.gcp }}
+      {{- if .Values.gcp }}
         - name: gcp-credentials
           mountPath: "/gcp"
           readOnly: true
       {{- end }}
         env:
-        - name: MY_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-      {{- if .Values.restore.gcp }}
+        - name: BACKUP_NAME
+          value: {{ .Values.name | quote }}
+      {{- if .Values.gcp }}
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: /gcp/credentials.json
       {{- end }}
+      {{- if .Values.ceph }}
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.ceph.secretName }}
+              key: access_key
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.ceph.secretName }}
+              key: secret_key
+      {{- end }}
+        - name: TIDB_USER
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.secretName }}
+              key: user
         - name: TIDB_PASSWORD
           valueFrom:
             secretKeyRef:
-              name: {{ .Values.clusterName }}-restore
-              key: user-password
+              name: {{ .Values.secretName }}
+              key: password
       volumes:
       - name: data
         persistentVolumeClaim:
-          claimName: {{ .Values.clusterName }}-restore
-    {{- if .Values.restore.gcp }}
+          claimName: {{ .Values.name }}
+    {{- if .Values.gcp }}
       - name: gcp-credentials
         secret:
-          secretName: {{ .Values.clusterName }}-restore
+          secretName: {{ .Values.gcp.secretName }}
     {{- end }}
 {{- end }}
-

charts/tidb-backup/templates/scripts/_start_backup.sh.tpl

@@ -1,29 +1,30 @@
 set -euo pipefail
-dirname=backup-`date +%Y-%m-%dT%H%M%S`-${MY_POD_NAME}
+
 host=`echo {{ .Values.clusterName }}_TIDB_SERVICE_HOST | tr '[a-z]' '[A-Z]' | tr '-' '_'`
 
-mkdir -p /data/${dirname}/
-cp /savepoint-dir/savepoint /data/${dirname}/
+dirname=/data/${BACKUP_NAME}
+mkdir -p ${dirname}
+cp /savepoint-dir/savepoint ${dirname}/
 
 /mydumper \
-  --outputdir=/data/${dirname} \
+  --outputdir=${dirname} \
   --host=`eval echo '${'$host'}'` \
   --port=4000 \
-  --user={{ .Values.backup.user }} \
+  --user=${TIDB_USER} \
   --password=${TIDB_PASSWORD} \
-  {{ .Values.backup.options }}
+  {{ .Values.backupOptions }}
 
-{{- if .Values.backup.gcp }}
+{{- if .Values.gcp }}
 uploader \
   --cloud=gcp \
-  --bucket={{ .Values.backup.gcp.bucket }} \
-  --backup-dir=/data/${dirname}
+  --bucket={{ .Values.gcp.bucket }} \
+  --backup-dir=${dirname}
 {{- end }}
 
-{{- if .Values.backup.ceph }}
+{{- if .Values.ceph }}
 uploader \
   --cloud=ceph \
-  --bucket={{ .Values.backup.ceph.bucket }} \
-  --endpoint={{ .Values.backup.ceph.endpoint }} \
-  --backup-dir=/data/${dirname}
+  --bucket={{ .Values.ceph.bucket }} \
+  --endpoint={{ .Values.ceph.endpoint }} \
+  --backup-dir=${dirname}
 {{- end }}

charts/tidb-backup/templates/scripts/_start_restore.sh.tpl

@@ -1,31 +1,30 @@
 set -euo pipefail
 
-dirname=restore-`date +%Y-%m-%dT%H%M%S`-${MY_POD_NAME}
-dataDir=/data/${dirname}
-mkdir -p ${dataDir}/
+dirname=/data/${BACKUP_NAME}
+mkdir -p ${dirname}
 host=`echo {{ .Values.clusterName }}_TIDB_SERVICE_HOST | tr '[a-z]' '[A-Z]' | tr '-' '_'`
 
-{{- if .Values.restore.gcp }}
+{{- if .Values.gcp }}
 downloader \
   --cloud=gcp \
-  --bucket={{ .Values.restore.gcp.bucket }} \
-  --srcDir={{ .Values.restore.srcDir }} \
-  --destDir=${dataDir}
+  --bucket={{ .Values.gcp.bucket }} \
+  --srcDir=${BACKUP_NAME} \
+  --destDir=${dirname}
 {{- end }}
 
-{{- if .Values.restore.ceph }}
+{{- if .Values.ceph }}
 downloader \
   --cloud=ceph \
-  --bucket={{ .Values.restore.ceph.bucket }} \
-  --endpoint={{ .Values.restore.ceph.endpoint }} \
-  --srcDir={{ .Values.restore.srcDir }} \
-  --destDir=${dataDir}
+  --bucket={{ .Values.ceph.bucket }} \
+  --endpoint={{ .Values.ceph.endpoint }} \
+  --srcDir=${BACKUP_NAME} \
+  --destDir=${dirname}
 {{- end }}
 
 /loader \
-  -d ${dataDir}/{{ .Values.restore.srcDir }} \
+  -d ${dirname} \
   -h `eval echo '${'$host'}'` \
-  -u {{ .Values.restore.user }} \
+  -u ${TIDB_USER} \
   -p ${TIDB_PASSWORD} \
   -P 4000 \
-  {{ .Values.restore.options }}
+  {{ .Values.restoreOptions }}