cherry-pick important features to release-4.0 branch

cmd/backup.go

@@ -5,7 +5,6 @@ package cmd
 import (
 	"github.com/pingcap/log"
 	"github.com/pingcap/tidb/ddl"
-	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
@@ -56,8 +55,6 @@ func NewBackupCommand() *cobra.Command {
 
 			// Do not run ddl worker in BR.
 			ddl.RunWorker = false
-			// Do not run stat worker in BR.
-			session.DisableStats4Test()
 
 			summary.SetUnit(summary.BackupUnit)
 			return nil

cmd/debug.go

@@ -22,6 +22,7 @@ import (
 	"go.uber.org/zap"
 
 	berrors "github.com/pingcap/br/pkg/errors"
+	"github.com/pingcap/br/pkg/logutil"
 	"github.com/pingcap/br/pkg/restore"
 	"github.com/pingcap/br/pkg/rtree"
 	"github.com/pingcap/br/pkg/task"
@@ -106,8 +107,8 @@ func newCheckSumCommand() *cobra.Command {
 						zap.Uint64("totalBytes", file.GetTotalBytes()),
 						zap.Uint64("startVersion", file.GetStartVersion()),
 						zap.Uint64("endVersion", file.GetEndVersion()),
-						zap.Stringer("startKey", utils.WrapKey(file.GetStartKey())),
-						zap.Stringer("endKey", utils.WrapKey(file.GetEndKey())),
+						zap.Stringer("startKey", logutil.WrapKey(file.GetStartKey())),
+						zap.Stringer("endKey", logutil.WrapKey(file.GetEndKey())),
 					)
 
 					var data []byte
@@ -186,7 +187,7 @@ func newBackupMetaCommand() *cobra.Command {
 					log.Error(
 						"file ranges overlapped",
 						zap.Stringer("out", out),
-						utils.ZapFile(file),
+						logutil.File(file),
 					)
 				}
 			}

cmd/restore.go

@@ -4,7 +4,6 @@ package cmd
 
 import (
 	"github.com/pingcap/log"
-	"github.com/pingcap/tidb/session"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
@@ -82,9 +81,6 @@ func NewRestoreCommand() *cobra.Command {
 			utils.LogBRInfo()
 			task.LogArguments(c)
 
-			// Do not run stat worker in BR.
-			session.DisableStats4Test()
-
 			summary.SetUnit(summary.RestoreUnit)
 			return nil
 		},

docker-compose.yaml

@@ -4,13 +4,15 @@ version: '3.2'
 
 services:
   control:
-    image: control:latest
+    image: control:nightly
     build:
       context: .
       dockerfile: ./docker/Dockerfile
     volumes:
-      - ./docker/data:/data
-      - ./docker/logs:/tmp
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/tmp
+      - /tmp/br/docker/logs:/logs
+      - ./bin:/go/src/github.com/pingcap/br/bin
     command: -c "/usr/bin/tail -f /dev/null"
     depends_on:
       - "tidb"
@@ -19,13 +21,13 @@ services:
       - ./docker/minio.env
 
   pd0:
-    image: pingcap/pd:latest
+    image: pingcap/pd:nightly
     ports:
       - "2379"
     volumes:
       - ./docker/config/pd.toml:/pd.toml:ro
-      - ./docker/data:/data
-      - ./docker/logs:/logs
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/logs
     command:
       - --name=pd0
       - --client-urls=http://0.0.0.0:2379
@@ -45,11 +47,11 @@ services:
     restart: on-failure
 
   tikv0:
-    image: pingcap/tikv:latest
+    image: pingcap/tikv:nightly
     volumes:
       - ./docker/config/tikv.toml:/tikv.toml:ro
-      - ./docker/data:/data
-      - ./docker/logs:/logs
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/logs
     command:
       - --addr=0.0.0.0:20160
       - --advertise-addr=tikv0:20160
@@ -70,11 +72,11 @@ services:
       - ./docker/minio.env
 
   tikv1:
-    image: pingcap/tikv:latest
+    image: pingcap/tikv:nightly
     volumes:
       - ./docker/config/tikv.toml:/tikv.toml:ro
-      - ./docker/data:/data
-      - ./docker/logs:/logs
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/logs
     command:
       - --addr=0.0.0.0:20160
       - --advertise-addr=tikv1:20160
@@ -95,11 +97,11 @@ services:
       - ./docker/minio.env
 
   tikv2:
-    image: pingcap/tikv:latest
+    image: pingcap/tikv:nightly
     volumes:
       - ./docker/config/tikv.toml:/tikv.toml:ro
-      - ./docker/data:/data
-      - ./docker/logs:/logs
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/logs
     command:
       - --addr=0.0.0.0:20160
       - --advertise-addr=tikv2:20160
@@ -120,11 +122,11 @@ services:
       - ./docker/minio.env
 
   tikv3:
-    image: pingcap/tikv:latest
+    image: pingcap/tikv:nightly
     volumes:
       - ./docker/config/tikv.toml:/tikv.toml:ro
-      - ./docker/data:/data
-      - ./docker/logs:/logs
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/logs
     command:
       - --addr=0.0.0.0:20160
       - --advertise-addr=tikv3:20160
@@ -145,11 +147,11 @@ services:
       - ./docker/minio.env
 
   tikv4:
-    image: pingcap/tikv:latest
+    image: pingcap/tikv:nightly
     volumes:
       - ./docker/config/tikv.toml:/tikv.toml:ro
-      - ./docker/data:/data
-      - ./docker/logs:/logs
+      - /tmp/br/docker/data:/data
+      - /tmp/br/docker/logs:/logs
     command:
       - --addr=0.0.0.0:20160
       - --advertise-addr=tikv4:20160
@@ -170,13 +172,13 @@ services:
       - ./docker/minio.env
 
   tidb:
-    image: pingcap/tidb:latest
+    image: pingcap/tidb:nightly
     ports:
-      - "4000:4000"
-      - "10080:10080"
+      - "4000"
+      - "10080"
     volumes:
       - ./docker/config/tidb.toml:/tidb.toml:ro
-      - ./docker/logs:/logs
+      - /tmp/br/docker/logs:/logs
     command:
       - --store=tikv
       - --path=pd0:2379
@@ -202,15 +204,8 @@ services:
     ports:
       - 24927:24927
     volumes:
-      - ./docker/data/s3:/data/s3
+      - /tmp/br/docker/data/s3:/data/s3
     command: server --address=:24927 /data/s3
     env_file:
       - ./docker/minio.env
 
-  tidb-vision:
-    image: pingcap/tidb-vision:latest
-    environment:
-      PD_ENDPOINT: pd0:2379
-    ports:
-      - "8010:8010"
-    restart: on-failure

go.mod

@@ -4,35 +4,43 @@ go 1.13
 
 require (
 	cloud.google.com/go/storage v1.6.0
-	github.com/aws/aws-sdk-go v1.35.2
+	github.com/aws/aws-sdk-go v1.35.3
 	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/coreos/go-semver v0.3.0
+	github.com/dgraph-io/ristretto v0.0.2 // indirect
 	github.com/fsouza/fake-gcs-server v1.19.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/mock v1.4.4
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
+	github.com/mattn/go-colorable v0.1.7 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/onsi/ginkgo v1.11.0 // indirect
+	github.com/onsi/gomega v1.8.1 // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
-	github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d
+	github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/kvproto v0.0.0-20200927025644-73dc27044686
-	github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463
-	github.com/pingcap/parser v0.0.0-20200921041333-cd2542b7a8a2
-	github.com/pingcap/tidb v1.1.0-beta.0.20200921082409-501466fb690d
-	github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible
-	github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3
+	github.com/pingcap/kvproto v0.0.0-20201124110645-494a2fb764b7
+	github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8
+	github.com/pingcap/parser v0.0.0-20201125084540-106549ed5502
+	github.com/pingcap/tidb v1.1.0-beta.0.20201126095401-55c106afb89b
+	github.com/pingcap/tidb-tools v4.0.6-0.20200828085514-03575b185007+incompatible
+	github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/common v0.9.1
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09
+	github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect
+	github.com/tikv/pd v1.1.0-beta.0.20201126071958-7445c235ddbf
+	go.etcd.io/bbolt v1.3.5 // indirect
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
-	go.uber.org/multierr v1.5.0
+	go.uber.org/multierr v1.6.0
 	go.uber.org/zap v1.16.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.22.0
 	google.golang.org/grpc v1.27.1
 )

go.mod1

@@ -4,35 +4,43 @@ go 1.13
 
 require (
 	cloud.google.com/go/storage v1.6.0
-	github.com/aws/aws-sdk-go v1.35.2
+	github.com/aws/aws-sdk-go v1.35.3
 	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/coreos/go-semver v0.3.0
+	github.com/dgraph-io/ristretto v0.0.2 // indirect
 	github.com/fsouza/fake-gcs-server v1.19.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/mock v1.4.4
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
+	github.com/mattn/go-colorable v0.1.7 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/onsi/ginkgo v1.11.0 // indirect
+	github.com/onsi/gomega v1.8.1 // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
-	github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d
+	github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/kvproto v0.0.0-20200927025644-73dc27044686
-	github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463
-	github.com/pingcap/parser v0.0.0-20200921041333-cd2542b7a8a2
-	github.com/pingcap/tidb v1.1.0-beta.0.20200921082409-501466fb690d
-	github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible
-	github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3
+	github.com/pingcap/kvproto v0.0.0-20201124110645-494a2fb764b7
+	github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8
+	github.com/pingcap/parser v0.0.0-20201125084540-106549ed5502
+	github.com/pingcap/tidb v1.1.0-beta.0.20201126095401-55c106afb89b
+	github.com/pingcap/tidb-tools v4.0.6-0.20200828085514-03575b185007+incompatible
+	github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/common v0.9.1
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09
+	github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect
+	github.com/tikv/pd v1.1.0-beta.0.20201126071958-7445c235ddbf
+	go.etcd.io/bbolt v1.3.5 // indirect
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
-	go.uber.org/multierr v1.5.0
+	go.uber.org/multierr v1.6.0
 	go.uber.org/zap v1.16.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/api v0.22.0
 	google.golang.org/grpc v1.27.1
 )