fix default-columns ci

backup/checksum: add cluster index support for ranges (pingcap#1120)

lightning/restore: support ingset multi ssts for same range (pingcap#1089)

* ingest ssts for the same range in a batch

* make ingest compatible with old tikv

tests: add row count check for br clustered index tests (pingcap#1151)

storage: mkdirAll for local storage even when SkipCheckPath is true (pingcap#1156)

tests/br: fix bug in issue pingcap#1158 (pingcap#1160)

tests/br/compatibility: fix failed for manifest unknown (pingcap#1161)

*: upgrade go version from 1.13/1.15 to 1.16 (pingcap#1159)

action trigger: fix compatibility trigger bug on push (pingcap#1170)

address comment

fix test

address comment

address comment

fix build

add ut for ignore columns

go.mod: update tidb to the new version (pingcap#1153)

lightning: support restore data into tables that contains data (pingcap#784)

fix conflict after merge master

.github/workflows/build.yml

@@ -4,10 +4,28 @@ on:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
   pull_request:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
 
 jobs:
   compile:
@@ -30,7 +48,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.16
 
       - name: Run build
         run: make build
@@ -44,7 +62,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.16
 
       - name: Compile for FreeBSD
         run: GOOS=freebsd make build

.github/workflows/compatible_test.yml

@@ -5,54 +5,59 @@ on:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
   pull_request:
     branches:
       - master
       - 'release-[0-9].[0-9]*'
-  issue_comment:
-    types:
-      - created
+    paths-ignore:
+      - '**.html'
+      - '**.md'
+      - 'CNAME'
+      - 'LICENSE'
+      - 'docs/**'
+      - 'tests/**'
+      - 'docker/**'
+      - '.github/workflows/**.yml'
 
 jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 25
     steps:
 
-    - uses: khan/pull-request-comment-trigger@master
-      id: check
-      with:
-        trigger: '/run-compatiblility-tests'
-
     - name: Free disk space
       run: |
         sudo rm -rf /usr/local/lib/android
         sudo rm -rf /usr/share/dotnet
         docker volume prune -f
         docker image prune -f
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - uses: actions/checkout@v2
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.15
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
+        go-version: 1.16
 
     - name: Generate compatibility test backup data
       timeout-minutes: 15
       run: sh compatibility/prepare_backup.sh
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - name: Start server
       run: |
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml rm -s -v
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml build
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml up --remove-orphans -d
         TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml exec -T control make compatibility_test
-      if: ${{ github.event_name == 'pull_request' || steps.check.outputs.triggered == 'true' }}
 
     - name: Collect component log
       if: ${{ failure() }}

CONTRIBUTING.md

@@ -14,7 +14,7 @@ your contribution accepted.
 
 Developing BR requires:
 
-* [Go 1.13+](http://golang.org/doc/code.html)
+* [Go 1.16+](http://golang.org/doc/code.html)
 * An internet connection to download the dependencies
 
 Simply run `make` to build the program.

README.md

@@ -34,7 +34,7 @@ $ make
 $ make test
 ```
 
-Notice BR supports building with Go version `Go >= 1.13`
+Notice BR supports building with Go version `Go >= 1.16`
 
 When BR is built successfully, you can find binary in the `bin` directory.
 
@@ -108,15 +108,15 @@ bin/br backup table --db test \
 	-s local:///tmp/backup_test/ \
 	--pd ${PD_ADDR}:2379 \
 	--log-file backup_test.log \
-					
+
 # Let's drop the table.
 mysql -uroot --host 127.0.0.1 -P4000 -E -e "USE test; DROP TABLE order_line; show tables" -u root -p
 
 # Restore from the backup.
 bin/br restore table --db test \
-	--table order_line \ 
-	-s local:///tmp/backup_test/ \ 
-	--pd ${PD_ADDR}:2379 \ 
+	--table order_line \
+	-s local:///tmp/backup_test/ \
+	--pd ${PD_ADDR}:2379 \
 	--log-file restore_test.log
 
 # How many rows do we get after restore? Expected to be 300242 rows.

cmd/br/cmd.go

@@ -15,7 +15,6 @@ import (
 	"github.com/pingcap/log"
 	tidbutils "github.com/pingcap/tidb-tools/pkg/utils"
 	"github.com/pingcap/tidb/util/logutil"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
 	"github.com/pingcap/br/pkg/gluetidb"
@@ -101,6 +100,27 @@ func AddFlags(cmd *cobra.Command) {
 // Init initializes BR cli.
 func Init(cmd *cobra.Command) (err error) {
 	initOnce.Do(func() {
+		slowLogFilename, e := cmd.Flags().GetString(FlagSlowLogFile)
+		if e != nil {
+			err = e
+			return
+		}
+		tidbLogCfg := logutil.LogConfig{}
+		if len(slowLogFilename) != 0 {
+			tidbLogCfg.SlowQueryFile = slowLogFilename
+			// Just for special grpc log file,
+			// otherwise the info will be print in stdout...
+			tidbLogCfg.File.Filename = timestampLogFileName()
+		} else {
+			// Disable annoying TiDB Log.
+			// TODO: some error logs outputs randomly, we need to fix them in TiDB.
+			tidbLogCfg.Level = "fatal"
+		}
+		e = logutil.InitLogger(&tidbLogCfg)
+		if e != nil {
+			err = e
+			return
+		}
 		// Initialize the logger.
 		conf := new(log.Config)
 		conf.Level, err = cmd.Flags().GetString(FlagLogLevel)
@@ -144,28 +164,6 @@ func Init(cmd *cobra.Command) (err error) {
 			return
 		}
 		redact.InitRedact(redactLog || redactInfoLog)
-
-		slowLogFilename, e := cmd.Flags().GetString(FlagSlowLogFile)
-		if e != nil {
-			err = e
-			return
-		}
-		tidbLogCfg := logutil.LogConfig{}
-		if len(slowLogFilename) != 0 {
-			tidbLogCfg.SlowQueryFile = slowLogFilename
-		} else {
-			// Hack! Discard slow log by setting log level to PanicLevel
-			logutil.SlowQueryLogger.SetLevel(logrus.PanicLevel)
-			// Disable annoying TiDB Log.
-			// TODO: some error logs outputs randomly, we need to fix them in TiDB.
-			tidbLogCfg.Level = "fatal"
-		}
-		e = logutil.InitLogger(&tidbLogCfg)
-		if e != nil {
-			err = e
-			return
-		}
-
 		err = startPProf(cmd)
 	})
 	return errors.Trace(err)

compatibility/get_last_tags.sh

@@ -0,0 +1,20 @@
+# update tags
+git fetch --tags
+
+getLatestTags() {
+  release_5_branch_regex="^release-5\.[0-9].*$"
+  release_4_branch_regex="^release-4\.[0-9].*$"
+  TOTAL_TAGS=$(git for-each-ref --sort=creatordate  refs/tags | awk -F '/' '{print $3}')
+  filter='alpha'
+  # latest tags
+  TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | grep -v $filter | tail -n3)
+  if git rev-parse --abbrev-ref HEAD | egrep -q $release_5_branch_regex
+  then
+    # If we are in release-5.0 branch, try to use latest 3 version of 5.x and last 4.x version
+    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | grep -v $filter | tail -n1 && echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v5." | grep -v $filter | tail -n3)
+  elif git rev-parse --abbrev-ref HEAD | egrep -q $release_4_branch_regex
+  then
+    # If we are in release-4.0 branch, try to use latest 3 version of 4.x
+    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | grep -v $filter | tail -n3)
+  fi
+}

compatibility/prepare_backup.sh

@@ -15,27 +15,9 @@
 
 set -eux
 
-# update tags
-git fetch --tags
+. compatibility/get_last_tags.sh
 
 TAGS="v5.0.0"
-getLatestTags() {
-  release_5_branch_regex="^release-5\.[0-9].*$"
-  release_4_branch_regex="^release-4\.[0-9].*$"
-  TOTAL_TAGS=$(git for-each-ref --sort=creatordate  refs/tags | awk -F '/' '{print $3}')
-  # latest tags
-  TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | tail -n3)
-  if git rev-parse --abbrev-ref HEAD | egrep -q $release_5_branch_regex
-  then
-    # If we are in release-5.0 branch, try to use latest 3 version of 5.x and last 4.x version
-    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | tail -n1 && echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v5." | tail -n3)
-  elif git rev-parse --abbrev-ref HEAD | egrep -q $release_4_branch_regex
-  then
-    # If we are in release-4.0 branch, try to use latest 3 version of 4.x
-    TAGS=$(echo $TOTAL_TAGS | tr ' ' '\n' | fgrep "v4." | tail -n3)
-  fi
-}
-
 getLatestTags
 echo "recent version of cluster is $TAGS"
 

docker/Dockerfile

@@ -1,5 +1,5 @@
 # For loading data to TiDB
-FROM golang:1.13.8-buster as go-ycsb-builder
+FROM golang:1.16.4-buster as go-ycsb-builder
 WORKDIR /go/src/github.com/pingcap/
 RUN git clone https://github.com/pingcap/go-ycsb.git && \
     cd go-ycsb && \
@@ -8,7 +8,7 @@ RUN git clone https://github.com/pingcap/go-ycsb.git && \
 # For operating minio S3 compatible storage
 FROM minio/mc as mc-builder
 
-FROM golang:1.13.8-buster
+FROM golang:1.16.4-buster
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git \

go.mod1

@@ -1,6 +1,6 @@
 module github.com/pingcap/br
 
-go 1.13
+go 1.16
 
 require (
 	cloud.google.com/go/storage v1.6.0
@@ -25,24 +25,23 @@ require (
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3
 	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd
-	github.com/pingcap/kvproto v0.0.0-20210507054410-a8152f8a876c
+	github.com/pingcap/kvproto v0.0.0-20210507074444-0ec2d0dc2e4b
 	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4
-	github.com/pingcap/parser v0.0.0-20210513020953-ae2c4497c07b
-	github.com/pingcap/tidb v1.1.0-beta.0.20210513061339-f2c2fbda476f
+	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307
+	github.com/pingcap/tidb v1.1.0-beta.0.20210602080826-ca3d88eba5c6
 	github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible
-	github.com/pingcap/tipb v0.0.0-20210422074242-57dd881b81b1
+	github.com/pingcap/tipb v0.0.0-20210525032549-b80be13ddf6c
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0
-	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d
 	github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457
 	github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
 	go.uber.org/atomic v1.7.0
-	go.uber.org/multierr v1.6.0
+	go.uber.org/multierr v1.7.0
 	go.uber.org/zap v1.16.0
 	golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d