Merge branch 'master' into zixiong-fix-move-table-startts

.github/workflows/integration.yml

@@ -9,7 +9,107 @@ on:
     branches: [ master ]
 
 jobs:
+  MySQL-integration:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.14.0'
+
+      - name: Cache Vendor
+        id: cache-vendor
+        uses: actions/cache@v2
+        with:
+          path: vendor
+          key: ${{ runner.os }}-cdc-integration-vendor-${{ hashFiles('go.sum') }}
+
+      - name: Update Vendor
+        run: go mod vendor
+
+      - name: Pull images
+        run: docker-compose -f docker-compose-avro.yml pull --ignore-pull-failures
+
+      - name: TiKV version
+        run: docker run pingcap/tikv:release-4.0-nightly -V
+
+      - name: Build Integration Framework
+        run: |
+          cd $GITHUB_WORKSPACE/integration
+          go build
+
+      - name: Run Integration Framework
+        timeout-minutes: 45
+        run: |
+          cd $GITHUB_WORKSPACE/integration
+          ./integration -protocol=mysql
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: logs
+          path: ${{ github.workspace }}/docker/logs/*
+
+      - name: Clean Up
+        if: ${{ always() }}
+        run: |
+          $GITHUB_WORKSPACE/scripts/avro-local-test.sh down
+
+  OldValue-integration:
+    needs: [MySQL-integration]
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.14.0'
+
+      - name: Cache Vendor
+        id: cache-vendor
+        uses: actions/cache@v2
+        with:
+          path: vendor
+          key: ${{ runner.os }}-cdc-integration-vendor-${{ hashFiles('go.sum') }}
+
+      - name: Update Vendor
+        run: go mod vendor
+
+      - name: Pull images
+        run: docker-compose -f docker-compose-avro.yml pull --ignore-pull-failures
+
+      - name: TiKV version
+        run: docker run pingcap/tikv:release-4.0-nightly -V
+
+      - name: Build Integration Framework
+        run: |
+          cd $GITHUB_WORKSPACE/integration
+          go build
+
+      - name: Run Integration Framework
+        timeout-minutes: 45
+        run: |
+          cd $GITHUB_WORKSPACE/integration
+          ./integration -protocol=simple-mysql-checking-old-value
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: logs
+          path: ${{ github.workspace }}/docker/logs/*
+
+      - name: Clean Up
+        if: ${{ always() }}
+        run: |
+          $GITHUB_WORKSPACE/scripts/avro-local-test.sh down
+
   CanalJson-integration:
+    needs: [OldValue-integration]
     runs-on: ubuntu-latest
 
     steps:

Dockerfile.development

@@ -4,7 +4,9 @@ WORKDIR /go/src/github.com/pingcap/ticdc
 COPY . .
 ENV CDC_ENABLE_VENDOR=1
 RUN go mod vendor
+RUN make failpoint-enable
 RUN make
+RUN make failpoint-disable
 
 FROM alpine:3.12
 RUN apk add --no-cache tzdata bash curl socat

Makefile

@@ -19,6 +19,7 @@ GOVENDORFLAG := -mod=vendor
 endif
 
 GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) -trimpath $(GOVENDORFLAG)
+GOBUILDNOVENDOR  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) -trimpath 
 ifeq ($(GOVERSION114), 1)
 GOTEST   := CGO_ENABLED=1 $(GO) test -p 3 --race -gcflags=all=-d=checkptr=0
 else
@@ -83,7 +84,7 @@ leak_test: check_failpoint_ctl
 	$(FAILPOINT_DISABLE)
 
 check_failpoint_ctl:
-	which $(FAILPOINT) >/dev/null 2>&1 || $(GOBUILD) -o $(FAILPOINT) github.com/pingcap/failpoint/failpoint-ctl
+	which $(FAILPOINT) >/dev/null 2>&1 || $(GOBUILDNOVENDOR) -o $(FAILPOINT) github.com/pingcap/failpoint/failpoint-ctl
 
 check_third_party_binary:
 	@which bin/tidb-server
@@ -142,8 +143,8 @@ check: check-copyright fmt lint check-static tidy errdoc
 
 coverage:
 	GO111MODULE=off go get github.com/wadey/gocovmerge
-	gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|$(CDC_PKG)/cdc/kv/testing.go|.*.__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
-	grep -vE ".*.pb.go|$(CDC_PKG)/cdc/kv/testing.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out"
+	gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|$(CDC_PKG)/cdc/kv/testing.go|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
+	grep -vE ".*.pb.go|$(CDC_PKG)/cdc/kv/testing.go|$(CDC_PKG)/cdc/sink/simple_mysql_tester.go|.*.__failpoint_binding__.go" "$(TEST_DIR)/cov.unit.out" > "$(TEST_DIR)/unit_cov.out"
 ifeq ("$(JenkinsCI)", "1")
 	GO111MODULE=off go get github.com/mattn/goveralls
 	@goveralls -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
@@ -173,8 +174,8 @@ tools/bin/golangci-lint: tools/check/go.mod
 	cd tools/check; test -e ../bin/golangci-lint || \
 	$(GO) build -o ../bin/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint
 
-failpoint-enable:
+failpoint-enable: check_failpoint_ctl
 	$(FAILPOINT_ENABLE)
 
-failpoint-disable:
+failpoint-disable: check_failpoint_ctl
 	$(FAILPOINT_DISABLE)

cdc/kv/store_op.go

@@ -15,7 +15,6 @@ package kv
 
 import (
 	"fmt"
-	"go.uber.org/zap"
 	"sync"
 	"time"
 
@@ -30,6 +29,7 @@ import (
 	"github.com/pingcap/tidb/meta"
 	"github.com/pingcap/tidb/store"
 	"github.com/pingcap/tidb/store/tikv"
+	"go.uber.org/zap"
 )
 
 const (
@@ -61,7 +61,6 @@ func newStorageWithCurVersionCache(storage tidbkv.Storage, cacheKey string) tidb
 		curVersionCache[cacheKey] = &curVersionCacheEntry{
 			ts:          0,
 			lastUpdated: time.Unix(0, 0),
-			mu:          sync.Mutex{},
 		}
 	}
 

cdc/sink/mysql.go

@@ -999,7 +999,7 @@ func reduceReplace(replaces map[string][][]interface{}, batchSize int) ([]string
 		cacheArgs := make([]interface{}, 0)
 		last := false
 		for i, val := range vals {
-			cacheCount += 1
+			cacheCount++
 			if i == len(vals)-1 || cacheCount >= batchSize {
 				last = true
 			}