Skip to content

Commit

Permalink
Merge branch 'master' into updateProto
Browse files Browse the repository at this point in the history
  • Loading branch information
lichunzhu authored Nov 1, 2021
2 parents f420249 + bb0d31f commit 804eb54
Show file tree
Hide file tree
Showing 6 changed files with 63 additions and 51 deletions.
4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ require (
github.com/pingcap/kvproto v0.0.0-20211011042309-a4518fcacbc8
github.com/pingcap/log v0.0.0-20210906054005-afc726e70354
github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348
github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible
github.com/pingcap/tidb-tools v5.2.3-0.20211101071251-40e8f0cfcb1d+incompatible
github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348
github.com/pingcap/tipb v0.0.0-20211026080602-ec68283c1735
github.com/prometheus/client_golang v1.7.1
Expand All @@ -52,5 +52,3 @@ require (
)

replace google.golang.org/grpc => google.golang.org/grpc v1.29.1

replace github.com/pingcap/tidb-tools => github.com/rleungx/tidb-tools v5.2.3-0.20211101050842-23301ee7b003+incompatible
5 changes: 3 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,9 @@ github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348/go.mod h1:0x9V
github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo=
github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ=
github.com/pingcap/tidb-dashboard v0.0.0-20211008050453-a25c25809529/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ=
github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tidb-tools v5.2.3-0.20211101071251-40e8f0cfcb1d+incompatible h1:5AvxynVmWAE4ucC7uz2A42wjyCe/eIuXz9nYEJdWIyk=
github.com/pingcap/tidb-tools v5.2.3-0.20211101071251-40e8f0cfcb1d+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg=
github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348 h1:Dt7Lsp9i6pMF/eKp/wMG2lO9ZsTM/gLSwRfeLUI3SDk=
github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg=
Expand Down Expand Up @@ -761,8 +764,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rleungx/tidb-tools v5.2.3-0.20211101050842-23301ee7b003+incompatible h1:idV3TvNNQ8kSpvfqL+7T3CfuD4UWHdltPwKprF+C1Ns=
github.com/rleungx/tidb-tools v5.2.3-0.20211101050842-23301ee7b003+incompatible/go.mod h1:LvRVcSoPQLLy2RBajXo+FjXA7CQORqIMFYO30C5JgJ4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
Expand Down
5 changes: 4 additions & 1 deletion tests/_utils/check_data
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
OUT_DIR=/tmp/tidb_binlog_test

CONFIG_FILE=$1
sync_diff_inspector -config=$CONFIG_FILE > $OUT_DIR/diff.log 2>&1
cp $CONFIG_FILE $OUT_DIR/diff.toml
rm -rf $OUT_DIR/sync_diff_$TEST_NAME
sed -i "s/test-name-placeholder/$TEST_NAME/g" $OUT_DIR/diff.toml
sync_diff_inspector --log-level debug --config=$OUT_DIR/diff.toml > $OUT_DIR/diff.log 2>&1
if [ $? -ne 0 ]; then
cat $OUT_DIR/diff.log
exit 1
Expand Down
2 changes: 1 addition & 1 deletion tests/reparo/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ run_drainer "$args" &

GO111MODULE=on go build -o out

sleep 5
sleep 8

run_sql "CREATE DATABASE IF NOT EXISTS \`reparo-test\`"

Expand Down
49 changes: 27 additions & 22 deletions tests/reparo/sync_diff_inspector.toml
Original file line number Diff line number Diff line change
@@ -1,37 +1,42 @@
# diff Configuration.

log-level = "debug"

# for example, the whole data is [1...100]
# we can split these data to [1...10], [11...20], ..., [91...100]
# the [1...10] is a chunk, and it's chunk size is 10
# size of the split chunk
chunk-size = 100
######################### Global config #########################

# how many goroutines are created to check data
check-thread-count = 1

# sampling check percent, for example 10 means only check 10% data
sample-percent = 100

# the name of the file which saves sqls used to fix different data
fix-sql-file = "/tmp/tidb_binlog_test/fix.sql"

use-checksum=true
# set false if just want compare data by checksum, will skip select data when checksum is not equal.
# set true if want compare all different rows, will slow down the total compare time.
export-fix-sql = true

[[check-tables]]
schema = "reparo-test"
tables = ["~.*"]
# ignore check table's data
check-struct-only = false

[target-db]
######################### Databases config #########################
[data-sources]
[data-sources.source1]
host = "127.0.0.1"
port = 3306
port = 4000
user = "root"
password = ""

[[source-db]]
[data-sources.target]
host = "127.0.0.1"
port = 4000
port = 3306
user = "root"
password = ""
instance-id = "source-1"

######################### Task config #########################
[task]
# 1 fix sql: fix-target-TIDB1.sql
# 2 log: sync-diff.log
# 3 summary: summary.txt
# 4 checkpoint: a dir
output-dir = "/tmp/tidb_binlog_test/sync_diff_test-name-placeholder"

source-instances = ["source1"]

target-instance = "target"

# tables need to check. *Include `schema` and `table`. Use `.` to split*
target-check-tables = ["/^reparo-test$/.*"]
49 changes: 27 additions & 22 deletions tests/restart/sync_diff_inspector.toml
Original file line number Diff line number Diff line change
@@ -1,37 +1,42 @@
# diff Configuration.

log-level = "debug"

# for example, the whole data is [1...100]
# we can split these data to [1...10], [11...20], ..., [91...100]
# the [1...10] is a chunk, and it's chunk size is 10
# size of the split chunk
chunk-size = 100
######################### Global config #########################

# how many goroutines are created to check data
check-thread-count = 1

# sampling check percent, for example 10 means only check 10% data
sample-percent = 100

# the name of the file which saves sqls used to fix different data
fix-sql-file = "/tmp/tidb_binlog_test/fix.sql"

use-checksum=true
# set false if just want compare data by checksum, will skip select data when checksum is not equal.
# set true if want compare all different rows, will slow down the total compare time.
export-fix-sql = true

[[check-tables]]
schema = "RESTART_TEST"
tables = ["TEST"]
# ignore check table's data
check-struct-only = false

[target-db]
######################### Databases config #########################
[data-sources]
[data-sources.source1]
host = "127.0.0.1"
port = 3306
port = 4000
user = "root"
password = ""

[[source-db]]
[data-sources.target]
host = "127.0.0.1"
port = 4000
port = 3306
user = "root"
password = ""
instance-id = "source-1"

######################### Task config #########################
[task]
# 1 fix sql: fix-target-TIDB1.sql
# 2 log: sync-diff.log
# 3 summary: summary.txt
# 4 checkpoint: a dir
output-dir = "/tmp/tidb_binlog_test/sync_diff_test-name-placeholder"

source-instances = ["source1"]

target-instance = "target"

# tables need to check. *Include `schema` and `table`. Use `.` to split*
target-check-tables = ["RESTART_TEST.TEST"]

0 comments on commit 804eb54

Please sign in to comment.