diff --git a/go.mod b/go.mod index 35e3f2dc4..f6f5ae872 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20211011042309-a4518fcacbc8 github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348 - github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible + github.com/pingcap/tidb-tools v5.2.3-0.20211101071251-40e8f0cfcb1d+incompatible github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348 github.com/pingcap/tipb v0.0.0-20211026080602-ec68283c1735 github.com/prometheus/client_golang v1.7.1 @@ -52,5 +52,3 @@ require ( ) replace google.golang.org/grpc => google.golang.org/grpc v1.29.1 - -replace github.com/pingcap/tidb-tools => github.com/rleungx/tidb-tools v5.2.3-0.20211101050842-23301ee7b003+incompatible diff --git a/go.sum b/go.sum index aeed6eac4..749fb775b 100644 --- a/go.sum +++ b/go.sum @@ -704,6 +704,9 @@ github.com/pingcap/tidb v1.1.0-beta.0.20211026030648-c497d5c06348/go.mod h1:0x9V github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-dashboard v0.0.0-20210716172320-2226872e3296/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-dashboard v0.0.0-20211008050453-a25c25809529/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= +github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb-tools v5.2.3-0.20211101071251-40e8f0cfcb1d+incompatible h1:5AvxynVmWAE4ucC7uz2A42wjyCe/eIuXz9nYEJdWIyk= +github.com/pingcap/tidb-tools v5.2.3-0.20211101071251-40e8f0cfcb1d+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348 h1:Dt7Lsp9i6pMF/eKp/wMG2lO9ZsTM/gLSwRfeLUI3SDk= github.com/pingcap/tidb/parser v0.0.0-20211026030648-c497d5c06348/go.mod h1:e1MGCA9Sg3T8jid8PKAEq5eYVuMMCq4n8gJ+Kqp4Plg= @@ -761,8 +764,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rleungx/tidb-tools v5.2.3-0.20211101050842-23301ee7b003+incompatible h1:idV3TvNNQ8kSpvfqL+7T3CfuD4UWHdltPwKprF+C1Ns= -github.com/rleungx/tidb-tools v5.2.3-0.20211101050842-23301ee7b003+incompatible/go.mod h1:LvRVcSoPQLLy2RBajXo+FjXA7CQORqIMFYO30C5JgJ4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/tests/_utils/check_data b/tests/_utils/check_data index 6ecfb7ff6..3fcfc09f6 100755 --- a/tests/_utils/check_data +++ b/tests/_utils/check_data @@ -3,7 +3,10 @@ OUT_DIR=/tmp/tidb_binlog_test CONFIG_FILE=$1 -sync_diff_inspector -config=$CONFIG_FILE > $OUT_DIR/diff.log 2>&1 +cp $CONFIG_FILE $OUT_DIR/diff.toml +rm -rf $OUT_DIR/sync_diff_$TEST_NAME +sed -i "s/test-name-placeholder/$TEST_NAME/g" $OUT_DIR/diff.toml +sync_diff_inspector --log-level debug --config=$OUT_DIR/diff.toml > $OUT_DIR/diff.log 2>&1 if [ $? -ne 0 ]; then cat $OUT_DIR/diff.log exit 1 diff --git a/tests/reparo/run.sh b/tests/reparo/run.sh index 1557e75c1..eaab87687 100755 --- a/tests/reparo/run.sh +++ b/tests/reparo/run.sh @@ -14,7 +14,7 @@ run_drainer "$args" & GO111MODULE=on go build -o out -sleep 5 +sleep 8 run_sql "CREATE DATABASE IF NOT EXISTS \`reparo-test\`" diff --git a/tests/reparo/sync_diff_inspector.toml b/tests/reparo/sync_diff_inspector.toml index 47754b11a..91373832c 100644 --- a/tests/reparo/sync_diff_inspector.toml +++ b/tests/reparo/sync_diff_inspector.toml @@ -1,37 +1,42 @@ # diff Configuration. -log-level = "debug" - -# for example, the whole data is [1...100] -# we can split these data to [1...10], [11...20], ..., [91...100] -# the [1...10] is a chunk, and it's chunk size is 10 -# size of the split chunk -chunk-size = 100 +######################### Global config ######################### # how many goroutines are created to check data check-thread-count = 1 -# sampling check percent, for example 10 means only check 10% data -sample-percent = 100 - -# the name of the file which saves sqls used to fix different data -fix-sql-file = "/tmp/tidb_binlog_test/fix.sql" - -use-checksum=true +# set false if just want compare data by checksum, will skip select data when checksum is not equal. +# set true if want compare all different rows, will slow down the total compare time. +export-fix-sql = true -[[check-tables]] -schema = "reparo-test" -tables = ["~.*"] +# ignore check table's data +check-struct-only = false -[target-db] +######################### Databases config ######################### +[data-sources] +[data-sources.source1] host = "127.0.0.1" -port = 3306 +port = 4000 user = "root" password = "" -[[source-db]] +[data-sources.target] host = "127.0.0.1" -port = 4000 +port = 3306 user = "root" password = "" -instance-id = "source-1" + +######################### Task config ######################### +[task] +# 1 fix sql: fix-target-TIDB1.sql +# 2 log: sync-diff.log +# 3 summary: summary.txt +# 4 checkpoint: a dir +output-dir = "/tmp/tidb_binlog_test/sync_diff_test-name-placeholder" + +source-instances = ["source1"] + +target-instance = "target" + +# tables need to check. *Include `schema` and `table`. Use `.` to split* +target-check-tables = ["/^reparo-test$/.*"] diff --git a/tests/restart/sync_diff_inspector.toml b/tests/restart/sync_diff_inspector.toml index 8538029d1..184a52385 100644 --- a/tests/restart/sync_diff_inspector.toml +++ b/tests/restart/sync_diff_inspector.toml @@ -1,37 +1,42 @@ # diff Configuration. -log-level = "debug" - -# for example, the whole data is [1...100] -# we can split these data to [1...10], [11...20], ..., [91...100] -# the [1...10] is a chunk, and it's chunk size is 10 -# size of the split chunk -chunk-size = 100 +######################### Global config ######################### # how many goroutines are created to check data check-thread-count = 1 -# sampling check percent, for example 10 means only check 10% data -sample-percent = 100 - -# the name of the file which saves sqls used to fix different data -fix-sql-file = "/tmp/tidb_binlog_test/fix.sql" - -use-checksum=true +# set false if just want compare data by checksum, will skip select data when checksum is not equal. +# set true if want compare all different rows, will slow down the total compare time. +export-fix-sql = true -[[check-tables]] -schema = "RESTART_TEST" -tables = ["TEST"] +# ignore check table's data +check-struct-only = false -[target-db] +######################### Databases config ######################### +[data-sources] +[data-sources.source1] host = "127.0.0.1" -port = 3306 +port = 4000 user = "root" password = "" -[[source-db]] +[data-sources.target] host = "127.0.0.1" -port = 4000 +port = 3306 user = "root" password = "" -instance-id = "source-1" + +######################### Task config ######################### +[task] +# 1 fix sql: fix-target-TIDB1.sql +# 2 log: sync-diff.log +# 3 summary: summary.txt +# 4 checkpoint: a dir +output-dir = "/tmp/tidb_binlog_test/sync_diff_test-name-placeholder" + +source-instances = ["source1"] + +target-instance = "target" + +# tables need to check. *Include `schema` and `table`. Use `.` to split* +target-check-tables = ["RESTART_TEST.TEST"]