From e4de23b08259514ca20797696b6cd021a6a6de1a Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 25 Sep 2020 12:37:38 +0800 Subject: [PATCH] support new collation for kv encoder (#407) * support new collation * add unit test * remove useless code * fix multi task * add a comment * add a integration for new collation * fix test * resolve comments --- lightning/restore/restore.go | 12 ++ lightning/restore/tidb.go | 14 ++ lightning/restore/tidb_test.go | 25 +++ tests/_utils/run_services | 260 ++++++++++++++++++++++++++++++++ tests/new_collation/config.toml | 0 tests/new_collation/run.sh | 65 ++++++++ tests/run.sh | 260 +------------------------------- 7 files changed, 379 insertions(+), 257 deletions(-) create mode 100755 tests/_utils/run_services create mode 100644 tests/new_collation/config.toml create mode 100644 tests/new_collation/run.sh diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 97b0db435..c0bad1deb 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -24,6 +24,8 @@ import ( "sync/atomic" "time" + "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/br/pkg/storage" "github.com/pingcap/errors" @@ -269,6 +271,7 @@ func (rc *RestoreController) Close() { func (rc *RestoreController) Run(ctx context.Context) error { opts := []func(context.Context) error{ rc.checkRequirements, + rc.setGlobalVariables, rc.restoreSchema, rc.restoreTables, rc.fullCompact, @@ -1296,6 +1299,15 @@ func (rc *RestoreController) checkRequirements(_ context.Context) error { return rc.backend.CheckRequirements() } +func (rc *RestoreController) setGlobalVariables(ctx context.Context) error { + // set new collation flag base on tidb config + enabled := ObtainNewCollationEnabled(ctx, rc.tidbMgr.db) + // we should enable/disable new collation here since in server mode, tidb config + // may be different in different tasks + collate.SetNewCollationEnabledForTest(enabled) + return nil +} + func (rc *RestoreController) waitCheckpointFinish() { // wait checkpoint process finish so that we can do cleanup safely close(rc.saveCpCh) diff --git a/lightning/restore/tidb.go b/lightning/restore/tidb.go index e2f07a149..2857dcd41 100644 --- a/lightning/restore/tidb.go +++ b/lightning/restore/tidb.go @@ -264,6 +264,20 @@ func ObtainRowFormatVersion(ctx context.Context, db *sql.DB) (rowFormatVersion s return } +func ObtainNewCollationEnabled(ctx context.Context, db *sql.DB) bool { + var newCollationVal string + err := common.SQLWithRetry{DB: db, Logger: log.L()}.QueryRow(ctx, "obtain new collation enabled", + "SELECT variable_value FROM mysql.tidb WHERE variable_name = 'new_collation_enabled'", + &newCollationVal, + ) + newCollationEnabled := false + if err == nil && newCollationVal == "True" { + newCollationEnabled = true + } + + return newCollationEnabled +} + func AlterAutoIncrement(ctx context.Context, db *sql.DB, tableName string, incr int64) error { sql := common.SQLWithRetry{ DB: db, diff --git a/lightning/restore/tidb_test.go b/lightning/restore/tidb_test.go index 6428d19c2..02a9ff679 100644 --- a/lightning/restore/tidb_test.go +++ b/lightning/restore/tidb_test.go @@ -352,3 +352,28 @@ func (s *tidbSuite) TestObtainRowFormatVersionFailure(c *C) { version := ObtainRowFormatVersion(ctx, s.timgr.db) c.Assert(version, Equals, "1") } + +func (s *tidbSuite) TestObtainNewCollationEnabled(c *C) { + ctx := context.Background() + + s.mockDB. + ExpectQuery("\\QSELECT variable_value FROM mysql.tidb WHERE variable_name = 'new_collation_enabled'\\E") + version := ObtainNewCollationEnabled(ctx, s.timgr.db) + c.Assert(version, Equals, false) + + kvMap := map[string]bool{ + "True": true, + "False": false, + } + for k, v := range kvMap { + s.mockDB. + ExpectQuery("\\QSELECT variable_value FROM mysql.tidb WHERE variable_name = 'new_collation_enabled'\\E"). + WillReturnRows(sqlmock.NewRows([]string{"variable_value"}).AddRow(k)) + + version := ObtainNewCollationEnabled(ctx, s.timgr.db) + c.Assert(version, Equals, v) + } + s.mockDB. + ExpectClose() + +} diff --git a/tests/_utils/run_services b/tests/_utils/run_services new file mode 100755 index 000000000..feab06522 --- /dev/null +++ b/tests/_utils/run_services @@ -0,0 +1,260 @@ +stop_services() { + killall -9 tikv-server || true + killall -9 pd-server || true + killall -9 tidb-server || true + killall -9 tikv-importer || true + killall -9 tiflash || true + + find "$TEST_DIR" -maxdepth 1 -not -path "$TEST_DIR" -not -name "cov.*" -not -name "*.log" | xargs rm -r || true +} + +start_services() { + stop_services + + TT="$TEST_DIR/tls" + mkdir -p "$TT" + rm -f "$TEST_DIR"/*.log + + # Ref: https://docs.microsoft.com/en-us/azure/application-gateway/self-signed-certificates + # gRPC only supports P-256 curves, see https://github.com/grpc/grpc/issues/6722 + echo "Generate TLS keys..." + cat - > "$TT/ipsan.cnf" < /dev/null + for cluster in tidb pd tikv importer lightning tiflash curl; do + openssl ecparam -out "$TT/$cluster.key" -name prime256v1 -genkey + openssl req -new -batch -sha256 -subj '/CN=localhost' -key "$TT/$cluster.key" -out "$TT/$cluster.csr" + openssl x509 -req -sha256 -days 1 -extensions EXT -extfile "$TT/ipsan.cnf" -in "$TT/$cluster.csr" -CA "$TT/ca.pem" -CAkey "$TT/ca.key" -CAcreateserial -out "$TT/$cluster.pem" 2> /dev/null + done + + cat - > "$TEST_DIR/pd-config.toml" < "$TEST_DIR/tikv-config.toml" < "$TEST_DIR/tidb-config.toml" <> "$TEST_DIR/tidb-config.toml" < "$TEST_DIR/importer-config.toml" < $TEST_DIR/tiflash-learner.toml < "$TEST_DIR/tiflash.toml" </dev/null 2>&1; do + i=$((i+1)) + if [ "$i" -gt 60 ]; then + echo "failed to start tiflash" + return 1 + fi + echo "TiFlash seems doesn't started, retrying..." + sleep 3 + done + fi +} \ No newline at end of file diff --git a/tests/new_collation/config.toml b/tests/new_collation/config.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/new_collation/run.sh b/tests/new_collation/run.sh new file mode 100644 index 000000000..ba983e59e --- /dev/null +++ b/tests/new_collation/run.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +check_cluster_version 4 0 0 'new collation' || { echo 'TiDB does not support new collation! skipping test'; exit 0; } + +set -euE + +cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +. $cur/../_utils/run_services + +COLLATION_ENABLED=$NEW_COLLATION +# restart cluster with new collation enabled +if [ -z "$NEW_COLLATION" ]; then + NEW_COLLATION=1 start_services +fi + +# Populate the mydumper source +DBPATH="$TEST_DIR/nc.mydump" +mkdir -p $DBPATH +echo 'CREATE DATABASE nc;' > "$DBPATH/nc-schema-create.sql" +# create table with collate `utf8_general_ci`, the index key will be different between old/new collation +echo "CREATE TABLE t(i INT PRIMARY KEY, s varchar(32), j TINYINT, KEY s_j (s, i)) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_general_ci;" > "$DBPATH/nc.t-schema.sql" +cat > "$DBPATH/nc.t.0.sql" << _EOF_ +INSERT INTO t (s, i, j) VALUES + ("this_is_test1", 1, 1), + ("this_is_test2", 2, 2), + ("this_is_test3", 3, 3), + ("this_is_test4", 4, 4), + ("this_is_test5", 5, 5); +_EOF_ +echo 'INSERT INTO t(s, i, j) VALUES ("another test case", 6, 6);' > "$DBPATH/nc.t.1.sql" + +for BACKEND in local importer tidb; do + # Start importing the tables. + run_sql 'DROP DATABASE IF EXISTS nc' + + run_lightning -d "$DBPATH" --backend $BACKEND 2> /dev/null + + run_sql 'SELECT count(*), sum(i) FROM `nc`.t' + check_contains "count(*): 6" + check_contains "sum(i): 21" + + # run sql with index `s_j`, if lightning don't support new collation, no result will be returned. + run_sql "SELECT j FROM nc.t WHERE s = 'This_Is_Test4'"; + check_contains "j: 4" + +done + +# restart with original config if needed +if [ -z "$COLLATION_ENABLED" ]; then + NEW_COLLATION= start_services +fi + diff --git a/tests/run.sh b/tests/run.sh index ea4b31cbe..a6ddebe5a 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -14,267 +14,13 @@ # limitations under the License. # enable start tiflash -TIFLASH="$TIFLASH" +export TIFLASH="$TIFLASH" +export NEW_COLLATION="$NEW_COLLATION" set -eu TEST_DIR=/tmp/lightning_test_result SELECTED_TEST_NAME="${TEST_NAME-$(find tests -mindepth 2 -maxdepth 2 -name run.sh | cut -d/ -f2 | sort)}" export PATH="tests/_utils:$PATH" - -stop_services() { - killall -9 tikv-server || true - killall -9 pd-server || true - killall -9 tidb-server || true - killall -9 tikv-importer || true - killall -9 tiflash || true - - find "$TEST_DIR" -maxdepth 1 -not -path "$TEST_DIR" -not -name "cov.*" -not -name "*.log" | xargs rm -r || true -} - -start_services() { - stop_services - - TT="$TEST_DIR/tls" - mkdir -p "$TT" - rm -f "$TEST_DIR"/*.log - - # Ref: https://docs.microsoft.com/en-us/azure/application-gateway/self-signed-certificates - # gRPC only supports P-256 curves, see https://github.com/grpc/grpc/issues/6722 - echo "Generate TLS keys..." - cat - > "$TT/ipsan.cnf" < /dev/null - for cluster in tidb pd tikv importer lightning tiflash curl; do - openssl ecparam -out "$TT/$cluster.key" -name prime256v1 -genkey - openssl req -new -batch -sha256 -subj '/CN=localhost' -key "$TT/$cluster.key" -out "$TT/$cluster.csr" - openssl x509 -req -sha256 -days 1 -extensions EXT -extfile "$TT/ipsan.cnf" -in "$TT/$cluster.csr" -CA "$TT/ca.pem" -CAkey "$TT/ca.key" -CAcreateserial -out "$TT/$cluster.pem" 2> /dev/null - done - - cat - > "$TEST_DIR/pd-config.toml" < "$TEST_DIR/tikv-config.toml" < "$TEST_DIR/tidb-config.toml" < "$TEST_DIR/importer-config.toml" < $TEST_DIR/tiflash-learner.toml < "$TEST_DIR/tiflash.toml" </dev/null 2>&1; do - i=$((i+1)) - if [ "$i" -gt 60 ]; then - echo "failed to start tiflash" - return 1 - fi - echo "TiFlash seems doesn't started, retrying..." - sleep 3 - done - fi -} +source tests/_utils/run_services trap stop_services EXIT start_services