Merge branch 'pingcap:master' into issue24031

br/pkg/lightning/backend/local/local.go

@@ -2351,11 +2351,9 @@ func (local *local) CleanupEngine(ctx context.Context, engineUUID uuid.UUID) err
 }
 
 func (local *local) CheckRequirements(ctx context.Context, checkCtx *backend.CheckCtx) error {
-	versionStr, err := local.g.GetSQLExecutor().ObtainStringWithLog(
-		ctx,
-		"SELECT version();",
-		"check TiDB version",
-		log.L())
+	// TODO: support lightning via SQL
+	db, _ := local.g.GetDB()
+	versionStr, err := version.FetchVersion(ctx, db)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -2369,8 +2367,8 @@ func (local *local) CheckRequirements(ctx context.Context, checkCtx *backend.Che
 		return err
 	}
 
-	tidbVersion, _ := version.ExtractTiDBVersion(versionStr)
-	return checkTiFlashVersion(ctx, local.g, checkCtx, *tidbVersion)
+	serverInfo := version.ParseServerInfo(versionStr)
+	return checkTiFlashVersion(ctx, local.g, checkCtx, *serverInfo.ServerVersion)
 }
 
 func checkTiDBVersion(_ context.Context, versionStr string, requiredMinVersion, requiredMaxVersion semver.Version) error {

br/pkg/lightning/restore/restore_test.go

@@ -986,8 +986,8 @@ func (s *tableRestoreSuite) TestTableRestoreMetrics(c *C) {
 	db, sqlMock, err := sqlmock.New()
 	c.Assert(err, IsNil)
 	g.EXPECT().GetDB().Return(db, nil).AnyTimes()
-	sqlMock.ExpectQuery("SELECT version\\(\\);").WillReturnRows(sqlmock.NewRows([]string{"version()"}).
-		AddRow("5.7.25-TiDB-v5.0.1"))
+	sqlMock.ExpectQuery("SELECT tidb_version\\(\\);").WillReturnRows(sqlmock.NewRows([]string{"tidb_version()"}).
+		AddRow("Release Version: v5.2.1\nEdition: Community\n"))
 
 	web.BroadcastInitProgress(rc.dbMetas)
 

br/pkg/version/version.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/log"
 	berrors "github.com/pingcap/tidb/br/pkg/errors"
+	"github.com/pingcap/tidb/br/pkg/logutil"
 	"github.com/pingcap/tidb/br/pkg/utils"
 	"github.com/pingcap/tidb/br/pkg/version/build"
 	pd "github.com/tikv/pd/client"
@@ -216,11 +217,11 @@ func ExtractTiDBVersion(version string) (*semver.Version, error) {
 
 // CheckTiDBVersion is equals to ExtractTiDBVersion followed by CheckVersion.
 func CheckTiDBVersion(versionStr string, requiredMinVersion, requiredMaxVersion semver.Version) error {
-	version, err := ExtractTiDBVersion(versionStr)
-	if err != nil {
-		return errors.Trace(err)
+	serverInfo := ParseServerInfo(versionStr)
+	if serverInfo.ServerType != ServerTypeTiDB {
+		return errors.Errorf("server with version '%s' is not TiDB", versionStr)
 	}
-	return CheckVersion("TiDB", *version, requiredMinVersion, requiredMaxVersion)
+	return CheckVersion("TiDB", *serverInfo.ServerVersion, requiredMinVersion, requiredMaxVersion)
 }
 
 // NormalizeBackupVersion normalizes the version string from backupmeta.
@@ -241,11 +242,22 @@ func NormalizeBackupVersion(version string) *semver.Version {
 }
 
 // FetchVersion gets the version information from the database server
+//
+// NOTE: the executed query will be:
+// - `select tidb_version()` if target db is tidb
+// - `select version()` if target db is not tidb
 func FetchVersion(ctx context.Context, db utils.QueryExecutor) (string, error) {
 	var versionInfo string
+	const queryTiDB = "SELECT tidb_version();"
+	tidbRow := db.QueryRowContext(ctx, queryTiDB)
+	err := tidbRow.Scan(&versionInfo)
+	if err == nil {
+		return versionInfo, nil
+	}
+	log.L().Warn("select tidb_version() failed, will fallback to 'select version();'", logutil.ShortError(err))
 	const query = "SELECT version();"
 	row := db.QueryRowContext(ctx, query)
-	err := row.Scan(&versionInfo)
+	err = row.Scan(&versionInfo)
 	if err != nil {
 		return "", errors.Annotatef(err, "sql: %s", query)
 	}
@@ -263,24 +275,51 @@ const (
 	ServerTypeMariaDB
 	// ServerTypeTiDB represents TiDB server type
 	ServerTypeTiDB
+
+	// ServerTypeAll represents All server types
+	ServerTypeAll
 )
 
+var serverTypeString = []string{
+	ServerTypeUnknown: "Unknown",
+	ServerTypeMySQL:   "MySQL",
+	ServerTypeMariaDB: "MariaDB",
+	ServerTypeTiDB:    "TiDB",
+}
+
+// String implements Stringer.String
+func (s ServerType) String() string {
+	if s >= ServerTypeAll {
+		return ""
+	}
+	return serverTypeString[s]
+}
+
 // ServerInfo is the combination of ServerType and ServerInfo
 type ServerInfo struct {
 	ServerType    ServerType
 	ServerVersion *semver.Version
+	HasTiKV       bool
 }
 
 var (
 	mysqlVersionRegex = regexp.MustCompile(`^\d+\.\d+\.\d+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?`)
-	tidbVersionRegex  = regexp.MustCompile(`-[v]?\d+\.\d+\.\d+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?`)
+	// `select version()` result
+	tidbVersionRegex = regexp.MustCompile(`-[v]?\d+\.\d+\.\d+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?`)
+	// `select tidb_version()` result
+	tidbReleaseVersionRegex = regexp.MustCompile(`v\d+\.\d+\.\d+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?`)
 )
 
 // ParseServerInfo parses exported server type and version info from version string
 func ParseServerInfo(src string) ServerInfo {
 	lowerCase := strings.ToLower(src)
 	serverInfo := ServerInfo{}
+	isReleaseVersion := false
 	switch {
+	case strings.Contains(lowerCase, "release version:"):
+		// this version string is tidb release version
+		serverInfo.ServerType = ServerTypeTiDB
+		isReleaseVersion = true
 	case strings.Contains(lowerCase, "tidb"):
 		serverInfo.ServerType = ServerTypeTiDB
 	case strings.Contains(lowerCase, "mariadb"):
@@ -293,7 +332,11 @@ func ParseServerInfo(src string) ServerInfo {
 
 	var versionStr string
 	if serverInfo.ServerType == ServerTypeTiDB {
-		versionStr = tidbVersionRegex.FindString(src)[1:]
+		if isReleaseVersion {
+			versionStr = tidbReleaseVersionRegex.FindString(src)
+		} else {
+			versionStr = tidbVersionRegex.FindString(src)[1:]
+		}
 		versionStr = strings.TrimPrefix(versionStr, "v")
 	} else {
 		versionStr = mysqlVersionRegex.FindString(src)
@@ -305,6 +348,13 @@ func ParseServerInfo(src string) ServerInfo {
 		log.L().Warn("fail to parse version",
 			zap.String("version", versionStr))
 	}
+	var version string
+	if serverInfo.ServerVersion != nil {
+		version = serverInfo.ServerVersion.String()
+	}
+	log.L().Info("detect server version",
+		zap.String("type", serverInfo.ServerType.String()),
+		zap.String("version", version))
 
 	return serverInfo
 }

br/pkg/version/version_test.go

@@ -4,7 +4,9 @@ package version
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
@@ -309,6 +311,8 @@ func (s *checkSuite) TestDetectServerInfo(c *C) {
 		{4, "5.7.25-TiDB-v3.0.7-58-g6adce2367", ServerTypeTiDB, mkVer(3, 0, 7, "58-g6adce2367")},
 		{5, "5.7.25-TiDB-3.0.6", ServerTypeTiDB, mkVer(3, 0, 6, "")},
 		{6, "invalid version", ServerTypeUnknown, (*semver.Version)(nil)},
+		{7, "Release Version: v5.2.1\nEdition: Community\nGit Commit Hash: cd8fb24c5f7ebd9d479ed228bb41848bd5e97445", ServerTypeTiDB, mkVer(5, 2, 1, "")},
+		{8, "Release Version: v5.4.0-alpha-21-g86caab907\nEdition: Community\nGit Commit Hash: 86caab907c481bbc4243b5a3346ec13907cc8721\nGit Branch: master", ServerTypeTiDB, mkVer(5, 4, 0, "alpha-21-g86caab907")},
 	}
 	dec := func(d []interface{}) (tag int, verStr string, tp ServerType, v *semver.Version) {
 		return d[0].(int), d[1].(string), ServerType(d[2].(int)), d[3].(*semver.Version)
@@ -318,8 +322,14 @@ func (s *checkSuite) TestDetectServerInfo(c *C) {
 		tag, r, serverTp, expectVer := dec(datum)
 		cmt := Commentf("test case number: %d", tag)
 
-		rows := sqlmock.NewRows([]string{"version"}).AddRow(r)
-		mock.ExpectQuery("SELECT version()").WillReturnRows(rows)
+		tidbVersionQuery := mock.ExpectQuery("SELECT tidb_version\\(\\);")
+		if strings.HasPrefix(r, "Release Version:") {
+			tidbVersionQuery.WillReturnRows(sqlmock.NewRows([]string{"tidb_version"}).AddRow(r))
+		} else {
+			tidbVersionQuery.WillReturnError(errors.New("mock error"))
+			rows := sqlmock.NewRows([]string{"version"}).AddRow(r)
+			mock.ExpectQuery("SELECT version\\(\\);").WillReturnRows(rows)
+		}
 
 		verStr, err := FetchVersion(context.Background(), db)
 		c.Assert(err, IsNil, cmt)
@@ -329,6 +339,7 @@ func (s *checkSuite) TestDetectServerInfo(c *C) {
 		if info.ServerVersion == nil {
 			c.Assert(expectVer, IsNil, cmt)
 		} else {
+			fmt.Printf("%v, %v\n", *info.ServerVersion, *expectVer)
 			c.Assert(info.ServerVersion.Equal(*expectVer), IsTrue)
 		}
 		c.Assert(mock.ExpectationsWereMet(), IsNil, cmt)
@@ -343,3 +354,39 @@ func makeVersion(major, minor, patch int64, preRelease string) *semver.Version {
 		Metadata:   "",
 	}
 }
+
+func (s *checkSuite) TestFetchVersion(c *C) {
+	db, mock, err := sqlmock.New()
+	c.Assert(err, IsNil)
+
+	tidbVersion := `Release Version: v5.2.1
+Edition: Community
+Git Commit Hash: cd8fb24c5f7ebd9d479ed228bb41848bd5e97445
+Git Branch: heads/refs/tags/v5.2.1
+UTC Build Time: 2021-09-08 02:32:56
+GoVersion: go1.16.4
+Race Enabled: false
+TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306
+Check Table Before Drop: false`
+
+	ctx := context.Background()
+	mock.ExpectQuery("SELECT tidb_version\\(\\);").WillReturnRows(sqlmock.
+		NewRows([]string{""}).AddRow(tidbVersion))
+	versionStr, err := FetchVersion(ctx, db)
+	c.Assert(err, IsNil)
+	c.Assert(versionStr, Equals, tidbVersion)
+
+	mock.ExpectQuery("SELECT tidb_version\\(\\);").WillReturnError(errors.New("mock failure"))
+	mock.ExpectQuery("SELECT version\\(\\);").WillReturnRows(sqlmock.
+		NewRows([]string{""}).AddRow("5.7.25"))
+	versionStr, err = FetchVersion(ctx, db)
+	c.Assert(err, IsNil)
+	c.Assert(versionStr, Equals, "5.7.25")
+
+	mock.ExpectQuery("SELECT tidb_version\\(\\);").WillReturnError(errors.New("mock failure"))
+	mock.ExpectQuery("SELECT version\\(\\);").WillReturnError(errors.New("mock failure"))
+
+	_, err = FetchVersion(ctx, db)
+	c.Assert(err, ErrorMatches, ".*mock failure")
+
+}

cmd/explaintest/r/new_character_set_builtin.result

@@ -163,3 +163,55 @@ select password(a), password(b), password(c) from t;
 password(a)	password(b)	password(c)
 *D13577D198CA3F0AF5C548195065991E0E3EE665	*A669F2B2DD49E2463FE62D8F72DDF4F858687EA5	*9FC0B2ABDF3EC9895E852B15BE432EE0EA0C26BA
 set @@tidb_enable_vectorized_expression = false;
+drop table if exists t;
+create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
+insert into t values ('一二三', '一二三', '一二三');
+select sha1(a), sha1(b), sha1(c) from t;
+sha1(a)	sha1(b)	sha1(c)
+01c1743ce7a7e822454a659f659bad61375ff10c	30cda4eed59a2ff592f2881f39d42fed6e10cad8	a112317779176843452f88e5b1fdbf3092ad124a
+select sha(a), sha(b), sha(c) from t;
+sha(a)	sha(b)	sha(c)
+01c1743ce7a7e822454a659f659bad61375ff10c	30cda4eed59a2ff592f2881f39d42fed6e10cad8	a112317779176843452f88e5b1fdbf3092ad124a
+set @@tidb_enable_vectorized_expression = true;
+select sha1(a), sha1(b), sha1(c) from t;
+sha1(a)	sha1(b)	sha1(c)
+01c1743ce7a7e822454a659f659bad61375ff10c	30cda4eed59a2ff592f2881f39d42fed6e10cad8	a112317779176843452f88e5b1fdbf3092ad124a
+select sha(a), sha(b), sha(c) from t;
+sha(a)	sha(b)	sha(c)
+01c1743ce7a7e822454a659f659bad61375ff10c	30cda4eed59a2ff592f2881f39d42fed6e10cad8	a112317779176843452f88e5b1fdbf3092ad124a
+set @@tidb_enable_vectorized_expression = false;
+drop table if exists t;
+create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
+insert into t values ('一二三', '一二三', '一二三');
+select sha2(a, 0), sha2(b, 0), sha2(c, 0) from t;
+sha2(a, 0)	sha2(b, 0)	sha2(c, 0)
+4fc9d8955b6155d931b24a583a6ad872f7d77fd4e4562cf8f619faa9c1a2cdc7	b6c1ae1f8d8a07426ddb13fca5124fb0b9f1f0ef1cca6730615099cf198ca8af	f98670d7ccdf803dfa2f24555cfd733d626dd24f5f2bb0b91e37cc4f54a0f359
+select sha2(a, 224), sha2(b, 224), sha2(c, 224) from t;
+sha2(a, 224)	sha2(b, 224)	sha2(c, 224)
+ae47a60dd96e1deed3988d8fff3d662165e0aac7ddf371f244d7c11e	2362f577783f6cd6cc10b0308f946f479fef868a39d6339b5d74cc6d	eb0de82dfada1205f65ac4aa6e4778d2f7730144f31688d42a316b9c
+select sha2(a, 256), sha2(b, 256), sha2(c, 256) from t;
+sha2(a, 256)	sha2(b, 256)	sha2(c, 256)
+4fc9d8955b6155d931b24a583a6ad872f7d77fd4e4562cf8f619faa9c1a2cdc7	b6c1ae1f8d8a07426ddb13fca5124fb0b9f1f0ef1cca6730615099cf198ca8af	f98670d7ccdf803dfa2f24555cfd733d626dd24f5f2bb0b91e37cc4f54a0f359
+select sha2(a, 384), sha2(b, 384), sha2(c, 384) from t;
+sha2(a, 384)	sha2(b, 384)	sha2(c, 384)
+cdb9c8d3e2579d021116ebe9d7d7bb4f5b3a489cae84768f7b3348c9b8d716897a409ea96fd92bfb95e3fd8aa91ffc74	54e75070f1faab03e7ce808ca2824ed4614ad1d58ee1409d8c1e4fd72ecab12c92ac3a2f919721c2aa09b23e5f3cc8aa	1b526bf926dbc53609628f145a6efe2b17eb29754ba2c6e0e33673dbd79d62ea50bbfc233252f06ba27712f8bd406963
+select sha2(a, 514), sha2(b, 514), sha2(c, 514) from t;
+sha2(a, 514)	sha2(b, 514)	sha2(c, 514)
+NULL	NULL	NULL
+set @@tidb_enable_vectorized_expression = true;
+select sha2(a, 0), sha2(b, 0), sha2(c, 0) from t;
+sha2(a, 0)	sha2(b, 0)	sha2(c, 0)
+4fc9d8955b6155d931b24a583a6ad872f7d77fd4e4562cf8f619faa9c1a2cdc7	4fc9d8955b6155d931b24a583a6ad872f7d77fd4e4562cf8f619faa9c1a2cdc7	f98670d7ccdf803dfa2f24555cfd733d626dd24f5f2bb0b91e37cc4f54a0f359
+select sha2(a, 224), sha2(b, 224), sha2(c, 224) from t;
+sha2(a, 224)	sha2(b, 224)	sha2(c, 224)
+ae47a60dd96e1deed3988d8fff3d662165e0aac7ddf371f244d7c11e	ae47a60dd96e1deed3988d8fff3d662165e0aac7ddf371f244d7c11e	eb0de82dfada1205f65ac4aa6e4778d2f7730144f31688d42a316b9c
+select sha2(a, 256), sha2(b, 256), sha2(c, 256) from t;
+sha2(a, 256)	sha2(b, 256)	sha2(c, 256)
+4fc9d8955b6155d931b24a583a6ad872f7d77fd4e4562cf8f619faa9c1a2cdc7	4fc9d8955b6155d931b24a583a6ad872f7d77fd4e4562cf8f619faa9c1a2cdc7	f98670d7ccdf803dfa2f24555cfd733d626dd24f5f2bb0b91e37cc4f54a0f359
+select sha2(a, 384), sha2(b, 384), sha2(c, 384) from t;
+sha2(a, 384)	sha2(b, 384)	sha2(c, 384)
+cdb9c8d3e2579d021116ebe9d7d7bb4f5b3a489cae84768f7b3348c9b8d716897a409ea96fd92bfb95e3fd8aa91ffc74	cdb9c8d3e2579d021116ebe9d7d7bb4f5b3a489cae84768f7b3348c9b8d716897a409ea96fd92bfb95e3fd8aa91ffc74	1b526bf926dbc53609628f145a6efe2b17eb29754ba2c6e0e33673dbd79d62ea50bbfc233252f06ba27712f8bd406963
+select sha2(a, 514), sha2(b, 514), sha2(c, 514) from t;
+sha2(a, 514)	sha2(b, 514)	sha2(c, 514)
+NULL	NULL	NULL
+set @@tidb_enable_vectorized_expression = false;

cmd/explaintest/t/new_character_set_builtin.test

@@ -84,3 +84,31 @@ select password(a), password(b), password(c) from t;
 set @@tidb_enable_vectorized_expression = true;
 select password(a), password(b), password(c) from t;
 set @@tidb_enable_vectorized_expression = false;
+
+-- test for builtin function sha1()/sha()
+drop table if exists t;
+create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
+insert into t values ('一二三', '一二三', '一二三');
+select sha1(a), sha1(b), sha1(c) from t;
+select sha(a), sha(b), sha(c) from t;
+set @@tidb_enable_vectorized_expression = true;
+select sha1(a), sha1(b), sha1(c) from t;
+select sha(a), sha(b), sha(c) from t;
+set @@tidb_enable_vectorized_expression = false;
+
+-- test for builtin function sha2()
+drop table if exists t;
+create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
+insert into t values ('一二三', '一二三', '一二三');
+select sha2(a, 0), sha2(b, 0), sha2(c, 0) from t;
+select sha2(a, 224), sha2(b, 224), sha2(c, 224) from t;
+select sha2(a, 256), sha2(b, 256), sha2(c, 256) from t;
+select sha2(a, 384), sha2(b, 384), sha2(c, 384) from t;
+select sha2(a, 514), sha2(b, 514), sha2(c, 514) from t;
+set @@tidb_enable_vectorized_expression = true;
+select sha2(a, 0), sha2(b, 0), sha2(c, 0) from t;
+select sha2(a, 224), sha2(b, 224), sha2(c, 224) from t;
+select sha2(a, 256), sha2(b, 256), sha2(c, 256) from t;
+select sha2(a, 384), sha2(b, 384), sha2(c, 384) from t;
+select sha2(a, 514), sha2(b, 514), sha2(c, 514) from t;
+set @@tidb_enable_vectorized_expression = false;

ddl/column_test.go

@@ -184,7 +184,7 @@ func testDropColumns(c *C, ctx sessionctx.Context, d *ddl, dbInfo *model.DBInfo,
 	return job
 }
 
-func (s *testColumnSuite) TestColumnAAA(c *C) {
+func (s *testColumnSuite) TestColumnBasic(c *C) {
 	d := testNewDDLAndStart(
 		context.Background(),
 		c,

ddl/db_cache_test.go

@@ -48,6 +48,8 @@ func (s *testDBSuite2) TestAlterTableCache(c *C) {
 	c.Assert(terror.ErrorEqual(domain.ErrInfoSchemaChanged, err), IsTrue)
 	/* Test can skip schema checker */
 	tk.MustExec("begin")
+	tk.MustExec("drop table if exists t1")
+	tk.MustExec("CREATE TABLE t1 (a int)")
 	tk.MustExec("insert into t1 set a=2;")
 	tk2.MustExec("alter table t2 cache")
 	tk.MustExec("commit")