ddl, config: add a variable to control the max index length (#15012)

config/config.go

@@ -41,6 +41,10 @@ const (
 	DefTxnEntryCountLimit = 300 * 1000
 	// DefTxnTotalSizeLimit is the default value of TxnTxnTotalSizeLimit.
 	DefTxnTotalSizeLimit = 100 * 1024 * 1024
+	// DefMaxIndexLength is the maximum index length(in bytes). This value is consistent with MySQL.
+	DefMaxIndexLength = 3072
+	// DefMaxOfMaxIndexLength is the maximum index length(in bytes) for TiDB v3.0.7 and previous version.
+	DefMaxOfMaxIndexLength = 3072 * 4
 )
 
 // Valid config maps
@@ -90,6 +94,7 @@ type Config struct {
 	Plugin              Plugin            `toml:"plugin" json:"plugin"`
 	PessimisticTxn      PessimisticTxn    `toml:"pessimistic-txn" json:"pessimistic-txn"`
 	CheckMb4ValueInUTF8 bool              `toml:"check-mb4-value-in-utf8" json:"check-mb4-value-in-utf8"`
+	MaxIndexLength      int               `toml:"max-index-length" json:"max-index-length"`
 	// AlterPrimaryKey is used to control alter primary key feature.
 	AlterPrimaryKey bool `toml:"alter-primary-key" json:"alter-primary-key"`
 	// TreatOldVersionUTF8AsUTF8MB4 is use to treat old version table/column UTF8 charset as UTF8MB4. This is for compatibility.
@@ -351,6 +356,7 @@ var defaultConf = Config{
 	EnableStreaming:              false,
 	EnableBatchDML:               false,
 	CheckMb4ValueInUTF8:          true,
+	MaxIndexLength:               3072,
 	AlterPrimaryKey:              false,
 	TreatOldVersionUTF8AsUTF8MB4: true,
 	SplitRegionMaxNum:            1000,
@@ -588,6 +594,9 @@ func (c *Config) Valid() error {
 	if c.Store == "mocktikv" && !c.RunDDL {
 		return fmt.Errorf("can't disable DDL on mocktikv")
 	}
+	if c.MaxIndexLength < DefMaxIndexLength || c.MaxIndexLength > DefMaxOfMaxIndexLength {
+		return fmt.Errorf("max-index-length should be [%d, %d]", DefMaxIndexLength, DefMaxOfMaxIndexLength)
+	}
 	if c.Log.File.MaxSize > MaxLogFileSize {
 		return fmt.Errorf("invalid max log file size=%v which is larger than max=%v", c.Log.File.MaxSize, MaxLogFileSize)
 	}

config/config.toml.example

@@ -57,6 +57,18 @@ check-mb4-value-in-utf8 = true
 # treat-old-version-utf8-as-utf8mb4 use for upgrade compatibility. Set to true will treat old version table/column UTF8 charset as UTF8MB4.
 treat-old-version-utf8-as-utf8mb4 = true
 
+<<<<<<< HEAD
+=======
+# max-index-length is used to deal with compatibility issues from v3.0.7 and previous version upgrades. It can only be in [3072, 3072*4].
+max-index-length = 3072
+
+# enable-table-lock is used to control table lock feature. Default is false, indicate the table lock feature is disabled.
+enable-table-lock = false
+
+# delay-clean-table-lock is used to control the time (Milliseconds) of delay before unlock the table in the abnormal situation.
+delay-clean-table-lock = 0
+
+>>>>>>> 1771cf8... ddl, config: add a variable to control the max index length (#15012)
 # Maximum number of the splitting region, which is used by the split region statement.
 split-region-max-num = 1000
 

config/config_test.go

@@ -67,6 +67,13 @@ token-limit = 0
 alter-primary-key = true
 split-region-max-num=10000
 server-version = "test_version"
+<<<<<<< HEAD
+=======
+repair-mode = true
+max-server-connections = 200
+mem-quota-query = 10000
+max-index-length = 3080
+>>>>>>> 1771cf8... ddl, config: add a variable to control the max index length (#15012)
 [performance]
 txn-entry-count-limit=2000
 txn-total-size-limit=2000
@@ -112,6 +119,25 @@ history-size=100
 	c.Assert(conf.StmtSummary.MaxSQLLength, Equals, uint(1024))
 	c.Assert(conf.StmtSummary.RefreshInterval, Equals, 100)
 	c.Assert(conf.StmtSummary.HistorySize, Equals, 100)
+<<<<<<< HEAD
+=======
+	c.Assert(conf.EnableBatchDML, Equals, true)
+	c.Assert(conf.RepairMode, Equals, true)
+	c.Assert(conf.MaxServerConnections, Equals, uint32(200))
+	c.Assert(conf.MemQuotaQuery, Equals, int64(10000))
+	c.Assert(conf.Experimental.AllowAutoRandom, IsTrue)
+	c.Assert(conf.IsolationRead.Engines, DeepEquals, []string{"tiflash"})
+	c.Assert(conf.MaxIndexLength, Equals, 3080)
+
+	_, err = f.WriteString(`
+[log.file]
+log-rotate = true`)
+	c.Assert(err, IsNil)
+	err = conf.Load(configFile)
+	tmp := err.(*ErrConfigValidationFailed)
+	c.Assert(isAllDeprecatedConfigItems(tmp.UndecodedItems), IsTrue)
+
+>>>>>>> 1771cf8... ddl, config: add a variable to control the max index length (#15012)
 	c.Assert(f.Close(), IsNil)
 	c.Assert(os.Remove(configFile), IsNil)
 
@@ -246,3 +272,65 @@ func (s *testConfigSuite) TestOOMActionValid(c *C) {
 		c.Assert(c1.Valid() == nil, Equals, tt.valid)
 	}
 }
+<<<<<<< HEAD
+=======
+
+func (s *testConfigSuite) TestTxnTotalSizeLimitValid(c *C) {
+	conf := NewConfig()
+	tests := []struct {
+		limit uint64
+		valid bool
+	}{
+		{4 << 10, true},
+		{10 << 30, true},
+		{10<<30 + 1, false},
+	}
+
+	for _, tt := range tests {
+		conf.Performance.TxnTotalSizeLimit = tt.limit
+		c.Assert(conf.Valid() == nil, Equals, tt.valid)
+	}
+
+	conf.Binlog.Enable = true
+	conf.Performance.TxnTotalSizeLimit = 100<<20 + 1
+	c.Assert(conf.Valid(), NotNil)
+}
+
+func (s *testConfigSuite) TestAllowAutoRandomValid(c *C) {
+	conf := NewConfig()
+	checkValid := func(allowAlterPK, allowAutoRand, shouldBeValid bool) {
+		conf.AlterPrimaryKey = allowAlterPK
+		conf.Experimental.AllowAutoRandom = allowAutoRand
+		c.Assert(conf.Valid() == nil, Equals, shouldBeValid)
+	}
+	checkValid(true, true, false)
+	checkValid(true, false, true)
+	checkValid(false, true, true)
+	checkValid(false, false, true)
+}
+
+func (s *testConfigSuite) TestMaxIndexLength(c *C) {
+	conf := NewConfig()
+	checkValid := func(indexLen int, shouldBeValid bool) {
+		conf.MaxIndexLength = indexLen
+		c.Assert(conf.Valid() == nil, Equals, shouldBeValid)
+	}
+	checkValid(DefMaxIndexLength, true)
+	checkValid(DefMaxIndexLength-1, false)
+	checkValid(DefMaxOfMaxIndexLength, true)
+	checkValid(DefMaxOfMaxIndexLength+1, false)
+}
+
+func (s *testConfigSuite) TestParsePath(c *C) {
+	etcdAddrs, disableGC, err := ParsePath("tikv://node1:2379,node2:2379")
+	c.Assert(err, IsNil)
+	c.Assert(etcdAddrs, DeepEquals, []string{"node1:2379", "node2:2379"})
+	c.Assert(disableGC, IsFalse)
+
+	_, _, err = ParsePath("tikv://node1:2379")
+	c.Assert(err, IsNil)
+	_, disableGC, err = ParsePath("tikv://node1:2379?disableGC=true")
+	c.Assert(err, IsNil)
+	c.Assert(disableGC, IsTrue)
+}
+>>>>>>> 1771cf8... ddl, config: add a variable to control the max index length (#15012)

ddl/ddl.go

@@ -32,6 +32,7 @@ import (
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
 	pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client"
+	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl/util"
 	"github.com/pingcap/tidb/infoschema"
 	"github.com/pingcap/tidb/kv"
@@ -90,6 +91,7 @@ var (
 	errInvalidStoreVer       = terror.ClassDDL.New(codeInvalidStoreVer, "invalid storage current version")
 
 	// We don't support dropping column with index covered now.
+<<<<<<< HEAD
 	errCantDropColWithIndex     = terror.ClassDDL.New(codeCantDropColWithIndex, "can't drop column with index")
 	errUnsupportedAddColumn     = terror.ClassDDL.New(codeUnsupportedAddColumn, "unsupported add column")
 	errUnsupportedModifyColumn  = terror.ClassDDL.New(codeUnsupportedModifyColumn, "unsupported modify column %s")
@@ -113,6 +115,28 @@ var (
 	errTooManyFields            = terror.ClassDDL.New(codeTooManyFields, "Too many columns")
 	errInvalidSplitRegionRanges = terror.ClassDDL.New(codeInvalidRanges, "Failed to split region ranges")
 	errReorgPanic               = terror.ClassDDL.New(codeReorgWorkerPanic, "reorg worker panic.")
+=======
+	errCantDropColWithIndex      = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "drop column with index"))
+	errUnsupportedAddColumn      = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "add column"))
+	errUnsupportedModifyColumn   = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "modify column: %s"))
+	errUnsupportedModifyCharset  = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "modify %s"))
+	errUnsupportedPKHandle       = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "drop integer primary key"))
+	errUnsupportedCharset        = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "charset %s and collate %s"))
+	errUnsupportedShardRowIDBits = terror.ClassDDL.New(mysql.ErrUnsupportedDDLOperation, fmt.Sprintf(mysql.MySQLErrName[mysql.ErrUnsupportedDDLOperation], "shard_row_id_bits for table with primary key as row id"))
+	errBlobKeyWithoutLength      = terror.ClassDDL.New(mysql.ErrBlobKeyWithoutLength, mysql.MySQLErrName[mysql.ErrBlobKeyWithoutLength])
+	errKeyPart0                  = terror.ClassDDL.New(mysql.ErrKeyPart0, mysql.MySQLErrName[mysql.ErrKeyPart0])
+	errIncorrectPrefixKey        = terror.ClassDDL.New(mysql.ErrWrongSubKey, mysql.MySQLErrName[mysql.ErrWrongSubKey])
+	errTooLongKey                = terror.ClassDDL.New(mysql.ErrTooLongKey,
+		fmt.Sprintf(mysql.MySQLErrName[mysql.ErrTooLongKey], config.GetGlobalConfig().MaxIndexLength))
+	errKeyColumnDoesNotExits    = terror.ClassDDL.New(mysql.ErrKeyColumnDoesNotExits, mysql.MySQLErrName[mysql.ErrKeyColumnDoesNotExits])
+	errUnknownTypeLength        = terror.ClassDDL.New(mysql.ErrUnknownTypeLength, mysql.MySQLErrName[mysql.ErrUnknownTypeLength])
+	errUnknownFractionLength    = terror.ClassDDL.New(mysql.ErrUnknownFractionLength, mysql.MySQLErrName[mysql.ErrUnknownFractionLength])
+	errInvalidDDLJobVersion     = terror.ClassDDL.New(mysql.ErrInvalidDDLJobVersion, mysql.MySQLErrName[mysql.ErrInvalidDDLJobVersion])
+	errInvalidUseOfNull         = terror.ClassDDL.New(mysql.ErrInvalidUseOfNull, mysql.MySQLErrName[mysql.ErrInvalidUseOfNull])
+	errTooManyFields            = terror.ClassDDL.New(mysql.ErrTooManyFields, mysql.MySQLErrName[mysql.ErrTooManyFields])
+	errInvalidSplitRegionRanges = terror.ClassDDL.New(mysql.ErrInvalidSplitRegionRanges, mysql.MySQLErrName[mysql.ErrInvalidSplitRegionRanges])
+	errReorgPanic               = terror.ClassDDL.New(mysql.ErrReorgPanic, mysql.MySQLErrName[mysql.ErrReorgPanic])
+>>>>>>> 1771cf8... ddl, config: add a variable to control the max index length (#15012)
 	errFkColumnCannotDrop       = terror.ClassDDL.New(mysql.ErrFkColumnCannotDrop, mysql.MySQLErrName[mysql.ErrFkColumnCannotDrop])
 	errFKIncompatibleColumns    = terror.ClassDDL.New(mysql.ErrFKIncompatibleColumns, mysql.MySQLErrName[mysql.ErrFKIncompatibleColumns])
 

ddl/index.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pingcap/parser/charset"
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/parser/mysql"
+	"github.com/pingcap/tidb/config"
 	ddlutil "github.com/pingcap/tidb/ddl/util"
 	"github.com/pingcap/tidb/expression"
 	"github.com/pingcap/tidb/infoschema"
@@ -49,7 +50,6 @@ import (
 )
 
 const (
-	maxPrefixLength = 3072
 	// MaxCommentLength is exported for testing.
 	MaxCommentLength = 1024
 )
@@ -77,7 +77,7 @@ func buildIndexColumns(columns []*model.ColumnInfo, idxColNames []*ast.IndexColN
 		sumLength += indexColumnLength
 
 		// The sum of all lengths must be shorter than the max length for prefix.
-		if sumLength > maxPrefixLength {
+		if sumLength > config.GetGlobalConfig().MaxIndexLength {
 			return nil, errors.Trace(errTooLongKey)
 		}
 
@@ -122,7 +122,7 @@ func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.Ind
 		}
 		sumLength += indexColumnLength
 		// The sum of all lengths must be shorter than the max length for prefix.
-		if sumLength > maxPrefixLength {
+		if sumLength > config.GetGlobalConfig().MaxIndexLength {
 			return errors.Trace(errTooLongKey)
 		}
 	}
@@ -156,7 +156,7 @@ func checkIndexColumn(col *model.ColumnInfo, ic *ast.IndexColName) error {
 	}
 
 	// Specified length must be shorter than the max length for prefix.
-	if ic.Length > maxPrefixLength {
+	if ic.Length > config.GetGlobalConfig().MaxIndexLength {
 		return errors.Trace(errTooLongKey)
 	}
 	return nil

ddl/serial_test.go

@@ -82,6 +82,24 @@ func (s *testSerialSuite) TearDownSuite(c *C) {
 	}
 }
 
+func (s *testSerialSuite) TestChangeMaxIndexLength(c *C) {
+	tk := testkit.NewTestKitWithInit(c, s.store)
+	cfg := config.GetGlobalConfig()
+	newCfg := *cfg
+	originalMaxIndexLen := cfg.MaxIndexLength
+	newCfg.MaxIndexLength = config.DefMaxOfMaxIndexLength
+	config.StoreGlobalConfig(&newCfg)
+	defer func() {
+		newCfg.MaxIndexLength = originalMaxIndexLen
+		config.StoreGlobalConfig(&newCfg)
+	}()
+
+	tk.MustExec("create table t (c1 varchar(3073), index(c1)) charset = ascii;")
+	tk.MustExec(fmt.Sprintf("create table t1 (c1 varchar(%d), index(c1)) charset = ascii;", config.DefMaxOfMaxIndexLength))
+	tk.MustGetErrCode(fmt.Sprintf("create table t2 (c1 varchar(%d), index(c1)) charset = ascii;", config.DefMaxOfMaxIndexLength+1), mysql.ErrTooLongKey)
+	tk.MustExec("drop table t, t1")
+}
+
 func (s *testSerialSuite) TestPrimaryKey(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")