Merge branch 'master' into master

cmd/importer/data.go

@@ -19,7 +19,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cznic/mathutil"
+	"github.com/pingcap/tidb/util/mathutil"
 )
 
 type datum struct {

config/config.go

@@ -104,6 +104,8 @@ type Config struct {
 	// RepairMode indicates that the TiDB is in the repair mode for table meta.
 	RepairMode      bool     `toml:"repair-mode" json:"repair-mode"`
 	RepairTableList []string `toml:"repair-table-list" json:"repair-table-list"`
+
+	Experimental Experimental `toml:"experimental" json:"experimental"`
 }
 
 // nullableBool defaults unset bool options to unset instead of false, which enables us to know if the user has set 2
@@ -429,6 +431,13 @@ type StmtSummary struct {
 	HistorySize int `toml:"history-size" json:"history-size"`
 }
 
+// Experimental controls the features that are still experimental: their semantics, interfaces are subject to change.
+// Using these features in the production environment is not recommended.
+type Experimental struct {
+	// Whether enable the syntax like `auto_random(3)` on the primary key column.
+	AllowAutoRandom bool `toml:"allow-auto-random" json:"allow-auto-random"`
+}
+
 var defaultConf = Config{
 	Host:                         "0.0.0.0",
 	AdvertiseAddress:             "",
@@ -543,6 +552,9 @@ var defaultConf = Config{
 		RefreshInterval: 1800,
 		HistorySize:     24,
 	},
+	Experimental: Experimental{
+		AllowAutoRandom: false,
+	},
 }
 
 var (
@@ -735,6 +747,10 @@ func (c *Config) Valid() error {
 	if c.StmtSummary.RefreshInterval <= 0 {
 		return fmt.Errorf("refresh-interval in [stmt-summary] should be greater than 0")
 	}
+
+	if c.AlterPrimaryKey && c.Experimental.AllowAutoRandom {
+		return fmt.Errorf("allow-auto-random is unavailable when alter-primary-key is enabled")
+	}
 	return nil
 }
 

config/config.toml.example

@@ -361,3 +361,9 @@ refresh-interval = 1800
 
 # the maximum history size of statement summary.
 history-size = 24
+
+# experimental section controls the features that are still experimental: their semantics,
+# interfaces are subject to change, using these features in the production environment is not recommended.
+[experimental]
+# enable column attribute `auto_random` to be defined on the primary key column.
+allow-auto-random = false

config/config_test.go

@@ -196,6 +196,8 @@ max-stmt-count=1000
 max-sql-length=1024
 refresh-interval=100
 history-size=100
+[experimental]
+allow-auto-random = true
 `)
 
 	c.Assert(err, IsNil)
@@ -228,6 +230,7 @@ history-size=100
 	c.Assert(conf.StmtSummary.HistorySize, Equals, 100)
 	c.Assert(conf.EnableBatchDML, Equals, true)
 	c.Assert(conf.RepairMode, Equals, true)
+	c.Assert(conf.Experimental.AllowAutoRandom, IsTrue)
 	c.Assert(f.Close(), IsNil)
 	c.Assert(os.Remove(configFile), IsNil)
 
@@ -383,3 +386,16 @@ func (s *testConfigSuite) TestTxnTotalSizeLimitValid(c *C) {
 	conf.Performance.TxnTotalSizeLimit = 100<<20 + 1
 	c.Assert(conf.Valid(), NotNil)
 }
+
+func (s *testConfigSuite) TestAllowAutoRandomValid(c *C) {
+	conf := NewConfig()
+	checkValid := func(allowAlterPK, allowAutoRand, shouldBeValid bool) {
+		conf.AlterPrimaryKey = allowAlterPK
+		conf.Experimental.AllowAutoRandom = allowAutoRand
+		c.Assert(conf.Valid() == nil, Equals, shouldBeValid)
+	}
+	checkValid(true, true, false)
+	checkValid(true, false, true)
+	checkValid(false, true, true)
+	checkValid(false, false, true)
+}

ddl/column_change_test.go

@@ -356,8 +356,8 @@ func getCurrentTable(d *ddl, schemaID, tableID int64) (table.Table, error) {
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	alloc := autoid.NewAllocator(d.store, schemaID, false)
-	tbl, err := table.TableFromMeta(alloc, tblInfo)
+	alloc := autoid.NewAllocator(d.store, schemaID, false, autoid.RowIDAllocType)
+	tbl, err := table.TableFromMeta(autoid.NewAllocators(alloc), tblInfo)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}

ddl/ddl.go

@@ -226,6 +226,8 @@ var (
 	ErrFieldNotFoundPart = terror.ClassDDL.New(mysql.ErrFieldNotFoundPart, mysql.MySQLErrName[mysql.ErrFieldNotFoundPart])
 	// ErrWrongTypeColumnValue returns 'Partition column values of incorrect type'
 	ErrWrongTypeColumnValue = terror.ClassDDL.New(mysql.ErrWrongTypeColumnValue, mysql.MySQLErrName[mysql.ErrWrongTypeColumnValue])
+	// ErrInvalidAutoRandom returns when auto_random is used incorrectly.
+	ErrInvalidAutoRandom = terror.ClassDDL.New(mysql.ErrInvalidAutoRandom, mysql.MySQLErrName[mysql.ErrInvalidAutoRandom])
 )
 
 // DDL is responsible for updating schema in data store and maintaining in-memory InfoSchema cache.
@@ -670,6 +672,7 @@ func init() {
 		mysql.ErrGeneratedColumnFunctionIsNotAllowed:  mysql.ErrGeneratedColumnFunctionIsNotAllowed,
 		mysql.ErrGeneratedColumnNonPrior:              mysql.ErrGeneratedColumnNonPrior,
 		mysql.ErrGeneratedColumnRefAutoInc:            mysql.ErrGeneratedColumnRefAutoInc,
+		mysql.ErrInvalidAutoRandom:                    mysql.ErrInvalidAutoRandom,
 		mysql.ErrInvalidDDLJob:                        mysql.ErrInvalidDDLJob,
 		mysql.ErrInvalidDDLState:                      mysql.ErrInvalidDDLState,
 		mysql.ErrInvalidDDLWorker:                     mysql.ErrInvalidDDLWorker,

ddl/ddl_api.go

@@ -26,7 +26,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/cznic/mathutil"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/ast"
 	"github.com/pingcap/parser/charset"
@@ -49,6 +48,7 @@ import (
 	"github.com/pingcap/tidb/util/chunk"
 	"github.com/pingcap/tidb/util/domainutil"
 	"github.com/pingcap/tidb/util/logutil"
+	"github.com/pingcap/tidb/util/mathutil"
 	"github.com/pingcap/tidb/util/mock"
 	"github.com/pingcap/tidb/util/set"
 	"go.uber.org/zap"
@@ -896,9 +896,9 @@ func checkDuplicateColumn(cols []interface{}) error {
 	return nil
 }
 
-func checkIsAutoIncrementColumn(colDefs *ast.ColumnDef) bool {
-	for _, option := range colDefs.Options {
-		if option.Tp == ast.ColumnOptionAutoIncrement {
+func containsColumnOption(colDef *ast.ColumnDef, opTp ast.ColumnOptionType) bool {
+	for _, option := range colDef.Options {
+		if option.Tp == opTp {
 			return true
 		}
 	}
@@ -917,7 +917,7 @@ func checkGeneratedColumn(colDefs []*ast.ColumnDef) error {
 				}
 			}
 		}
-		if checkIsAutoIncrementColumn(colDef) {
+		if containsColumnOption(colDef, ast.ColumnOptionAutoIncrement) {
 			exists, autoIncrementColumn = true, colDef.Name.Name.L
 		}
 		generated, depCols := findDependedColumnNames(colDef)
@@ -1096,6 +1096,58 @@ func checkConstraintNames(constraints []*ast.Constraint) error {
 	return nil
 }
 
+func setTableAutoRandomBits(tbInfo *model.TableInfo, colDefs []*ast.ColumnDef) error {
+	allowAutoRandom := config.GetGlobalConfig().Experimental.AllowAutoRandom
+	pkColName := tbInfo.GetPkName()
+	for _, col := range colDefs {
+		if containsColumnOption(col, ast.ColumnOptionAutoRandom) {
+			if !allowAutoRandom {
+				return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomExperimentalDisabledErrMsg)
+			}
+			if !tbInfo.PKIsHandle || col.Name.Name.L != pkColName.L {
+				return ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(autoid.AutoRandomPKisNotHandleErrMsg, col.Name.Name.O))
+			}
+			if containsColumnOption(col, ast.ColumnOptionAutoIncrement) {
+				return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg)
+			}
+			if containsColumnOption(col, ast.ColumnOptionDefaultValue) {
+				return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg)
+			}
+
+			autoRandBits, err := extractAutoRandomBitsFromColDef(col)
+			if err != nil {
+				return errors.Trace(err)
+			}
+			maxFieldTypeBitsLength := uint64(mysql.DefaultLengthOfMysqlTypes[col.Tp.Tp] * 8)
+			if autoRandBits == 0 {
+				return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNonPositive)
+			} else if autoRandBits >= maxFieldTypeBitsLength {
+				return ErrInvalidAutoRandom.GenWithStackByArgs(fmt.Sprintf(autoid.AutoRandomOverflowErrMsg, autoRandBits, maxFieldTypeBitsLength))
+			}
+			tbInfo.AutoRandomBits = autoRandBits
+		}
+	}
+	return nil
+}
+
+func extractAutoRandomBitsFromColDef(colDef *ast.ColumnDef) (uint64, error) {
+	for _, op := range colDef.Options {
+		if op.Tp == ast.ColumnOptionAutoRandom {
+			return convertAutoRandomBitsToUnsigned(op.AutoRandomBitLength)
+		}
+	}
+	return 0, nil
+}
+
+func convertAutoRandomBitsToUnsigned(autoRandomBits int) (uint64, error) {
+	if autoRandomBits == types.UnspecifiedLength {
+		return autoid.DefaultAutoRandomBits, nil
+	} else if autoRandomBits < 0 {
+		return 0, ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomNonPositive)
+	}
+	return uint64(autoRandomBits), nil
+}
+
 func buildTableInfo(ctx sessionctx.Context, d *ddl, tableName model.CIStr, cols []*table.Column, constraints []*ast.Constraint) (tbInfo *model.TableInfo, err error) {
 	tbInfo = &model.TableInfo{
 		Name:    tableName,
@@ -1156,7 +1208,7 @@ func buildTableInfo(ctx sessionctx.Context, d *ddl, tableName model.CIStr, cols
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
-		//check if the index is primary or uniqiue.
+		// check if the index is primary or unique.
 		switch constr.Tp {
 		case ast.ConstraintPrimaryKey:
 			idxInfo.Primary = true
@@ -1334,6 +1386,10 @@ func buildTableInfoWithCheck(ctx sessionctx.Context, d *ddl, s *ast.CreateTableS
 	tbInfo.Collate = tableCollate
 	tbInfo.Charset = tableCharset
 
+	if err = setTableAutoRandomBits(tbInfo, colDefs); err != nil {
+		return nil, errors.Trace(err)
+	}
+
 	pi, err := buildTablePartitionInfo(ctx, d, s)
 	if err != nil {
 		return nil, errors.Trace(err)
@@ -1735,9 +1791,9 @@ func checkCharsetAndCollation(cs string, co string) error {
 // handleAutoIncID handles auto_increment option in DDL. It creates a ID counter for the table and initiates the counter to a proper value.
 // For example if the option sets auto_increment to 10. The counter will be set to 9. So the next allocated ID will be 10.
 func (d *ddl) handleAutoIncID(tbInfo *model.TableInfo, schemaID int64) error {
-	alloc := autoid.NewAllocator(d.store, tbInfo.GetDBID(schemaID), tbInfo.IsAutoIncColUnsigned())
+	allocs := autoid.NewAllocatorsFromTblInfo(d.store, schemaID, tbInfo)
 	tbInfo.State = model.StatePublic
-	tb, err := table.TableFromMeta(alloc, tbInfo)
+	tb, err := table.TableFromMeta(allocs, tbInfo)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -2001,7 +2057,7 @@ func (d *ddl) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int6
 	if err != nil {
 		return errors.Trace(err)
 	}
-	autoIncID, err := t.Allocator(ctx).NextGlobalAutoID(t.Meta().ID)
+	autoIncID, err := t.Allocator(ctx, autoid.RowIDAllocType).NextGlobalAutoID(t.Meta().ID)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -2591,6 +2647,8 @@ func processColumnOptions(ctx sessionctx.Context, col *table.Column, options []*
 			return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs("can't modify with references"))
 		case ast.ColumnOptionFulltext:
 			return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs("can't modify with full text"))
+		// Ignore ColumnOptionAutoRandom. It will be handled later.
+		case ast.ColumnOptionAutoRandom:
 		default:
 			return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("unknown column option type: %d", opt.Tp)))
 		}
@@ -2735,6 +2793,10 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or
 		return nil, errors.Trace(err)
 	}
 
+	if err = checkAutoRandom(t.Meta(), col, specNewColumn); err != nil {
+		return nil, errors.Trace(err)
+	}
+
 	job := &model.Job{
 		SchemaID:   schema.ID,
 		TableID:    t.Meta().ID,
@@ -2783,6 +2845,36 @@ func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol
 	return nil
 }
 
+func checkAutoRandom(tableInfo *model.TableInfo, originCol *table.Column, specNewColumn *ast.ColumnDef) error {
+	if !config.GetGlobalConfig().Experimental.AllowAutoRandom && containsColumnOption(specNewColumn, ast.ColumnOptionAutoRandom) {
+		return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomExperimentalDisabledErrMsg)
+	}
+	// Disallow add/drop/modify actions on auto_random.
+	newAutoRandomBit, err := extractAutoRandomBitsFromColDef(specNewColumn)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	if tableInfo.AutoRandomBits != newAutoRandomBit {
+		return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomAlterErrMsg)
+	}
+
+	if tableInfo.AutoRandomBits != 0 {
+		// Disallow changing the column field type.
+		if originCol.Tp != specNewColumn.Tp.Tp {
+			return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomModifyColTypeErrMsg)
+		}
+		// Disallow changing auto_increment on auto_random column.
+		if containsColumnOption(specNewColumn, ast.ColumnOptionAutoIncrement) != mysql.HasAutoIncrementFlag(originCol.Flag) {
+			return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithAutoIncErrMsg)
+		}
+		// Disallow specifying a default value on auto_random column.
+		if containsColumnOption(specNewColumn, ast.ColumnOptionDefaultValue) {
+			return ErrInvalidAutoRandom.GenWithStackByArgs(autoid.AutoRandomIncompatibleWithDefaultValueErrMsg)
+		}
+	}
+	return nil
+}
+
 // ChangeColumn renames an existing column and modifies the column's definition,
 // currently we only support limited kind of changes
 // that do not need to change or check data on the table.