Merge branch 'master' into ttl_scan

README.md

@@ -1,4 +1,4 @@
-![](docs/logo_with_text.png)
+<img src="docs/tidb-logo-with-text.png" alt="TiDB, a distributed SQL database" height=100></img>
 
 [![LICENSE](https://img.shields.io/github/license/pingcap/tidb.svg)](https://github.com/pingcap/tidb/blob/master/LICENSE)
 [![Language](https://img.shields.io/badge/Language-Go-blue.svg)](https://golang.org/)
@@ -12,15 +12,15 @@
 
 ## What is TiDB?
 
-TiDB ("Ti" stands for Titanium) is an open-source NewSQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability.
+TiDB (/’taɪdiːbi:/, "Ti" stands for Titanium) is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability.
 
 - [Key features](https://docs.pingcap.com/tidb/stable/overview#key-features)
 - [Architecture](#architecture)
-- [MySQL Compatibility](https://docs.pingcap.com/tidb/stable/mysql-compatibility)
+- [MySQL compatibility](https://docs.pingcap.com/tidb/stable/mysql-compatibility)
 
-For more details and latest updates, see [TiDB docs](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
+For more details and latest updates, see [TiDB documentation](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
 
-For future plans, see [TiDB Roadmap](roadmap.md).
+For future plans, see the [TiDB roadmap](roadmap.md).
 
 ## Quick start
 
@@ -38,40 +38,42 @@ See [TiDB Quick Start Guide](https://docs.pingcap.com/tidb/stable/quick-start-wi
 
 ### Start developing TiDB
 
-See [Get Started](https://pingcap.github.io/tidb-dev-guide/get-started/introduction.html) chapter of [TiDB Dev Guide](https://pingcap.github.io/tidb-dev-guide/index.html).
+See the [Get Started](https://pingcap.github.io/tidb-dev-guide/get-started/introduction.html) chapter of [TiDB Development Guide](https://pingcap.github.io/tidb-dev-guide/index.html).
 
 ## Community
 
-You can join these groups and chats to discuss and ask TiDB related questions:
+You can join the following groups or channels to discuss or ask questions about TiDB, and to keep yourself informed of the latest TiDB updates:
 
-- [TiDB Internals Forum](https://internals.tidb.io/)
-- [Slack Channel](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb)
-- [TiDB User Group Forum (Chinese)](https://asktug.com)
-
-In addition, you may enjoy following:
-
-- [@PingCAP](https://twitter.com/PingCAP) on Twitter
-- Question tagged [#tidb on StackOverflow](https://stackoverflow.com/questions/tagged/tidb)
-- The PingCAP Team [English Blog](https://en.pingcap.com/blog) and [Chinese Blog](https://pingcap.com/blog-cn/)
+- Discuss TiDB's implementation and design
+  - [TiDB Internals forum](https://internals.tidb.io/)
+- Seek help when you use TiDB
+  - Slack channels: [#everyone](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb) (English), [#tidb-japan](https://slack.tidb.io/invite?team=tidb-community&channel=tidb-japan&ref=github-tidb) (Japanese)
+  - [TiDB User Group forum](https://asktug.com) (Chinese)
+  - [Stack Overflow](https://stackoverflow.com/questions/tagged/tidb) (questions tagged with #tidb)
+- Get the latest TiDB news or updates
+    - Follow [@PingCAP](https://twitter.com/PingCAP) on Twitter
+    - Read the PingCAP [English Blog](https://www.pingcap.com/blog/?from=en) or [Chinese Blog](https://cn.pingcap.com/blog/)
 
 For support, please contact [PingCAP](http://bit.ly/contact_us_via_github).
 
 ## Contributing
 
-The [community repository](https://github.com/pingcap/community) hosts all information about the TiDB community, including how to contribute to TiDB, how TiDB community is governed, how special interest groups are organized, etc.
+The [community repository](https://github.com/pingcap/community) hosts all information about the TiDB community, including [how to contribute](https://github.com/pingcap/community/blob/master/contributors/README.md) to TiDB, how the TiDB community is governed, how [teams](https://github.com/pingcap/community/blob/master/teams/README.md) are organized.
+
+Contributions are welcomed and greatly appreciated. You can get started with one of the [good first issues](https://github.com/pingcap/tidb/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) or [help wanted issues](https://github.com/pingcap/tidb/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22). For more details on typical contribution workflows, see [Contribute to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html). For more contributing information about where to start, click the contributor icon below.
 
 [<img src="docs/contribution-map.png" alt="contribution-map" width="180">](https://github.com/pingcap/tidb-map/blob/master/maps/contribution-map.md#tidb-is-an-open-source-distributed-htap-database-compatible-with-the-mysql-protocol)
 
-Contributions are welcomed and greatly appreciated. All the contributors are welcomed to claim your reward by filing this [form](https://forms.pingcap.com/f/tidb-contribution-swag). See [Contribution to TiDB](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/introduction.html) for details on typical contribution workflows. For more contributing information, click on the contributor icon above.
+Every contributor is welcome to claim your contribution swag by filling in and submitting this [form](https://forms.pingcap.com/f/tidb-contribution-swag).
 
 ## Case studies
 
-- [English](https://pingcap.com/case-studies)
-- [简体中文](https://pingcap.com/cases-cn/)
+- [Case studies in English](https://www.pingcap.com/customers/)
+- [中文用户案例](https://cn.pingcap.com/case/)
 
 ## Architecture
 
-![architecture](./docs/architecture.png)
+![TiDB architecture](./docs/tidb-architecture.png)
 
 ## License
 

br/pkg/stream/meta_kv.go

@@ -111,15 +111,34 @@ const (
 	flagShortValuePrefix   = byte('v')
 	flagOverlappedRollback = byte('R')
 	flagGCFencePrefix      = byte('F')
+	flagLastChangePrefix   = byte('l')
+	flagTxnSourcePrefix    = byte('S')
 )
 
+// RawWriteCFValue represents the value in write columnFamily.
+// Detail see line: https://github.com/tikv/tikv/blob/release-6.5/components/txn_types/src/write.rs#L70
 type RawWriteCFValue struct {
 	t                     WriteType
 	startTs               uint64
 	shortValue            []byte
 	hasOverlappedRollback bool
-	hasGCFence            bool
-	gcFence               uint64
+
+	// Records the next version after this version when overlapping rollback
+	// happens on an already existed commit record.
+	//
+	// See [`Write::gc_fence`] for more detail.
+	hasGCFence bool
+	gcFence    uint64
+
+	// The number of versions that need skipping from this record
+	// to find the latest PUT/DELETE record.
+	// If versions_to_last_change > 0 but last_change_ts == 0, the key does not
+	// have a PUT/DELETE record before this write record.
+	lastChangeTs         uint64
+	versionsToLastChange uint64
+
+	// The source of this txn.
+	txnSource uint64
 }
 
 // ParseFrom decodes the value to get the struct `RawWriteCFValue`.
@@ -146,6 +165,10 @@ l_for:
 		switch data[0] {
 		case flagShortValuePrefix:
 			vlen := data[1]
+			if len(data[2:]) < int(vlen) {
+				return errors.Annotatef(berrors.ErrInvalidArgument,
+					"the length of short value is invalid, vlen: %v", int(vlen))
+			}
 			v.shortValue = data[2 : vlen+2]
 			data = data[vlen+2:]
 		case flagOverlappedRollback:
@@ -157,13 +180,37 @@ l_for:
 			if err != nil {
 				return errors.Annotate(berrors.ErrInvalidArgument, "decode gc fence failed")
 			}
+		case flagLastChangePrefix:
+			data, v.lastChangeTs, err = codec.DecodeUint(data[1:])
+			if err != nil {
+				return errors.Annotate(berrors.ErrInvalidArgument, "decode last change ts failed")
+			}
+			data, v.versionsToLastChange, err = codec.DecodeUvarint(data)
+			if err != nil {
+				return errors.Annotate(berrors.ErrInvalidArgument, "decode versions to last change failed")
+			}
+		case flagTxnSourcePrefix:
+			data, v.txnSource, err = codec.DecodeUvarint(data[1:])
+			if err != nil {
+				return errors.Annotate(berrors.ErrInvalidArgument, "decode txn source failed")
+			}
 		default:
 			break l_for
 		}
 	}
 	return nil
 }
 
+// IsRollback checks whether the value in cf is a `rollback` record.
+func (v *RawWriteCFValue) IsRollback() bool {
+	return v.GetWriteType() == WriteTypeRollback
+}
+
+// IsRollback checks whether the value in cf is a `delete` record.
+func (v *RawWriteCFValue) IsDelete() bool {
+	return v.GetWriteType() == WriteTypeDelete
+}
+
 // HasShortValue checks whether short value is stored in write cf.
 func (v *RawWriteCFValue) HasShortValue() bool {
 	return len(v.shortValue) > 0
@@ -204,5 +251,14 @@ func (v *RawWriteCFValue) EncodeTo() []byte {
 		data = append(data, flagGCFencePrefix)
 		data = codec.EncodeUint(data, v.gcFence)
 	}
+	if v.lastChangeTs > 0 || v.versionsToLastChange > 0 {
+		data = append(data, flagLastChangePrefix)
+		data = codec.EncodeUint(data, v.lastChangeTs)
+		data = codec.EncodeUvarint(data, v.versionsToLastChange)
+	}
+	if v.txnSource > 0 {
+		data = append(data, flagTxnSourcePrefix)
+		data = codec.EncodeUvarint(data, v.txnSource)
+	}
 	return data
 }

br/pkg/stream/meta_kv_test.go

@@ -68,29 +68,49 @@ func TestWriteType(t *testing.T) {
 }
 
 func TestWriteCFValueNoShortValue(t *testing.T) {
+	var (
+		ts        uint64 = 400036290571534337
+		txnSource uint64 = 9527
+	)
+
 	buff := make([]byte, 0, 9)
-	buff = append(buff, byte('P'))
-	buff = codec.EncodeUvarint(buff, 400036290571534337)
+	buff = append(buff, WriteTypePut)
+	buff = codec.EncodeUvarint(buff, ts)
+	buff = append(buff, flagTxnSourcePrefix)
+	buff = codec.EncodeUvarint(buff, txnSource)
 
 	v := new(RawWriteCFValue)
 	err := v.ParseFrom(buff)
 	require.NoError(t, err)
+	require.False(t, v.IsDelete())
+	require.False(t, v.IsRollback())
 	require.False(t, v.HasShortValue())
+	require.False(t, v.hasGCFence)
+	require.Equal(t, v.lastChangeTs, uint64(0))
+	require.Equal(t, v.versionsToLastChange, uint64(0))
+	require.Equal(t, v.txnSource, txnSource)
 
 	encodedBuff := v.EncodeTo()
 	require.True(t, bytes.Equal(buff, encodedBuff))
 }
 
 func TestWriteCFValueWithShortValue(t *testing.T) {
-	var ts uint64 = 400036290571534337
-	shortValue := []byte("pingCAP")
+	var (
+		ts                   uint64 = 400036290571534337
+		shortValue                  = []byte("pingCAP")
+		lastChangeTs         uint64 = 9527
+		versionsToLastChange uint64 = 95271
+	)
 
 	buff := make([]byte, 0, 9)
-	buff = append(buff, byte('P'))
+	buff = append(buff, WriteTypePut)
 	buff = codec.EncodeUvarint(buff, ts)
 	buff = append(buff, flagShortValuePrefix)
 	buff = append(buff, byte(len(shortValue)))
 	buff = append(buff, shortValue...)
+	buff = append(buff, flagLastChangePrefix)
+	buff = codec.EncodeUint(buff, lastChangeTs)
+	buff = codec.EncodeUvarint(buff, versionsToLastChange)
 
 	v := new(RawWriteCFValue)
 	err := v.ParseFrom(buff)
@@ -99,7 +119,78 @@ func TestWriteCFValueWithShortValue(t *testing.T) {
 	require.True(t, bytes.Equal(v.GetShortValue(), shortValue))
 	require.False(t, v.hasGCFence)
 	require.False(t, v.hasOverlappedRollback)
+	require.Equal(t, v.lastChangeTs, lastChangeTs)
+	require.Equal(t, v.versionsToLastChange, versionsToLastChange)
+	require.Equal(t, v.txnSource, uint64(0))
 
 	data := v.EncodeTo()
 	require.True(t, bytes.Equal(data, buff))
 }
+
+func TestWriteCFValueWithRollback(t *testing.T) {
+	var (
+		ts                          uint64 = 400036290571534337
+		protectedRollbackShortValue        = []byte{'P'}
+	)
+
+	buff := make([]byte, 0, 9)
+	buff = append(buff, WriteTypeRollback)
+	buff = codec.EncodeUvarint(buff, ts)
+	buff = append(buff, flagShortValuePrefix, byte(len(protectedRollbackShortValue)))
+	buff = append(buff, protectedRollbackShortValue...)
+
+	v := new(RawWriteCFValue)
+	err := v.ParseFrom(buff)
+	require.NoError(t, err)
+	require.True(t, v.IsRollback())
+	require.True(t, v.HasShortValue())
+	require.Equal(t, v.GetShortValue(), protectedRollbackShortValue)
+	require.Equal(t, v.startTs, ts)
+	require.Equal(t, v.lastChangeTs, uint64(0))
+	require.Equal(t, v.versionsToLastChange, uint64(0))
+	require.Equal(t, v.txnSource, uint64(0))
+
+	data := v.EncodeTo()
+	require.Equal(t, data, buff)
+}
+
+func TestWriteCFValueWithDelete(t *testing.T) {
+	var ts uint64 = 400036290571534337
+	buff := make([]byte, 0, 9)
+	buff = append(buff, byte('D'))
+	buff = codec.EncodeUvarint(buff, ts)
+
+	v := new(RawWriteCFValue)
+	err := v.ParseFrom(buff)
+	require.NoError(t, err)
+	require.True(t, v.IsDelete())
+	require.False(t, v.HasShortValue())
+
+	data := v.EncodeTo()
+	require.Equal(t, data, buff)
+}
+
+func TestWriteCFValueWithGcFence(t *testing.T) {
+	var (
+		ts      uint64 = 400036290571534337
+		gcFence uint64 = 9527
+	)
+
+	buff := make([]byte, 0, 9)
+	buff = append(buff, WriteTypePut)
+	buff = codec.EncodeUvarint(buff, ts)
+	buff = append(buff, flagOverlappedRollback)
+	buff = append(buff, flagGCFencePrefix)
+	buff = codec.EncodeUint(buff, gcFence)
+
+	v := new(RawWriteCFValue)
+	err := v.ParseFrom(buff)
+	require.NoError(t, err)
+	require.Equal(t, v.startTs, ts)
+	require.True(t, v.hasGCFence)
+	require.Equal(t, v.gcFence, gcFence)
+	require.True(t, v.hasOverlappedRollback)
+
+	data := v.EncodeTo()
+	require.Equal(t, data, buff)
+}

br/pkg/stream/rewrite_meta_rawkv.go

@@ -451,13 +451,20 @@ func (sr *SchemasReplace) rewriteValueV2(value []byte, cf string, rewrite func([
 			return rewriteResult{}, errors.Trace(err)
 		}
 
-		if rawWriteCFValue.t == WriteTypeDelete {
+		if rawWriteCFValue.IsDelete() {
 			return rewriteResult{
 				NewValue:    value,
 				NeedRewrite: true,
 				Deleted:     true,
 			}, nil
 		}
+		if rawWriteCFValue.IsRollback() {
+			return rewriteResult{
+				NewValue:    value,
+				NeedRewrite: true,
+				Deleted:     false,
+			}, nil
+		}
 		if !rawWriteCFValue.HasShortValue() {
 			return rewriteResult{
 				NewValue:    value,
@@ -467,6 +474,9 @@ func (sr *SchemasReplace) rewriteValueV2(value []byte, cf string, rewrite func([
 
 		shortValue, needWrite, err := rewrite(rawWriteCFValue.GetShortValue())
 		if err != nil {
+			log.Info("failed to rewrite short value",
+				zap.ByteString("write-type", []byte{rawWriteCFValue.GetWriteType()}),
+				zap.Int("short-value-len", len(rawWriteCFValue.GetShortValue())))
 			return rewriteResult{}, errors.Trace(err)
 		}
 		if !needWrite {

config/config.go

@@ -557,6 +557,8 @@ type Security struct {
 	AuthTokenJWKS string `toml:"auth-token-jwks" json:"auth-token-jwks"`
 	// The refresh time interval of JWKS
 	AuthTokenRefreshInterval string `toml:"auth-token-refresh-interval" json:"auth-token-refresh-interval"`
+	// Disconnect directly when the password is expired
+	DisconnectOnExpiredPassword bool `toml:"disconnect-on-expired-password" json:"disconnect-on-expired-password"`
 }
 
 // The ErrConfigValidationFailed error is used so that external callers can do a type assertion
@@ -975,6 +977,7 @@ var defaultConf = Config{
 		RSAKeySize:                  4096,
 		AuthTokenJWKS:               "",
 		AuthTokenRefreshInterval:    DefAuthTokenRefreshInterval.String(),
+		DisconnectOnExpiredPassword: true,
 	},
 	DeprecateIntegerDisplayWidth:         false,
 	EnableEnumLengthLimit:                true,

ddl/db_test.go

@@ -1763,3 +1763,23 @@ func TestDDLBlockedCreateView(t *testing.T) {
 	dom.DDL().SetHook(hook)
 	tk.MustExec("alter table t modify column a char(10)")
 }
+
+func TestHashPartitionAddColumn(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("create table t(a int, b int) partition by hash(a) partitions 4")
+
+	hook := &ddl.TestDDLCallback{Do: dom}
+	hook.OnJobRunBeforeExported = func(job *model.Job) {
+		if job.SchemaState != model.StateWriteOnly {
+			return
+		}
+		tk2 := testkit.NewTestKit(t, store)
+		tk2.MustExec("use test")
+		tk2.MustExec("delete from t")
+	}
+	dom.DDL().SetHook(hook)
+	tk.MustExec("alter table t add column c int")
+}