Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl: Add index fast path to be a default add index path #39267

Merged
merged 14 commits into from
Nov 23, 2022
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ddl/cancel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,8 @@ func TestCancel(t *testing.T) {

// Prepare schema.
tk.MustExec("use test")
// TODO: Will check why tidb_ddl_enable_fast_reorg could not default be on in another PR.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("drop table if exists t_partition;")
tk.MustExec(`create table t_partition (
c1 int, c2 int, c3 int
Expand Down
1 change: 1 addition & 0 deletions ddl/column_modify_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,7 @@ func TestWriteReorgForColumnTypeChangeOnAmendTxn(t *testing.T) {

tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0")
tk.MustExec("set global tidb_enable_amend_pessimistic_txn = ON")
defer tk.MustExec("set global tidb_enable_amend_pessimistic_txn = OFF")

Expand Down
2 changes: 2 additions & 0 deletions ddl/db_change_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1719,6 +1719,8 @@ func TestCreateUniqueExpressionIndex(t *testing.T) {

tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
// TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.pr.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("create table t(a int default 0, b int default 0)")
tk.MustExec("insert into t values (1, 1), (2, 2), (3, 3), (4, 4)")

Expand Down
2 changes: 2 additions & 0 deletions ddl/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -985,6 +985,7 @@ func TestCommitTxnWithIndexChange(t *testing.T) {
// Prepare work.
tk := testkit.NewTestKit(t, store)
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk.MustExec("use test")
tk.MustExec("create table t1 (c1 int primary key, c2 int, c3 int, index ok2(c2))")
Expand Down Expand Up @@ -1385,6 +1386,7 @@ func TestAmendTxnSavepointWithDDL(t *testing.T) {
tk.MustExec("use test;")
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk2.MustExec("use test;")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")

prepareFn := func() {
Expand Down
5 changes: 5 additions & 0 deletions ddl/failtest/fail_db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,8 @@ func TestRunDDLJobPanicEnableClusteredIndex(t *testing.T) {
s := createFailDBSuite(t)
testAddIndexWorkerNum(t, s, func(tk *testkit.TestKit) {
tk.Session().GetSessionVars().EnableClusteredIndex = variable.ClusteredIndexDefModeOn
// TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1, c3))")
})
}
Expand All @@ -328,6 +330,8 @@ func TestRunDDLJobPanicEnableClusteredIndex(t *testing.T) {
func TestRunDDLJobPanicDisableClusteredIndex(t *testing.T) {
s := createFailDBSuite(t)
testAddIndexWorkerNum(t, s, func(tk *testkit.TestKit) {
// TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1))")
})
}
Expand Down Expand Up @@ -420,6 +424,7 @@ func TestPartitionAddIndexGC(t *testing.T) {
s := createFailDBSuite(t)
tk := testkit.NewTestKit(t, s.store)
tk.MustExec("use test")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec(`create table partition_add_idx (
id int not null,
hired date not null
Expand Down
2 changes: 2 additions & 0 deletions ddl/index_change_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ func TestIndexChange(t *testing.T) {
ddl.SetWaitTimeWhenErrorOccurred(1 * time.Microsecond)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
// TODO: Will check why tidb_ddl_enable_fast_reorg could not default be on in another PR.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("create table t (c1 int primary key, c2 int)")
tk.MustExec("insert t values (1, 1), (2, 2), (3, 3);")

Expand Down
1 change: 0 additions & 1 deletion ddl/index_merge_tmp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,6 @@ func TestPessimisticAmendIncompatibleWithFastReorg(t *testing.T) {
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 1;")

tk.MustGetErrMsg("set @@tidb_enable_amend_pessimistic_txn = 1;",
"amend pessimistic transactions is not compatible with tidb_ddl_enable_fast_reorg")
Expand Down
3 changes: 3 additions & 0 deletions ddl/multi_schema_change_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,8 @@ func TestMultiSchemaChangeAddIndexesCancelled(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
// TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
originHook := dom.DDL().GetHook()

// Test cancel successfully.
Expand Down Expand Up @@ -1014,6 +1016,7 @@ func TestMultiSchemaChangeMixCancelled(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")

tk.MustExec("create table t (a int, b int, c int, index i1(c), index i2(c));")
tk.MustExec("insert into t values (1, 2, 3);")
Expand Down
2 changes: 2 additions & 0 deletions ddl/serial_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,8 @@ func TestCancelAddIndexPanic(t *testing.T) {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/errorMockPanic"))
}()
tk.MustExec("use test")
// TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(c1 int, c2 int)")

Expand Down
9 changes: 6 additions & 3 deletions ddl/stat_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,16 @@ func TestDDLStatsInfo(t *testing.T) {
store, domain := testkit.CreateMockStoreAndDomainWithSchemaLease(t, testLease)
d := domain.DDL()

tk := testkit.NewTestKit(t, store)
ctx := tk.Session()
dbInfo, err := testSchemaInfo(store, "test_stat")
require.NoError(t, err)
testCreateSchema(t, testkit.NewTestKit(t, store).Session(), d, dbInfo)
testCreateSchema(t, ctx, d, dbInfo)
tblInfo, err := testTableInfo(store, "t", 2)
require.NoError(t, err)
testCreateTable(t, testkit.NewTestKit(t, store).Session(), d, dbInfo, tblInfo)
ctx := testkit.NewTestKit(t, store).Session()
testCreateTable(t, ctx, d, dbInfo, tblInfo)
// TODO: will check why tidb_ddl_enable_fast_reorg could not default be on in another pr.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
err = sessiontxn.NewTxn(context.Background(), ctx)
require.NoError(t, err)

Expand Down
1 change: 1 addition & 0 deletions ddl/table_modify_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ func TestLockTableReadOnly(t *testing.T) {
tk1.MustExec("admin cleanup table lock t1")
tk2.MustExec("insert into t1 set a=1, b=2")

tk1.MustExec("set global tidb_ddl_enable_fast_reorg = 0")
tk1.MustExec("set tidb_enable_amend_pessimistic_txn = 1")
tk1.MustExec("begin pessimistic")
tk1.MustQuery("select * from t1 where a = 1").Check(testkit.Rows("1 2"))
Expand Down
1 change: 1 addition & 0 deletions expression/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4956,6 +4956,7 @@ func TestSchemaDMLNotChange(t *testing.T) {
tk2 := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk2.MustExec("use test")
tk.MustExec("drop table if exists t")
Expand Down
1 change: 1 addition & 0 deletions session/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ func TestRetrySchemaChangeForEmptyChange(t *testing.T) {
tk1.MustExec("commit")

// TODO remove this enable after fixing table delta map.
tk1.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk1.MustExec("set tidb_enable_amend_pessimistic_txn = 1")
tk1.MustExec("begin pessimistic")
tk2.MustExec("alter table t add k int")
Expand Down
4 changes: 2 additions & 2 deletions sessionctx/variable/sysvar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -692,8 +692,8 @@ func TestSetTIDBFastDDL(t *testing.T) {
vars.GlobalVarsAccessor = mock
fastDDL := GetSysVar(TiDBDDLEnableFastReorg)

// Default off
require.Equal(t, fastDDL.Value, Off)
// Default true
require.Equal(t, fastDDL.Value, On)

// Set to On
err := mock.SetGlobalSysVar(context.Background(), TiDBDDLEnableFastReorg, On)
Expand Down
2 changes: 1 addition & 1 deletion sessionctx/variable/tidb_vars.go
Original file line number Diff line number Diff line change
Expand Up @@ -1065,7 +1065,7 @@ const (
DefTiFlashFastScan = false
DefMemoryUsageAlarmRatio = 0.7
DefMemoryUsageAlarmKeepRecordNum = 5
DefTiDBEnableFastReorg = false
DefTiDBEnableFastReorg = true
DefTiDBDDLDiskQuota = 100 * 1024 * 1024 * 1024 // 100GB
DefExecutorConcurrency = 5
DefTiDBEnableGeneralPlanCache = false
Expand Down
8 changes: 4 additions & 4 deletions telemetry/data_feature_usage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ func TestAddIndexAccelerationAndMDL(t *testing.T) {
require.NoError(t, err)

allow := ddl.IsEnableFastReorg()
require.Equal(t, false, allow)
require.Equal(t, true, allow)
tk.MustExec("set global tidb_enable_metadata_lock = 0")
tk.MustExec("use test")
tk.MustExec("drop table if exists tele_t")
Expand All @@ -507,7 +507,7 @@ func TestAddIndexAccelerationAndMDL(t *testing.T) {
tk.MustExec("alter table tele_t add index idx_org(b)")
usage, err = telemetry.GetFeatureUsage(tk.Session())
require.NoError(t, err)
require.Equal(t, int64(0), usage.DDLUsageCounter.AddIndexIngestUsed)
require.Equal(t, int64(1), usage.DDLUsageCounter.AddIndexIngestUsed)
require.Equal(t, false, usage.DDLUsageCounter.MetadataLockUsed)

tk.MustExec("set @@global.tidb_ddl_enable_fast_reorg = on")
Expand All @@ -516,11 +516,11 @@ func TestAddIndexAccelerationAndMDL(t *testing.T) {
require.Equal(t, true, allow)
usage, err = telemetry.GetFeatureUsage(tk.Session())
require.NoError(t, err)
require.Equal(t, int64(0), usage.DDLUsageCounter.AddIndexIngestUsed)
require.Equal(t, int64(1), usage.DDLUsageCounter.AddIndexIngestUsed)
tk.MustExec("alter table tele_t add index idx_new(b)")
usage, err = telemetry.GetFeatureUsage(tk.Session())
require.NoError(t, err)
require.Equal(t, int64(1), usage.DDLUsageCounter.AddIndexIngestUsed)
require.Equal(t, int64(2), usage.DDLUsageCounter.AddIndexIngestUsed)
require.Equal(t, true, usage.DDLUsageCounter.MetadataLockUsed)
}

Expand Down
12 changes: 12 additions & 0 deletions tests/realtikvtest/pessimistictest/pessimistic_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1850,6 +1850,7 @@ func TestPessimisticTxnWithDDLAddDropColumn(t *testing.T) {

// tk2 starts a pessimistic transaction and make some changes on table t1.
// tk executes some ddl statements add/drop column on table t1.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk.MustExec("begin pessimistic")
tk.MustExec("update t1 set c2 = c1 * 10")
Expand Down Expand Up @@ -1880,6 +1881,7 @@ func TestPessimisticTxnWithDDLChangeColumn(t *testing.T) {
tk.MustExec("insert t1 values (1, 77, 'a'), (2, 88, 'b')")

// Extend column field length is acceptable.
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk.MustExec("begin pessimistic")
tk.MustExec("update t1 set c2 = c1 * 10")
Expand Down Expand Up @@ -2153,6 +2155,7 @@ func TestAmendTxnVariable(t *testing.T) {
tk3.MustExec("set tidb_enable_amend_pessimistic_txn = 0;")
tk3.MustExec("begin pessimistic")
tk3.MustExec("insert into t1 values(3, 3, 3)")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk.MustExec("begin pessimistic")
tk.MustExec("insert into t1 values(4, 4, 4)")
Expand All @@ -2173,6 +2176,7 @@ func TestAmendTxnVariable(t *testing.T) {
tk4.MustExec("insert into t1 values(5, 5, 5, 5)")
tk2.MustExec("alter table t1 drop column new_col")
require.Error(t, tk4.ExecToErr("commit"))
tk4.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk4.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk4.MustExec("begin pessimistic")
tk4.MustExec("insert into t1 values(5, 5, 5)")
Expand Down Expand Up @@ -2303,6 +2307,7 @@ func TestAsyncCommitWithSchemaChange(t *testing.T) {
tk.MustExec("insert into tk values(1, 1, 1)")
tk2 := createAsyncCommitTestKit(t, store)
tk3 := createAsyncCommitTestKit(t, store)
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk2.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk3.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
Expand Down Expand Up @@ -2377,6 +2382,7 @@ func Test1PCWithSchemaChange(t *testing.T) {
tk.MustExec("drop table if exists tk")
tk.MustExec("create table tk (c1 int primary key, c2 int)")
tk.MustExec("insert into tk values (1, 1)")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk2.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
tk3.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")
Expand Down Expand Up @@ -2424,6 +2430,7 @@ func TestAmendForUniqueIndex(t *testing.T) {
tk2 := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk2.MustExec("use test")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")

tk2.MustExec("drop table if exists t1")
Expand Down Expand Up @@ -2546,6 +2553,7 @@ func TestAmendWithColumnTypeChange(t *testing.T) {
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk2.MustExec("use test")

tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1;")

tk2.MustExec("drop table if exists t")
Expand All @@ -2563,6 +2571,7 @@ func TestIssue21498(t *testing.T) {
tk2 := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk2.MustExec("use test")
tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1")

for _, partition := range []bool{false, true} {
Expand Down Expand Up @@ -2732,6 +2741,7 @@ func TestPlanCacheSchemaChange(t *testing.T) {
tk.MustExec("create table t (id int primary key, v int, unique index iv (v), vv int)")
tk.MustExec("insert into t values(1, 1, 1), (2, 2, 2), (4, 4, 4)")

tk.MustExec("set global tidb_ddl_enable_fast_reorg = 0")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = 1")
tk2.MustExec("set tidb_enable_amend_pessimistic_txn = 1")

Expand Down Expand Up @@ -2899,6 +2909,7 @@ func TestAmendForIndexChange(t *testing.T) {
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk2.MustExec("use test")

tk.MustExec("set global tidb_ddl_enable_fast_reorg = OFF;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = ON;")
tk.Session().GetSessionVars().EnableAsyncCommit = false
tk.Session().GetSessionVars().Enable1PC = false
Expand Down Expand Up @@ -2974,6 +2985,7 @@ func TestAmendForColumnChange(t *testing.T) {
tk.MustExec("set global tidb_enable_metadata_lock=0")
tk2.MustExec("use test")

tk.MustExec("set global tidb_ddl_enable_fast_reorg = OFF;")
tk.MustExec("set tidb_enable_amend_pessimistic_txn = ON;")
tk2.MustExec("drop table if exists t1")

Expand Down