From 3d995e8e2b88d2ca543e437f09d1582878c251f8 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 15:56:11 +0800
Subject: [PATCH 01/27] fixup

---
 pkg/planner/index_advisor/model.go          | 159 ++++++++++++
 pkg/planner/index_advisor/optimizer.go      | 264 ++++++++++++++++++++
 pkg/planner/index_advisor/optimizer_test.go | 262 +++++++++++++++++++
 pkg/planner/index_advisor/set.go            | 183 ++++++++++++++
 pkg/planner/index_advisor/set_test.go       |  88 +++++++
 pkg/planner/optimize.go                     |  18 ++
 6 files changed, 974 insertions(+)
 create mode 100644 pkg/planner/index_advisor/model.go
 create mode 100644 pkg/planner/index_advisor/optimizer.go
 create mode 100644 pkg/planner/index_advisor/optimizer_test.go
 create mode 100644 pkg/planner/index_advisor/set.go
 create mode 100644 pkg/planner/index_advisor/set_test.go

diff --git a/pkg/planner/index_advisor/model.go b/pkg/planner/index_advisor/model.go
new file mode 100644
index 0000000000000..be4f621500857
--- /dev/null
+++ b/pkg/planner/index_advisor/model.go
@@ -0,0 +1,159 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index_advisor
+
+import (
+	"fmt"
+	"math"
+	"strings"
+)
+
+// Query represents a Query statement.
+type Query struct { // DQL or DML
+	Alias      string
+	SchemaName string
+	Text       string
+	Frequency  int
+	CostPerMon float64
+}
+
+// Key returns the key of the Query.
+func (q Query) Key() string {
+	return q.Text
+}
+
+// Column represents a column.
+type Column struct {
+	SchemaName string
+	TableName  string
+	ColumnName string
+}
+
+// NewColumn creates a new column.
+func NewColumn(schemaName, tableName, columnName string) Column {
+	return Column{SchemaName: strings.ToLower(schemaName), TableName: strings.ToLower(tableName), ColumnName: strings.ToLower(columnName)}
+}
+
+// NewColumns creates new columns.
+func NewColumns(schemaName, tableName string, columnNames ...string) []Column {
+	var cols []Column
+	for _, col := range columnNames {
+		cols = append(cols, NewColumn(schemaName, tableName, col))
+	}
+	return cols
+}
+
+// Key returns the key of the column.
+func (c Column) Key() string {
+	return fmt.Sprintf("%v.%v.%v", c.SchemaName, c.TableName, c.ColumnName)
+}
+
+// Index represents an index.
+type Index struct {
+	SchemaName string
+	TableName  string
+	IndexName  string
+	Columns    []Column
+}
+
+// NewIndex creates a new index.
+func NewIndex(schemaName, tableName, indexName string, columns ...string) Index {
+	return Index{SchemaName: strings.ToLower(schemaName), TableName: strings.ToLower(tableName), IndexName: strings.ToLower(indexName), Columns: NewColumns(schemaName, tableName, columns...)}
+}
+
+// NewIndexWithColumns creates a new index with columns.
+func NewIndexWithColumns(indexName string, columns ...Column) Index {
+	names := make([]string, len(columns))
+	for i, col := range columns {
+		names[i] = col.ColumnName
+	}
+	return NewIndex(columns[0].SchemaName, columns[0].TableName, indexName, names...)
+}
+
+// Key returns the key of the index.
+func (i Index) Key() string {
+	var names []string
+	for _, col := range i.Columns {
+		names = append(names, col.ColumnName)
+	}
+	return fmt.Sprintf("%v.%v(%v)", i.SchemaName, i.TableName, strings.Join(names, ","))
+}
+
+// PrefixContain returns whether j is a prefix of i.
+func (i Index) PrefixContain(j Index) bool {
+	if i.SchemaName != j.SchemaName || i.TableName != j.TableName || len(i.Columns) < len(j.Columns) {
+		return false
+	}
+	for k := range j.Columns {
+		if i.Columns[k].ColumnName != j.Columns[k].ColumnName {
+			return false
+		}
+	}
+	return true
+}
+
+// IndexSetCost is the cost of a index configuration.
+type IndexSetCost struct {
+	TotalWorkloadQueryCost    float64
+	TotalNumberOfIndexColumns int
+	IndexKeysStr              string // IndexKeysStr is the string representation of the index keys.
+}
+
+// Less returns whether the cost of c is less than the cost of other.
+func (c IndexSetCost) Less(other IndexSetCost) bool {
+	if c.TotalWorkloadQueryCost == 0 { // not initialized
+		return false
+	}
+	if other.TotalWorkloadQueryCost == 0 { // not initialized
+		return true
+	}
+	cc, cOther := c.TotalWorkloadQueryCost, other.TotalWorkloadQueryCost
+	if math.Abs(cc-cOther) > 10 && math.Abs(cc-cOther)/math.Max(cc, cOther) > 0.001 {
+		// their cost is very different, then the less cost, the better.
+		return cc < cOther
+	}
+
+	if c.TotalNumberOfIndexColumns != other.TotalNumberOfIndexColumns {
+		// if they have the same cost, then the less columns, the better.
+		return c.TotalNumberOfIndexColumns < other.TotalNumberOfIndexColumns
+	}
+
+	// if they have the same cost and the same number of columns, then use the IndexKeysStr to compare to make the result stable.
+	return c.IndexKeysStr < other.IndexKeysStr
+}
+
+// ImpactedQuery represents the impacted query.
+type ImpactedQuery struct {
+	Query            string
+	Improvement      float64
+	CostSavingPerMon float64
+}
+
+type WorkloadImpact struct {
+	WorkloadImprovement  float64
+	WorkloadSavingPerMon float64
+}
+
+// IndexAdvisorResult represents the result of the index advisor.
+type IndexAdvisorResult struct {
+	Database           string
+	Table              string
+	IndexName          string
+	IndexColumns       []string
+	IndexSize          uint64
+	Reason             string
+	WorkloadImpact     *WorkloadImpact
+	TopImpactedQueries []*ImpactedQuery
+}
diff --git a/pkg/planner/index_advisor/optimizer.go b/pkg/planner/index_advisor/optimizer.go
new file mode 100644
index 0000000000000..b5ec98439747a
--- /dev/null
+++ b/pkg/planner/index_advisor/optimizer.go
@@ -0,0 +1,264 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index_advisor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/pingcap/tidb/pkg/domain"
+	"github.com/pingcap/tidb/pkg/infoschema"
+	"github.com/pingcap/tidb/pkg/parser"
+	"github.com/pingcap/tidb/pkg/parser/ast"
+	"github.com/pingcap/tidb/pkg/parser/model"
+	"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
+	"github.com/pingcap/tidb/pkg/sessionctx"
+	"github.com/pingcap/tidb/pkg/types"
+)
+
+// QueryPlanCostHook is used to calculate the cost of the query plan on this sctx.
+var QueryPlanCostHook func(sctx sessionctx.Context, stmt ast.StmtNode) (float64, error)
+
+// Optimizer is the interface of a what-if optimizer.
+// This interface encapsulates all methods the Index Advisor needs to interact with the TiDB optimizer.
+// This interface is not thread-safe.
+type Optimizer interface {
+	// ColumnType returns the column type of the specified column.
+	ColumnType(c Column) (*types.FieldType, error)
+
+	// PrefixContainIndex returns whether the specified index is a prefix of an existing index.
+	PrefixContainIndex(idx Index) (bool, error)
+
+	// PossibleColumns returns the possible columns that match the specified column name.
+	PossibleColumns(schema, colName string) ([]Column, error)
+
+	// TableColumns returns the columns of the specified table.
+	TableColumns(schema, table string) ([]Column, error)
+
+	// IndexNameExist returns whether the specified index name exists in the specified table.
+	IndexNameExist(schema, table, indexName string) (bool, error)
+
+	// EstIndexSize return the estimated index size of the specified table and columns
+	EstIndexSize(db, table string, cols ...string) (indexSize float64, err error)
+
+	// QueryPlanCost return the cost of the query plan.
+	QueryPlanCost(sql string, hypoIndexes ...Index) (cost float64, err error)
+}
+
+// optimizerImpl is the implementation of Optimizer.
+type optimizerImpl struct {
+	sctx sessionctx.Context
+}
+
+// NewOptimizer creates a new Optimizer.
+func NewOptimizer(sctx sessionctx.Context) Optimizer {
+	return &optimizerImpl{sctx}
+}
+
+func (opt *optimizerImpl) is() infoschema.InfoSchema {
+	return opt.sctx.GetDomainInfoSchema().(infoschema.InfoSchema)
+}
+
+// IndexNameExist returns whether the specified index name exists in the specified table.
+func (opt *optimizerImpl) IndexNameExist(schema, table, indexName string) (bool, error) {
+	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(schema), model.NewCIStr(table))
+	if err != nil {
+		return false, err
+	}
+	for _, idx := range tbl.Indices() {
+		if idx.Meta().Name.L == indexName {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// TableColumns returns the columns of the specified table.
+func (opt *optimizerImpl) TableColumns(schema, table string) ([]Column, error) {
+	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(schema), model.NewCIStr(table))
+	if err != nil {
+		return nil, err
+	}
+	cols := make([]Column, 0)
+	for _, col := range tbl.Cols() {
+		cols = append(cols, Column{
+			SchemaName: schema,
+			TableName:  table,
+			ColumnName: col.Name.L,
+		})
+	}
+	return cols, nil
+}
+
+// PossibleColumns returns the possible columns that match the specified column name.
+func (opt *optimizerImpl) PossibleColumns(schema, colName string) ([]Column, error) {
+	// filtering system schema
+	schema = strings.ToLower(schema)
+	if schema == "information_schema" || schema == "metrics_schema" ||
+		schema == "performance_schema" || schema == "mysql" {
+		return nil, nil
+	}
+
+	cols := make([]Column, 0)
+	tbls, err := opt.is().SchemaTableInfos(context.Background(), model.NewCIStr(schema))
+	if err != nil {
+		return nil, err
+	}
+	for _, tbl := range tbls {
+		for _, col := range tbl.Cols() {
+			if strings.ToLower(col.Name.L) == colName {
+				cols = append(cols, Column{
+					SchemaName: schema,
+					TableName:  tbl.Name.L,
+					ColumnName: col.Name.L,
+				})
+			}
+		}
+	}
+	return cols, nil
+}
+
+// PrefixContainIndex returns whether the specified index is a prefix of an existing index.
+func (opt *optimizerImpl) PrefixContainIndex(idx Index) (bool, error) {
+	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(idx.SchemaName), model.NewCIStr(idx.TableName))
+	if err != nil {
+		return false, err
+	}
+	for _, tblIndex := range tbl.Indices() {
+		if len(tblIndex.Meta().Columns) < len(idx.Columns) {
+			continue
+		}
+		prefixMatched := true
+		for i, idxCol := range idx.Columns {
+			if tblIndex.Meta().Columns[i].Name.L != strings.ToLower(idxCol.ColumnName) {
+				prefixMatched = false
+				break
+			}
+		}
+		if prefixMatched {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// ColumnType returns the column type of the specified column.
+func (opt *optimizerImpl) ColumnType(c Column) (*types.FieldType, error) {
+	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(c.SchemaName), model.NewCIStr(c.TableName))
+	if err != nil {
+		return nil, err
+	}
+	for _, col := range tbl.Cols() {
+		if col.Name.L == strings.ToLower(c.ColumnName) {
+			return &col.FieldType, nil
+		}
+	}
+	return nil, fmt.Errorf("column %v not found in table %v.%v", c.ColumnName, c.SchemaName, c.TableName)
+}
+
+func (opt *optimizerImpl) addHypoIndex(hypoIndexes ...Index) error {
+	for _, h := range hypoIndexes {
+		tInfo, err := opt.is().TableByName(context.Background(), model.NewCIStr(h.SchemaName), model.NewCIStr(h.TableName))
+		if err != nil {
+			return err
+		}
+
+		var cols []*model.IndexColumn
+		for _, col := range h.Columns {
+			colOffset := -1
+			for i, tCol := range tInfo.Cols() {
+				if tCol.Name.L == strings.ToLower(col.ColumnName) {
+					colOffset = i
+					break
+				}
+			}
+			if colOffset == -1 {
+				return fmt.Errorf("column %v not found in table %v.%v", col.ColumnName, h.SchemaName, h.TableName)
+			}
+			cols = append(cols, &model.IndexColumn{
+				Name:   model.NewCIStr(col.ColumnName),
+				Offset: colOffset,
+				Length: types.UnspecifiedLength,
+			})
+		}
+		idxInfo := &model.IndexInfo{
+			Name:    model.NewCIStr(h.IndexName),
+			Columns: cols,
+			State:   model.StatePublic,
+			Tp:      model.IndexTypeHypo,
+		}
+
+		if opt.sctx.GetSessionVars().HypoIndexes == nil {
+			opt.sctx.GetSessionVars().HypoIndexes = make(map[string]map[string]map[string]*model.IndexInfo)
+		}
+		if opt.sctx.GetSessionVars().HypoIndexes[h.SchemaName] == nil {
+			opt.sctx.GetSessionVars().HypoIndexes[h.SchemaName] = make(map[string]map[string]*model.IndexInfo)
+		}
+		if opt.sctx.GetSessionVars().HypoIndexes[h.SchemaName][h.TableName] == nil {
+			opt.sctx.GetSessionVars().HypoIndexes[h.SchemaName][h.TableName] = make(map[string]*model.IndexInfo)
+		}
+		opt.sctx.GetSessionVars().HypoIndexes[h.SchemaName][h.TableName][h.IndexName] = idxInfo
+	}
+	return nil
+}
+
+// QueryPlanCost return the cost of the query plan.
+func (opt *optimizerImpl) QueryPlanCost(sql string, hypoIndexes ...Index) (cost float64, err error) {
+	p := parser.New()
+	stmt, err := p.ParseOneStmt(sql, "", "")
+	if err != nil {
+		return 0, err
+	}
+
+	originalFix43817 := opt.sctx.GetSessionVars().OptimizerFixControl[fixcontrol.Fix43817]
+	originalWarns := opt.sctx.GetSessionVars().StmtCtx.GetWarnings()
+	originalExtraWarns := opt.sctx.GetSessionVars().StmtCtx.GetExtraWarnings()
+	originalHypoIndexes := opt.sctx.GetSessionVars().HypoIndexes
+	defer func() {
+		opt.sctx.GetSessionVars().OptimizerFixControl[fixcontrol.Fix43817] = originalFix43817
+		opt.sctx.GetSessionVars().StmtCtx.SetWarnings(originalWarns)
+		opt.sctx.GetSessionVars().StmtCtx.SetExtraWarnings(originalExtraWarns)
+		opt.sctx.GetSessionVars().HypoIndexes = originalHypoIndexes
+		opt.sctx.GetSessionVars().StmtCtx.InExplainStmt = false
+	}()
+	opt.sctx.GetSessionVars().OptimizerFixControl[fixcontrol.Fix43817] = "on"
+	opt.sctx.GetSessionVars().StmtCtx.InExplainStmt = true
+	opt.sctx.GetSessionVars().HypoIndexes = nil
+
+	if err := opt.addHypoIndex(hypoIndexes...); err != nil {
+		return 0, err
+	}
+	return QueryPlanCostHook(opt.sctx, stmt)
+}
+
+// EstIndexSize return the estimated index size of the specified table and columns
+func (opt *optimizerImpl) EstIndexSize(db, table string, cols ...string) (indexSize float64, err error) {
+	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(db), model.NewCIStr(table))
+	if err != nil {
+		return 0, err
+	}
+	stats := domain.GetDomain(opt.sctx).StatsHandle()
+	tblStats := stats.GetTableStats(tbl.Meta())
+	for _, colName := range cols {
+		colStats := tblStats.ColumnByName(colName)
+		if colStats == nil { // might be not loaded
+			indexSize += float64(8) * float64(tblStats.RealtimeCount)
+		} else {
+			indexSize += float64(colStats.TotColSize)
+		}
+	}
+	return indexSize, nil
+}
diff --git a/pkg/planner/index_advisor/optimizer_test.go b/pkg/planner/index_advisor/optimizer_test.go
new file mode 100644
index 0000000000000..91bf1ca28d4d9
--- /dev/null
+++ b/pkg/planner/index_advisor/optimizer_test.go
@@ -0,0 +1,262 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index_advisor_test
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/pingcap/tidb/pkg/parser/mysql"
+	"github.com/pingcap/tidb/pkg/planner/index_advisor"
+	"github.com/pingcap/tidb/pkg/testkit"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptimizerColumnType(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b float, c varchar(255))`)
+	tk.MustExec(`create table t2 (a int, b decimal(10,2), c varchar(1024))`)
+	opt := index_advisor.NewOptimizer(tk.Session())
+
+	tp, err := opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "a"})
+	require.NoError(t, err)
+	require.Equal(t, mysql.TypeLong, tp.GetType())
+
+	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "b"})
+	require.NoError(t, err)
+	require.Equal(t, mysql.TypeFloat, tp.GetType())
+
+	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "c"})
+	require.NoError(t, err)
+	require.Equal(t, mysql.TypeVarchar, tp.GetType())
+
+	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "a"})
+	require.NoError(t, err)
+	require.Equal(t, mysql.TypeLong, tp.GetType())
+
+	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "b"})
+	require.NoError(t, err)
+	require.Equal(t, mysql.TypeNewDecimal, tp.GetType())
+
+	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "c"})
+	require.NoError(t, err)
+	require.Equal(t, mysql.TypeVarchar, tp.GetType())
+
+	_, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "d"})
+	require.Error(t, err)
+
+	_, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t3", ColumnName: "a"})
+	require.Error(t, err)
+}
+
+func TestOptimizerPrefixContainIndex(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b, c))`)
+	tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a, b, c, d), key(d, c, b, a))`)
+	opt := index_advisor.NewOptimizer(tk.Session())
+
+	check := func(expected bool, tableName string, columns ...string) {
+		ok, err := opt.PrefixContainIndex(index_advisor.NewIndex("test", tableName, "idx", columns...))
+		require.NoError(t, err)
+		require.Equal(t, expected, ok)
+	}
+
+	check(true, "t1", "a")
+	check(true, "t1", "b")
+	check(true, "t1", "b", "c")
+	check(false, "t1", "c")
+	check(false, "t1", "a", "b")
+	check(false, "t1", "b", "c", "a")
+	check(true, "t2", "a")
+	check(true, "t2", "a", "b")
+	check(true, "t2", "a", "b", "c")
+	check(true, "t2", "a", "b", "c", "d")
+	check(true, "t2", "d")
+	check(true, "t2", "d", "c")
+	check(false, "t2", "b")
+	check(false, "t2", "b", "a")
+	check(false, "t2", "b", "a", "c")
+}
+
+func TestOptimizerPossibleColumns(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b int, c int, d int)`)
+	tk.MustExec(`create table t2 (a int, b int, c int, d int)`)
+	tk.MustExec(`create table t3 (c int, d int, e int, f int)`)
+	opt := index_advisor.NewOptimizer(tk.Session())
+
+	check := func(schema, colName string, expected []string) {
+		cols, err := opt.PossibleColumns(schema, colName)
+		require.NoError(t, err)
+		var tmp []string
+		for _, col := range cols {
+			tmp = append(tmp, fmt.Sprintf("%v.%v", col.TableName, col.ColumnName))
+		}
+		sort.Strings(tmp)
+		require.Equal(t, expected, tmp)
+	}
+
+	check("test", "a", []string{"t1.a", "t2.a"})
+	check("test", "b", []string{"t1.b", "t2.b"})
+	check("test", "c", []string{"t1.c", "t2.c", "t3.c"})
+	check("test", "d", []string{"t1.d", "t2.d", "t3.d"})
+	check("test", "e", []string{"t3.e"})
+	check("test", "f", []string{"t3.f"})
+	check("test", "g", nil)
+}
+
+func TestOptimizerTableColumns(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b int, c int, d int)`)
+	tk.MustExec(`create table t2 (a int, b int, c int, d int)`)
+	tk.MustExec(`create table t3 (c int, d int, e int, f int)`)
+	opt := index_advisor.NewOptimizer(tk.Session())
+
+	check := func(schemaName, tableName string, columns []string) {
+		cols, err := opt.TableColumns(schemaName, tableName)
+		require.NoError(t, err)
+		var tmp []string
+		for _, col := range cols {
+			require.Equal(t, schemaName, col.SchemaName)
+			require.Equal(t, tableName, col.TableName)
+			tmp = append(tmp, col.ColumnName)
+		}
+		sort.Strings(tmp)
+		require.Equal(t, columns, tmp)
+	}
+
+	check("test", "t1", []string{"a", "b", "c", "d"})
+	check("test", "t2", []string{"a", "b", "c", "d"})
+	check("test", "t3", []string{"c", "d", "e", "f"})
+}
+
+func TestOptimizerIndexNameExist(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b int, c int, d int, index ka(a), index kbc(b, c))`)
+	tk.MustExec(`create table t2 (a int, b int, c int, d int, index ka(a), index kbc(b, c))`)
+	opt := index_advisor.NewOptimizer(tk.Session())
+
+	check := func(schema, table, indexName string, expected bool) {
+		ok, err := opt.IndexNameExist(schema, table, indexName)
+		require.NoError(t, err)
+		require.Equal(t, expected, ok)
+	}
+
+	check("test", "t1", "ka", true)
+	check("test", "t1", "kbc", true)
+	check("test", "t1", "kbc2", false)
+	check("test", "t2", "ka", true)
+	check("test", "t2", "kbc", true)
+	check("test", "t2", "kbc2", false)
+}
+
+func TestOptimizerEstIndexSize(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	h := dom.StatsHandle()
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t (a int, b varchar(64))`)
+	opt := index_advisor.NewOptimizer(tk.Session())
+
+	s, err := opt.EstIndexSize("test", "t", "a")
+	require.NoError(t, err)
+	require.Equal(t, float64(0), s)
+
+	s, err = opt.EstIndexSize("test", "t", "b")
+	require.NoError(t, err)
+	require.Equal(t, float64(0), s)
+
+	tk.MustExec(`insert into t values (1, space(32))`)
+	require.NoError(t, h.DumpStatsDeltaToKV(true))
+	require.NoError(t, h.Update(context.Background(), dom.InfoSchema()))
+	s, err = opt.EstIndexSize("test", "t", "a")
+	require.NoError(t, err)
+	require.Equal(t, float64(1), s)
+
+	s, err = opt.EstIndexSize("test", "t", "b")
+	require.NoError(t, err)
+	require.Equal(t, float64(33), s) // 32 + 1
+
+	s, err = opt.EstIndexSize("test", "t", "a", "b")
+	require.NoError(t, err)
+	require.Equal(t, float64(34), s) // 32 + 1 + 1
+
+	tk.MustExec(`insert into t values (1, space(64))`)
+	require.NoError(t, h.DumpStatsDeltaToKV(true))
+	require.NoError(t, h.Update(context.Background(), dom.InfoSchema()))
+	s, err = opt.EstIndexSize("test", "t", "a")
+	require.NoError(t, err)
+	require.Equal(t, float64(2), s) // 2 rows
+
+	s, err = opt.EstIndexSize("test", "t", "b")
+	require.NoError(t, err)
+	require.Equal(t, float64(99), s) // 32 + 64 + x
+
+	s, err = opt.EstIndexSize("test", "t", "b", "a")
+	require.NoError(t, err)
+	require.Equal(t, float64(99+2), s)
+}
+
+func TestOptimizerQueryCost(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b int, c int, d int, index ka(a), index kbc(b, c))`)
+	tk.MustExec(`create table t2 (a int, b int, c int, d int, index ka(a), index kbc(b, c))`)
+}
+
+func TestOptimizerQueryPlanCost(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t0 (a int, b int, c int)`)
+
+	opt := index_advisor.NewOptimizer(tk.Session())
+	cost1, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1")
+	require.NoError(t, err)
+
+	cost2, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1", index_advisor.Index{
+		SchemaName: "test",
+		TableName:  "t0",
+		IndexName:  "idx_a",
+		Columns: []index_advisor.Column{
+			{SchemaName: "test", TableName: "t0", ColumnName: "a"}},
+	})
+	require.NoError(t, err)
+	require.True(t, cost2 < cost1)
+
+	cost3, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1", index_advisor.Index{
+		SchemaName: "test",
+		TableName:  "t0",
+		IndexName:  "idx_a",
+		Columns: []index_advisor.Column{
+			{SchemaName: "test", TableName: "t0", ColumnName: "a"},
+			{SchemaName: "test", TableName: "t0", ColumnName: "b"}},
+	})
+	require.NoError(t, err)
+	require.True(t, cost3 < cost2)
+}
diff --git a/pkg/planner/index_advisor/set.go b/pkg/planner/index_advisor/set.go
new file mode 100644
index 0000000000000..4e8060c472d5d
--- /dev/null
+++ b/pkg/planner/index_advisor/set.go
@@ -0,0 +1,183 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index_advisor
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// SetKey is the interface for the key of a set item.
+type SetKey interface {
+	Key() string
+}
+
+// Set is the interface for a set.
+type Set[T SetKey] interface {
+	Add(items ...T)
+	Contains(item T) bool
+	Remove(item T)
+	ToList() []T
+	Size() int
+	Clone() Set[T]
+	String() string
+}
+
+type setImpl[T SetKey] struct {
+	s map[string]T
+}
+
+func NewSet[T SetKey]() Set[T] {
+	return new(setImpl[T])
+}
+
+func (s *setImpl[T]) Add(items ...T) {
+	if s.s == nil {
+		s.s = make(map[string]T)
+	}
+	for _, item := range items {
+		s.s[item.Key()] = item
+	}
+}
+
+func (s *setImpl[T]) Contains(item T) bool {
+	if s.s == nil {
+		return false
+	}
+	_, ok := s.s[item.Key()]
+	return ok
+}
+
+func (s *setImpl[T]) ToList() []T {
+	if s == nil {
+		return nil
+	}
+	var list []T
+	for _, v := range s.s {
+		list = append(list, v)
+	}
+	sort.Slice(list, func(i, j int) bool {
+		return list[i].Key() < list[j].Key()
+	}) // to make the result stable
+	return list
+}
+
+func (s *setImpl[T]) Remove(item T) {
+	delete(s.s, item.Key())
+}
+
+func (s *setImpl[T]) Size() int {
+	if s == nil {
+		return 0
+	}
+	return len(s.s)
+}
+
+func (s *setImpl[T]) Clone() Set[T] {
+	clone := NewSet[T]()
+	clone.Add(s.ToList()...)
+	return clone
+}
+
+func (s *setImpl[T]) String() string {
+	var items []string
+	for _, item := range s.s {
+		items = append(items, item.Key())
+	}
+	sort.Strings(items)
+	return fmt.Sprintf("{%v}", strings.Join(items, ", "))
+}
+
+// ListToSet converts a list to a set.
+func ListToSet[T SetKey](items ...T) Set[T] {
+	s := NewSet[T]()
+	for _, item := range items {
+		s.Add(item)
+	}
+	return s
+}
+
+// UnionSet returns the union set of the given sets.
+func UnionSet[T SetKey](ss ...Set[T]) Set[T] {
+	if len(ss) == 0 {
+		return NewSet[T]()
+	}
+	if len(ss) == 1 {
+		return ss[0].Clone()
+	}
+	s := NewSet[T]()
+	for _, set := range ss {
+		s.Add(set.ToList()...)
+	}
+	return s
+}
+
+// AndSet returns the intersection set of the given sets.
+func AndSet[T SetKey](ss ...Set[T]) Set[T] {
+	if len(ss) == 0 {
+		return NewSet[T]()
+	}
+	if len(ss) == 1 {
+		return ss[0].Clone()
+	}
+	s := NewSet[T]()
+	for _, item := range ss[0].ToList() {
+		contained := true
+		for _, set := range ss[1:] {
+			if !set.Contains(item) {
+				contained = false
+				break
+			}
+		}
+		if contained {
+			s.Add(item)
+		}
+	}
+	return s
+}
+
+// DiffSet returns a set of items that are in s1 but not in s2.
+// DiffSet({1, 2, 3, 4}, {2, 3}) = {1, 4}
+func DiffSet[T SetKey](s1, s2 Set[T]) Set[T] {
+	s := NewSet[T]()
+	for _, item := range s1.ToList() {
+		if !s2.Contains(item) {
+			s.Add(item)
+		}
+	}
+	return s
+}
+
+// CombSet returns all combinations of `numberOfItems` items in the given set.
+// For example ({a, b, c}, 2) returns {ab, ac, bc}.
+func CombSet[T SetKey](s Set[T], numberOfItems int) []Set[T] {
+	return combSetIterate(s.ToList(), NewSet[T](), 0, numberOfItems)
+}
+
+func combSetIterate[T SetKey](itemList []T, currSet Set[T], depth, numberOfItems int) []Set[T] {
+	if currSet.Size() == numberOfItems {
+		return []Set[T]{currSet.Clone()}
+	}
+	if depth == len(itemList) || currSet.Size() > numberOfItems {
+		return nil
+	}
+	var res []Set[T]
+	currSet.Add(itemList[depth])
+	res = append(res, combSetIterate(itemList, currSet, depth+1, numberOfItems)...)
+	currSet.Remove(itemList[depth])
+	res = append(res, combSetIterate(itemList, currSet, depth+1, numberOfItems)...)
+	return res
+}
diff --git a/pkg/planner/index_advisor/set_test.go b/pkg/planner/index_advisor/set_test.go
new file mode 100644
index 0000000000000..7808573127e76
--- /dev/null
+++ b/pkg/planner/index_advisor/set_test.go
@@ -0,0 +1,88 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index_advisor
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetBasic(t *testing.T) {
+	s := NewSet[Query]()
+	s.Add(Query{Text: "q1"}, Query{Text: "q2"}, Query{Text: "q3"})
+	require.True(t, s.Contains(Query{Text: "q1"}))
+	require.True(t, s.Contains(Query{Text: "q2"}))
+	require.True(t, s.Contains(Query{Text: "q3"}))
+	require.False(t, s.Contains(Query{Text: "q4"}))
+	require.Equal(t, 3, s.Size())
+	require.Equal(t, []Query{{Text: "q1"}, {Text: "q2"}, {Text: "q3"}}, s.ToList())
+	s.Remove(Query{Text: "q2"})
+	require.False(t, s.Contains(Query{Text: "q2"}))
+	require.Equal(t, 2, s.Size())
+
+	clonedS := s.Clone()
+	require.True(t, clonedS.Contains(Query{Text: "q1"}))
+	s.Remove(Query{Text: "q1"})
+	require.False(t, s.Contains(Query{Text: "q1"}))
+	require.True(t, clonedS.Contains(Query{Text: "q1"}))
+	require.Equal(t, 2, clonedS.Size())
+}
+
+func TestSetOperation(t *testing.T) {
+	s1 := NewSet[Query]()
+	s1.Add(Query{Text: "q1"}, Query{Text: "q2"}, Query{Text: "q3"})
+	s2 := NewSet[Query]()
+	s2.Add(Query{Text: "q2"}, Query{Text: "q3"}, Query{Text: "q4"})
+	unionSet := UnionSet(s1, s2)
+	require.Equal(t, []Query{{Text: "q1"}, {Text: "q2"}, {Text: "q3"}, {Text: "q4"}}, unionSet.ToList())
+
+	andSet := AndSet(s1, s2)
+	require.Equal(t, []Query{{Text: "q2"}, {Text: "q3"}}, andSet.ToList())
+
+	diffSet := DiffSet(s1, s2)
+	require.Equal(t, []Query{{Text: "q1"}}, diffSet.ToList())
+	diffSet = DiffSet(s2, s1)
+	require.Equal(t, []Query{{Text: "q4"}}, diffSet.ToList())
+}
+
+func TestSetCombination(t *testing.T) {
+	s := NewSet[Query]()
+	s.Add(Query{Text: "q1"}, Query{Text: "q2"}, Query{Text: "q3"}, Query{Text: "q4"})
+
+	setListStr := func(setList []Set[Query]) string {
+		var tmp []string
+		for _, set := range setList {
+			tmp = append(tmp, set.String())
+		}
+		return strings.Join(tmp, ", ")
+	}
+
+	s1 := CombSet(s, 1)
+	require.Equal(t, "{q1}, {q2}, {q3}, {q4}", setListStr(s1))
+
+	s2 := CombSet(s, 2)
+	require.Equal(t, "{q1, q2}, {q1, q3}, {q1, q4}, {q2, q3}, {q2, q4}, {q3, q4}", setListStr(s2))
+
+	s3 := CombSet(s, 3)
+	require.Equal(t, "{q1, q2, q3}, {q1, q2, q4}, {q1, q3, q4}, {q2, q3, q4}", setListStr(s3))
+
+	s4 := CombSet(s, 4)
+	require.Equal(t, "{q1, q2, q3, q4}", setListStr(s4))
+
+	s5 := CombSet(s, 5)
+	require.Equal(t, "", setListStr(s5))
+}
diff --git a/pkg/planner/optimize.go b/pkg/planner/optimize.go
index ac596e7491112..3d5e0262ed989 100644
--- a/pkg/planner/optimize.go
+++ b/pkg/planner/optimize.go
@@ -16,6 +16,9 @@ package planner
 
 import (
 	"context"
+	"github.com/pingcap/tidb/pkg/planner/index_advisor"
+	"github.com/pingcap/tidb/pkg/planner/property"
+	"github.com/pingcap/tidb/pkg/planner/util/optimizetrace"
 	"math"
 	"math/rand"
 	"sync"
@@ -623,9 +626,24 @@ func hypoIndexChecker(ctx context.Context, is infoschema.InfoSchema) func(db, tb
 	}
 }
 
+// queryPlanCost returns the plan cost of this node, which is mainly for the Index Advisor.
+func queryPlanCost(sctx sessionctx.Context, stmt ast.StmtNode) (float64, error) {
+	nodeW := resolve.NewNodeW(stmt)
+	plan, _, err := Optimize(context.Background(), sctx, nodeW, sctx.GetDomainInfoSchema().(infoschema.InfoSchema))
+	if err != nil {
+		return 0, err
+	}
+	pp, ok := plan.(base.PhysicalPlan)
+	if !ok {
+		return 0, errors.Errorf("plan is not a physical plan: %T", plan)
+	}
+	return core.GetPlanCost(pp, property.RootTaskType, optimizetrace.NewDefaultPlanCostOption())
+}
+
 func init() {
 	core.OptimizeAstNode = Optimize
 	core.IsReadOnly = IsReadOnly
+	index_advisor.QueryPlanCostHook = queryPlanCost
 	bindinfo.GetGlobalBindingHandle = func(sctx sessionctx.Context) bindinfo.GlobalBindingHandle {
 		return domain.GetDomain(sctx).BindHandle()
 	}

From 971f19b31c2719bc6a0fffd384123f988c5249dd Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 17:19:11 +0800
Subject: [PATCH 02/27] fixup

---
 pkg/planner/BUILD.bazel               |  3 +++
 pkg/planner/index_advisor/BUILD.bazel | 39 +++++++++++++++++++++++++++
 pkg/planner/index_advisor/set.go      |  1 +
 pkg/planner/optimize.go               |  6 ++---
 4 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 pkg/planner/index_advisor/BUILD.bazel

diff --git a/pkg/planner/BUILD.bazel b/pkg/planner/BUILD.bazel
index 7103c9c40fc51..351e865691cc4 100644
--- a/pkg/planner/BUILD.bazel
+++ b/pkg/planner/BUILD.bazel
@@ -18,7 +18,10 @@ go_library(
         "//pkg/planner/core",
         "//pkg/planner/core/base",
         "//pkg/planner/core/resolve",
+        "//pkg/planner/index_advisor",
+        "//pkg/planner/property",
         "//pkg/planner/util/debugtrace",
+        "//pkg/planner/util/optimizetrace",
         "//pkg/privilege",
         "//pkg/sessionctx",
         "//pkg/sessionctx/variable",
diff --git a/pkg/planner/index_advisor/BUILD.bazel b/pkg/planner/index_advisor/BUILD.bazel
new file mode 100644
index 0000000000000..585816c053e36
--- /dev/null
+++ b/pkg/planner/index_advisor/BUILD.bazel
@@ -0,0 +1,39 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "index_advisor",
+    srcs = [
+        "model.go",
+        "optimizer.go",
+        "set.go",
+    ],
+    importpath = "github.com/pingcap/tidb/pkg/planner/index_advisor",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/domain",
+        "//pkg/infoschema",
+        "//pkg/parser",
+        "//pkg/parser/ast",
+        "//pkg/parser/model",
+        "//pkg/planner/util/fixcontrol",
+        "//pkg/sessionctx",
+        "//pkg/types",
+    ],
+)
+
+go_test(
+    name = "index_advisor_test",
+    timeout = "short",
+    srcs = [
+        "optimizer_test.go",
+        "set_test.go",
+    ],
+    embed = [":index_advisor"],
+    flaky = True,
+    shard_count = 11,
+    deps = [
+        "//pkg/parser/mysql",
+        "//pkg/testkit",
+        "@com_github_stretchr_testify//require",
+    ],
+)
diff --git a/pkg/planner/index_advisor/set.go b/pkg/planner/index_advisor/set.go
index 4e8060c472d5d..20e153e6c4902 100644
--- a/pkg/planner/index_advisor/set.go
+++ b/pkg/planner/index_advisor/set.go
@@ -40,6 +40,7 @@ type setImpl[T SetKey] struct {
 	s map[string]T
 }
 
+// NewSet creates a new set.
 func NewSet[T SetKey]() Set[T] {
 	return new(setImpl[T])
 }
diff --git a/pkg/planner/optimize.go b/pkg/planner/optimize.go
index 3d5e0262ed989..f374b54010962 100644
--- a/pkg/planner/optimize.go
+++ b/pkg/planner/optimize.go
@@ -16,9 +16,6 @@ package planner
 
 import (
 	"context"
-	"github.com/pingcap/tidb/pkg/planner/index_advisor"
-	"github.com/pingcap/tidb/pkg/planner/property"
-	"github.com/pingcap/tidb/pkg/planner/util/optimizetrace"
 	"math"
 	"math/rand"
 	"sync"
@@ -38,7 +35,10 @@ import (
 	"github.com/pingcap/tidb/pkg/planner/core"
 	"github.com/pingcap/tidb/pkg/planner/core/base"
 	"github.com/pingcap/tidb/pkg/planner/core/resolve"
+	"github.com/pingcap/tidb/pkg/planner/index_advisor"
+	"github.com/pingcap/tidb/pkg/planner/property"
 	"github.com/pingcap/tidb/pkg/planner/util/debugtrace"
+	"github.com/pingcap/tidb/pkg/planner/util/optimizetrace"
 	"github.com/pingcap/tidb/pkg/privilege"
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"

From 6dfdb5196ca85f935672765a352fdd0a49302642 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 21:11:54 +0800
Subject: [PATCH 03/27] fixup

---
 pkg/planner/index_advisor/model.go | 21 +++++++++++----------
 pkg/planner/index_advisor/set.go   |  4 ++--
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/pkg/planner/index_advisor/model.go b/pkg/planner/index_advisor/model.go
index be4f621500857..295d0bf04a263 100644
--- a/pkg/planner/index_advisor/model.go
+++ b/pkg/planner/index_advisor/model.go
@@ -43,12 +43,13 @@ type Column struct {
 
 // NewColumn creates a new column.
 func NewColumn(schemaName, tableName, columnName string) Column {
-	return Column{SchemaName: strings.ToLower(schemaName), TableName: strings.ToLower(tableName), ColumnName: strings.ToLower(columnName)}
+	return Column{SchemaName: strings.ToLower(schemaName),
+		TableName: strings.ToLower(tableName), ColumnName: strings.ToLower(columnName)}
 }
 
 // NewColumns creates new columns.
 func NewColumns(schemaName, tableName string, columnNames ...string) []Column {
-	var cols []Column
+	cols := make([]Column, 0, len(columnNames))
 	for _, col := range columnNames {
 		cols = append(cols, NewColumn(schemaName, tableName, col))
 	}
@@ -70,7 +71,8 @@ type Index struct {
 
 // NewIndex creates a new index.
 func NewIndex(schemaName, tableName, indexName string, columns ...string) Index {
-	return Index{SchemaName: strings.ToLower(schemaName), TableName: strings.ToLower(tableName), IndexName: strings.ToLower(indexName), Columns: NewColumns(schemaName, tableName, columns...)}
+	return Index{SchemaName: strings.ToLower(schemaName), TableName: strings.ToLower(tableName),
+		IndexName: strings.ToLower(indexName), Columns: NewColumns(schemaName, tableName, columns...)}
 }
 
 // NewIndexWithColumns creates a new index with columns.
@@ -84,7 +86,7 @@ func NewIndexWithColumns(indexName string, columns ...Column) Index {
 
 // Key returns the key of the index.
 func (i Index) Key() string {
-	var names []string
+	names := make([]string, 0, len(i.Columns))
 	for _, col := range i.Columns {
 		names = append(names, col.ColumnName)
 	}
@@ -130,20 +132,19 @@ func (c IndexSetCost) Less(other IndexSetCost) bool {
 		return c.TotalNumberOfIndexColumns < other.TotalNumberOfIndexColumns
 	}
 
-	// if they have the same cost and the same number of columns, then use the IndexKeysStr to compare to make the result stable.
+	// to make the result stable.
 	return c.IndexKeysStr < other.IndexKeysStr
 }
 
 // ImpactedQuery represents the impacted query.
 type ImpactedQuery struct {
-	Query            string
-	Improvement      float64
-	CostSavingPerMon float64
+	Query       string
+	Improvement float64
 }
 
+// WorkloadImpact represents the workload impact.
 type WorkloadImpact struct {
-	WorkloadImprovement  float64
-	WorkloadSavingPerMon float64
+	WorkloadImprovement float64
 }
 
 // IndexAdvisorResult represents the result of the index advisor.
diff --git a/pkg/planner/index_advisor/set.go b/pkg/planner/index_advisor/set.go
index 20e153e6c4902..c2cb79c369ce0 100644
--- a/pkg/planner/index_advisor/set.go
+++ b/pkg/planner/index_advisor/set.go
@@ -66,7 +66,7 @@ func (s *setImpl[T]) ToList() []T {
 	if s == nil {
 		return nil
 	}
-	var list []T
+	list := make([]T, 0, len(s.s))
 	for _, v := range s.s {
 		list = append(list, v)
 	}
@@ -94,7 +94,7 @@ func (s *setImpl[T]) Clone() Set[T] {
 }
 
 func (s *setImpl[T]) String() string {
-	var items []string
+	items := make([]string, 0, len(s.s))
 	for _, item := range s.s {
 		items = append(items, item.Key())
 	}

From 9149bf23cb9bf33fbfc707a4df46b632e5a98c29 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 21:12:10 +0800
Subject: [PATCH 04/27] fixup

---
 .../BUILD.bazel                               |  0
 .../{index_advisor => indexadvisor}/model.go  |  2 +-
 .../optimizer.go                              |  2 +-
 .../optimizer_test.go                         | 44 +++++++++----------
 .../{index_advisor => indexadvisor}/set.go    |  2 +-
 .../set_test.go                               |  2 +-
 pkg/planner/optimize.go                       |  4 +-
 7 files changed, 28 insertions(+), 28 deletions(-)
 rename pkg/planner/{index_advisor => indexadvisor}/BUILD.bazel (100%)
 rename pkg/planner/{index_advisor => indexadvisor}/model.go (99%)
 rename pkg/planner/{index_advisor => indexadvisor}/optimizer.go (99%)
 rename pkg/planner/{index_advisor => indexadvisor}/optimizer_test.go (83%)
 rename pkg/planner/{index_advisor => indexadvisor}/set.go (99%)
 rename pkg/planner/{index_advisor => indexadvisor}/set_test.go (99%)

diff --git a/pkg/planner/index_advisor/BUILD.bazel b/pkg/planner/indexadvisor/BUILD.bazel
similarity index 100%
rename from pkg/planner/index_advisor/BUILD.bazel
rename to pkg/planner/indexadvisor/BUILD.bazel
diff --git a/pkg/planner/index_advisor/model.go b/pkg/planner/indexadvisor/model.go
similarity index 99%
rename from pkg/planner/index_advisor/model.go
rename to pkg/planner/indexadvisor/model.go
index 295d0bf04a263..0c2b4798c89f5 100644
--- a/pkg/planner/index_advisor/model.go
+++ b/pkg/planner/indexadvisor/model.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package index_advisor
+package indexadvisor
 
 import (
 	"fmt"
diff --git a/pkg/planner/index_advisor/optimizer.go b/pkg/planner/indexadvisor/optimizer.go
similarity index 99%
rename from pkg/planner/index_advisor/optimizer.go
rename to pkg/planner/indexadvisor/optimizer.go
index b5ec98439747a..0f500a06ae95d 100644
--- a/pkg/planner/index_advisor/optimizer.go
+++ b/pkg/planner/indexadvisor/optimizer.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package index_advisor
+package indexadvisor
 
 import (
 	"context"
diff --git a/pkg/planner/index_advisor/optimizer_test.go b/pkg/planner/indexadvisor/optimizer_test.go
similarity index 83%
rename from pkg/planner/index_advisor/optimizer_test.go
rename to pkg/planner/indexadvisor/optimizer_test.go
index 91bf1ca28d4d9..69af075d041ef 100644
--- a/pkg/planner/index_advisor/optimizer_test.go
+++ b/pkg/planner/indexadvisor/optimizer_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package index_advisor_test
+package indexadvisor_test
 
 import (
 	"context"
@@ -21,7 +21,7 @@ import (
 	"testing"
 
 	"github.com/pingcap/tidb/pkg/parser/mysql"
-	"github.com/pingcap/tidb/pkg/planner/index_advisor"
+	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/stretchr/testify/require"
 )
@@ -32,36 +32,36 @@ func TestOptimizerColumnType(t *testing.T) {
 	tk.MustExec(`use test`)
 	tk.MustExec(`create table t1 (a int, b float, c varchar(255))`)
 	tk.MustExec(`create table t2 (a int, b decimal(10,2), c varchar(1024))`)
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 
-	tp, err := opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "a"})
+	tp, err := opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "a"})
 	require.NoError(t, err)
 	require.Equal(t, mysql.TypeLong, tp.GetType())
 
-	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "b"})
+	tp, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "b"})
 	require.NoError(t, err)
 	require.Equal(t, mysql.TypeFloat, tp.GetType())
 
-	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "c"})
+	tp, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t1", ColumnName: "c"})
 	require.NoError(t, err)
 	require.Equal(t, mysql.TypeVarchar, tp.GetType())
 
-	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "a"})
+	tp, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "a"})
 	require.NoError(t, err)
 	require.Equal(t, mysql.TypeLong, tp.GetType())
 
-	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "b"})
+	tp, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "b"})
 	require.NoError(t, err)
 	require.Equal(t, mysql.TypeNewDecimal, tp.GetType())
 
-	tp, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "c"})
+	tp, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "c"})
 	require.NoError(t, err)
 	require.Equal(t, mysql.TypeVarchar, tp.GetType())
 
-	_, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "d"})
+	_, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t2", ColumnName: "d"})
 	require.Error(t, err)
 
-	_, err = opt.ColumnType(index_advisor.Column{SchemaName: "test", TableName: "t3", ColumnName: "a"})
+	_, err = opt.ColumnType(indexadvisor.Column{SchemaName: "test", TableName: "t3", ColumnName: "a"})
 	require.Error(t, err)
 }
 
@@ -71,10 +71,10 @@ func TestOptimizerPrefixContainIndex(t *testing.T) {
 	tk.MustExec(`use test`)
 	tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b, c))`)
 	tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a, b, c, d), key(d, c, b, a))`)
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 
 	check := func(expected bool, tableName string, columns ...string) {
-		ok, err := opt.PrefixContainIndex(index_advisor.NewIndex("test", tableName, "idx", columns...))
+		ok, err := opt.PrefixContainIndex(indexadvisor.NewIndex("test", tableName, "idx", columns...))
 		require.NoError(t, err)
 		require.Equal(t, expected, ok)
 	}
@@ -103,7 +103,7 @@ func TestOptimizerPossibleColumns(t *testing.T) {
 	tk.MustExec(`create table t1 (a int, b int, c int, d int)`)
 	tk.MustExec(`create table t2 (a int, b int, c int, d int)`)
 	tk.MustExec(`create table t3 (c int, d int, e int, f int)`)
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 
 	check := func(schema, colName string, expected []string) {
 		cols, err := opt.PossibleColumns(schema, colName)
@@ -132,7 +132,7 @@ func TestOptimizerTableColumns(t *testing.T) {
 	tk.MustExec(`create table t1 (a int, b int, c int, d int)`)
 	tk.MustExec(`create table t2 (a int, b int, c int, d int)`)
 	tk.MustExec(`create table t3 (c int, d int, e int, f int)`)
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 
 	check := func(schemaName, tableName string, columns []string) {
 		cols, err := opt.TableColumns(schemaName, tableName)
@@ -158,7 +158,7 @@ func TestOptimizerIndexNameExist(t *testing.T) {
 	tk.MustExec(`use test`)
 	tk.MustExec(`create table t1 (a int, b int, c int, d int, index ka(a), index kbc(b, c))`)
 	tk.MustExec(`create table t2 (a int, b int, c int, d int, index ka(a), index kbc(b, c))`)
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 
 	check := func(schema, table, indexName string, expected bool) {
 		ok, err := opt.IndexNameExist(schema, table, indexName)
@@ -180,7 +180,7 @@ func TestOptimizerEstIndexSize(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec(`use test`)
 	tk.MustExec(`create table t (a int, b varchar(64))`)
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 
 	s, err := opt.EstIndexSize("test", "t", "a")
 	require.NoError(t, err)
@@ -235,25 +235,25 @@ func TestOptimizerQueryPlanCost(t *testing.T) {
 	tk.MustExec(`use test`)
 	tk.MustExec(`create table t0 (a int, b int, c int)`)
 
-	opt := index_advisor.NewOptimizer(tk.Session())
+	opt := indexadvisor.NewOptimizer(tk.Session())
 	cost1, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1")
 	require.NoError(t, err)
 
-	cost2, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1", index_advisor.Index{
+	cost2, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1", indexadvisor.Index{
 		SchemaName: "test",
 		TableName:  "t0",
 		IndexName:  "idx_a",
-		Columns: []index_advisor.Column{
+		Columns: []indexadvisor.Column{
 			{SchemaName: "test", TableName: "t0", ColumnName: "a"}},
 	})
 	require.NoError(t, err)
 	require.True(t, cost2 < cost1)
 
-	cost3, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1", index_advisor.Index{
+	cost3, err := opt.QueryPlanCost("select a, b from t0 where a=1 and b=1", indexadvisor.Index{
 		SchemaName: "test",
 		TableName:  "t0",
 		IndexName:  "idx_a",
-		Columns: []index_advisor.Column{
+		Columns: []indexadvisor.Column{
 			{SchemaName: "test", TableName: "t0", ColumnName: "a"},
 			{SchemaName: "test", TableName: "t0", ColumnName: "b"}},
 	})
diff --git a/pkg/planner/index_advisor/set.go b/pkg/planner/indexadvisor/set.go
similarity index 99%
rename from pkg/planner/index_advisor/set.go
rename to pkg/planner/indexadvisor/set.go
index c2cb79c369ce0..641639e03a558 100644
--- a/pkg/planner/index_advisor/set.go
+++ b/pkg/planner/indexadvisor/set.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package index_advisor
+package indexadvisor
 
 import (
 	"fmt"
diff --git a/pkg/planner/index_advisor/set_test.go b/pkg/planner/indexadvisor/set_test.go
similarity index 99%
rename from pkg/planner/index_advisor/set_test.go
rename to pkg/planner/indexadvisor/set_test.go
index 7808573127e76..54f4da41ced13 100644
--- a/pkg/planner/index_advisor/set_test.go
+++ b/pkg/planner/indexadvisor/set_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package index_advisor
+package indexadvisor
 
 import (
 	"strings"
diff --git a/pkg/planner/optimize.go b/pkg/planner/optimize.go
index f374b54010962..808d467a76bb9 100644
--- a/pkg/planner/optimize.go
+++ b/pkg/planner/optimize.go
@@ -35,7 +35,7 @@ import (
 	"github.com/pingcap/tidb/pkg/planner/core"
 	"github.com/pingcap/tidb/pkg/planner/core/base"
 	"github.com/pingcap/tidb/pkg/planner/core/resolve"
-	"github.com/pingcap/tidb/pkg/planner/index_advisor"
+	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
 	"github.com/pingcap/tidb/pkg/planner/property"
 	"github.com/pingcap/tidb/pkg/planner/util/debugtrace"
 	"github.com/pingcap/tidb/pkg/planner/util/optimizetrace"
@@ -643,7 +643,7 @@ func queryPlanCost(sctx sessionctx.Context, stmt ast.StmtNode) (float64, error)
 func init() {
 	core.OptimizeAstNode = Optimize
 	core.IsReadOnly = IsReadOnly
-	index_advisor.QueryPlanCostHook = queryPlanCost
+	indexadvisor.QueryPlanCostHook = queryPlanCost
 	bindinfo.GetGlobalBindingHandle = func(sctx sessionctx.Context) bindinfo.GlobalBindingHandle {
 		return domain.GetDomain(sctx).BindHandle()
 	}

From 608213319ebb2477534a29a6f9ec2e1b2fdde495 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 21:12:51 +0800
Subject: [PATCH 05/27] fixup

---
 pkg/planner/BUILD.bazel              |  2 +-
 pkg/planner/indexadvisor/BUILD.bazel | 35 ++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/pkg/planner/BUILD.bazel b/pkg/planner/BUILD.bazel
index 351e865691cc4..3d6398176dc79 100644
--- a/pkg/planner/BUILD.bazel
+++ b/pkg/planner/BUILD.bazel
@@ -18,7 +18,7 @@ go_library(
         "//pkg/planner/core",
         "//pkg/planner/core/base",
         "//pkg/planner/core/resolve",
-        "//pkg/planner/index_advisor",
+        "//pkg/planner/indexadvisor",
         "//pkg/planner/property",
         "//pkg/planner/util/debugtrace",
         "//pkg/planner/util/optimizetrace",
diff --git a/pkg/planner/indexadvisor/BUILD.bazel b/pkg/planner/indexadvisor/BUILD.bazel
index 585816c053e36..74dada3560c9c 100644
--- a/pkg/planner/indexadvisor/BUILD.bazel
+++ b/pkg/planner/indexadvisor/BUILD.bazel
@@ -37,3 +37,38 @@ go_test(
         "@com_github_stretchr_testify//require",
     ],
 )
+
+go_library(
+    name = "indexadvisor",
+    srcs = [
+        "model.go",
+        "optimizer.go",
+        "set.go",
+    ],
+    importpath = "github.com/pingcap/tidb/pkg/planner/indexadvisor",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/domain",
+        "//pkg/infoschema",
+        "//pkg/parser",
+        "//pkg/parser/ast",
+        "//pkg/parser/model",
+        "//pkg/planner/util/fixcontrol",
+        "//pkg/sessionctx",
+        "//pkg/types",
+    ],
+)
+
+go_test(
+    name = "indexadvisor_test",
+    srcs = [
+        "optimizer_test.go",
+        "set_test.go",
+    ],
+    embed = [":indexadvisor"],
+    deps = [
+        "//pkg/parser/mysql",
+        "//pkg/testkit",
+        "@com_github_stretchr_testify//require",
+    ],
+)

From 888dd9fa1d347c211cdeaee36e9433430ea9fdb7 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 21:21:57 +0800
Subject: [PATCH 06/27] fixup

---
 pkg/planner/indexadvisor/model.go | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/pkg/planner/indexadvisor/model.go b/pkg/planner/indexadvisor/model.go
index 0c2b4798c89f5..14e767235480f 100644
--- a/pkg/planner/indexadvisor/model.go
+++ b/pkg/planner/indexadvisor/model.go
@@ -135,26 +135,3 @@ func (c IndexSetCost) Less(other IndexSetCost) bool {
 	// to make the result stable.
 	return c.IndexKeysStr < other.IndexKeysStr
 }
-
-// ImpactedQuery represents the impacted query.
-type ImpactedQuery struct {
-	Query       string
-	Improvement float64
-}
-
-// WorkloadImpact represents the workload impact.
-type WorkloadImpact struct {
-	WorkloadImprovement float64
-}
-
-// IndexAdvisorResult represents the result of the index advisor.
-type IndexAdvisorResult struct {
-	Database           string
-	Table              string
-	IndexName          string
-	IndexColumns       []string
-	IndexSize          uint64
-	Reason             string
-	WorkloadImpact     *WorkloadImpact
-	TopImpactedQueries []*ImpactedQuery
-}

From c724933a10c5ecaa46d14c4b78b92b81467ea559 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Tue, 3 Sep 2024 21:41:02 +0800
Subject: [PATCH 07/27] fixup

---
 pkg/planner/indexadvisor/BUILD.bazel | 41 ++--------------------------
 1 file changed, 3 insertions(+), 38 deletions(-)

diff --git a/pkg/planner/indexadvisor/BUILD.bazel b/pkg/planner/indexadvisor/BUILD.bazel
index 74dada3560c9c..23fd4fd5643ac 100644
--- a/pkg/planner/indexadvisor/BUILD.bazel
+++ b/pkg/planner/indexadvisor/BUILD.bazel
@@ -1,43 +1,5 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
-go_library(
-    name = "index_advisor",
-    srcs = [
-        "model.go",
-        "optimizer.go",
-        "set.go",
-    ],
-    importpath = "github.com/pingcap/tidb/pkg/planner/index_advisor",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//pkg/domain",
-        "//pkg/infoschema",
-        "//pkg/parser",
-        "//pkg/parser/ast",
-        "//pkg/parser/model",
-        "//pkg/planner/util/fixcontrol",
-        "//pkg/sessionctx",
-        "//pkg/types",
-    ],
-)
-
-go_test(
-    name = "index_advisor_test",
-    timeout = "short",
-    srcs = [
-        "optimizer_test.go",
-        "set_test.go",
-    ],
-    embed = [":index_advisor"],
-    flaky = True,
-    shard_count = 11,
-    deps = [
-        "//pkg/parser/mysql",
-        "//pkg/testkit",
-        "@com_github_stretchr_testify//require",
-    ],
-)
-
 go_library(
     name = "indexadvisor",
     srcs = [
@@ -61,11 +23,14 @@ go_library(
 
 go_test(
     name = "indexadvisor_test",
+    timeout = "short",
     srcs = [
         "optimizer_test.go",
         "set_test.go",
     ],
     embed = [":indexadvisor"],
+    flaky = True,
+    shard_count = 11,
     deps = [
         "//pkg/parser/mysql",
         "//pkg/testkit",

From e17417c863e9ba4844a2c83c2618c5aced46298d Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 10:46:47 +0800
Subject: [PATCH 08/27] fixup

---
 pkg/planner/indexadvisor/optimizer.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/planner/indexadvisor/optimizer.go b/pkg/planner/indexadvisor/optimizer.go
index 0f500a06ae95d..c36de1d0b187f 100644
--- a/pkg/planner/indexadvisor/optimizer.go
+++ b/pkg/planner/indexadvisor/optimizer.go
@@ -30,6 +30,7 @@ import (
 )
 
 // QueryPlanCostHook is used to calculate the cost of the query plan on this sctx.
+// This hook is used to avoid cyclic import.
 var QueryPlanCostHook func(sctx sessionctx.Context, stmt ast.StmtNode) (float64, error)
 
 // Optimizer is the interface of a what-if optimizer.

From d0327ff8f7a1f960b3451699f6b6a349fbf2f872 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:12:19 +0800
Subject: [PATCH 09/27] fixup

---
 pkg/planner/indexadvisor/utils.go      | 460 +++++++++++++++++++++++++
 pkg/planner/indexadvisor/utils_test.go |  75 ++++
 2 files changed, 535 insertions(+)
 create mode 100644 pkg/planner/indexadvisor/utils.go
 create mode 100644 pkg/planner/indexadvisor/utils_test.go

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
new file mode 100644
index 0000000000000..2f9946ecc04e7
--- /dev/null
+++ b/pkg/planner/indexadvisor/utils.go
@@ -0,0 +1,460 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexadvisor
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pingcap/tidb/pkg/parser"
+	"github.com/pingcap/tidb/pkg/parser/ast"
+	"github.com/pingcap/tidb/pkg/parser/mysql"
+	"github.com/pingcap/tidb/pkg/parser/opcode"
+	"github.com/pingcap/tidb/pkg/types"
+	driver "github.com/pingcap/tidb/pkg/types/parser_driver"
+	"github.com/pingcap/tidb/pkg/util/logutil"
+	parser2 "github.com/pingcap/tidb/pkg/util/parser"
+	"go.uber.org/zap"
+)
+
+// ParseOneSQL parses the given Query text and returns the AST.
+func ParseOneSQL(sqlText string) (ast.StmtNode, error) {
+	p := parser.New()
+	return p.ParseOneStmt(sqlText, "", "")
+}
+
+// NormalizeDigest normalizes the given Query text and returns the normalized Query text and its digest.
+func NormalizeDigest(sqlText string) (string, string) {
+	norm, d := parser.NormalizeDigest(sqlText)
+	return norm, d.String()
+}
+
+type nodeVisitor struct {
+	enter func(n ast.Node) (skip bool)
+	leave func(n ast.Node) (skip bool)
+}
+
+func (v *nodeVisitor) Enter(n ast.Node) (out ast.Node, skipChildren bool) {
+	if v.enter != nil {
+		return n, v.enter(n)
+	}
+	return n, false
+}
+
+func (c *nodeVisitor) Leave(n ast.Node) (out ast.Node, ok bool) {
+	if c.leave != nil {
+		return n, c.leave(n)
+	}
+	return n, true
+}
+
+func visitNode(n ast.Node, enter, leave func(n ast.Node) (skip bool)) {
+	n.Accept(&nodeVisitor{enter, leave})
+}
+
+// collectTableNamesFromQuery returns all referenced table names in the given Query text.
+func collectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
+	node, err := ParseOneSQL(query)
+	if err != nil {
+		return nil, err
+	}
+	cteNames := make(map[string]struct{})
+	var tableNames []string
+	visitNode(node, func(n ast.Node) bool {
+		switch x := n.(type) {
+		case *ast.WithClause:
+			for _, cte := range x.CTEs {
+				cteNames[fmt.Sprintf("%v.%v", defaultSchema, cte.Name.String())] = struct{}{}
+			}
+		case *ast.TableName:
+			var tableName string
+			if x.Schema.L == "" {
+				tableName = fmt.Sprintf("%v.%v", defaultSchema, x.Name.String())
+			} else {
+				tableName = fmt.Sprintf("%v.%v", x.Schema.L, x.Name.String())
+			}
+			if _, ok := cteNames[tableName]; !ok {
+				tableNames = append(tableNames, tableName)
+			}
+		}
+		return false
+	}, nil)
+	return tableNames, nil
+}
+
+func collectSelectColumnsFromQuery(q Query) (Set[Column], error) {
+	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+	if err != nil {
+		return nil, err
+	}
+	if len(names) != 1 { // unsupported yet
+		return nil, nil
+	}
+	tmp := strings.Split(names[0], ".")
+	node, err := ParseOneSQL(q.Text)
+	if err != nil {
+		return nil, err
+	}
+	underSelectField := false
+	selectCols := NewSet[Column]()
+	visitNode(node, func(n ast.Node) bool {
+		switch x := n.(type) {
+		case *ast.SelectField:
+			underSelectField = true
+		case *ast.ColumnNameExpr:
+			if underSelectField {
+				selectCols.Add(Column{
+					SchemaName: tmp[0],
+					TableName:  tmp[1],
+					ColumnName: x.Name.Name.O})
+			}
+		}
+		return false
+	}, func(n ast.Node) bool {
+		switch n.(type) {
+		case *ast.SelectField:
+			underSelectField = false
+		}
+		return true
+	})
+	return selectCols, nil
+}
+
+func collectOrderByColumnsFromQuery(q Query) ([]Column, error) {
+	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+	if err != nil {
+		return nil, err
+	}
+	if len(names) != 1 { // unsupported yet
+		return nil, nil
+	}
+	tmp := strings.Split(names[0], ".")
+	node, err := ParseOneSQL(q.Text)
+	if err != nil {
+		return nil, err
+	}
+	var orderByCols []Column
+	exit := false
+	visitNode(node, func(n ast.Node) bool {
+		if exit {
+			return true
+		}
+		switch x := n.(type) {
+		case *ast.OrderByClause:
+			for _, byItem := range x.Items {
+				colExpr, ok := byItem.Expr.(*ast.ColumnNameExpr)
+				if !ok {
+					orderByCols = nil
+					exit = true
+					return true
+				}
+				orderByCols = append(orderByCols, Column{
+					SchemaName: tmp[0],
+					TableName:  tmp[1],
+					ColumnName: colExpr.Name.Name.O})
+			}
+		}
+		return false
+	}, nil)
+	return orderByCols, nil
+}
+
+// collectDNFColumnsFromQuery parses the given Query text and returns the DNF columns.
+// For a query `select ... where c1=1 or c2=2 or c3=3`, the DNF columns are `c1`, `c2` and `c3`.
+func collectDNFColumnsFromQuery(q Query) (Set[Column], error) {
+	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+	if err != nil {
+		return nil, err
+	}
+	if len(names) != 1 { // unsupported yet
+		return nil, nil
+	}
+	tmp := strings.Split(names[0], ".")
+	node, err := ParseOneSQL(q.Text)
+	if err != nil {
+		return nil, err
+	}
+	dnfColSet := NewSet[Column]()
+
+	visitNode(node, func(n ast.Node) bool {
+		if dnfColSet.Size() > 0 { // already collected
+			return true
+		}
+		switch x := n.(type) {
+		case *ast.SelectStmt:
+			cnf := flattenCNF(x.Where)
+			for _, expr := range cnf {
+				dnf := flattenDNF(expr)
+				if len(dnf) <= 1 {
+					continue
+				}
+				// c1=1 or c2=2 or c3=3
+				var dnfCols []*ast.ColumnNameExpr
+				fail := false
+				for _, dnfExpr := range dnf {
+					col, _ := flattenColEQConst(dnfExpr)
+					if col == nil {
+						fail = true
+						break
+					}
+					dnfCols = append(dnfCols, col)
+				}
+				if fail {
+					continue
+				}
+				for _, col := range dnfCols {
+					dnfColSet.Add(Column{SchemaName: tmp[0], TableName: tmp[1], ColumnName: col.Name.Name.O})
+				}
+			}
+		}
+		return false
+	}, nil)
+
+	return dnfColSet, nil
+}
+
+func flattenColEQConst(expr ast.ExprNode) (*ast.ColumnNameExpr, *driver.ValueExpr) {
+	if _, ok := expr.(*ast.ParenthesesExpr); ok {
+		return flattenColEQConst(expr.(*ast.ParenthesesExpr).Expr)
+	}
+
+	if op, ok := expr.(*ast.BinaryOperationExpr); ok && op.Op == opcode.EQ {
+		l, r := op.L, op.R
+		_, lIsCol := l.(*ast.ColumnNameExpr)
+		_, lIsCon := l.(*driver.ValueExpr)
+		_, rIsCol := r.(*ast.ColumnNameExpr)
+		_, rIsCon := r.(*driver.ValueExpr)
+		if lIsCol && rIsCon {
+			return l.(*ast.ColumnNameExpr), r.(*driver.ValueExpr)
+		}
+		if lIsCon && rIsCol {
+			return r.(*ast.ColumnNameExpr), l.(*driver.ValueExpr)
+		}
+	}
+	return nil, nil
+}
+
+func flattenCNF(expr ast.ExprNode) []ast.ExprNode {
+	if _, ok := expr.(*ast.ParenthesesExpr); ok {
+		return flattenCNF(expr.(*ast.ParenthesesExpr).Expr)
+	}
+
+	var cnf []ast.ExprNode
+	if op, ok := expr.(*ast.BinaryOperationExpr); ok && op.Op == opcode.LogicAnd {
+		cnf = append(cnf, flattenCNF(op.L)...)
+		cnf = append(cnf, flattenCNF(op.R)...)
+	} else {
+		cnf = append(cnf, expr)
+	}
+	return cnf
+}
+
+func flattenDNF(expr ast.ExprNode) []ast.ExprNode {
+	if _, ok := expr.(*ast.ParenthesesExpr); ok {
+		return flattenDNF(expr.(*ast.ParenthesesExpr).Expr)
+	}
+
+	var cnf []ast.ExprNode
+	if op, ok := expr.(*ast.BinaryOperationExpr); ok && op.Op == opcode.LogicOr {
+		cnf = append(cnf, flattenDNF(op.L)...)
+		cnf = append(cnf, flattenDNF(op.R)...)
+	} else {
+		cnf = append(cnf, expr)
+	}
+	return cnf
+}
+
+func restoreSchemaName(defaultSchema string, sqls Set[Query], returnErr bool) (Set[Query], error) {
+	s := NewSet[Query]()
+	for _, sql := range sqls.ToList() {
+		if sql.SchemaName == "" {
+			sql.SchemaName = defaultSchema
+		}
+		stmt, err := ParseOneSQL(sql.Text)
+		if err != nil {
+			if returnErr {
+				return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
+			}
+			continue
+		}
+		sql.Text = parser2.RestoreWithDefaultDB(stmt, sql.SchemaName, sql.Text)
+		s.Add(sql)
+	}
+	return s, nil
+}
+
+// some queries might be forbidden by the fix-control 43817.
+func filterInvalidQueries(opt Optimizer, sqls Set[Query], returnErr bool) (Set[Query], error) {
+	s := NewSet[Query]()
+	for _, sql := range sqls.ToList() {
+		_, err := opt.QueryPlanCost(sql.Text)
+		if err == nil {
+			s.Add(sql)
+		} else if err != nil && returnErr {
+			return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
+		}
+	}
+	return s, nil
+}
+
+func filterSQLAccessingSystemTables(sqls Set[Query], returnErr bool) (Set[Query], error) {
+	s := NewSet[Query]()
+	for _, sql := range sqls.ToList() {
+		accessSystemTable := false
+		names, err := collectTableNamesFromQuery(sql.SchemaName, sql.Text)
+		if err != nil {
+			if returnErr {
+				return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
+			}
+			continue
+		}
+		if len(names) == 0 {
+			// `select @@some_var` or `select some_func()`
+			continue
+		}
+		for _, name := range names {
+			schemaName := strings.ToLower(strings.Split(name, ".")[0])
+			if schemaName == "information_schema" || schemaName == "metrics_schema" ||
+				schemaName == "performance_schema" || schemaName == "mysql" {
+				accessSystemTable = true
+				break
+			}
+		}
+		if !accessSystemTable {
+			s.Add(sql)
+		}
+	}
+	return s, nil
+}
+
+// CollectIndexableColumnsForQuerySet finds all columns that appear in any range-filter, order-by, or group-by clause.
+func CollectIndexableColumnsForQuerySet(opt Optimizer, querySet Set[Query]) (Set[Column], error) {
+	indexableColumnSet := NewSet[Column]()
+	queryList := querySet.ToList()
+	for _, q := range queryList {
+		cols, err := CollectIndexableColumnsFromQuery(q, opt)
+		if err != nil {
+			return nil, err
+		}
+		querySet.Add(q)
+		indexableColumnSet.Add(cols.ToList()...)
+	}
+	return indexableColumnSet, nil
+}
+
+// CollectIndexableColumnsFromQuery parses the given Query text and returns the indexable columns.
+func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (Set[Column], error) {
+	tableNames, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+	if err != nil {
+		return nil, err
+	}
+	possibleSchemas := make(map[string]bool)
+	possibleSchemas[q.SchemaName] = true
+	for _, name := range tableNames {
+		schemaName := strings.Split(name, ".")[0]
+		possibleSchemas[strings.ToLower(schemaName)] = true
+	}
+
+	stmt, err := ParseOneSQL(q.Text)
+	if err != nil {
+		return nil, err
+	}
+	cols := NewSet[Column]()
+	var collectColumn func(n ast.Node)
+	collectColumn = func(n ast.Node) {
+		switch x := n.(type) {
+		case *ast.ColumnNameExpr:
+			collectColumn(x.Name)
+		case *ast.ColumnName:
+			var schemaNames []string
+			if x.Schema.L != "" {
+				schemaNames = append(schemaNames, x.Schema.L)
+			} else {
+				for schemaName := range possibleSchemas {
+					schemaNames = append(schemaNames, schemaName)
+				}
+			}
+
+			var possibleColumns []Column
+			for _, schemaName := range schemaNames {
+				cols, err := opt.PossibleColumns(schemaName, x.Name.L)
+				if err != nil {
+					// TODO: log or return this error?
+					continue
+				}
+				possibleColumns = append(possibleColumns, cols...)
+			}
+
+			for _, c := range possibleColumns {
+				colType, err := opt.ColumnType(c)
+				if err != nil {
+					// TODO: log?
+					continue
+				}
+				if !isIndexableColumnType(colType) {
+					continue
+				}
+				cols.Add(c)
+			}
+		}
+	}
+
+	visitNode(stmt, func(n ast.Node) (skip bool) {
+		switch x := n.(type) {
+		case *ast.GroupByClause: // group by {col}
+			for _, item := range x.Items {
+				collectColumn(item.Expr)
+			}
+			return true
+		case *ast.OrderByClause: // order by {col}
+			for _, item := range x.Items {
+				collectColumn(item.Expr)
+			}
+			return true
+		case *ast.BetweenExpr: // {col} between ? and ?
+			collectColumn(x.Expr)
+		case *ast.PatternInExpr: // {col} in (?, ?, ...)
+			collectColumn(x.Expr)
+		case *ast.BinaryOperationExpr: // range predicates like `{col} > ?`
+			switch x.Op {
+			case opcode.EQ, opcode.LT, opcode.LE, opcode.GT, opcode.GE: // {col} = ?
+				collectColumn(x.L)
+				collectColumn(x.R)
+			}
+		default:
+		}
+		return false
+	}, nil)
+	return cols, nil
+}
+
+func isIndexableColumnType(tp *types.FieldType) bool {
+	if tp == nil {
+		return false
+	}
+	switch tp.GetType() {
+	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeYear,
+		mysql.TypeFloat, mysql.TypeDouble, mysql.TypeNewDecimal,
+		mysql.TypeDuration, mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp:
+		return true
+	case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString:
+		return tp.GetFlen() <= 512
+	}
+	return false
+}
+
+func l() *zap.Logger {
+	return logutil.BgLogger().With(zap.String("component", "index_advisor"))
+}
diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
new file mode 100644
index 0000000000000..f3d8576e186dd
--- /dev/null
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -0,0 +1,75 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexadvisor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCollectTableFromQuery(t *testing.T) {
+	names, err := collectTableNamesFromQuery("test", "select * from t where a = 1")
+	require.NoError(t, err)
+	require.Equal(t, names[0], "test.t")
+
+	names, err = collectTableNamesFromQuery("test", "select * from t1, t2")
+	require.NoError(t, err)
+	require.Equal(t, names[0], "test.t1")
+	require.Equal(t, names[1], "test.t2")
+
+	names, err = collectTableNamesFromQuery("test", "select * from t1 where t1.a < (select max(b) from t2)")
+	require.NoError(t, err)
+	require.Equal(t, names[0], "test.t1")
+	require.Equal(t, names[1], "test.t2")
+
+	names, err = collectTableNamesFromQuery("test", "select * from t1 where t1.a < (select max(b) from db2.t2)")
+	require.NoError(t, err)
+	require.Equal(t, names[0], "test.t1")
+	require.Equal(t, names[1], "db2.t2")
+}
+
+func TestCollectSelectColumnsFromQuery(t *testing.T) {
+	names, err := collectSelectColumnsFromQuery(Query{Text: "select a, b from test.t"})
+	require.NoError(t, err)
+	require.True(t, names.String() == "{test.t.a, test.t.b}")
+
+	names, err = collectSelectColumnsFromQuery(Query{Text: "select a, b, c from test.t"})
+	require.NoError(t, err)
+	require.True(t, names.String() == "{test.t.a, test.t.b, test.t.c}")
+}
+
+func TestCollectOrderByColumnsFromQuery(t *testing.T) {
+	cols, err := collectOrderByColumnsFromQuery(Query{Text: "select a, b from test.t order by a"})
+	require.NoError(t, err)
+	require.Equal(t, len(cols), 1)
+	require.Equal(t, cols[0].Key(), "test.t.a")
+
+	cols, err = collectOrderByColumnsFromQuery(Query{Text: "select a, b from test.t order by a, b"})
+	require.NoError(t, err)
+	require.Equal(t, len(cols), 2)
+	require.Equal(t, cols[0].Key(), "test.t.a")
+	require.Equal(t, cols[1].Key(), "test.t.b")
+}
+
+func TestCollectDNFColumnsFromQuery(t *testing.T) {
+	cols, err := collectDNFColumnsFromQuery(Query{Text: "select a, b from test.t where a = 1 or b = 2"})
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{test.t.a, test.t.b}")
+
+	cols, err = collectDNFColumnsFromQuery(Query{Text: "select a, b from test.t where a = 1 or b = 2 or c=3"})
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{test.t.a, test.t.b, test.t.c}")
+}

From 936b9fb712bca7f4247565b89f92d6b15e8837e9 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:24:19 +0800
Subject: [PATCH 10/27] fixup

---
 pkg/planner/indexadvisor/utils.go      | 17 +++++++++++------
 pkg/planner/indexadvisor/utils_test.go | 16 ++++++++++++++++
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index 2f9946ecc04e7..5ce1041aa51d1 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -43,7 +43,7 @@ func NormalizeDigest(sqlText string) (string, string) {
 
 type nodeVisitor struct {
 	enter func(n ast.Node) (skip bool)
-	leave func(n ast.Node) (skip bool)
+	leave func(n ast.Node) (ok bool)
 }
 
 func (v *nodeVisitor) Enter(n ast.Node) (out ast.Node, skipChildren bool) {
@@ -60,11 +60,12 @@ func (c *nodeVisitor) Leave(n ast.Node) (out ast.Node, ok bool) {
 	return n, true
 }
 
-func visitNode(n ast.Node, enter, leave func(n ast.Node) (skip bool)) {
+func visitNode(n ast.Node, enter func(n ast.Node) (skip bool), leave func(n ast.Node) (ok bool)) {
 	n.Accept(&nodeVisitor{enter, leave})
 }
 
 // collectTableNamesFromQuery returns all referenced table names in the given Query text.
+// The returned format is []string{"schema.table", "schema.table", ...}.
 func collectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
 	node, err := ParseOneSQL(query)
 	if err != nil {
@@ -94,6 +95,8 @@ func collectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
 	return tableNames, nil
 }
 
+// collectSelectColumnsFromQuery parses the given Query text and returns the selected columns.
+// For example, "select a, b, c from t" returns []string{"a", "b", "c"}.
 func collectSelectColumnsFromQuery(q Query) (Set[Column], error) {
 	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
@@ -132,6 +135,8 @@ func collectSelectColumnsFromQuery(q Query) (Set[Column], error) {
 	return selectCols, nil
 }
 
+// collectOrderByColumnsFromQuery parses the given Query text and returns the order-by columns.
+// For example, "select a, b from t order by a, b" returns []string{"a", "b"}.
 func collectOrderByColumnsFromQuery(q Query) ([]Column, error) {
 	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
@@ -276,7 +281,7 @@ func flattenDNF(expr ast.ExprNode) []ast.ExprNode {
 	return cnf
 }
 
-func restoreSchemaName(defaultSchema string, sqls Set[Query], returnErr bool) (Set[Query], error) {
+func restoreSchemaName(defaultSchema string, sqls Set[Query], ignoreErr bool) (Set[Query], error) {
 	s := NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		if sql.SchemaName == "" {
@@ -284,10 +289,10 @@ func restoreSchemaName(defaultSchema string, sqls Set[Query], returnErr bool) (S
 		}
 		stmt, err := ParseOneSQL(sql.Text)
 		if err != nil {
-			if returnErr {
-				return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
+			if ignoreErr {
+				continue
 			}
-			continue
+			return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
 		}
 		sql.Text = parser2.RestoreWithDefaultDB(stmt, sql.SchemaName, sql.Text)
 		s.Add(sql)
diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index f3d8576e186dd..e8dde4b7b41fa 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -73,3 +73,19 @@ func TestCollectDNFColumnsFromQuery(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, cols.String(), "{test.t.a, test.t.b, test.t.c}")
 }
+
+func TestRestoreSchemaName(t *testing.T) {
+	q1 := Query{Text: "select * from t1"}
+	q2 := Query{Text: "select * from t2", SchemaName: "test2"}
+	q3 := Query{Text: "select * from t3"}
+	q4 := Query{Text: "wrong"}
+	set1 := NewSet[Query]()
+	set1.Add(q1, q2, q3, q4)
+
+	set2, err := restoreSchemaName("test", set1, true)
+	require.NoError(t, err)
+	require.Equal(t, set2.String(), "{SELECT * FROM `test2`.`t2`, SELECT * FROM `test`.`t1`, SELECT * FROM `test`.`t3`}")
+
+	_, err = restoreSchemaName("test", set1, false)
+	require.Error(t, err)
+}

From a4ffad2f1999c4fb867b40f36b29a8ad1ad1c878 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:39:43 +0800
Subject: [PATCH 11/27] fixup

---
 pkg/planner/indexadvisor/utils.go      | 45 ++++++++++++++------------
 pkg/planner/indexadvisor/utils_test.go | 37 ++++++++++-----------
 2 files changed, 44 insertions(+), 38 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index 5ce1041aa51d1..d9b806fcc3b20 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -64,9 +64,9 @@ func visitNode(n ast.Node, enter func(n ast.Node) (skip bool), leave func(n ast.
 	n.Accept(&nodeVisitor{enter, leave})
 }
 
-// collectTableNamesFromQuery returns all referenced table names in the given Query text.
+// CollectTableNamesFromQuery returns all referenced table names in the given Query text.
 // The returned format is []string{"schema.table", "schema.table", ...}.
-func collectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
+func CollectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
 	node, err := ParseOneSQL(query)
 	if err != nil {
 		return nil, err
@@ -95,10 +95,10 @@ func collectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
 	return tableNames, nil
 }
 
-// collectSelectColumnsFromQuery parses the given Query text and returns the selected columns.
+// CollectSelectColumnsFromQuery parses the given Query text and returns the selected columns.
 // For example, "select a, b, c from t" returns []string{"a", "b", "c"}.
-func collectSelectColumnsFromQuery(q Query) (Set[Column], error) {
-	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+func CollectSelectColumnsFromQuery(q Query) (Set[Column], error) {
+	names, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
 	}
@@ -135,10 +135,10 @@ func collectSelectColumnsFromQuery(q Query) (Set[Column], error) {
 	return selectCols, nil
 }
 
-// collectOrderByColumnsFromQuery parses the given Query text and returns the order-by columns.
+// CollectOrderByColumnsFromQuery parses the given Query text and returns the order-by columns.
 // For example, "select a, b from t order by a, b" returns []string{"a", "b"}.
-func collectOrderByColumnsFromQuery(q Query) ([]Column, error) {
-	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+func CollectOrderByColumnsFromQuery(q Query) ([]Column, error) {
+	names, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
 	}
@@ -176,10 +176,10 @@ func collectOrderByColumnsFromQuery(q Query) ([]Column, error) {
 	return orderByCols, nil
 }
 
-// collectDNFColumnsFromQuery parses the given Query text and returns the DNF columns.
+// CollectDNFColumnsFromQuery parses the given Query text and returns the DNF columns.
 // For a query `select ... where c1=1 or c2=2 or c3=3`, the DNF columns are `c1`, `c2` and `c3`.
-func collectDNFColumnsFromQuery(q Query) (Set[Column], error) {
-	names, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+func CollectDNFColumnsFromQuery(q Query) (Set[Column], error) {
+	names, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
 	}
@@ -281,7 +281,8 @@ func flattenDNF(expr ast.ExprNode) []ast.ExprNode {
 	return cnf
 }
 
-func restoreSchemaName(defaultSchema string, sqls Set[Query], ignoreErr bool) (Set[Query], error) {
+// RestoreSchemaName restores the schema name of the given Query set.
+func RestoreSchemaName(defaultSchema string, sqls Set[Query], ignoreErr bool) (Set[Query], error) {
 	s := NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		if sql.SchemaName == "" {
@@ -300,25 +301,29 @@ func restoreSchemaName(defaultSchema string, sqls Set[Query], ignoreErr bool) (S
 	return s, nil
 }
 
+// FilterInvalidQueries filters out invalid queries from the given query set.
 // some queries might be forbidden by the fix-control 43817.
-func filterInvalidQueries(opt Optimizer, sqls Set[Query], returnErr bool) (Set[Query], error) {
+func FilterInvalidQueries(opt Optimizer, sqls Set[Query], ignoreErr bool) (Set[Query], error) {
 	s := NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		_, err := opt.QueryPlanCost(sql.Text)
-		if err == nil {
-			s.Add(sql)
-		} else if err != nil && returnErr {
-			return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
+		if err != nil {
+			if ignoreErr {
+				continue
+			}
+			return nil, err
 		}
+		s.Add(sql)
 	}
 	return s, nil
 }
 
-func filterSQLAccessingSystemTables(sqls Set[Query], returnErr bool) (Set[Query], error) {
+// FilterSQLAccessingSystemTables filters out queries that access system tables.
+func FilterSQLAccessingSystemTables(sqls Set[Query], returnErr bool) (Set[Query], error) {
 	s := NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		accessSystemTable := false
-		names, err := collectTableNamesFromQuery(sql.SchemaName, sql.Text)
+		names, err := CollectTableNamesFromQuery(sql.SchemaName, sql.Text)
 		if err != nil {
 			if returnErr {
 				return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
@@ -361,7 +366,7 @@ func CollectIndexableColumnsForQuerySet(opt Optimizer, querySet Set[Query]) (Set
 
 // CollectIndexableColumnsFromQuery parses the given Query text and returns the indexable columns.
 func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (Set[Column], error) {
-	tableNames, err := collectTableNamesFromQuery(q.SchemaName, q.Text)
+	tableNames, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index e8dde4b7b41fa..a71504703f502 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -12,52 +12,53 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package indexadvisor
+package indexadvisor_test
 
 import (
+	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestCollectTableFromQuery(t *testing.T) {
-	names, err := collectTableNamesFromQuery("test", "select * from t where a = 1")
+	names, err := indexadvisor.CollectTableNamesFromQuery("test", "select * from t where a = 1")
 	require.NoError(t, err)
 	require.Equal(t, names[0], "test.t")
 
-	names, err = collectTableNamesFromQuery("test", "select * from t1, t2")
+	names, err = indexadvisor.CollectTableNamesFromQuery("test", "select * from t1, t2")
 	require.NoError(t, err)
 	require.Equal(t, names[0], "test.t1")
 	require.Equal(t, names[1], "test.t2")
 
-	names, err = collectTableNamesFromQuery("test", "select * from t1 where t1.a < (select max(b) from t2)")
+	names, err = indexadvisor.CollectTableNamesFromQuery("test", "select * from t1 where t1.a < (select max(b) from t2)")
 	require.NoError(t, err)
 	require.Equal(t, names[0], "test.t1")
 	require.Equal(t, names[1], "test.t2")
 
-	names, err = collectTableNamesFromQuery("test", "select * from t1 where t1.a < (select max(b) from db2.t2)")
+	names, err = indexadvisor.CollectTableNamesFromQuery("test", "select * from t1 where t1.a < (select max(b) from db2.t2)")
 	require.NoError(t, err)
 	require.Equal(t, names[0], "test.t1")
 	require.Equal(t, names[1], "db2.t2")
 }
 
 func TestCollectSelectColumnsFromQuery(t *testing.T) {
-	names, err := collectSelectColumnsFromQuery(Query{Text: "select a, b from test.t"})
+	names, err := indexadvisor.CollectSelectColumnsFromQuery(indexadvisor.Query{Text: "select a, b from test.t"})
 	require.NoError(t, err)
 	require.True(t, names.String() == "{test.t.a, test.t.b}")
 
-	names, err = collectSelectColumnsFromQuery(Query{Text: "select a, b, c from test.t"})
+	names, err = indexadvisor.CollectSelectColumnsFromQuery(indexadvisor.Query{Text: "select a, b, c from test.t"})
 	require.NoError(t, err)
 	require.True(t, names.String() == "{test.t.a, test.t.b, test.t.c}")
 }
 
 func TestCollectOrderByColumnsFromQuery(t *testing.T) {
-	cols, err := collectOrderByColumnsFromQuery(Query{Text: "select a, b from test.t order by a"})
+	cols, err := indexadvisor.CollectOrderByColumnsFromQuery(indexadvisor.Query{Text: "select a, b from test.t order by a"})
 	require.NoError(t, err)
 	require.Equal(t, len(cols), 1)
 	require.Equal(t, cols[0].Key(), "test.t.a")
 
-	cols, err = collectOrderByColumnsFromQuery(Query{Text: "select a, b from test.t order by a, b"})
+	cols, err = indexadvisor.CollectOrderByColumnsFromQuery(indexadvisor.Query{Text: "select a, b from test.t order by a, b"})
 	require.NoError(t, err)
 	require.Equal(t, len(cols), 2)
 	require.Equal(t, cols[0].Key(), "test.t.a")
@@ -65,27 +66,27 @@ func TestCollectOrderByColumnsFromQuery(t *testing.T) {
 }
 
 func TestCollectDNFColumnsFromQuery(t *testing.T) {
-	cols, err := collectDNFColumnsFromQuery(Query{Text: "select a, b from test.t where a = 1 or b = 2"})
+	cols, err := indexadvisor.CollectDNFColumnsFromQuery(indexadvisor.Query{Text: "select a, b from test.t where a = 1 or b = 2"})
 	require.NoError(t, err)
 	require.Equal(t, cols.String(), "{test.t.a, test.t.b}")
 
-	cols, err = collectDNFColumnsFromQuery(Query{Text: "select a, b from test.t where a = 1 or b = 2 or c=3"})
+	cols, err = indexadvisor.CollectDNFColumnsFromQuery(indexadvisor.Query{Text: "select a, b from test.t where a = 1 or b = 2 or c=3"})
 	require.NoError(t, err)
 	require.Equal(t, cols.String(), "{test.t.a, test.t.b, test.t.c}")
 }
 
 func TestRestoreSchemaName(t *testing.T) {
-	q1 := Query{Text: "select * from t1"}
-	q2 := Query{Text: "select * from t2", SchemaName: "test2"}
-	q3 := Query{Text: "select * from t3"}
-	q4 := Query{Text: "wrong"}
-	set1 := NewSet[Query]()
+	q1 := indexadvisor.Query{Text: "select * from t1"}
+	q2 := indexadvisor.Query{Text: "select * from t2", SchemaName: "test2"}
+	q3 := indexadvisor.Query{Text: "select * from t3"}
+	q4 := indexadvisor.Query{Text: "wrong"}
+	set1 := indexadvisor.NewSet[indexadvisor.Query]()
 	set1.Add(q1, q2, q3, q4)
 
-	set2, err := restoreSchemaName("test", set1, true)
+	set2, err := indexadvisor.RestoreSchemaName("test", set1, true)
 	require.NoError(t, err)
 	require.Equal(t, set2.String(), "{SELECT * FROM `test2`.`t2`, SELECT * FROM `test`.`t1`, SELECT * FROM `test`.`t3`}")
 
-	_, err = restoreSchemaName("test", set1, false)
+	_, err = indexadvisor.RestoreSchemaName("test", set1, false)
 	require.Error(t, err)
 }

From 24dcec851dc391dca52e070b3dd391967b97c835 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:48:35 +0800
Subject: [PATCH 12/27] fixup

---
 pkg/planner/indexadvisor/utils.go      |  8 ++++----
 pkg/planner/indexadvisor/utils_test.go | 21 ++++++++++++++++++++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index d9b806fcc3b20..2ab4b3e3392e8 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -319,16 +319,16 @@ func FilterInvalidQueries(opt Optimizer, sqls Set[Query], ignoreErr bool) (Set[Q
 }
 
 // FilterSQLAccessingSystemTables filters out queries that access system tables.
-func FilterSQLAccessingSystemTables(sqls Set[Query], returnErr bool) (Set[Query], error) {
+func FilterSQLAccessingSystemTables(sqls Set[Query], ignoreErr bool) (Set[Query], error) {
 	s := NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		accessSystemTable := false
 		names, err := CollectTableNamesFromQuery(sql.SchemaName, sql.Text)
 		if err != nil {
-			if returnErr {
-				return nil, fmt.Errorf("invalid query: %v, err: %v", sql.Text, err)
+			if ignoreErr {
+				continue
 			}
-			continue
+			return nil, err
 		}
 		if len(names) == 0 {
 			// `select @@some_var` or `select some_func()`
diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index a71504703f502..de5b2be69bdd7 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -15,9 +15,9 @@
 package indexadvisor_test
 
 import (
-	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
 	"testing"
 
+	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
 	"github.com/stretchr/testify/require"
 )
 
@@ -90,3 +90,22 @@ func TestRestoreSchemaName(t *testing.T) {
 	_, err = indexadvisor.RestoreSchemaName("test", set1, false)
 	require.Error(t, err)
 }
+
+func TestFilterSQLAccessingSystemTables(t *testing.T) {
+	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta"})
+	set1.Add(indexadvisor.Query{Text: "select * from information_schema.test"})
+	set1.Add(indexadvisor.Query{Text: "select * from metrics_schema.test"})
+	set1.Add(indexadvisor.Query{Text: "select * from performance_schema.test"})
+	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta", SchemaName: "test"})
+	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta, test.t1", SchemaName: "test"})
+	set1.Add(indexadvisor.Query{Text: "select * from test.t1", SchemaName: "mysql"})
+	set1.Add(indexadvisor.Query{Text: "wrong", SchemaName: "information_schema"})
+
+	set2, err := indexadvisor.FilterSQLAccessingSystemTables(set1, true)
+	require.NoError(t, err)
+	require.Equal(t, set2.String(), "{select * from test.t1}")
+
+	_, err = indexadvisor.FilterSQLAccessingSystemTables(set1, false)
+	require.Error(t, err)
+}

From f439921c45f30d4d59e78b09d8be910111f86a34 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:49:30 +0800
Subject: [PATCH 13/27] fixup

---
 pkg/planner/indexadvisor/utils_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index de5b2be69bdd7..2a0d2545b7923 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -100,6 +100,8 @@ func TestFilterSQLAccessingSystemTables(t *testing.T) {
 	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta", SchemaName: "test"})
 	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta, test.t1", SchemaName: "test"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t1", SchemaName: "mysql"})
+	set1.Add(indexadvisor.Query{Text: "select @@var", SchemaName: "test"})
+	set1.Add(indexadvisor.Query{Text: "select sleep(1)", SchemaName: "test"})
 	set1.Add(indexadvisor.Query{Text: "wrong", SchemaName: "information_schema"})
 
 	set2, err := indexadvisor.FilterSQLAccessingSystemTables(set1, true)

From e74292e9a90a7e7e6343517ef7df4292e6ab774a Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:53:54 +0800
Subject: [PATCH 14/27] fixup

---
 pkg/planner/indexadvisor/utils_test.go | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index 2a0d2545b7923..c95f6a4913036 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -15,6 +15,7 @@
 package indexadvisor_test
 
 import (
+	"github.com/pingcap/tidb/pkg/testkit"
 	"testing"
 
 	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
@@ -111,3 +112,26 @@ func TestFilterSQLAccessingSystemTables(t *testing.T) {
 	_, err = indexadvisor.FilterSQLAccessingSystemTables(set1, false)
 	require.Error(t, err)
 }
+
+func TestFilterInvalidQueries(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t1 (a int, b int, c int)`)
+	tk.MustExec(`create table t2 (a int, b int, c int)`)
+	opt := indexadvisor.NewOptimizer(tk.Session())
+
+	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1.Add(indexadvisor.Query{Text: "select * from test.t1"})
+	set1.Add(indexadvisor.Query{Text: "select * from test.t3"})                            // table t3 does not exist
+	set1.Add(indexadvisor.Query{Text: "select d from t1"})                                 // column d does not exist
+	set1.Add(indexadvisor.Query{Text: "select * from t1 where a<(select max(b) from t2)"}) // Fix43817
+	set1.Add(indexadvisor.Query{Text: "wrong"})                                            // invalid query
+
+	set2, err := indexadvisor.FilterInvalidQueries(opt, set1, true)
+	require.NoError(t, err)
+	require.Equal(t, set2.String(), "{select * from test.t1}")
+
+	_, err = indexadvisor.FilterInvalidQueries(opt, set1, false)
+	require.Error(t, err)
+}

From 5f2fc1ff1a9015e7b907f1bb25d0d2e39323451a Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 11:58:42 +0800
Subject: [PATCH 15/27] fixup

---
 pkg/planner/indexadvisor/utils_test.go | 55 ++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index c95f6a4913036..217a008cd47b6 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -135,3 +135,58 @@ func TestFilterInvalidQueries(t *testing.T) {
 	_, err = indexadvisor.FilterInvalidQueries(opt, set1, false)
 	require.Error(t, err)
 }
+
+func TestCollectIndexableColumnsFromQuery(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t (a int, b int, c int, d int, e int)`)
+	opt := indexadvisor.NewOptimizer(tk.Session())
+
+	cols, err := indexadvisor.CollectIndexableColumnsFromQuery(
+		indexadvisor.Query{SchemaName: "test", Text: "select * from t where a<1 and b>1 and e like 'abc'"}, opt)
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{test.t.a, test.t.b}")
+
+	cols, err = indexadvisor.CollectIndexableColumnsFromQuery(
+		indexadvisor.Query{SchemaName: "test", Text: "select * from t where c in (1, 2, 3) order by d"}, opt)
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{test.t.c, test.t.d}")
+
+	cols, err = indexadvisor.CollectIndexableColumnsFromQuery(
+		indexadvisor.Query{SchemaName: "test", Text: "select 1 from t where c in (1, 2, 3) group by d"}, opt)
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{test.t.c, test.t.d}")
+
+	tk.MustExec("drop table t")
+
+	tk.MustExec(`create table t1 (a int)`)
+	tk.MustExec(`create table t2 (a int)`)
+	cols, err = indexadvisor.CollectIndexableColumnsFromQuery(
+		indexadvisor.Query{SchemaName: "test", Text: "select * from t2 tx where a<1"}, opt)
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{test.t1.a, test.t2.a}")
+	tk.MustExec("drop table t1")
+	tk.MustExec("drop table t2")
+
+	tk.MustExec(`create database tpch`)
+	tk.MustExec(`use tpch`)
+	tk.MustExec(`CREATE TABLE tpch.nation ( N_NATIONKEY bigint(20) NOT NULL,
+		N_NAME char(25) NOT NULL, N_REGIONKEY bigint(20) NOT NULL, N_COMMENT varchar(152) DEFAULT NULL,
+		PRIMARY KEY (N_NATIONKEY) /*T![clustered_index] CLUSTERED */)`)
+	q := ` select supp_nation, cust_nation, l_year, sum(volume) as revenue from
+	( select n1.n_name as supp_nation, n2.n_name as cust_nation,
+			extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume
+		from supplier, lineitem, orders, customer, nation n1, nation n2
+		where s_suppkey = l_suppkey and o_orderkey = l_orderkey and c_custkey = o_custkey
+		  and s_nationkey = n1.n_nationkey and c_nationkey = n2.n_nationkey
+			and ( (n1.n_name = 'MOZAMBIQUE' and n2.n_name = 'UNITED KINGDOM')
+				or (n1.n_name = 'UNITED KINGDOM' and n2.n_name = 'MOZAMBIQUE')
+			) and l_shipdate between date '1995-01-01' and date '1996-12-31'
+	) as shipping group by supp_nation, cust_nation, l_year
+	order by supp_nation, cust_nation, l_year`
+	cols, err = indexadvisor.CollectIndexableColumnsFromQuery(
+		indexadvisor.Query{SchemaName: "tpch", Text: q}, opt)
+	require.NoError(t, err)
+	require.Equal(t, cols.String(), "{tpch.nation.n_name, tpch.nation.n_nationkey}")
+}

From a459176d9e7901f5c692e610692d12c9c7fa5c7e Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 12:01:10 +0800
Subject: [PATCH 16/27] fixup

---
 pkg/planner/indexadvisor/utils_test.go | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index 217a008cd47b6..0a6b81d7dc2c9 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -136,6 +136,24 @@ func TestFilterInvalidQueries(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestCollectIndexableColumnsForQuerySet(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`use test`)
+	tk.MustExec(`create table t (a int, b int, c int, d int, e int)`)
+	opt := indexadvisor.NewOptimizer(tk.Session())
+
+	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1.Add(indexadvisor.Query{Text: "select * from t where a=1 and b=1 and e like 'abc'"})
+	set1.Add(indexadvisor.Query{Text: "select * from t where a<1 and b>1 and e like 'abc'"})
+	set1.Add(indexadvisor.Query{Text: "select * from t where c in (1, 2, 3) order by d"})
+	set1.Add(indexadvisor.Query{Text: "select 1 from t where c in (1, 2, 3) group by e"})
+
+	set2, err := indexadvisor.CollectIndexableColumnsForQuerySet(opt, set1)
+	require.NoError(t, err)
+	require.Equal(t, "{test.t.a, test.t.b, test.t.c, test.t.d}", set2.String())
+}
+
 func TestCollectIndexableColumnsFromQuery(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)

From 1f6bb19d854deb68ffdb7a93d21eb82c95089464 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 12:02:33 +0800
Subject: [PATCH 17/27] fixup

---
 pkg/planner/indexadvisor/utils_test.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index 0a6b81d7dc2c9..b5922f1b2d23b 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -140,18 +140,18 @@ func TestCollectIndexableColumnsForQuerySet(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec(`use test`)
-	tk.MustExec(`create table t (a int, b int, c int, d int, e int)`)
+	tk.MustExec(`create table t (a int, b int, c int, d int, e int, f int)`)
 	opt := indexadvisor.NewOptimizer(tk.Session())
 
 	set1 := indexadvisor.NewSet[indexadvisor.Query]()
-	set1.Add(indexadvisor.Query{Text: "select * from t where a=1 and b=1 and e like 'abc'"})
-	set1.Add(indexadvisor.Query{Text: "select * from t where a<1 and b>1 and e like 'abc'"})
-	set1.Add(indexadvisor.Query{Text: "select * from t where c in (1, 2, 3) order by d"})
-	set1.Add(indexadvisor.Query{Text: "select 1 from t where c in (1, 2, 3) group by e"})
+	set1.Add(indexadvisor.Query{Text: "select * from test.t where a=1 and b=1 and e like 'abc'"})
+	set1.Add(indexadvisor.Query{Text: "select * from test.t where a<1 and b>1 and e like 'abc'"})
+	set1.Add(indexadvisor.Query{Text: "select * from test.t where c in (1, 2, 3) order by d"})
+	set1.Add(indexadvisor.Query{Text: "select 1 from test.t where c in (1, 2, 3) group by e"})
 
 	set2, err := indexadvisor.CollectIndexableColumnsForQuerySet(opt, set1)
 	require.NoError(t, err)
-	require.Equal(t, "{test.t.a, test.t.b, test.t.c, test.t.d}", set2.String())
+	require.Equal(t, "{test.t.a, test.t.b, test.t.c, test.t.d, test.t.e}", set2.String())
 }
 
 func TestCollectIndexableColumnsFromQuery(t *testing.T) {

From df2ded279a2849ce0f4f51ef57037d8b5e75c580 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 12:03:33 +0800
Subject: [PATCH 18/27] fixup

---
 pkg/planner/indexadvisor/optimizer.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pkg/planner/indexadvisor/optimizer.go b/pkg/planner/indexadvisor/optimizer.go
index c36de1d0b187f..98dd607154f47 100644
--- a/pkg/planner/indexadvisor/optimizer.go
+++ b/pkg/planner/indexadvisor/optimizer.go
@@ -21,7 +21,6 @@ import (
 
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/infoschema"
-	"github.com/pingcap/tidb/pkg/parser"
 	"github.com/pingcap/tidb/pkg/parser/ast"
 	"github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
@@ -218,8 +217,7 @@ func (opt *optimizerImpl) addHypoIndex(hypoIndexes ...Index) error {
 
 // QueryPlanCost return the cost of the query plan.
 func (opt *optimizerImpl) QueryPlanCost(sql string, hypoIndexes ...Index) (cost float64, err error) {
-	p := parser.New()
-	stmt, err := p.ParseOneStmt(sql, "", "")
+	stmt, err := ParseOneSQL(sql)
 	if err != nil {
 		return 0, err
 	}

From 3e0031db13010b18671532cdd7947ee95f2b8351 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 12:05:32 +0800
Subject: [PATCH 19/27] fixup

---
 pkg/planner/indexadvisor/BUILD.bazel | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pkg/planner/indexadvisor/BUILD.bazel b/pkg/planner/indexadvisor/BUILD.bazel
index 23fd4fd5643ac..36fed4c54da10 100644
--- a/pkg/planner/indexadvisor/BUILD.bazel
+++ b/pkg/planner/indexadvisor/BUILD.bazel
@@ -6,6 +6,7 @@ go_library(
         "model.go",
         "optimizer.go",
         "set.go",
+        "utils.go",
     ],
     importpath = "github.com/pingcap/tidb/pkg/planner/indexadvisor",
     visibility = ["//visibility:public"],
@@ -15,9 +16,15 @@ go_library(
         "//pkg/parser",
         "//pkg/parser/ast",
         "//pkg/parser/model",
+        "//pkg/parser/mysql",
+        "//pkg/parser/opcode",
         "//pkg/planner/util/fixcontrol",
         "//pkg/sessionctx",
         "//pkg/types",
+        "//pkg/types/parser_driver",
+        "//pkg/util/logutil",
+        "//pkg/util/parser",
+        "@org_uber_go_zap//:zap",
     ],
 )
 
@@ -27,10 +34,11 @@ go_test(
     srcs = [
         "optimizer_test.go",
         "set_test.go",
+        "utils_test.go",
     ],
     embed = [":indexadvisor"],
     flaky = True,
-    shard_count = 11,
+    shard_count = 20,
     deps = [
         "//pkg/parser/mysql",
         "//pkg/testkit",

From f85a3aa72eed8b92f15080220f9da1ea89a604bf Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 14:28:10 +0800
Subject: [PATCH 20/27] fixup

---
 pkg/planner/indexadvisor/utils.go | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index 2ab4b3e3392e8..fb73b964f8885 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -36,7 +36,7 @@ func ParseOneSQL(sqlText string) (ast.StmtNode, error) {
 }
 
 // NormalizeDigest normalizes the given Query text and returns the normalized Query text and its digest.
-func NormalizeDigest(sqlText string) (string, string) {
+func NormalizeDigest(sqlText string) (normalizedSQL, digest string) {
 	norm, d := parser.NormalizeDigest(sqlText)
 	return norm, d.String()
 }
@@ -53,9 +53,9 @@ func (v *nodeVisitor) Enter(n ast.Node) (out ast.Node, skipChildren bool) {
 	return n, false
 }
 
-func (c *nodeVisitor) Leave(n ast.Node) (out ast.Node, ok bool) {
-	if c.leave != nil {
-		return n, c.leave(n)
+func (v *nodeVisitor) Leave(n ast.Node) (out ast.Node, ok bool) {
+	if v.leave != nil {
+		return n, v.leave(n)
 	}
 	return n, true
 }
@@ -126,8 +126,7 @@ func CollectSelectColumnsFromQuery(q Query) (Set[Column], error) {
 		}
 		return false
 	}, func(n ast.Node) bool {
-		switch n.(type) {
-		case *ast.SelectField:
+		if _, ok := n.(*ast.SelectField); ok {
 			underSelectField = false
 		}
 		return true
@@ -156,8 +155,7 @@ func CollectOrderByColumnsFromQuery(q Query) ([]Column, error) {
 		if exit {
 			return true
 		}
-		switch x := n.(type) {
-		case *ast.OrderByClause:
+		if x, ok := n.(*ast.OrderByClause); ok {
 			for _, byItem := range x.Items {
 				colExpr, ok := byItem.Expr.(*ast.ColumnNameExpr)
 				if !ok {
@@ -197,8 +195,7 @@ func CollectDNFColumnsFromQuery(q Query) (Set[Column], error) {
 		if dnfColSet.Size() > 0 { // already collected
 			return true
 		}
-		switch x := n.(type) {
-		case *ast.SelectStmt:
+		if x, ok := n.(*ast.SelectStmt); ok {
 			cnf := flattenCNF(x.Where)
 			for _, expr := range cnf {
 				dnf := flattenDNF(expr)

From fec0f5a12ccf25d4e41f796f0405e79718f41a87 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 14:39:51 +0800
Subject: [PATCH 21/27] fixup

---
 pkg/planner/indexadvisor/optimizer.go | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/pkg/planner/indexadvisor/optimizer.go b/pkg/planner/indexadvisor/optimizer.go
index 98dd607154f47..c24036e7da645 100644
--- a/pkg/planner/indexadvisor/optimizer.go
+++ b/pkg/planner/indexadvisor/optimizer.go
@@ -21,8 +21,9 @@ import (
 
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/infoschema"
+	"github.com/pingcap/tidb/pkg/meta/model"
 	"github.com/pingcap/tidb/pkg/parser/ast"
-	"github.com/pingcap/tidb/pkg/parser/model"
+	model2 "github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/types"
@@ -74,7 +75,7 @@ func (opt *optimizerImpl) is() infoschema.InfoSchema {
 
 // IndexNameExist returns whether the specified index name exists in the specified table.
 func (opt *optimizerImpl) IndexNameExist(schema, table, indexName string) (bool, error) {
-	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(schema), model.NewCIStr(table))
+	tbl, err := opt.is().TableByName(context.Background(), model2.NewCIStr(schema), model2.NewCIStr(table))
 	if err != nil {
 		return false, err
 	}
@@ -88,7 +89,7 @@ func (opt *optimizerImpl) IndexNameExist(schema, table, indexName string) (bool,
 
 // TableColumns returns the columns of the specified table.
 func (opt *optimizerImpl) TableColumns(schema, table string) ([]Column, error) {
-	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(schema), model.NewCIStr(table))
+	tbl, err := opt.is().TableByName(context.Background(), model2.NewCIStr(schema), model2.NewCIStr(table))
 	if err != nil {
 		return nil, err
 	}
@@ -113,7 +114,7 @@ func (opt *optimizerImpl) PossibleColumns(schema, colName string) ([]Column, err
 	}
 
 	cols := make([]Column, 0)
-	tbls, err := opt.is().SchemaTableInfos(context.Background(), model.NewCIStr(schema))
+	tbls, err := opt.is().SchemaTableInfos(context.Background(), model2.NewCIStr(schema))
 	if err != nil {
 		return nil, err
 	}
@@ -133,7 +134,7 @@ func (opt *optimizerImpl) PossibleColumns(schema, colName string) ([]Column, err
 
 // PrefixContainIndex returns whether the specified index is a prefix of an existing index.
 func (opt *optimizerImpl) PrefixContainIndex(idx Index) (bool, error) {
-	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(idx.SchemaName), model.NewCIStr(idx.TableName))
+	tbl, err := opt.is().TableByName(context.Background(), model2.NewCIStr(idx.SchemaName), model2.NewCIStr(idx.TableName))
 	if err != nil {
 		return false, err
 	}
@@ -157,7 +158,7 @@ func (opt *optimizerImpl) PrefixContainIndex(idx Index) (bool, error) {
 
 // ColumnType returns the column type of the specified column.
 func (opt *optimizerImpl) ColumnType(c Column) (*types.FieldType, error) {
-	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(c.SchemaName), model.NewCIStr(c.TableName))
+	tbl, err := opt.is().TableByName(context.Background(), model2.NewCIStr(c.SchemaName), model2.NewCIStr(c.TableName))
 	if err != nil {
 		return nil, err
 	}
@@ -171,7 +172,7 @@ func (opt *optimizerImpl) ColumnType(c Column) (*types.FieldType, error) {
 
 func (opt *optimizerImpl) addHypoIndex(hypoIndexes ...Index) error {
 	for _, h := range hypoIndexes {
-		tInfo, err := opt.is().TableByName(context.Background(), model.NewCIStr(h.SchemaName), model.NewCIStr(h.TableName))
+		tInfo, err := opt.is().TableByName(context.Background(), model2.NewCIStr(h.SchemaName), model2.NewCIStr(h.TableName))
 		if err != nil {
 			return err
 		}
@@ -189,16 +190,16 @@ func (opt *optimizerImpl) addHypoIndex(hypoIndexes ...Index) error {
 				return fmt.Errorf("column %v not found in table %v.%v", col.ColumnName, h.SchemaName, h.TableName)
 			}
 			cols = append(cols, &model.IndexColumn{
-				Name:   model.NewCIStr(col.ColumnName),
+				Name:   model2.NewCIStr(col.ColumnName),
 				Offset: colOffset,
 				Length: types.UnspecifiedLength,
 			})
 		}
 		idxInfo := &model.IndexInfo{
-			Name:    model.NewCIStr(h.IndexName),
+			Name:    model2.NewCIStr(h.IndexName),
 			Columns: cols,
 			State:   model.StatePublic,
-			Tp:      model.IndexTypeHypo,
+			Tp:      model2.IndexTypeHypo,
 		}
 
 		if opt.sctx.GetSessionVars().HypoIndexes == nil {
@@ -245,7 +246,7 @@ func (opt *optimizerImpl) QueryPlanCost(sql string, hypoIndexes ...Index) (cost
 
 // EstIndexSize return the estimated index size of the specified table and columns
 func (opt *optimizerImpl) EstIndexSize(db, table string, cols ...string) (indexSize float64, err error) {
-	tbl, err := opt.is().TableByName(context.Background(), model.NewCIStr(db), model.NewCIStr(table))
+	tbl, err := opt.is().TableByName(context.Background(), model2.NewCIStr(db), model2.NewCIStr(table))
 	if err != nil {
 		return 0, err
 	}

From d334b0aee0336b61d1c1f4bb7325d552cc0ae390 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 14:46:24 +0800
Subject: [PATCH 22/27] fixup

---
 pkg/planner/indexadvisor/utils.go             |  1 +
 pkg/planner/indexadvisor/utils_test.go        | 11 ++--
 pkg/{planner/indexadvisor => util/set}/set.go |  2 +-
 .../indexadvisor => util/set}/set_test.go     | 58 +++++++++++--------
 4 files changed, 41 insertions(+), 31 deletions(-)
 rename pkg/{planner/indexadvisor => util/set}/set.go (99%)
 rename pkg/{planner/indexadvisor => util/set}/set_test.go (55%)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index fb73b964f8885..4c1dc3e19aa71 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -26,6 +26,7 @@ import (
 	driver "github.com/pingcap/tidb/pkg/types/parser_driver"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	parser2 "github.com/pingcap/tidb/pkg/util/parser"
+	. "github.com/pingcap/tidb/pkg/util/set"
 	"go.uber.org/zap"
 )
 
diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index b5922f1b2d23b..b04e7a15b2015 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -15,10 +15,11 @@
 package indexadvisor_test
 
 import (
-	"github.com/pingcap/tidb/pkg/testkit"
 	"testing"
 
 	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
+	"github.com/pingcap/tidb/pkg/testkit"
+	. "github.com/pingcap/tidb/pkg/util/set"
 	"github.com/stretchr/testify/require"
 )
 
@@ -81,7 +82,7 @@ func TestRestoreSchemaName(t *testing.T) {
 	q2 := indexadvisor.Query{Text: "select * from t2", SchemaName: "test2"}
 	q3 := indexadvisor.Query{Text: "select * from t3"}
 	q4 := indexadvisor.Query{Text: "wrong"}
-	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1 := NewSet[indexadvisor.Query]()
 	set1.Add(q1, q2, q3, q4)
 
 	set2, err := indexadvisor.RestoreSchemaName("test", set1, true)
@@ -93,7 +94,7 @@ func TestRestoreSchemaName(t *testing.T) {
 }
 
 func TestFilterSQLAccessingSystemTables(t *testing.T) {
-	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1 := NewSet[indexadvisor.Query]()
 	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta"})
 	set1.Add(indexadvisor.Query{Text: "select * from information_schema.test"})
 	set1.Add(indexadvisor.Query{Text: "select * from metrics_schema.test"})
@@ -121,7 +122,7 @@ func TestFilterInvalidQueries(t *testing.T) {
 	tk.MustExec(`create table t2 (a int, b int, c int)`)
 	opt := indexadvisor.NewOptimizer(tk.Session())
 
-	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1 := NewSet[indexadvisor.Query]()
 	set1.Add(indexadvisor.Query{Text: "select * from test.t1"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t3"})                            // table t3 does not exist
 	set1.Add(indexadvisor.Query{Text: "select d from t1"})                                 // column d does not exist
@@ -143,7 +144,7 @@ func TestCollectIndexableColumnsForQuerySet(t *testing.T) {
 	tk.MustExec(`create table t (a int, b int, c int, d int, e int, f int)`)
 	opt := indexadvisor.NewOptimizer(tk.Session())
 
-	set1 := indexadvisor.NewSet[indexadvisor.Query]()
+	set1 := NewSet[indexadvisor.Query]()
 	set1.Add(indexadvisor.Query{Text: "select * from test.t where a=1 and b=1 and e like 'abc'"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t where a<1 and b>1 and e like 'abc'"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t where c in (1, 2, 3) order by d"})
diff --git a/pkg/planner/indexadvisor/set.go b/pkg/util/set/set.go
similarity index 99%
rename from pkg/planner/indexadvisor/set.go
rename to pkg/util/set/set.go
index 641639e03a558..15e607427ac3b 100644
--- a/pkg/planner/indexadvisor/set.go
+++ b/pkg/util/set/set.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package indexadvisor
+package set
 
 import (
 	"fmt"
diff --git a/pkg/planner/indexadvisor/set_test.go b/pkg/util/set/set_test.go
similarity index 55%
rename from pkg/planner/indexadvisor/set_test.go
rename to pkg/util/set/set_test.go
index 54f4da41ced13..270a21cad6b66 100644
--- a/pkg/planner/indexadvisor/set_test.go
+++ b/pkg/util/set/set_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package indexadvisor
+package set
 
 import (
 	"strings"
@@ -21,49 +21,57 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type item struct {
+	Text string
+}
+
+func (i item) Key() string {
+	return i.Text
+}
+
 func TestSetBasic(t *testing.T) {
-	s := NewSet[Query]()
-	s.Add(Query{Text: "q1"}, Query{Text: "q2"}, Query{Text: "q3"})
-	require.True(t, s.Contains(Query{Text: "q1"}))
-	require.True(t, s.Contains(Query{Text: "q2"}))
-	require.True(t, s.Contains(Query{Text: "q3"}))
-	require.False(t, s.Contains(Query{Text: "q4"}))
+	s := NewSet[item]()
+	s.Add(item{Text: "q1"}, item{Text: "q2"}, item{Text: "q3"})
+	require.True(t, s.Contains(item{Text: "q1"}))
+	require.True(t, s.Contains(item{Text: "q2"}))
+	require.True(t, s.Contains(item{Text: "q3"}))
+	require.False(t, s.Contains(item{Text: "q4"}))
 	require.Equal(t, 3, s.Size())
-	require.Equal(t, []Query{{Text: "q1"}, {Text: "q2"}, {Text: "q3"}}, s.ToList())
-	s.Remove(Query{Text: "q2"})
-	require.False(t, s.Contains(Query{Text: "q2"}))
+	require.Equal(t, []item{{Text: "q1"}, {Text: "q2"}, {Text: "q3"}}, s.ToList())
+	s.Remove(item{Text: "q2"})
+	require.False(t, s.Contains(item{Text: "q2"}))
 	require.Equal(t, 2, s.Size())
 
 	clonedS := s.Clone()
-	require.True(t, clonedS.Contains(Query{Text: "q1"}))
-	s.Remove(Query{Text: "q1"})
-	require.False(t, s.Contains(Query{Text: "q1"}))
-	require.True(t, clonedS.Contains(Query{Text: "q1"}))
+	require.True(t, clonedS.Contains(item{Text: "q1"}))
+	s.Remove(item{Text: "q1"})
+	require.False(t, s.Contains(item{Text: "q1"}))
+	require.True(t, clonedS.Contains(item{Text: "q1"}))
 	require.Equal(t, 2, clonedS.Size())
 }
 
 func TestSetOperation(t *testing.T) {
-	s1 := NewSet[Query]()
-	s1.Add(Query{Text: "q1"}, Query{Text: "q2"}, Query{Text: "q3"})
-	s2 := NewSet[Query]()
-	s2.Add(Query{Text: "q2"}, Query{Text: "q3"}, Query{Text: "q4"})
+	s1 := NewSet[item]()
+	s1.Add(item{Text: "q1"}, item{Text: "q2"}, item{Text: "q3"})
+	s2 := NewSet[item]()
+	s2.Add(item{Text: "q2"}, item{Text: "q3"}, item{Text: "q4"})
 	unionSet := UnionSet(s1, s2)
-	require.Equal(t, []Query{{Text: "q1"}, {Text: "q2"}, {Text: "q3"}, {Text: "q4"}}, unionSet.ToList())
+	require.Equal(t, []item{{Text: "q1"}, {Text: "q2"}, {Text: "q3"}, {Text: "q4"}}, unionSet.ToList())
 
 	andSet := AndSet(s1, s2)
-	require.Equal(t, []Query{{Text: "q2"}, {Text: "q3"}}, andSet.ToList())
+	require.Equal(t, []item{{Text: "q2"}, {Text: "q3"}}, andSet.ToList())
 
 	diffSet := DiffSet(s1, s2)
-	require.Equal(t, []Query{{Text: "q1"}}, diffSet.ToList())
+	require.Equal(t, []item{{Text: "q1"}}, diffSet.ToList())
 	diffSet = DiffSet(s2, s1)
-	require.Equal(t, []Query{{Text: "q4"}}, diffSet.ToList())
+	require.Equal(t, []item{{Text: "q4"}}, diffSet.ToList())
 }
 
 func TestSetCombination(t *testing.T) {
-	s := NewSet[Query]()
-	s.Add(Query{Text: "q1"}, Query{Text: "q2"}, Query{Text: "q3"}, Query{Text: "q4"})
+	s := NewSet[item]()
+	s.Add(item{Text: "q1"}, item{Text: "q2"}, item{Text: "q3"}, item{Text: "q4"})
 
-	setListStr := func(setList []Set[Query]) string {
+	setListStr := func(setList []Set[item]) string {
 		var tmp []string
 		for _, set := range setList {
 			tmp = append(tmp, set.String())

From 033ef9b89c304ca3ae9ce6242a31f74f4e13fbce Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 14:48:33 +0800
Subject: [PATCH 23/27] fixup

---
 pkg/planner/indexadvisor/BUILD.bazel | 9 +++++----
 pkg/util/set/BUILD.bazel             | 3 +++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/pkg/planner/indexadvisor/BUILD.bazel b/pkg/planner/indexadvisor/BUILD.bazel
index 36fed4c54da10..fa19fb7585b0a 100644
--- a/pkg/planner/indexadvisor/BUILD.bazel
+++ b/pkg/planner/indexadvisor/BUILD.bazel
@@ -5,7 +5,6 @@ go_library(
     srcs = [
         "model.go",
         "optimizer.go",
-        "set.go",
         "utils.go",
     ],
     importpath = "github.com/pingcap/tidb/pkg/planner/indexadvisor",
@@ -13,6 +12,7 @@ go_library(
     deps = [
         "//pkg/domain",
         "//pkg/infoschema",
+        "//pkg/meta/model",
         "//pkg/parser",
         "//pkg/parser/ast",
         "//pkg/parser/model",
@@ -24,6 +24,7 @@ go_library(
         "//pkg/types/parser_driver",
         "//pkg/util/logutil",
         "//pkg/util/parser",
+        "//pkg/util/set",
         "@org_uber_go_zap//:zap",
     ],
 )
@@ -33,15 +34,15 @@ go_test(
     timeout = "short",
     srcs = [
         "optimizer_test.go",
-        "set_test.go",
         "utils_test.go",
     ],
-    embed = [":indexadvisor"],
     flaky = True,
-    shard_count = 20,
+    shard_count = 17,
     deps = [
+        ":indexadvisor",
         "//pkg/parser/mysql",
         "//pkg/testkit",
+        "//pkg/util/set",
         "@com_github_stretchr_testify//require",
     ],
 )
diff --git a/pkg/util/set/BUILD.bazel b/pkg/util/set/BUILD.bazel
index 443f3c384023a..5e3b1d0343d1a 100644
--- a/pkg/util/set/BUILD.bazel
+++ b/pkg/util/set/BUILD.bazel
@@ -6,6 +6,7 @@ go_library(
         "float64_set.go",
         "int_set.go",
         "mem_aware_map.go",
+        "set.go",
         "set_with_memory_usage.go",
         "string_set.go",
     ],
@@ -26,6 +27,7 @@ go_test(
         "int_set_test.go",
         "main_test.go",
         "mem_aware_map_test.go",
+        "set_test.go",
         "set_with_memory_usage_test.go",
         "string_set_test.go",
     ],
@@ -34,6 +36,7 @@ go_test(
     deps = [
         "//pkg/testkit/testsetup",
         "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",
     ],
 )

From af553440d73a606dd48a65688d0dd7436bfb8985 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 14:50:06 +0800
Subject: [PATCH 24/27] fixup

---
 pkg/planner/indexadvisor/utils.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index 4c1dc3e19aa71..1103f2fc31c30 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -399,7 +399,7 @@ func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (Set[Column], erro
 			for _, schemaName := range schemaNames {
 				cols, err := opt.PossibleColumns(schemaName, x.Name.L)
 				if err != nil {
-					// TODO: log or return this error?
+					l().Warn("failed to get possible columns", zap.String("schema", schemaName), zap.String("column", x.Name.L))
 					continue
 				}
 				possibleColumns = append(possibleColumns, cols...)
@@ -408,7 +408,7 @@ func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (Set[Column], erro
 			for _, c := range possibleColumns {
 				colType, err := opt.ColumnType(c)
 				if err != nil {
-					// TODO: log?
+					l().Warn("failed to get column type", zap.String("schema", c.SchemaName), zap.String("table", c.TableName), zap.String("column", c.ColumnName))
 					continue
 				}
 				if !isIndexableColumnType(colType) {

From 5fccc0245d367966975589b362f0d3e7ffcc85b8 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 14:57:47 +0800
Subject: [PATCH 25/27] fixup

---
 pkg/util/set/set.go | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/pkg/util/set/set.go b/pkg/util/set/set.go
index 15e607427ac3b..65777b72481a0 100644
--- a/pkg/util/set/set.go
+++ b/pkg/util/set/set.go
@@ -20,13 +20,13 @@ import (
 	"strings"
 )
 
-// SetKey is the interface for the key of a set item.
-type SetKey interface {
+// Key is the interface for the key of a set item.
+type Key interface {
 	Key() string
 }
 
 // Set is the interface for a set.
-type Set[T SetKey] interface {
+type Set[T Key] interface {
 	Add(items ...T)
 	Contains(item T) bool
 	Remove(item T)
@@ -36,12 +36,12 @@ type Set[T SetKey] interface {
 	String() string
 }
 
-type setImpl[T SetKey] struct {
+type setImpl[T Key] struct {
 	s map[string]T
 }
 
 // NewSet creates a new set.
-func NewSet[T SetKey]() Set[T] {
+func NewSet[T Key]() Set[T] {
 	return new(setImpl[T])
 }
 
@@ -103,7 +103,7 @@ func (s *setImpl[T]) String() string {
 }
 
 // ListToSet converts a list to a set.
-func ListToSet[T SetKey](items ...T) Set[T] {
+func ListToSet[T Key](items ...T) Set[T] {
 	s := NewSet[T]()
 	for _, item := range items {
 		s.Add(item)
@@ -112,7 +112,7 @@ func ListToSet[T SetKey](items ...T) Set[T] {
 }
 
 // UnionSet returns the union set of the given sets.
-func UnionSet[T SetKey](ss ...Set[T]) Set[T] {
+func UnionSet[T Key](ss ...Set[T]) Set[T] {
 	if len(ss) == 0 {
 		return NewSet[T]()
 	}
@@ -127,7 +127,7 @@ func UnionSet[T SetKey](ss ...Set[T]) Set[T] {
 }
 
 // AndSet returns the intersection set of the given sets.
-func AndSet[T SetKey](ss ...Set[T]) Set[T] {
+func AndSet[T Key](ss ...Set[T]) Set[T] {
 	if len(ss) == 0 {
 		return NewSet[T]()
 	}
@@ -152,7 +152,7 @@ func AndSet[T SetKey](ss ...Set[T]) Set[T] {
 
 // DiffSet returns a set of items that are in s1 but not in s2.
 // DiffSet({1, 2, 3, 4}, {2, 3}) = {1, 4}
-func DiffSet[T SetKey](s1, s2 Set[T]) Set[T] {
+func DiffSet[T Key](s1, s2 Set[T]) Set[T] {
 	s := NewSet[T]()
 	for _, item := range s1.ToList() {
 		if !s2.Contains(item) {
@@ -164,11 +164,11 @@ func DiffSet[T SetKey](s1, s2 Set[T]) Set[T] {
 
 // CombSet returns all combinations of `numberOfItems` items in the given set.
 // For example ({a, b, c}, 2) returns {ab, ac, bc}.
-func CombSet[T SetKey](s Set[T], numberOfItems int) []Set[T] {
+func CombSet[T Key](s Set[T], numberOfItems int) []Set[T] {
 	return combSetIterate(s.ToList(), NewSet[T](), 0, numberOfItems)
 }
 
-func combSetIterate[T SetKey](itemList []T, currSet Set[T], depth, numberOfItems int) []Set[T] {
+func combSetIterate[T Key](itemList []T, currSet Set[T], depth, numberOfItems int) []Set[T] {
 	if currSet.Size() == numberOfItems {
 		return []Set[T]{currSet.Clone()}
 	}

From d37f0a5e8ee25c35ed8391fb0e33f6311a6c7955 Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 15:07:06 +0800
Subject: [PATCH 26/27] fixup

---
 pkg/planner/indexadvisor/utils.go      | 30 +++++++++++++-------------
 pkg/planner/indexadvisor/utils_test.go | 10 ++++-----
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index 1103f2fc31c30..fd89bfbe22210 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -26,7 +26,7 @@ import (
 	driver "github.com/pingcap/tidb/pkg/types/parser_driver"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	parser2 "github.com/pingcap/tidb/pkg/util/parser"
-	. "github.com/pingcap/tidb/pkg/util/set"
+	s "github.com/pingcap/tidb/pkg/util/set"
 	"go.uber.org/zap"
 )
 
@@ -98,7 +98,7 @@ func CollectTableNamesFromQuery(defaultSchema, query string) ([]string, error) {
 
 // CollectSelectColumnsFromQuery parses the given Query text and returns the selected columns.
 // For example, "select a, b, c from t" returns []string{"a", "b", "c"}.
-func CollectSelectColumnsFromQuery(q Query) (Set[Column], error) {
+func CollectSelectColumnsFromQuery(q Query) (s.Set[Column], error) {
 	names, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
@@ -112,7 +112,7 @@ func CollectSelectColumnsFromQuery(q Query) (Set[Column], error) {
 		return nil, err
 	}
 	underSelectField := false
-	selectCols := NewSet[Column]()
+	selectCols := s.NewSet[Column]()
 	visitNode(node, func(n ast.Node) bool {
 		switch x := n.(type) {
 		case *ast.SelectField:
@@ -177,7 +177,7 @@ func CollectOrderByColumnsFromQuery(q Query) ([]Column, error) {
 
 // CollectDNFColumnsFromQuery parses the given Query text and returns the DNF columns.
 // For a query `select ... where c1=1 or c2=2 or c3=3`, the DNF columns are `c1`, `c2` and `c3`.
-func CollectDNFColumnsFromQuery(q Query) (Set[Column], error) {
+func CollectDNFColumnsFromQuery(q Query) (s.Set[Column], error) {
 	names, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
@@ -190,7 +190,7 @@ func CollectDNFColumnsFromQuery(q Query) (Set[Column], error) {
 	if err != nil {
 		return nil, err
 	}
-	dnfColSet := NewSet[Column]()
+	dnfColSet := s.NewSet[Column]()
 
 	visitNode(node, func(n ast.Node) bool {
 		if dnfColSet.Size() > 0 { // already collected
@@ -280,8 +280,8 @@ func flattenDNF(expr ast.ExprNode) []ast.ExprNode {
 }
 
 // RestoreSchemaName restores the schema name of the given Query set.
-func RestoreSchemaName(defaultSchema string, sqls Set[Query], ignoreErr bool) (Set[Query], error) {
-	s := NewSet[Query]()
+func RestoreSchemaName(defaultSchema string, sqls s.Set[Query], ignoreErr bool) (s.Set[Query], error) {
+	s := s.NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		if sql.SchemaName == "" {
 			sql.SchemaName = defaultSchema
@@ -301,8 +301,8 @@ func RestoreSchemaName(defaultSchema string, sqls Set[Query], ignoreErr bool) (S
 
 // FilterInvalidQueries filters out invalid queries from the given query set.
 // some queries might be forbidden by the fix-control 43817.
-func FilterInvalidQueries(opt Optimizer, sqls Set[Query], ignoreErr bool) (Set[Query], error) {
-	s := NewSet[Query]()
+func FilterInvalidQueries(opt Optimizer, sqls s.Set[Query], ignoreErr bool) (s.Set[Query], error) {
+	s := s.NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		_, err := opt.QueryPlanCost(sql.Text)
 		if err != nil {
@@ -317,8 +317,8 @@ func FilterInvalidQueries(opt Optimizer, sqls Set[Query], ignoreErr bool) (Set[Q
 }
 
 // FilterSQLAccessingSystemTables filters out queries that access system tables.
-func FilterSQLAccessingSystemTables(sqls Set[Query], ignoreErr bool) (Set[Query], error) {
-	s := NewSet[Query]()
+func FilterSQLAccessingSystemTables(sqls s.Set[Query], ignoreErr bool) (s.Set[Query], error) {
+	s := s.NewSet[Query]()
 	for _, sql := range sqls.ToList() {
 		accessSystemTable := false
 		names, err := CollectTableNamesFromQuery(sql.SchemaName, sql.Text)
@@ -348,8 +348,8 @@ func FilterSQLAccessingSystemTables(sqls Set[Query], ignoreErr bool) (Set[Query]
 }
 
 // CollectIndexableColumnsForQuerySet finds all columns that appear in any range-filter, order-by, or group-by clause.
-func CollectIndexableColumnsForQuerySet(opt Optimizer, querySet Set[Query]) (Set[Column], error) {
-	indexableColumnSet := NewSet[Column]()
+func CollectIndexableColumnsForQuerySet(opt Optimizer, querySet s.Set[Query]) (s.Set[Column], error) {
+	indexableColumnSet := s.NewSet[Column]()
 	queryList := querySet.ToList()
 	for _, q := range queryList {
 		cols, err := CollectIndexableColumnsFromQuery(q, opt)
@@ -363,7 +363,7 @@ func CollectIndexableColumnsForQuerySet(opt Optimizer, querySet Set[Query]) (Set
 }
 
 // CollectIndexableColumnsFromQuery parses the given Query text and returns the indexable columns.
-func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (Set[Column], error) {
+func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (s.Set[Column], error) {
 	tableNames, err := CollectTableNamesFromQuery(q.SchemaName, q.Text)
 	if err != nil {
 		return nil, err
@@ -379,7 +379,7 @@ func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (Set[Column], erro
 	if err != nil {
 		return nil, err
 	}
-	cols := NewSet[Column]()
+	cols := s.NewSet[Column]()
 	var collectColumn func(n ast.Node)
 	collectColumn = func(n ast.Node) {
 		switch x := n.(type) {
diff --git a/pkg/planner/indexadvisor/utils_test.go b/pkg/planner/indexadvisor/utils_test.go
index b04e7a15b2015..6fe1893f4e5d5 100644
--- a/pkg/planner/indexadvisor/utils_test.go
+++ b/pkg/planner/indexadvisor/utils_test.go
@@ -19,7 +19,7 @@ import (
 
 	"github.com/pingcap/tidb/pkg/planner/indexadvisor"
 	"github.com/pingcap/tidb/pkg/testkit"
-	. "github.com/pingcap/tidb/pkg/util/set"
+	s "github.com/pingcap/tidb/pkg/util/set"
 	"github.com/stretchr/testify/require"
 )
 
@@ -82,7 +82,7 @@ func TestRestoreSchemaName(t *testing.T) {
 	q2 := indexadvisor.Query{Text: "select * from t2", SchemaName: "test2"}
 	q3 := indexadvisor.Query{Text: "select * from t3"}
 	q4 := indexadvisor.Query{Text: "wrong"}
-	set1 := NewSet[indexadvisor.Query]()
+	set1 := s.NewSet[indexadvisor.Query]()
 	set1.Add(q1, q2, q3, q4)
 
 	set2, err := indexadvisor.RestoreSchemaName("test", set1, true)
@@ -94,7 +94,7 @@ func TestRestoreSchemaName(t *testing.T) {
 }
 
 func TestFilterSQLAccessingSystemTables(t *testing.T) {
-	set1 := NewSet[indexadvisor.Query]()
+	set1 := s.NewSet[indexadvisor.Query]()
 	set1.Add(indexadvisor.Query{Text: "select * from mysql.stats_meta"})
 	set1.Add(indexadvisor.Query{Text: "select * from information_schema.test"})
 	set1.Add(indexadvisor.Query{Text: "select * from metrics_schema.test"})
@@ -122,7 +122,7 @@ func TestFilterInvalidQueries(t *testing.T) {
 	tk.MustExec(`create table t2 (a int, b int, c int)`)
 	opt := indexadvisor.NewOptimizer(tk.Session())
 
-	set1 := NewSet[indexadvisor.Query]()
+	set1 := s.NewSet[indexadvisor.Query]()
 	set1.Add(indexadvisor.Query{Text: "select * from test.t1"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t3"})                            // table t3 does not exist
 	set1.Add(indexadvisor.Query{Text: "select d from t1"})                                 // column d does not exist
@@ -144,7 +144,7 @@ func TestCollectIndexableColumnsForQuerySet(t *testing.T) {
 	tk.MustExec(`create table t (a int, b int, c int, d int, e int, f int)`)
 	opt := indexadvisor.NewOptimizer(tk.Session())
 
-	set1 := NewSet[indexadvisor.Query]()
+	set1 := s.NewSet[indexadvisor.Query]()
 	set1.Add(indexadvisor.Query{Text: "select * from test.t where a=1 and b=1 and e like 'abc'"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t where a<1 and b>1 and e like 'abc'"})
 	set1.Add(indexadvisor.Query{Text: "select * from test.t where c in (1, 2, 3) order by d"})

From b387a3c2f2982cc41ece2075f4f443670037d2ff Mon Sep 17 00:00:00 2001
From: qw4990 <zhangyuanjia@pingcap.com>
Date: Wed, 4 Sep 2024 15:37:01 +0800
Subject: [PATCH 27/27] fixup

---
 pkg/planner/indexadvisor/utils.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pkg/planner/indexadvisor/utils.go b/pkg/planner/indexadvisor/utils.go
index fd89bfbe22210..3d7a8d8126a51 100644
--- a/pkg/planner/indexadvisor/utils.go
+++ b/pkg/planner/indexadvisor/utils.go
@@ -399,7 +399,9 @@ func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (s.Set[Column], er
 			for _, schemaName := range schemaNames {
 				cols, err := opt.PossibleColumns(schemaName, x.Name.L)
 				if err != nil {
-					l().Warn("failed to get possible columns", zap.String("schema", schemaName), zap.String("column", x.Name.L))
+					l().Warn("failed to get possible columns",
+						zap.String("schema", schemaName),
+						zap.String("column", x.Name.L))
 					continue
 				}
 				possibleColumns = append(possibleColumns, cols...)
@@ -408,7 +410,10 @@ func CollectIndexableColumnsFromQuery(q Query, opt Optimizer) (s.Set[Column], er
 			for _, c := range possibleColumns {
 				colType, err := opt.ColumnType(c)
 				if err != nil {
-					l().Warn("failed to get column type", zap.String("schema", c.SchemaName), zap.String("table", c.TableName), zap.String("column", c.ColumnName))
+					l().Warn("failed to get column type",
+						zap.String("schema", c.SchemaName),
+						zap.String("table", c.TableName),
+						zap.String("column", c.ColumnName))
 					continue
 				}
 				if !isIndexableColumnType(colType) {